From ff0c92ab13dbf3de56f8e90e72b4435ca4647820 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Wed, 15 Mar 2023 11:29:28 -0400 Subject: [PATCH 001/189] Bump mvn version to 2.3. (#2002) A bump to 2.2 was intended with the mono repo changes, but it was only partially done (a v2.2 tag exists, but pom.xml still lists 2.1). --- jitsi-media-transform/pom.xml | 2 +- jvb/pom.xml | 2 +- pom.xml | 2 +- rtp/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/jitsi-media-transform/pom.xml b/jitsi-media-transform/pom.xml index 3e17e814ca..6d9eb2cdb7 100644 --- a/jitsi-media-transform/pom.xml +++ b/jitsi-media-transform/pom.xml @@ -7,7 +7,7 @@ org.jitsi jvb-parent - 2.1-SNAPSHOT + 2.3-SNAPSHOT org.jitsi diff --git a/jvb/pom.xml b/jvb/pom.xml index 8cbee49944..a6b4313ba6 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -5,7 +5,7 @@ org.jitsi jvb-parent - 2.1-SNAPSHOT + 2.3-SNAPSHOT diff --git a/pom.xml b/pom.xml index 734604223f..36cde248ae 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ org.jitsi jvb-parent - 2.1-SNAPSHOT + 2.3-SNAPSHOT pom diff --git a/rtp/pom.xml b/rtp/pom.xml index f192bf864d..465a19557e 100644 --- a/rtp/pom.xml +++ b/rtp/pom.xml @@ -7,7 +7,7 @@ org.jitsi jvb-parent - 2.1-SNAPSHOT + 2.3-SNAPSHOT org.jitsi From dcff6f64d5dd34b4d01e8e796eb8f3b404f25929 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 16 Mar 2023 10:34:03 -0400 Subject: [PATCH 002/189] feat: Support setting the initial value of last-n through colibri2. (#2003) * feat: Support setting the initial value of last-n through colibri2. * fix: initial-last-n only has effect before constraints are received. * chore: Update jitsi-xmpp-extensions. --- .../kotlin/org/jitsi/videobridge/Endpoint.kt | 16 ++++++++++++++++ .../colibri2/Colibri2ConferenceHandler.kt | 4 ++++ pom.xml | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index 91bdddc305..cb524b153f 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -166,6 +166,12 @@ class Endpoint @JvmOverloads constructor( */ var acceptVideo = false + /** + * Whether a [ReceiverVideoConstraintsMessage] has been received from the endpoint. Setting initial-last-n only + * has an effect prior to constraints being received on the message transport. + */ + var initialReceiverConstraintsReceived = false + /** * The queue we put outgoing SRTP packets onto so they can be sent * out via the [IceTransport] on an IO thread. @@ -843,6 +849,7 @@ class Endpoint @JvmOverloads constructor( fun numForwardedEndpoints(): Int = bitrateController.numForwardedEndpoints() fun setBandwidthAllocationSettings(message: ReceiverVideoConstraintsMessage) { + initialReceiverConstraintsReceived = true bitrateController.setBandwidthAllocationSettings(message) } @@ -1137,6 +1144,15 @@ class Endpoint @JvmOverloads constructor( logger.info("Expired.") } + fun setInitialLastN(initialLastN: Int) { + if (initialReceiverConstraintsReceived) { + logger.info("Ignoring initialLastN, message transport already connected.") + } else { + logger.info("Setting initialLastN = $initialLastN") + bitrateController.lastN = initialLastN + } + } + companion object { /** * Whether or not the bridge should be the peer which opens the data channel diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt index bbf9a354f3..6a05d099c7 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt @@ -225,6 +225,10 @@ class Colibri2ConferenceHandler( it.mediaType == MediaType.VIDEO } + c2endpoint.initialLastN?.value?.let { + endpoint.setInitialLastN(it) + } + c2endpoint.transport?.iceUdpTransport?.let { endpoint.setTransportInfo(it) } if (c2endpoint.create) { val transBuilder = Transport.getBuilder() diff --git a/pom.xml b/pom.xml index 36cde248ae..4418f19043 100644 --- a/pom.xml +++ b/pom.xml @@ -110,7 +110,7 @@ ${project.groupId} jitsi-xmpp-extensions - 1.0-66-g379c87b + 1.0-71-g8c6cdeb From 1f4c4d40d1d5c5bc109612cc703f6dda3ee2318f Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Thu, 16 Mar 2023 16:35:16 -0400 Subject: [PATCH 003/189] Add datachannel support for relay message transport. (#2001) * Move SctpHandler and DataChannelHandler out of Endpoint. * Add datachannel support for relay message transport. * Put sctp in transport response if it was in the request. * Update docs to mention that relays can use SCTP. * Don't advertise websockets for relay connections if told to use data channels in the signaling. * Fix typo in exception message. --- doc/relay.md | 4 +- .../kotlin/org/jitsi/videobridge/Endpoint.kt | 102 +------- .../colibri2/Colibri2ConferenceHandler.kt | 48 +++- .../org/jitsi/videobridge/relay/Relay.kt | 220 ++++++++++++++++-- .../relay/RelayMessageTransport.kt | 116 ++++++++- .../videobridge/sctp/DataChannelHandler.kt | 75 ++++++ .../org/jitsi/videobridge/sctp/SctpHandler.kt | 68 ++++++ 7 files changed, 492 insertions(+), 141 deletions(-) create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/sctp/DataChannelHandler.kt create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpHandler.kt diff --git a/doc/relay.md b/doc/relay.md index 879c2f7170..7d75542455 100644 --- a/doc/relay.md +++ b/doc/relay.md @@ -2,8 +2,8 @@ ### Relays Relays (aka secure octo) use ICE and DTLS/SRTP between each pair of bridges, so a secure -network is not required. It uses and requires colibri websockets for the -bridge-bridge connections (endpoints can still use SCTP). +network is not required. It uses and requires either SCTP or colibri websockets for the +bridge-bridge connections. ## Jitsi Videobridge configuration diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index cb524b153f..cf0d785fb2 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -32,7 +32,6 @@ import org.jitsi.nlj.rtp.SsrcAssociationType import org.jitsi.nlj.rtp.VideoRtpPacket import org.jitsi.nlj.srtp.TlsRole import org.jitsi.nlj.stats.EndpointConnectionStats -import org.jitsi.nlj.stats.NodeStatsBlock import org.jitsi.nlj.transform.node.ConsumerNode import org.jitsi.nlj.util.Bandwidth import org.jitsi.nlj.util.LocalSsrcAssociation @@ -69,7 +68,8 @@ import org.jitsi.videobridge.message.SenderSourceConstraintsMessage import org.jitsi.videobridge.message.SenderVideoConstraintsMessage import org.jitsi.videobridge.relay.AudioSourceDesc import org.jitsi.videobridge.rest.root.debug.EndpointDebugFeatures -import org.jitsi.videobridge.sctp.SctpConfig +import org.jitsi.videobridge.sctp.DataChannelHandler +import org.jitsi.videobridge.sctp.SctpHandler import org.jitsi.videobridge.sctp.SctpManager import org.jitsi.videobridge.stats.PacketTransitStats import org.jitsi.videobridge.transport.dtls.DtlsTransport @@ -85,13 +85,11 @@ import org.jitsi_modified.sctp4j.SctpDataCallback import org.jitsi_modified.sctp4j.SctpServerSocket import org.jitsi_modified.sctp4j.SctpSocket import org.json.simple.JSONObject -import java.nio.ByteBuffer import java.security.SecureRandom import java.time.Clock import java.time.Duration import java.time.Instant import java.util.Optional -import java.util.concurrent.LinkedBlockingQueue import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicLong import java.util.function.Supplier @@ -1221,100 +1219,4 @@ class Endpoint @JvmOverloads constructor( bandwidthProbing.bandwidthEstimationChanged(newValue) } } - - /** - * A node which can be placed in the pipeline to cache Data channel packets - * until the DataChannelStack is ready to handle them. - */ - private class DataChannelHandler : ConsumerNode("Data channel handler") { - private val dataChannelStackLock = Any() - private var dataChannelStack: DataChannelStack? = null - private val cachedDataChannelPackets = LinkedBlockingQueue() - - public override fun consume(packetInfo: PacketInfo) { - synchronized(dataChannelStackLock) { - when (val packet = packetInfo.packet) { - is DataChannelPacket -> { - dataChannelStack?.onIncomingDataChannelPacket( - ByteBuffer.wrap(packet.buffer), packet.sid, packet.ppid - ) ?: run { - cachedDataChannelPackets.add(packetInfo) - } - } - else -> Unit - } - } - } - - fun setDataChannelStack(dataChannelStack: DataChannelStack) { - // Submit this to the pool since we wait on the lock and process any - // cached packets here as well - - // Submit this to the pool since we wait on the lock and process any - // cached packets here as well - TaskPools.IO_POOL.execute { - // We grab the lock here so that we can set the SCTP manager and - // process any previously-cached packets as an atomic operation. - // It also prevents another thread from coming in via - // #doProcessPackets and processing packets at the same time in - // another thread, which would be a problem. - synchronized(dataChannelStackLock) { - this.dataChannelStack = dataChannelStack - cachedDataChannelPackets.forEach { - val dcp = it.packet as DataChannelPacket - dataChannelStack.onIncomingDataChannelPacket( - ByteBuffer.wrap(dcp.buffer), dcp.sid, dcp.ppid - ) - } - } - } - } - - override fun trace(f: () -> Unit) = f.invoke() - } - - /** - * A node which can be placed in the pipeline to cache SCTP packets until - * the SCTPManager is ready to handle them. - */ - private class SctpHandler : ConsumerNode("SCTP handler") { - private val sctpManagerLock = Any() - private var sctpManager: SctpManager? = null - private val numCachedSctpPackets = AtomicLong(0) - private val cachedSctpPackets = LinkedBlockingQueue(100) - - override fun consume(packetInfo: PacketInfo) { - synchronized(sctpManagerLock) { - if (SctpConfig.config.enabled) { - sctpManager?.handleIncomingSctp(packetInfo) ?: run { - numCachedSctpPackets.incrementAndGet() - cachedSctpPackets.add(packetInfo) - } - } - } - } - - override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { - addNumber("num_cached_packets", numCachedSctpPackets.get()) - } - - fun setSctpManager(sctpManager: SctpManager) { - // Submit this to the pool since we wait on the lock and process any - // cached packets here as well - TaskPools.IO_POOL.execute { - // We grab the lock here so that we can set the SCTP manager and - // process any previously-cached packets as an atomic operation. - // It also prevents another thread from coming in via - // #doProcessPackets and processing packets at the same time in - // another thread, which would be a problem. - synchronized(sctpManagerLock) { - this.sctpManager = sctpManager - cachedSctpPackets.forEach { sctpManager.handleIncomingSctp(it) } - cachedSctpPackets.clear() - } - } - } - - override fun trace(f: () -> Unit) = f.invoke() - } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt index 6a05d099c7..e2d88df61b 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt @@ -74,14 +74,16 @@ class Colibri2ConferenceHandler( } for (r in conferenceModifyIQ.relays) { if (!RelayConfig.config.enabled) { - throw IqProcessingException(Condition.feature_not_implemented, "Octo is disable in configuration.") + throw IqProcessingException(Condition.feature_not_implemented, "Octo is disabled in configuration.") } - if (!WebsocketServiceConfig.config.enabled) { + if (!WebsocketServiceConfig.config.enabled && !SctpConfig.config.enabled) { logger.warn( - "Can not use a colibri2 relay, because colibri web sockets are not enabled. See " + - "https://github.com/jitsi/jitsi-videobridge/blob/master/doc/octo.md" + "Can not use a colibri2 relay, because neither SCTP nor colibri web sockets are enabled. See " + + "https://github.com/jitsi/jitsi-videobridge/blob/master/doc/relay.md" + ) + throw UnsupportedOperationException( + "Colibri websockets or SCTP need to be enabled to use a colibri2 relay." ) - throw UnsupportedOperationException("Colibri websockets need to be enabled to use a colibri2 relay.") } responseBuilder.addRelay(handleColibri2Relay(r)) } @@ -356,16 +358,42 @@ class Colibri2ConferenceHandler( ) } - if (c2relay.transport?.sctp != null) throw IqProcessingException( - Condition.feature_not_implemented, - "SCTP is not supported for relays." - ) + c2relay.transport?.sctp?.let { sctp -> + if (!SctpConfig.config.enabled) { + throw IqProcessingException( + Condition.feature_not_implemented, + "SCTP support is not configured" + ) + } + if (sctp.port != null && sctp.port != SctpManager.DEFAULT_SCTP_PORT) { + throw IqProcessingException( + Condition.bad_request, + "Specific SCTP port requested, not supported." + ) + } + + relay.createSctpConnection(sctp) + } c2relay.transport?.iceUdpTransport?.let { relay.setTransportInfo(it) } if (c2relay.create) { val transBuilder = Transport.getBuilder() transBuilder.setIceUdpExtension(relay.describeTransport()) - respBuilder.setTransport(transBuilder.build()) + c2relay.transport?.sctp?.let { + val role = if (it.role == Sctp.Role.CLIENT) { + Sctp.Role.SERVER + } else { + Sctp.Role.CLIENT + } + transBuilder.setSctp( + Sctp.Builder() + .setPort(SctpManager.DEFAULT_SCTP_PORT) + .setRole(role) + .build() + ) + + respBuilder.setTransport(transBuilder.build()) + } } for (media: Media in c2relay.media) { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt index 9cea5b165f..5960898b03 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt @@ -71,9 +71,15 @@ import org.jitsi.videobridge.EncodingsManager import org.jitsi.videobridge.Endpoint import org.jitsi.videobridge.PotentialPacketHandler import org.jitsi.videobridge.TransportConfig +import org.jitsi.videobridge.datachannel.DataChannelStack +import org.jitsi.videobridge.datachannel.protocol.DataChannelPacket +import org.jitsi.videobridge.datachannel.protocol.DataChannelProtocolConstants import org.jitsi.videobridge.message.BridgeChannelMessage import org.jitsi.videobridge.message.SourceVideoTypeMessage import org.jitsi.videobridge.rest.root.debug.EndpointDebugFeatures +import org.jitsi.videobridge.sctp.DataChannelHandler +import org.jitsi.videobridge.sctp.SctpHandler +import org.jitsi.videobridge.sctp.SctpManager import org.jitsi.videobridge.stats.PacketTransitStats import org.jitsi.videobridge.transport.dtls.DtlsTransport import org.jitsi.videobridge.transport.ice.IceTransport @@ -82,15 +88,24 @@ import org.jitsi.videobridge.util.TaskPools import org.jitsi.videobridge.util.looksLikeDtls import org.jitsi.videobridge.websocket.colibriWebSocketServiceSupplier import org.jitsi.xmpp.extensions.colibri.WebSocketPacketExtension +import org.jitsi.xmpp.extensions.colibri2.Sctp import org.jitsi.xmpp.extensions.jingle.DtlsFingerprintPacketExtension import org.jitsi.xmpp.extensions.jingle.IceUdpTransportPacketExtension +import org.jitsi_modified.sctp4j.SctpClientSocket +import org.jitsi_modified.sctp4j.SctpDataCallback +import org.jitsi_modified.sctp4j.SctpServerSocket +import org.jitsi_modified.sctp4j.SctpSocket import org.json.simple.JSONObject import java.time.Clock import java.time.Instant +import java.util.* import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicLong import java.util.function.Supplier +import kotlin.collections.ArrayList +import kotlin.collections.HashMap +import kotlin.collections.HashSet import kotlin.collections.sumOf /** @@ -149,6 +164,9 @@ class Relay @JvmOverloads constructor( */ private var expired = false + private val sctpHandler = SctpHandler() + private val dataChannelHandler = DataChannelHandler() + private val iceTransport = IceTransport(id, iceControlling, useUniquePort, logger, clock) private val dtlsTransport = DtlsTransport(logger).also { it.cryptex = CryptexConfig.relay } @@ -160,6 +178,19 @@ class Relay @JvmOverloads constructor( private val timelineLogger = logger.createChildLogger("timeline.${this.javaClass.name}") + /** + * The [SctpManager] instance we'll use to manage the SCTP connection + */ + private var sctpManager: SctpManager? = null + + private var dataChannelStack: DataChannelStack? = null + + /** + * The [SctpSocket] for this endpoint, if an SCTP connection was + * negotiated. + */ + private var sctpSocket: SctpSocket? = null + private val relayedEndpoints = HashMap() private val endpointsBySsrc = HashMap() private val endpointsLock = Any() @@ -251,6 +282,20 @@ class Relay @JvmOverloads constructor( setErrorHandler(queueErrorCounter) } + /** + * The queue which enforces sequential processing of incoming data channel messages + * to maintain processing order. + */ + private val incomingDataChannelMessagesQueue = PacketInfoQueue( + "${javaClass.simpleName}-incoming-data-channel-queue", + TaskPools.IO_POOL, + { packetInfo -> + dataChannelHandler.consume(packetInfo) + true + }, + TransportConfig.queueSize + ) + val debugState: JSONObject get() = JSONObject().apply { put("iceTransport", iceTransport.getDebugState()) @@ -330,7 +375,7 @@ class Relay @JvmOverloads constructor( private fun setupDtlsTransport() { dtlsTransport.incomingDataHandler = object : DtlsTransport.IncomingDataHandler { override fun dtlsAppDataReceived(buf: ByteArray, off: Int, len: Int) { - // TODO this@Relay.dtlsAppPacketReceived(buf, off, len) + dtlsAppPacketReceived(buf, off, len) } } dtlsTransport.outgoingDataHandler = object : DtlsTransport.OutgoingDataHandler { @@ -346,6 +391,11 @@ class Relay @JvmOverloads constructor( ) { logger.info("DTLS handshake complete") setSrtpInformation(chosenSrtpProtectionProfile, tlsRole, keyingMaterial) + when (val socket = sctpSocket) { + is SctpClientSocket -> connectSctpConnection(socket) + is SctpServerSocket -> acceptSctpConnection(socket) + else -> Unit + } scheduleRelayMessageTransportTimeout() } } @@ -378,6 +428,118 @@ class Relay @JvmOverloads constructor( senders.values.forEach { it.setSrtpInformation(srtpTransformers) } } + /** + * Create an SCTP connection for this Relay. If [sctpDesc.role] is [Sctp.Role.CLIENT], + * we will create the data channel locally, otherwise we will wait for the remote side + * to open it. + */ + fun createSctpConnection(sctpDesc: Sctp) { + val openDataChannelLocally = sctpDesc.role == Sctp.Role.CLIENT + + logger.cdebug { "Creating SCTP manager" } + // Create the SctpManager and provide it a method for sending SCTP data + val sctpManager = SctpManager( + { data, offset, length -> + dtlsTransport.sendDtlsData(data, offset, length) + 0 + }, + logger + ) + this.sctpManager = sctpManager + sctpHandler.setSctpManager(sctpManager) + val socket = if (sctpDesc.role == Sctp.Role.CLIENT) { + sctpManager.createClientSocket() + } else { + sctpManager.createServerSocket() + } + socket.eventHandler = object : SctpSocket.SctpSocketEventHandler { + override fun onReady() { + logger.info("SCTP connection is ready, creating the Data channel stack") + val dataChannelStack = DataChannelStack( + { data, sid, ppid -> socket.send(data, true, sid, ppid) }, + logger + ) + this@Relay.dataChannelStack = dataChannelStack + // This handles if the remote side will be opening the data channel + dataChannelStack.onDataChannelStackEvents { dataChannel -> + logger.info("Remote side opened a data channel.") + messageTransport.setDataChannel(dataChannel) + } + dataChannelHandler.setDataChannelStack(dataChannelStack) + if (openDataChannelLocally) { + // This logic is for opening the data channel locally + logger.info("Will open the data channel.") + val dataChannel = dataChannelStack.createDataChannel( + DataChannelProtocolConstants.RELIABLE, + 0, + 0, + 0, + "default" + ) + messageTransport.setDataChannel(dataChannel) + dataChannel.open() + } else { + logger.info("Will wait for the remote side to open the data channel.") + } + } + + override fun onDisconnected() { + logger.info("SCTP connection is disconnected") + } + } + socket.dataCallback = SctpDataCallback { data, sid, ssn, tsn, ppid, context, flags -> + // We assume all data coming over SCTP will be datachannel data + val dataChannelPacket = DataChannelPacket(data, 0, data.size, sid, ppid.toInt()) + // Post the rest of the task here because the current context is + // holding a lock inside the SctpSocket which can cause a deadlock + // if two endpoints are trying to send datachannel messages to one + // another (with stats broadcasting it can happen often) + incomingDataChannelMessagesQueue.add(PacketInfo(dataChannelPacket)) + } + if (socket is SctpServerSocket) { + socket.listen() + } + sctpSocket = socket + } + + fun connectSctpConnection(sctpClientSocket: SctpClientSocket) { + TaskPools.IO_POOL.execute { + // We don't want to block the thread calling + // onDtlsHandshakeComplete so run the socket acceptance in an IO + // pool thread + logger.info("Attempting to establish SCTP socket connection") + + if (!sctpClientSocket.connect(SctpManager.DEFAULT_SCTP_PORT)) { + logger.error("Failed to establish SCTP connection to remote side") + } + } + } + + fun acceptSctpConnection(sctpServerSocket: SctpServerSocket) { + TaskPools.IO_POOL.execute { + // We don't want to block the thread calling + // onDtlsHandshakeComplete so run the socket acceptance in an IO + // pool thread + // FIXME: This runs forever once the socket is closed ( + // accept never returns true). + logger.info("Attempting to establish SCTP socket connection") + var attempts = 0 + while (!sctpServerSocket.accept()) { + attempts++ + try { + Thread.sleep(100) + } catch (e: InterruptedException) { + break + } + if (attempts > 100) { + logger.error("Timed out waiting for SCTP connection from remote side") + break + } + } + logger.cdebug { "SCTP socket ${sctpServerSocket.hashCode()} accepted connection" } + } + } + /** * Sets the remote transport information (ICE candidates, DTLS fingerprints). * @@ -406,7 +568,7 @@ class Relay @JvmOverloads constructor( iceTransport.startConnectivityEstablishment(transportInfo) val websocketExtension = transportInfo.getFirstChildOfType(WebSocketPacketExtension::class.java) - websocketExtension?.url?.let { messageTransport.connectTo(it) } + websocketExtension?.url?.let { messageTransport.connectToWebsocket(it) } } fun describeTransport(): IceUdpTransportPacketExtension { @@ -414,29 +576,31 @@ class Relay @JvmOverloads constructor( iceTransport.describe(iceUdpTransportPacketExtension) dtlsTransport.describe(iceUdpTransportPacketExtension) - /* TODO: this should be dependent on videobridge.websockets.enabled, if we support that being - * disabled for relay. - */ - if (messageTransport.isActive) { - iceUdpTransportPacketExtension.addChildExtension( - WebSocketPacketExtension().apply { active = true } - ) - } else { - colibriWebSocketServiceSupplier.get()?.let { colibriWebsocketService -> - val urls = colibriWebsocketService.getColibriRelayWebSocketUrls( - conference.id, - id, - iceTransport.icePassword + if (sctpSocket != null) { + /* TODO: this should be dependent on videobridge.websockets.enabled, if we support that being + * disabled for relay. + */ + if (messageTransport.isActive) { + iceUdpTransportPacketExtension.addChildExtension( + WebSocketPacketExtension().apply { active = true } ) - if (urls.isEmpty()) { - logger.warn("No colibri relay URLs configured") - } - urls.forEach { - iceUdpTransportPacketExtension.addChildExtension( - WebSocketPacketExtension().apply { - url = it - } + } else { + colibriWebSocketServiceSupplier.get()?.let { colibriWebsocketService -> + val urls = colibriWebsocketService.getColibriRelayWebSocketUrls( + conference.id, + id, + iceTransport.icePassword ) + if (urls.isEmpty()) { + logger.warn("No colibri relay URLs configured") + } + urls.forEach { + iceUdpTransportPacketExtension.addChildExtension( + WebSocketPacketExtension().apply { + url = it + } + ) + } } } } @@ -561,6 +725,14 @@ class Relay @JvmOverloads constructor( } } + /** + * Handle a DTLS app packet (that is, a packet of some other protocol sent + * over DTLS) which has just been received. + */ + // TODO(brian): change sctp handler to take buf, off, len + fun dtlsAppPacketReceived(data: ByteArray, off: Int, len: Int) = + sctpHandler.processPacket(PacketInfo(UnparsedPacket(data, off, len))) + fun addRemoteEndpoint( id: String, statsId: String?, @@ -923,6 +1095,8 @@ class Relay @JvmOverloads constructor( transceiver.teardown() messageTransport.close() + sctpHandler.stop() + sctpManager?.closeConnection() } catch (t: Throwable) { logger.error("Exception while expiring: ", t) } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt index bd537a5da6..2754ed9ea5 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt @@ -23,6 +23,10 @@ import org.jitsi.utils.logging2.Logger import org.jitsi.videobridge.AbstractEndpointMessageTransport import org.jitsi.videobridge.VersionConfig import org.jitsi.videobridge.Videobridge +import org.jitsi.videobridge.datachannel.DataChannel +import org.jitsi.videobridge.datachannel.DataChannelStack.DataChannelMessageListener +import org.jitsi.videobridge.datachannel.protocol.DataChannelMessage +import org.jitsi.videobridge.datachannel.protocol.DataChannelStringMessage import org.jitsi.videobridge.message.AddReceiverMessage import org.jitsi.videobridge.message.BridgeChannelMessage import org.jitsi.videobridge.message.ClientHelloMessage @@ -35,6 +39,7 @@ import org.jitsi.videobridge.message.VideoTypeMessage import org.jitsi.videobridge.util.TaskPools import org.jitsi.videobridge.websocket.ColibriWebSocket import org.json.simple.JSONObject +import java.lang.ref.WeakReference import java.net.URI import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicInteger @@ -43,14 +48,15 @@ import java.util.function.Supplier /** * Handles the functionality related to sending and receiving COLIBRI messages - * for a [Relay]. + * for a [Relay]. Supports two underlying transport mechanisms -- + * WebRTC data channels and {@code WebSocket}s. */ class RelayMessageTransport( private val relay: Relay, private val statisticsSupplier: Supplier, private val eventHandler: EndpointMessageTransportEventHandler, parentLogger: Logger -) : AbstractEndpointMessageTransport(parentLogger), ColibriWebSocket.EventHandler { +) : AbstractEndpointMessageTransport(parentLogger), ColibriWebSocket.EventHandler, DataChannelMessageListener { /** * The last connected/accepted web-socket by this instance, if any. */ @@ -70,6 +76,16 @@ class RelayMessageTransport( * Use to synchronize access to [webSocket] */ private val webSocketSyncRoot = Any() + + /** + * Whether the last active transport channel (i.e. the last to receive a + * message from the remote endpoint) was the web socket (if `true`), + * or the WebRTC data channel (if `false`). + */ + private var webSocketLastActive = false + + private var dataChannel = WeakReference(null) + private val numOutgoingMessagesDropped = AtomicInteger(0) /** @@ -82,7 +98,7 @@ class RelayMessageTransport( /** * Connect the bridge channel message to the websocket URL specified */ - fun connectTo(url: String) { + fun connectToWebsocket(url: String) { if (this.url != null && this.url == url) { return } @@ -202,11 +218,23 @@ class RelayMessageTransport( super.sendMessage(dst, message) // Log message if (dst is ColibriWebSocket) { sendMessage(dst, message) + } else if (dst is DataChannel) { + sendMessage(dst, message) } else { throw IllegalArgumentException("unknown transport:$dst") } } + /** + * Sends a string via a particular [DataChannel]. + * @param dst the data channel to send through. + * @param message the message to send. + */ + private fun sendMessage(dst: DataChannel, message: BridgeChannelMessage) { + dst.sendString(message.toJson()) + statisticsSupplier.get().dataChannelMessagesSent.inc() + } + /** * Sends a string via a particular [ColibriWebSocket] instance. * @param dst the [ColibriWebSocket] through which to send the message. @@ -220,21 +248,69 @@ class RelayMessageTransport( statisticsSupplier.get().colibriWebSocketMessagesSent.inc() } + override fun onDataChannelMessage(dataChannelMessage: DataChannelMessage?) { + webSocketLastActive = false + statisticsSupplier.get().dataChannelMessagesReceived.inc() + if (dataChannelMessage is DataChannelStringMessage) { + onMessage(dataChannel.get(), dataChannelMessage.data) + } + } + /** * {@inheritDoc} */ public override fun sendMessage(msg: BridgeChannelMessage) { - if (webSocket == null) { + val dst = getActiveTransportChannel() + if (dst == null) { logger.debug("No available transport channel, can't send a message") numOutgoingMessagesDropped.incrementAndGet() } else { sentMessagesCounts.computeIfAbsent(msg.javaClass.simpleName) { AtomicLong() }.incrementAndGet() - sendMessage(webSocket, msg) + sendMessage(dst, msg) + } + } + + /** + * @return the active transport channel for this + * [RelayMessageTransport] (either the [.webSocket], or + * the WebRTC data channel represented by a [DataChannel]). + * + * The "active" channel is determined based on what channels are available, + * and which one was the last to receive data. That is, if only one channel + * is available, it will be returned. If two channels are available, the + * last one to have received data will be returned. Otherwise, `null` + * will be returned. + */ + // TODO(brian): seems like it'd be nice to have the websocket and datachannel + // share a common parent class (or, at least, have a class that is returned + // here and provides a common API but can wrap either a websocket or + // datachannel) + private fun getActiveTransportChannel(): Any? { + val dataChannel = dataChannel.get() + val webSocket = webSocket + var dst: Any? = null + if (webSocketLastActive) { + dst = webSocket + } + + // Either the socket was not the last active channel, + // or it has been closed. + if (dst == null) { + if (dataChannel != null && dataChannel.isReady) { + dst = dataChannel + } + } + + // Maybe the WebRTC data channel is the last active, but it is not + // currently available. If so, and a web-socket is available -- use it. + if (dst == null && webSocket != null) { + dst = webSocket } + return dst } override val isConnected: Boolean - get() = webSocket != null + get() = getActiveTransportChannel() != null val isActive: Boolean get() = outgoingWebsocket != null @@ -248,6 +324,7 @@ class RelayMessageTransport( if (ws != webSocket) { logger.info("Replacing an existing websocket.") webSocket?.session?.close(CloseStatus.NORMAL, "replaced") + webSocketLastActive = true webSocket = ws sendMessage(ws, createServerHello()) } else { @@ -276,6 +353,7 @@ class RelayMessageTransport( synchronized(webSocketSyncRoot) { if (ws == webSocket) { webSocket = null + webSocketLastActive = false logger.debug { "Web socket closed, statusCode $statusCode ( $reason)." } } } @@ -325,9 +403,35 @@ class RelayMessageTransport( return } statisticsSupplier.get().colibriWebSocketMessagesReceived.inc() + webSocketLastActive = true onMessage(ws, message) } + /** + * Sets the data channel for this endpoint. + * @param dataChannel the [DataChannel] to use for this transport + */ + fun setDataChannel(dataChannel: DataChannel) { + val prevDataChannel = this.dataChannel.get() + if (prevDataChannel == null) { + this.dataChannel = WeakReference(dataChannel) + // We install the handler first, otherwise the 'ready' might fire after we check it but before we + // install the handler + dataChannel.onDataChannelEvents { notifyTransportChannelConnected() } + if (dataChannel.isReady) { + notifyTransportChannelConnected() + } + dataChannel.onDataChannelMessage(this) + } else if (prevDataChannel === dataChannel) { + // TODO: i think we should be able to ensure this doesn't happen, + // so throwing for now. if there's a good + // reason for this, we can make this a no-op + throw Error("Re-setting the same data channel") + } else { + throw Error("Overwriting a previous data channel!") + } + } + override val debugState: JSONObject get() { val debugState = super.debugState diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/DataChannelHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/DataChannelHandler.kt new file mode 100644 index 0000000000..df69991226 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/DataChannelHandler.kt @@ -0,0 +1,75 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.sctp + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.transform.node.ConsumerNode +import org.jitsi.videobridge.datachannel.DataChannelStack +import org.jitsi.videobridge.datachannel.protocol.DataChannelPacket +import org.jitsi.videobridge.util.TaskPools +import java.nio.ByteBuffer +import java.util.concurrent.LinkedBlockingQueue + +/** + * A node which can be placed in the pipeline to cache Data channel packets + * until the DataChannelStack is ready to handle them. + */ +class DataChannelHandler : ConsumerNode("Data channel handler") { + private val dataChannelStackLock = Any() + private var dataChannelStack: DataChannelStack? = null + private val cachedDataChannelPackets = LinkedBlockingQueue() + + public override fun consume(packetInfo: PacketInfo) { + synchronized(dataChannelStackLock) { + when (val packet = packetInfo.packet) { + is DataChannelPacket -> { + dataChannelStack?.onIncomingDataChannelPacket( + ByteBuffer.wrap(packet.buffer), packet.sid, packet.ppid + ) ?: run { + cachedDataChannelPackets.add(packetInfo) + } + } + else -> Unit + } + } + } + + fun setDataChannelStack(dataChannelStack: DataChannelStack) { + // Submit this to the pool since we wait on the lock and process any + // cached packets here as well + + // Submit this to the pool since we wait on the lock and process any + // cached packets here as well + TaskPools.IO_POOL.execute { + // We grab the lock here so that we can set the SCTP manager and + // process any previously-cached packets as an atomic operation. + // It also prevents another thread from coming in via + // #doProcessPackets and processing packets at the same time in + // another thread, which would be a problem. + synchronized(dataChannelStackLock) { + this.dataChannelStack = dataChannelStack + cachedDataChannelPackets.forEach { + val dcp = it.packet as DataChannelPacket + dataChannelStack.onIncomingDataChannelPacket( + ByteBuffer.wrap(dcp.buffer), dcp.sid, dcp.ppid + ) + } + } + } + } + + override fun trace(f: () -> Unit) = f.invoke() +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpHandler.kt new file mode 100644 index 0000000000..3fc1be6ed0 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpHandler.kt @@ -0,0 +1,68 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.sctp + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.ConsumerNode +import org.jitsi.videobridge.util.TaskPools +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.atomic.AtomicLong + +/** + * A node which can be placed in the pipeline to cache SCTP packets until + * the SCTPManager is ready to handle them. + */ +class SctpHandler : ConsumerNode("SCTP handler") { + private val sctpManagerLock = Any() + private var sctpManager: SctpManager? = null + private val numCachedSctpPackets = AtomicLong(0) + private val cachedSctpPackets = LinkedBlockingQueue(100) + + override fun consume(packetInfo: PacketInfo) { + synchronized(sctpManagerLock) { + if (SctpConfig.config.enabled) { + sctpManager?.handleIncomingSctp(packetInfo) ?: run { + numCachedSctpPackets.incrementAndGet() + cachedSctpPackets.add(packetInfo) + } + } + } + } + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + addNumber("num_cached_packets", numCachedSctpPackets.get()) + } + + fun setSctpManager(sctpManager: SctpManager) { + // Submit this to the pool since we wait on the lock and process any + // cached packets here as well + TaskPools.IO_POOL.execute { + // We grab the lock here so that we can set the SCTP manager and + // process any previously-cached packets as an atomic operation. + // It also prevents another thread from coming in via + // #doProcessPackets and processing packets at the same time in + // another thread, which would be a problem. + synchronized(sctpManagerLock) { + this.sctpManager = sctpManager + cachedSctpPackets.forEach { sctpManager.handleIncomingSctp(it) } + cachedSctpPackets.clear() + } + } + } + + override fun trace(f: () -> Unit) = f.invoke() +} From 7a3271b6dd45a9bff10d80e16966076509a2c077 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Fri, 17 Mar 2023 10:41:09 -0400 Subject: [PATCH 004/189] Bump jitsi-sctp, pass logger to sctp sockets. (#2004) --- jvb/pom.xml | 2 +- .../main/java/org/jitsi/videobridge/sctp/SctpManager.java | 8 ++++---- jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt | 2 +- jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/jvb/pom.xml b/jvb/pom.xml index a6b4313ba6..2ba5eeecc2 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -108,7 +108,7 @@ ${project.groupId} sctp - 1.0-9-g45bf9f2 + 1.0-11-gcd70942 diff --git a/jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java b/jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java index fb99973cea..7dd23f4cae 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java +++ b/jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java @@ -109,9 +109,9 @@ public void handleIncomingSctp(PacketInfo sctpPacket) * Create an {@link SctpServerSocket} to be used to wait for incoming SCTP connections * @return an {@link SctpServerSocket} */ - public SctpServerSocket createServerSocket() + public SctpServerSocket createServerSocket(Logger parentLogger) { - socket = Sctp4j.createServerSocket(DEFAULT_SCTP_PORT); + socket = Sctp4j.createServerSocket(DEFAULT_SCTP_PORT, parentLogger); socket.outgoingDataSender = this.dataSender; logger.debug(() -> "Created SCTP server socket " + socket.hashCode()); return (SctpServerSocket)socket; @@ -121,9 +121,9 @@ public SctpServerSocket createServerSocket() * Create an {@link SctpClientSocket} to be used to open an SCTP connection * @return an {@link SctpClientSocket} */ - public SctpClientSocket createClientSocket() + public SctpClientSocket createClientSocket(Logger parentLogger) { - socket = Sctp4j.createClientSocket(DEFAULT_SCTP_PORT); + socket = Sctp4j.createClientSocket(DEFAULT_SCTP_PORT, parentLogger); socket.outgoingDataSender = this.dataSender; if (logger.isDebugEnabled()) { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index cf0d785fb2..7278ce17be 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -610,7 +610,7 @@ class Endpoint @JvmOverloads constructor( sctpHandler.setSctpManager(sctpManager!!) // NOTE(brian): as far as I know we always act as the 'server' for sctp // connections, but if not we can make which type we use dynamic - val socket = sctpManager!!.createServerSocket() + val socket = sctpManager!!.createServerSocket(logger) socket.eventHandler = object : SctpSocket.SctpSocketEventHandler { override fun onReady() { logger.info("SCTP connection is ready, creating the Data channel stack") diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt index 5960898b03..0b8cf79b95 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt @@ -448,9 +448,9 @@ class Relay @JvmOverloads constructor( this.sctpManager = sctpManager sctpHandler.setSctpManager(sctpManager) val socket = if (sctpDesc.role == Sctp.Role.CLIENT) { - sctpManager.createClientSocket() + sctpManager.createClientSocket(logger) } else { - sctpManager.createServerSocket() + sctpManager.createServerSocket(logger) } socket.eventHandler = object : SctpSocket.SctpSocketEventHandler { override fun onReady() { From 33ef732e8b0d2ffb742a8b8a84bf7af1b465deff Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Mon, 20 Mar 2023 12:57:05 -0400 Subject: [PATCH 005/189] Fix setting transport for relays not using sctp. (#2005) --- .../jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt index e2d88df61b..e6fc120397 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt @@ -391,9 +391,8 @@ class Colibri2ConferenceHandler( .setRole(role) .build() ) - - respBuilder.setTransport(transBuilder.build()) } + respBuilder.setTransport(transBuilder.build()) } for (media: Media in c2relay.media) { From fb3a86ccd03b1ed5a7a2e4f3e05f20faa331d8c7 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Tue, 21 Mar 2023 14:26:50 -0400 Subject: [PATCH 006/189] Set prometheus namespace to jitsi_jvb. (#2006) --- .../jitsi/videobridge/metrics/VideobridgeMetricsContainer.kt | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetricsContainer.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetricsContainer.kt index d54589aaa8..d7e58f9d42 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetricsContainer.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetricsContainer.kt @@ -21,7 +21,7 @@ import org.jitsi.metrics.MetricsContainer * `VideobridgeMetricsContainer` gathers and exports metrics * from a [Videobridge][org.jitsi.videobridge.Videobridge] instance. */ -class VideobridgeMetricsContainer private constructor() : MetricsContainer() { +class VideobridgeMetricsContainer private constructor() : MetricsContainer(namespace = "jitsi_jvb") { companion object { /** diff --git a/pom.xml b/pom.xml index 4418f19043..29a42890e8 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 5.3.0 5.8.2 1.0-124-ge57838f - 1.1-122-g7bba0d6 + 1.1-123-g68019c1 1.13.1 3.2.2 4.6.0 From bcfde3f245537e35ce9e13f18ef17c3f188049fb Mon Sep 17 00:00:00 2001 From: bgrozev Date: Wed, 22 Mar 2023 12:28:58 -0400 Subject: [PATCH 007/189] Send the set of forwarded sources when the message transport connects. (#2007) --- jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt | 5 +++++ .../jitsi/videobridge/cc/allocation/BitrateController.kt | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index 7278ce17be..2f39fae79d 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -513,6 +513,11 @@ class Endpoint @JvmOverloads constructor( // TODO: this should be part of an EndpointMessageTransport.EventHandler interface fun endpointMessageTransportConnected() { sendAllVideoConstraints() + if (isUsingSourceNames) { + sendForwardedSourcesMessage(bitrateController.forwardedSources) + } else { + sendForwardedEndpointsMessage(bitrateController.forwardedEndpoints) + } videoSsrcs.sendAllMappings() audioSsrcs.sendAllMappings() } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt index 33d14a3224..6570ff4b8a 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt @@ -58,11 +58,13 @@ class BitrateController @JvmOverloads constructor( * Keep track of the "forwarded" endpoints, i.e. the endpoints for which we are forwarding *some* layer. */ @Deprecated("", ReplaceWith("forwardedSources"), DeprecationLevel.WARNING) - private var forwardedEndpoints: Set = emptySet() + var forwardedEndpoints: Set = emptySet() + private set /** * Keep track of the "forwarded" sources, i.e. the media sources for which we are forwarding *some* layer. */ - private var forwardedSources: Set = emptySet() + var forwardedSources: Set = emptySet() + private set /** * Keep track of how much time we spend knowingly oversending (due to enableOnstageVideoSuspend being false) @@ -275,14 +277,12 @@ class BitrateController @JvmOverloads constructor( packetHandler.allocationChanged(allocation) if (useSourceNames) { - // TODO as per George's comment above: should this message be sent on message transport connect? val newForwardedSources = allocation.forwardedSources if (forwardedSources != newForwardedSources) { forwardedSources = newForwardedSources eventEmitter.fireEvent { forwardedSourcesChanged(newForwardedSources) } } } else { - // TODO(george) bring back sending this message on message transport connect val newForwardedEndpoints = allocation.forwardedEndpoints if (forwardedEndpoints != newForwardedEndpoints) { forwardedEndpoints = newForwardedEndpoints From 7aedd9aba43ad4cda7acf5e319e0ba0d7977cce2 Mon Sep 17 00:00:00 2001 From: Misha <104766779+mykhailo-ua@users.noreply.github.com> Date: Fri, 24 Mar 2023 12:56:56 +0100 Subject: [PATCH 008/189] ref(colibri2 session participant re-invite): specific colibri2 unknown endpoint error (#1999) * Added functionality to send back UNKNOWN_ENDPOINT Colibri2Error when there is no such endpoint. This addition reason will help jicofo to recognise this colibri2 error and re-invite participant instead of invalidating whole bridge and re-inviting all participants. * ref: Move UnknownEndpointException definition. * Update jitsi-xmpp-extensions. * feat: Return feature_not_implemented for colibri2 requests that use features known not to be properly supported by jitsi-videobridge (and not used in jicofo). This is mostly to help spot problems early if developing a different colibri2 client (not jicofo). * feat: Suppress UNKNOWN_ENDPOINT errors for some requests When the request is just trying to expire and endpoint or relay that is missing (presumably already expired) we don't need to throw an error. Same if the request only updates the force-mute state of an endpoint (the only other operation that is currently performed in a batch by jicofo). Other requests, which create an endpoint/relay or update the media/sources/transport of an endpoint still throw UNKNOWN_ENDPOINT erorrs. These requests are now required to reference a single endpoint or relay (to simplify error handling). --------- Co-authored-by: Boris Grozev --- .../colibri2/Colibri2ConferenceHandler.kt | 96 ++++++++++++++++--- .../videobridge/colibri2/Colibri2Util.kt | 18 ++++ .../colibri2/IqProcessingException.kt | 10 +- 3 files changed, 108 insertions(+), 16 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt index e6fc120397..f1a8306da9 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt @@ -62,15 +62,17 @@ class Colibri2ConferenceHandler( * expired. */ fun handleConferenceModifyIQ(conferenceModifyIQ: ConferenceModifyIQ): Pair = try { + validateRequest(conferenceModifyIQ) + + val ignoreUnknownEndpoints = shouldIgnoreUnknownEndpoints(conferenceModifyIQ) val responseBuilder = ConferenceModifiedIQ.builder(ConferenceModifiedIQ.Builder.createResponse(conferenceModifyIQ)) var expire = conferenceModifyIQ.expire.also { if (it) logger.info("Received request to expire conference.") } - /* TODO: is there any reason we might need to handle Endpoints and Relays in in-message order? */ for (e in conferenceModifyIQ.endpoints) { - responseBuilder.addEndpoint(handleColibri2Endpoint(e)) + responseBuilder.addEndpoint(handleColibri2Endpoint(e, ignoreUnknownEndpoints)) } for (r in conferenceModifyIQ.relays) { if (!RelayConfig.config.enabled) { @@ -85,7 +87,7 @@ class Colibri2ConferenceHandler( "Colibri websockets or SCTP need to be enabled to use a colibri2 relay." ) } - responseBuilder.addRelay(handleColibri2Relay(r)) + responseBuilder.addRelay(handleColibri2Relay(r, ignoreUnknownEndpoints)) } // Include feedback sources with any "create conference" or "create endpoint" request. This allows async @@ -101,6 +103,13 @@ class Colibri2ConferenceHandler( } Pair(responseBuilder.build(), expire) + } catch (e: UnknownEndpointException) { + logger.warn("Unknown Endpoint during processing conference-modify IQ: $e") + val error = createEndpointNotFoundError(conferenceModifyIQ, e.endpointId) + Pair(error, false) + } catch (e: FeatureNotImplementedException) { + logger.warn("Unsupported request (${e.message}): ${conferenceModifyIQ.toXML()}") + Pair(createFeatureNotImplementedError(conferenceModifyIQ, e.message), false) } catch (e: IqProcessingException) { // Item not found conditions are assumed to be less critical, as they often happen in case a request // arrives late for an expired endpoint. @@ -144,7 +153,10 @@ class Colibri2ConferenceHandler( * the conference-modified. */ @Throws(IqProcessingException::class) - private fun handleColibri2Endpoint(c2endpoint: Colibri2Endpoint): Colibri2Endpoint { + private fun handleColibri2Endpoint( + c2endpoint: Colibri2Endpoint, + ignoreUnknownEndpoints: Boolean + ): Colibri2Endpoint { val respBuilder = Colibri2Endpoint.getBuilder().apply { setId(c2endpoint.id) } if (c2endpoint.expire) { conference.getLocalEndpoint(c2endpoint.id)?.expire() @@ -196,11 +208,15 @@ class Colibri2ConferenceHandler( } } } else { - conference.getLocalEndpoint(c2endpoint.id) ?: throw IqProcessingException( - // TODO: this should be Condition.item_not_found but this conflicts with some error codes from the Muc. - Condition.bad_request, - "Unknown endpoint ${c2endpoint.id}" - ) + conference.getLocalEndpoint(c2endpoint.id) + } + + if (endpoint == null) { + if (ignoreUnknownEndpoints) { + return respBuilder.build() + } else { + throw UnknownEndpointException(c2endpoint.id) + } } for (media in c2endpoint.media) { @@ -325,7 +341,10 @@ class Colibri2ConferenceHandler( * the conference-modified. */ @Throws(IqProcessingException::class) - private fun handleColibri2Relay(c2relay: Colibri2Relay): Colibri2Relay { + private fun handleColibri2Relay( + c2relay: Colibri2Relay, + ignoreUnknownRelays: Boolean + ): Colibri2Relay { val respBuilder = Colibri2Relay.getBuilder() respBuilder.setId(c2relay.id) if (c2relay.expire) { @@ -334,7 +353,7 @@ class Colibri2ConferenceHandler( return respBuilder.build() } - val relay: Relay + val relay: Relay? if (c2relay.create) { if (conference.getRelay(c2relay.id) != null) { throw IqProcessingException(Condition.conflict, "Relay with ID ${c2relay.id} already exists") @@ -351,11 +370,16 @@ class Colibri2ConferenceHandler( transport.useUniquePort ) } else { - relay = conference.getRelay(c2relay.id) ?: throw IqProcessingException( + relay = conference.getRelay(c2relay.id) + } + + if (relay == null) { + if (ignoreUnknownRelays) { + return respBuilder.build() + } else { // TODO: this should be Condition.item_not_found but this conflicts with some error codes from the Muc. - Condition.bad_request, - "Unknown relay ${c2relay.id}" - ) + throw IqProcessingException(Condition.bad_request, "Unknown relay ${c2relay.id}") + } } c2relay.transport?.sctp?.let { sctp -> @@ -459,4 +483,46 @@ class Colibri2ConferenceHandler( } return Pair(audioSources, videoSources) } + + /** + * Check if the request satisfies the additional restrictions that jitsi-videobridge imposes on colibri2 requests + * and throw [FeatureNotImplementedException] if it doesn't. + * + * Mostly, we don't support requests that contain arbitrary multiple [Colibri2Endpoint]s or [Colibri2Relay]s. We do + * support them when they expire a set of endpoints, or solely update the "force-mute" state of a set of endpoints, + * because in these cases it's reasonable to ignore errors when endpoints expired. + */ + @kotlin.jvm.Throws(FeatureNotImplementedException::class) + private fun validateRequest(iq: ConferenceModifyIQ) { + if (iq.endpoints.size > 0 && iq.relays.size > 0) { + throw FeatureNotImplementedException("Using both 'relay' and 'endpoint' in a request") + } + + if (iq.endpoints.size > 1) { + if (iq.endpoints.any { it.create || it.media.size > 0 || it.transport != null || it.sources != null }) { + throw FeatureNotImplementedException( + "Creating or updating media, sources or transport for more than one endpoint in a request." + ) + } + } + + if (iq.relays.size > 1) { + throw FeatureNotImplementedException("Updating more than one 'relay' in a request.") + } + } +} + +/** + * Return true if "unknown endpoint" and "unknown relay" errors should be ignored when handling [iq]. We ignore errors + * if the request simply expires endpoints or a relay (since presumably the entity is already expired), or if it is a + * batch update of force-mute state (for the same reason, in order to simplify error handling). + */ +private fun shouldIgnoreUnknownEndpoints(iq: ConferenceModifyIQ): Boolean { + if (iq.endpoints.any { it.create || it.media.size > 0 || it.transport != null || it.sources != null }) { + return false + } + if (iq.relays.any { !it.expire }) { + return false + } + return true } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2Util.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2Util.kt index 6898ef3fdb..526b345608 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2Util.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2Util.kt @@ -15,6 +15,7 @@ */ package org.jitsi.videobridge.colibri2 +import org.jitsi.xmpp.extensions.colibri2.Colibri2Endpoint import org.jitsi.xmpp.extensions.colibri2.Colibri2Error import org.jitsi.xmpp.util.createError import org.jivesoftware.smack.packet.IQ @@ -40,3 +41,20 @@ fun createGracefulShutdownErrorResponse(iq: IQ): IQ = createError( "In graceful shutdown", Colibri2Error(Colibri2Error.Reason.GRACEFUL_SHUTDOWN) ) + +fun createEndpointNotFoundError(iq: IQ, endpointId: String) = createError( + iq, + StanzaError.Condition.item_not_found, + "Endpoint not found for ID: $endpointId", + listOf( + Colibri2Error(Colibri2Error.Reason.UNKNOWN_ENDPOINT), + Colibri2Endpoint.getBuilder().setId(endpointId).build() + ) +) + +fun createFeatureNotImplementedError(iq: IQ, message: String?) = createError( + iq, + StanzaError.Condition.feature_not_implemented, + message ?: "", + Colibri2Error(Colibri2Error.Reason.FEATURE_NOT_IMPLEMENTED), +) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/IqProcessingException.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/IqProcessingException.kt index e54c0c5b05..44da1844e0 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/IqProcessingException.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/IqProcessingException.kt @@ -18,9 +18,17 @@ package org.jitsi.videobridge.colibri2 import org.jivesoftware.smack.packet.StanzaError import java.lang.Exception -internal class IqProcessingException( +internal open class IqProcessingException( val condition: StanzaError.Condition, message: String ) : Exception(message) { override fun toString() = "$condition $message" } + +internal class UnknownEndpointException(val endpointId: String) : IqProcessingException( + StanzaError.Condition.item_not_found, "Unknown endpoint $endpointId" +) + +internal class FeatureNotImplementedException(message: String) : IqProcessingException( + StanzaError.Condition.feature_not_implemented, message +) From 4339a9f7dec277f027d625cd9ab6ff8d25f2023c Mon Sep 17 00:00:00 2001 From: bgrozev Date: Mon, 3 Apr 2023 14:45:58 -0500 Subject: [PATCH 009/189] Put assumed-bandwidth-bps behind a config flag. (#2008) * Put assumed-bandwidth-bps behind a config flag. --- .../videobridge/cc/allocation/AllocationSettings.kt | 9 ++++++--- .../videobridge/cc/config/BitrateControllerConfig.kt | 5 +++++ jvb/src/main/resources/reference.conf | 4 ++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt index 167aaeffd4..c644595069 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt @@ -166,9 +166,12 @@ internal class AllocationSettingsWrapper( } } message.assumedBandwidthBps?.let { - logger.warn("Setting assumed bandwidth ${it.bps}") - this.assumedBandwidthBps = it - changed = true + config.assumedBandwidthLimit?.let { limit -> + val limited = it.coerceAtMost(limit.bps.toLong()) + logger.warn("Setting assumed bandwidth ${limited.bps} (receiver asked for $it).") + this.assumedBandwidthBps = limited + changed = true + } ?: logger.info("Ignoring assumed-bandwidth-bps, not allowed in config.") } if (changed) { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt index 79de6f4072..347708b7d2 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt @@ -19,6 +19,7 @@ package org.jitsi.videobridge.cc.config import org.jitsi.config.JitsiConfig import org.jitsi.metaconfig.config import org.jitsi.metaconfig.from +import org.jitsi.metaconfig.optionalconfig import org.jitsi.nlj.util.Bandwidth import java.time.Duration @@ -116,6 +117,10 @@ class BitrateControllerConfig private constructor() { ) fun maxTimeBetweenCalculations() = maxTimeBetweenCalculations + val assumedBandwidthLimit: Bandwidth? by optionalconfig { + "videobridge.cc.assumed-bandwidth-limit".from(JitsiConfig.newConfig) + } + companion object { @JvmField val config = BitrateControllerConfig() diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index e7186b242e..ffaaa0238b 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -58,6 +58,10 @@ videobridge { # will take the minimum of their setting and this one (-1 implies # no last-n limit) jvb-last-n = -1 + + # If set allows receivers to override bandwidth estimation (BWE) with a specific value signaled over the bridge + # channel (limited to the configured value). If not set, receivers are not allowed to override BWE. + // assumed-bandwidth-limit = 10 Mbps } # Whether to indicate support for cryptex header extension encryption (RFC 9335) cryptex { From c3028dea834adb6cbeb43b46c4fc0ac202460d23 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Mon, 10 Apr 2023 12:46:44 -0500 Subject: [PATCH 010/189] Call conference.addEndpoints() a single time when adding RelayedEndpoints. (#2010) * Call conference.addEndpoints() a single time when adding RelayedEndpoints. --- .../videobridge/colibri2/Colibri2ConferenceHandler.kt | 9 ++++++++- .../main/kotlin/org/jitsi/videobridge/relay/Relay.kt | 11 +++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt index f1a8306da9..f517ba36b1 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt @@ -24,6 +24,7 @@ import org.jitsi.nlj.util.SsrcAssociation import org.jitsi.utils.MediaType import org.jitsi.utils.logging2.Logger import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.videobridge.AbstractEndpoint import org.jitsi.videobridge.Conference import org.jitsi.videobridge.relay.AudioSourceDesc import org.jitsi.videobridge.relay.Relay @@ -436,13 +437,18 @@ class Colibri2ConferenceHandler( /* No need to put media in conference-modified. */ } + // Calls to conference.addEndpoint re-run bandwidth allocation for the existing endpoints in the conference, + // so call it only once. + val newEndpoints = mutableSetOf() c2relay.endpoints?.endpoints?.forEach { endpoint -> if (endpoint.expire) { relay.removeRemoteEndpoint(endpoint.id) } else { val sources = endpoint.parseSourceDescs() if (endpoint.create) { - relay.addRemoteEndpoint(endpoint.id, endpoint.statsId, sources.first, sources.second) + relay.addRemoteEndpoint(endpoint.id, endpoint.statsId, sources.first, sources.second)?.let { + newEndpoints.add(it) + } } else { relay.updateRemoteEndpoint(endpoint.id, sources.first, sources.second) } @@ -455,6 +461,7 @@ class Colibri2ConferenceHandler( } } } + conference.addEndpoints(newEndpoints) /* TODO: handle the rest of the relay's fields: feedback sources. */ return respBuilder.build() diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt index 0b8cf79b95..71e3a85e11 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt @@ -733,18 +733,22 @@ class Relay @JvmOverloads constructor( fun dtlsAppPacketReceived(data: ByteArray, off: Int, len: Int) = sctpHandler.processPacket(PacketInfo(UnparsedPacket(data, off, len))) + /** + * Return the newly created endpoint, or null if an endpoint with that ID already existed. Note that the new + * endpoint has to be added to the [Conference] separately. + */ fun addRemoteEndpoint( id: String, statsId: String?, audioSources: Collection, videoSources: Collection - ) { + ): RelayedEndpoint? { val ep: RelayedEndpoint synchronized(endpointsLock) { if (relayedEndpoints.containsKey(id)) { logger.warn("Relay already contains remote endpoint with ID $id") updateRemoteEndpoint(id, audioSources, videoSources) - return + return null } ep = RelayedEndpoint( conference, @@ -765,8 +769,6 @@ class Relay @JvmOverloads constructor( ep.ssrcs.forEach { ssrc -> endpointsBySsrc[ssrc] = ep } } - conference.addEndpoints(setOf(ep)) - srtpTransformers?.let { ep.setSrtpInformation(it) } payloadTypes.forEach { payloadType -> ep.addPayloadType(payloadType) } rtpExtensions.forEach { rtpExtension -> ep.addRtpExtension(rtpExtension) } @@ -775,6 +777,7 @@ class Relay @JvmOverloads constructor( setEndpointMediaSources(ep, audioSources, videoSources) ep.setFeature(Features.TRANSCEIVER_PCAP_DUMP, transceiver.isFeatureEnabled(Features.TRANSCEIVER_PCAP_DUMP)) + return ep } fun updateRemoteEndpoint( From d808da2e95746cd849f0cec1fabd8b630ede0c50 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Mon, 10 Apr 2023 14:11:55 -0400 Subject: [PATCH 011/189] Don't expire a conference that's in multiple meshes. (#2012) Even if it has no participants - it might be bridging to a visitor relay. --- jvb/src/main/java/org/jitsi/videobridge/Conference.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/jvb/src/main/java/org/jitsi/videobridge/Conference.java b/jvb/src/main/java/org/jitsi/videobridge/Conference.java index 8d163dd30c..4af444322e 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Conference.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Conference.java @@ -1075,6 +1075,14 @@ public Logger getLogger() return logger; } + /** + * @return {@code true} if this {@link Conference} is in more than one relay mesh. + */ + private boolean inMultipleMeshes() + { + return relaysById.values().stream().map(Relay::getMeshId).collect(Collectors.toSet()).size() > 1; + } + /** * @return {@code true} if this {@link Conference} is ready to be expired. */ @@ -1082,6 +1090,7 @@ public boolean shouldExpire() { // Allow a conference to have no endpoints in the first 20 seconds. return getEndpointCount() == 0 + && !inMultipleMeshes() && (System.currentTimeMillis() - creationTime > 20000); } From 06796321cd40947f94ca84e83560353e1084ed37 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Mon, 10 Apr 2023 14:12:19 -0400 Subject: [PATCH 012/189] Don't call _mediaSources.getMediaSources() twice when setting RelayedEndpoint.mediaSources. (#2011) --- .../main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt index 2509d55639..08b58a8604 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt @@ -176,7 +176,7 @@ class RelayedEndpoint( val mergedMediaSources = _mediaSources.getMediaSources() val signaledMediaSources = value.copy() if (changed) { - val setMediaSourcesEvent = SetMediaSourcesEvent(mediaSources, signaledMediaSources) + val setMediaSourcesEvent = SetMediaSourcesEvent(mergedMediaSources, signaledMediaSources) rtpReceiver.handleEvent(setMediaSourcesEvent) mediaSources.forEach { From 18476a94ed10fb716bb6367dadc1cf7ce0bf210d Mon Sep 17 00:00:00 2001 From: bgrozev Date: Mon, 10 Apr 2023 13:41:30 -0500 Subject: [PATCH 013/189] Add meshId to debug state. (#2009) * Add meshId to debug state. --- jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt index 71e3a85e11..cbc545835a 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt @@ -301,6 +301,7 @@ class Relay @JvmOverloads constructor( put("iceTransport", iceTransport.getDebugState()) put("dtlsTransport", dtlsTransport.getDebugState()) put("transceiver", transceiver.getNodeStats().toJson()) + put("meshId", meshId) put("messageTransport", messageTransport.debugState) val remoteEndpoints = JSONObject() val endpointsBySsrcMap = JSONObject() From 4fb508203d6339b56d28b35ab6248a591c00c9b7 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Wed, 12 Apr 2023 15:12:55 -0500 Subject: [PATCH 014/189] Add stats for packet rate organized by conference size. (#2013) * Add stats for packet/byte count organized by conference size. --- .../videobridge/rest/root/debug/Debug.java | 3 + .../rest/root/debug/DebugFeatures.java | 3 +- .../stats/VideobridgeStatistics.java | 11 +++ .../stats/ConferencePacketStats.kt | 82 +++++++++++++++++++ .../stats/config/StatsManagerConfig.kt | 4 +- 5 files changed, 99 insertions(+), 4 deletions(-) create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/stats/ConferencePacketStats.kt diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/Debug.java b/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/Debug.java index 9e20fc95ef..8a670e0c1d 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/Debug.java +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/Debug.java @@ -386,6 +386,9 @@ public String getJvbFeatureStats(@PathParam("feature") DebugFeatures feature) case ICE_STATS: { return IceStatistics.Companion.getStats().toJson().toJSONString(); } + case CONFERENCE_PACKET_STATS: { + return ConferencePacketStats.stats.toJson().toJSONString(); + } case TOSSED_PACKET_STATS: { return videobridge.getStatistics().tossedPacketsEnergy.toJson().toJSONString(); } diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/DebugFeatures.java b/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/DebugFeatures.java index 10b4ce1ca9..dc1ca6bc20 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/DebugFeatures.java +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/DebugFeatures.java @@ -33,7 +33,8 @@ public enum DebugFeatures NODE_TRACING("node-tracing"), ICE_STATS("ice-stats"), XMPP_DELAY_STATS("xmpp-delay-stats"), - TOSSED_PACKET_STATS("tossed-packet-stats"); + TOSSED_PACKET_STATS("tossed-packet-stats"), + CONFERENCE_PACKET_STATS("conference-packet-stats"); private final String value; diff --git a/jvb/src/main/java/org/jitsi/videobridge/stats/VideobridgeStatistics.java b/jvb/src/main/java/org/jitsi/videobridge/stats/VideobridgeStatistics.java index 5442c8d3c2..2613d033b6 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/stats/VideobridgeStatistics.java +++ b/jvb/src/main/java/org/jitsi/videobridge/stats/VideobridgeStatistics.java @@ -275,6 +275,8 @@ private void generate0() for (Conference conference : videobridge.getConferences()) { + long conferenceBitrate = 0; + long conferencePacketRate = 0; if (conference.isP2p()) { p2pConferences++; @@ -345,6 +347,8 @@ private void generate0() = transceiverStats.getRtpReceiverStats().getPacketStreamStats(); bitrateDownloadBps += incomingPacketStreamStats.getBitrateBps(); packetRateDownload += incomingPacketStreamStats.getPacketRate(); + conferenceBitrate += incomingPacketStreamStats.getBitrateBps(); + conferencePacketRate += incomingPacketStreamStats.getPacketRate(); for (IncomingSsrcStats.Snapshot ssrcStats : incomingStats.getSsrcStats().values()) { double ssrcJitter = ssrcStats.getJitter(); @@ -360,6 +364,8 @@ private void generate0() PacketStreamStats.Snapshot outgoingStats = transceiverStats.getOutgoingPacketStreamStats(); bitrateUploadBps += outgoingStats.getBitrateBps(); packetRateUpload += outgoingStats.getPacketRate(); + conferenceBitrate += outgoingStats.getBitrateBps(); + conferencePacketRate += outgoingStats.getPacketRate(); EndpointConnectionStats.Snapshot endpointConnectionStats = transceiverStats.getEndpointConnectionStats(); @@ -394,9 +400,13 @@ private void generate0() { relayBitrateIncomingBps += relay.getIncomingBitrateBps(); relayPacketRateIncoming += relay.getIncomingPacketRate(); + conferenceBitrate += relay.getIncomingBitrateBps(); + conferencePacketRate += relay.getIncomingPacketRate(); relayBitrateOutgoingBps += relay.getOutgoingBitrateBps(); relayPacketRateOutgoing += relay.getOutgoingPacketRate(); + conferenceBitrate += relay.getOutgoingBitrateBps(); + conferencePacketRate += relay.getOutgoingPacketRate(); /* TODO: report Relay RTT and loss, like we do for Endpoints? */ } @@ -405,6 +415,7 @@ private void generate0() numAudioSenders += conferenceAudioSenders; updateBuckets(videoSendersBuckets, conferenceVideoSenders); numVideoSenders += conferenceVideoSenders; + ConferencePacketStats.stats.addValue(numConferenceEndpoints, conferencePacketRate, conferenceBitrate); } // JITTER_AGGREGATE diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/ConferencePacketStats.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/ConferencePacketStats.kt new file mode 100644 index 0000000000..6f71b9c4ca --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/ConferencePacketStats.kt @@ -0,0 +1,82 @@ +/* + * Copyright @ 2023 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.stats + +import org.jitsi.videobridge.stats.config.StatsManagerConfig +import org.json.simple.JSONArray +import org.json.simple.JSONObject +import java.util.concurrent.atomic.AtomicLong + +/** + * Saves statistics for packets and bytes based on conference size. + * + * We assume the stats are updated periodically at an interval of [periodSeconds] seconds. This way we can convert + * the rates passed to [addValue] to number of packets and bytes. + */ +class ConferencePacketStats private constructor() { + private val periodSeconds = StatsManagerConfig.config.interval.toMillis().toDouble() / 1000 + + /** Maps a conference size to a [Stats] instance that sums the added packet and bit rates. */ + private val stats: Map = mutableMapOf().apply { + (0..MAX_SIZE).forEach { + this[it] = Stats() + } + } + + val totalPackets = AtomicLong() + val totalBytes = AtomicLong() + + fun addValue(conferenceSize: Int, packetRatePps: Long, bitrateBps: Long) { + stats[conferenceSize.coerceAtMost(MAX_SIZE)]?.let { + val packets = (packetRatePps * periodSeconds).toLong() + val bytes = (bitrateBps * periodSeconds / 8).toLong() + + it.packets.addAndGet(packets) + it.bytes.addAndGet(bytes) + totalPackets.addAndGet(packets) + totalBytes.addAndGet(bytes) + } + } + + fun toJson() = JSONObject().apply { + val packetRates = JSONArray() + val bitrates = JSONArray() + + (0..MAX_SIZE).forEach { conferenceSize -> + stats[conferenceSize]?.let { + packetRates.add(it.packets.get()) + bitrates.add(it.bytes.get()) + } + } + + put("packets", packetRates) + put("bytes", bitrates) + put("total_packets", totalPackets.get()) + put("total_bytes", totalBytes.get()) + } + + companion object { + const val MAX_SIZE = 500 + + @JvmField + val stats = ConferencePacketStats() + } + + private class Stats { + val packets = AtomicLong() + val bytes = AtomicLong() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/config/StatsManagerConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/config/StatsManagerConfig.kt index 5f8eee8546..9c7a790b58 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/config/StatsManagerConfig.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/config/StatsManagerConfig.kt @@ -26,9 +26,7 @@ import org.jitsi.videobridge.xmpp.XmppConnection import java.time.Duration class StatsManagerConfig private constructor() { - /** - * The interval at which the stats are pushed - */ + /** The interval at which the stats are collected. */ val interval: Duration by config { "org.jitsi.videobridge.STATISTICS_INTERVAL".from(JitsiConfig.legacyConfig).convertFrom(Duration::ofMillis) "videobridge.stats.interval".from(JitsiConfig.newConfig) From 45b29663c15c55f7d000ae3e9fb11559cce808c6 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Wed, 19 Apr 2023 18:30:31 -0400 Subject: [PATCH 015/189] Update to Jetty 11.0.14. (#2014) * Update to Jetty 11.0.14. (Including jicoco dependency.) * Bump jicoco, and jersey version. This also necessitated bumping junit and kotest versions for tests, and fixing one test syntax. * Exclude some spotbugs errors from new kotest. * Pull in the correct junit-jupiter-api version. * Fix some deprecated kotest API usages. --- jitsi-media-transform/pom.xml | 8 +++++++ jitsi-media-transform/spotbugs-exclude.xml | 2 ++ .../org/jitsi/nlj/KotestProjectConfig.kt | 2 +- jvb/pom.xml | 23 ++++++++----------- .../org/jitsi/videobridge/VideobridgeTest.kt | 14 +++++------ pom.xml | 10 ++++---- rtp/spotbugs-exclude.xml | 1 + .../jitsi/test_helpers/matchers/RtpPacket.kt | 6 +++-- 8 files changed, 37 insertions(+), 29 deletions(-) diff --git a/jitsi-media-transform/pom.xml b/jitsi-media-transform/pom.xml index 6d9eb2cdb7..39432a5578 100644 --- a/jitsi-media-transform/pom.xml +++ b/jitsi-media-transform/pom.xml @@ -115,6 +115,14 @@ ${kotest.version} test + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + io.mockk mockk diff --git a/jitsi-media-transform/spotbugs-exclude.xml b/jitsi-media-transform/spotbugs-exclude.xml index ac890dcfab..3e072793f8 100644 --- a/jitsi-media-transform/spotbugs-exclude.xml +++ b/jitsi-media-transform/spotbugs-exclude.xml @@ -34,6 +34,8 @@ + + diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/KotestProjectConfig.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/KotestProjectConfig.kt index db36cfd7d3..c03b59a8a2 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/KotestProjectConfig.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/KotestProjectConfig.kt @@ -20,7 +20,7 @@ import io.kotest.core.config.AbstractProjectConfig import org.jitsi.metaconfig.MetaconfigSettings class KotestProjectConfig : AbstractProjectConfig() { - override fun beforeAll() = super.beforeAll().also { + override suspend fun beforeProject() = super.beforeProject().also { // The only purpose of config caching is performance. We always want caching disabled in tests (so we can // freely modify the config without affecting other tests executing afterwards). MetaconfigSettings.cacheEnabled = false diff --git a/jvb/pom.xml b/jvb/pom.xml index 2ba5eeecc2..2dbd3f4659 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -217,12 +217,7 @@ org.glassfish.jersey.test-framework jersey-test-framework-core ${jersey.version} - - - junit - junit - - + test org.glassfish.jersey.test-framework.providers @@ -255,6 +250,14 @@ ${junit.version} test + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + io.kotest kotest-runner-junit5-jvm @@ -272,14 +275,6 @@ jicoco-test-kotlin test - - - org.junit.platform - junit-platform-engine - 1.8.1 - test - javax.xml.bind jaxb-api diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/VideobridgeTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/VideobridgeTest.kt index 60a009c99d..960a2084a2 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/VideobridgeTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/VideobridgeTest.kt @@ -73,15 +73,15 @@ class VideobridgeTest : ShouldSpec() { verify(exactly = 1) { shutdownService.beginShutdown() } } context("When the graceful shutdown period expires") { + fakeExecutor.clock.elapse(ShutdownConfig.config.gracefulShutdownMaxDuration) + fakeExecutor.run() should("go to SHUTTING_DOWN") { - fakeExecutor.clock.elapse(ShutdownConfig.config.gracefulShutdownMaxDuration) - fakeExecutor.run() videobridge.shutdownState shouldBe ShutdownState.SHUTTING_DOWN - should("and then shut down after shuttingDownDelay") { - fakeExecutor.clock.elapse(ShutdownConfig.config.shuttingDownDelay) - fakeExecutor.run() - verify(exactly = 1) { shutdownService.beginShutdown() } - } + } + should("and then shut down after shuttingDownDelay") { + fakeExecutor.clock.elapse(ShutdownConfig.config.shuttingDownDelay) + fakeExecutor.run() + verify(exactly = 1) { shutdownService.beginShutdown() } } } } diff --git a/pom.xml b/pom.xml index 29a42890e8..e6dc087f0e 100644 --- a/pom.xml +++ b/pom.xml @@ -23,16 +23,16 @@ pom - 11.0.10 + 11.0.14 1.6.21 - 5.3.0 - 5.8.2 + 5.5.5 + 5.9.1 1.0-124-ge57838f - 1.1-123-g68019c1 + 1.1-125-g805c0d8 1.13.1 3.2.2 4.6.0 - 3.0.4 + 3.0.10 2.12.4 1.70 0.16.0 diff --git a/rtp/spotbugs-exclude.xml b/rtp/spotbugs-exclude.xml index 7ccd876806..b9956fab85 100644 --- a/rtp/spotbugs-exclude.xml +++ b/rtp/spotbugs-exclude.xml @@ -14,6 +14,7 @@ + diff --git a/rtp/src/test/kotlin/org/jitsi/test_helpers/matchers/RtpPacket.kt b/rtp/src/test/kotlin/org/jitsi/test_helpers/matchers/RtpPacket.kt index ba05ee5159..01bdba0ec2 100644 --- a/rtp/src/test/kotlin/org/jitsi/test_helpers/matchers/RtpPacket.kt +++ b/rtp/src/test/kotlin/org/jitsi/test_helpers/matchers/RtpPacket.kt @@ -38,8 +38,10 @@ fun haveSamePayload(expected: RtpPacket) = object : Matcher { return MatcherResult( valuePayload.hasSameContentAs(expectedPayload), - "\n${valuePayload.toHex()}\nwas supposed to be:\n${expectedPayload.toHex()}", - "\n${valuePayload.toHex()}\nshould not have equaled \n${expectedPayload.toHex()}" + { "\n${valuePayload.toHex()}\nwas supposed to be:\n${expectedPayload.toHex()}" }, + { + "\n${valuePayload.toHex()}\nshould not have equaled \n${expectedPayload.toHex()}" + } ) } } From 6351e0dce9c99c5c72ff6324b0121e4167047180 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 20 Apr 2023 13:51:54 -0500 Subject: [PATCH 016/189] Fix XMPP health checks (#2015) * Remove unnecessary try/catch. * Actually return health check result. --- .../org/jitsi/videobridge/Videobridge.java | 38 ++++++------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java b/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java index 82e3a82e8c..68b3078eec 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java @@ -18,6 +18,7 @@ import kotlin.*; import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.*; +import org.jitsi.health.Result; import org.jitsi.metrics.*; import org.jitsi.nlj.*; import org.jitsi.shutdown.*; @@ -492,31 +493,6 @@ private void handleColibriRequest(XmppConnection.ColibriRequest request) } } - /** - * Handles HealthCheckIQ by performing health check on this - * Videobridge instance. - * - * @param healthCheckIQ the HealthCheckIQ to be handled. - * @return IQ with "result" type if the health check succeeded or - * IQ with "error" type if something went wrong. - * {@link StanzaError.Condition#internal_server_error} is returned when the - * health check fails or {@link StanzaError.Condition#not_authorized} if the - * request comes from a JID that is not authorized to do health checks on - * this instance. - */ - public IQ handleHealthCheckIQ(HealthCheckIQ healthCheckIQ) - { - try - { - return IQ.createResultIQ(healthCheckIQ); - } - catch (Exception e) - { - logger.warn("Exception while handling health check IQ request", e); - return createError(healthCheckIQ, StanzaError.Condition.internal_server_error, e.getMessage()); - } - } - /** * Handles a shutdown request. */ @@ -799,7 +775,17 @@ public IQ versionIqReceived(@NotNull org.jivesoftware.smackx.iqversion.packet.Ve @Override public IQ healthCheckIqReceived(@NotNull HealthCheckIQ iq) { - return handleHealthCheckIQ(iq); + Result result = jvbHealthChecker.getResult(); + if (result.getSuccess()) + { + return IQ.createResultIQ(iq); + } + else + { + return IQ.createErrorResponse( + iq, + StanzaError.from(StanzaError.Condition.internal_server_error, result.getMessage()).build()); + } } } From 1fd69725a78d633e693f37fe85db204c4df98532 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Fri, 21 Apr 2023 16:12:53 -0400 Subject: [PATCH 017/189] Fix reversed condition adding websockets to transport parameters. (#2016) This fixes inter-relay bridge channels when SCTP datachannels are disabled. --- jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt index cbc545835a..398ddfc082 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt @@ -577,7 +577,7 @@ class Relay @JvmOverloads constructor( iceTransport.describe(iceUdpTransportPacketExtension) dtlsTransport.describe(iceUdpTransportPacketExtension) - if (sctpSocket != null) { + if (sctpSocket == null) { /* TODO: this should be dependent on videobridge.websockets.enabled, if we support that being * disabled for relay. */ From 6befa625c2ac3372ab20a984bba13a8e404cca37 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Mon, 24 Apr 2023 13:58:58 -0500 Subject: [PATCH 018/189] fix: Fix sending preemptive keyframe requests with source names, do not send for screensharing. (#2017) * fix: Fix sending preemptive keyframe requests with source names, do not send for screensharing. --- .../org/jitsi/videobridge/Conference.java | 21 ++++++++++++++++--- .../kotlin/org/jitsi/videobridge/Endpoint.kt | 6 ++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/Conference.java b/jvb/src/main/java/org/jitsi/videobridge/Conference.java index 4af444322e..6b5de44800 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Conference.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Conference.java @@ -491,12 +491,27 @@ private void maybeSendKeyframeRequest(AbstractEndpoint dominantSpeaker) } boolean anyEndpointInStageView = false; + Set allOnStageSourceNames = new HashSet<>(); for (Endpoint otherEndpoint : getLocalEndpoints()) { - if (otherEndpoint != dominantSpeaker && otherEndpoint.isInStageView()) + if (otherEndpoint != dominantSpeaker) { - anyEndpointInStageView = true; - break; + allOnStageSourceNames.addAll(otherEndpoint.getOnStageSources()); + } + } + + for (String onStageSourceName : allOnStageSourceNames) + { + AbstractEndpoint owner = findSourceOwner(onStageSourceName); + if (owner != null) + { + // Do not anticipate a switch if all on-stage sources are DESKTOP + MediaSourceDesc onStageSource = owner.findMediaSourceDesc(onStageSourceName); + if (onStageSource != null && onStageSource.getVideoType() == VideoType.CAMERA) + { + anyEndpointInStageView = true; + break; + } } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index 2f39fae79d..04eec27ffc 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -401,9 +401,11 @@ class Endpoint @JvmOverloads constructor( } /** - * Whether this endpoint has any endpoints "on-stage". + * Return the list of sources the endpoint has selected as "on stage". We just concatenate with the old + * "onStageEndpoints" since even with the old API we have matching source names. */ - fun isInStageView() = bitrateController.allocationSettings.onStageEndpoints.isNotEmpty() + fun getOnStageSources() = + bitrateController.allocationSettings.onStageEndpoints + bitrateController.allocationSettings.onStageSources private fun setupDtlsTransport() { dtlsTransport.incomingDataHandler = object : DtlsTransport.IncomingDataHandler { From 0ad3fc6b6473d8141092619bc18789983925cfb9 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Tue, 25 Apr 2023 10:41:06 -0500 Subject: [PATCH 019/189] fix: Correctly read assumed-bandwidth-limit as a Bandwidth. (#2018) --- .../org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt index 347708b7d2..6f25167015 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt @@ -119,6 +119,7 @@ class BitrateControllerConfig private constructor() { val assumedBandwidthLimit: Bandwidth? by optionalconfig { "videobridge.cc.assumed-bandwidth-limit".from(JitsiConfig.newConfig) + .convertFrom { Bandwidth.fromString(it) } } companion object { From b286dc0cad073331c9a40972f9de8c2acd3b9e28 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Tue, 25 Apr 2023 18:53:49 -0500 Subject: [PATCH 020/189] fix: Fix PacketCacher stats aggregation. (#2019) --- .../main/kotlin/org/jitsi/nlj/transform/node/PacketCacher.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PacketCacher.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PacketCacher.kt index a4dfa693db..9ccdbc8967 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PacketCacher.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PacketCacher.kt @@ -37,7 +37,8 @@ class PacketCacher : ObserverNode("Packet cache") { override fun getNodeStats(): NodeStatsBlock { return super.getNodeStats().apply { - addBlock(packetCache.getNodeStats()) + // Put the values directly in the node stats and not inside a block to make aggregation work correctly. + aggregate(packetCache.getNodeStats()) } } } From fc17337e46c96747fa89738c836255aeda21e957 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Tue, 2 May 2023 17:20:53 -0500 Subject: [PATCH 021/189] Log uncaught exceptions. (#2020) --- jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt index 7dd1cf0897..5c8b8d8669 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt @@ -51,6 +51,10 @@ import org.jitsi.videobridge.websocket.singleton as webSocketServiceSingleton fun main() { val logger = LoggerImpl("org.jitsi.videobridge.Main") + Thread.setDefaultUncaughtExceptionHandler { thread, exception -> + logger.error("An uncaught exception occurred in thread=$thread", exception) + } + setupMetaconfigLogger() setSystemPropertyDefaults() From 9eefa65d62030e8733ed9c9cff999a8e310dd7ad Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Mon, 8 May 2023 13:58:24 -0400 Subject: [PATCH 022/189] Disable SCTP UDP encapsulation. (#2022) --- .../main/java/org/jitsi/videobridge/sctp/SctpManager.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java b/jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java index 7dd23f4cae..72e0cacce8 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java +++ b/jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java @@ -60,9 +60,9 @@ public class SctpManager if (config.enabled()) { classLogger.info("Initializing Sctp4j"); - // TODO: We pass DEFAULT_SCTP_PORT as the udp_port parameter to usrsctp_init, which probably doesn't make - // sense. - Sctp4j.init(DEFAULT_SCTP_PORT); + // "If UDP encapsulation is not necessary, the UDP port has to be set to 0" + // All our SCTP is encapsulated in DTLS, we don't use direct UDP encapsulation. + Sctp4j.init(0); } else { From 6d865d1197bf25625c435b5a0ae46d8a9c48db08 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 18 May 2023 05:00:26 -0700 Subject: [PATCH 023/189] fix: Fix broken audio when SSRC rewriting with relays (#2024) Fixes finding the AudioSourceDesc when it belongs to a RelayedEndpoint. --- jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt | 9 ++++++--- jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt | 6 +++--- .../org/jitsi/videobridge/relay/RelayedEndpoint.kt | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index 04eec27ffc..afd3beae28 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -67,6 +67,7 @@ import org.jitsi.videobridge.message.ReceiverVideoConstraintsMessage import org.jitsi.videobridge.message.SenderSourceConstraintsMessage import org.jitsi.videobridge.message.SenderVideoConstraintsMessage import org.jitsi.videobridge.relay.AudioSourceDesc +import org.jitsi.videobridge.relay.RelayedEndpoint import org.jitsi.videobridge.rest.root.debug.EndpointDebugFeatures import org.jitsi.videobridge.sctp.DataChannelHandler import org.jitsi.videobridge.sctp.SctpHandler @@ -878,9 +879,11 @@ class Endpoint @JvmOverloads constructor( */ override fun findAudioSourceProps(ssrc: Long): AudioSourceDesc? { conference.getEndpointBySsrc(ssrc)?.let { ep -> - if (ep !is Endpoint) - return null - return ep.audioSources.find { s -> s.ssrc == ssrc } + return when (ep) { + is Endpoint -> ep.audioSources + is RelayedEndpoint -> ep.audioSources + else -> emptyList() + }.find { it.ssrc == ssrc } } logger.error { "No properties found for SSRC $ssrc." } return null diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt index 398ddfc082..4e4aea9441 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt @@ -762,7 +762,7 @@ class Relay @JvmOverloads constructor( } ) ep.statsId = statsId - ep.audioSources = audioSources.toTypedArray() + ep.audioSources = audioSources.toList() ep.mediaSources = videoSources.toTypedArray() relayedEndpoints[id] = ep @@ -794,7 +794,7 @@ class Relay @JvmOverloads constructor( } val oldSsrcs = ep.ssrcs - ep.audioSources = audioSources.toTypedArray() + ep.audioSources = audioSources.toList() ep.mediaSources = videoSources.toTypedArray() val newSsrcs = ep.ssrcs @@ -883,7 +883,7 @@ class Relay @JvmOverloads constructor( audioSources: Collection, videoSources: Collection ) { - ep.audioSources = audioSources.toTypedArray() + ep.audioSources = audioSources.toList() ep.mediaSources = videoSources.toTypedArray() } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt index 08b58a8604..5f5c92d66b 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt @@ -57,7 +57,7 @@ class RelayedEndpoint( parentLogger: Logger, diagnosticContext: DiagnosticContext ) : AbstractEndpoint(conference, id, parentLogger), Relay.IncomingRelayPacketHandler { - var audioSources: Array = arrayOf() + var audioSources: List = listOf() set(value) { field = value value.forEach { From 9563221a45561ec00f15d58175bbe0a8242fc47a Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 18 May 2023 07:47:51 -0700 Subject: [PATCH 024/189] log: Remove redundant log context. (#2023) The Relay already has its ID in the context resulting in: relayId=jvb-foo-bar relay-id=jvb-foo-bar --- .../kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt index 2754ed9ea5..3cf8ba5424 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt @@ -93,8 +93,6 @@ class RelayMessageTransport( */ private val sentMessagesCounts: MutableMap = ConcurrentHashMap() - init { logger.addContext("relay-id", relay.id) } - /** * Connect the bridge channel message to the websocket URL specified */ From 0c842f0da26cec185a9f2f0fb826abf4f8940062 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 25 May 2023 14:39:38 -0700 Subject: [PATCH 025/189] fix: Do not update RTT when the calculated value is invalid (#2025) or suspiciously high, or the sender's "delay since last SR" values is too high. --- .../nlj/stats/EndpointConnectionStats.kt | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/EndpointConnectionStats.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/EndpointConnectionStats.kt index fb5a974d65..92e666999b 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/EndpointConnectionStats.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/EndpointConnectionStats.kt @@ -144,20 +144,30 @@ class EndpointConnectionStats( // The delaySinceLastSr value is given in 1/65536ths of a second, so divide it by .000065536 to get it // in nanoseconds val remoteProcessingDelay = Duration.ofNanos((reportBlock.delaySinceLastSr / .000065536).toLong()) - rtt = (Duration.between(srSentTime, receivedTime) - remoteProcessingDelay).toDoubleMillis() - if (rtt > 7.secs.toMillis()) { + if (remoteProcessingDelay > Duration.ofMinutes(5)) { + logger.warn("Ignoring report block with suspiciously long DLSR: $remoteProcessingDelay") + return@let + } + + val newRtt = (Duration.between(srSentTime, receivedTime) - remoteProcessingDelay).toDoubleMillis() + if (newRtt > 7.secs.toMillis()) { logger.warn( - "Suspiciously high rtt value: $rtt ms, remote processing delay was " + + "Ignoring suspiciously high rtt value: $newRtt ms, remote processing delay was " + "$remoteProcessingDelay (${reportBlock.delaySinceLastSr}), srSentTime was $srSentTime, " + "received time was $receivedTime" ) - } else if (rtt < 0) { + return@let + } + if (newRtt < 0) { logger.warn( - "Negative rtt value: $rtt ms, remote processing delay was " + + "Negative rtt value: $newRtt ms, remote processing delay was " + "$remoteProcessingDelay (${reportBlock.delaySinceLastSr}), srSentTime was $srSentTime, " + "received time was $receivedTime" ) + return@let } + + rtt = newRtt endpointConnectionStatsListeners.forEach { it.onRttUpdate(rtt) } } ?: run { logger.cdebug { From 1da507fa553f7076a7bd4edbf403826af3fc1768 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Fri, 2 Jun 2023 15:51:28 -0400 Subject: [PATCH 026/189] Use source names as keys to effective constraints debug state. (#2026) --- .../org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt index a87e1d792d..e36761673a 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt @@ -124,7 +124,7 @@ internal class BandwidthAllocator( debugState["bweBps"] = bweBps debugState["allocation"] = allocation.debugState debugState["allocationSettings"] = allocationSettings.toJson() - debugState["effectiveConstraints"] = effectiveConstraints + debugState["effectiveConstraints"] = effectiveConstraints.mapKeys { it.key.sourceName } return debugState } From 5e55fdf078585dacedfe246674ce20758f501c2b Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Thu, 8 Jun 2023 12:42:28 -0400 Subject: [PATCH 027/189] Generalize SsrcCache codec-specific packet rewriting code. (#2027) --- .../kotlin/org/jitsi/videobridge/SsrcCache.kt | 147 +++++++++++------- 1 file changed, 93 insertions(+), 54 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt index 30b6af2824..f75523a4ac 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt @@ -18,6 +18,7 @@ package org.jitsi.videobridge import org.jitsi.nlj.MediaSourceDesc import org.jitsi.nlj.VideoType +import org.jitsi.nlj.codec.vpx.VpxUtils import org.jitsi.nlj.rtp.SsrcAssociationType import org.jitsi.nlj.rtp.codec.vp8.Vp8Packet import org.jitsi.nlj.rtp.codec.vp9.Vp9Packet @@ -38,39 +39,6 @@ import org.jitsi.videobridge.message.VideoSourcesMap import org.jitsi.videobridge.relay.AudioSourceDesc import org.json.simple.JSONObject -/** - * Get tl0PicIdx field for codecs that have it. - * Return -1 if not applicable. - */ -private fun RtpPacket.getTl0Index(): Int { - return when (this) { - is Vp9Packet -> this.TL0PICIDX - is Vp8Packet -> this.TL0PICIDX - else -> -1 - } -} - -/** - * Set tl0PicIdx field for codecs that have it. - * No-op if not applicable. - */ -private fun RtpPacket.setTl0Index(tl0Index: Int) { - when (this) { - is Vp9Packet -> this.TL0PICIDX = tl0Index - is Vp8Packet -> this.TL0PICIDX = tl0Index - } -} - -/** - * Addition clipped to 8 unsigned bits. - */ -private infix fun Int.bytePlus(x: Int) = this.plus(x) and 0xff - -/** - * Subtraction clipped to 8 unsigned bits. - */ -private infix fun Int.byteMinus(x: Int) = this.minus(x) and 0xff - /** * Align common fields from different source types. * Perhaps this could become a base class of those types. @@ -105,20 +73,20 @@ class SourceDesc private constructor( class RtpState { var lastSequenceNumber = 0 var lastTimestamp = 0L - var lastTl0Index = -1 + var codecState: CodecState? = null var valid = false fun update(packet: RtpPacket) { lastSequenceNumber = packet.sequenceNumber lastTimestamp = packet.timestamp - lastTl0Index = packet.getTl0Index() + codecState = packet.getCodecState() valid = true } /** * {@inheritDoc} */ - override fun toString(): String = if (valid) "$lastSequenceNumber/$lastTimestamp/$lastTl0Index" else "-" + override fun toString(): String = if (valid) "$lastSequenceNumber/$lastTimestamp/$codecState" else "-" } /** @@ -146,7 +114,7 @@ class SendSsrc(val ssrc: Long) { private val state = RtpState() private var sequenceNumberDelta = 0 private var timestampDelta = 0L - private var tl0IndexDelta = 0 + private var codecDeltas: CodecDeltas? = null /** * Update RTP state and apply deltas. @@ -154,8 +122,6 @@ class SendSsrc(val ssrc: Long) { fun rewriteRtp(packet: RtpPacket, sending: Boolean, recv: ReceiveSsrc) { if (sending) { - val tl0Index = packet.getTl0Index() - if (!recv.hasDeltas) { /* Calculate new deltas the first time a receive ssrc is mapped to a send ssrc. */ if (state.valid) { @@ -164,10 +130,7 @@ class SendSsrc(val ssrc: Long) { RtpUtils.getSequenceNumberDelta(state.lastSequenceNumber, recv.state.lastSequenceNumber) timestampDelta = RtpUtils.getTimestampDiff(state.lastTimestamp, recv.state.lastTimestamp) - if (state.lastTl0Index != -1 && recv.state.lastTl0Index != 1) - tl0IndexDelta = state.lastTl0Index byteMinus recv.state.lastTl0Index - else - tl0IndexDelta = 0 + codecDeltas = state.codecState?.getDeltas(recv.state.codecState) } else { val prevSequenceNumber = RtpUtils.applySequenceNumberDelta(packet.sequenceNumber, -1) @@ -177,10 +140,7 @@ class SendSsrc(val ssrc: Long) { RtpUtils.getSequenceNumberDelta(state.lastSequenceNumber, prevSequenceNumber) timestampDelta = RtpUtils.getTimestampDiff(state.lastTimestamp, prevTimestamp) - if (state.lastTl0Index != -1 && tl0Index != -1) - tl0IndexDelta = state.lastTl0Index byteMinus (tl0Index - 1) - else - tl0IndexDelta = 0 + codecDeltas = state.codecState?.getDeltas(packet) } } recv.hasDeltas = true @@ -191,9 +151,7 @@ class SendSsrc(val ssrc: Long) { packet.ssrc = ssrc packet.sequenceNumber = RtpUtils.applySequenceNumberDelta(packet.sequenceNumber, sequenceNumberDelta) packet.timestamp = RtpUtils.applyTimestampDelta(packet.timestamp, timestampDelta) - if (tl0Index != -1) { - packet.setTl0Index(tl0Index bytePlus tl0IndexDelta) - } + codecDeltas?.rewritePacket(packet) state.update(packet) } else { @@ -218,7 +176,7 @@ class SendSsrc(val ssrc: Long) { /** * {@inheritDoc} */ - override fun toString(): String = "$ssrc{$state,\u2206=$sequenceNumberDelta/$timestampDelta/$tl0IndexDelta}" + override fun toString(): String = "$ssrc{$state,\u2206=$sequenceNumberDelta/$timestampDelta/$codecDeltas}" } /** @@ -329,9 +287,9 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: * Print packet fields relevant to rewriting mode. */ private fun debugInfo(packet: RtpPacket): String { - val tl0Index = packet.getTl0Index() - val tl0Info = if (tl0Index != -1) " tl0PicIdx=$tl0Index" else "" - return "ssrc=${packet.ssrc} seq=${packet.sequenceNumber} ts=${packet.timestamp}" + tl0Info + val codecState = packet.getCodecState() + val codecInfo = codecState?.toString() ?: "" + return "ssrc=${packet.ssrc} seq=${packet.sequenceNumber} ts=${packet.timestamp}" + codecInfo } } @@ -619,3 +577,84 @@ class VideoSsrcCache(size: Int, ep: SsrcRewriter, parentLogger: Logger) : } } } + +/** Codec-specific packet state. */ +interface CodecState { + fun getDeltas(otherState: CodecState?): CodecDeltas? + fun getDeltas(packet: RtpPacket): CodecDeltas? +} + +/** Codec-specific packet deltas. */ +interface CodecDeltas { + fun rewritePacket(packet: RtpPacket) +} + +private class Vp8CodecState(val lastTl0Index: Int) : CodecState { + constructor(packet: Vp8Packet) : this(packet.TL0PICIDX) + + override fun getDeltas(otherState: CodecState?): CodecDeltas? { + if (otherState !is Vp8CodecState) { + return null + } + val tl0IndexDelta = VpxUtils.getTl0PicIdxDelta(lastTl0Index, otherState.lastTl0Index) + return Vp8CodecDeltas(tl0IndexDelta) + } + + override fun getDeltas(packet: RtpPacket): CodecDeltas? { + if (packet !is Vp8Packet) { + return null + } + val tl0IndexDelta = VpxUtils.getTl0PicIdxDelta(lastTl0Index, (packet.TL0PICIDX - 1)) + return Vp8CodecDeltas(tl0IndexDelta) + } + + override fun toString() = "[VP8 TL0Idx]$lastTl0Index" +} + +private class Vp8CodecDeltas(val tl0IndexDelta: Int) : CodecDeltas { + override fun rewritePacket(packet: RtpPacket) { + require(packet is Vp8Packet) + packet.TL0PICIDX = VpxUtils.applyTl0PicIdxDelta(packet.TL0PICIDX, tl0IndexDelta) + } + + override fun toString() = "[VP8 TL0Idx]$tl0IndexDelta" +} + +private class Vp9CodecState(val lastTl0Index: Int) : CodecState { + constructor(packet: Vp9Packet) : this(packet.TL0PICIDX) + + override fun getDeltas(otherState: CodecState?): CodecDeltas? { + if (otherState !is Vp9CodecState) { + return null + } + val tl0IndexDelta = VpxUtils.getTl0PicIdxDelta(lastTl0Index, otherState.lastTl0Index) + return Vp9CodecDeltas(tl0IndexDelta) + } + + override fun getDeltas(packet: RtpPacket): CodecDeltas? { + if (packet !is Vp9Packet) { + return null + } + val tl0IndexDelta = VpxUtils.getTl0PicIdxDelta(lastTl0Index, (packet.TL0PICIDX - 1)) + return Vp9CodecDeltas(tl0IndexDelta) + } + + override fun toString() = "[VP9 TL0Idx]$lastTl0Index" +} + +private class Vp9CodecDeltas(val tl0IndexDelta: Int) : CodecDeltas { + override fun rewritePacket(packet: RtpPacket) { + require(packet is Vp9Packet) + packet.TL0PICIDX = VpxUtils.applyTl0PicIdxDelta(packet.TL0PICIDX, tl0IndexDelta) + } + + override fun toString() = "[VP9 TL0Idx]$tl0IndexDelta" +} + +private fun RtpPacket.getCodecState(): CodecState? { + return when (this) { + is Vp8Packet -> Vp8CodecState(this) + is Vp9Packet -> Vp9CodecState(this) + else -> null + } +} From 7fabc45dc3ad19a0f468ba411fc1ca81a983f534 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Thu, 15 Jun 2023 16:31:45 -0400 Subject: [PATCH 028/189] Update to ktlint-maven-plugin 1.16.0. (#2029) Disable some rules that are annoying. Corresponding code format changes (should be no substantive change). --- .editorconfig | 10 +- jitsi-media-transform/pom.xml | 2 +- .../kotlin/org/jitsi/nlj/MediaSourceDesc.kt | 5 +- .../main/kotlin/org/jitsi/nlj/PacketInfo.kt | 2 +- .../kotlin/org/jitsi/nlj/RtpEncodingDesc.kt | 4 +- .../main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt | 7 +- .../main/kotlin/org/jitsi/nlj/RtpReceiver.kt | 2 + .../kotlin/org/jitsi/nlj/RtpSenderImpl.kt | 1 + .../main/kotlin/org/jitsi/nlj/Transceiver.kt | 3 +- .../main/kotlin/org/jitsi/nlj/VideoType.kt | 1 + .../org/jitsi/nlj/codec/vp8/Vp8Utils.kt | 11 +- .../kotlin/org/jitsi/nlj/dtls/DtlsServer.kt | 4 +- .../kotlin/org/jitsi/nlj/dtls/DtlsStack.kt | 2 + .../kotlin/org/jitsi/nlj/dtls/DtlsUtils.kt | 4 +- .../org/jitsi/nlj/dtls/TlsClientImpl.kt | 6 +- .../org/jitsi/nlj/dtls/TlsServerImpl.kt | 13 +- .../org/jitsi/nlj/format/PayloadType.kt | 1 + .../src/main/kotlin/org/jitsi/nlj/main.kt | 56 ----- .../org/jitsi/nlj/rtcp/KeyframeRequester.kt | 4 + .../kotlin/org/jitsi/nlj/rtcp/RtcpParsers.kt | 6 +- .../org/jitsi/nlj/rtp/RedAudioRtpPacket.kt | 7 +- .../jitsi/nlj/rtp/ResumableStreamRewriter.kt | 13 +- .../org/jitsi/nlj/rtp/TransportCcEngine.kt | 26 +- .../org/jitsi/nlj/rtp/VideoRtpPacket.kt | 4 +- .../bandwidthestimation/GoogleCcEstimator.kt | 7 +- .../org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt | 10 +- .../org/jitsi/nlj/rtp/codec/vp8/Vp8Parser.kt | 3 +- .../org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt | 14 +- .../org/jitsi/nlj/srtp/SrtpTransformer.kt | 1 + .../kotlin/org/jitsi/nlj/srtp/SrtpUtil.kt | 8 +- .../nlj/stats/EndpointConnectionStats.kt | 2 + .../kotlin/org/jitsi/nlj/stats/JitterStats.kt | 1 + .../org/jitsi/nlj/stats/NodeStatsBlock.kt | 1 + .../org/jitsi/nlj/stats/PacketIOActivity.kt | 2 + .../nlj/transform/node/AudioRedHandler.kt | 74 +++--- .../org/jitsi/nlj/transform/node/Node.kt | 3 + .../nlj/transform/node/PacketLossNode.kt | 4 +- .../nlj/transform/node/SrtpTransformerNode.kt | 6 +- .../node/debug/PayloadVerificationPlugin.kt | 1 - .../node/incoming/BitrateCalculator.kt | 4 +- .../incoming/IncomingStatisticsTracker.kt | 8 +- .../node/incoming/RemoteBandwidthEstimator.kt | 9 +- .../node/incoming/RtcpTermination.kt | 1 + .../nlj/transform/node/incoming/RtxHandler.kt | 1 + .../node/incoming/TccGeneratorNode.kt | 6 +- .../node/outgoing/HeaderExtStripper.kt | 3 +- .../outgoing/OutgoingStatisticsTracker.kt | 1 + .../node/outgoing/RetransmissionSender.kt | 5 +- .../kotlin/org/jitsi/nlj/util/ArrayCache.kt | 1 + .../org/jitsi/nlj/util/BitrateTracker.kt | 1 + .../org/jitsi/nlj/util/ExecutorUtils.kt | 1 + .../nlj/util/NodeStatsBlockExtensions.kt | 1 - .../kotlin/org/jitsi/nlj/util/PacketCache.kt | 6 +- .../jitsi/nlj/util/SsrcAssociationStore.kt | 1 + .../SendSideBandwidthEstimationConfig.kt | 5 +- .../org/jitsi/nlj/MediaSourceDescTest.kt | 29 ++- .../codec/vpx/PictureIdIndexTrackerTest.kt | 2 +- .../jitsi/nlj/module_tests/EndToEndHarness.kt | 16 +- .../nlj/module_tests/RtpReceiverHarness.kt | 8 +- .../RtpReceiverNoNewBufferTest.kt | 8 +- .../nlj/module_tests/RtpSenderHarness.kt | 8 +- .../module_tests/SrtpTransformerFactory.kt | 2 +- .../nlj/resources/logging/StdoutLogger.kt | 1 + .../nlj/resources/srtp_samples/SrtpSample.kt | 10 +- .../nlj/rtcp/StreamPacketRequesterTest.kt | 6 +- .../org/jitsi/nlj/rtp/AudioRedHandlerTest.kt | 2 +- .../jitsi/nlj/rtp/codec/vp9/Vp9PacketTest.kt | 17 +- .../incoming/RemoteBandwidthEstimatorTest.kt | 1 + .../node/incoming/SrtpDecryptTest.kt | 2 +- .../node/incoming/TccGeneratorNodeTest.kt | 1 + .../node/outgoing/RetransmissionSenderTest.kt | 1 + .../node/outgoing/SrtpEncryptTest.kt | 5 +- .../jitsi/nlj/util/Rfc3711IndexTrackerTest.kt | 2 +- .../kotlin/org/jitsi/videobridge/Endpoint.kt | 11 +- .../EndpointConnectionStatusMonitor.kt | 1 + .../org/jitsi/videobridge/LoudestConfig.kt | 4 + .../main/kotlin/org/jitsi/videobridge/Main.kt | 6 +- .../kotlin/org/jitsi/videobridge/SsrcCache.kt | 41 ++-- .../cc/allocation/BandwidthAllocator.kt | 4 +- .../cc/allocation/BitrateController.kt | 3 + .../videobridge/cc/allocation/Prioritize.kt | 1 - .../cc/allocation/SingleSourceAllocation.kt | 2 - .../vp9/Vp9AdaptiveSourceProjectionContext.kt | 23 +- .../org/jitsi/videobridge/cc/vp9/Vp9Frame.kt | 6 +- .../videobridge/cc/vp9/Vp9FrameProjection.kt | 4 +- .../jitsi/videobridge/cc/vp9/Vp9Picture.kt | 1 - .../jitsi/videobridge/cc/vp9/Vp9PictureMap.kt | 2 +- .../colibri2/IqProcessingException.kt | 6 +- .../load_management/PacketRateMeasurement.kt | 1 + .../message/BridgeChannelMessage.kt | 9 +- .../org/jitsi/videobridge/relay/Relay.kt | 4 +- .../relay/RelayMessageTransport.kt | 6 +- .../videobridge/relay/RelayedEndpoint.kt | 2 +- .../org/jitsi/videobridge/rest/RestConfig.kt | 1 + .../videobridge/sctp/DataChannelHandler.kt | 8 +- .../transport/dtls/DtlsTransport.kt | 3 + .../videobridge/transport/ice/IceTransport.kt | 6 +- .../websocket/ColibriWebSocketService.kt | 1 + .../allocation/BitrateControllerPerfTest.kt | 1 - .../allocation/SingleSourceAllocationTest.kt | 91 ++++++- .../cc/vp9/Vp9AdaptiveSourceProjectionTest.kt | 225 ++++++++++++------ .../cc/vp9/Vp9QualityFilterTest.kt | 19 +- pom.xml | 2 +- rtp/pom.xml | 2 +- rtp/src/main/kotlin/org/jitsi/rtp/Packet.kt | 6 +- .../bytearray/ByteArrayExtensions.kt | 1 + .../org/jitsi/rtp/rtcp/RtcpReportBlock.kt | 18 +- .../payload_specific_fb/RtcpFbFirPacket.kt | 1 + .../transport_layer_fb/tcc/LastChunk.kt | 12 +- .../transport_layer_fb/tcc/RtcpFbTccPacket.kt | 29 ++- .../org/jitsi/rtp/rtp/RedPacketParser.kt | 36 ++- .../kotlin/org/jitsi/rtp/rtp/RtpHeader.kt | 1 + .../kotlin/org/jitsi/rtp/rtp/RtpPacket.kt | 7 +- .../org/jitsi/rtp/rtp/RtpSequenceNumber.kt | 2 +- .../AbsSendTimeHeaderExtension.kt | 1 + .../header_extensions/SdesHeaderExtension.kt | 11 +- .../org/jitsi/rtp/rtcp/RtcpByePacketTest.kt | 3 +- .../tcc/RtcpFbTccPacketTest.kt | 4 +- .../kotlin/org/jitsi/rtp/rtp/RtpPacketTest.kt | 20 +- .../SdesHeaderExtensionTest.kt | 12 +- 120 files changed, 773 insertions(+), 393 deletions(-) delete mode 100644 jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/main.kt diff --git a/.editorconfig b/.editorconfig index f5be276e0b..fbe6d50b34 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,5 +1,9 @@ [*.{kt,kts}] -# Comma-separated list of rules to disable (Since 0.34.0) -# Note that rules in any ruleset other than the standard ruleset will need to be prefixed -# by the ruleset identifier. max_line_length=120 + +# I find trailing commas annoying +ktlint_standard_trailing-comma-on-call-site = disabled +ktlint_standard_trailing-comma-on-declaration-site = disabled + +# This forbids underscores in package names, which we use +ktlint_standard_package-name = disabled diff --git a/jitsi-media-transform/pom.xml b/jitsi-media-transform/pom.xml index 39432a5578..3b6499c91b 100644 --- a/jitsi-media-transform/pom.xml +++ b/jitsi-media-transform/pom.xml @@ -248,7 +248,7 @@ com.github.gantsign.maven ktlint-maven-plugin - 1.13.1 + ${ktlint-maven-plugin.version} ${project.basedir}/src/main/kotlin diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt index 48efc0f1f6..3da9ba9bb9 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt @@ -162,7 +162,10 @@ class MediaSourceDesc */ @Synchronized fun copy() = MediaSourceDesc( - Array(this.rtpEncodings.size) { i -> this.rtpEncodings[i].copy() }, this.owner, this.sourceName, this.videoType + Array(this.rtpEncodings.size) { i -> this.rtpEncodings[i].copy() }, + this.owner, + this.sourceName, + this.videoType ) override fun toString(): String = "MediaSourceDesc[name=$sourceName owner=$owner, videoType=$videoType, " + diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketInfo.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketInfo.kt index 28c5c387f6..b43e31fc13 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketInfo.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketInfo.kt @@ -158,7 +158,7 @@ open class PacketInfo @JvmOverloads constructor( clone.endpointId = endpointId clone.layeringChanged = layeringChanged clone.payloadVerification = payloadVerification - @Suppress("UNCHECKED_CAST") /* ArrayList.clone() really does return ArrayList, not Object. */ + @Suppress("UNCHECKED_CAST") // ArrayList.clone() really does return ArrayList, not Object. clone.onSentActions = onSentActions?.clone() as ArrayList<() -> Unit>? return clone } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt index c461cb346a..2b9873f445 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt @@ -144,7 +144,9 @@ constructor( fun matches(ssrc: Long): Boolean { return if (primarySSRC == ssrc) { true - } else secondarySsrcs.containsKey(ssrc) + } else { + secondarySsrcs.containsKey(ssrc) + } } /** diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt index c7aba0cb7d..29d4586cbc 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt @@ -288,7 +288,10 @@ constructor( */ @JvmStatic fun indexString(index: Int): String = - if (index == SUSPENDED_INDEX) "SUSP" - else "E${getEidFromIndex(index)}S${getSidFromIndex(index)}T${getTidFromIndex(index)}" + if (index == SUSPENDED_INDEX) { + "SUSP" + } else { + "E${getEidFromIndex(index)}S${getSidFromIndex(index)}T${getTidFromIndex(index)}" + } } } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiver.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiver.kt index 156a863702..5a55659b8c 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiver.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiver.kt @@ -34,6 +34,7 @@ abstract class RtpReceiver : * input chain). */ abstract var packetHandler: PacketHandler? + /** * Enqueue an incoming packet to be processed */ @@ -69,6 +70,7 @@ interface RtpReceiverEventHandler { * We received an audio level indication from the remote endpoint. */ fun audioLevelReceived(sourceSsrc: Long, level: Long): Boolean = false + /** * The estimation of the available send bandwidth changed. */ diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt index 2890cb6531..b3f21a718b 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt @@ -95,6 +95,7 @@ class RtpSenderImpl( var running = true private var localVideoSsrc: Long? = null private var localAudioSsrc: Long? = null + // TODO(brian): this is changed to a handler instead of a queue because we want to use // a PacketQueue, and the handler for a PacketQueue must be set at the time of creation. // since we want the handler to be another entity (something in jvb) we just use diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Transceiver.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Transceiver.kt index bc7f79f909..afa45577b0 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Transceiver.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Transceiver.kt @@ -43,7 +43,6 @@ import java.util.concurrent.ExecutorService import java.util.concurrent.ScheduledExecutorService // This is an API class, so its usages will largely be outside of this library -@Suppress("unused") /** * Handles all packets (incoming and outgoing) for a particular stream. * (TODO: 'stream' defined as what, exactly, here?) @@ -56,6 +55,7 @@ import java.util.concurrent.ScheduledExecutorService * have the one thread just read from the queue and send, rather than that thread * having to read from a bunch of individual queues) */ +@Suppress("unused") class Transceiver( private val id: String, receiverExecutor: ExecutorService, @@ -80,6 +80,7 @@ class Transceiver( private val endpointConnectionStats = EndpointConnectionStats(logger) private val streamInformationStore = StreamInformationStoreImpl() val readOnlyStreamInformationStore: ReadOnlyStreamInformationStore = streamInformationStore + /** * A central place to subscribe to be notified on the reception or transmission of RTCP packets for * this transceiver. This is intended to be used by internal entities: mainly logic for things like generating diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/VideoType.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/VideoType.kt index 2b59bfe46a..b9a0cd9965 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/VideoType.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/VideoType.kt @@ -20,6 +20,7 @@ enum class VideoType { DESKTOP, DESKTOP_HIGH_FPS, DISABLED, + // NONE was used in the context where an Endpoint has always one media source description. It used to cover both // lack of the actual source and the source being temporarily disabled. With the support for multiple sources per // endpoint DISABLED means a source is turned off. Lack of a MediaSourceDesc is equivalent to NONE. diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/codec/vp8/Vp8Utils.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/codec/vp8/Vp8Utils.kt index 0b97be38b3..ae861dc87e 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/codec/vp8/Vp8Utils.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/codec/vp8/Vp8Utils.kt @@ -63,7 +63,8 @@ class Vp8Utils { vp8Packet.payloadLength ) return DePacketizer.VP8KeyframeHeader.getHeight( - vp8Packet.buffer, vp8Packet.payloadOffset + payloadDescriptorLen + VP8_PAYLOAD_HEADER_LEN + vp8Packet.buffer, + vp8Packet.payloadOffset + payloadDescriptorLen + VP8_PAYLOAD_HEADER_LEN ) } @@ -80,12 +81,16 @@ class Vp8Utils { fun getTemporalLayerIdOfFrame(vp8Payload: ByteBuffer) = DePacketizer.VP8PayloadDescriptor.getTemporalLayerIndex( - vp8Payload.array(), vp8Payload.arrayOffset(), vp8Payload.limit() + vp8Payload.array(), + vp8Payload.arrayOffset(), + vp8Payload.limit() ) fun getTemporalLayerIdOfFrame(vp8Packet: RtpPacket) = DePacketizer.VP8PayloadDescriptor.getTemporalLayerIndex( - vp8Packet.buffer, vp8Packet.payloadOffset, vp8Packet.payloadLength + vp8Packet.buffer, + vp8Packet.payloadOffset, + vp8Packet.payloadLength ) } } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsServer.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsServer.kt index 495aec3cc1..2d8d7c69a2 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsServer.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsServer.kt @@ -45,7 +45,9 @@ class DtlsServer( return dtlsServerProtocol.accept(tlsServer, datagramTransport).also { logger.cdebug { "DTLS handshake finished" } handshakeCompleteHandler( - tlsServer.chosenSrtpProtectionProfile, TlsRole.SERVER, tlsServer.srtpKeyingMaterial + tlsServer.chosenSrtpProtectionProfile, + TlsRole.SERVER, + tlsServer.srtpKeyingMaterial ) } } catch (t: Throwable) { diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsStack.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsStack.kt index 44f854358a..4d7c32ba5c 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsStack.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsStack.kt @@ -139,6 +139,7 @@ class DtlsStack( ) roleSet.countDown() } + /** * 'start' this stack, in whatever role it has been told to operate (client or server). If a role * has not yet been yet (via [actAsServer] or [actAsClient]), then it will block until the role @@ -243,6 +244,7 @@ class DtlsStack( companion object { private const val QUEUE_SIZE = 50 + /** * Because generating the certificateInfo can be expensive, we generate a single * one to be used everywhere which expires in 24 hours (when we'll generate diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsUtils.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsUtils.kt index 5ef0fe475f..ce41481594 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsUtils.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsUtils.kt @@ -163,7 +163,6 @@ class DtlsUtils { certificateInfo: org.bouncycastle.tls.Certificate, remoteFingerprints: Map ) { - if (certificateInfo.certificateList.isEmpty()) { throw DtlsException("No remote fingerprints.") } @@ -224,7 +223,7 @@ class DtlsUtils { hashFunction = hashFunctionUpgrade } } - */ + */ val certificateFingerprint = certificate.getFingerprint(hashFunction) @@ -265,6 +264,7 @@ class DtlsUtils { } private val HEX_CHARS = "0123456789ABCDEF".toCharArray() + /** * Helper function to convert a [ByteArray] to a colon-delimited hex string */ diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsClientImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsClientImpl.kt index 0c8f80be40..8ee1c76ba0 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsClientImpl.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsClientImpl.kt @@ -87,12 +87,14 @@ class TlsClientImpl( (context.crypto as BcTlsCrypto), PrivateKeyFactory.createKey(certificateInfo.keyPair.private.encoded), certificateInfo.certificate, - if (TlsUtils.isSignatureAlgorithmsExtensionAllowed(context.serverVersion)) + if (TlsUtils.isSignatureAlgorithmsExtensionAllowed(context.serverVersion)) { SignatureAndHashAlgorithm( HashAlgorithm.sha256, SignatureAlgorithm.ecdsa ) - else null + } else { + null + } ) } return clientCredentials!! diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsServerImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsServerImpl.kt index 5fcd0c92cd..3a9007fe24 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsServerImpl.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsServerImpl.kt @@ -115,11 +115,14 @@ class TlsServerImpl( PrivateKeyFactory.createKey(certificateInfo.keyPair.private.encoded), certificateInfo.certificate, /* For DTLS 1.0 support (needed for Jigasi) we can't set this to sha256 fixed */ - if (TlsUtils.isSignatureAlgorithmsExtensionAllowed(context.serverVersion)) SignatureAndHashAlgorithm( - HashAlgorithm.sha256, - SignatureAlgorithm.ecdsa - ) - else null + if (TlsUtils.isSignatureAlgorithmsExtensionAllowed(context.serverVersion)) { + SignatureAndHashAlgorithm( + HashAlgorithm.sha256, + SignatureAlgorithm.ecdsa + ) + } else { + null + } ) } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/format/PayloadType.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/format/PayloadType.kt index bb29e90f91..6f0c5bba25 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/format/PayloadType.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/format/PayloadType.kt @@ -27,6 +27,7 @@ fun RtcpFeedbackSet.supportsPli(): Boolean = this.contains("nack pli") fun RtcpFeedbackSet.supportsFir(): Boolean = this.contains("ccm fir") fun RtcpFeedbackSet.supportsRemb(): Boolean = this.contains("goog-remb") fun RtcpFeedbackSet.supportsTcc(): Boolean = this.contains("transport-cc") + /** * Represents an RTP payload type. * diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/main.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/main.kt deleted file mode 100644 index f25e35103f..0000000000 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/main.kt +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright @ 2018 - Present, 8x8 Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.jitsi.nlj - -fun main() { - - /* - val pg = PacketGenerator() - val packets = mutableListOf() - repeat(20) { - packets.add(pg.generatePacket()) - } - - val stream1 = IncomingMediaSource1() - val stream2 = RtpReceiverImpl() - // IncomingMediaStreamTrack currently implements the following simulated packet pipeline: - // - // RTP / --> Packet loss monitor --> RTP handler - // / - // --> Packet Stats --> SRTP --> RTP/RTCP splitter - // \ - // RTCP \ --> RTCP handler - - packets.forEach { - stream2.processPackets(listOf(it)) - } - - println(stream2.getNodeStats()) - - - - println("Outgoing") - val outgoingStream2 = RtpSenderImpl() - packets.forEach { - if (it is RtpPacket) { - outgoingStream2.outgoingRtpChain.processPackets(listOf(it)) - - } else if (it is RtcpPacket) { - outgoingStream2.outgoingRtcpChain.processPackets(listOf(it)) - } - } - */ -} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/KeyframeRequester.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/KeyframeRequester.kt index eaf294cc53..c305eea277 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/KeyframeRequester.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/KeyframeRequester.kt @@ -65,14 +65,18 @@ class KeyframeRequester @JvmOverloads constructor( // Number of PLI/FIRs received and forwarded to the endpoint. private var numPlisForwarded: Int = 0 private var numFirsForwarded: Int = 0 + // Number of PLI/FIRs received but dropped due to throttling. private var numPlisDropped: Int = 0 private var numFirsDropped: Int = 0 + // Number of PLI/FIRs generated as a result of an API request or due to translation between PLI/FIR. private var numPlisGenerated: Int = 0 private var numFirsGenerated: Int = 0 + // Number of calls to requestKeyframe private var numApiRequests: Int = 0 + // Number of calls to requestKeyframe ignored due to throttling private var numApiRequestsDropped: Int = 0 diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpParsers.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpParsers.kt index 97cadb06bf..8d75a79454 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpParsers.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpParsers.kt @@ -22,7 +22,8 @@ import org.jitsi.rtp.rtcp.RtcpPacket import org.jitsi.utils.logging2.Logger class CompoundRtcpParser(parentLogger: Logger) : PacketParser( - "Compound RTCP parser", parentLogger, + "Compound RTCP parser", + parentLogger, { CompoundRtcpPacket(it.buffer, it.offset, it.length).also { compoundPacket -> // Force packets to be evaluated to trigger any parsing errors @@ -34,7 +35,8 @@ class CompoundRtcpParser(parentLogger: Logger) : PacketParser( } class SingleRtcpParser(parentLogger: Logger) : PacketParser( - "Single RTCP parser", parentLogger, + "Single RTCP parser", + parentLogger, { RtcpPacket.parse(it.buffer, it.offset, it.length) } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RedAudioRtpPacket.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RedAudioRtpPacket.kt index 0a5451f27b..6da1e747c0 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RedAudioRtpPacket.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RedAudioRtpPacket.kt @@ -34,8 +34,11 @@ class RedAudioRtpPacket( } fun removeRedAndGetRedundancyPackets(): List = - if (removed) throw IllegalStateException("RED encapsulation already removed.") - else parser.decapsulate(this, parseRedundancy = true).also { removed = true } + if (removed) { + throw IllegalStateException("RED encapsulation already removed.") + } else { + parser.decapsulate(this, parseRedundancy = true).also { removed = true } + } override fun clone(): RedAudioRtpPacket = RedAudioRtpPacket( diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ResumableStreamRewriter.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ResumableStreamRewriter.kt index 839fe24018..84507bedd8 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ResumableStreamRewriter.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ResumableStreamRewriter.kt @@ -126,8 +126,9 @@ class ResumableStreamRewriter(val keepHistory: Boolean = false) { private val rfc3711IndexTracker = Rfc3711IndexTracker() private fun fillBetween(start: Int, end: Int, firstNewIndex: Int) { - if (end <= lastIndex - size + 1) + if (end <= lastIndex - size + 1) { return + } val actualStart = if (start <= lastIndex - size) { lastIndex - size + 1 } else { @@ -187,8 +188,9 @@ class ResumableStreamRewriter(val keepHistory: Boolean = false) { if (item.accept == null) { item.accept = accept - if (!accept) + if (!accept) { gapsLeft++ + } } newIndex = item.newIndex @@ -204,7 +206,7 @@ class ResumableStreamRewriter(val keepHistory: Boolean = false) { val oldestNewIndex = oldest.item!!.newIndex - val indexGap = index - oldestIndex /* Negative */ + val indexGap = index - oldestIndex // Negative val newGap = indexGap + if (accept) 0 else 1 @@ -230,8 +232,9 @@ class ResumableStreamRewriter(val keepHistory: Boolean = false) { } if (accept) return toSequenceNumber(newIndex) - return sequenceNumber /* Don't care about sequence numbers for non-accepted packets, - so make sure rewriteRtp does nothing. */ + /* Don't care about sequence numbers for non-accepted packets, + so make sure rewriteRtp does nothing. */ + return sequenceNumber } init { diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/TransportCcEngine.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/TransportCcEngine.kt index 08ac5cbf0a..a2b648ce06 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/TransportCcEngine.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/TransportCcEngine.kt @@ -147,9 +147,9 @@ class TransportCcEngine( when (packetReport) { is UnreceivedPacketReport -> { - if (packetDetail.state == PacketDetailState.unreported) { + if (packetDetail.state == PacketDetailState.Unreported) { bandwidthEstimator.processPacketLoss(now, packetDetail.packetSendTime, tccSeqNum) - packetDetail.state = PacketDetailState.reportedLost + packetDetail.state = PacketDetailState.ReportedLost numPacketsReported.increment() numPacketsReportedLost.increment() synchronized(this) { @@ -163,8 +163,8 @@ class TransportCcEngine( currArrivalTimestamp += packetReport.deltaDuration when (packetDetail.state) { - PacketDetailState.unreported, PacketDetailState.reportedLost -> { - val previouslyReportedLost = packetDetail.state == PacketDetailState.reportedLost + PacketDetailState.Unreported, PacketDetailState.ReportedLost -> { + val previouslyReportedLost = packetDetail.state == PacketDetailState.ReportedLost if (previouslyReportedLost) { numPacketsReportedAfterLost.increment() numPacketsReportedLost.decrement() @@ -177,8 +177,11 @@ class TransportCcEngine( currArrivalTimestamp - Duration.between(localReferenceTime, remoteReferenceTime) bandwidthEstimator.processPacketArrival( - now, packetDetail.packetSendTime, arrivalTimeInLocalClock, - tccSeqNum, packetDetail.packetLength, + now, + packetDetail.packetSendTime, + arrivalTimeInLocalClock, + tccSeqNum, + packetDetail.packetLength, previouslyReportedLost = previouslyReportedLost ) synchronized(this) { @@ -186,10 +189,10 @@ class TransportCcEngine( it.packetReceived(previouslyReportedLost) } } - packetDetail.state = PacketDetailState.reportedReceived + packetDetail.state = PacketDetailState.ReportedReceived } - PacketDetailState.reportedReceived -> + PacketDetailState.ReportedReceived -> numDuplicateReports.increment() } } @@ -253,7 +256,7 @@ class TransportCcEngine( * [PacketDetailState] is the state of a [PacketDetail] */ private enum class PacketDetailState { - unreported, reportedLost, reportedReceived + Unreported, ReportedLost, ReportedReceived } /** @@ -272,7 +275,7 @@ class TransportCcEngine( * as [unreported]: once we receive a TCC feedback from the remote side referring * to this packet, the state will transition to either [reportedLost] or [reportedReceived]. */ - var state = PacketDetailState.unreported + var state = PacketDetailState.Unreported } data class StatisticsSnapshot( @@ -302,8 +305,9 @@ class TransportCcEngine( clock = clock ) { override fun discardItem(item: PacketDetail) { - if (item.state == PacketDetailState.unreported) + if (item.state == PacketDetailState.Unreported) { numPacketsUnreported.increment() + } } private val rfc3711IndexTracker = Rfc3711IndexTracker() diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/VideoRtpPacket.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/VideoRtpPacket.kt index 11de9dcbc0..dc4835e3c1 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/VideoRtpPacket.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/VideoRtpPacket.kt @@ -34,7 +34,9 @@ open class VideoRtpPacket protected constructor( offset: Int, length: Int ) : this( - buffer, offset, length, + buffer, + offset, + length, qualityIndex = null ) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/GoogleCcEstimator.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/GoogleCcEstimator.kt index db49566d05..82af123036 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/GoogleCcEstimator.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/GoogleCcEstimator.kt @@ -78,7 +78,9 @@ class GoogleCcEstimator(diagnosticContext: DiagnosticContext, parentLogger: Logg if (sendTime != null && recvTime != null) { bitrateEstimatorAbsSendTime.incomingPacketInfo( now.toEpochMilli(), - sendTime.toEpochMilli(), recvTime.toEpochMilli(), size.bytes.toInt() + sendTime.toEpochMilli(), + recvTime.toEpochMilli(), + size.bytes.toInt() ) } sendSideBandwidthEstimation.updateReceiverEstimate(bitrateEstimatorAbsSendTime.latestEstimate) @@ -105,7 +107,8 @@ class GoogleCcEstimator(diagnosticContext: DiagnosticContext, parentLogger: Logg } override fun getStats(now: Instant): StatisticsSnapshot = StatisticsSnapshot( - "GoogleCcEstimator", getCurrentBw(now) + "GoogleCcEstimator", + getCurrentBw(now) ).apply { addNumber("incomingEstimateExpirations", bitrateEstimatorAbsSendTime.incomingEstimateExpirations) addNumber("latestDelayEstimate", sendSideBandwidthEstimation.latestREMB) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt index 8b1f6a41af..ca822b4250 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt @@ -87,7 +87,10 @@ class Vp8Packet private constructor( set(newValue) { _TL0PICIDX = newValue if (newValue != -1 && !DePacketizer.VP8PayloadDescriptor.setTL0PICIDX( - buffer, payloadOffset, payloadLength, newValue + buffer, + payloadOffset, + payloadLength, + newValue ) ) { logger.cwarn { "Failed to set the TL0PICIDX of a VP8 packet." } @@ -100,7 +103,10 @@ class Vp8Packet private constructor( set(newValue) { _pictureId = newValue if (!DePacketizer.VP8PayloadDescriptor.setExtendedPictureId( - buffer, payloadOffset, payloadLength, newValue + buffer, + payloadOffset, + payloadLength, + newValue ) ) { logger.cwarn { "Failed to set the picture id of a VP8 packet." } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Parser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Parser.kt index aa2a06eff3..024f45d683 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Parser.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Parser.kt @@ -58,7 +58,8 @@ class Vp8Parser( "Packet Data: ${vp8Packet.toHex(80)}" } tidWithoutTl0PicIdxState.setState( - vp8Packet.hasTL0PICIDX || !vp8Packet.hasTemporalLayerIndex, vp8Packet + vp8Packet.hasTL0PICIDX || !vp8Packet.hasTemporalLayerIndex, + vp8Packet ) { "Packet Data: ${vp8Packet.toHex(80)}" } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt index 15ba9b98f2..52038026d1 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt @@ -101,7 +101,10 @@ class Vp9Packet private constructor( get() = _TL0PICIDX set(newValue) { if (newValue != -1 && !DePacketizer.VP9PayloadDescriptor.setTL0PICIDX( - buffer, payloadOffset, payloadLength, newValue + buffer, + payloadOffset, + payloadLength, + newValue ) ) { logger.cwarn { "Failed to set the TL0PICIDX of a VP9 packet." } @@ -119,7 +122,10 @@ class Vp9Packet private constructor( get() = _pictureId set(newValue) { if (!DePacketizer.VP9PayloadDescriptor.setExtendedPictureId( - buffer, payloadOffset, payloadLength, newValue + buffer, + payloadOffset, + payloadLength, + newValue ) ) { logger.cwarn { "Failed to set the picture id of a VP9 packet." } @@ -237,7 +243,9 @@ class Vp9Packet private constructor( */ var off = DePacketizer.VP9PayloadDescriptor.getScalabilityStructureOffset( - buffer, payloadOffset, payloadLength + buffer, + payloadOffset, + payloadLength ) if (off == -1) { return null diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/srtp/SrtpTransformer.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/srtp/SrtpTransformer.kt index 62ba3685da..4a14242cb2 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/srtp/SrtpTransformer.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/srtp/SrtpTransformer.kt @@ -39,6 +39,7 @@ abstract class AbstractSrtpTransformer() + /** * Holds stats that are computed based on other values in the map (to e.g. calculate the * ratio of two values). Restricted to [Number] because this makes it easier to implement and diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/PacketIOActivity.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/PacketIOActivity.kt index 3150e29fd2..8e0d5cebc8 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/PacketIOActivity.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/PacketIOActivity.kt @@ -29,12 +29,14 @@ class PacketIOActivity { var lastRtpPacketReceivedInstant: Instant by threadSafeVetoable(NEVER) { _, oldValue, newValue -> newValue.isAfter(oldValue) } + /** * The last time an RTP or RTCP packet was received. */ var lastRtpPacketSentInstant: Instant by threadSafeVetoable(NEVER) { _, oldValue, newValue -> newValue.isAfter(oldValue) } + /** * The last time ICE consent was refreshed. */ diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/AudioRedHandler.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/AudioRedHandler.kt index c699349f08..5404975677 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/AudioRedHandler.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/AudioRedHandler.kt @@ -221,46 +221,48 @@ class AudioRedHandler( } } - return if (strip) buildList { - val redPacket = packetInfo.packetAs() - - val seq = redPacket.sequenceNumber - val prev = applySequenceNumberDelta(seq, -1) - val prev2 = applySequenceNumberDelta(seq, -2) - val prevMissing = !sentAudioCache.contains(prev) - val prev2Missing = !sentAudioCache.contains(prev2) - - try { - if (prevMissing || prev2Missing) { - redPacket.removeRedAndGetRedundancyPackets().forEach { - if ((it.sequenceNumber == prev && prevMissing) || - (it.sequenceNumber == prev2 && prev2Missing) - ) { - add(PacketInfo(it)) - stats.lostPacketRecovered() + return if (strip) { + buildList { + val redPacket = packetInfo.packetAs() + + val seq = redPacket.sequenceNumber + val prev = applySequenceNumberDelta(seq, -1) + val prev2 = applySequenceNumberDelta(seq, -2) + val prevMissing = !sentAudioCache.contains(prev) + val prev2Missing = !sentAudioCache.contains(prev2) + + try { + if (prevMissing || prev2Missing) { + redPacket.removeRedAndGetRedundancyPackets().forEach { + if ((it.sequenceNumber == prev && prevMissing) || + (it.sequenceNumber == prev2 && prev2Missing) + ) { + add(PacketInfo(it)) + stats.lostPacketRecovered() + } + sentAudioCache.insert(it) } - sentAudioCache.insert(it) + } else { + redPacket.removeRed() } - } else { - redPacket.removeRed() + } catch (e: IllegalArgumentException) { + logger.warn( + "Dropping invalid RED packet from ep=${packetInfo.endpointId} (${e.message}): " + + "$redPacket. Contents (50B): ${redPacket.toHex(50)}" + ) + stats.invalidRedPacketDropped() + return@buildList } - } catch (e: IllegalArgumentException) { - logger.warn( - "Dropping invalid RED packet from ep=${packetInfo.endpointId} (${e.message}): " + - "$redPacket. Contents (50B): ${redPacket.toHex(50)}" - ) - stats.invalidRedPacketDropped() - return@buildList - } - stats.redPacketDecapsulated() - packetInfo.packet = redPacket.toOtherType(::AudioRtpPacket) + stats.redPacketDecapsulated() + packetInfo.packet = redPacket.toOtherType(::AudioRtpPacket) - // It's possible we already forwarded the primary packet if we recovered it from a previously received - // packet. - if (!sentAudioCache.contains(seq)) { - sentAudioCache.insert(packetInfo.packetAs()) - add(packetInfo) + // It's possible we already forwarded the primary packet if we recovered it from a previously received + // packet. + if (!sentAudioCache.contains(seq)) { + sentAudioCache.insert(packetInfo.packetAs()) + add(packetInfo) + } } } else { stats.redPacketForwarded() @@ -279,10 +281,12 @@ enum class RedPolicy { * No change. */ NOOP, + /** * Always strip. */ STRIP, + /** * Add RED for all endpoints. */ diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/Node.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/Node.kt index 9004376123..7e75e9d719 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/Node.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/Node.kt @@ -48,6 +48,7 @@ sealed class Node( private var nextNode: Node? = null private val inputNodes: MutableList by lazy { mutableListOf() } + // Create these once here so we don't allocate a new string every time protected val nodeEntryString = "Entered node $name" protected val nodeExitString = "Exited node $name" @@ -114,6 +115,7 @@ sealed class Node( nextNode?.processPacket(packetInfo) } } + /** * This function must be implemented by leaf nodes, as * ``` @@ -130,6 +132,7 @@ sealed class Node( companion object { var TRACE_ENABLED = false var PLUGINS_ENABLED = false + // 'Plugins' are observers which, when enabled, will be passed every packet that passes through // every node val plugins: MutableSet = mutableSetOf() diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PacketLossNode.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PacketLossNode.kt index a08043fa57..0cbf8f3d82 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PacketLossNode.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PacketLossNode.kt @@ -48,7 +48,9 @@ class PacketLossNode(val config: PacketLossConfig) : FilterNode("PacketLossNode( currentBurstPacketsDropped = 0 } false - } else true + } else { + true + } } override fun trace(f: () -> Unit) { } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/SrtpTransformerNode.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/SrtpTransformerNode.kt index 20f39c26e5..dfd686f0e8 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/SrtpTransformerNode.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/SrtpTransformerNode.kt @@ -58,6 +58,7 @@ abstract class SrtpTransformerNode(name: String) : MultipleOutputTransformerNode private var firstPacketReceivedTimestamp = -1L private var firstPacketForwardedTimestamp = -1L + /** * How many packets, total, we put into the cache while waiting for the transformer * (this includes packets which may have been dropped due to the cache filling up) @@ -81,8 +82,9 @@ abstract class SrtpTransformerNode(name: String) : MultipleOutputTransformerNode } else { val err = transformer.transform(packetInfo) countErrorStatus(err) - outPackets = if (err == SrtpErrorStatus.OK) - listOf(packetInfo) else { + outPackets = if (err == SrtpErrorStatus.OK) { + listOf(packetInfo) + } else { packetDiscarded(packetInfo) emptyList() } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/debug/PayloadVerificationPlugin.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/debug/PayloadVerificationPlugin.kt index 42dfa46cd3..952ffb5093 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/debug/PayloadVerificationPlugin.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/debug/PayloadVerificationPlugin.kt @@ -41,7 +41,6 @@ class PayloadVerificationPlugin { if (PacketInfo.ENABLE_PAYLOAD_VERIFICATION && packetInfo.payloadVerification != null ) { - val expected = packetInfo.payloadVerification val actual = packetInfo.packet.payloadVerification if (expected != actual) { diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/BitrateCalculator.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/BitrateCalculator.kt index 8bcbd1369a..62105d6994 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/BitrateCalculator.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/BitrateCalculator.kt @@ -102,7 +102,9 @@ open class BitrateCalculator( // In the grace period any received data counts, and we check the bitrate because we can only access the // packet rate rounded to an Int. bitrate.bps > 0 - } else packetRatePps >= activePacketRateThreshold + } else { + packetRatePps >= activePacketRateThreshold + } override fun observe(packetInfo: PacketInfo) { val now = clock.millis() diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/IncomingStatisticsTracker.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/IncomingStatisticsTracker.kt index 324cfaf619..a9136d7af7 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/IncomingStatisticsTracker.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/IncomingStatisticsTracker.kt @@ -135,8 +135,10 @@ class IncomingSsrcStats( private val jitterStats = JitterStats() private var numReceivedPackets: Int = 0 private var numReceivedBytes: Int = 0 + /** How long this SSRC has been active. */ private var durationActive = Duration.ZERO + /** The receiveTime of the last packet */ private var lastPacketReceivedTime: Instant? = null @@ -157,6 +159,7 @@ class IncomingSsrcStats( * from the cumulative loss amount) */ const val MAX_OOO_AMOUNT = 100 + /** * https://tools.ietf.org/html/rfc3550#appendix-A.1 * "...a source is declared valid only after MIN_SEQUENTIAL packets have been received in @@ -320,10 +323,11 @@ class IncomingSsrcStats( val numReceivedPacketsInterval = numReceivedPackets - previousSnapshot.numReceivedPackets val numLostPacketsInterval = numExpectedPacketsInterval - numReceivedPacketsInterval - return if (numExpectedPacketsInterval == 0 || numLostPacketsInterval <= 0) + return if (numExpectedPacketsInterval == 0 || numLostPacketsInterval <= 0) { 0 - else + } else { (((numLostPacketsInterval shl 8) / numExpectedPacketsInterval.toDouble())).toInt() + } } fun toJson() = OrderedJsonObject().apply { diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RemoteBandwidthEstimator.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RemoteBandwidthEstimator.kt index c4d81d559c..3d94ec3cd0 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RemoteBandwidthEstimator.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RemoteBandwidthEstimator.kt @@ -52,6 +52,7 @@ class RemoteBandwidthEstimator( private val clock: Clock = Clock.systemUTC() ) : ObserverNode("Remote Bandwidth Estimator") { private val logger = createChildLogger(parentLogger) + /** * The remote bandwidth estimation is enabled when REMB support is signaled, but TCC is not signaled. */ @@ -60,13 +61,19 @@ class RemoteBandwidthEstimator( logger.debug { "Setting enabled=$newValue." } } private var astExtId: Int? = null + /** * We use the full [GoogleCcEstimator] here, but we don't notify it of packet loss, effectively using only the * delay-based part. */ private val bwe: BandwidthEstimator by lazy { GoogleCcEstimator(diagnosticContext, logger) } private val ssrcs: MutableSet = - Collections.synchronizedSet(LRUCache.lruSet(MAX_SSRCS, true /* accessOrder */)) + Collections.synchronizedSet( + LRUCache.lruSet( + MAX_SSRCS, + true // accessOrder + ) + ) private var numRembsCreated = 0 private var numPacketsWithoutAbsSendTime = 0 private var localSsrc = 0L diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RtcpTermination.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RtcpTermination.kt index ba6e5385e9..cf92023c31 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RtcpTermination.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RtcpTermination.kt @@ -44,6 +44,7 @@ class RtcpTermination( ) : TransformerNode("RTCP termination") { private val logger = createChildLogger(parentLogger) private var packetReceiveCounts = mutableMapOf() + /** * Number of packets we failed to forward because a compound packet contained more than one * packet we wanted to forward. Ideally this shouldn't happen. diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RtxHandler.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RtxHandler.kt index 3dd754a795..074e0a2db4 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RtxHandler.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RtxHandler.kt @@ -40,6 +40,7 @@ class RtxHandler( private val logger = createChildLogger(parentLogger) private var numPaddingPacketsReceived = 0 private var numRtxPacketsReceived = 0 + /** * Maps the Integer payload type of RTX to the [RtxPayloadType] instance. We do this * so we can look up the associated (original) payload type from the [RtxPayloadType] diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNode.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNode.kt index a7c5fd251b..e22fb088da 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNode.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNode.kt @@ -57,8 +57,10 @@ class TccGeneratorNode( private var currTccSeqNum: Int = 0 private var lastTccSentTime: Instant = NEVER private val lock = Any() + // Tcc seq num -> arrival time in ms private val packetArrivalTimes = TreeMap() + // The first sequence number of the current tcc feedback packet private var windowStartSeq: Int = -1 private val tccFeedbackBitrate = BitrateTracker(1.secs, 10.ms) @@ -148,8 +150,8 @@ class TccGeneratorNode( } } else if (tccSeqNum < windowStartSeq || !packetArrivalTimes.containsKey(tccSeqNum)) { /* If we've already cleared the arrival info about this packet, assume it was previously - * reported as lost - there are some corner cases where this isn't true, but they should be rare. - */ + * reported as lost - there are some corner cases where this isn't true, but they should be rare. + */ lossListeners.forEach { it.packetReceived(true) } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/HeaderExtStripper.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/HeaderExtStripper.kt index 6e9183ee51..009bf1288e 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/HeaderExtStripper.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/HeaderExtStripper.kt @@ -49,7 +49,8 @@ class HeaderExtStripper( companion object { private val retainedExtTypes: Set = setOf( - RtpExtensionType.SSRC_AUDIO_LEVEL, RtpExtensionType.VIDEO_ORIENTATION + RtpExtensionType.SSRC_AUDIO_LEVEL, + RtpExtensionType.VIDEO_ORIENTATION ) } } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/OutgoingStatisticsTracker.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/OutgoingStatisticsTracker.kt index 0a237d8c0d..2f2c56403a 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/OutgoingStatisticsTracker.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/OutgoingStatisticsTracker.kt @@ -94,6 +94,7 @@ class OutgoingSsrcStats( private val ssrc: Long ) { private var statsLock = Any() + // Start variables protected by statsLock private var packetCount: Int = 0 private var octetCount: Int = 0 diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/RetransmissionSender.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/RetransmissionSender.kt index 23a781c448..4ed27c5f09 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/RetransmissionSender.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/RetransmissionSender.kt @@ -34,10 +34,12 @@ class RetransmissionSender( parentLogger: Logger ) : ModifierNode("Retransmission sender") { private val logger = createChildLogger(parentLogger) + /** * Maps an original payload type (Int) to its [RtxPayloadType] */ private val origPtToRtxPayloadType: MutableMap = ConcurrentHashMap() + /** * A map of rtx stream ssrc to the current sequence number for that stream */ @@ -112,7 +114,8 @@ class RetransmissionSender( addNumber("num_retransmissions_rtx_sent", numRetransmittedRtxPackets) addNumber("num_retransmissions_plain_sent", numRetransmittedPlainPackets) addString( - "rtx_payload_types(orig -> rtx)", this@RetransmissionSender.origPtToRtxPayloadType.toString() + "rtx_payload_types(orig -> rtx)", + this@RetransmissionSender.origPtToRtxPayloadType.toString() ) } } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ArrayCache.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ArrayCache.kt index 88da428a92..958a8874c0 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ArrayCache.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ArrayCache.kt @@ -43,6 +43,7 @@ open class ArrayCache( ) : NodeStatsProducer { private val cache: Array = Array(size) { Container() } protected val syncRoot = Any() + /** * The index in [cache] where the item with the highest index is stored. */ diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/BitrateTracker.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/BitrateTracker.kt index e795c94fad..b399939cb8 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/BitrateTracker.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/BitrateTracker.kt @@ -29,6 +29,7 @@ open class BitrateTracker @JvmOverloads constructor( // that RateTracker uses. private val tracker = RateTracker(windowSize, bucketSize, clock) open fun getRate(nowMs: Long = clock.millis()): Bandwidth = tracker.getRate(nowMs).bps + @JvmOverloads open fun getRateBps(nowMs: Long = clock.millis()): Long = tracker.getRate(nowMs) val rate: Bandwidth diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ExecutorUtils.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ExecutorUtils.kt index 5fa3a326ac..83ec21b075 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ExecutorUtils.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ExecutorUtils.kt @@ -21,6 +21,7 @@ import java.util.concurrent.ExecutorService import java.util.concurrent.TimeUnit class ExecutorShutdownTimeoutException : Exception("Timed out trying to shutdown executor service") + /** * Shutdown [executorService] normally via [ExecutorService.shutdown]. If, after [timeout] / 2, the * service has still not shutdown, try to stop it more forcefully via [ExecutorService.shutdownNow]. If, diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/NodeStatsBlockExtensions.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/NodeStatsBlockExtensions.kt index eee6c2b6c4..17e569d7cc 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/NodeStatsBlockExtensions.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/NodeStatsBlockExtensions.kt @@ -49,7 +49,6 @@ fun NodeStatsBlock.addRatio( denominatorKey: String, defaultDenominator: Number = 1 ) = addCompoundValue(name) { - val numerator = it.getNumber(numeratorKey) ?: 0 val denominator = it.getNumber(denominatorKey) ?: defaultDenominator diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/PacketCache.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/PacketCache.kt index cf386e15f6..a9a3ed426a 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/PacketCache.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/PacketCache.kt @@ -39,9 +39,9 @@ class PacketCache( LinkedHashMap( // These are the default values of initialCapacity and loadFactor - we have to set them to be able to set // accessOrder - 16, /* initialCapacity */ - 0.75F, /* loadFactor */ - true /* accessOrder */ + 16, // initialCapacity + 0.75F, // loadFactor + true // accessOrder ) ) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/SsrcAssociationStore.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/SsrcAssociationStore.kt index e1f1d24bf2..f1ef149ec7 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/SsrcAssociationStore.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/SsrcAssociationStore.kt @@ -27,6 +27,7 @@ class SsrcAssociationStore( private val name: String = "SSRC Associations" ) : NodeStatsProducer { private val ssrcAssociations: MutableList = CopyOnWriteArrayList() + /** * The SSRC associations indexed by the primary SSRC. Since an SSRC may have * multiple secondary SSRC mappings, the primary SSRC maps to a list of its diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi_modified/impl/neomedia/rtp/sendsidebandwidthestimation/config/SendSideBandwidthEstimationConfig.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi_modified/impl/neomedia/rtp/sendsidebandwidthestimation/config/SendSideBandwidthEstimationConfig.kt index 49cf601d83..5d3a0de9db 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi_modified/impl/neomedia/rtp/sendsidebandwidthestimation/config/SendSideBandwidthEstimationConfig.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi_modified/impl/neomedia/rtp/sendsidebandwidthestimation/config/SendSideBandwidthEstimationConfig.kt @@ -25,7 +25,7 @@ import org.jitsi.nlj.util.kbps class SendSideBandwidthEstimationConfig { companion object { private val defaultLowLossThreshold: Double by - config("jmt.bwe.send-side.low-loss-threshold".from(JitsiConfig.newConfig)) + config("jmt.bwe.send-side.low-loss-threshold".from(JitsiConfig.newConfig)) /** * The low-loss threshold (expressed as a proportion of lost packets) when the loss probability @@ -35,7 +35,7 @@ class SendSideBandwidthEstimationConfig { fun defaultLowLossThreshold() = defaultLowLossThreshold private val defaultHighLossThreshold: Double by - config("jmt.bwe.send-side.high-loss-threshold".from(JitsiConfig.newConfig)) + config("jmt.bwe.send-side.high-loss-threshold".from(JitsiConfig.newConfig)) /** * The high-loss threshold (expressed as a proportion of lost packets) when the loss probability @@ -48,6 +48,7 @@ class SendSideBandwidthEstimationConfig { "jmt.bwe.send-side.bitrate-threshold".from(JitsiConfig.newConfig) .convertFrom { Bandwidth.fromString(it) } } + /** * The bitrate threshold when the loss probability experiment is *not* active. */ diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/MediaSourceDescTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/MediaSourceDescTest.kt index bce5a8065d..25f144afe1 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/MediaSourceDescTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/MediaSourceDescTest.kt @@ -141,13 +141,15 @@ private fun createRTPLayerDescs( arrayOf( rtpLayers[ idx( - spatialIdx, temporalIdx - 1, + spatialIdx, + temporalIdx - 1, temporalLen ) ]!!, rtpLayers[ idx( - spatialIdx - 1, temporalIdx, + spatialIdx - 1, + temporalIdx, temporalLen ) ]!! @@ -157,7 +159,8 @@ private fun createRTPLayerDescs( arrayOf( rtpLayers[ idx( - spatialIdx - 1, temporalIdx, + spatialIdx - 1, + temporalIdx, temporalLen ) ]!! @@ -167,7 +170,8 @@ private fun createRTPLayerDescs( arrayOf( rtpLayers[ idx( - spatialIdx, temporalIdx - 1, + spatialIdx, + temporalIdx - 1, temporalLen ) ]!! @@ -180,7 +184,11 @@ private fun createRTPLayerDescs( val spatialId = if (spatialLen > 1) spatialIdx else -1 rtpLayers[idx] = RtpLayerDesc( encodingIdx, - temporalId, spatialId, height, frameRate, dependencies + temporalId, + spatialId, + height, + frameRate, + dependencies ) frameRate *= 2.0 } @@ -207,8 +215,10 @@ private fun createRtpEncodingDesc( height: Int ): RtpEncodingDesc { val layers: Array = createRTPLayerDescs( - spatialLen, temporalLen, - encodingIdx, height + spatialLen, + temporalLen, + encodingIdx, + height ) val enc = RtpEncodingDesc(primarySsrc, layers) return enc @@ -228,7 +238,10 @@ private fun createSource( val primarySsrc: Long = primarySsrcs[encodingIdx] val ret = createRtpEncodingDesc( primarySsrc, - numSpatialLayersPerStream, numTemporalLayersPerStream, encodingIdx, height + numSpatialLayersPerStream, + numTemporalLayersPerStream, + encodingIdx, + height ) height *= 2 ret diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/codec/vpx/PictureIdIndexTrackerTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/codec/vpx/PictureIdIndexTrackerTest.kt index ec5b9b8e2b..e94ff210d6 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/codec/vpx/PictureIdIndexTrackerTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/codec/vpx/PictureIdIndexTrackerTest.kt @@ -40,7 +40,7 @@ internal class PictureIdIndexTrackerTest : ShouldSpec() { context("and then another which does roll over") { val rollOverIndex = indexTracker.update(2) should("return the proper index") { - rollOverIndex shouldBe 1L /* roc */ * 0x8000 + 2 + rollOverIndex shouldBe 1L * 0x8000 + 2 } context("and then a sequence number from the previous rollover") { val prevRollOverIndex = indexTracker.update(32002) diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/EndToEndHarness.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/EndToEndHarness.kt index e1b03e5248..c22b46d481 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/EndToEndHarness.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/EndToEndHarness.kt @@ -59,13 +59,21 @@ fun main() { val executor = Executors.newSingleThreadExecutor() val sender = SenderFactory.createSender( - executor, backgroundExecutor, pcap.srtpData, - pcap.payloadTypes, pcap.headerExtensions, pcap.ssrcAssociations + executor, + backgroundExecutor, + pcap.srtpData, + pcap.payloadTypes, + pcap.headerExtensions, + pcap.ssrcAssociations ) val receiver = ReceiverFactory.createReceiver( - executor, backgroundExecutor, pcap.srtpData, - pcap.payloadTypes, pcap.headerExtensions, pcap.ssrcAssociations, + executor, + backgroundExecutor, + pcap.srtpData, + pcap.payloadTypes, + pcap.headerExtensions, + pcap.ssrcAssociations, { rtcpPacket -> sender.processPacket(PacketInfo(rtcpPacket)) } ) diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpReceiverHarness.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpReceiverHarness.kt index 681f9706fc..a098679635 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpReceiverHarness.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpReceiverHarness.kt @@ -48,8 +48,12 @@ fun main() { val receivers = mutableListOf() repeat(numReceivers) { val receiver = ReceiverFactory.createReceiver( - executor, backgroundExecutor, pcap.srtpData, - pcap.payloadTypes, pcap.headerExtensions, pcap.ssrcAssociations, + executor, + backgroundExecutor, + pcap.srtpData, + pcap.payloadTypes, + pcap.headerExtensions, + pcap.ssrcAssociations, logger = StdoutLogger("receiver", Level.ALL) ) receivers.add(receiver) diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpReceiverNoNewBufferTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpReceiverNoNewBufferTest.kt index ffebaf7af1..15219ba873 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpReceiverNoNewBufferTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpReceiverNoNewBufferTest.kt @@ -43,8 +43,12 @@ fun main() { val executor = Executors.newSingleThreadExecutor() val receiver = ReceiverFactory.createReceiver( - executor, backgroundExecutor, pcap.srtpData, - pcap.payloadTypes, pcap.headerExtensions, pcap.ssrcAssociations + executor, + backgroundExecutor, + pcap.srtpData, + pcap.payloadTypes, + pcap.headerExtensions, + pcap.ssrcAssociations ) val sentArrays = LinkedBlockingQueue() diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpSenderHarness.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpSenderHarness.kt index 3c113ae911..fd348cfad6 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpSenderHarness.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpSenderHarness.kt @@ -37,8 +37,12 @@ fun main() { val senders = mutableListOf() repeat(numSenders) { val sender = SenderFactory.createSender( - senderExecutor, backgroundExecutor, pcap.srtpData, - pcap.payloadTypes, pcap.headerExtensions, pcap.ssrcAssociations + senderExecutor, + backgroundExecutor, + pcap.srtpData, + pcap.payloadTypes, + pcap.headerExtensions, + pcap.ssrcAssociations ) senders.add(sender) } diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/SrtpTransformerFactory.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/SrtpTransformerFactory.kt index e8c3f7939f..2bc1e2f9a0 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/SrtpTransformerFactory.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/SrtpTransformerFactory.kt @@ -28,7 +28,7 @@ class SrtpTransformerFactory { srtpData.srtpProfileInformation, srtpData.keyingMaterial, srtpData.tlsRole, - cryptex = false, /* TODO: add tests for the cryptex=true case */ + cryptex = false, // TODO: add tests for the cryptex=true case StdoutLogger() ) } diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/resources/logging/StdoutLogger.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/resources/logging/StdoutLogger.kt index 12636bdfb7..bc9336ade0 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/resources/logging/StdoutLogger.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/resources/logging/StdoutLogger.kt @@ -131,6 +131,7 @@ class StdoutLogger( override fun setLevel(level: Level) { _level = level } + /* These can be stubs */ override fun setUseParentHandlers(useParentHandlers: Boolean) = Unit override fun addHandler(handler: Handler?) = Unit diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/resources/srtp_samples/SrtpSample.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/resources/srtp_samples/SrtpSample.kt index 12426b5b94..49161fc990 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/resources/srtp_samples/SrtpSample.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/resources/srtp_samples/SrtpSample.kt @@ -35,9 +35,13 @@ import org.jitsi.rtp.util.byteBufferOf class SrtpSample { companion object { val srtpProfileInformation = SrtpProfileInformation( - cipherKeyLength = 16, cipherSaltLength = 14, cipherName = 1, - authFunctionName = 1, authKeyLength = 20, - rtcpAuthTagLength = 10, rtpAuthTagLength = 10 + cipherKeyLength = 16, + cipherSaltLength = 14, + cipherName = 1, + authFunctionName = 1, + authKeyLength = 20, + rtcpAuthTagLength = 10, + rtpAuthTagLength = 10 ) val keyingMaterial = byteBufferOf( 0xB4, 0x04, 0x3B, 0x87, 0x67, 0xF6, 0xC4, 0x67, diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtcp/StreamPacketRequesterTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtcp/StreamPacketRequesterTest.kt index 32f7c673f2..99315059c6 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtcp/StreamPacketRequesterTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtcp/StreamPacketRequesterTest.kt @@ -37,7 +37,11 @@ class StreamPacketRequesterTest : ShouldSpec() { } private val streamPacketRequester = RetransmissionRequester.StreamPacketRequester( - 123L, scheduler, scheduler.clock, ::rtcpSender, StdoutLogger() + 123L, + scheduler, + scheduler.clock, + ::rtcpSender, + StdoutLogger() ) init { diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/AudioRedHandlerTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/AudioRedHandlerTest.kt index f2f524379d..d01297121e 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/AudioRedHandlerTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/AudioRedHandlerTest.kt @@ -199,7 +199,7 @@ class AudioRedHandlerTest : ShouldSpec() { packet4WasAvailable: Boolean ) { size shouldBe 6 - map { it.sequenceNumber }.toList() shouldBe listOf(0, 1, 2, 3, /* 4 is lost */ 5, 6) + map { it.sequenceNumber }.toList() shouldBe listOf(0, 1, 2, 3, 5, 6) // 4 is lost forEach { it.payloadType shouldBe 112 it.shouldBeTypeOf() diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9PacketTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9PacketTest.kt index b87cb6609b..3aee52fc19 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9PacketTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9PacketTest.kt @@ -137,14 +137,15 @@ class Vp9PacketTest : ShouldSpec() { 0x6098017bL, arrayOf( RtpLayerDesc(0, 0, 0, 180, 7.5), - RtpLayerDesc(0, 1, 0, 180, 15.0 /* TODO: dependencies */), - RtpLayerDesc(0, 2, 0, 180, 30.0 /* TODO: dependencies */), - RtpLayerDesc(0, 0, 1, 360, 7.5 /* TODO: dependencies */), - RtpLayerDesc(0, 1, 1, 360, 15.0 /* TODO: dependencies */), - RtpLayerDesc(0, 2, 1, 360, 30.0 /* TODO: dependencies */), - RtpLayerDesc(0, 0, 2, 720, 7.5 /* TODO: dependencies */), - RtpLayerDesc(0, 1, 2, 720, 15.0 /* TODO: dependencies */), - RtpLayerDesc(0, 2, 2, 720, 30.0 /* TODO: dependencies */) + /* TODO: dependencies */ + RtpLayerDesc(0, 1, 0, 180, 15.0), + RtpLayerDesc(0, 2, 0, 180, 30.0), + RtpLayerDesc(0, 0, 1, 360, 7.5), + RtpLayerDesc(0, 1, 1, 360, 15.0), + RtpLayerDesc(0, 2, 1, 360, 30.0), + RtpLayerDesc(0, 0, 2, 720, 7.5), + RtpLayerDesc(0, 1, 2, 720, 15.0), + RtpLayerDesc(0, 2, 2, 720, 30.0) ) ) ), diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/RemoteBandwidthEstimatorTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/RemoteBandwidthEstimatorTest.kt index b09b716cb4..3496bf7ad5 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/RemoteBandwidthEstimatorTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/RemoteBandwidthEstimatorTest.kt @@ -40,6 +40,7 @@ class RemoteBandwidthEstimatorTest : ShouldSpec() { private val clock: FakeClock = FakeClock() private val astExtensionId = 3 + // REMB is enabled by having at least one payload type which has "goog-remb" signaled as a rtcp-fb, and TCC is // disabled. private val vp8PayloadType = Vp8PayloadType(100, emptyMap(), setOf("goog-remb")) diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/SrtpDecryptTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/SrtpDecryptTest.kt index 82f0afc15f..e28bc9ea4c 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/SrtpDecryptTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/SrtpDecryptTest.kt @@ -36,7 +36,7 @@ internal class SrtpDecryptTest : ShouldSpec() { SrtpSample.srtpProfileInformation, SrtpSample.keyingMaterial.array(), SrtpSample.tlsRole, - cryptex = false, /* TODO: add tests for cryptex case */ + cryptex = false, // TODO: add tests for cryptex case StdoutLogger() ) diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNodeTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNodeTest.kt index 58550e2e07..c1a559dd39 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNodeTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNodeTest.kt @@ -39,6 +39,7 @@ class TccGeneratorNodeTest : ShouldSpec() { private val onTccReady = { tccPacket: RtcpPacket -> tccPackets.add(tccPacket); Unit } private val streamInformationStore = StreamInformationStoreImpl() private val tccExtensionId = 5 + // TCC is enabled by having at least one payload type which has "transport-cc" signaled as a rtcp-fb. private val vp8PayloadType = Vp8PayloadType(100, emptyMap(), setOf("transport-cc")) diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/RetransmissionSenderTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/RetransmissionSenderTest.kt index bcca035d0f..79e08e3e31 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/RetransmissionSenderTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/RetransmissionSenderTest.kt @@ -38,6 +38,7 @@ class RetransmissionSenderTest : ShouldSpec() { private val originalSsrc = 1234L private val rtxSsrc = 5678L private val streamInformationStore = StreamInformationStoreImpl() + // NOTE(brian): unfortunately i ran into issues trying to use mock frameworks to mock // a packet, notably i ran into issues when trying to mock the byte[] property in // the parent java class, mocking frameworks seem to struggle with this diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/SrtpEncryptTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/SrtpEncryptTest.kt index ac9e4c478e..c6db04be94 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/SrtpEncryptTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/SrtpEncryptTest.kt @@ -41,7 +41,7 @@ internal class SrtpEncryptTest : ShouldSpec() { SrtpSample.srtpProfileInformation, SrtpSample.keyingMaterial.array(), SrtpSample.tlsRole, - cryptex = false, /* TODO: add tests for cryptex case */ + cryptex = false, // TODO: add tests for cryptex case StdoutLogger() ) @@ -69,7 +69,8 @@ internal class SrtpEncryptTest : ShouldSpec() { println("original packet:\n${originalPacket.buffer.toHex()}") println("packet after:\n${encryptedPacket.buffer.toHex()}") RtcpHeader.getPacketType( - encryptedPacket.buffer, encryptedPacket.offset + encryptedPacket.buffer, + encryptedPacket.offset ) shouldBe TransportLayerRtcpFbPacket.PT } } diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/Rfc3711IndexTrackerTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/Rfc3711IndexTrackerTest.kt index 128bbab3d7..282f7016f6 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/Rfc3711IndexTrackerTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/Rfc3711IndexTrackerTest.kt @@ -40,7 +40,7 @@ internal class Rfc3711IndexTrackerTest : ShouldSpec() { context("and then another which does roll over") { val rollOverIndex = indexTracker.update(2) should("return the proper index") { - rollOverIndex shouldBe 1 /* roc */ * 0x1_0000 + 2L + rollOverIndex shouldBe 1 * 0x1_0000 + 2L } context("and then a sequence number from the previous rollover") { val prevRollOverIndex = indexTracker.update(65002) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index afd3beae28..f3226e50e9 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -895,12 +895,14 @@ class Endpoint @JvmOverloads constructor( override fun getNextSendSsrc(): Long { synchronized(sendSsrcs) { while (true) { - val ssrc = if (useRandomSendSsrcs) + val ssrc = if (useRandomSendSsrcs) { random.nextLong().and(0xFFFF_FFFFL) - else + } else { nextSendSsrc++ - if (sendSsrcs.add(ssrc)) + } + if (sendSsrcs.add(ssrc)) { return ssrc + } } } } @@ -928,8 +930,9 @@ class Endpoint @JvmOverloads constructor( bitrateController.transformRtcp(packet) if (doSsrcRewriting) { // Just check both tables instead of looking up the type first. - if (!videoSsrcs.rewriteRtcp(packet) && !audioSsrcs.rewriteRtcp(packet)) + if (!videoSsrcs.rewriteRtcp(packet) && !audioSsrcs.rewriteRtcp(packet)) { return + } } logger.trace { "relaying an sr from ssrc=${packet.senderSsrc}, timestamp=${packet.senderInfo.rtpTimestamp}" diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/EndpointConnectionStatusMonitor.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/EndpointConnectionStatusMonitor.kt index ba12300155..8e9c310a83 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/EndpointConnectionStatusMonitor.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/EndpointConnectionStatusMonitor.kt @@ -159,6 +159,7 @@ class EndpointConnectionStatusMonitor @JvmOverloads constructor( "endpoints_disconnected", "Endpoints detected as temporarily inactive/disconnected due to inactivity." ) + @JvmField val endpointsReconnected = VideobridgeMetricsContainer.instance.registerCounter( "endpoints_reconnected", diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/LoudestConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/LoudestConfig.kt index 5118e96da0..256bae7507 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/LoudestConfig.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/LoudestConfig.kt @@ -26,18 +26,22 @@ class LoudestConfig private constructor() { val routeLoudestOnly: Boolean by config( "videobridge.loudest.route-loudest-only".from(JitsiConfig.newConfig) ) + @JvmStatic val numLoudest: Int by config( "videobridge.loudest.num-loudest".from(JitsiConfig.newConfig) ) + @JvmStatic val alwaysRouteDominant: Boolean by config( "videobridge.loudest.always-route-dominant".from(JitsiConfig.newConfig) ) + @JvmStatic val energyExpireTime: Duration by config( "videobridge.loudest.energy-expire-time".from(JitsiConfig.newConfig) ) + @JvmStatic val energyAlphaPct: Int by config( "videobridge.loudest.energy-alpha-pct".from(JitsiConfig.newConfig) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt index 5c8b8d8669..0e5416bf98 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt @@ -93,7 +93,11 @@ fun main() { val xmppConnection = XmppConnection().apply { start() } val shutdownService = ShutdownServiceImpl() val videobridge = Videobridge( - xmppConnection, shutdownService, versionService.currentVersion, VersionConfig.config.release, Clock.systemUTC() + xmppConnection, + shutdownService, + versionService.currentVersion, + VersionConfig.config.release, + Clock.systemUTC() ).apply { start() } val healthChecker = videobridge.jvbHealthChecker val statsCollector = StatsCollector(VideobridgeStatistics(videobridge, xmppConnection)).apply { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt index f75523a4ac..15e59d85af 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt @@ -51,7 +51,11 @@ class SourceDesc private constructor( val ssrc2: Long ) { constructor(s: AudioSourceDesc) : this( - s.sourceName ?: "anon", s.owner ?: "unknown", VideoType.DISABLED, s.ssrc, -1 + s.sourceName ?: "anon", + s.owner ?: "unknown", + VideoType.DISABLED, + s.ssrc, + -1 ) constructor(s: MediaSourceDesc) : this(s.sourceName, s.owner, s.videoType, s.primarySSRC, getRtx(s)) companion object { @@ -121,7 +125,6 @@ class SendSsrc(val ssrc: Long) { */ fun rewriteRtp(packet: RtpPacket, sending: Boolean, recv: ReceiveSsrc) { if (sending) { - if (!recv.hasDeltas) { /* Calculate new deltas the first time a receive ssrc is mapped to a send ssrc. */ if (state.valid) { @@ -135,7 +138,7 @@ class SendSsrc(val ssrc: Long) { val prevSequenceNumber = RtpUtils.applySequenceNumberDelta(packet.sequenceNumber, -1) val prevTimestamp = - RtpUtils.applyTimestampDelta(packet.timestamp, -960) /* guessing */ + RtpUtils.applyTimestampDelta(packet.timestamp, -960) // guessing sequenceNumberDelta = RtpUtils.getSequenceNumberDelta(state.lastSequenceNumber, prevSequenceNumber) timestampDelta = @@ -168,7 +171,8 @@ class SendSsrc(val ssrc: Long) { packet.senderSsrc = ssrc if (packet is RtcpSrPacket) { packet.senderInfo.rtpTimestamp = RtpUtils.applyTimestampDelta( - packet.senderInfo.rtpTimestamp, timestampDelta + packet.senderInfo.rtpTimestamp, + timestampDelta ) } } @@ -269,7 +273,10 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: * forwarded to the endpoint, the element at the front of this list will be removed and that element's * local SSRC will be used. Note: indexed by primary SSRC. */ - private val sendSources = LRUCache(size, true /* accessOrder */) + private val sendSources = LRUCache( + size, + true // accessOrder + ) /** * Whether an incoming RTP packet can automatically activate its source (i.e. acquire a send SSRC). @@ -316,13 +323,13 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: allowCreate: Boolean, remappings: MutableList ): SendSource? { - /* Moves to end of LRU when found. */ var sendSource = sendSources.get(ssrc) if (sendSource == null) { - if (!allowCreate) + if (!allowCreate) { return null + } if (sendSources.size == size) { val eldest = sendSources.eldest() sendSource = SendSource(props, eldest.value.send1, eldest.value.send2) @@ -350,7 +357,6 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: * Assign send SSRCs to the given sources. Any remapped SSRCs will be notified to the client. */ fun activate(sources: List) { - val remappings = mutableListOf() synchronized(sendSources) { @@ -366,8 +372,9 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: logger.debug { this.toString() } } - if (remappings.isNotEmpty()) + if (remappings.isNotEmpty()) { notifyMappings(remappings) + } } /** @@ -381,8 +388,9 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: remappings = sendSources.values.toList() } - if (remappings.isNotEmpty()) + if (remappings.isNotEmpty()) { notifyMappings(remappings) + } } /** @@ -393,7 +401,6 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: * @return whether to send this packet. */ fun rewriteRtp(packet: RtpPacket, start: Boolean = true): Boolean { - val remappings = mutableListOf() var send = false @@ -424,8 +431,9 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: } } - if (remappings.isNotEmpty()) + if (remappings.isNotEmpty()) { notifyMappings(remappings) + } return send } @@ -435,8 +443,7 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: * For packets in the same direction as media flow; feedback messages handled separately. */ fun rewriteRtcp(packet: RtcpPacket): Boolean { - - val remappings = mutableListOf() /* unused */ + val remappings = mutableListOf() // unused val senderSsrc = packet.senderSsrc synchronized(sendSources) { @@ -458,7 +465,6 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: * then do not modify the packet and return null. */ fun unmapRtcpFbSsrc(packet: RtcpFbPacket): String? { - val mediaSsrc = packet.mediaSourceSsrc synchronized(sendSources) { @@ -531,10 +537,11 @@ class AudioSsrcCache(size: Int, ep: SsrcRewriter, parentLogger: Logger) : */ override fun findSourceProps(ssrc: Long): SourceDesc? { val p = ep.findAudioSourceProps(ssrc) - if (p == null || p.sourceName == null || p.owner == null) + if (p == null || p.sourceName == null || p.owner == null) { return null - else + } else { return SourceDesc(p) + } } /** diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt index e36761673a..2e575feeda 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt @@ -241,7 +241,9 @@ internal class BandwidthAllocator( var remainingBandwidth = if (allocationSettings.assumedBandwidthBps >= 0) { logger.warn("Allocating with assumed bandwidth ${allocationSettings.assumedBandwidthBps.bps}.") allocationSettings.assumedBandwidthBps - } else availableBandwidth + } else { + availableBandwidth + } var oldRemainingBandwidth: Long = -1 var oversending = false while (oldRemainingBandwidth != remainingBandwidth) { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt index 6570ff4b8a..f21f02751d 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt @@ -54,12 +54,14 @@ class BitrateController @JvmOverloads constructor( val eventEmitter = SyncEventEmitter() private val bitrateAllocatorEventHandler = BitrateAllocatorEventHandler() + /** * Keep track of the "forwarded" endpoints, i.e. the endpoints for which we are forwarding *some* layer. */ @Deprecated("", ReplaceWith("forwardedSources"), DeprecationLevel.WARNING) var forwardedEndpoints: Set = emptySet() private set + /** * Keep track of the "forwarded" sources, i.e. the media sources for which we are forwarding *some* layer. */ @@ -264,6 +266,7 @@ class BitrateController @JvmOverloads constructor( newEffectiveConstraints: EffectiveConstraintsMap, ) fun keyframeNeeded(endpointId: String?, ssrc: Long) + /** * This is meant to be internal to BitrateAllocator, but is exposed here temporarily for the purposes of testing. */ diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/Prioritize.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/Prioritize.kt index f205c23ca3..d265c8ac1b 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/Prioritize.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/Prioritize.kt @@ -62,7 +62,6 @@ fun prioritize( */ fun getEffectiveConstraints(sources: List, allocationSettings: AllocationSettings): EffectiveConstraintsMap { - // FIXME figure out before merge - is using source count instead of endpoints // Add 1 for the receiver endpoint, which is not in the list. val effectiveLastN = effectiveLastN(allocationSettings.lastN, sources.size + 1) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt index e9a1fc9243..44a171a83b 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt @@ -212,7 +212,6 @@ internal class SingleSourceAllocation( constraints: VideoConstraints, onStage: Boolean ): Layers { - var activeLayers = layers.filter { it.bitrate > 0 } // No active layers usually happens when the source has just been signaled and we haven't received // any packets yet. Add the layers here, so one gets selected and we can start forwarding sooner. @@ -290,7 +289,6 @@ internal class SingleSourceAllocation( layers: List, constraints: VideoConstraints, ): Layers { - val minHeight = layers.map { it.layer.height }.minOrNull() ?: return Layers.noLayers val noActiveLayers = layers.none { (_, bitrate) -> bitrate > 0 } val (preferredHeight, preferredFps) = getPreferred(constraints) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt index a2b6811993..57376b0bdb 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt @@ -68,7 +68,9 @@ class Vp9AdaptiveSourceProjectionContext( private var lastVp9FrameProjection = Vp9FrameProjection( diagnosticContext, - rtpState.ssrc, rtpState.maxSequenceNumber, rtpState.maxTimestamp + rtpState.ssrc, + rtpState.maxSequenceNumber, + rtpState.maxTimestamp ) /** @@ -116,8 +118,12 @@ class Vp9AdaptiveSourceProjectionContext( val projection: Vp9FrameProjection try { projection = createProjection( - frame = frame, initialPacket = packet, isResumption = acceptResult.isResumption, - isReset = result.isReset, mark = acceptResult.mark, receivedTime = receivedTime + frame = frame, + initialPacket = packet, + isResumption = acceptResult.isResumption, + isReset = result.isReset, + mark = acceptResult.mark, + receivedTime = receivedTime ) } catch (e: Exception) { logger.warn("Failed to create frame projection", e) @@ -352,7 +358,7 @@ class Vp9AdaptiveSourceProjectionContext( /* These must be non-null because we don't execute this function unless frameIsNewSsrc has returned false. - */ + */ val lastFrame = prevFrame(frame)!! val lastProjectedFrame = lastVp9FrameProjection.vp9Frame!! @@ -393,7 +399,7 @@ class Vp9AdaptiveSourceProjectionContext( ): Vp9FrameProjection { /* This must be non-null because we don't execute this function unless frameIsNewSsrc has returned false. - */ + */ val lastFrame = lastVp9FrameProjection.vp9Frame!! /* Apply the latest projected frame's projections out, linearly. */ @@ -452,8 +458,11 @@ class Vp9AdaptiveSourceProjectionContext( frameIsNewSsrc has returned false.) */ return createInEncodingProjection( - frame, lastVp9FrameProjection.vp9Frame!!, - initialPacket, mark, receivedTime + frame, + lastVp9FrameProjection.vp9Frame!!, + initialPacket, + mark, + receivedTime ) } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Frame.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Frame.kt index 0220fb6450..98387f0d7d 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Frame.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Frame.kt @@ -207,6 +207,7 @@ class Vp9Frame internal constructor( isKeyframe = packet.isKeyframe, numSpatialLayers = packet.scalabilityStructureNumSpatial ) + /** * Remember another packet of this frame. * Note: this assumes every packet is received only once, i.e. a filter @@ -305,7 +306,8 @@ class Vp9Frame internal constructor( isUpperLevelReference == pkt.isUpperLevelReference && usesInterLayerDependency == pkt.usesInterLayerDependency && isInterPicturePredicted == pkt.isInterPicturePredicted - ) /* TODO: also check start, end, seq nums? */ { + // TODO: also check start, end, seq nums? + ) { return } throw RuntimeException( @@ -391,7 +393,7 @@ class Vp9Frame internal constructor( delta == 0 -> spatialLayer == otherFrame.spatialLayer + 1 delta == 1 -> - spatialLayer == 0 && otherFrame.spatialLayer == 2 /* ??? */ + spatialLayer == 0 && otherFrame.spatialLayer == 2 // ??? else -> false } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9FrameProjection.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9FrameProjection.kt index 8f334bb8e7..87598ee961 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9FrameProjection.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9FrameProjection.kt @@ -175,7 +175,9 @@ internal constructor( synchronized(vp9Frame) { return if (closedSeq < 0) { true - } else rtpPacket.sequenceNumber isOlderThan closedSeq + } else { + rtpPacket.sequenceNumber isOlderThan closedSeq + } } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Picture.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Picture.kt index 2407e4aac4..fad626785e 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Picture.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Picture.kt @@ -153,7 +153,6 @@ class Vp9Picture(packet: Vp9Packet, index: Int) { * @throws RuntimeException if the specified RTP packet is inconsistent with this frame */ fun validateConsistency(pkt: Vp9Packet) { - val f = frame(pkt) if (f != null) { f.validateConsistency(pkt) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9PictureMap.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9PictureMap.kt index b91a3aa761..2913590852 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9PictureMap.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9PictureMap.kt @@ -153,7 +153,7 @@ class Vp9PictureMap( } companion object { - const val PICTURE_MAP_SIZE = 500 /* Matches PacketCache default size. */ + const val PICTURE_MAP_SIZE = 500 // Matches PacketCache default size. } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/IqProcessingException.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/IqProcessingException.kt index 44da1844e0..4464338418 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/IqProcessingException.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/IqProcessingException.kt @@ -26,9 +26,11 @@ internal open class IqProcessingException( } internal class UnknownEndpointException(val endpointId: String) : IqProcessingException( - StanzaError.Condition.item_not_found, "Unknown endpoint $endpointId" + StanzaError.Condition.item_not_found, + "Unknown endpoint $endpointId" ) internal class FeatureNotImplementedException(message: String) : IqProcessingException( - StanzaError.Condition.feature_not_implemented, message + StanzaError.Condition.feature_not_implemented, + message ) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/PacketRateMeasurement.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/PacketRateMeasurement.kt index ed570b1ba1..584405c016 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/PacketRateMeasurement.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/PacketRateMeasurement.kt @@ -36,6 +36,7 @@ class PacketRateMeasurement(private val packetRate: Long) : JvbLoadMeasurement { @JvmStatic val loadedThreshold = config.loadThreshold + @JvmStatic val recoveryThreshold = config.recoverThreshold } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt index 382e8316b5..045abfe1b7 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt @@ -68,12 +68,14 @@ sealed class BridgeChannelMessage( val type: String ) { private val jsonCacheDelegate = ResettableLazy { createJson() } + /** * Caches the JSON string representation of this object. Note that after any changes to state (e.g. vars being set) * the cache needs to be invalidated via [resetJsonCache]. */ private val jsonCache: String by jsonCacheDelegate protected fun resetJsonCache() = jsonCacheDelegate.reset() + /** * Get a JSON representation of this [BridgeChannelMessage]. */ @@ -89,6 +91,7 @@ sealed class BridgeChannelMessage( private val mapper = jacksonObjectMapper().apply { enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS) } + @JvmStatic @Throws(JsonProcessingException::class, JsonMappingException::class) fun parse(string: String): BridgeChannelMessage { @@ -273,7 +276,9 @@ class DominantSpeakerMessage @JvmOverloads constructor( * element. */ constructor(previousSpeakers: List, silence: Boolean) : this( - previousSpeakers[0], previousSpeakers.drop(1), silence + previousSpeakers[0], + previousSpeakers.drop(1), + silence ) companion object { const val TYPE = "DominantSpeakerEndpointChangeEvent" @@ -308,10 +313,10 @@ class EndpointConnectionStatusMessage( */ @Deprecated("Use ForwardedSourcesMessage", ReplaceWith("ForwardedSourcesMessage"), DeprecationLevel.WARNING) class ForwardedEndpointsMessage( - @get:JsonProperty("lastNEndpoints") /** * The set of endpoints for which the bridge is currently sending video. */ + @get:JsonProperty("lastNEndpoints") val forwardedEndpoints: Collection ) : BridgeChannelMessage(TYPE) { companion object { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt index 4e4aea9441..5d5d2d59c7 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt @@ -579,8 +579,8 @@ class Relay @JvmOverloads constructor( if (sctpSocket == null) { /* TODO: this should be dependent on videobridge.websockets.enabled, if we support that being - * disabled for relay. - */ + * disabled for relay. + */ if (messageTransport.isActive) { iceUdpTransportPacketExtension.addChildExtension( WebSocketPacketExtension().apply { active = true } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt index 3cf8ba5424..1a5214ce71 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt @@ -464,7 +464,11 @@ class RelayMessageTransport( return null } - conference.sendMessage(message, listOf(targetEndpoint), false /* sendToRelays */) + conference.sendMessage( + message, + listOf(targetEndpoint), + false // sendToRelays + ) } return null } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt index 5f5c92d66b..ab0379ef0d 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt @@ -143,7 +143,7 @@ class RelayedEndpoint( AddReceiverMessage( RelayConfig.config.relayId, id, - null, /* source name - used in multi-stream */ + null, // source name - used in multi-stream maxVideoConstraints ) ) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/rest/RestConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/rest/RestConfig.kt index 46d5743e54..3b298cc7e9 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/rest/RestConfig.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/rest/RestConfig.kt @@ -65,6 +65,7 @@ class RestConfig private constructor() { "org.jitsi.videobridge.ENABLE_REST_SHUTDOWN".from(JitsiConfig.legacyConfig) "videobridge.rest.shutdown.enabled".from(JitsiConfig.newConfig) } + /** * Due to historical reasons the shutdown API is only enabled when the COLIBRI API is enabled. */ diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/DataChannelHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/DataChannelHandler.kt index df69991226..4d346e05b4 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/DataChannelHandler.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/DataChannelHandler.kt @@ -37,7 +37,9 @@ class DataChannelHandler : ConsumerNode("Data channel handler") { when (val packet = packetInfo.packet) { is DataChannelPacket -> { dataChannelStack?.onIncomingDataChannelPacket( - ByteBuffer.wrap(packet.buffer), packet.sid, packet.ppid + ByteBuffer.wrap(packet.buffer), + packet.sid, + packet.ppid ) ?: run { cachedDataChannelPackets.add(packetInfo) } @@ -64,7 +66,9 @@ class DataChannelHandler : ConsumerNode("Data channel handler") { cachedDataChannelPackets.forEach { val dcp = it.packet as DataChannelPacket dataChannelStack.onIncomingDataChannelPacket( - ByteBuffer.wrap(dcp.buffer), dcp.sid, dcp.ppid + ByteBuffer.wrap(dcp.buffer), + dcp.sid, + dcp.ppid ) } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/dtls/DtlsTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/dtls/DtlsTransport.kt index 2ad3b234d5..eb7ce55d96 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/dtls/DtlsTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/dtls/DtlsTransport.kt @@ -43,10 +43,13 @@ class DtlsTransport(parentLogger: Logger) { private val logger = createChildLogger(parentLogger) private val running = AtomicBoolean(true) + @JvmField var incomingDataHandler: IncomingDataHandler? = null + @JvmField var outgoingDataHandler: OutgoingDataHandler? = null + @JvmField var eventHandler: EventHandler? = null private var dtlsHandshakeComplete = false diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt index dc2b303b00..326e11aa32 100755 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt @@ -93,6 +93,7 @@ class IceTransport @JvmOverloads constructor( * Whether or not this [IceTransport] has connected. */ private val iceConnected = AtomicBoolean(false) + /** * Whether or not this [IceTransport] has failed to connect. */ @@ -460,10 +461,12 @@ class IceTransport @JvmOverloads constructor( * Notify the event handler that ICE connected successfully */ fun connected() + /** * Notify the event handler that ICE failed to connect */ fun failed() + /** * Notify the event handler that ICE consent was updated */ @@ -497,7 +500,8 @@ private fun CandidatePacketExtension.ipNeedsResolution(): Boolean = !InetAddresses.isInetAddress(ip) private fun TransportAddress.isPrivateAddress(): Boolean = address.isSiteLocalAddress || - /* 0xfc00::/7 */ ((address is Inet6Address) && ((addressBytes[0].toInt() and 0xfe) == 0xfc)) + /* 0xfc00::/7 */ + ((address is Inet6Address) && ((addressBytes[0].toInt() and 0xfe) == 0xfc)) private fun Transport.isTcpType(): Boolean = this == Transport.TCP || this == Transport.SSLTCP diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/ColibriWebSocketService.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/ColibriWebSocketService.kt index 2ffbd74c99..6ed779a0f0 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/ColibriWebSocketService.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/ColibriWebSocketService.kt @@ -100,6 +100,7 @@ class ColibriWebSocketService( companion object { private val logger = createLogger() + /** * The root path of the HTTP endpoint for COLIBRI WebSockets. */ diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerPerfTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerPerfTest.kt index cbff65d8cd..44d2c5b946 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerPerfTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerPerfTest.kt @@ -110,7 +110,6 @@ class BitrateControllerPerfTest : StringSpec() { } private fun run(testName: String, selectedEndpoints: List, maxFrameHeight: Int) { - val start = System.nanoTime() bc.lastN = 7 diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocationTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocationTest.kt index 5c7a71ffb7..4868548329 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocationTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocationTest.kt @@ -65,7 +65,12 @@ class SingleSourceAllocationTest : ShouldSpec() { context("Without constraints") { val allocation = SingleSourceAllocation( - endpointId, mediaSource, VideoConstraints(720), false, diagnosticContext, clock + endpointId, + mediaSource, + VideoConstraints(720), + false, + diagnosticContext, + clock ) // We include all resolutions up to the preferred resolution, and only high-FPS (at least @@ -77,7 +82,12 @@ class SingleSourceAllocationTest : ShouldSpec() { context("With constraints") { val allocation = SingleSourceAllocation( - endpointId, mediaSource, VideoConstraints(360), false, diagnosticContext, clock + endpointId, + mediaSource, + VideoConstraints(360), + false, + diagnosticContext, + clock ) // We include all resolutions up to the preferred resolution, and only high-FPS (at least @@ -100,7 +110,12 @@ class SingleSourceAllocationTest : ShouldSpec() { context("Non-zero constraints") { val allocation = SingleSourceAllocation( - endpointId, mediaSource, VideoConstraints(360), false, diagnosticContext, clock + endpointId, + mediaSource, + VideoConstraints(360), + false, + diagnosticContext, + clock ) // The receiver set 360p constraints, but we only have a 720p stream. @@ -111,7 +126,12 @@ class SingleSourceAllocationTest : ShouldSpec() { context("Zero constraints") { val allocation = SingleSourceAllocation( - endpointId, mediaSource, VideoConstraints(0), false, diagnosticContext, clock + endpointId, + mediaSource, + VideoConstraints(0), + false, + diagnosticContext, + clock ) // The receiver set a maxHeight=0 constraint. @@ -141,7 +161,12 @@ class SingleSourceAllocationTest : ShouldSpec() { val allocation = SingleSourceAllocation( - endpointId, mediaSource, VideoConstraints(720), false, diagnosticContext, clock + endpointId, + mediaSource, + VideoConstraints(720), + false, + diagnosticContext, + clock ) // We include all resolutions up to the preferred resolution, and only high-FPS (at least @@ -168,7 +193,12 @@ class SingleSourceAllocationTest : ShouldSpec() { context("With no constraints") { val allocation = SingleSourceAllocation( - endpointId, mediaSource, VideoConstraints(720), true, diagnosticContext, clock + endpointId, + mediaSource, + VideoConstraints(720), + true, + diagnosticContext, + clock ) // For screensharing the "preferred" layer should be the highest -- always prioritized over other @@ -181,7 +211,12 @@ class SingleSourceAllocationTest : ShouldSpec() { context("With 360p constraints") { val allocation = SingleSourceAllocation( - endpointId, mediaSource, VideoConstraints(360), true, diagnosticContext, clock + endpointId, + mediaSource, + VideoConstraints(360), + true, + diagnosticContext, + clock ) allocation.preferredLayer shouldBe sd30 @@ -207,7 +242,12 @@ class SingleSourceAllocationTest : ShouldSpec() { val allocation = SingleSourceAllocation( - "A", mediaSource, VideoConstraints(720), true, diagnosticContext, clock + "A", + mediaSource, + VideoConstraints(720), + true, + diagnosticContext, + clock ) // For screensharing the "preferred" layer should be the highest -- always prioritized over other @@ -238,7 +278,12 @@ class SingleSourceAllocationTest : ShouldSpec() { context("With no constraints") { val allocation = SingleSourceAllocation( - "A", mediaSource, VideoConstraints(720), true, diagnosticContext, clock + "A", + mediaSource, + VideoConstraints(720), + true, + diagnosticContext, + clock ) // For screensharing the "preferred" layer should be the highest -- always prioritized over other @@ -250,7 +295,12 @@ class SingleSourceAllocationTest : ShouldSpec() { context("With 180p constraints") { val allocation = SingleSourceAllocation( - "A", mediaSource, VideoConstraints(180), true, diagnosticContext, clock + "A", + mediaSource, + VideoConstraints(180), + true, + diagnosticContext, + clock ) // For screensharing the "preferred" layer should be the highest -- always prioritized over other @@ -280,7 +330,12 @@ class SingleSourceAllocationTest : ShouldSpec() { context("With no constraints") { val allocation = SingleSourceAllocation( - "A", mediaSource, VideoConstraints(720), true, diagnosticContext, clock + "A", + mediaSource, + VideoConstraints(720), + true, + diagnosticContext, + clock ) allocation.preferredLayer shouldBe l3 @@ -294,7 +349,12 @@ class SingleSourceAllocationTest : ShouldSpec() { // these layers. context("On stage") { val allocation = SingleSourceAllocation( - "A", mediaSource, VideoConstraints(180), true, diagnosticContext, clock + "A", + mediaSource, + VideoConstraints(180), + true, + diagnosticContext, + clock ) allocation.preferredLayer shouldBe l3 @@ -303,7 +363,12 @@ class SingleSourceAllocationTest : ShouldSpec() { } context("Off stage") { val allocation = SingleSourceAllocation( - "A", mediaSource, VideoConstraints(180), false, diagnosticContext, clock + "A", + mediaSource, + VideoConstraints(180), + false, + diagnosticContext, + clock ) allocation.preferredLayer shouldBe l1 diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionTest.kt index 44f8f44822..174c814445 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionTest.kt @@ -55,7 +55,8 @@ class Vp9AdaptiveSourceProjectionTest { private val logger: Logger = LoggerImpl(javaClass.name) private val payloadType: PayloadType = Vp9PayloadType( 96.toByte(), - ConcurrentHashMap(), CopyOnWriteArraySet() + ConcurrentHashMap(), + CopyOnWriteArraySet() ) @Test @@ -64,8 +65,10 @@ class Vp9AdaptiveSourceProjectionTest { diagnosticContext["test"] = "singlePacketProjectionTest" val initialState = RtpState(1, 10000, 1000000) val context = Vp9AdaptiveSourceProjectionContext( - diagnosticContext, payloadType, - initialState, logger + diagnosticContext, + payloadType, + initialState, + logger ) val generator = ScalableVp9PacketGenerator(1) val packetInfo = generator.nextPacket() @@ -74,7 +77,8 @@ class Vp9AdaptiveSourceProjectionTest { Assert.assertTrue( context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), targetIndex + getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + targetIndex ) ) context.rewriteRtp(packetInfo) @@ -90,8 +94,10 @@ class Vp9AdaptiveSourceProjectionTest { diagnosticContext["test"] = Thread.currentThread().stackTrace[2].methodName val initialState = RtpState(1, 10000, 1000000) val context = Vp9AdaptiveSourceProjectionContext( - diagnosticContext, payloadType, - initialState, logger + diagnosticContext, + payloadType, + initialState, + logger ) var expectedSeq = 10001 var expectedTs: Long = 1003000 @@ -104,7 +110,8 @@ class Vp9AdaptiveSourceProjectionTest { val packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), targetIndex + getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + targetIndex ) if (!packet.hasLayerIndices) { expectedTl0PicIdx = -1 @@ -170,7 +177,8 @@ class Vp9AdaptiveSourceProjectionTest { val context = Vp9AdaptiveSourceProjectionContext( diagnosticContext, payloadType, - initialState, logger + initialState, + logger ) var latestSeq = buffer[0]!!.packetAs().sequenceNumber val projectedPackets = TreeMap() @@ -187,7 +195,8 @@ class Vp9AdaptiveSourceProjectionTest { } val accepted = context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), targetIndex + getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + targetIndex ) val oldestValidSeq: Int = RtpUtils.applySequenceNumberDelta( @@ -200,9 +209,9 @@ class Vp9AdaptiveSourceProjectionTest { * that are part of an existing accepted frame will be accepted. */ } else if (packet.temporalLayerIndex <= targetTid && ( - packet.spatialLayerIndex == targetSid || - (packet.isUpperLevelReference && packet.spatialLayerIndex < targetSid) - ) + packet.spatialLayerIndex == targetSid || + (packet.isUpperLevelReference && packet.spatialLayerIndex < targetSid) + ) ) { Assert.assertTrue(accepted) @@ -271,7 +280,8 @@ class Vp9AdaptiveSourceProjectionTest { doRunOutOfOrderTest(generator, targetIndex, initialOrderedCount, seed) } catch (e: Throwable) { logger.error( - "Exception thrown in randomized test, seed = $seed", e + "Exception thrown in randomized test, seed = $seed", + e ) throw e } @@ -490,8 +500,10 @@ class Vp9AdaptiveSourceProjectionTest { diagnosticContext["test"] = "slightlyDelayedKeyframeTest" val initialState = RtpState(1, 10000, 1000000) val context = Vp9AdaptiveSourceProjectionContext( - diagnosticContext, payloadType, - initialState, logger + diagnosticContext, + payloadType, + initialState, + logger ) val firstPacketInfo = generator.nextPacket() val firstPacket = firstPacketInfo.packetAs() @@ -502,14 +514,16 @@ class Vp9AdaptiveSourceProjectionTest { Assert.assertFalse( context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), targetIndex + getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + targetIndex ) ) } Assert.assertTrue( context.accept( firstPacketInfo, - getIndex(0, firstPacket.spatialLayerIndex, firstPacket.temporalLayerIndex), targetIndex + getIndex(0, firstPacket.spatialLayerIndex, firstPacket.temporalLayerIndex), + targetIndex ) ) context.rewriteRtp(firstPacketInfo) @@ -519,7 +533,8 @@ class Vp9AdaptiveSourceProjectionTest { Assert.assertTrue( context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), targetIndex + getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + targetIndex ) ) context.rewriteRtp(packetInfo) @@ -533,8 +548,10 @@ class Vp9AdaptiveSourceProjectionTest { diagnosticContext["test"] = "veryDelayedKeyframeTest" val initialState = RtpState(1, 10000, 1000000) val context = Vp9AdaptiveSourceProjectionContext( - diagnosticContext, payloadType, - initialState, logger + diagnosticContext, + payloadType, + initialState, + logger ) val firstPacketInfo = generator.nextPacket() val firstPacket = firstPacketInfo.packetAs() @@ -545,14 +562,16 @@ class Vp9AdaptiveSourceProjectionTest { Assert.assertFalse( context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), targetIndex + getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + targetIndex ) ) } Assert.assertFalse( context.accept( firstPacketInfo, - getIndex(0, firstPacket.spatialLayerIndex, firstPacket.temporalLayerIndex), targetIndex + getIndex(0, firstPacket.spatialLayerIndex, firstPacket.temporalLayerIndex), + targetIndex ) ) for (i in 0..9) { @@ -561,7 +580,8 @@ class Vp9AdaptiveSourceProjectionTest { Assert.assertFalse( context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), targetIndex + getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + targetIndex ) ) } @@ -572,7 +592,8 @@ class Vp9AdaptiveSourceProjectionTest { Assert.assertTrue( context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), targetIndex + getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + targetIndex ) ) context.rewriteRtp(packetInfo) @@ -586,8 +607,10 @@ class Vp9AdaptiveSourceProjectionTest { diagnosticContext["test"] = "delayedPartialKeyframeTest" val initialState = RtpState(1, 10000, 1000000) val context = Vp9AdaptiveSourceProjectionContext( - diagnosticContext, payloadType, - initialState, logger + diagnosticContext, + payloadType, + initialState, + logger ) val firstPacketInfo = generator.nextPacket() val firstPacket = firstPacketInfo.packetAs() @@ -599,7 +622,8 @@ class Vp9AdaptiveSourceProjectionTest { Assert.assertTrue( context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), targetIndex + getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + targetIndex ) ) context.rewriteRtp(packetInfo) @@ -623,7 +647,8 @@ class Vp9AdaptiveSourceProjectionTest { Assert.assertTrue( context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), targetIndex + getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + targetIndex ) ) context.rewriteRtp(packetInfo) @@ -639,8 +664,10 @@ class Vp9AdaptiveSourceProjectionTest { diagnosticContext["test"] = "twoStreamsNoSwitchingTest" val initialState = RtpState(1, 10000, 1000000) val context = Vp9AdaptiveSourceProjectionContext( - diagnosticContext, payloadType, - initialState, logger + diagnosticContext, + payloadType, + initialState, + logger ) val targetIndex = getIndex(eid = 1, sid = 0, tid = 2) var expectedSeq = 10001 @@ -651,7 +678,8 @@ class Vp9AdaptiveSourceProjectionTest { Assert.assertTrue( context.accept( packetInfo1, - getIndex(1, packet1.spatialLayerIndex, packet1.temporalLayerIndex), targetIndex + getIndex(1, packet1.spatialLayerIndex, packet1.temporalLayerIndex), + targetIndex ) ) val packetInfo2 = generator2.nextPacket() @@ -659,7 +687,8 @@ class Vp9AdaptiveSourceProjectionTest { Assert.assertFalse( context.accept( packetInfo2, - getIndex(0, packet2.spatialLayerIndex, packet2.temporalLayerIndex), targetIndex + getIndex(0, packet2.spatialLayerIndex, packet2.temporalLayerIndex), + targetIndex ) ) context.rewriteRtp(packetInfo1) @@ -681,8 +710,10 @@ class Vp9AdaptiveSourceProjectionTest { diagnosticContext["test"] = "twoStreamsSwitchingTest" val initialState = RtpState(1, 10000, 1000000) val context = Vp9AdaptiveSourceProjectionContext( - diagnosticContext, payloadType, - initialState, logger + diagnosticContext, + payloadType, + initialState, + logger ) var expectedSeq = 10001 var expectedTs: Long = 1003000 @@ -703,7 +734,8 @@ class Vp9AdaptiveSourceProjectionTest { packetInfo1, getIndex( 0, - packet1.spatialLayerIndex, packet1.temporalLayerIndex + packet1.spatialLayerIndex, + packet1.temporalLayerIndex ), targetIndex ) @@ -720,7 +752,8 @@ class Vp9AdaptiveSourceProjectionTest { packetInfo2, getIndex( 1, - packet2.spatialLayerIndex, packet2.temporalLayerIndex + packet2.spatialLayerIndex, + packet2.temporalLayerIndex ), targetIndex ) @@ -781,7 +814,8 @@ class Vp9AdaptiveSourceProjectionTest { generator1.requestKeyframe() generator2.requestKeyframe() - /* After a keyframe we should accept spatial layer 1 */for (i in 0..8999) { + /* After a keyframe we should accept spatial layer 1 */ + for (i in 0..8999) { val srPacket1 = generator1.srPacket val packetInfo1 = generator1.nextPacket() val packet1 = packetInfo1.packetAs() @@ -794,7 +828,8 @@ class Vp9AdaptiveSourceProjectionTest { i == 0, context.accept( packetInfo1, - getIndex(0, packet1.spatialLayerIndex, packet1.temporalLayerIndex), targetIndex + getIndex(0, packet1.spatialLayerIndex, packet1.temporalLayerIndex), + targetIndex ) ) Assert.assertEquals(i == 0, context.rewriteRtcp(srPacket1)) @@ -812,7 +847,8 @@ class Vp9AdaptiveSourceProjectionTest { Assert.assertTrue( context.accept( packetInfo2, - getIndex(1, packet2.spatialLayerIndex, packet2.temporalLayerIndex), targetIndex + getIndex(1, packet2.spatialLayerIndex, packet2.temporalLayerIndex), + targetIndex ) ) context.rewriteRtp(packetInfo2) @@ -846,8 +882,10 @@ class Vp9AdaptiveSourceProjectionTest { diagnosticContext["test"] = "temporalLayerSwitchingTest" val initialState = RtpState(1, 10000, 1000000) val context = Vp9AdaptiveSourceProjectionContext( - diagnosticContext, payloadType, - initialState, logger + diagnosticContext, + payloadType, + initialState, + logger ) var targetTid = 0 var decodableTid = 0 @@ -861,7 +899,8 @@ class Vp9AdaptiveSourceProjectionTest { val packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), targetIndex + getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + targetIndex ) if (packet.isStartOfFrame && packet.temporalLayerIndex == 0) { expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) @@ -889,7 +928,7 @@ class Vp9AdaptiveSourceProjectionTest { if (packet.isEndOfFrame) { expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) expectedPicId = applyExtendedPictureIdDelta(expectedPicId, 1) - if (i % 97 == 0) /* Prime number so it's out of sync with packet cycles. */ { + if (i % 97 == 0) { // Prime number so it's out of sync with packet cycles. */ targetTid = (targetTid + 2) % 3 targetIndex = getIndex(0, 0, targetTid) } @@ -904,7 +943,8 @@ class Vp9AdaptiveSourceProjectionTest { val context = Vp9AdaptiveSourceProjectionContext( diagnosticContext, payloadType, - initialState, logger + initialState, + logger ) var expectedSeq = 10001 var expectedTs: Long = 1003000 @@ -917,7 +957,8 @@ class Vp9AdaptiveSourceProjectionTest { val packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), targetIndex + getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + targetIndex ) if (packet.isStartOfFrame && packet.temporalLayerIndex == 0) { expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) @@ -957,7 +998,8 @@ class Vp9AdaptiveSourceProjectionTest { Assert.assertTrue( context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), targetIndex + getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + targetIndex ) ) context.rewriteRtp(packetInfo) @@ -976,7 +1018,8 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), targetIndex + getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + targetIndex ) if (packet.isStartOfFrame && packet.temporalLayerIndex == 0) { expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) @@ -1032,7 +1075,8 @@ class Vp9AdaptiveSourceProjectionTest { val context = Vp9AdaptiveSourceProjectionContext( diagnosticContext, payloadType, - initialState, logger + initialState, + logger ) var expectedSeq = 10001 var expectedTs: Long = 1003000 @@ -1052,7 +1096,8 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), targetIndex + getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + targetIndex ) if (packet.isStartOfFrame && packet.temporalLayerIndex == 0) { expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) @@ -1091,7 +1136,8 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), targetIndex + getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + targetIndex ) Assert.assertTrue(accepted) context.rewriteRtp(packetInfo) @@ -1108,7 +1154,8 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), RtpLayerDesc.SUSPENDED_INDEX + getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + RtpLayerDesc.SUSPENDED_INDEX ) Assert.assertFalse(accepted) if (packet.isEndOfPicture) { @@ -1123,7 +1170,8 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), targetIndex + getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + targetIndex ) Assert.assertFalse(accepted) if (packet.isEndOfPicture) { @@ -1139,7 +1187,8 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), targetIndex + getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + targetIndex ) Assert.assertFalse(accepted) if (packet.isEndOfPicture) { @@ -1153,7 +1202,8 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), targetIndex + getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + targetIndex ) if (packet.isStartOfFrame && packet.temporalLayerIndex == 0) { expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) @@ -1225,7 +1275,7 @@ class Vp9AdaptiveSourceProjectionTest { } companion object { - val baseReceivedTime = Instant.ofEpochMilli(1577836800000L) /* 2020-01-01 00:00:00 UTC */ + val baseReceivedTime = Instant.ofEpochMilli(1577836800000L) // 2020-01-01 00:00:00 UTC } } @@ -1268,15 +1318,21 @@ class Vp9AdaptiveSourceProjectionTest { Vp9Packet computes values at construct-time. */ DePacketizer.VP9PayloadDescriptor.setStartOfFrame( rtpPacket.buffer, - rtpPacket.payloadOffset, rtpPacket.payloadLength, startOfFrame + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + startOfFrame ) DePacketizer.VP9PayloadDescriptor.setEndOfFrame( rtpPacket.buffer, - rtpPacket.payloadOffset, rtpPacket.payloadLength, endOfFrame + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + endOfFrame ) DePacketizer.VP9PayloadDescriptor.setInterPicturePredicted( rtpPacket.buffer, - rtpPacket.payloadOffset, rtpPacket.payloadLength, !keyframePicture + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + !keyframePicture ) rtpPacket.isMarked = endOfFrame @@ -1317,13 +1373,13 @@ class Vp9AdaptiveSourceProjectionTest { } } companion object { - private val vp9PacketTemplate = DatatypeConverter.parseHexBinary( /* RTP Header */ - "80" + /* V, P, X, CC */ - "60" + /* M, PT */ - "0000" + /* Seq */ - "00000000" + /* TS */ - "cafebabe" + /* SSRC */ - /* VP9 Payload descriptor */ + private val vp9PacketTemplate = DatatypeConverter.parseHexBinary( // RTP Header + "80" + // V, P, X, CC + "60" + // M, PT + "0000" + // Seq + "00000000" + // TS + "cafebabe" + // SSRC + // VP9 Payload descriptor // I=1,P=0,L=0,F=0,B=1,E=0,V=0,Z=0 "88" + // M=1,PID=0x653e=25918 @@ -1381,7 +1437,7 @@ class Vp9AdaptiveSourceProjectionTest { 2 -> 1 1, 3 -> 2 else -> { - assert(false /* Math is broken */) + assert(false) // Math is broken -1 } } @@ -1402,25 +1458,37 @@ class Vp9AdaptiveSourceProjectionTest { Vp9Packet computes values at construct-time. */ DePacketizer.VP9PayloadDescriptor.setStartOfFrame( rtpPacket.buffer, - rtpPacket.payloadOffset, rtpPacket.payloadLength, startOfFrame + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + startOfFrame ) DePacketizer.VP9PayloadDescriptor.setEndOfFrame( rtpPacket.buffer, - rtpPacket.payloadOffset, rtpPacket.payloadLength, endOfFrame + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + endOfFrame ) DePacketizer.VP9PayloadDescriptor.setInterPicturePredicted( rtpPacket.buffer, - rtpPacket.payloadOffset, rtpPacket.payloadLength, !keyframePicture + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + !keyframePicture ) DePacketizer.VP9PayloadDescriptor.setUpperLevelReference( rtpPacket.buffer, - rtpPacket.payloadOffset, rtpPacket.payloadLength, sid != numLayers - 1 + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + sid != numLayers - 1 ) Assert.assertTrue( DePacketizer.VP9PayloadDescriptor.setLayerIndices( rtpPacket.buffer, - rtpPacket.payloadOffset, rtpPacket.payloadLength, sid, tid, tid > 0, + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + sid, + tid, + tid > 0, sid > 0 && (isKsvc || keyframePicture) ) ) @@ -1495,13 +1563,13 @@ class Vp9AdaptiveSourceProjectionTest { } companion object { - private val vp9SvcPacketTemplate = DatatypeConverter.parseHexBinary( /* RTP Header */ - "80" + /* V, P, X, CC */ - "60" + /* M, PT */ - "0000" + /* Seq */ - "00000000" + /* TS */ - "cafebabe" + /* SSRC */ - /* VP9 Payload descriptor */ + private val vp9SvcPacketTemplate = DatatypeConverter.parseHexBinary( // RTP Header + "80" + // V, P, X, CC + "60" + // M, PT + "0000" + // Seq + "00000000" + // TS + "cafebabe" + // SSRC + // VP9 Payload descriptor // I=1,P=0,L=1,F=0,B=1,E=0,V=0,Z=0 "a8" + // M=1,PID=0x653e=25918 @@ -1514,6 +1582,7 @@ class Vp9AdaptiveSourceProjectionTest { // Dummy payload data "000000" ) + /* TODO: move this to jitsi-rtp */ fun setSIBuilderNtp(siBuilder: SenderInfoBuilder, wallTime: Long) { val JAVA_TO_NTP_EPOCH_OFFSET_SECS = 2208988800L diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilterTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilterTest.kt index 5cdc0a9963..07bb90c2d0 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilterTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilterTest.kt @@ -271,8 +271,11 @@ internal class Vp9QualityFilterTest : ShouldSpec() { testGenerator(generator, filter, targetIndex2, numFrames = 1200) { f, result -> if (f.spatialLayer == 2) sawTargetLayer = true if (f.isKeyframe) sawKeyframe = true - result.accept shouldBe if (!sawKeyframe) (f.spatialLayer == 0) else + result.accept shouldBe if (!sawKeyframe) { + (f.spatialLayer == 0) + } else { (f.spatialLayer == 2 || !f.isInterPicturePredicted) + } if (result.accept) { result.mark shouldBe if (!sawKeyframe) (f.spatialLayer == 0) else (f.spatialLayer == 2) filter.needsKeyframe shouldBe (sawTargetLayer && !sawKeyframe) @@ -287,9 +290,11 @@ internal class Vp9QualityFilterTest : ShouldSpec() { testGenerator(generator, filter, targetIndex3) { f, result -> if (f.spatialLayer == 1) sawTargetLayer = true if (f.isKeyframe) sawKeyframe = true - result.accept shouldBe if (!sawKeyframe) - ((f.spatialLayer == 2 || !f.isInterPicturePredicted) && f.temporalLayer == 0) else + result.accept shouldBe if (!sawKeyframe) { + ((f.spatialLayer == 2 || !f.isInterPicturePredicted) && f.temporalLayer == 0) + } else { (f.spatialLayer == 1 || (!f.isInterPicturePredicted && f.spatialLayer < 1)) + } if (result.accept) { result.mark shouldBe if (!sawKeyframe) (f.spatialLayer == 2) else (f.spatialLayer == 1) filter.needsKeyframe shouldBe !sawKeyframe @@ -384,9 +389,11 @@ internal class Vp9QualityFilterTest : ShouldSpec() { val targetIndex3 = RtpLayerDesc.getIndex(1, 0, 2) testGenerator(generator, filter, targetIndex3) { f, result -> if (f.isKeyframe) sawKeyframe = true - result.accept shouldBe if (!sawKeyframe) - (f.temporalLayer == 0 && (f.ssrc == 2L || f.isKeyframe)) else + result.accept shouldBe if (!sawKeyframe) { + (f.temporalLayer == 0 && (f.ssrc == 2L || f.isKeyframe)) + } else { (f.ssrc == 1L || (f.isKeyframe && f.ssrc < 1L)) + } if (result.accept) { result.mark shouldBe true filter.needsKeyframe shouldBe !sawKeyframe @@ -638,7 +645,7 @@ private class SimulcastFrameGenerator : FrameGenerator() { val keyframePicture = (pictureCount % 48) == 0 val f = Vp9Frame( - ssrc = enc.toLong(), /* Use the encoding ID as the SSRC to make testing easier. */ + ssrc = enc.toLong(), // Use the encoding ID as the SSRC to make testing easier. timestamp = pictureCount * 3000L, earliestKnownSequenceNumber = pictureCount + (enc * 10000), latestKnownSequenceNumber = pictureCount + (enc * 10000), diff --git a/pom.xml b/pom.xml index e6dc087f0e..eeeed36a4a 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ 5.9.1 1.0-124-ge57838f 1.1-125-g805c0d8 - 1.13.1 + 1.16.0 3.2.2 4.6.0 3.0.10 diff --git a/rtp/pom.xml b/rtp/pom.xml index 465a19557e..f03c567fca 100644 --- a/rtp/pom.xml +++ b/rtp/pom.xml @@ -171,7 +171,7 @@ com.github.gantsign.maven ktlint-maven-plugin - 1.13.1 + ${ktlint-maven-plugin.version} ${project.basedir}/src/main/kotlin diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/Packet.kt b/rtp/src/main/kotlin/org/jitsi/rtp/Packet.kt index 7bfea4f9b2..0a7dd503d2 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/Packet.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/Packet.kt @@ -51,8 +51,10 @@ abstract class Packet( protected fun cloneBuffer(bytesToLeaveAtStart: Int): ByteArray = BufferPool.getArray(bytesToLeaveAtStart + length + BYTES_TO_LEAVE_AT_END_OF_PACKET).apply { System.arraycopy( - buffer, offset, - this, bytesToLeaveAtStart, + buffer, + offset, + this, + bytesToLeaveAtStart, length ) } diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/extensions/bytearray/ByteArrayExtensions.kt b/rtp/src/main/kotlin/org/jitsi/rtp/extensions/bytearray/ByteArrayExtensions.kt index 2e1e39c198..c353b9baed 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/extensions/bytearray/ByteArrayExtensions.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/extensions/bytearray/ByteArrayExtensions.kt @@ -108,6 +108,7 @@ operator fun ByteArray.plus(other: ByteArray): ByteArray { } private val HEX_CHARS = "0123456789ABCDEF".toCharArray() + /** * Print the contents of the [ByteArray] as hex digits. */ diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpReportBlock.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpReportBlock.kt index 8df65a9de0..3f1ed76522 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpReportBlock.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpReportBlock.kt @@ -58,9 +58,13 @@ class RtcpReportBlock( lastSrTimestamp: Long, delaySinceLastSr: Long ) : this( - ssrc, fractionLost, cumulativePacketsLost, + ssrc, + fractionLost, + cumulativePacketsLost, ((seqNumCycles shl 16) + seqNum.toShort()).toPositiveLong(), - interarrivalJitter, lastSrTimestamp, delaySinceLastSr + interarrivalJitter, + lastSrTimestamp, + delaySinceLastSr ) val seqNumCycles: Int by lazy { @@ -82,6 +86,7 @@ class RtcpReportBlock( companion object { const val SIZE_BYTES = 24 + // Offsets relative to the start of an RTCP Report Block const val SSRC_OFFSET = 0 const val FRACTION_LOST_OFFSET = 4 @@ -100,8 +105,13 @@ class RtcpReportBlock( val delaySinceLastSr = getDelaySinceLastSr(buffer, offset) return RtcpReportBlock( - ssrc, fractionLost, cumulativePacketsLost, extendedHighestSeqNum, - interarrivalJitter, lastSrTimestamp, delaySinceLastSr + ssrc, + fractionLost, + cumulativePacketsLost, + extendedHighestSeqNum, + interarrivalJitter, + lastSrTimestamp, + delaySinceLastSr ) } diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbFirPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbFirPacket.kt index 08a92b43c9..a68dbed81c 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbFirPacket.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbFirPacket.kt @@ -75,6 +75,7 @@ class RtcpFbFirPacket( companion object { const val FMT = 4 + // TODO: support multiple FCI? const val SIZE_BYTES = HEADER_SIZE + 8 diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/LastChunk.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/LastChunk.kt index d72b7504c5..3522780c0a 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/LastChunk.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/LastChunk.kt @@ -45,19 +45,23 @@ class LastChunk { // Return if delta sizes still can be encoded into single chunk with added // |delta_size|. fun CanAdd(deltaSize: DeltaSize): Boolean { - if (size_ < kMaxTwoBitCapacity) + if (size_ < kMaxTwoBitCapacity) { return true - if (size_ < kMaxOneBitCapacity && !has_large_delta_ && deltaSize != kLarge) + } + if (size_ < kMaxOneBitCapacity && !has_large_delta_ && deltaSize != kLarge) { return true - if (size_ < kMaxRunLengthCapacity && all_same_ && delta_sizes_[0] == deltaSize) + } + if (size_ < kMaxRunLengthCapacity && all_same_ && delta_sizes_[0] == deltaSize) { return true + } return false } // Add |delta_size|, assumes |CanAdd(delta_size)|, fun Add(deltaSize: DeltaSize) { - if (size_ < kMaxVectorCapacity) + if (size_ < kMaxVectorCapacity) { delta_sizes_[size_] = deltaSize + } size_++ all_same_ = all_same_ && deltaSize == delta_sizes_[0] has_large_delta_ = has_large_delta_ || deltaSize == kLarge diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacket.kt index 8f4d7e12d4..bf44629206 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacket.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacket.kt @@ -75,18 +75,23 @@ class RtcpFbTccPacketBuilder( ) { var base_seq_no_: RtpSequenceNumber = RtpSequenceNumber.INVALID private set + // The reference time, in ticks. Chrome passes this into BuildFeedbackPacket, but we don't // hold the times in the same way, so we'll just assign it the first time we see // a packet in AddReceivedPacket private var base_time_ticks_: Long = -1 + // The amount of packets_ whose status are represented var num_seq_no_ = 0 private set + // The current chunk we're 'filling out' as packets // are received private var last_chunk_ = LastChunk() + // All but last encoded packet chunks. private val encoded_chunks_ = mutableListOf() + // The size of the entire packet, in bytes private var size_bytes_ = kTransportFeedbackHeaderSizeBytes private var last_timestamp_us_: Long = 0 @@ -119,14 +124,16 @@ class RtcpFbTccPacketBuilder( return false } while (next_seq_no != sequence_number) { - if (!AddDeltaSize(0)) + if (!AddDeltaSize(0)) { return false + } next_seq_no += 1 } } val delta_size = if (delta >= 0 && delta <= 0xff) 1 else 2 - if (!AddDeltaSize(delta_size)) + if (!AddDeltaSize(delta_size)) { return false + } packets_.add(ReceivedPacketReport(sequence_number.value, delta)) last_timestamp_us_ += delta * kDeltaScaleFactor @@ -139,12 +146,14 @@ class RtcpFbTccPacketBuilder( base_time_ticks_ * kBaseScaleFactor private fun AddDeltaSize(deltaSize: DeltaSize): Boolean { - if (num_seq_no_ == kMaxReportedPackets) + if (num_seq_no_ == kMaxReportedPackets) { return false + } val add_chunk_size = if (last_chunk_.Empty()) kChunkSizeBytes else 0 - if (size_bytes_ + deltaSize + add_chunk_size > kMaxSizeBytes) + if (size_bytes_ + deltaSize + add_chunk_size > kMaxSizeBytes) { return false + } if (last_chunk_.CanAdd(deltaSize)) { size_bytes_ += add_chunk_size @@ -153,8 +162,9 @@ class RtcpFbTccPacketBuilder( return true } - if (size_bytes_ + deltaSize + kChunkSizeBytes > kMaxSizeBytes) + if (size_bytes_ + deltaSize + kChunkSizeBytes > kMaxSizeBytes) { return false + } encoded_chunks_.add(last_chunk_.Emit()) size_bytes_ += kChunkSizeBytes @@ -374,6 +384,7 @@ class RtcpFbTccPacket( // All but last encoded packet chunks. private val encoded_chunks_: MutableList get() = data.encoded_chunks_ + // The current chunk we're 'filling out' as packets // are received private var last_chunk_: LastChunk @@ -395,6 +406,7 @@ class RtcpFbTccPacket( set(value) { data.last_timestamp_us_ = value } + // The reference time, in ticks. private var base_time_ticks_: Long get() = data.base_time_ticks_ @@ -413,20 +425,26 @@ class RtcpFbTccPacket( companion object { const val FMT = 15 + // Convert to multiples of 0.25ms const val kDeltaScaleFactor = 250 + // Maximum number of packets_ (including missing) TransportFeedback can report. const val kMaxReportedPackets = 0xFFFF const val kChunkSizeBytes = 2 + // Fit TCC packets within an MTU and allow for further encapsulation (and perhaps compound RTCP) const val kMaxSizeBytes = 1200 + // Header size: // * 4 bytes Common RTCP Packet Header // * 8 bytes Common Packet Format for RTCP Feedback Messages // * 8 bytes FeedbackPacket header const val kTransportFeedbackHeaderSizeBytes = 4 + 8 + 8 + // Used to convert from microseconds to multiples of 64ms const val kBaseScaleFactor = kDeltaScaleFactor * (1 shl 8) + // The reference time field is 24 bits and are represented as multiples of 64ms // When the reference time field would need to wrap around const val kTimeWrapPeriodUs: Long = (1 shl 24).toLong() * kBaseScaleFactor @@ -436,6 +454,7 @@ class RtcpFbTccPacket( const val REFERENCE_TIME_OFFSET = RtcpFbPacket.HEADER_SIZE + 4 const val FB_PACKET_COUNT_OFFSET = RtcpFbPacket.HEADER_SIZE + 7 const val PACKET_CHUNKS_OFFSET = RtcpFbPacket.HEADER_SIZE + 8 + // baseOffset in all of these refers to the start of the entire RTCP TCC packet fun getBaseSeqNum(buf: ByteArray, baseOffset: Int): Int = buf.getShortAsInt(baseOffset + BASE_SEQ_NUM_OFFSET) diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RedPacketParser.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RedPacketParser.kt index f160166fbb..3e50868f45 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RedPacketParser.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RedPacketParser.kt @@ -74,15 +74,19 @@ class RedPacketParser( ) System.arraycopy( - buffer, offset, - byteArray, BYTES_TO_LEAVE_AT_START_OF_PACKET, + buffer, + offset, + byteArray, + BYTES_TO_LEAVE_AT_START_OF_PACKET, FIXED_HEADER_SIZE_BYTES ) RtpHeader.setCsrcCount(byteArray, BYTES_TO_LEAVE_AT_START_OF_PACKET, 0) RtpHeader.setHasExtensions(byteArray, BYTES_TO_LEAVE_AT_START_OF_PACKET, false) System.arraycopy( - buffer, currentOffset, - byteArray, BYTES_TO_LEAVE_AT_START_OF_PACKET + FIXED_HEADER_SIZE_BYTES, + buffer, + currentOffset, + byteArray, + BYTES_TO_LEAVE_AT_START_OF_PACKET + FIXED_HEADER_SIZE_BYTES, blockLength ) @@ -110,8 +114,10 @@ class RedPacketParser( val newOffset = currentOffset - headerLength val newLength = length - currentOffset + offset + headerLength System.arraycopy( - buffer, offset, - buffer, newOffset, + buffer, + offset, + buffer, + newOffset, headerLength ) offset = newOffset @@ -241,8 +247,10 @@ class RedPacketBuilder(val createPacket: (ByteArray, Int val primaryHeaderLength = primary.headerLength System.arraycopy( - primary.buffer, primary.offset, - buf, currentOffset, + primary.buffer, + primary.offset, + buf, + currentOffset, primaryHeaderLength ) currentOffset += primaryHeaderLength @@ -263,8 +271,10 @@ class RedPacketBuilder(val createPacket: (ByteArray, Int redHeaderOffset += header.write(buf, redHeaderOffset) System.arraycopy( - it.buffer, it.payloadOffset, - buf, currentOffset, + it.buffer, + it.payloadOffset, + buf, + currentOffset, payloadLength ) currentOffset += payloadLength @@ -274,8 +284,10 @@ class RedPacketBuilder(val createPacket: (ByteArray, Int redHeaderOffset += primaryHeader.write(buf, redHeaderOffset) System.arraycopy( - primary.buffer, primary.payloadOffset, - buf, currentOffset, + primary.buffer, + primary.payloadOffset, + buf, + currentOffset, primary.payloadLength ) diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpHeader.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpHeader.kt index 582d64cd66..afa1b71c87 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpHeader.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpHeader.kt @@ -54,6 +54,7 @@ class RtpHeader { companion object { const val FIXED_HEADER_SIZE_BYTES = 12 const val CSRCS_OFFSET = 12 + // The size of the RTP Extension header block const val EXT_HEADER_SIZE_BYTES = 4 const val VERSION = 2 diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpPacket.kt index cdc6ee6842..6741a65f8b 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpPacket.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpPacket.kt @@ -561,11 +561,14 @@ open class RtpPacket( // This treats unknown header extension types as no extensions, which is what we want. if (headerExtensionParser != null) { val extensionBlockLength = HeaderExtensionHelpers.getExtensionsTotalLength( - buffer, offset + RtpHeader.FIXED_HEADER_SIZE_BYTES + csrcCount * 4 + buffer, + offset + RtpHeader.FIXED_HEADER_SIZE_BYTES + csrcCount * 4 ) extensionBlockLength - HeaderExtensionHelpers.TOP_LEVEL_EXT_HEADER_SIZE_BYTES - } else 0 + } else { + 0 + } if (extLength <= 0) { // No extensions diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpSequenceNumber.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpSequenceNumber.kt index e5d73707e9..1240c995ed 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpSequenceNumber.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpSequenceNumber.kt @@ -71,7 +71,7 @@ class RtpSequenceNumberProgression( val start: RtpSequenceNumber, val endInclusive: RtpSequenceNumber, val step: Int = 1 -) : Iterable /*, ClosedRange */ { +) : Iterable { // , ClosedRange override fun iterator(): Iterator = RtpSequenceNumberProgressionIterator(start, endInclusive, step) diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/AbsSendTimeHeaderExtension.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/AbsSendTimeHeaderExtension.kt index 85acd16b94..60de3e3264 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/AbsSendTimeHeaderExtension.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/AbsSendTimeHeaderExtension.kt @@ -34,6 +34,7 @@ import java.time.Instant class AbsSendTimeHeaderExtension { companion object { const val DATA_SIZE_BYTES = 3 + /** * One billion. */ diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/SdesHeaderExtension.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/SdesHeaderExtension.kt index 61114d4d4b..3f0292c013 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/SdesHeaderExtension.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/SdesHeaderExtension.kt @@ -38,15 +38,18 @@ class SdesHeaderExtension { private fun getTextValue(buf: ByteArray, offset: Int, dataLength: Int): String { /* RFC 7941 says the value in RTP is UTF-8. But we use this for MID and RID values - * which are define for SDP in RFC 5888 and RFC 4566 as ASCII only. Thus we don't - * support UTF-8 to keep things simpler. */ + * which are define for SDP in RFC 5888 and RFC 4566 as ASCII only. Thus we don't + * support UTF-8 to keep things simpler. */ return String(buf, offset, dataLength, StandardCharsets.US_ASCII) } private fun setTextValue(buf: ByteArray, offset: Int, sdesValue: String) { System.arraycopy( - sdesValue.toByteArray(StandardCharsets.US_ASCII), 0, buf, - offset, sdesValue.length + sdesValue.toByteArray(StandardCharsets.US_ASCII), + 0, + buf, + offset, + sdesValue.length ) } } diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpByePacketTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpByePacketTest.kt index 487dc5cb76..29e5e4e3b0 100644 --- a/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpByePacketTest.kt +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpByePacketTest.kt @@ -44,7 +44,8 @@ internal class RtcpByePacketTest : ShouldSpec() { private val byeReason = "Connection terminated" private val byeReasonData = byeReason.toByteArray(StandardCharsets.US_ASCII) private val rtcpByeReasonData = byteArrayOf( - byeReasonData.size.toByte(), *byeReasonData.toTypedArray() + byeReasonData.size.toByte(), + *byeReasonData.toTypedArray() ) private val reasonSize = 1 + byeReasonData.size private val padding = byteArrayOf(0x00, 0x00) diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacketTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacketTest.kt index f7c8894e11..191a300f35 100644 --- a/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacketTest.kt +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacketTest.kt @@ -152,7 +152,9 @@ class RtcpFbTccPacketTest : ShouldSpec() { } context("with mixed chunk types and a negative delta") { val rtcpFbTccPacket = RtcpFbTccPacket( - tccMixedChunkTypeData.array(), tccMixedChunkTypeData.arrayOffset(), tccMixedChunkTypeData.limit() + tccMixedChunkTypeData.array(), + tccMixedChunkTypeData.arrayOffset(), + tccMixedChunkTypeData.limit() ) should("parse the values correctly") { rtcpFbTccPacket.forEach { diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpPacketTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpPacketTest.kt index 6f6e347021..2d5f96e00b 100644 --- a/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpPacketTest.kt +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpPacketTest.kt @@ -55,9 +55,15 @@ class RtpPacketTest : ShouldSpec() { private val oneByteHeaderExtensions = byteArrayOf( // BEDE, length=1 - 0xbe, 0xde, 0x00, 0x01, + 0xbe, + 0xde, + 0x00, + 0x01, // ExtId=1,Length=0(1 byte),Data=FF,Padding - 0x10, 0xff, 0x00, 0x00 + 0x10, + 0xff, + 0x00, + 0x00 ) private val oneByteHeaderExtensionsPaddingBetween = byteArrayOf( @@ -87,8 +93,14 @@ class RtpPacketTest : ShouldSpec() { ) private val cryptexHeaderExtensions = byteArrayOf( - 0xc0, 0xde, 0x00, 0x01, - 0xeb, 0x92, 0x36, 0x52 + 0xc0, + 0xde, + 0x00, + 0x01, + 0xeb, + 0x92, + 0x36, + 0x52 ) private val rtpHeaderWithNoExtensions = byteArrayOf( diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/SdesHeaderExtensionTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/SdesHeaderExtensionTest.kt index 0f7710dabd..4803c0941a 100644 --- a/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/SdesHeaderExtensionTest.kt +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/SdesHeaderExtensionTest.kt @@ -44,9 +44,15 @@ class SdesHeaderExtensionTest : ShouldSpec() { private val rtpHeaderSimpleSdesExtension = byteArrayOf( // BEDE, length=1 - 0xbe, 0xde, 0x00, 0x01, + 0xbe, + 0xde, + 0x00, + 0x01, // ExtId=1,Length=0(1 byte),Data='1',Padding - 0x10, 0x31, 0x00, 0x00 + 0x10, + 0x31, + 0x00, + 0x00 ) private val rtpHeaderEmojiSdesExtension = byteArrayOf( @@ -140,7 +146,7 @@ class SdesHeaderExtensionTest : ShouldSpec() { sdesExt.dataLengthBytes shouldBe 4 val payload = SdesHeaderExtension.getTextValue(sdesExt) /* The payload here is UTF-8, but we only parse ASCII. - * Parsing doesn't fail, but the string contains garbage. */ + * Parsing doesn't fail, but the string contains garbage. */ payload shouldNotBe null } } From aa706bc5c4a9cb7b71f16e4db4ddf76fcc48ddba Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Fri, 16 Jun 2023 15:28:44 -0400 Subject: [PATCH 029/189] Update jitsi-xmpp-extensions and jitsi-utils. (#2030) Performance improvements. --- .../java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java | 2 +- pom.xml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java b/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java index 3a6fdc6fea..943b66f5da 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java +++ b/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java @@ -402,7 +402,7 @@ private static List getSourceSsrcs( logger.warn( "Unprocessed source groups: " + sourceGroupsCopy.stream() - .map(e -> e.toXML(XmlEnvironment.EMPTY)) + .map(e -> e.toXML(XmlEnvironment.EMPTY).toString()) .reduce(String::concat)); } diff --git a/pom.xml b/pom.xml index eeeed36a4a..d534eb680b 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ 1.6.21 5.5.5 5.9.1 - 1.0-124-ge57838f + 1.0-126-g02b0c86 1.1-125-g805c0d8 1.16.0 3.2.2 @@ -110,7 +110,7 @@ ${project.groupId} jitsi-xmpp-extensions - 1.0-71-g8c6cdeb + 1.0-72-gc9dde6c From 25bb1ed6053d38833157638a8af31ccc641a3cf5 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 20 Jun 2023 15:19:55 -0400 Subject: [PATCH 030/189] Don't consider visitor endpoints in ConferenceSpeechActivity. (#2031) * Convert AbstractEndpoint to Kotlin. * Add visitor val to AbstractEndpoint. --- .../jitsi/videobridge/AbstractEndpoint.java | 509 ------------------ .../org/jitsi/videobridge/Conference.java | 28 +- .../org/jitsi/videobridge/AbstractEndpoint.kt | 370 +++++++++++++ .../kotlin/org/jitsi/videobridge/Endpoint.kt | 42 +- .../videobridge/relay/RelayedEndpoint.kt | 19 +- 5 files changed, 425 insertions(+), 543 deletions(-) delete mode 100644 jvb/src/main/java/org/jitsi/videobridge/AbstractEndpoint.java create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt diff --git a/jvb/src/main/java/org/jitsi/videobridge/AbstractEndpoint.java b/jvb/src/main/java/org/jitsi/videobridge/AbstractEndpoint.java deleted file mode 100644 index 93013f2dca..0000000000 --- a/jvb/src/main/java/org/jitsi/videobridge/AbstractEndpoint.java +++ /dev/null @@ -1,509 +0,0 @@ -/* - * Copyright @ 2015 - Present, 8x8 Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.jitsi.videobridge; - -import org.jetbrains.annotations.*; -import org.jitsi.nlj.*; -import org.jitsi.nlj.format.*; -import org.jitsi.nlj.rtp.*; -import org.jitsi.nlj.util.*; -import org.jitsi.utils.event.*; -import org.jitsi.utils.logging2.*; -import org.jitsi.videobridge.cc.allocation.*; -import org.json.simple.*; - -import java.time.*; -import java.util.*; - -/** - * Represents an endpoint in a conference (i.e. the entity associated with - * a participant in the conference, which connects the participant's audio - * and video channel). This might be an endpoint connected to this instance of - * jitsi-videobridge, or a relayed endpoint connected to another bridge in the - * same conference. - * - * @author Boris Grozev - * @author Brian Baldino - */ -public abstract class AbstractEndpoint - implements MediaSourceContainer -{ - /** - * The (unique) identifier/ID of the endpoint of a participant in a - * Conference. - */ - private final String id; - - /** - * The {@link Logger} used by the {@link AbstractEndpoint} class to print debug - * information. - */ - protected final Logger logger; - - /** - * A reference to the Conference this Endpoint belongs to. - */ - private final Conference conference; - - /** - * The map of source name -> ReceiverConstraintsMap. - */ - private final Map receiverVideoConstraints = new HashMap<>(); - - /** - * The statistic Id of this Endpoint. - */ - private String statsId; - - /** - * The indicator which determines whether {@link #expire()} has been called - * on this Endpoint. - */ - private boolean expired = false; - - /** - * The maximum set of constraints applied by all receivers of this endpoint - * in the conference. The client needs to send _at least_ this to satisfy - * all receivers. - * - * @deprecated Use maxReceiverVideoConstraintsMap. - */ - @Deprecated - protected VideoConstraints maxReceiverVideoConstraints = new VideoConstraints(0, 0.0); - - /** - * A map of source name -> VideoConstraints. - * - * Stores the maximum set of constraints applied by all receivers for each media source sent by this endpoint. - * The client needs to send _at least_ this for a media source to satisfy all it's receivers. - */ - protected Map maxReceiverVideoConstraintsMap = new HashMap<>(); - - protected final EventEmitter eventEmitter = new SyncEventEmitter<>(); - - protected Map videoTypeCache = new HashMap<>(); - - /** - * Initializes a new {@link AbstractEndpoint} instance. - * @param conference the {@link Conference} which this endpoint is to be a - * part of. - * @param id the ID of the endpoint. - */ - protected AbstractEndpoint(Conference conference, String id, Logger parentLogger) - { - this.conference = Objects.requireNonNull(conference, "conference"); - Map context = new HashMap<>(); - context.put("epId", id); - logger = parentLogger.createChildLogger(this.getClass().getName(), context); - this.id = Objects.requireNonNull(id, "id"); - } - - public boolean hasVideoAvailable() - { - for (MediaSourceDesc source : getMediaSources()) - { - if (source.getVideoType() != VideoType.NONE) - { - return true; - } - } - if (videoTypeCache.values().stream().anyMatch(t -> t != VideoType.NONE)) - { - // Video type cached for a source that hasn't been signaled yet. - return true; - } - return false; - } - - public void setVideoType(@NotNull String sourceName, VideoType videoType) - { - MediaSourceDesc mediaSourceDesc = findMediaSourceDesc(sourceName); - - if (mediaSourceDesc != null) - { - if (mediaSourceDesc.getVideoType() != videoType) - { - mediaSourceDesc.setVideoType(videoType); - conference.getSpeechActivity().endpointVideoAvailabilityChanged(); - } - } - - videoTypeCache.put(sourceName, videoType); - } - - protected void applyVideoTypeCache(MediaSourceDesc[] mediaSourceDescs) - { - // Video types are signaled over JVB data channel while MediaStreamDesc over Colibri. The two channels need - // to be synchronized. - for (MediaSourceDesc mediaSourceDesc : mediaSourceDescs) - { - VideoType videoType = videoTypeCache.get(mediaSourceDesc.getSourceName()); - if (videoType != null) - { - mediaSourceDesc.setVideoType(videoType); - videoTypeCache.remove(mediaSourceDesc.getSourceName()); - } - } - } - - /** - * Checks whether a specific SSRC belongs to this endpoint. - * @param ssrc - * @return - */ - public abstract boolean receivesSsrc(long ssrc); - - /** - * Get the set of SSRCs received from this endpoint. - * @return - */ - public abstract Set getSsrcs(); - - /** - * @return the {@link AbstractEndpointMessageTransport} associated with - * this endpoint. - */ - public AbstractEndpointMessageTransport getMessageTransport() - { - return null; - } - - /** - * Gets the description of the video {@link MediaSourceDesc} that this endpoint has advertised, or {@code null} if - * it hasn't advertised any video sources. - */ - @Nullable - protected MediaSourceDesc getMediaSource() - { - return Arrays.stream(getMediaSources()).findFirst().orElse(null); - } - - protected MediaSourceDesc findMediaSourceDesc(@NotNull String sourceName) - { - for (MediaSourceDesc desc : getMediaSources()) - { - if (sourceName.equals(desc.getSourceName())) - { - return desc; - } - } - - return null; - } - - /** - * Returns the stats Id of this Endpoint. - * - * @return the stats Id of this Endpoint. - */ - public String getStatsId() - { - return statsId; - } - - /** - * Gets the (unique) identifier/ID of this instance. - * - * @return the (unique) identifier/ID of this instance - */ - @NotNull - @Override - public final String getId() - { - return id; - } - - /** - * Gets the Conference to which this Endpoint belongs. - * - * @return the Conference to which this Endpoint belongs. - */ - public Conference getConference() - { - return conference; - } - - void addEventHandler(EventHandler eventHandler) - { - eventEmitter.addHandler(eventHandler); - } - - void removeEventHandler(EventHandler eventHandler) - { - eventEmitter.removeHandler(eventHandler); - } - - /** - * Checks whether or not this Endpoint is considered "expired" - * ({@link #expire()} method has been called). - * - * @return true if this instance is "expired" or false - * otherwise. - */ - public boolean isExpired() - { - return expired; - } - - /** - * Sets the stats Id of this Endpoint. - * - * @param value the stats Id value to set on this Endpoint. - */ - public void setStatsId(String value) - { - this.statsId = value; - if (value != null) - { - logger.addContext("stats_id", value); - } - } - - /** - * {@inheritDoc} - */ - @Override - public String toString() - { - return getClass().getName() + " " + getId(); - } - - /** - * Expires this {@link AbstractEndpoint}. - */ - public void expire() - { - logger.info("Expiring."); - this.expired = true; - - Conference conference = getConference(); - if (conference != null) - { - conference.endpointExpired(this); - } - } - - /** - * Return true if this endpoint should expire (based on whatever logic is - * appropriate for that endpoint implementation. - * - * @return true if this endpoint should expire, false otherwise - */ - public abstract boolean shouldExpire(); - - /** - * Get the last 'incoming activity' (packets received) this endpoint has seen - * @return the timestamp, in milliseconds, of the last activity of this endpoint - */ - public Instant getLastIncomingActivity() - { - return ClockUtils.NEVER; - } - - /** - * Requests a keyframe from this endpoint for the specified media SSRC. - * - * @param mediaSsrc the media SSRC to request a keyframe from. - */ - public abstract void requestKeyframe(long mediaSsrc); - - /** - * Requests a keyframe from this endpoint on the first video SSRC - * it finds. Being able to request a keyframe without passing a specific - * SSRC is useful for things like requesting a pre-emptive keyframes when a new - * active speaker is detected (where it isn't convenient to try and look up - * a particular SSRC). - */ - public abstract void requestKeyframe(); - - /** - * Gets a JSON representation of the parts of this object's state that - * are deemed useful for debugging. - */ - @SuppressWarnings("unchecked") - public JSONObject getDebugState() - { - JSONObject debugState = new JSONObject(); - - JSONObject receiverVideoConstraints = new JSONObject(); - - this.receiverVideoConstraints.forEach( - (sourceName, receiverConstraints) -> - receiverVideoConstraints.put(sourceName, receiverConstraints.getDebugState())); - - debugState.put("receiverVideoConstraints", receiverVideoConstraints); - debugState.put("maxReceiverVideoConstraintsMap", new HashMap<>(maxReceiverVideoConstraintsMap)); - debugState.put("expired", expired); - debugState.put("statsId", statsId); - - return debugState; - } - - /** - * Computes and sets the {@link #maxReceiverVideoConstraints} from the specified video constraints of the media - * source identified by the given source name. - * - * @param sourceName the name of the media source for which the constraints have changed. - * @param newMaxHeight the maximum height resulting from the current set of constraints. - * (Currently we only support constraining the height, and not frame rate.) - */ - private void receiverVideoConstraintsChanged(String sourceName, int newMaxHeight) - { - VideoConstraints oldReceiverMaxVideoConstraints = this.maxReceiverVideoConstraintsMap.get(sourceName); - - VideoConstraints newReceiverMaxVideoConstraints = new VideoConstraints(newMaxHeight, -1.0); - - if (!newReceiverMaxVideoConstraints.equals(oldReceiverMaxVideoConstraints)) - { - this.maxReceiverVideoConstraintsMap.put(sourceName, newReceiverMaxVideoConstraints); - sendVideoConstraintsV2(sourceName, newReceiverMaxVideoConstraints); - } - } - - /** - * Whether the remote endpoint is currently sending (non-silence) audio. - */ - public abstract boolean isSendingAudio(); - - /** - * Whether the remote endpoint is currently sending video. - */ - public abstract boolean isSendingVideo(); - - /** - * Adds a payload type to this endpoint. - */ - public abstract void addPayloadType(PayloadType payloadType); - - /** - * Adds an RTP extension to this endpoint - */ - public abstract void addRtpExtension(RtpExtension rtpExtension); - - /** - * Sets extmap-allow-mixed for this endpoint - */ - public abstract void setExtmapAllowMixed(Boolean allow); - - /** - * Notifies this instance that the max video constraints that the bridge - * needs to receive from this endpoint has changed. Each implementation - * handles this notification differently. - * - * @param maxVideoConstraints the max video constraints that the bridge - * needs to receive from this endpoint - * @deprecated use sendVideoConstraintsV2 - */ - @Deprecated - protected abstract void - sendVideoConstraints(@NotNull VideoConstraints maxVideoConstraints); - - /** - * Notifies this instance that the max video constraints that the bridge needs to receive from a source of this - * endpoint has changed. Each implementation handles this notification differently. - * - * @param sourceName the name of the media source - * @param maxVideoConstraints the max video constraints that the bridge needs to receive from the source - */ - protected abstract void - sendVideoConstraintsV2(@NotNull String sourceName, @NotNull VideoConstraints maxVideoConstraints); - - /** - * Notifies this instance that a specified received wants to receive the specified video constraints from the media - * source with the given source name. - * - * The receiver can be either another endpoint, or a remote bridge. - * - * @param receiverId the id that specifies the receiver endpoint. - * @param sourceName the name of the media source for which the constraints are to be applied. - * @param newVideoConstraints the video constraints that the receiver wishes to receive. - */ - public void addReceiver( - @NotNull String receiverId, - @NotNull String sourceName, - @NotNull VideoConstraints newVideoConstraints - ) - { - ReceiverConstraintsMap sourceConstraints = receiverVideoConstraints.get(sourceName); - - if (sourceConstraints == null) - { - sourceConstraints = new ReceiverConstraintsMap(); - receiverVideoConstraints.put(sourceName, sourceConstraints); - } - - VideoConstraints oldVideoConstraints = sourceConstraints.put(receiverId, newVideoConstraints); - - if (oldVideoConstraints == null || !oldVideoConstraints.equals(newVideoConstraints)) - { - logger.debug( - () -> "Changed receiver constraints: " + receiverId + "->" + sourceName + ": " + - newVideoConstraints.getMaxHeight()); - receiverVideoConstraintsChanged(sourceName, sourceConstraints.getMaxHeight()); - } - } - - /** - * Notifies this instance that the specified receiver no longer wants or - * needs to receive anything from the endpoint attached to this - * instance (the sender). - * - * @param receiverId the id that specifies the receiver endpoint - */ - public void removeReceiver(String receiverId) - { - for (Map.Entry sourceConstraintsEntry - : receiverVideoConstraints.entrySet()) - { - String sourceName = sourceConstraintsEntry.getKey(); - ReceiverConstraintsMap sourceConstraints = sourceConstraintsEntry.getValue(); - - if (sourceConstraints.remove(receiverId) != null) - { - logger.debug(() -> "Removed receiver " + receiverId + " for " + sourceName); - receiverVideoConstraintsChanged(sourceName, sourceConstraints.getMaxHeight()); - } - } - } - - /** - * Notifies this instance that the specified receiver no longer wants or needs to receive anything from the media - * source attached to this instance (the sender). - * - * @param receiverId the id that specifies the receiver endpoint - * @param sourceName the media source name - */ - public void removeSourceReceiver(String receiverId, String sourceName) - { - ReceiverConstraintsMap sourceConstraints = receiverVideoConstraints.get(sourceName); - - if (sourceConstraints != null) - { - if (sourceConstraints.remove(receiverId) != null) - { - logger.debug(() -> "Removed receiver " + receiverId + " for " + sourceName); - receiverVideoConstraintsChanged(sourceName, sourceConstraints.getMaxHeight()); - } - } - } - - public interface EventHandler - { - void iceSucceeded(); - void iceFailed(); - void sourcesChanged(); - } -} diff --git a/jvb/src/main/java/org/jitsi/videobridge/Conference.java b/jvb/src/main/java/org/jitsi/videobridge/Conference.java index 6b5de44800..a0c1f38168 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Conference.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Conference.java @@ -781,11 +781,15 @@ public void sourcesChanged() } /** - * An endpoint was added or removed. + * One or more endpoints was added or removed. + * @param includesNonVisitors Whether any of the endpoints changed was not a visitor. */ - private void endpointsChanged() + private void endpointsChanged(boolean includesNonVisitors) { - speechActivity.endpointsChanged(getEndpoints()); + if (includesNonVisitors) + { + speechActivity.endpointsChanged(getNonVisitorEndpoints()); + } } /** @@ -860,6 +864,14 @@ public List getEndpoints() return new ArrayList<>(this.endpointsById.values()); } + /** + * Gets the Endpoints participating in this conference that are not visitors. + */ + public List getNonVisitorEndpoints() + { + return this.endpointsById.values().stream().filter(ep -> !ep.getVisitor()).collect(Collectors.toList()); + } + List getOrderedEndpoints() { return speechActivity.getOrderedEndpoints(); @@ -981,12 +993,12 @@ public void endpointExpired(AbstractEndpoint endpoint) // The removed endpoint was a local Endpoint as opposed to a RelayedEndpoint. updateEndpointsCache(); endpointsById.forEach((i, senderEndpoint) -> senderEndpoint.removeReceiver(id)); - videobridge.localEndpointExpired(((Endpoint) removedEndpoint).getVisitor()); + videobridge.localEndpointExpired(removedEndpoint.getVisitor()); } relaysById.forEach((i, relay) -> relay.endpointExpired(id)); endpoint.getSsrcs().forEach(ssrc -> endpointsBySsrc.remove(ssrc, endpoint)); - endpointsChanged(); + endpointsChanged(removedEndpoint.getVisitor()); } /** @@ -1024,7 +1036,11 @@ public void addEndpoints(Set endpoints) updateEndpointsCache(); - endpointsChanged(); + boolean hasNonVisitor = endpoints.stream().anyMatch( endpoint -> + !(endpoint instanceof Endpoint) || !((Endpoint)endpoint).getVisitor() + ); + + endpointsChanged(hasNonVisitor); } /** diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt new file mode 100644 index 0000000000..1c329cf9bb --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt @@ -0,0 +1,370 @@ +/* + * Copyright @ 2015 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge + +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.VideoType +import org.jitsi.nlj.format.PayloadType +import org.jitsi.nlj.rtp.RtpExtension +import org.jitsi.nlj.util.NEVER +import org.jitsi.utils.event.EventEmitter +import org.jitsi.utils.event.SyncEventEmitter +import org.jitsi.utils.logging2.Logger +import org.jitsi.videobridge.cc.allocation.MediaSourceContainer +import org.jitsi.videobridge.cc.allocation.ReceiverConstraintsMap +import org.jitsi.videobridge.cc.allocation.VideoConstraints +import org.json.simple.JSONObject +import java.time.Instant +import java.util.* + +/** + * Represents an endpoint in a conference (i.e. the entity associated with + * a participant in the conference, which connects the participant's audio + * and video channel). This might be an endpoint connected to this instance of + * jitsi-videobridge, or a relayed endpoint connected to another bridge in the + * same conference. + * + * @author Boris Grozev + * @author Brian Baldino + */ +abstract class AbstractEndpoint protected constructor( + /** + * The [Conference] which this endpoint is to be part of. + */ + val conference: Conference, + /** + * The ID of the endpoint. + */ + final override val id: String, + parentLogger: Logger +) : MediaSourceContainer { + + /** + * The [Logger] used by the [AbstractEndpoint] class to print debug + * information. + */ + protected val logger: Logger = parentLogger.createChildLogger(this.javaClass.name, mapOf("epId" to id)) + + /** + * The map of source name -> ReceiverConstraintsMap. + */ + private val receiverVideoConstraints: MutableMap = HashMap() + + /** + * The statistics id of this Endpoint. + */ + var statsId: String? = null + set(value) { + field = value + if (value != null) { + logger.addContext("stats_id", value) + } + } + + /** + * The indicator which determines whether [.expire] has been called + * on this Endpoint. + */ + var isExpired = false + private set + + /** + * The maximum set of constraints applied by all receivers of this endpoint + * in the conference. The client needs to send _at least_ this to satisfy + * all receivers. + * + */ + @Deprecated("Use maxReceiverVideoConstraintsMap.") + protected var maxReceiverVideoConstraints = VideoConstraints(0, 0.0) + + /** + * A map of source name -> VideoConstraints. + * + * Stores the maximum set of constraints applied by all receivers for each media source sent by this endpoint. + * The client needs to send _at least_ this for a media source to satisfy all its receivers. + */ + protected var maxReceiverVideoConstraintsMap: MutableMap = HashMap() + + protected val eventEmitter: EventEmitter = SyncEventEmitter() + + protected var videoTypeCache: MutableMap = HashMap() + + fun hasVideoAvailable(): Boolean { + for (source in mediaSources) { + if (source.videoType !== VideoType.NONE) { + return true + } + } + return videoTypeCache.values.any { it !== VideoType.NONE } + } + + fun setVideoType(sourceName: String, videoType: VideoType) { + val mediaSourceDesc = findMediaSourceDesc(sourceName) + if (mediaSourceDesc != null) { + if (mediaSourceDesc.videoType !== videoType) { + mediaSourceDesc.videoType = videoType + conference.speechActivity.endpointVideoAvailabilityChanged() + } + } + videoTypeCache[sourceName] = videoType + } + + protected fun applyVideoTypeCache(mediaSourceDescs: Array) { + // Video types are signaled over JVB data channel while MediaStreamDesc over Colibri. The two channels need + // to be synchronized. + for (mediaSourceDesc in mediaSourceDescs) { + val videoType = videoTypeCache[mediaSourceDesc.sourceName] + if (videoType != null) { + mediaSourceDesc.videoType = videoType + videoTypeCache.remove(mediaSourceDesc.sourceName) + } + } + } + + /** + * Checks whether a specific SSRC belongs to this endpoint. + * @param ssrc + * @return + */ + abstract fun receivesSsrc(ssrc: Long): Boolean + + /** + * The set of SSRCs received from this endpoint. + * @return + */ + abstract val ssrcs: Set + + /** + * The [AbstractEndpointMessageTransport] associated with + * this endpoint. + */ + open val messageTransport: AbstractEndpointMessageTransport? + get() = null + + /** Whether this endpoint represents a visitor. */ + abstract val visitor: Boolean + + /** + * Gets the description of the video [MediaSourceDesc] that this endpoint has advertised, or `null` if + * it hasn't advertised any video sources. + */ + protected val mediaSource: MediaSourceDesc? + get() = mediaSources.firstOrNull() + + fun findMediaSourceDesc(sourceName: String): MediaSourceDesc? = + mediaSources.firstOrNull { + sourceName == it.sourceName + } + + fun addEventHandler(eventHandler: EventHandler) { + eventEmitter.addHandler(eventHandler) + } + + fun removeEventHandler(eventHandler: EventHandler) { + eventEmitter.removeHandler(eventHandler) + } + + /** + * {@inheritDoc} + */ + override fun toString(): String { + return javaClass.name + " " + id + } + + /** + * Expires this [AbstractEndpoint]. + */ + open fun expire() { + logger.info("Expiring.") + isExpired = true + conference.endpointExpired(this) + } + + /** + * Return true if this endpoint should expire (based on whatever logic is + * appropriate for that endpoint implementation. + * + * @return true if this endpoint should expire, false otherwise + */ + abstract fun shouldExpire(): Boolean + + /** + * Get the last 'incoming activity' (packets received) this endpoint has seen + * @return the timestamp, in milliseconds, of the last activity of this endpoint + */ + open val lastIncomingActivity: Instant? + get() = NEVER + + /** + * Requests a keyframe from this endpoint for the specified media SSRC. + * + * @param mediaSsrc the media SSRC to request a keyframe from. + */ + abstract fun requestKeyframe(mediaSsrc: Long) + + /** + * Requests a keyframe from this endpoint on the first video SSRC + * it finds. Being able to request a keyframe without passing a specific + * SSRC is useful for things like requesting a pre-emptive keyframes when a new + * active speaker is detected (where it isn't convenient to try and look up + * a particular SSRC). + */ + abstract fun requestKeyframe() + + /** + * A JSON representation of the parts of this object's state that + * are deemed useful for debugging. + */ + open val debugState: JSONObject + get() { + val debugState = JSONObject() + val receiverVideoConstraints = JSONObject() + this.receiverVideoConstraints.forEach { (sourceName, receiverConstraints) -> + receiverVideoConstraints[sourceName] = receiverConstraints.getDebugState() + } + debugState["receiverVideoConstraints"] = receiverVideoConstraints + debugState["maxReceiverVideoConstraintsMap"] = HashMap(maxReceiverVideoConstraintsMap) + debugState["expired"] = isExpired + debugState["statsId"] = statsId + return debugState + } + + /** + * Computes and sets the [.maxReceiverVideoConstraints] from the specified video constraints of the media + * source identified by the given source name. + * + * @param sourceName the name of the media source for which the constraints have changed. + * @param newMaxHeight the maximum height resulting from the current set of constraints. + * (Currently we only support constraining the height, and not frame rate.) + */ + private fun receiverVideoConstraintsChanged(sourceName: String, newMaxHeight: Int) { + val oldReceiverMaxVideoConstraints = maxReceiverVideoConstraintsMap[sourceName] + val newReceiverMaxVideoConstraints = VideoConstraints(newMaxHeight, -1.0) + if (newReceiverMaxVideoConstraints != oldReceiverMaxVideoConstraints) { + maxReceiverVideoConstraintsMap[sourceName] = newReceiverMaxVideoConstraints + sendVideoConstraintsV2(sourceName, newReceiverMaxVideoConstraints) + } + } + + /** + * Whether the remote endpoint is currently sending (non-silence) audio. + */ + abstract val isSendingAudio: Boolean + + /** + * Whether the remote endpoint is currently sending video. + */ + abstract val isSendingVideo: Boolean + + /** + * Adds a payload type to this endpoint. + */ + abstract fun addPayloadType(payloadType: PayloadType) + + /** + * Adds an RTP extension to this endpoint + */ + abstract fun addRtpExtension(rtpExtension: RtpExtension) + + /** + * Sets extmap-allow-mixed for this endpoint + */ + abstract fun setExtmapAllowMixed(allow: Boolean) + + /** + * Notifies this instance that the max video constraints that the bridge + * needs to receive from this endpoint has changed. Each implementation + * handles this notification differently. + * + * @param maxVideoConstraints the max video constraints that the bridge + * needs to receive from this endpoint + */ + @Deprecated("use sendVideoConstraintsV2") + protected abstract fun sendVideoConstraints(maxVideoConstraints: VideoConstraints) + + /** + * Notifies this instance that the max video constraints that the bridge needs to receive from a source of this + * endpoint has changed. Each implementation handles this notification differently. + * + * @param sourceName the name of the media source + * @param maxVideoConstraints the max video constraints that the bridge needs to receive from the source + */ + protected abstract fun sendVideoConstraintsV2(sourceName: String, maxVideoConstraints: VideoConstraints) + + /** + * Notifies this instance that a specified received wants to receive the specified video constraints from the media + * source with the given source name. + * + * The receiver can be either another endpoint, or a remote bridge. + * + * @param receiverId the id that specifies the receiver endpoint. + * @param sourceName the name of the media source for which the constraints are to be applied. + * @param newVideoConstraints the video constraints that the receiver wishes to receive. + */ + fun addReceiver( + receiverId: String, + sourceName: String, + newVideoConstraints: VideoConstraints + ) { + val sourceConstraints = receiverVideoConstraints.computeIfAbsent(sourceName) { ReceiverConstraintsMap() } + val oldVideoConstraints = sourceConstraints.put(receiverId, newVideoConstraints) + if (oldVideoConstraints == null || oldVideoConstraints != newVideoConstraints) { + logger.debug { + "Changed receiver constraints: $receiverId->$sourceName: ${newVideoConstraints.maxHeight}" + } + receiverVideoConstraintsChanged(sourceName, sourceConstraints.maxHeight) + } + } + + /** + * Notifies this instance that the specified receiver no longer wants or + * needs to receive anything from the endpoint attached to this + * instance (the sender). + * + * @param receiverId the id that specifies the receiver endpoint + */ + fun removeReceiver(receiverId: String) { + for ((sourceName, sourceConstraints) in receiverVideoConstraints) { + if (sourceConstraints.remove(receiverId) != null) { + logger.debug { "Removed receiver $receiverId for $sourceName" } + receiverVideoConstraintsChanged(sourceName, sourceConstraints.maxHeight) + } + } + } + + /** + * Notifies this instance that the specified receiver no longer wants or needs to receive anything from the media + * source attached to this instance (the sender). + * + * @param receiverId the id that specifies the receiver endpoint + * @param sourceName the media source name + */ + fun removeSourceReceiver(receiverId: String, sourceName: String) { + val sourceConstraints = receiverVideoConstraints[sourceName] + if (sourceConstraints != null) { + if (sourceConstraints.remove(receiverId) != null) { + logger.debug { "Removed receiver $receiverId for $sourceName" } + receiverVideoConstraintsChanged(sourceName, sourceConstraints.maxHeight) + } + } + } + + interface EventHandler { + fun iceSucceeded() + fun iceFailed() + fun sourcesChanged() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index f3226e50e9..8d7fd21d79 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -112,7 +112,7 @@ class Endpoint @JvmOverloads constructor( /** * Whether this endpoint is in "visitor" mode, i.e. should be invisible to other endpoints. */ - val visitor: Boolean, + override val visitor: Boolean, private val clock: Clock = Clock.systemUTC() ) : AbstractEndpoint(conference, id, parentLogger), PotentialPacketHandler, @@ -232,15 +232,13 @@ class Endpoint @JvmOverloads constructor( * The instance which manages the Colibri messaging (over a data channel * or web sockets). */ - private val _messageTransport = EndpointMessageTransport( + override val messageTransport = EndpointMessageTransport( this, Supplier { conference.videobridge.statistics }, conference, logger ) - override fun getMessageTransport(): EndpointMessageTransport = _messageTransport - /** * Gets the endpoints in the conference in LastN order, with this {@link Endpoint} removed. */ @@ -481,14 +479,15 @@ class Endpoint @JvmOverloads constructor( } } - override fun isSendingAudio(): Boolean { - // The endpoint is sending audio if we (the transceiver) are receiving audio. - return transceiver.isReceivingAudio() - } - override fun isSendingVideo(): Boolean { - // The endpoint is sending video if we (the transceiver) are receiving video. - return transceiver.isReceivingVideo() - } + override val isSendingAudio: Boolean + get() = + // The endpoint is sending audio if we (the transceiver) are receiving audio. + transceiver.isReceivingAudio() + + override val isSendingVideo: Boolean + get() = + // The endpoint is sending video if we (the transceiver) are receiving video. + transceiver.isReceivingVideo() private fun doSendSrtp(packetInfo: PacketInfo): Boolean { packetInfo.addEvent(SRTP_QUEUE_EXIT_EVENT) @@ -629,7 +628,7 @@ class Endpoint @JvmOverloads constructor( // This handles if the remote side will be opening the data channel dataChannelStack!!.onDataChannelStackEvents { dataChannel -> logger.info("Remote side opened a data channel.") - _messageTransport.setDataChannel(dataChannel) + messageTransport.setDataChannel(dataChannel) } dataChannelHandler.setDataChannelStack(dataChannelStack!!) if (openDataChannelLocally) { @@ -642,7 +641,7 @@ class Endpoint @JvmOverloads constructor( 0, "default" ) - _messageTransport.setDataChannel(dataChannel) + messageTransport.setDataChannel(dataChannel) dataChannel.open() } else { logger.info("Will wait for the remote side to open the data channel.") @@ -838,9 +837,11 @@ class Endpoint @JvmOverloads constructor( fun unmapRtcpFbSsrc(packet: RtcpFbPacket) = videoSsrcs.unmapRtcpFbSsrc(packet) - override fun getSsrcs() = HashSet(transceiver.receiveSsrcs) + override val ssrcs + get() = HashSet(transceiver.receiveSsrcs) - override fun getLastIncomingActivity(): Instant = transceiver.packetIOActivity.lastIncomingActivityInstant + override val lastIncomingActivity + get() = transceiver.packetIOActivity.lastIncomingActivityInstant override fun requestKeyframe() = transceiver.requestKeyFrame() @@ -1101,8 +1102,8 @@ class Endpoint @JvmOverloads constructor( } } - override fun getDebugState(): JSONObject { - return super.getDebugState().apply { + override val debugState: JSONObject + get() = super.debugState.apply { put("bitrateController", bitrateController.debugState) put("bandwidthProbing", bandwidthProbing.getDebugState()) put("iceTransport", iceTransport.getDebugState()) @@ -1117,10 +1118,9 @@ class Endpoint @JvmOverloads constructor( put("videoSsrcs", videoSsrcs.getDebugState()) } } - } override fun expire() { - if (super.isExpired()) { + if (super.isExpired) { return } super.expire() @@ -1137,7 +1137,7 @@ class Endpoint @JvmOverloads constructor( logger.info("Spent ${bitrateController.getTotalOversendingTime().seconds} seconds oversending") transceiver.teardown() - _messageTransport.close() + messageTransport.close() sctpHandler.stop() sctpManager?.closeConnection() } catch (t: Throwable) { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt index ab0379ef0d..017a90ccb9 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt @@ -118,7 +118,11 @@ class RelayedEndpoint( return streamInformationStore.receiveSsrcs.contains(ssrc) } - override fun getSsrcs() = HashSet(streamInformationStore.receiveSsrcs) + override val ssrcs + get() = HashSet(streamInformationStore.receiveSsrcs) + + // Visitors are never advertised between relays, so relayed endpoints are never visitors. + override val visitor = false fun hasReceiveSsrcs(): Boolean = streamInformationStore.receiveSsrcs.isNotEmpty() @@ -129,8 +133,10 @@ class RelayedEndpoint( override fun requestKeyframe() = relay.transceiver.requestKeyFrame(mediaSource?.primarySSRC) - override fun isSendingAudio(): Boolean = rtpReceiver.isReceivingAudio() - override fun isSendingVideo(): Boolean = rtpReceiver.isReceivingVideo() + override val isSendingAudio + get() = rtpReceiver.isReceivingAudio() + override val isSendingVideo + get() = rtpReceiver.isReceivingVideo() override fun addPayloadType(payloadType: PayloadType) = streamInformationStore.addRtpPayloadType(payloadType) override fun addRtpExtension(rtpExtension: RtpExtension) = @@ -210,12 +216,11 @@ class RelayedEndpoint( } } - override fun getDebugState(): JSONObject { - return super.getDebugState().apply { + override val debugState: JSONObject + get() = super.debugState.apply { val block = getNodeStats() put(block.name, block.toJson()) } - } private fun updateStatsOnExpire() { val relayStats = relay.statistics @@ -231,7 +236,7 @@ class RelayedEndpoint( } override fun expire() { - if (super.isExpired()) { + if (super.isExpired) { return } super.expire() From 64f0751b39eb3f721beab5c875f3e5cce2340653 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Fri, 30 Jun 2023 17:21:14 -0400 Subject: [PATCH 031/189] Include packet handler stats in RtpSenderImpl stats. (#2032) --- .../src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt index b3f21a718b..b568116e5f 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt @@ -299,6 +299,7 @@ class RtpSenderImpl( } override fun getNodeStats(): NodeStatsBlock = NodeStatsBlock("RTP sender $id").apply { + addBlock(super.getNodeStats()) addBlock(nackHandler.getNodeStats()) addBlock(probingDataSender.getNodeStats()) addJson("packetQueue", incomingPacketQueue.debugState) From 03a0a80db92eef60b39eb6521a77d9d8760124c7 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Mon, 3 Jul 2023 10:45:51 -0400 Subject: [PATCH 032/189] Improve naming of MediaSources stats. (#2033) --- .../src/main/kotlin/org/jitsi/nlj/MediaSources.kt | 5 ++--- .../src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt | 1 + .../src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSources.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSources.kt index 39cf6beffd..e6e8cade9e 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSources.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSources.kt @@ -61,10 +61,9 @@ class MediaSources : NodeStatsProducer { fun getMediaSources(): Array = sources override fun getNodeStats(): NodeStatsBlock = NodeStatsBlock("MediaStreamSources").apply { - sources.forEachIndexed { i, source -> - val sourceBlock = NodeStatsBlock("source_$i") + sources.forEach { source -> + val sourceBlock = NodeStatsBlock(source.sourceName) sourceBlock.addString("owner", source.owner) - sourceBlock.addString("name", source.sourceName) sourceBlock.addString("video_type", source.videoType.toString()) source.rtpEncodings.forEach { sourceBlock.addBlock(it.getNodeStats()) } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt index 2b9873f445..1533589e85 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt @@ -155,6 +155,7 @@ constructor( fun getNodeStats() = NodeStatsBlock(primarySSRC.toString()).apply { addNumber("rtx_ssrc", getSecondarySsrc(SsrcAssociationType.RTX)) addNumber("fec_ssrc", getSecondarySsrc(SsrcAssociationType.FEC)) + addNumber("eid", eid) for (layer in layers) { addBlock(layer.getNodeStats()) } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt index 29d4586cbc..4d52c322e9 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt @@ -217,7 +217,7 @@ constructor( /** * Extracts a [NodeStatsBlock] from an [RtpLayerDesc]. */ - fun getNodeStats() = NodeStatsBlock(layerId.toString()).apply { + fun getNodeStats() = NodeStatsBlock(indexString(index)).apply { addNumber("frameRate", frameRate) addNumber("height", height) addNumber("index", index) From 165c6dbd16980507f94e6e38b01148a019049262 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Mon, 3 Jul 2023 16:49:10 -0400 Subject: [PATCH 033/189] Improve trace logs for BandwidthAllocation. (#2034) --- .../org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt | 2 +- .../org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt index 7fe9e1836c..c029bdcfbf 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt @@ -93,5 +93,5 @@ data class SingleAllocation( fun isForwarded(): Boolean = targetIndex > -1 override fun toString(): String = "[id=$endpointId target=${targetLayer?.height}/${targetLayer?.frameRate} " + - "ideal=${idealLayer?.height}/${idealLayer?.frameRate}" + "ideal=${idealLayer?.height}/${idealLayer?.frameRate}]" } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt index 2e575feeda..4d4eb0341e 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt @@ -202,7 +202,8 @@ internal class BandwidthAllocator( logger.trace { "Finished allocation: allocationChanged=$allocationChanged, " + - "effectiveConstraintsChanged=$effectiveConstraintsChanged" + "effectiveConstraintsChanged=$effectiveConstraintsChanged, " + + "allocation=[$allocation]" } if (effectiveConstraintsChanged) { eventEmitter.fireEvent { From 8a01e052eb525aac01d5221e431bc19f8c17f87e Mon Sep 17 00:00:00 2001 From: luocq3 Date: Fri, 7 Jul 2023 17:11:54 +0800 Subject: [PATCH 034/189] use correct JVB_CONFIG_FILE var --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fedc5aaed7..abf11eba62 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ JVB_CONFIG_DIR_LOCATION="~/" JVB_CONFIG_DIR_NAME=".jvb" JVB_CONFIG_FILE="$JVB_CONFIG_DIR_LOCATION/$JVB_JVB_CONFIG_DIR_NAME/jvb.conf" -mvn compile exec:exec -Dexec.executable=java -Dexec.args="-cp %classpath org.jitsi.videobridge.MainKt -Djava.library.path=$JVB_HOME/lib/native/linux-64 -Djava.util.logging.config.file=$JVB_HOME/lib/logging.properties -Dnet.java.sip.communicator.SC_HOME_DIR_LOCATION=$JVB_CONFIG_DIR_LOCATION -Dnet.java.sip.communicator.SC_HOME_DIR_NAME=$JVB_CONFIG_DIR_NAME -Dconfig.file=$JVB_CONFIG_.FILE" +mvn compile exec:exec -Dexec.executable=java -Dexec.args="-cp %classpath org.jitsi.videobridge.MainKt -Djava.library.path=$JVB_HOME/lib/native/linux-64 -Djava.util.logging.config.file=$JVB_HOME/lib/logging.properties -Dnet.java.sip.communicator.SC_HOME_DIR_LOCATION=$JVB_CONFIG_DIR_LOCATION -Dnet.java.sip.communicator.SC_HOME_DIR_NAME=$JVB_CONFIG_DIR_NAME -Dconfig.file=$JVB_CONFIG_FILE" ``` # Configuration From 44a053343d051b64894abf760777f6e4ad0135de Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 11 Jul 2023 15:16:56 -0400 Subject: [PATCH 035/189] Bump Bouncycastle to version 1.75. (#2036) --- jitsi-media-transform/pom.xml | 8 ++++---- pom.xml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jitsi-media-transform/pom.xml b/jitsi-media-transform/pom.xml index 3b6499c91b..0f58c9ea44 100644 --- a/jitsi-media-transform/pom.xml +++ b/jitsi-media-transform/pom.xml @@ -22,7 +22,7 @@ ${project.groupId} jitsi-srtp - 1.1-12-ga64adcc + 1.1-13-g1d0db60 ${project.groupId} @@ -55,17 +55,17 @@ org.bouncycastle - bctls-jdk15on + bctls-jdk18on ${bouncycastle.version} org.bouncycastle - bcprov-jdk15on + bcprov-jdk18on ${bouncycastle.version} org.bouncycastle - bcpkix-jdk15on + bcpkix-jdk18on ${bouncycastle.version} diff --git a/pom.xml b/pom.xml index d534eb680b..b48673715f 100644 --- a/pom.xml +++ b/pom.xml @@ -34,7 +34,7 @@ 4.6.0 3.0.10 2.12.4 - 1.70 + 1.75 0.16.0 UTF-8 From 99534de45eec06eeb94c9d1eeee23da865d7d00a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Aug 2023 08:12:49 -0700 Subject: [PATCH 036/189] chore(deps): Bump guava from 31.0.1-jre to 32.0.0-jre in /jvb (#2028) Bumps [guava](https://github.com/google/guava) from 31.0.1-jre to 32.0.0-jre. - [Release notes](https://github.com/google/guava/releases) - [Commits](https://github.com/google/guava/commits) --- updated-dependencies: - dependency-name: com.google.guava:guava dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- jvb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/pom.xml b/jvb/pom.xml index 2dbd3f4659..bd813e24fa 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -61,7 +61,7 @@ com.google.guava guava - 31.0.1-jre + 32.0.0-jre org.eclipse.jetty From 5fd429e826fddbeafc95a9e22dd751f1b0bf8222 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 3 Aug 2023 08:14:11 -0700 Subject: [PATCH 037/189] Remove maxReceiverVideoConstraints (unused). (#2038) --- .../org/jitsi/videobridge/AbstractEndpoint.kt | 34 +++++-------------- .../kotlin/org/jitsi/videobridge/Endpoint.kt | 4 +-- .../videobridge/relay/RelayedEndpoint.kt | 2 +- 3 files changed, 11 insertions(+), 29 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt index 1c329cf9bb..531dc1acb0 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt @@ -28,7 +28,6 @@ import org.jitsi.videobridge.cc.allocation.ReceiverConstraintsMap import org.jitsi.videobridge.cc.allocation.VideoConstraints import org.json.simple.JSONObject import java.time.Instant -import java.util.* /** * Represents an endpoint in a conference (i.e. the entity associated with @@ -61,7 +60,7 @@ abstract class AbstractEndpoint protected constructor( /** * The map of source name -> ReceiverConstraintsMap. */ - private val receiverVideoConstraints: MutableMap = HashMap() + private val receiverVideoConstraints = mutableMapOf() /** * The statistics id of this Endpoint. @@ -81,26 +80,17 @@ abstract class AbstractEndpoint protected constructor( var isExpired = false private set - /** - * The maximum set of constraints applied by all receivers of this endpoint - * in the conference. The client needs to send _at least_ this to satisfy - * all receivers. - * - */ - @Deprecated("Use maxReceiverVideoConstraintsMap.") - protected var maxReceiverVideoConstraints = VideoConstraints(0, 0.0) - /** * A map of source name -> VideoConstraints. * * Stores the maximum set of constraints applied by all receivers for each media source sent by this endpoint. * The client needs to send _at least_ this for a media source to satisfy all its receivers. */ - protected var maxReceiverVideoConstraintsMap: MutableMap = HashMap() + protected var maxReceiverVideoConstraints = mutableMapOf() protected val eventEmitter: EventEmitter = SyncEventEmitter() - protected var videoTypeCache: MutableMap = HashMap() + private val videoTypeCache = mutableMapOf() fun hasVideoAvailable(): Boolean { for (source in mediaSources) { @@ -177,12 +167,7 @@ abstract class AbstractEndpoint protected constructor( eventEmitter.removeHandler(eventHandler) } - /** - * {@inheritDoc} - */ - override fun toString(): String { - return javaClass.name + " " + id - } + override fun toString() = "${javaClass.name} $id" /** * Expires this [AbstractEndpoint]. @@ -224,10 +209,7 @@ abstract class AbstractEndpoint protected constructor( */ abstract fun requestKeyframe() - /** - * A JSON representation of the parts of this object's state that - * are deemed useful for debugging. - */ + /** A JSON representation of the parts of this object's state that are deemed useful for debugging. */ open val debugState: JSONObject get() { val debugState = JSONObject() @@ -236,7 +218,7 @@ abstract class AbstractEndpoint protected constructor( receiverVideoConstraints[sourceName] = receiverConstraints.getDebugState() } debugState["receiverVideoConstraints"] = receiverVideoConstraints - debugState["maxReceiverVideoConstraintsMap"] = HashMap(maxReceiverVideoConstraintsMap) + debugState["maxReceiverVideoConstraints"] = HashMap(maxReceiverVideoConstraints) debugState["expired"] = isExpired debugState["statsId"] = statsId return debugState @@ -251,10 +233,10 @@ abstract class AbstractEndpoint protected constructor( * (Currently we only support constraining the height, and not frame rate.) */ private fun receiverVideoConstraintsChanged(sourceName: String, newMaxHeight: Int) { - val oldReceiverMaxVideoConstraints = maxReceiverVideoConstraintsMap[sourceName] + val oldReceiverMaxVideoConstraints = maxReceiverVideoConstraints[sourceName] val newReceiverMaxVideoConstraints = VideoConstraints(newMaxHeight, -1.0) if (newReceiverMaxVideoConstraints != oldReceiverMaxVideoConstraints) { - maxReceiverVideoConstraintsMap[sourceName] = newReceiverMaxVideoConstraints + maxReceiverVideoConstraints[sourceName] = newReceiverMaxVideoConstraints sendVideoConstraintsV2(sourceName, newReceiverMaxVideoConstraints) } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index 8d7fd21d79..e47acfadbc 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -525,7 +525,7 @@ class Endpoint @JvmOverloads constructor( } private fun sendAllVideoConstraints() { - maxReceiverVideoConstraintsMap.forEach { (sourceName, constraints) -> + maxReceiverVideoConstraints.forEach { (sourceName, constraints) -> sendVideoConstraintsV2(sourceName, constraints) } } @@ -591,7 +591,7 @@ class Endpoint @JvmOverloads constructor( logger.cdebug { "Sender constraints changed: ${senderSourceConstraintsMessage.toJson()}" } sendMessage(senderSourceConstraintsMessage) } else { - maxReceiverVideoConstraintsMap[sourceName]?.let { + maxReceiverVideoConstraints[sourceName]?.let { sendVideoConstraints(it) } ?: logger.error("No max receiver constraints mapping found for: $sourceName") diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt index 017a90ccb9..3255e4a78e 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt @@ -167,7 +167,7 @@ class RelayedEndpoint( } fun relayMessageTransportConnected() { - maxReceiverVideoConstraintsMap.forEach { (sourceName, constraints) -> + maxReceiverVideoConstraints.forEach { (sourceName, constraints) -> sendVideoConstraintsV2(sourceName, constraints) } } From 2edebdd6e9b72f45b1505dc5df6a44aa8e605ae3 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 3 Aug 2023 09:35:50 -0700 Subject: [PATCH 038/189] ref: Simplify (avoid unnecessary cast). (#2037) --- jvb/src/main/java/org/jitsi/videobridge/Conference.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/Conference.java b/jvb/src/main/java/org/jitsi/videobridge/Conference.java index a0c1f38168..58f1481d9d 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Conference.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Conference.java @@ -1036,9 +1036,7 @@ public void addEndpoints(Set endpoints) updateEndpointsCache(); - boolean hasNonVisitor = endpoints.stream().anyMatch( endpoint -> - !(endpoint instanceof Endpoint) || !((Endpoint)endpoint).getVisitor() - ); + boolean hasNonVisitor = endpoints.stream().anyMatch(endpoint -> !endpoint.getVisitor()); endpointsChanged(hasNonVisitor); } From 1a8621977a7e7166b0a2ed717251d2d7d4b3e5fc Mon Sep 17 00:00:00 2001 From: bgrozev Date: Tue, 8 Aug 2023 10:19:17 -0500 Subject: [PATCH 039/189] ref: Cleanup BridgeChannelMessage (remove unnecessary type field). (#2039) --- .../videobridge/EndpointMessageTransport.java | 4 +- .../message/BridgeChannelMessage.kt | 53 ++++++++++--------- .../relay/RelayMessageTransport.kt | 2 +- .../message/BridgeChannelMessageTest.kt | 2 +- 4 files changed, 31 insertions(+), 30 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java b/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java index 6bcf3624f0..629985a222 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java +++ b/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java @@ -166,9 +166,9 @@ public BridgeChannelMessage sourceVideoType(SourceVideoTypeMessage sourceVideoTy } @Override - public void unhandledMessage(BridgeChannelMessage message) + public void unhandledMessage(@NotNull BridgeChannelMessage message) { - getLogger().warn("Received a message with an unexpected type: " + message.getType()); + getLogger().warn("Received a message with an unexpected type: " + message.getClass().getSimpleName()); } /** diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt index 045abfe1b7..7ea41014e9 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt @@ -41,7 +41,7 @@ import java.util.concurrent.atomic.AtomicLong * The messages are formatted in JSON with a required "colibriClass" field, which indicates the message type. Different * message types have different (if any) additional fields. */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "colibriClass") +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = BridgeChannelMessage.TYPE_PROPERTY_NAME) @JsonSubTypes( JsonSubTypes.Type(value = ClientHelloMessage::class, name = ClientHelloMessage.TYPE), JsonSubTypes.Type(value = ServerHelloMessage::class, name = ServerHelloMessage.TYPE), @@ -62,11 +62,7 @@ import java.util.concurrent.atomic.AtomicLong JsonSubTypes.Type(value = SourceVideoTypeMessage::class, name = SourceVideoTypeMessage.TYPE), JsonSubTypes.Type(value = VideoTypeMessage::class, name = VideoTypeMessage.TYPE) ) -sealed class BridgeChannelMessage( - // The type is included as colibriClass (as it has to be on the wire) by the annotation above. - @JsonIgnore - val type: String -) { +sealed class BridgeChannelMessage { private val jsonCacheDelegate = ResettableLazy { createJson() } /** @@ -97,6 +93,7 @@ sealed class BridgeChannelMessage( fun parse(string: String): BridgeChannelMessage { return mapper.readValue(string) } + const val TYPE_PROPERTY_NAME = "colibriClass" } } @@ -162,7 +159,7 @@ open class MessageHandler { /** * A message sent from a client to a bridge in the beginning of a session. */ -class ClientHelloMessage : BridgeChannelMessage(TYPE) { +class ClientHelloMessage : BridgeChannelMessage() { companion object { const val TYPE = "ClientHello" } @@ -174,7 +171,7 @@ class ClientHelloMessage : BridgeChannelMessage(TYPE) { class ServerHelloMessage @JvmOverloads constructor( @JsonInclude(JsonInclude.Include.NON_NULL) val version: String? = null -) : BridgeChannelMessage(TYPE) { +) : BridgeChannelMessage() { override fun createJson(): String = if (version == null) JSON_STRING_NO_VERSION else """{"colibriClass":"$TYPE","version":"$version"}""" @@ -194,7 +191,7 @@ class ServerHelloMessage @JvmOverloads constructor( * bridge, sent to a client or sent to a bridge. */ @JsonIgnoreProperties(ignoreUnknown = true) -class EndpointMessage(val to: String) : BridgeChannelMessage(TYPE) { +class EndpointMessage(val to: String) : BridgeChannelMessage() { @JsonInclude(JsonInclude.Include.NON_NULL) var from: String? = null set(value) { @@ -213,7 +210,9 @@ class EndpointMessage(val to: String) : BridgeChannelMessage(TYPE) { @JsonAnySetter fun put(key: String, value: Any?) { - otherFields[key] = value + if (key != TYPE_PROPERTY_NAME) { + otherFields[key] = value + } } companion object { @@ -231,7 +230,7 @@ class EndpointMessage(val to: String) : BridgeChannelMessage(TYPE) { * bridge, sent to a client or sent to a bridge. */ @JsonIgnoreProperties(ignoreUnknown = true) -class EndpointStats : BridgeChannelMessage(TYPE) { +class EndpointStats : BridgeChannelMessage() { @JsonInclude(JsonInclude.Include.NON_NULL) var from: String? = null set(value) { @@ -244,7 +243,9 @@ class EndpointStats : BridgeChannelMessage(TYPE) { @JsonAnySetter fun put(key: String, value: Any?) { - otherFields[key] = value + if (key != TYPE_PROPERTY_NAME) { + otherFields[key] = value + } } companion object { @@ -256,7 +257,7 @@ class EndpointStats : BridgeChannelMessage(TYPE) { * A message sent from a client, indicating that it wishes to change its "lastN" (i.e. the maximum number of video * streams to be received). */ -class LastNMessage(val lastN: Int) : BridgeChannelMessage(TYPE) { +class LastNMessage(val lastN: Int) : BridgeChannelMessage() { companion object { const val TYPE = "LastNChangedEvent" } @@ -270,7 +271,7 @@ class DominantSpeakerMessage @JvmOverloads constructor( val dominantSpeakerEndpoint: String, val previousSpeakers: List? = null, val silence: Boolean = false -) : BridgeChannelMessage(TYPE) { +) : BridgeChannelMessage() { /** * Construct a message from a list of speakers with the dominant speaker on top. The list must have at least one * element. @@ -292,7 +293,7 @@ class DominantSpeakerMessage @JvmOverloads constructor( class EndpointConnectionStatusMessage( val endpoint: String, activeBoolean: Boolean -) : BridgeChannelMessage(TYPE) { +) : BridgeChannelMessage() { // For whatever reason we encode the boolean in JSON as a string. val active: String = activeBoolean.toString() @@ -318,7 +319,7 @@ class ForwardedEndpointsMessage( */ @get:JsonProperty("lastNEndpoints") val forwardedEndpoints: Collection -) : BridgeChannelMessage(TYPE) { +) : BridgeChannelMessage() { companion object { const val TYPE = "LastNEndpointsChangeEvent" } @@ -332,7 +333,7 @@ class ForwardedSourcesMessage( * The set of media sources for which the bridge is currently sending video. */ val forwardedSources: Collection -) : BridgeChannelMessage(TYPE) { +) : BridgeChannelMessage() { companion object { const val TYPE = "ForwardedSources" } @@ -360,7 +361,7 @@ data class VideoSourceMapping( class VideoSourcesMap( /* The current list of maps of sources to ssrcs. */ val mappedSources: Collection -) : BridgeChannelMessage(TYPE) { +) : BridgeChannelMessage() { companion object { const val TYPE = "VideoSourcesMap" } @@ -384,7 +385,7 @@ data class AudioSourceMapping( class AudioSourcesMap( /* The current list of maps of sources to ssrcs. */ val mappedSources: Collection -) : BridgeChannelMessage(TYPE) { +) : BridgeChannelMessage() { companion object { const val TYPE = "AudioSourcesMap" } @@ -397,7 +398,7 @@ class AudioSourcesMap( * TODO: update https://github.com/jitsi/jitsi-videobridge/blob/master/doc/constraints.md before removing. */ @Deprecated("", ReplaceWith("SenderSourceConstraints"), DeprecationLevel.WARNING) -class SenderVideoConstraintsMessage(val videoConstraints: VideoConstraints) : BridgeChannelMessage(TYPE) { +class SenderVideoConstraintsMessage(val videoConstraints: VideoConstraints) : BridgeChannelMessage() { constructor(maxHeight: Int) : this(VideoConstraints(maxHeight)) /** @@ -422,7 +423,7 @@ class SenderVideoConstraintsMessage(val videoConstraints: VideoConstraints) : Br class SenderSourceConstraintsMessage( val sourceName: String, val maxHeight: Int -) : BridgeChannelMessage(TYPE) { +) : BridgeChannelMessage() { /** * Serialize manually because it's faster than Jackson. @@ -444,7 +445,7 @@ class AddReceiverMessage( val endpointId: String?, // Used in single stream per endpoint mode and wil be removed val sourceName: String?, // Used in the multi-stream mode val videoConstraints: VideoConstraints -) : BridgeChannelMessage(TYPE) { +) : BridgeChannelMessage() { /** * Serialize manually because it's faster than Jackson. */ @@ -466,7 +467,7 @@ class AddReceiverMessage( class RemoveReceiverMessage( val bridgeId: String, val endpointId: String -) : BridgeChannelMessage(TYPE) { +) : BridgeChannelMessage() { /** * Serialize manually because it's faster than Jackson. */ @@ -489,7 +490,7 @@ class ReceiverVideoConstraintsMessage( val defaultConstraints: VideoConstraints? = null, val constraints: Map? = null, val assumedBandwidthBps: Long? = null -) : BridgeChannelMessage(TYPE) { +) : BridgeChannelMessage() { companion object { const val TYPE = "ReceiverVideoConstraints" } @@ -506,7 +507,7 @@ class SourceVideoTypeMessage( * message was received on (non-null values are needed only for Relays). */ endpointId: String? = null -) : BridgeChannelMessage(TYPE) { +) : BridgeChannelMessage() { @JsonInclude(JsonInclude.Include.NON_NULL) var endpointId: String? = endpointId set(value) { @@ -536,7 +537,7 @@ class VideoTypeMessage( * message was received on (non-null values are needed only for Relays). */ endpointId: String? = null -) : BridgeChannelMessage(TYPE) { +) : BridgeChannelMessage() { @JsonInclude(JsonInclude.Include.NON_NULL) var endpointId: String? = endpointId set(value) { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt index 1a5214ce71..edb0a978ca 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt @@ -204,7 +204,7 @@ class RelayMessageTransport( } override fun unhandledMessage(message: BridgeChannelMessage) { - logger.warn("Received a message with an unexpected type: " + message.type) + logger.warn("Received a message with an unexpected type: ${message.javaClass.simpleName}") } /** diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/message/BridgeChannelMessageTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/message/BridgeChannelMessageTest.kt index 3758d14b2a..10cee7ffeb 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/message/BridgeChannelMessageTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/message/BridgeChannelMessageTest.kt @@ -46,7 +46,7 @@ class BridgeChannelMessageTest : ShouldSpec() { parsed.shouldBeInstanceOf() val parsedColibriClass = parsed["colibriClass"] parsedColibriClass.shouldBeInstanceOf() - parsedColibriClass shouldBe message.type + parsedColibriClass shouldBe ClientHelloMessage.TYPE } } context("parsing an invalid message") { From 8bb3315d845e25afeb1da974e266e0d364bba25a Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Mon, 21 Aug 2023 15:52:35 -0400 Subject: [PATCH 040/189] Expose some internal BWE statistics in GoogleCcEstimator stats. (#2041) --- .../BandwidthUsage.java | 2 +- .../OveruseDetector.java | 5 +++ .../RemoteBitrateEstimatorAbsSendTime.java | 32 +++++++++++++++++++ .../bandwidthestimation/GoogleCcEstimator.kt | 7 +++- 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/BandwidthUsage.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/BandwidthUsage.java index 32ad8aa4c1..16a5cf1118 100644 --- a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/BandwidthUsage.java +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/BandwidthUsage.java @@ -20,7 +20,7 @@ * * @author Lyubomir Marinov */ -enum BandwidthUsage +public enum BandwidthUsage { kBwNormal(0), kBwUnderusing(-1), kBwOverusing(1); diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/OveruseDetector.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/OveruseDetector.java index 223e0120d2..150f7ed6ce 100644 --- a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/OveruseDetector.java +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/OveruseDetector.java @@ -195,4 +195,9 @@ private void updateThreshold(double modifiedOffset, long nowMs) lastUpdateMs = nowMs; } + + public double getThreshold() + { + return threshold; + } } diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RemoteBitrateEstimatorAbsSendTime.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RemoteBitrateEstimatorAbsSendTime.java index 906988a60d..071e9649a9 100644 --- a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RemoteBitrateEstimatorAbsSendTime.java +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RemoteBitrateEstimatorAbsSendTime.java @@ -389,6 +389,22 @@ public synchronized void setMinBitrate(int minBitrateBps) remoteRate.setMinBitrate(minBitrateBps); } + /** + * Get various statistics about the estimation process. [Local addition, not in original C++.] + */ + public synchronized @Nullable Statistics getStatistics() + { + if (detector == null) + { + return null; + } + return new Statistics( + detector.estimator.getOffset(), + detector.detector.getThreshold(), + detector.detector.getState() + ); + } + /** * Holds the {@link InterArrival}, {@link OveruseEstimator} and * {@link OveruseDetector} instances that estimate the remote bitrate of a @@ -439,4 +455,20 @@ public static long convertMsTo24Bits(long timeMs) { return (((timeMs << kAbsSendTimeFraction) + 500) / 1000) & 0x00FFFFFF; } + + /** + * Various statistics about the estimation process. [Local addition, not in original C++.] + */ + public static class Statistics { + public double offset; + public double threshold; + public BandwidthUsage hypothesis; + + public Statistics(double o, double t, BandwidthUsage h) + { + offset = o; + threshold = t; + hypothesis = h; + } + } } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/GoogleCcEstimator.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/GoogleCcEstimator.kt index 82af123036..c5ccc64f86 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/GoogleCcEstimator.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/GoogleCcEstimator.kt @@ -111,7 +111,12 @@ class GoogleCcEstimator(diagnosticContext: DiagnosticContext, parentLogger: Logg getCurrentBw(now) ).apply { addNumber("incomingEstimateExpirations", bitrateEstimatorAbsSendTime.incomingEstimateExpirations) - addNumber("latestDelayEstimate", sendSideBandwidthEstimation.latestREMB) + bitrateEstimatorAbsSendTime.statistics?.run { + addNumber("delayBasedEstimatorOffset", offset) + addNumber("delayBasedEstimatorThreshold", threshold) + addNumber("delayBasedEstimatorHypothesis", hypothesis.value) + } + addNumber("latestDelayBasedEstimate", sendSideBandwidthEstimation.latestREMB) addNumber("latestLossFraction", sendSideBandwidthEstimation.latestFractionLoss / 256.0) with(sendSideBandwidthEstimation.statistics) { update(now.toEpochMilli()) From 194c8f7b11c97ebe3445f5f03780595b9e395696 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Wed, 23 Aug 2023 12:12:46 -0500 Subject: [PATCH 041/189] log: Log "meeting_id" instead of "meetingId" for easier matching. (#2043) --- jvb/src/main/java/org/jitsi/videobridge/Videobridge.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java b/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java index 68b3078eec..4160002ad9 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java @@ -280,7 +280,7 @@ void localEndpointExpired(boolean visitor) { final Conference conference = doCreateConference(name, meetingId, isRtcStatsEnabled); - logger.info(() -> "create_conf, id=" + conference.getID() + " meetingId=" + meetingId); + logger.info(() -> "create_conf, id=" + conference.getID() + " meeting_id=" + meetingId); return conference; } @@ -485,7 +485,7 @@ private void handleColibriRequest(XmppConnection.ColibriRequest request) { if (conference == null) { - logger.warn("Conference with meetingId=" + meetingId + " not found."); + logger.warn("Conference with meeting_id=" + meetingId + " not found."); throw new ConferenceNotFoundException(); } return conference; From a8d8a786c2a055bbdc5434847ea381aa5cdeccb5 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 24 Aug 2023 09:50:41 -0500 Subject: [PATCH 042/189] Increase consent check and inactivity intervals. (#2044) * Bump consent check and inactivity intervals. * Add missing whitespace in reference.conf. * chore: Update ice4j. --- jvb/src/main/resources/application.conf | 4 +- jvb/src/main/resources/reference.conf | 57 ++++++++++++------------- pom.xml | 2 +- 3 files changed, 31 insertions(+), 32 deletions(-) diff --git a/jvb/src/main/resources/application.conf b/jvb/src/main/resources/application.conf index 956ac8019c..9afb3c9bf8 100644 --- a/jvb/src/main/resources/application.conf +++ b/jvb/src/main/resources/application.conf @@ -1,8 +1,8 @@ # This file contains overrides for libraries JVB uses ice4j { consent-freshness { - // Sends "consent freshness" check every 3 seconds - interval = 3 seconds + // Sends "consent freshness" check every 5 seconds. + interval = 5 seconds // Retry max 5 times which will take up to 2500ms, that is before the next "consent freshness" transaction starts. max-retransmissions = 5 } diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index ffaaa0238b..d4a33aab03 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -1,54 +1,54 @@ videobridge { entity-expiration { # If an entity has no activity after this timeout, it is expired - timeout=1 minute + timeout = 1 minute # The interval at which the videobridge will check for expired entities - check-interval=${videobridge.entity-expiration.timeout} + check-interval = ${videobridge.entity-expiration.timeout} } health { # The interval between health checks - interval=1 minute + interval = 1 minute # The timeout for a health check. This needs to be higher than [interval], otherwise health checks timeout because # none were scheduled. - timeout=90 seconds + timeout = 90 seconds # If performing a health check takes longer than this, it is considered unsuccessful. - max-check-duration=3 seconds + max-check-duration = 3 seconds # Whether or not health check failures should be 'sticky' # (i.e. once the bridge becomes unhealthy, it will never # go back to a healthy state) - sticky-failures=false + sticky-failures = false } ep-connection-status { # How long we'll wait for an endpoint to *start* sending # data before we consider it 'inactive' - first-transfer-timeout=15 seconds + first-transfer-timeout = 15 seconds # How long an endpoint can be 'inactive' before it will # be considered disconnected - max-inactivity-limit=3 seconds + max-inactivity-limit = 8 seconds # How often we check endpoint's connectivity status - check-interval=500 milliseconds + check-interval = 500 milliseconds } cc { - bwe-change-threshold=0.15 - thumbnail-max-height-px=180 - onstage-ideal-height-px=1080 - onstage-preferred-height-px=360 - onstage-preferred-framerate=30 + bwe-change-threshold = 0.15 + thumbnail-max-height-px = 180 + onstage-ideal-height-px = 1080 + onstage-preferred-height-px = 360 + onstage-preferred-framerate = 30 // Whether the bridge is allowed to oversend (send the lowest layer regardless of BWE) for on-stage endpoints. If // allowed, it's only used when an endpoint is screensharing. - allow-oversend-onstage=true + allow-oversend-onstage = true // The maximum bitrate by which the estimation will be exceeded when oversending (if oversending is allowed). - max-oversend-bitrate=500 kbps - trust-bwe=true + max-oversend-bitrate = 500 kbps + trust-bwe = true # How often we check to send probing data - padding-period=15ms + padding-period = 15ms # How often we'll run bandwidth re-allocation. This is in addition to re-allocating when otherwise required (e.g. # a new endpoint joins or the available layers change). @@ -143,16 +143,16 @@ videobridge { } relay { # Whether or not relays (octo) are enabled - enabled=false + enabled = false # A string denoting the 'region' of this JVB. This region will be used by Jicofo in the selection of a bridge for # a client by comparing it to the client's region. # Must be set when 'enabled' is true. - #region="us-west-1" + #region = "us-west-1" # The unique identifier of the jitsi-videobridge instance as a relay. # Must be set when 'enabled' is true. - #relay-id="jvb-1234" + #relay-id = "jvb-1234" } load-management { # Whether or not the reducer will be enabled to take actions to mitigate load @@ -172,8 +172,7 @@ videobridge { reduction-scale = .75 # The factor by which we'll increase the current last-n when trying to recover recover-scale = 1.25 - # The minimum time in between runs of the last-n reducer to reduce or recover from - # load + # The minimum time in between runs of the last-n reducer to reduce or recover from load impact-time = 1 minute # The lowest value we'll set for last-n minimum-last-n-value = 1 @@ -200,22 +199,22 @@ videobridge { } sctp { // Whether SCTP data channels are enabled. - enabled=true + enabled = true } stats { // The interval at which stats are gathered. interval = 5 seconds } websockets { - enabled=false - server-id="default-id" + enabled = false + server-id = "default-id" // Whether to negotiate WebSocket compression (permessage-deflate) enable-compression = true // Optional, even when 'enabled' is set to true - #tls=true + #tls = true // The domains used when advertising a colibri-ws URL. Must be set when enabled = true - domains= [] + domains = [] // The domain used when advertising a colibri-relay-ws URL. If empty defaults to the value of `domains`. relay-domains = [] } @@ -272,7 +271,7 @@ videobridge { # 100pps for low-definition, last-n 20 and 2 participants talking, so # 2*50pps for audio, this queue is fed 300+19*100+2*50 = 2300pps, so its # size in terms of millis is 1024/2300*1000 ~= 445ms. - queue-size=1024 + queue-size = 1024 } } diff --git a/pom.xml b/pom.xml index b48673715f..ab93ef259a 100644 --- a/pom.xml +++ b/pom.xml @@ -105,7 +105,7 @@ ${project.groupId} ice4j - 3.0-58-gf41542d + 3.0-63-g9fe2d69 ${project.groupId} From 461d80588b187fe8360f98663d4053ddb715bf4a Mon Sep 17 00:00:00 2001 From: bgrozev Date: Wed, 13 Sep 2023 13:27:36 -0500 Subject: [PATCH 043/189] Enable private addresses for ICE based on a per-endpoint flag. (#2047) * Enable private addresses for ICE based on a per-endpoint flag. --- .../main/java/org/jitsi/videobridge/Conference.java | 6 ++++-- .../main/kotlin/org/jitsi/videobridge/Endpoint.kt | 3 ++- .../colibri2/Colibri2ConferenceHandler.kt | 4 +++- .../kotlin/org/jitsi/videobridge/ice/IceConfig.kt | 2 +- .../kotlin/org/jitsi/videobridge/relay/Relay.kt | 11 ++++++++++- .../jitsi/videobridge/transport/ice/IceTransport.kt | 13 ++++++++++--- jvb/src/main/resources/reference.conf | 4 +++- .../kotlin/org/jitsi/videobridge/ConferenceTest.kt | 3 ++- pom.xml | 2 +- 9 files changed, 36 insertions(+), 12 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/Conference.java b/jvb/src/main/java/org/jitsi/videobridge/Conference.java index 58f1481d9d..7b2bc5d0aa 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Conference.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Conference.java @@ -722,7 +722,8 @@ public Endpoint createLocalEndpoint( boolean iceControlling, boolean sourceNames, boolean doSsrcRewriting, - boolean visitor) + boolean visitor, + boolean privateAddresses) { final AbstractEndpoint existingEndpoint = getEndpoint(id); if (existingEndpoint != null) @@ -730,7 +731,8 @@ public Endpoint createLocalEndpoint( throw new IllegalArgumentException("Local endpoint with ID = " + id + "already created"); } - final Endpoint endpoint = new Endpoint(id, this, logger, iceControlling, sourceNames, doSsrcRewriting, visitor); + final Endpoint endpoint = new Endpoint( + id, this, logger, iceControlling, sourceNames, doSsrcRewriting, visitor, privateAddresses); videobridge.localEndpointCreated(visitor); subscribeToEndpointEvents(endpoint); diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index e47acfadbc..f04ead42b7 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -113,6 +113,7 @@ class Endpoint @JvmOverloads constructor( * Whether this endpoint is in "visitor" mode, i.e. should be invisible to other endpoints. */ override val visitor: Boolean, + supportsPrivateAddresses: Boolean, private val clock: Clock = Clock.systemUTC() ) : AbstractEndpoint(conference, id, parentLogger), PotentialPacketHandler, @@ -127,7 +128,7 @@ class Endpoint @JvmOverloads constructor( private val dataChannelHandler = DataChannelHandler() /* TODO: do we ever want to support useUniquePort for an Endpoint? */ - private val iceTransport = IceTransport(id, iceControlling, false, logger) + private val iceTransport = IceTransport(id, iceControlling, false, supportsPrivateAddresses, logger) private val dtlsTransport = DtlsTransport(logger).also { it.cryptex = CryptexConfig.endpoint } private var cryptex: Boolean = CryptexConfig.endpoint diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt index f517ba36b1..5a4860d3f2 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt @@ -175,12 +175,14 @@ class Colibri2ConferenceHandler( ) val sourceNames = c2endpoint.hasCapability(Capability.CAP_SOURCE_NAME_SUPPORT) val ssrcRewriting = sourceNames && c2endpoint.hasCapability(Capability.CAP_SSRC_REWRITING_SUPPORT) + val privateAddresses = c2endpoint.hasCapability(Capability.CAP_PRIVATE_ADDRESS_CONNECTIVITY) conference.createLocalEndpoint( c2endpoint.id, transport.iceControlling, sourceNames, ssrcRewriting, - c2endpoint.mucRole == MUCRole.visitor + c2endpoint.mucRole == MUCRole.visitor, + privateAddresses ).apply { c2endpoint.statsId?.let { statsId = it diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/ice/IceConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/ice/IceConfig.kt index d072a81a26..6090b444b1 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/ice/IceConfig.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/ice/IceConfig.kt @@ -106,7 +106,7 @@ class IceConfig private constructor() { /** * Whether to advertise ICE candidates with private IP addresses (RFC1918 IPv4 addresses and - * fec0::/10 or fc00::/7 IPv6 addresses). + * fec0::/10 or fc00::/7 IPv6 addresses) even to endpoints that have not signaled support for private addresses. */ val advertisePrivateCandidates: Boolean by config( "videobridge.ice.advertise-private-candidates".from(JitsiConfig.newConfig) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt index 5d5d2d59c7..8018bec71f 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt @@ -167,7 +167,16 @@ class Relay @JvmOverloads constructor( private val sctpHandler = SctpHandler() private val dataChannelHandler = DataChannelHandler() - private val iceTransport = IceTransport(id, iceControlling, useUniquePort, logger, clock) + private val iceTransport = IceTransport( + id = id, + controlling = iceControlling, + useUniquePort = useUniquePort, + // There's no good reason to disable private addresses. + advertisePrivateAddresses = true, + parentLogger = logger, + clock = clock + ) + private val dtlsTransport = DtlsTransport(logger).also { it.cryptex = CryptexConfig.relay } private var cryptex = CryptexConfig.relay diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt index 326e11aa32..a1fca51284 100755 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt @@ -62,6 +62,10 @@ class IceTransport @JvmOverloads constructor( * unique local ports, rather than the configured port. */ useUniquePort: Boolean, + /** + * Use private addresses for this [IceTransport] even if [IceConfig.advertisePrivateCandidates] is false. + */ + private val advertisePrivateAddresses: Boolean, parentLogger: Logger, private val clock: Clock = Clock.systemUTC() ) { @@ -276,7 +280,7 @@ class IceTransport @JvmOverloads constructor( password = iceAgent.localPassword ufrag = iceAgent.localUfrag iceComponent.localCandidates?.forEach { cand -> - cand.toCandidatePacketExtension()?.let { pe.addChildExtension(it) } + cand.toCandidatePacketExtension(advertisePrivateAddresses)?.let { pe.addChildExtension(it) } } addChildExtension(IceRtcpmuxPacketExtension()) } @@ -512,8 +516,11 @@ private fun generateCandidateId(candidate: LocalCandidate): String = buildString append(java.lang.Long.toHexString(candidate.hashCode().toLong())) } -private fun LocalCandidate.toCandidatePacketExtension(): CandidatePacketExtension? { - if (!IceConfig.config.advertisePrivateCandidates && transportAddress.isPrivateAddress()) { +private fun LocalCandidate.toCandidatePacketExtension(advertisePrivateAddresses: Boolean): CandidatePacketExtension? { + if (transportAddress.isPrivateAddress() && + !advertisePrivateAddresses && + !IceConfig.config.advertisePrivateCandidates + ) { return null } val cpe = IceCandidatePacketExtension() diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index d4a33aab03..3e93c45ff7 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -255,7 +255,9 @@ videobridge { # "NominateFirstValid", "NominateHighestPriority", "NominateFirstHostOrReflexiveValid", or "NominateBestRTT". nomination-strategy = "NominateFirstHostOrReflexiveValid" - # Whether to advertise private ICE candidates, i.e. RFC 1918 IPv4 addresses and fec0::/10 and fc00::/7 IPv6 addresses. + # Whether to advertise private ICE candidates, i.e. RFC 1918 IPv4 addresses and fec0::/10 and fc00::/7 IPv6 + # addresses even for endpoints that have not signaled support for private addresses. + # Note: Jicofo signals support for private addresses for jigasi and jibri. advertise-private-candidates = true } diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/ConferenceTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/ConferenceTest.kt index 7c8a272961..95aaebcb24 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/ConferenceTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/ConferenceTest.kt @@ -37,7 +37,8 @@ class ConferenceTest : ConfigTest() { context("Adding local endpoints should work") { with(Conference(videobridge, "id", name, null, false)) { endpointCount shouldBe 0 - createLocalEndpoint("abcdabcd", true, false, false, false) // TODO cover the case when they're true + // TODO cover the case when they're true + createLocalEndpoint("abcdabcd", true, false, false, false, false) endpointCount shouldBe 1 debugState.shouldBeValidJson() } diff --git a/pom.xml b/pom.xml index ab93ef259a..e84f5dc2f6 100644 --- a/pom.xml +++ b/pom.xml @@ -110,7 +110,7 @@ ${project.groupId} jitsi-xmpp-extensions - 1.0-72-gc9dde6c + 1.0-74-gc88e006 From e69ee348b98b2a78dbc73441a5c25560e8f52495 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Wed, 13 Sep 2023 16:45:10 -0500 Subject: [PATCH 044/189] chore: Bump jitsi-xmpp-extensions. (#2048) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e84f5dc2f6..5507949e6a 100644 --- a/pom.xml +++ b/pom.xml @@ -110,7 +110,7 @@ ${project.groupId} jitsi-xmpp-extensions - 1.0-74-gc88e006 + 1.0-75-g4664207 From 8983b11f979e49a069c5a5bb81b7805875ca7cce Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 14 Sep 2023 09:21:55 -0500 Subject: [PATCH 045/189] Revert "Bump Bouncycastle to version 1.75. (#2036)" (#2049) The update to 1.75 was found to cause a failure for some endpoints to finish DTLS, leaving them unable to join a conference. We are investigating and planning to update once the problem is fixed. The CVEs that affect 1.70 but not 1.75 do not affect jitsi-videobridge. This reverts commit 44a053343d051b64894abf760777f6e4ad0135de. --- jitsi-media-transform/pom.xml | 8 ++++---- pom.xml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jitsi-media-transform/pom.xml b/jitsi-media-transform/pom.xml index 0f58c9ea44..3b6499c91b 100644 --- a/jitsi-media-transform/pom.xml +++ b/jitsi-media-transform/pom.xml @@ -22,7 +22,7 @@ ${project.groupId} jitsi-srtp - 1.1-13-g1d0db60 + 1.1-12-ga64adcc ${project.groupId} @@ -55,17 +55,17 @@ org.bouncycastle - bctls-jdk18on + bctls-jdk15on ${bouncycastle.version} org.bouncycastle - bcprov-jdk18on + bcprov-jdk15on ${bouncycastle.version} org.bouncycastle - bcpkix-jdk18on + bcpkix-jdk15on ${bouncycastle.version} diff --git a/pom.xml b/pom.xml index 5507949e6a..b9fa179c35 100644 --- a/pom.xml +++ b/pom.xml @@ -34,7 +34,7 @@ 4.6.0 3.0.10 2.12.4 - 1.75 + 1.70 0.16.0 UTF-8 From 37da7e13f2afea81128e5bf6da17dd431764a6a0 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Mon, 25 Sep 2023 09:34:59 -0500 Subject: [PATCH 046/189] fix: Fix a leak of WebSocketClients. (#2051) Every time we establish an active WS to another relay we create a new WebSocketClient instance. This comes with its own QueuedThreadPool with 8 threads and if we lose the reference without calling stop() it leaks. When the existing web socket closing arrives before the signaling that expires the relay, we try do WebSocketClient.stop() and then doConnect(). The call to stop() can take a while, and if in the meantime the relay is expired in signaling we end up running doConnect() on an expired relay. In this case we create a new WebSocketClient, which is never stopped. This fix uses a single WebSocketClient instance shared between all relays, and does not attempt to re-connect if the reason for the disconnect is RELAY_CLOSED. It re-uses the "webSocket: ColibriWebSocket" field for sockets created both actively and passively. --- .../relay/RelayMessageTransport.kt | 57 ++++++++----------- 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt index edb0a978ca..699dc17bd4 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt @@ -36,7 +36,6 @@ import org.jitsi.videobridge.message.EndpointStats import org.jitsi.videobridge.message.ServerHelloMessage import org.jitsi.videobridge.message.SourceVideoTypeMessage import org.jitsi.videobridge.message.VideoTypeMessage -import org.jitsi.videobridge.util.TaskPools import org.jitsi.videobridge.websocket.ColibriWebSocket import org.json.simple.JSONObject import java.lang.ref.WeakReference @@ -67,11 +66,6 @@ class RelayMessageTransport( */ private var url: String? = null - /** - * An active websocket client. - */ - private var outgoingWebsocket: WebSocketClient? = null - /** * Use to synchronize access to [webSocket] */ @@ -113,15 +107,11 @@ class RelayMessageTransport( webSocket = null } - // this.webSocket should only be initialized when it has connected (via [webSocketConnected]). - val newWebSocket = ColibriWebSocket(relay.id, this) - outgoingWebsocket?.let { - logger.warn("Re-connecting while outgoingWebsocket != null, possible leak.") - it.stop() - } - outgoingWebsocket = WebSocketClient().also { - it.start() - it.connect(newWebSocket, URI(url), ClientUpgradeRequest()) + ColibriWebSocket(relay.id, this).also { + webSocketClient.connect(it, URI(url), ClientUpgradeRequest()) + synchronized(webSocketSyncRoot) { + webSocket = it + } } } @@ -311,7 +301,7 @@ class RelayMessageTransport( get() = getActiveTransportChannel() != null val isActive: Boolean - get() = outgoingWebsocket != null + get() = url != null /** * {@inheritDoc} @@ -319,7 +309,7 @@ class RelayMessageTransport( override fun webSocketConnected(ws: ColibriWebSocket) { synchronized(webSocketSyncRoot) { // If we already have a web-socket, discard it and use the new one. - if (ws != webSocket) { + if (ws != webSocket && webSocket != null) { logger.info("Replacing an existing websocket.") webSocket?.session?.close(CloseStatus.NORMAL, "replaced") webSocketLastActive = true @@ -355,10 +345,10 @@ class RelayMessageTransport( logger.debug { "Web socket closed, statusCode $statusCode ( $reason)." } } } - outgoingWebsocket?.let { - // Try to reconnect. TODO: how to handle failures? - it.stop() - outgoingWebsocket = null + + // This check avoids trying to establish a new WS when the closing of the existing WS races the signaling to + // expire the relay. 1001 with RELAY_CLOSED means that the remote side willingly closed the socket. + if (statusCode != 1001 || reason != RELAY_CLOSED) { doConnect() } } @@ -374,22 +364,11 @@ class RelayMessageTransport( if (webSocket != null) { // 410 Gone indicates that the resource requested is no longer // available and will not be available again. - webSocket?.session?.close(CloseStatus.SHUTDOWN, "relay closed") + webSocket?.session?.close(CloseStatus.SHUTDOWN, RELAY_CLOSED) webSocket = null logger.debug { "Relay expired, closed colibri web-socket." } } } - outgoingWebsocket?.let { - // Stopping might block and we don't want to hold the thread processing signaling. - TaskPools.IO_POOL.submit { - try { - it.stop() - } catch (e: Exception) { - logger.warn("Error while stopping outgoing web socket", e) - } - } - } - outgoingWebsocket = null } /** @@ -509,4 +488,16 @@ class RelayMessageTransport( conference.sendMessageFromRelay(message, true, relay.meshId) return null } + + companion object { + /** + * The single [WebSocketClient] instance that all [Relay]s use to initiate a web socket connection. + */ + val webSocketClient = WebSocketClient().apply { start() } + + /** + * Reason to use when closing a WS due to the relay being expired. + */ + const val RELAY_CLOSED = "relay_closed" + } } From 48d00e97d762510b8032208c93120f9e6e590d4d Mon Sep 17 00:00:00 2001 From: bgrozev Date: Mon, 25 Sep 2023 16:23:31 -0500 Subject: [PATCH 047/189] chore: Update jicoco (default to not send jetty version). (#2053) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b9fa179c35..5a8d078d51 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 5.5.5 5.9.1 1.0-126-g02b0c86 - 1.1-125-g805c0d8 + 1.1-126-g1f776be 1.16.0 3.2.2 4.6.0 From f5f3991e993e62ee29cbc31c1056bb5b14d61aca Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 26 Sep 2023 11:40:11 -0400 Subject: [PATCH 048/189] ktlint fixes for 0.50.0. (#2050) (ktlint 1.0.0 will require a lot more; I haven't analyzed that yet.) --- .../src/main/kotlin/org/jitsi/nlj/PacketInfo.kt | 5 ++++- .../nlj/transform/node/incoming/TccGeneratorNodeTest.kt | 5 ++++- .../test/kotlin/org/jitsi/rtp/rtcp/RtcpReportBlockTest.kt | 8 ++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketInfo.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketInfo.kt index b43e31fc13..0adcd480ac 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketInfo.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketInfo.kt @@ -198,7 +198,10 @@ open class PacketInfo @JvmOverloads constructor( fun sent() { var actions: List<() -> Unit> = Collections.emptyList() synchronized(this) { - onSentActions?.let { actions = it; onSentActions = null } ?: run { return@sent } + onSentActions?.let { + actions = it + onSentActions = null + } ?: run { return@sent } } for (action in actions) { action.invoke() diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNodeTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNodeTest.kt index c1a559dd39..eeecd7ef1b 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNodeTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNodeTest.kt @@ -36,7 +36,10 @@ class TccGeneratorNodeTest : ShouldSpec() { private val clock: FakeClock = FakeClock() private val tccPackets = mutableListOf() - private val onTccReady = { tccPacket: RtcpPacket -> tccPackets.add(tccPacket); Unit } + private val onTccReady = { tccPacket: RtcpPacket -> + tccPackets.add(tccPacket) + Unit + } private val streamInformationStore = StreamInformationStoreImpl() private val tccExtensionId = 5 diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpReportBlockTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpReportBlockTest.kt index b4e5d4d511..6e91a4669d 100644 --- a/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpReportBlockTest.kt +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpReportBlockTest.kt @@ -37,8 +37,12 @@ class RtcpReportBlockTest : ShouldSpec() { val reportBlockData = with(ByteBuffer.allocate(24)) { putInt(expectedSsrc.toInt()) - put(expectedFractionLost.toByte()); put3Bytes(expectedCumulativeLost) - putShort(expectedSeqNumCycles.toShort()); putShort(expectedSeqNum.toShort()) + put(expectedFractionLost.toByte()) + put3Bytes(expectedCumulativeLost) + + putShort(expectedSeqNumCycles.toShort()) + putShort(expectedSeqNum.toShort()) + putInt(expectedInterarrivalJitter.toInt()) putInt(expectedLastSrTimestamp.toInt()) putInt(expectedDelaySinceLastSr.toInt()) From c706fa9c2ebf1ca1272728b8244e1f3b78939351 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Wed, 27 Sep 2023 11:14:48 -0500 Subject: [PATCH 049/189] ref: Fix some warnings. (#2054) --- .../java/org/jitsi/videobridge/Conference.java | 16 ++++------------ .../kotlin/org/jitsi/videobridge/Endpoint.kt | 9 ++++----- .../kotlin/org/jitsi/videobridge/SsrcCache.kt | 8 ++++---- 3 files changed, 12 insertions(+), 21 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/Conference.java b/jvb/src/main/java/org/jitsi/videobridge/Conference.java index 7b2bc5d0aa..3eb24e83a6 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Conference.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Conference.java @@ -84,7 +84,7 @@ public class Conference /** * A map of the endpoints in this conference, by their ssrcs. */ - private ConcurrentHashMap endpointsBySsrc = new ConcurrentHashMap<>(); + private final ConcurrentHashMap endpointsBySsrc = new ConcurrentHashMap<>(); /** * The relays participating in this conference. @@ -643,11 +643,9 @@ private void updateStatisticsOnExpire() if (logger.isInfoEnabled()) { - StringBuilder sb = new StringBuilder("expire_conf,"); - sb.append("duration=").append(durationSeconds) - .append(",has_failed=").append(hasFailed) - .append(",has_partially_failed=").append(hasPartiallyFailed); - logger.info(sb.toString()); + logger.info("expire_conf,duration=" + durationSeconds + + ",has_failed=" + hasFailed + + ",has_partially_failed=" + hasPartiallyFailed); } } @@ -1083,11 +1081,6 @@ public void addEndpointSsrc(@NotNull AbstractEndpoint endpoint, long ssrc) } } - public void removeEndpointSsrc(@NotNull AbstractEndpoint endpoint, long ssrc) - { - endpointsBySsrc.remove(ssrc, endpoint); - } - /** * Gets the conference name. * @@ -1199,7 +1192,6 @@ public boolean hasRelays() /** * Handles an RTP/RTCP packet coming from a specific endpoint. - * @param packetInfo */ public void handleIncomingPacket(PacketInfo packetInfo) { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index f04ead42b7..f8eaf607f4 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -93,7 +93,6 @@ import java.time.Instant import java.util.Optional import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicLong -import java.util.function.Supplier /** * Models a local endpoint (participant) in a [Conference] @@ -220,7 +219,7 @@ class Endpoint @JvmOverloads constructor( override fun keyframeNeeded(endpointId: String?, ssrc: Long) = conference.requestKeyframe(endpointId, ssrc) }, - Supplier { getOrderedEndpoints() }, + { getOrderedEndpoints() }, diagnosticContext, logger, isUsingSourceNames, @@ -235,7 +234,7 @@ class Endpoint @JvmOverloads constructor( */ override val messageTransport = EndpointMessageTransport( this, - Supplier { conference.videobridge.statistics }, + { conference.videobridge.statistics }, conference, logger ) @@ -288,7 +287,7 @@ class Endpoint @JvmOverloads constructor( return transceiver.sendProbing(mediaSsrcs, numBytes) } }, - Supplier { bitrateController.getStatusSnapshot() } + { bitrateController.getStatusSnapshot() } ).apply { diagnosticsContext = this@Endpoint.diagnosticContext enabled = true @@ -559,7 +558,7 @@ class Endpoint @JvmOverloads constructor( if (doSsrcRewriting) { val newActiveSources = newEffectiveConstraints.entries.filter { !it.value.isDisabled() }.map { it.key }.toList() - val newActiveSourceNames = newActiveSources.mapNotNull { it.sourceName }.toSet() + val newActiveSourceNames = newActiveSources.map { it.sourceName }.toSet() /* safe unlocked access of activeSources. BitrateController will not overlap calls to this method. */ if (activeSources != newActiveSourceNames) { activeSources = newActiveSourceNames diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt index 15e59d85af..297693bc0f 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt @@ -448,7 +448,7 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: synchronized(sendSources) { /* Don't activate a source on RTCP. */ - var rs = receivedSsrcs.get(packet.senderSsrc) ?: return false + val rs = receivedSsrcs.get(packet.senderSsrc) ?: return false val ss = getSendSource(rs.props.ssrc1, rs.props, allowCreate = false, remappings) ?: return false ss.rewriteRtcp(packet) logger.debug { @@ -537,10 +537,10 @@ class AudioSsrcCache(size: Int, ep: SsrcRewriter, parentLogger: Logger) : */ override fun findSourceProps(ssrc: Long): SourceDesc? { val p = ep.findAudioSourceProps(ssrc) - if (p == null || p.sourceName == null || p.owner == null) { - return null + return if (p?.sourceName == null || p.owner == null) { + null } else { - return SourceDesc(p) + SourceDesc(p) } } From ea562782eec24dadaa61b796d471f5d9d0fbe47d Mon Sep 17 00:00:00 2001 From: bgrozev Date: Wed, 27 Sep 2023 11:27:59 -0500 Subject: [PATCH 050/189] Fix routing of audio and stats when routeLoudestOnly is disabled (#2055) * fix: Don't drop audio when routeLoudestOnly is disabled in config. * fix: Do not drop stats when routeLoudestOnly is disabled. --- .../main/java/org/jitsi/videobridge/Conference.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/Conference.java b/jvb/src/main/java/org/jitsi/videobridge/Conference.java index 3eb24e83a6..95ca0a1444 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Conference.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Conference.java @@ -162,6 +162,11 @@ public long getLocalVideoSsrc() @NotNull private final EncodingsManager encodingsManager = new EncodingsManager(); + /** + * Cache here because it's accessed on every packet. + */ + private final boolean routeLoudestOnly = LoudestConfig.getRouteLoudestOnly(); + /** * The task of updating the ordered list of endpoints in the conference. It runs periodically in order to adapt to * endpoints stopping or starting to their video streams (which affects the order). @@ -1126,7 +1131,9 @@ public boolean isRankedSpeaker(AbstractEndpoint ep) { if (!LoudestConfig.Companion.getRouteLoudestOnly()) { - return false; + // When "route loudest only" is disabled all speakers should be considered "ranked" (we forward all audio + // and stats). + return true; } return speechActivity.isAmongLoudest(ep.getId()); } @@ -1275,7 +1282,7 @@ else if (pph.wants(packetInfo)) public boolean levelChanged(@NotNull AbstractEndpoint endpoint, long level) { SpeakerRanking ranking = speechActivity.levelChanged(endpoint, level); - if (ranking == null) + if (ranking == null || !routeLoudestOnly) return false; if (ranking.isDominant && LoudestConfig.Companion.getAlwaysRouteDominant()) return false; From 13803c84ee8512b363903211426b681b65035fbc Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Wed, 27 Sep 2023 17:22:40 -0400 Subject: [PATCH 051/189] Update Kotlin to 1.9.10, and related Maven packages. (#2056) * Update Kotlin to 1.9.10, and related Maven packages. * Update jitsi-xmpp-extensions version. * Update jicoco version. --- jitsi-media-transform/pom.xml | 5 +++-- jvb/pom.xml | 4 ++-- .../kotlin/org/jitsi/videobridge/Endpoint.kt | 1 + .../jitsi/videobridge/relay/RelayedEndpoint.kt | 1 + .../jitsi/videobridge/KotestProjectConfig.kt | 2 +- pom.xml | 17 +++++++++-------- rtp/spotbugs-exclude.xml | 3 +++ 7 files changed, 20 insertions(+), 13 deletions(-) diff --git a/jitsi-media-transform/pom.xml b/jitsi-media-transform/pom.xml index 3b6499c91b..1e54e4d958 100644 --- a/jitsi-media-transform/pom.xml +++ b/jitsi-media-transform/pom.xml @@ -125,8 +125,9 @@ io.mockk - mockk - 1.12.4 + mockk-jvm + ${mockk.version} + test javax.xml.bind diff --git a/jvb/pom.xml b/jvb/pom.xml index bd813e24fa..0f0cbb501b 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -209,8 +209,8 @@ io.mockk - mockk - 1.12.4 + mockk-jvm + ${mockk.version} test diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index f8eaf607f4..9718a5f71a 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -567,6 +567,7 @@ class Endpoint @JvmOverloads constructor( } } + @Deprecated("use sendVideoConstraintsV2") override fun sendVideoConstraints(maxVideoConstraints: VideoConstraints) { // Note that it's up to the client to respect these constraints. if (mediaSources.isEmpty()) { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt index 3255e4a78e..64abaa4e6e 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt @@ -144,6 +144,7 @@ class RelayedEndpoint( override fun setExtmapAllowMixed(allow: Boolean) = streamInformationStore.setExtmapAllowMixed(allow) + @Deprecated("use sendVideoConstraintsV2") override fun sendVideoConstraints(maxVideoConstraints: VideoConstraints) { relay.sendMessage( AddReceiverMessage( diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/KotestProjectConfig.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/KotestProjectConfig.kt index 24818e67dd..1ad89dcc0f 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/KotestProjectConfig.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/KotestProjectConfig.kt @@ -21,7 +21,7 @@ import org.jitsi.metaconfig.MetaconfigSettings import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer class KotestProjectConfig : AbstractProjectConfig() { - override fun beforeAll() = super.beforeAll().also { + override suspend fun beforeProject() = super.beforeProject().also { // The only purpose of config caching is performance. We always want caching disabled in tests (so we can // freely modify the config without affecting other tests executing afterwards). MetaconfigSettings.cacheEnabled = false diff --git a/pom.xml b/pom.xml index 5a8d078d51..1d9a16b7f2 100644 --- a/pom.xml +++ b/pom.xml @@ -24,13 +24,14 @@ 11.0.14 - 1.6.21 - 5.5.5 - 5.9.1 - 1.0-126-g02b0c86 - 1.1-126-g1f776be - 1.16.0 - 3.2.2 + 1.9.10 + 5.7.2 + 5.10.0 + 1.0-127-g6c65524 + 1.1-127-gf49982f + 1.13.8 + 2.0.0 + 3.5.1 4.6.0 3.0.10 2.12.4 @@ -110,7 +111,7 @@ ${project.groupId} jitsi-xmpp-extensions - 1.0-75-g4664207 + 1.0-76-ge98f8af diff --git a/rtp/spotbugs-exclude.xml b/rtp/spotbugs-exclude.xml index b9956fab85..f7cf298ceb 100644 --- a/rtp/spotbugs-exclude.xml +++ b/rtp/spotbugs-exclude.xml @@ -30,6 +30,9 @@ + + + From 30dcf51e4990786e6be1cc80179ac8a5c1a3b2f7 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 28 Sep 2023 17:03:53 -0500 Subject: [PATCH 052/189] log: Log colibri2 requests/responses with context from the Conference. (#2057) --- jvb/src/main/java/org/jitsi/videobridge/Conference.java | 2 ++ .../org/jitsi/videobridge/EndpointMessageTransport.java | 4 ++-- .../kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt | 6 ++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/Conference.java b/jvb/src/main/java/org/jitsi/videobridge/Conference.java index 95ca0a1444..cd2cfae219 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Conference.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Conference.java @@ -230,6 +230,7 @@ public Conference(Videobridge videobridge, { try { + logger.info("RECV colibri2 request: " + request.getRequest().toXML()); long start = System.currentTimeMillis(); Pair p = colibri2Handler.handleConferenceModifyIQ(request.getRequest()); IQ response = p.getFirst(); @@ -244,6 +245,7 @@ public Conference(Videobridge videobridge, logger.warn("Took " + processingDelay + " ms to process an IQ (total delay " + totalDelay + " ms): " + request.getRequest().toXML()); } + logger.info("SENT colibri2 response: " + response.toXML()); request.getCallback().invoke(response); if (expire) videobridge.expireConference(this); } diff --git a/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java b/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java index 629985a222..f45918c5bc 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java +++ b/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java @@ -151,9 +151,9 @@ public BridgeChannelMessage sourceVideoType(SourceVideoTypeMessage sourceVideoTy Conference conference = endpoint.getConference(); - if (conference == null || conference.isExpired()) + if (conference.isExpired()) { - getLogger().warn("Unable to forward SourceVideoTypeMessage, conference is null or expired"); + getLogger().warn("Unable to forward SourceVideoTypeMessage, conference is expired"); return null; } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt index f6882e08b3..f4367dd308 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt @@ -183,7 +183,10 @@ class XmppConnection : IQListener { if (iq == null) { return null } - logger.cdebug { "RECV: ${iq.toXML()}" } + // colibri2 requests are logged at the conference level. + if (iq !is ConferenceModifyIQ) { + logger.cdebug { "RECV: ${iq.toXML()}" } + } return when (iq.type) { IQ.Type.get, IQ.Type.set -> handleIqRequest(iq, mucClient)?.also { @@ -208,7 +211,6 @@ class XmppConnection : IQListener { handler.colibriRequestReceived( ColibriRequest(iq, colibriDelayStats, colibriProcessingDelayStats) { response -> response.setResponseTo(iq) - logger.debug { "SENT: ${response.toXML()}" } mucClient.sendStanza(response) } ) From be7701fa7ff5f582b26ff28802e7d6b6b4671a90 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 3 Oct 2023 11:20:01 -0400 Subject: [PATCH 053/189] Update for ktlint 1.0.0 (ktlint-maven-plugin 3.0.0). (#2059) --- .editorconfig | 1 + .../kotlin/org/jitsi/nlj/MediaSourceDesc.kt | 19 +- .../main/kotlin/org/jitsi/nlj/PacketInfo.kt | 1 + .../kotlin/org/jitsi/nlj/RtpEncodingDesc.kt | 10 +- .../main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt | 15 +- .../main/kotlin/org/jitsi/nlj/Transceiver.kt | 3 +- .../org/jitsi/nlj/codec/vp8/Vp8Utils.kt | 22 +- .../org/jitsi/nlj/codec/vpx/VpxUtils.kt | 6 +- .../kotlin/org/jitsi/nlj/dtls/DtlsUtils.kt | 10 +- .../org/jitsi/nlj/dtls/TlsClientImpl.kt | 3 +- .../org/jitsi/nlj/rtcp/RtcpRrGenerator.kt | 4 +- .../org/jitsi/nlj/rtp/AudioRtpPacket.kt | 11 +- .../org/jitsi/nlj/rtp/PaddingVideoPacket.kt | 3 +- .../org/jitsi/nlj/rtp/RedAudioRtpPacket.kt | 22 +- .../kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt | 3 +- .../org/jitsi/nlj/rtp/TransportCcEngine.kt | 4 +- .../org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt | 1 + .../org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt | 7 +- .../kotlin/org/jitsi/nlj/srtp/SrtpUtil.kt | 24 +- .../org/jitsi/nlj/transform/PipelineDsl.kt | 4 +- .../org/jitsi/nlj/transform/node/Node.kt | 3 + .../incoming/IncomingStatisticsTracker.kt | 8 +- .../node/incoming/RemoteBandwidthEstimator.kt | 3 +- .../outgoing/OutgoingStatisticsTracker.kt | 2 +- .../kotlin/org/jitsi/nlj/util/ArrayCache.kt | 47 +- .../kotlin/org/jitsi/nlj/util/Bandwidth.kt | 21 +- .../kotlin/org/jitsi/nlj/util/DataSize.kt | 9 +- .../kotlin/org/jitsi/nlj/util/InstantUtils.kt | 3 +- .../kotlin/org/jitsi/nlj/util/PacketCache.kt | 9 +- .../kotlin/org/jitsi/nlj/util/RateUtils.kt | 1 + .../jitsi/nlj/util/SsrcAssociationStore.kt | 3 +- .../org/jitsi/nlj/util/StreamInformation.kt | 9 +- .../org/jitsi/nlj/MediaSourceDescTest.kt | 10 +- .../module_tests/SrtpTransformerFactory.kt | 3 +- .../org/jitsi/nlj/test_utils/TimelineTest.kt | 3 +- .../node/incoming/SrtpDecryptTest.kt | 3 +- .../node/outgoing/SrtpEncryptTest.kt | 3 +- .../org/jitsi/videobridge/AbstractEndpoint.kt | 13 +- .../kotlin/org/jitsi/videobridge/Endpoint.kt | 12 +- .../kotlin/org/jitsi/videobridge/SsrcCache.kt | 10 +- .../cc/allocation/BandwidthAllocation.kt | 19 +- .../cc/allocation/BandwidthAllocator.kt | 39 +- .../cc/allocation/BitrateController.kt | 5 +- .../cc/allocation/PacketHandler.kt | 4 +- .../videobridge/cc/allocation/Prioritize.kt | 6 +- .../cc/allocation/SingleSourceAllocation.kt | 5 +- .../cc/allocation/VideoConstraints.kt | 3 +- .../vp9/Vp9AdaptiveSourceProjectionContext.kt | 24 +- .../org/jitsi/videobridge/cc/vp9/Vp9Frame.kt | 4 +- .../jitsi/videobridge/cc/vp9/Vp9Picture.kt | 3 +- .../videobridge/cc/vp9/Vp9QualityFilter.kt | 5 +- .../colibri2/Colibri2ConferenceHandler.kt | 5 +- .../load_management/JvbLoadManager.kt | 3 +- .../message/BridgeChannelMessage.kt | 21 +- .../relay/RelayMessageTransport.kt | 3 +- .../videobridge/relay/RelayedEndpoint.kt | 6 +- .../transport/dtls/DtlsTransport.kt | 6 +- .../videobridge/transport/ice/IceTransport.kt | 11 +- .../jitsi/videobridge/util/PayloadTypeUtil.kt | 5 +- .../videobridge/version/JvbVersionService.kt | 8 +- .../websocket/ColibriWebSocketService.kt | 5 +- .../jitsi/videobridge/xmpp/XmppConnection.kt | 9 +- .../EndpointConnectionStatusMonitorTest.kt | 2 +- .../allocation/BitrateControllerPerfTest.kt | 3 +- .../cc/allocation/BitrateControllerTest.kt | 669 +++++++++--------- .../allocation/BitrateControllerTraceTest.kt | 38 +- .../cc/allocation/EffectiveConstraintsTest.kt | 6 +- .../allocation/SingleSourceAllocationTest.kt | 109 +-- .../cc/vp9/Vp9AdaptiveSourceProjectionTest.kt | 39 +- .../cc/vp9/Vp9QualityFilterTest.kt | 9 +- pom.xml | 2 +- .../kotlin/org/jitsi/rtp/UnparsedPacket.kt | 11 +- .../org/jitsi/rtp/extensions/ByteBuffer.kt | 3 +- .../bytearray/ByteArrayExtensions.kt | 3 +- .../org/jitsi/rtp/rtcp/RtcpReportBlock.kt | 9 +- .../kotlin/org/jitsi/rtp/rtcp/RtcpRrPacket.kt | 3 +- .../org/jitsi/rtp/rtcp/RtcpSdesPacket.kt | 9 +- .../kotlin/org/jitsi/rtp/rtcp/RtcpSrPacket.kt | 9 +- .../jitsi/rtp/rtcp/UnsupportedRtcpPacket.kt | 3 +- .../org/jitsi/rtp/rtcp/rtcpfb/RtcpFbPacket.kt | 3 +- .../rtcp/rtcpfb/UnsupportedRtcpFbPacket.kt | 3 +- .../payload_specific_fb/RtcpFbFirPacket.kt | 3 +- .../payload_specific_fb/RtcpFbRembPacket.kt | 6 +- .../transport_layer_fb/tcc/LastChunk.kt | 4 +- .../transport_layer_fb/tcc/RtcpFbTccPacket.kt | 10 +- .../org/jitsi/rtp/rtp/RedPacketParser.kt | 16 +- .../kotlin/org/jitsi/rtp/rtp/RtpHeader.kt | 37 +- .../org/jitsi/rtp/rtp/RtpSequenceNumber.kt | 3 +- .../AbsSendTimeHeaderExtension.kt | 8 +- .../AudioLevelHeaderExtension.kt | 6 +- .../HeaderExtensionHelpers.kt | 22 +- .../header_extensions/TccHeaderExtension.kt | 6 +- .../kotlin/org/jitsi/rtp/util/FieldParsers.kt | 12 +- .../kotlin/org/jitsi/rtp/util/RtpUtils.kt | 59 +- .../extensions/ByteBufferExtensionsTest.kt | 4 +- .../tcc/RtcpFbTccPacketTest.kt | 6 +- .../kotlin/org/jitsi/rtp/rtp/RtpPacketTest.kt | 3 +- 97 files changed, 788 insertions(+), 897 deletions(-) diff --git a/.editorconfig b/.editorconfig index fbe6d50b34..9c60a17f53 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,5 +1,6 @@ [*.{kt,kts}] max_line_length=120 +ktlint_code_style = intellij_idea # I find trailing commas annoying ktlint_standard_trailing-comma-on-call-site = disabled diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt index 3da9ba9bb9..e09c7f2ee7 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt @@ -87,19 +87,21 @@ class MediaSourceDesc private fun updateLayerCache() { layersById.clear() layersByIndex.clear() - val layers_ = ArrayList() + val tempLayers = ArrayList() for (encoding in rtpEncodings) { for (layer in encoding.layers) { layersById[encoding.encodingId(layer)] = layer layersByIndex[layer.index] = layer - layers_.add(layer) + tempLayers.add(layer) } } - layers = Collections.unmodifiableList(layers_) + layers = Collections.unmodifiableList(tempLayers) } - init { updateLayerCache() } + init { + updateLayerCache() + } /** * Gets the last "stable" bitrate (in bps) of the encoding of the specified @@ -126,15 +128,13 @@ class MediaSourceDesc fun hasRtpLayers(): Boolean = layers.isNotEmpty() @Synchronized - fun numRtpLayers(): Int = - layersByIndex.size + fun numRtpLayers(): Int = layersByIndex.size val primarySSRC: Long get() = rtpEncodings[0].primarySSRC @Synchronized - fun getRtpLayerByQualityIdx(idx: Int): RtpLayerDesc? = - layersByIndex[idx] + fun getRtpLayerByQualityIdx(idx: Int): RtpLayerDesc? = layersByIndex[idx] @Synchronized fun findRtpLayerDesc(videoRtpPacket: VideoRtpPacket): RtpLayerDesc? { @@ -147,8 +147,7 @@ class MediaSourceDesc } @Synchronized - fun findRtpEncodingDesc(ssrc: Long): RtpEncodingDesc? = - rtpEncodings.find { it.matches(ssrc) } + fun findRtpEncodingDesc(ssrc: Long): RtpEncodingDesc? = rtpEncodings.find { it.matches(ssrc) } @Synchronized fun setEncodingLayers(layers: Array, ssrc: Long) { diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketInfo.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketInfo.kt index 0adcd480ac..f23e064045 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketInfo.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketInfo.kt @@ -215,6 +215,7 @@ open class PacketInfo @JvmOverloads constructor( /** * If this is enabled all [Node]s will verify that the payload didn't unexpectedly change. This is expensive. */ + @field:Suppress("ktlint:standard:property-naming") var ENABLE_PAYLOAD_VERIFICATION = false } } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt index 1533589e85..8cfee2f807 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt @@ -69,7 +69,9 @@ constructor( require(layer.eid == eid) { "Cannot add layer with EID ${layer.eid} to encoding with EID $eid" } } } - init { validateLayerEids(initialLayers) } + init { + validateLayerEids(initialLayers) + } internal var layers = initialLayers set(newLayers) { @@ -97,8 +99,7 @@ constructor( * rid). This server-side id is used in the layer lookup table that is * maintained in [MediaSourceDesc]. */ - fun encodingId(layer: RtpLayerDesc): Long = - calcEncodingId(primarySSRC, layer.layerId) + fun encodingId(layer: RtpLayerDesc): Long = calcEncodingId(primarySSRC, layer.layerId) /** * Get the secondary ssrc for this encoding that corresponds to the given @@ -162,8 +163,7 @@ constructor( } companion object { - fun calcEncodingId(ssrc: Long, layerId: Int) = - ssrc or (layerId.toLong() shl 32) + fun calcEncodingId(ssrc: Long, layerId: Int) = ssrc or (layerId.toLong() shl 32) } } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt index 4d52c322e9..2f7019b06d 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt @@ -49,9 +49,9 @@ constructor( /** * The max height of the bitstream that this instance represents. The actual * height may be less due to bad network or system load. + * + * XXX we should be able to sniff the actual height from the RTP packets. */ - // XXX we should be able to sniff the actual height from the RTP - // packets. val height: Int, /** * The max frame rate (in fps) of the bitstream that this instance @@ -287,11 +287,10 @@ constructor( * Get a string description of a layer index. */ @JvmStatic - fun indexString(index: Int): String = - if (index == SUSPENDED_INDEX) { - "SUSP" - } else { - "E${getEidFromIndex(index)}S${getSidFromIndex(index)}T${getTidFromIndex(index)}" - } + fun indexString(index: Int): String = if (index == SUSPENDED_INDEX) { + "SUSP" + } else { + "E${getEidFromIndex(index)}S${getSidFromIndex(index)}T${getTidFromIndex(index)}" + } } } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Transceiver.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Transceiver.kt index afa45577b0..d2302cf63f 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Transceiver.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Transceiver.kt @@ -299,8 +299,7 @@ class Transceiver( internalTransformers = internal } - fun setSrtpInformation(srtpTransformers: SrtpTransformers) = - setSrtpInformationInternal(srtpTransformers, false) + fun setSrtpInformation(srtpTransformers: SrtpTransformers) = setSrtpInformationInternal(srtpTransformers, false) /** * Forcibly mute or unmute the incoming audio stream diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/codec/vp8/Vp8Utils.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/codec/vp8/Vp8Utils.kt index ae861dc87e..1f189ef0dc 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/codec/vp8/Vp8Utils.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/codec/vp8/Vp8Utils.kt @@ -79,18 +79,16 @@ class Vp8Utils { } } - fun getTemporalLayerIdOfFrame(vp8Payload: ByteBuffer) = - DePacketizer.VP8PayloadDescriptor.getTemporalLayerIndex( - vp8Payload.array(), - vp8Payload.arrayOffset(), - vp8Payload.limit() - ) + fun getTemporalLayerIdOfFrame(vp8Payload: ByteBuffer) = DePacketizer.VP8PayloadDescriptor.getTemporalLayerIndex( + vp8Payload.array(), + vp8Payload.arrayOffset(), + vp8Payload.limit() + ) - fun getTemporalLayerIdOfFrame(vp8Packet: RtpPacket) = - DePacketizer.VP8PayloadDescriptor.getTemporalLayerIndex( - vp8Packet.buffer, - vp8Packet.payloadOffset, - vp8Packet.payloadLength - ) + fun getTemporalLayerIdOfFrame(vp8Packet: RtpPacket) = DePacketizer.VP8PayloadDescriptor.getTemporalLayerIndex( + vp8Packet.buffer, + vp8Packet.payloadOffset, + vp8Packet.payloadLength + ) } } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/codec/vpx/VpxUtils.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/codec/vpx/VpxUtils.kt index 09a670af30..bac5d7c1cc 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/codec/vpx/VpxUtils.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/codec/vpx/VpxUtils.kt @@ -54,8 +54,7 @@ class VpxUtils { * @return the extended picture ID resulting from doing "start + delta" */ @JvmStatic - fun applyExtendedPictureIdDelta(start: Int, delta: Int): Int = - (start + delta) and EXTENDED_PICTURE_ID_MASK + fun applyExtendedPictureIdDelta(start: Int, delta: Int): Int = (start + delta) and EXTENDED_PICTURE_ID_MASK /** * Returns the delta between two VP8/VP9 Tl0PicIdx values, taking into account @@ -86,7 +85,6 @@ class VpxUtils { * @return the Tl0PicIdx resulting from doing "start + delta" */ @JvmStatic - fun applyTl0PicIdxDelta(start: Int, delta: Int): Int = - (start + delta) and TL0PICIDX_MASK + fun applyTl0PicIdxDelta(start: Int, delta: Int): Int = (start + delta) and TL0PICIDX_MASK } } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsUtils.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsUtils.kt index ce41481594..c1259c3ba1 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsUtils.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsUtils.kt @@ -100,10 +100,7 @@ class DtlsUtils { * * TODO: make the algorithm dynamic (passed in) to support older dtls versions/clients */ - private fun generateCertificate( - subject: X500Name, - keyPair: KeyPair - ): Certificate { + private fun generateCertificate(subject: X500Name, keyPair: KeyPair): Certificate { val now = System.currentTimeMillis() val startDate = Date(now - Duration.ofDays(1).toMillis()) val expiryDate = Date(now + Duration.ofDays(7).toMillis()) @@ -182,10 +179,7 @@ class DtlsUtils { * and validate against the fingerprints presented by the remote endpoint * via the signaling path. */ - private fun verifyAndValidateCertificate( - certificate: Certificate, - remoteFingerprints: Map - ) { + private fun verifyAndValidateCertificate(certificate: Certificate, remoteFingerprints: Map) { // RFC 4572 "Connection-Oriented Media Transport over the Transport // Layer Security (TLS) Protocol in the Session Description Protocol // (SDP)" defines that "[a] certificate fingerprint MUST be computed diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsClientImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsClientImpl.kt index 8ee1c76ba0..10423a5087 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsClientImpl.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsClientImpl.kt @@ -169,8 +169,7 @@ class TlsClientImpl( logger.cinfo { "Negotiated DTLS version $serverVersion" } } - override fun getSupportedVersions(): Array = - arrayOf(ProtocolVersion.DTLSv12) + override fun getSupportedVersions(): Array = arrayOf(ProtocolVersion.DTLSv12) override fun notifyAlertRaised(alertLevel: Short, alertDescription: Short, message: String?, cause: Throwable?) = logger.notifyAlertRaised(alertLevel, alertDescription, message, cause) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpRrGenerator.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpRrGenerator.kt index 5146541621..18aa740e4d 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpRrGenerator.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpRrGenerator.kt @@ -127,7 +127,9 @@ class RtcpRrGenerator( when (packets.size) { 0 -> {} 1 -> rtcpSender(packets.first()) - else -> for (packet in CompoundRtcpPacket.createWithMtu(packets)) { rtcpSender(packet) } + else -> for (packet in CompoundRtcpPacket.createWithMtu(packets)) { + rtcpSender(packet) + } } backgroundExecutor.schedule(this::doWork, reportingInterval) } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/AudioRtpPacket.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/AudioRtpPacket.kt index a425a3c003..65c1269da6 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/AudioRtpPacket.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/AudioRtpPacket.kt @@ -23,10 +23,9 @@ open class AudioRtpPacket( length: Int ) : RtpPacket(buffer, offset, length) { - override fun clone(): AudioRtpPacket = - AudioRtpPacket( - cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET), - BYTES_TO_LEAVE_AT_START_OF_PACKET, - length - ) + override fun clone(): AudioRtpPacket = AudioRtpPacket( + cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET), + BYTES_TO_LEAVE_AT_START_OF_PACKET, + length + ) } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/PaddingVideoPacket.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/PaddingVideoPacket.kt index aa8abb2b5b..12a4dbfbd8 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/PaddingVideoPacket.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/PaddingVideoPacket.kt @@ -25,8 +25,7 @@ class PaddingVideoPacket private constructor( length: Int ) : VideoRtpPacket(buffer, offset, length) { - override fun clone(): PaddingVideoPacket = - throw NotImplementedError("clone() not supported for padding packets.") + override fun clone(): PaddingVideoPacket = throw NotImplementedError("clone() not supported for padding packets.") companion object { /** diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RedAudioRtpPacket.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RedAudioRtpPacket.kt index 6da1e747c0..ff0e028561 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RedAudioRtpPacket.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RedAudioRtpPacket.kt @@ -33,19 +33,17 @@ class RedAudioRtpPacket( removed = true } - fun removeRedAndGetRedundancyPackets(): List = - if (removed) { - throw IllegalStateException("RED encapsulation already removed.") - } else { - parser.decapsulate(this, parseRedundancy = true).also { removed = true } - } + fun removeRedAndGetRedundancyPackets(): List = if (removed) { + throw IllegalStateException("RED encapsulation already removed.") + } else { + parser.decapsulate(this, parseRedundancy = true).also { removed = true } + } - override fun clone(): RedAudioRtpPacket = - RedAudioRtpPacket( - cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET), - BYTES_TO_LEAVE_AT_START_OF_PACKET, - length - ) + override fun clone(): RedAudioRtpPacket = RedAudioRtpPacket( + cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET), + BYTES_TO_LEAVE_AT_START_OF_PACKET, + length + ) companion object { val parser = RedPacketParser(::AudioRtpPacket) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt index dc730843ee..1e869642ec 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt @@ -100,7 +100,6 @@ enum class RtpExtensionType(val uri: String) { companion object { private val uriMap = RtpExtensionType.values().associateBy(RtpExtensionType::uri) - fun createFromUri(uri: String): RtpExtensionType? = - uriMap.getOrDefault(uri, null) + fun createFromUri(uri: String): RtpExtensionType? = uriMap.getOrDefault(uri, null) } } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/TransportCcEngine.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/TransportCcEngine.kt index a2b648ce06..8e43a057e7 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/TransportCcEngine.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/TransportCcEngine.kt @@ -256,7 +256,9 @@ class TransportCcEngine( * [PacketDetailState] is the state of a [PacketDetail] */ private enum class PacketDetailState { - Unreported, ReportedLost, ReportedReceived + Unreported, + ReportedLost, + ReportedReceived } /** diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt index ca822b4250..93ea7b741a 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt @@ -79,6 +79,7 @@ class Vp8Packet private constructor( val hasTL0PICIDX = DePacketizer.VP8PayloadDescriptor.hasTL0PICIDX(buffer, payloadOffset, payloadLength) + @field:Suppress("ktlint:standard:property-naming") private var _TL0PICIDX = TL0PICIDX ?: DePacketizer.VP8PayloadDescriptor.getTL0PICIDX(buffer, payloadOffset, payloadLength) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt index 52038026d1..bacff0b341 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt @@ -94,6 +94,7 @@ class Vp9Packet private constructor( val isInterPicturePredicted = DePacketizer.VP9PayloadDescriptor.isInterPicturePredicted(buffer, payloadOffset, payloadLength) + @field:Suppress("ktlint:standard:property-naming") private var _TL0PICIDX = TL0PICIDX ?: DePacketizer.VP9PayloadDescriptor.getTL0PICIDX(buffer, payloadOffset, payloadLength) @@ -153,10 +154,8 @@ class Vp9Packet private constructor( val usesInterLayerDependency: Boolean = DePacketizer.VP9PayloadDescriptor.usesInterLayerDependency(buffer, payloadOffset, payloadLength) - fun getScalabilityStructure( - eid: Int = 0, - baseFrameRate: Double = 30.0 - ) = Companion.getScalabilityStructure(buffer, payloadOffset, payloadLength, ssrc, eid, baseFrameRate) + fun getScalabilityStructure(eid: Int = 0, baseFrameRate: Double = 30.0) = + Companion.getScalabilityStructure(buffer, payloadOffset, payloadLength, ssrc, eid, baseFrameRate) val scalabilityStructureNumSpatial: Int get() { diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/srtp/SrtpUtil.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/srtp/SrtpUtil.kt index 1eefb1bff8..550d3a0365 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/srtp/SrtpUtil.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/srtp/SrtpUtil.kt @@ -34,12 +34,24 @@ class SrtpUtil { fun getSrtpProtectionProfileFromName(profileName: String): Int { return when (profileName) { - "SRTP_AES128_CM_HMAC_SHA1_80" -> { SRTPProtectionProfile.SRTP_AES128_CM_HMAC_SHA1_80 } - "SRTP_AES128_CM_HMAC_SHA1_32" -> { SRTPProtectionProfile.SRTP_AES128_CM_HMAC_SHA1_32 } - "SRTP_NULL_HMAC_SHA1_32" -> { SRTPProtectionProfile.SRTP_NULL_HMAC_SHA1_32 } - "SRTP_NULL_HMAC_SHA1_80" -> { SRTPProtectionProfile.SRTP_NULL_HMAC_SHA1_80 } - "SRTP_AEAD_AES_128_GCM" -> { SRTPProtectionProfile.SRTP_AEAD_AES_128_GCM } - "SRTP_AEAD_AES_256_GCM" -> { SRTPProtectionProfile.SRTP_AEAD_AES_256_GCM } + "SRTP_AES128_CM_HMAC_SHA1_80" -> { + SRTPProtectionProfile.SRTP_AES128_CM_HMAC_SHA1_80 + } + "SRTP_AES128_CM_HMAC_SHA1_32" -> { + SRTPProtectionProfile.SRTP_AES128_CM_HMAC_SHA1_32 + } + "SRTP_NULL_HMAC_SHA1_32" -> { + SRTPProtectionProfile.SRTP_NULL_HMAC_SHA1_32 + } + "SRTP_NULL_HMAC_SHA1_80" -> { + SRTPProtectionProfile.SRTP_NULL_HMAC_SHA1_80 + } + "SRTP_AEAD_AES_128_GCM" -> { + SRTPProtectionProfile.SRTP_AEAD_AES_128_GCM + } + "SRTP_AEAD_AES_256_GCM" -> { + SRTPProtectionProfile.SRTP_AEAD_AES_256_GCM + } else -> throw IllegalArgumentException("Unsupported SRTP protection profile: $profileName") } } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/PipelineDsl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/PipelineDsl.kt index 47acd7f473..c276353079 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/PipelineDsl.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/PipelineDsl.kt @@ -48,7 +48,9 @@ class PipelineBuilder { } fun node(node: Node, condition: () -> Boolean = { true }) { - if (condition()) { addNode(node) } + if (condition()) { + addNode(node) + } } /** diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/Node.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/Node.kt index 7e75e9d719..a2ece3eacd 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/Node.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/Node.kt @@ -130,7 +130,10 @@ sealed class Node( abstract fun trace(f: () -> Unit) companion object { + @field:Suppress("ktlint:standard:property-naming") var TRACE_ENABLED = false + + @field:Suppress("ktlint:standard:property-naming") var PLUGINS_ENABLED = false // 'Plugins' are observers which, when enabled, will be passed every packet that passes through diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/IncomingStatisticsTracker.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/IncomingStatisticsTracker.kt index a9136d7af7..e2059a1e94 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/IncomingStatisticsTracker.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/IncomingStatisticsTracker.kt @@ -99,7 +99,7 @@ class IncomingStatisticsSnapshot( val ssrcStats: Map ) { fun toJson(): OrderedJsonObject = OrderedJsonObject().apply { - ssrcStats.forEach() { (ssrc, snapshot) -> + ssrcStats.forEach { (ssrc, snapshot) -> put(ssrc, snapshot.toJson()) } } @@ -233,11 +233,7 @@ class IncomingSsrcStats( * raw RTP timestamp, but the 'translated' timestamp which is a function of the RTP timestamp and the clockrate) * and was received at [packetReceivedTime] */ - fun packetReceived( - packet: RtpPacket, - packetSentTimestamp: Instant, - packetReceivedTime: Instant - ) { + fun packetReceived(packet: RtpPacket, packetSentTimestamp: Instant, packetReceivedTime: Instant) { val packetSequenceNumber = packet.sequenceNumber synchronized(statsLock) { activitySinceLastSnapshot = true diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RemoteBandwidthEstimator.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RemoteBandwidthEstimator.kt index 3d94ec3cd0..323961c198 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RemoteBandwidthEstimator.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RemoteBandwidthEstimator.kt @@ -71,7 +71,8 @@ class RemoteBandwidthEstimator( Collections.synchronizedSet( LRUCache.lruSet( MAX_SSRCS, - true // accessOrder + // accessOrder + true ) ) private var numRembsCreated = 0 diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/OutgoingStatisticsTracker.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/OutgoingStatisticsTracker.kt index 2f2c56403a..d2d07bec5f 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/OutgoingStatisticsTracker.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/OutgoingStatisticsTracker.kt @@ -84,7 +84,7 @@ class OutgoingStatisticsSnapshot( val ssrcStats: Map ) { fun toJson() = OrderedJsonObject().apply { - ssrcStats.forEach() { (ssrc, snapshot) -> + ssrcStats.forEach { (ssrc, snapshot) -> put(ssrc, snapshot.toJson()) } } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ArrayCache.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ArrayCache.kt index 958a8874c0..f68ed72874 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ArrayCache.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ArrayCache.kt @@ -69,21 +69,19 @@ open class ArrayCache( /** * Inserts an item with a specific index in the cache. Stores a copy. */ - fun insertItem(item: T, index: Int, timeAdded: Long): Boolean = - if (synchronize) { - synchronized(syncRoot) { - doInsert(item, index, timeAdded) - } - } else { + fun insertItem(item: T, index: Int, timeAdded: Long): Boolean = if (synchronize) { + synchronized(syncRoot) { doInsert(item, index, timeAdded) } + } else { + doInsert(item, index, timeAdded) + } /** * Inserts an item with a specific index in the cache, computing time * from [clock]. Stores a copy. */ - fun insertItem(item: T, index: Int): Boolean = - insertItem(item, index, clock.millis()) + fun insertItem(item: T, index: Int): Boolean = insertItem(item, index, clock.millis()) private fun doInsert(item: T, index: Int, timeAdded: Long): Boolean { val diff = if (head == -1) -1 else index - cache[head].index @@ -187,14 +185,13 @@ open class ArrayCache( /** * Updates the [timeAdded] value of an item with a particular index, if it is in the cache. */ - protected fun updateTimeAdded(index: Int, timeAdded: Long) = - if (synchronize) { - synchronized(syncRoot) { - doUpdateTimeAdded(index, timeAdded) - } - } else { + protected fun updateTimeAdded(index: Int, timeAdded: Long) = if (synchronize) { + synchronized(syncRoot) { doUpdateTimeAdded(index, timeAdded) } + } else { + doUpdateTimeAdded(index, timeAdded) + } private fun doUpdateTimeAdded(index: Int, timeAdded: Long) { if (head == -1 || index > cache[head].index) { @@ -213,14 +210,13 @@ open class ArrayCache( * iteration and returns. Note that if the caller must clone the item on their own if they want to keep or modify * it in any way. */ - fun forEachDescending(predicate: (T) -> Boolean) = - if (synchronize) { - synchronized(syncRoot) { - doForEachDescending(predicate) - } - } else { + fun forEachDescending(predicate: (T) -> Boolean) = if (synchronize) { + synchronized(syncRoot) { doForEachDescending(predicate) } + } else { + doForEachDescending(predicate) + } private fun doForEachDescending(predicate: (T) -> Boolean) { if (head == -1) return @@ -240,14 +236,13 @@ open class ArrayCache( /** * Removes all items stored in the cache, calling [discardItem] for each one. */ - fun flush() = - if (synchronize) { - synchronized(syncRoot) { - doFlush() - } - } else { + fun flush() = if (synchronize) { + synchronized(syncRoot) { doFlush() } + } else { + doFlush() + } private fun doFlush() { for (container in cache) { diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/Bandwidth.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/Bandwidth.kt index 5790a51066..f1d7c33200 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/Bandwidth.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/Bandwidth.kt @@ -30,11 +30,9 @@ value class Bandwidth(val bps: Double) : Comparable { val mbps: Double get() = bps / (1000 * 1000) - operator fun minus(other: Bandwidth): Bandwidth = - Bandwidth(bps - other.bps) + operator fun minus(other: Bandwidth): Bandwidth = Bandwidth(bps - other.bps) - operator fun plus(other: Bandwidth): Bandwidth = - Bandwidth(bps + other.bps) + operator fun plus(other: Bandwidth): Bandwidth = Bandwidth(bps + other.bps) /** * For multiplication, we support multiplying against @@ -45,25 +43,20 @@ value class Bandwidth(val bps: Double) : Comparable { * * to reduce 'currentBandwidth' by 5% */ - operator fun times(other: Double): Bandwidth = - Bandwidth(bps * other) + operator fun times(other: Double): Bandwidth = Bandwidth(bps * other) - operator fun times(other: Int): Bandwidth = - Bandwidth(bps * other) + operator fun times(other: Int): Bandwidth = Bandwidth(bps * other) /** * For division, we support both dividing by * a normal number (giving a bandwidth), and dividing * by another bandwidth, giving a number */ - operator fun div(other: Double): Bandwidth = - Bandwidth(bps / other) + operator fun div(other: Double): Bandwidth = Bandwidth(bps / other) - operator fun div(other: Int): Bandwidth = - Bandwidth(bps / other) + operator fun div(other: Int): Bandwidth = Bandwidth(bps / other) - operator fun div(other: Bandwidth): Double = - bps / other.bps + operator fun div(other: Bandwidth): Double = bps / other.bps override fun compareTo(other: Bandwidth): Int = sign(bps - other.bps).toInt() diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/DataSize.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/DataSize.kt index 66cc539bfa..4cfab6ff4e 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/DataSize.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/DataSize.kt @@ -32,22 +32,19 @@ class DataSize( val kiloBytes: Double = bytes / 1000.0 val megaBytes: Double = kiloBytes / 1000.0 - operator fun minus(other: DataSize): DataSize = - DataSize(bits - other.bits) + operator fun minus(other: DataSize): DataSize = DataSize(bits - other.bits) operator fun minusAssign(other: DataSize) { bits -= other.bits } - operator fun plus(other: DataSize): DataSize = - DataSize(bits + other.bits) + operator fun plus(other: DataSize): DataSize = DataSize(bits + other.bits) operator fun plusAssign(other: DataSize) { bits += other.bits } - operator fun times(other: Int): DataSize = - DataSize(bits * other) + operator fun times(other: Int): DataSize = DataSize(bits * other) operator fun timesAssign(other: Int) { bits *= other diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/InstantUtils.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/InstantUtils.kt index 4507f75628..8d1701e41b 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/InstantUtils.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/InstantUtils.kt @@ -18,5 +18,4 @@ package org.jitsi.nlj.util import java.time.Instant -fun latest(vararg instants: Instant): Instant = - instants.reduce { a, b -> maxOf(a, b) } +fun latest(vararg instants: Instant): Instant = instants.reduce { a, b -> maxOf(a, b) } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/PacketCache.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/PacketCache.kt index a9a3ed426a..9d5a5ff7a9 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/PacketCache.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/PacketCache.kt @@ -39,9 +39,12 @@ class PacketCache( LinkedHashMap( // These are the default values of initialCapacity and loadFactor - we have to set them to be able to set // accessOrder - 16, // initialCapacity - 0.75F, // loadFactor - true // accessOrder + // initialCapacity + 16, + // loadFactor + 0.75F, + // accessOrder + true ) ) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/RateUtils.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/RateUtils.kt index ece26c21dc..ace6beeb99 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/RateUtils.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/RateUtils.kt @@ -36,6 +36,7 @@ infix fun DataSize.atRate(bw: Bandwidth): Duration { fun howMuchCanISendAtRate(bw: Bandwidth): Bandwidth = bw +@Suppress("ktlint:standard:function-naming") infix fun Bandwidth.`in`(time: Duration): DataSize { return DataSize((bps * (time.seconds + time.nano / 1e9)).toLong()) } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/SsrcAssociationStore.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/SsrcAssociationStore.kt index f1ef149ec7..2a8799d459 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/SsrcAssociationStore.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/SsrcAssociationStore.kt @@ -59,8 +59,7 @@ class SsrcAssociationStore( ssrcAssociationsBySecondarySsrc = ssrcAssociations.associateBy(SsrcAssociation::secondarySsrc) } - fun getPrimarySsrc(secondarySsrc: Long): Long? = - ssrcAssociationsBySecondarySsrc[secondarySsrc]?.primarySsrc + fun getPrimarySsrc(secondarySsrc: Long): Long? = ssrcAssociationsBySecondarySsrc[secondarySsrc]?.primarySsrc fun getSecondarySsrc(primarySsrc: Long, associationType: SsrcAssociationType): Long? = ssrcAssociationsByPrimarySsrc[primarySsrc]?.find { it.type == associationType }?.secondarySsrc diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/StreamInformation.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/StreamInformation.kt index fbb99da5d5..23de158830 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/StreamInformation.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/StreamInformation.kt @@ -205,8 +205,7 @@ class StreamInformationStoreImpl : StreamInformationStore { // NOTE(brian): Currently, we only have a use case to do a mapping of // secondary -> primary for local SSRCs and primary -> secondary for // remote SSRCs - override fun getLocalPrimarySsrc(secondarySsrc: Long): Long? = - localSsrcAssociations.getPrimarySsrc(secondarySsrc) + override fun getLocalPrimarySsrc(secondarySsrc: Long): Long? = localSsrcAssociations.getPrimarySsrc(secondarySsrc) override fun getRemoteSecondarySsrc(primarySsrc: Long, associationType: SsrcAssociationType): Long? = remoteSsrcAssociations.getSecondarySsrc(primarySsrc, associationType) @@ -218,11 +217,9 @@ class StreamInformationStoreImpl : StreamInformationStore { } } - override fun addReceiveSsrc(ssrc: Long, mediaType: MediaType) = - receiveSsrcStore.addReceiveSsrc(ssrc, mediaType) + override fun addReceiveSsrc(ssrc: Long, mediaType: MediaType) = receiveSsrcStore.addReceiveSsrc(ssrc, mediaType) - override fun removeReceiveSsrc(ssrc: Long) = - receiveSsrcStore.removeReceiveSsrc(ssrc) + override fun removeReceiveSsrc(ssrc: Long) = receiveSsrcStore.removeReceiveSsrc(ssrc) override fun getNodeStats(): NodeStatsBlock = NodeStatsBlock("Stream Information Store").apply { addBlock( diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/MediaSourceDescTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/MediaSourceDescTest.kt index 25f144afe1..b1526978bc 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/MediaSourceDescTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/MediaSourceDescTest.kt @@ -113,8 +113,7 @@ class MediaSourceDescTest : ShouldSpec() { * @return the subjective quality index of the flow specified in the * arguments. */ -private fun idx(spatialIdx: Int, temporalIdx: Int, temporalLen: Int) = - spatialIdx * temporalLen + temporalIdx +private fun idx(spatialIdx: Int, temporalIdx: Int, temporalLen: Int) = spatialIdx * temporalLen + temporalIdx /* * Creates layers for an encoding. @@ -124,12 +123,7 @@ private fun idx(spatialIdx: Int, temporalIdx: Int, temporalLen: Int) = * @param height the maximum height of the top spatial layer * @return an array that holds the layer descriptions. */ -private fun createRTPLayerDescs( - spatialLen: Int, - temporalLen: Int, - encodingIdx: Int, - height: Int -): Array { +private fun createRTPLayerDescs(spatialLen: Int, temporalLen: Int, encodingIdx: Int, height: Int): Array { val rtpLayers = arrayOfNulls(spatialLen * temporalLen) for (spatialIdx in 0 until spatialLen) { var frameRate = 30.toDouble() / (1 shl temporalLen - 1) diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/SrtpTransformerFactory.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/SrtpTransformerFactory.kt index 2bc1e2f9a0..4818d9516b 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/SrtpTransformerFactory.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/SrtpTransformerFactory.kt @@ -28,7 +28,8 @@ class SrtpTransformerFactory { srtpData.srtpProfileInformation, srtpData.keyingMaterial, srtpData.tlsRole, - cryptex = false, // TODO: add tests for the cryptex=true case + // TODO: add tests for the cryptex=true case + cryptex = false, StdoutLogger() ) } diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/test_utils/TimelineTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/test_utils/TimelineTest.kt index 5f43c31f0c..4f7a07efb2 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/test_utils/TimelineTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/test_utils/TimelineTest.kt @@ -40,5 +40,4 @@ internal class TimelineTest(private val clock: FakeClock) { } } -internal fun timeline(clock: FakeClock, block: TimelineTest.() -> Unit): TimelineTest = - TimelineTest(clock).apply(block) +internal fun timeline(clock: FakeClock, block: TimelineTest.() -> Unit): TimelineTest = TimelineTest(clock).apply(block) diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/SrtpDecryptTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/SrtpDecryptTest.kt index e28bc9ea4c..736d463866 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/SrtpDecryptTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/SrtpDecryptTest.kt @@ -36,7 +36,8 @@ internal class SrtpDecryptTest : ShouldSpec() { SrtpSample.srtpProfileInformation, SrtpSample.keyingMaterial.array(), SrtpSample.tlsRole, - cryptex = false, // TODO: add tests for cryptex case + // TODO: add tests for cryptex case + cryptex = false, StdoutLogger() ) diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/SrtpEncryptTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/SrtpEncryptTest.kt index c6db04be94..3496090d8e 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/SrtpEncryptTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/SrtpEncryptTest.kt @@ -41,7 +41,8 @@ internal class SrtpEncryptTest : ShouldSpec() { SrtpSample.srtpProfileInformation, SrtpSample.keyingMaterial.array(), SrtpSample.tlsRole, - cryptex = false, // TODO: add tests for cryptex case + // TODO: add tests for cryptex case + cryptex = false, StdoutLogger() ) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt index 531dc1acb0..b52f121552 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt @@ -154,10 +154,9 @@ abstract class AbstractEndpoint protected constructor( protected val mediaSource: MediaSourceDesc? get() = mediaSources.firstOrNull() - fun findMediaSourceDesc(sourceName: String): MediaSourceDesc? = - mediaSources.firstOrNull { - sourceName == it.sourceName - } + fun findMediaSourceDesc(sourceName: String): MediaSourceDesc? = mediaSources.firstOrNull { + sourceName == it.sourceName + } fun addEventHandler(eventHandler: EventHandler) { eventEmitter.addHandler(eventHandler) @@ -296,11 +295,7 @@ abstract class AbstractEndpoint protected constructor( * @param sourceName the name of the media source for which the constraints are to be applied. * @param newVideoConstraints the video constraints that the receiver wishes to receive. */ - fun addReceiver( - receiverId: String, - sourceName: String, - newVideoConstraints: VideoConstraints - ) { + fun addReceiver(receiverId: String, sourceName: String, newVideoConstraints: VideoConstraints) { val sourceConstraints = receiverVideoConstraints.computeIfAbsent(sourceName) { ReceiverConstraintsMap() } val oldVideoConstraints = sourceConstraints.put(receiverId, newVideoConstraints) if (oldVideoConstraints == null || oldVideoConstraints != newVideoConstraints) { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index 9718a5f71a..011146144e 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -216,8 +216,7 @@ class Endpoint @JvmOverloads constructor( newEffectiveConstraints: EffectiveConstraintsMap, ) = this@Endpoint.effectiveVideoConstraintsChanged(oldEffectiveConstraints, newEffectiveConstraints) - override fun keyframeNeeded(endpointId: String?, ssrc: Long) = - conference.requestKeyframe(endpointId, ssrc) + override fun keyframeNeeded(endpointId: String?, ssrc: Long) = conference.requestKeyframe(endpointId, ssrc) }, { getOrderedEndpoints() }, diagnosticContext, @@ -242,8 +241,7 @@ class Endpoint @JvmOverloads constructor( /** * Gets the endpoints in the conference in LastN order, with this {@link Endpoint} removed. */ - fun getOrderedEndpoints(): List = - conference.orderedEndpoints.filterNot { it == this } + fun getOrderedEndpoints(): List = conference.orderedEndpoints.filterNot { it == this } /** * Listen for RTT updates from [transceiver] and update the ICE stats the first time an RTT is available. Note that @@ -601,7 +599,7 @@ class Endpoint @JvmOverloads constructor( } /** - * Create an SCTP connection for this Endpoint. If [openDataChannelLocally] is true, + * Create an SCTP connection for this Endpoint. If [OPEN_DATA_CHANNEL_LOCALLY] is true, * we will create the data channel locally, otherwise we will wait for the remote side * to open it. */ @@ -632,7 +630,7 @@ class Endpoint @JvmOverloads constructor( messageTransport.setDataChannel(dataChannel) } dataChannelHandler.setDataChannelStack(dataChannelStack!!) - if (openDataChannelLocally) { + if (OPEN_DATA_CHANNEL_LOCALLY) { // This logic is for opening the data channel locally logger.info("Will open the data channel.") val dataChannel = dataChannelStack!!.createDataChannel( @@ -1170,7 +1168,7 @@ class Endpoint @JvmOverloads constructor( * Whether or not the bridge should be the peer which opens the data channel * (as opposed to letting the far peer/client open it). */ - private const val openDataChannelLocally = false + private const val OPEN_DATA_CHANNEL_LOCALLY = false /** * Count the number of dropped packets and exceptions. diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt index 297693bc0f..5189d7f391 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt @@ -220,7 +220,9 @@ class SendSource(val props: SourceDesc, val send1: SendSsrc, val send2: SendSsrc * Fix SSRC and timestamps in an RTCP packet. * For packets in the same direction as media flow; feedback messages handled separately. */ - fun rewriteRtcp(packet: RtcpPacket) { getSender(packet.senderSsrc).rewriteRtcp(packet) } + fun rewriteRtcp(packet: RtcpPacket) { + getSender(packet.senderSsrc).rewriteRtcp(packet) + } /** * {@inheritDoc} @@ -275,7 +277,8 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: */ private val sendSources = LRUCache( size, - true // accessOrder + // accessOrder + true ) /** @@ -569,8 +572,7 @@ class VideoSsrcCache(size: Int, ep: SsrcRewriter, parentLogger: Logger) : /** * {@inheritDoc} */ - override fun findSourceProps(ssrc: Long): SourceDesc? = - ep.findVideoSourceProps(ssrc)?.let { SourceDesc(it) } + override fun findSourceProps(ssrc: Long): SourceDesc? = ep.findVideoSourceProps(ssrc)?.let { SourceDesc(it) } /** * {@inheritDoc} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt index c029bdcfbf..ebca3fd157 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt @@ -40,17 +40,16 @@ class BandwidthAllocation @JvmOverloads constructor( /** * Whether the two allocations have the same endpoints and same layers. */ - fun isTheSameAs(other: BandwidthAllocation) = - allocations.size == other.allocations.size && - oversending == other.oversending && - allocations.all { allocation -> - other.allocations.any { otherAllocation -> - allocation.endpointId == otherAllocation.endpointId && - allocation.mediaSource?.primarySSRC == - otherAllocation.mediaSource?.primarySSRC && - allocation.targetLayer?.index == otherAllocation.targetLayer?.index - } + fun isTheSameAs(other: BandwidthAllocation) = allocations.size == other.allocations.size && + oversending == other.oversending && + allocations.all { allocation -> + other.allocations.any { otherAllocation -> + allocation.endpointId == otherAllocation.endpointId && + allocation.mediaSource?.primarySSRC == + otherAllocation.mediaSource?.primarySSRC && + allocation.targetLayer?.index == otherAllocation.targetLayer?.index } + } override fun toString(): String = "oversending=$oversending " + allocations.joinToString() diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt index 4d4eb0341e..b26e141600 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt @@ -297,26 +297,25 @@ internal class BandwidthAllocator( } @Synchronized - private fun createAllocations( - conferenceMediaSources: List - ): List = conferenceMediaSources.map { source -> - SingleSourceAllocation( - source.owner, - source, - // Note that we use the effective constraints and not the receiver's constraints - // directly. This means we never even try to allocate bitrate to sources "outside - // lastN". For example, if LastN=1 and the first endpoint sends a non-scalable - // stream with bitrate higher that the available bandwidth, we will forward no - // video at all instead of going to the second endpoint in the list. - // I think this is not desired behavior. However, it is required for the "effective - // constraints" to work as designed. - effectiveConstraints[source]!!, - allocationSettings.onStageSources.contains(source.sourceName), - diagnosticContext, - clock, - logger - ) - }.toList() + private fun createAllocations(conferenceMediaSources: List): List = + conferenceMediaSources.map { source -> + SingleSourceAllocation( + source.owner, + source, + // Note that we use the effective constraints and not the receiver's constraints + // directly. This means we never even try to allocate bitrate to sources "outside + // lastN". For example, if LastN=1 and the first endpoint sends a non-scalable + // stream with bitrate higher that the available bandwidth, we will forward no + // video at all instead of going to the second endpoint in the list. + // I think this is not desired behavior. However, it is required for the "effective + // constraints" to work as designed. + effectiveConstraints[source]!!, + allocationSettings.onStageSources.contains(source.sourceName), + diagnosticContext, + clock, + logger + ) + }.toList() /** * Expire this bandwidth allocator. diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt index f21f02751d..851468a01c 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt @@ -181,9 +181,8 @@ class BitrateController @JvmOverloads constructor( * Query whether this source is on stage or selected, as of the most recent * video constraints */ - fun isOnStageOrSelected(source: MediaSourceDesc) = - allocationSettings.onStageSources.contains(source.sourceName) || - allocationSettings.selectedSources.contains(source.sourceName) + fun isOnStageOrSelected(source: MediaSourceDesc) = allocationSettings.onStageSources.contains(source.sourceName) || + allocationSettings.selectedSources.contains(source.sourceName) /** * Query whether this allocator has non-zero effective constraints for a given source diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/PacketHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/PacketHandler.kt index 59f9ebe2f0..ed0d2c040c 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/PacketHandler.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/PacketHandler.kt @@ -173,7 +173,9 @@ internal class PacketHandler( fun timeSinceFirstMedia(): Duration = firstMedia?.let { Duration.between(it, clock.instant()) } ?: Duration.ZERO - fun addPayloadType(payloadType: PayloadType) { payloadTypes[payloadType.pt] = payloadType } + fun addPayloadType(payloadType: PayloadType) { + payloadTypes[payloadType.pt] = payloadType + } val debugState: JSONObject get() = JSONObject().apply { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/Prioritize.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/Prioritize.kt index d265c8ac1b..e76a5be00b 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/Prioritize.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/Prioritize.kt @@ -60,8 +60,10 @@ fun prioritize( /** * Return the "effective" constraints for the given media sources, i.e. the constraints adjusted for LastN. */ -fun getEffectiveConstraints(sources: List, allocationSettings: AllocationSettings): - EffectiveConstraintsMap { +fun getEffectiveConstraints( + sources: List, + allocationSettings: AllocationSettings +): EffectiveConstraintsMap { // FIXME figure out before merge - is using source count instead of endpoints // Add 1 for the receiver endpoint, which is not in the list. val effectiveLastN = effectiveLastN(allocationSettings.lastN, sources.size + 1) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt index 44a171a83b..509b42616d 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt @@ -285,10 +285,7 @@ internal class SingleSourceAllocation( * require a high frame rate, with preconfigured values for the "preferred" height and frame rate, and we do not allow * oversending. */ - private fun selectLayersForCamera( - layers: List, - constraints: VideoConstraints, - ): Layers { + private fun selectLayersForCamera(layers: List, constraints: VideoConstraints): Layers { val minHeight = layers.map { it.layer.height }.minOrNull() ?: return Layers.noLayers val noActiveLayers = layers.none { (_, bitrate) -> bitrate > 0 } val (preferredHeight, preferredFps) = getPreferred(constraints) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/VideoConstraints.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/VideoConstraints.kt index 1e76700348..7e1e8ecc60 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/VideoConstraints.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/VideoConstraints.kt @@ -46,5 +46,4 @@ data class VideoConstraints @JvmOverloads constructor( } } -fun Map.prettyPrint(): String = - entries.joinToString { "${it.key}->${it.value.maxHeight}" } +fun Map.prettyPrint(): String = entries.joinToString { "${it.key}->${it.value.maxHeight}" } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt index 57376b0bdb..1fe7d7b1e8 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt @@ -81,11 +81,7 @@ class Vp9AdaptiveSourceProjectionContext( private var lastPicIdIndexResumption = -1 @Synchronized - override fun accept( - packetInfo: PacketInfo, - incomingIndex: Int, - targetIndex: Int - ): Boolean { + override fun accept(packetInfo: PacketInfo, incomingIndex: Int, targetIndex: Int): Boolean { val packet = packetInfo.packet if (packet !is Vp9Packet) { logger.warn("Packet is not Vp9 packet") @@ -203,42 +199,36 @@ class Vp9AdaptiveSourceProjectionContext( return seqGap } - private fun frameIsNewSsrc(frame: Vp9Frame): Boolean = - lastVp9FrameProjection.vp9Frame?.matchesSSRC(frame) != true + private fun frameIsNewSsrc(frame: Vp9Frame): Boolean = lastVp9FrameProjection.vp9Frame?.matchesSSRC(frame) != true /** * Find the previous frame before the given one. */ @Synchronized - private fun prevFrame(frame: Vp9Frame) = - vp9PictureMaps.get(frame.ssrc)?.prevFrame(frame) + private fun prevFrame(frame: Vp9Frame) = vp9PictureMaps.get(frame.ssrc)?.prevFrame(frame) /** * Find the next frame after the given one. */ @Synchronized - private fun nextFrame(frame: Vp9Frame) = - vp9PictureMaps.get(frame.ssrc)?.nextFrame(frame) + private fun nextFrame(frame: Vp9Frame) = vp9PictureMaps.get(frame.ssrc)?.nextFrame(frame) /** * Find the previous accepted frame before the given one. */ - private fun findPrevAcceptedFrame(frame: Vp9Frame) = - vp9PictureMaps.get(frame.ssrc)?.findPrevAcceptedFrame(frame) + private fun findPrevAcceptedFrame(frame: Vp9Frame) = vp9PictureMaps.get(frame.ssrc)?.findPrevAcceptedFrame(frame) /** * Find the next accepted frame after the given one. */ - private fun findNextAcceptedFrame(frame: Vp9Frame) = - vp9PictureMaps.get(frame.ssrc)?.findNextAcceptedFrame(frame) + private fun findNextAcceptedFrame(frame: Vp9Frame) = vp9PictureMaps.get(frame.ssrc)?.findNextAcceptedFrame(frame) /** * Find a subsequent base-layer TL0 frame after the given frame * @param frame The frame to query * @return A subsequent base-layer TL0 frame, or null */ - private fun findNextBaseTl0(frame: Vp9Frame) = - vp9PictureMaps.get(frame.ssrc)?.findNextBaseTl0(frame) + private fun findNextBaseTl0(frame: Vp9Frame) = vp9PictureMaps.get(frame.ssrc)?.findNextBaseTl0(frame) /** * Create a projection for this frame. diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Frame.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Frame.kt index 98387f0d7d..528e3debcd 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Frame.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Frame.kt @@ -185,7 +185,9 @@ class Vp9Frame internal constructor( get() = if (spatialLayer >= 0) spatialLayer else 0 // Validate that the index matches the pictureId - init { assert((index and 0x7fff) == pictureId) } + init { + assert((index and 0x7fff) == pictureId) + } constructor(packet: Vp9Packet, index: Int) : this( ssrc = packet.ssrc, diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Picture.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Picture.kt index fad626785e..7905db39f1 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Picture.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Picture.kt @@ -42,8 +42,7 @@ class Vp9Picture(packet: Vp9Packet, index: Int) { fun frame(packet: Vp9Packet) = frame(packet.effectiveSpatialLayerIndex) - private fun setFrameAtSid(frame: Vp9Frame, sid: Int) = - frames.setAndExtend(sid, frame, null) + private fun setFrameAtSid(frame: Vp9Frame, sid: Int) = frames.setAndExtend(sid, frame, null) /** * Return the first (lowest-sid, earliest in decoding order) frame that we've received so far. diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilter.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilter.kt index 79ea8c63d7..c17e94665b 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilter.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilter.kt @@ -337,10 +337,7 @@ internal class Vp9QualityFilter(parentLogger: Logger) { * @return true to accept the VP9 keyframe, otherwise false. */ @Synchronized - private fun acceptKeyframe( - incomingIndex: Int, - receivedTime: Instant? - ): Boolean { + private fun acceptKeyframe(incomingIndex: Int, receivedTime: Instant?): Boolean { val encodingIdOfKeyframe = getEidFromIndex(incomingIndex) // This branch writes the {@link #currentSpatialLayerId} and it // determines whether or not we should switch to another simulcast diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt index 5a4860d3f2..ad51657edd 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt @@ -344,10 +344,7 @@ class Colibri2ConferenceHandler( * the conference-modified. */ @Throws(IqProcessingException::class) - private fun handleColibri2Relay( - c2relay: Colibri2Relay, - ignoreUnknownRelays: Boolean - ): Colibri2Relay { + private fun handleColibri2Relay(c2relay: Colibri2Relay, ignoreUnknownRelays: Boolean): Colibri2Relay { val respBuilder = Colibri2Relay.getBuilder() respBuilder.setId(c2relay.id) if (c2relay.expire) { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadManager.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadManager.kt index 0e6b60c873..991b0ca5b6 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadManager.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadManager.kt @@ -89,8 +89,7 @@ class JvbLoadManager @JvmOverloads constructor( } } - fun getCurrentStressLevel(): Double = - mostRecentLoadMeasurement?.div(jvbLoadThreshold) ?: 0.0 + fun getCurrentStressLevel(): Double = mostRecentLoadMeasurement?.div(jvbLoadThreshold) ?: 0.0 fun getStats() = OrderedJsonObject().apply { put("state", state.toString()) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt index 7ea41014e9..f7f887e84c 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt @@ -301,8 +301,7 @@ class EndpointConnectionStatusMessage( /** * Serialize manually because it's faster than Jackson. */ - override fun createJson(): String = - """{"colibriClass":"$TYPE","endpoint":"$endpoint","active":"$active"}""" + override fun createJson(): String = """{"colibriClass":"$TYPE","endpoint":"$endpoint","active":"$active"}""" companion object { const val TYPE = "EndpointConnectivityStatusChangeEvent" @@ -442,18 +441,19 @@ class SenderSourceConstraintsMessage( */ class AddReceiverMessage( val bridgeId: String, - val endpointId: String?, // Used in single stream per endpoint mode and wil be removed - val sourceName: String?, // Used in the multi-stream mode + // Used in single stream per endpoint mode and wil be removed + val endpointId: String?, + // Used in the multi-stream mode + val sourceName: String?, val videoConstraints: VideoConstraints ) : BridgeChannelMessage() { /** * Serialize manually because it's faster than Jackson. */ - override fun createJson(): String = - "{\"colibriClass\":\"$TYPE\",\"bridgeId\":\"$bridgeId\"," + - (if (endpointId != null) "\"endpointId\":\"$endpointId\"," else "") + - (if (sourceName != null) "\"sourceName\":\"$sourceName\"," else "") + - "\"videoConstraints\":$videoConstraints}" + override fun createJson(): String = "{\"colibriClass\":\"$TYPE\",\"bridgeId\":\"$bridgeId\"," + + (if (endpointId != null) "\"endpointId\":\"$endpointId\"," else "") + + (if (sourceName != null) "\"sourceName\":\"$sourceName\"," else "") + + "\"videoConstraints\":$videoConstraints}" companion object { const val TYPE = "AddReceiver" @@ -471,8 +471,7 @@ class RemoveReceiverMessage( /** * Serialize manually because it's faster than Jackson. */ - override fun createJson(): String = - """{"colibriClass":"$TYPE","bridgeId":"$bridgeId","endpointId":"$endpointId"}""" + override fun createJson(): String = """{"colibriClass":"$TYPE","bridgeId":"$bridgeId","endpointId":"$endpointId"}""" companion object { const val TYPE = "RemoveReceiver" diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt index 699dc17bd4..73c09aed94 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt @@ -446,7 +446,8 @@ class RelayMessageTransport( conference.sendMessage( message, listOf(targetEndpoint), - false // sendToRelays + // sendToRelays + false ) } return null diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt index 64abaa4e6e..079fe2dd8f 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt @@ -150,7 +150,8 @@ class RelayedEndpoint( AddReceiverMessage( RelayConfig.config.relayId, id, - null, // source name - used in multi-stream + // source name - used in multi-stream + null, maxVideoConstraints ) ) @@ -160,7 +161,8 @@ class RelayedEndpoint( relay.sendMessage( AddReceiverMessage( RelayConfig.config.relayId, - null, // Endpoint ID - will be removed + // Endpoint ID - will be removed + null, sourceName, maxVideoConstraints ) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/dtls/DtlsTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/dtls/DtlsTransport.kt index eb7ce55d96..ed9c2276a0 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/dtls/DtlsTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/dtls/DtlsTransport.kt @@ -173,14 +173,12 @@ class DtlsTransport(parentLogger: Logger) { /** * Notify this layer that DTLS data has been received from the network */ - fun dtlsDataReceived(data: ByteArray, off: Int, len: Int) = - dtlsStack.processIncomingProtocolData(data, off, len) + fun dtlsDataReceived(data: ByteArray, off: Int, len: Int) = dtlsStack.processIncomingProtocolData(data, off, len) /** * Send out DTLS data */ - fun sendDtlsData(data: ByteArray, off: Int, len: Int) = - dtlsStack.sendApplicationData(data, off, len) + fun sendDtlsData(data: ByteArray, off: Int, len: Int) = dtlsStack.sendApplicationData(data, off, len) fun stop() { if (running.compareAndSet(true, false)) { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt index a1fca51284..328e775991 100755 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt @@ -290,10 +290,7 @@ class IceTransport @JvmOverloads constructor( * @return the number of network reachable remote candidates contained in * the given list of candidates. */ - private fun addRemoteCandidates( - remoteCandidates: List, - iceAgentIsRunning: Boolean - ): Int { + private fun addRemoteCandidates(remoteCandidates: List, iceAgentIsRunning: Boolean): Int { var remoteCandidateCount = 0 // Sort the remote candidates (host < reflexive < relayed) in order to // create first the host, then the reflexive, the relayed candidates and @@ -497,11 +494,9 @@ private data class IceProcessingStateTransition( } } -private fun IceMediaStream.remoteUfragAndPasswordKnown(): Boolean = - remoteUfrag != null && remotePassword != null +private fun IceMediaStream.remoteUfragAndPasswordKnown(): Boolean = remoteUfrag != null && remotePassword != null -private fun CandidatePacketExtension.ipNeedsResolution(): Boolean = - !InetAddresses.isInetAddress(ip) +private fun CandidatePacketExtension.ipNeedsResolution(): Boolean = !InetAddresses.isInetAddress(ip) private fun TransportAddress.isPrivateAddress(): Boolean = address.isSiteLocalAddress || /* 0xfc00::/7 */ diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/util/PayloadTypeUtil.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/util/PayloadTypeUtil.kt index 24503bec86..026a07980c 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/util/PayloadTypeUtil.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/util/PayloadTypeUtil.kt @@ -60,10 +60,7 @@ class PayloadTypeUtil { * @param ext the XML extension which describes the payload type. */ @JvmStatic - fun create( - ext: PayloadTypePacketExtension, - mediaType: MediaType - ): PayloadType? { + fun create(ext: PayloadTypePacketExtension, mediaType: MediaType): PayloadType? { val parameters: MutableMap = ConcurrentHashMap() ext.parameters.forEach { parameter -> // In SDP, format parameters don't necessarily come in name=value pairs (see e.g. the format used in diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/version/JvbVersionService.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/version/JvbVersionService.kt index ca80831fc9..7d1ee13e28 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/version/JvbVersionService.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/version/JvbVersionService.kt @@ -35,8 +35,8 @@ class JvbVersionService : VersionService { override fun parseVersionString(version: String?): Version { val matcher = Pattern.compile("(\\d*)\\.(\\d*)-(.*)").matcher(version ?: "").apply { find() } - val majorVersion = matcher.groupOrNull(1)?.toInt() ?: defaultMajorVersion - val minorVersion = matcher.groupOrNull(2)?.toInt() ?: defaultMinorVersion + val majorVersion = matcher.groupOrNull(1)?.toInt() ?: DEFAULT_MAJOR_VERSION + val minorVersion = matcher.groupOrNull(2)?.toInt() ?: DEFAULT_MINOR_VERSION val buildId = matcher.groupOrNull(3) ?: defaultBuildId return VersionImpl( @@ -48,8 +48,8 @@ class JvbVersionService : VersionService { } companion object { - private const val defaultMajorVersion = 2 - private const val defaultMinorVersion = 1 + private const val DEFAULT_MAJOR_VERSION = 2 + private const val DEFAULT_MINOR_VERSION = 1 private val defaultBuildId: String? = null } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/ColibriWebSocketService.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/ColibriWebSocketService.kt index 6ed779a0f0..f4e0c56367 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/ColibriWebSocketService.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/ColibriWebSocketService.kt @@ -79,10 +79,7 @@ class ColibriWebSocketService( return relayBaseUrls.map { "$it/$conferenceId/$relayId?pwd=$pwd" } } - fun registerServlet( - servletContextHandler: ServletContextHandler, - videobridge: Videobridge - ) { + fun registerServlet(servletContextHandler: ServletContextHandler, videobridge: Videobridge) { if (config.enabled) { logger.info("Registering servlet with baseUrls = $baseUrls, relayBaseUrls = $relayBaseUrls") val holder = ServletHolder().apply { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt index f4367dd308..7833a4794d 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt @@ -139,11 +139,10 @@ class XmppConnection : IQListener { return true } - private fun MucClientConfiguration.matches(other: MucClientConfiguration) = - hostname == other.hostname && - port == other.port && - domain == other.domain && - username == other.username + private fun MucClientConfiguration.matches(other: MucClientConfiguration) = hostname == other.hostname && + port == other.port && + domain == other.domain && + username == other.username /** * Returns ids of [MucClient] that have been added. diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/EndpointConnectionStatusMonitorTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/EndpointConnectionStatusMonitorTest.kt index 2784fd83bf..bd3cce7740 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/EndpointConnectionStatusMonitorTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/EndpointConnectionStatusMonitorTest.kt @@ -202,7 +202,7 @@ class EndpointConnectionStatusMonitorTest : ShouldSpec({ } } context("and then a new ep joins") { - every { conference.getLocalEndpoint("4") } returns mockk() { every { id } returns "4" } + every { conference.getLocalEndpoint("4") } returns mockk { every { id } returns "4" } monitor.endpointConnected("4") should("update the new endpoint of the other non-visitor endpoints' statuses") { sendMessageCalls shouldHaveSize 2 diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerPerfTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerPerfTest.kt index 44d2c5b946..6b7058fbfd 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerPerfTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerPerfTest.kt @@ -79,7 +79,8 @@ class BitrateControllerPerfTest : StringSpec() { Supplier { endpoints.toList() }, DiagnosticContext(), createLogger(), - false, // TODO cover the case for true? + // TODO cover the case for true? + false, clock, ).apply { // The BC only starts working 10 seconds after it first received media, so fake that. diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt index 494e5d6e6a..bd9435c513 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt @@ -56,10 +56,10 @@ class BitrateControllerTest : ShouldSpec() { private val logger = createLogger() private val clock = FakeClock() private val bc = BitrateControllerWrapper(createEndpoints("A", "B", "C", "D"), clock = clock) - private val A = bc.endpoints.find { it.id == "A" }!! as TestEndpoint - private val B = bc.endpoints.find { it.id == "B" }!! as TestEndpoint - private val C = bc.endpoints.find { it.id == "C" }!! as TestEndpoint - private val D = bc.endpoints.find { it.id == "D" }!! as TestEndpoint + private val a = bc.endpoints.find { it.id == "A" }!! as TestEndpoint + private val b = bc.endpoints.find { it.id == "B" }!! as TestEndpoint + private val c = bc.endpoints.find { it.id == "C" }!! as TestEndpoint + private val d = bc.endpoints.find { it.id == "D" }!! as TestEndpoint override suspend fun beforeSpec(spec: Spec) = super.beforeSpec(spec).also { // We disable the threshold, causing [BandwidthAllocator] to make a new decision every time BWE changes. This is @@ -131,9 +131,9 @@ class BitrateControllerTest : ShouldSpec() { listOf(true, false).forEach { screensharing -> context("With ${if (screensharing) "screensharing" else "camera"}") { if (screensharing) { - A.mediaSources[0].videoType = VideoType.DESKTOP + a.mediaSources[0].videoType = VideoType.DESKTOP } - bc.setEndpointOrdering(A, B, C, D) + bc.setEndpointOrdering(a, b, c, d) bc.setStageView("A-v0") bc.bc.allocationSettings.lastN shouldBe -1 @@ -147,7 +147,7 @@ class BitrateControllerTest : ShouldSpec() { } } context("and a non-dominant speaker is on stage") { - bc.setEndpointOrdering(B, A, C, D) + bc.setEndpointOrdering(b, a, c, d) bc.setStageView("A-v0") bc.bc.allocationSettings.lastN shouldBe -1 @@ -160,7 +160,7 @@ class BitrateControllerTest : ShouldSpec() { } context("When LastN=0") { // LastN=0 is used when the client goes in "audio-only" mode. - bc.setEndpointOrdering(A, B, C, D) + bc.setEndpointOrdering(a, b, c, d) bc.setStageView("A", lastN = 0) bc.bc.allocationSettings.lastN shouldBe 0 @@ -174,7 +174,7 @@ class BitrateControllerTest : ShouldSpec() { context("When LastN=1") { // LastN=1 is used when the client goes in "audio-only" mode, but someone starts a screenshare. context("and the dominant speaker is on-stage") { - bc.setEndpointOrdering(A, B, C, D) + bc.setEndpointOrdering(a, b, c, d) bc.setStageView("A-v0", lastN = 1) bc.bc.allocationSettings.lastN shouldBe 1 @@ -186,7 +186,7 @@ class BitrateControllerTest : ShouldSpec() { verifyStageViewLastN1() } context("and a non-dominant speaker is on stage") { - bc.setEndpointOrdering(B, A, C, D) + bc.setEndpointOrdering(b, a, c, d) bc.setStageView("A-v0", lastN = 1) bc.bc.allocationSettings.lastN shouldBe 1 @@ -200,7 +200,7 @@ class BitrateControllerTest : ShouldSpec() { } } context("Tile view") { - bc.setEndpointOrdering(A, B, C, D) + bc.setEndpointOrdering(a, b, c, d) bc.setTileView("A-v0", "B-v0", "C-v0", "D-v0") bc.bc.allocationSettings.lastN shouldBe -1 @@ -226,7 +226,7 @@ class BitrateControllerTest : ShouldSpec() { } } context("Tile view 360p") { - bc.setEndpointOrdering(A, B, C, D) + bc.setEndpointOrdering(a, b, c, d) bc.setTileView("A-v0", "B-v0", "C-v0", "D-v0", maxFrameHeight = 360) bc.bc.allocationSettings.lastN shouldBe -1 @@ -257,7 +257,7 @@ class BitrateControllerTest : ShouldSpec() { // A is dominant speaker, A and B are selected. With LastN=2 we should always forward the selected // sources regardless of who is speaking. // The exact flow of this scenario was taken from a (non-jitsi-meet) client. - bc.setEndpointOrdering(A, B, C, D) + bc.setEndpointOrdering(a, b, c, d) bc.bc.setBandwidthAllocationSettings( ReceiverVideoConstraintsMessage( selectedSources = listOf("A-v0", "B-v0"), @@ -289,7 +289,7 @@ class BitrateControllerTest : ShouldSpec() { clock.elapse(2.secs) // B becomes dominant speaker. - bc.setEndpointOrdering(B, A, C, D) + bc.setEndpointOrdering(b, a, c, d) bc.forwardedSourcesHistory.last().event.shouldBe(setOf("A-v0", "B-v0")) clock.elapse(2.secs) @@ -334,7 +334,7 @@ class BitrateControllerTest : ShouldSpec() { clock.elapse(2.secs) // D is now dominant speaker, but it should not override the selected endpoints. - bc.setEndpointOrdering(D, B, A, C) + bc.setEndpointOrdering(d, b, a, c) bc.forwardedSourcesHistory.last().event.shouldBe(setOf("A-v0", "B-v0")) bc.bwe = 10.mbps @@ -348,7 +348,7 @@ class BitrateControllerTest : ShouldSpec() { clock.elapse(2.secs) // C is now dominant speaker, but it should not override the selected endpoints. - bc.setEndpointOrdering(C, D, A, B) + bc.setEndpointOrdering(c, d, a, b) bc.forwardedSourcesHistory.last().event.shouldBe(setOf("A-v0", "B-v0")) } } @@ -391,10 +391,10 @@ class BitrateControllerTest : ShouldSpec() { 0.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = sd30), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ), oversending = true ) @@ -403,10 +403,10 @@ class BitrateControllerTest : ShouldSpec() { 160.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd7_5), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = hd7_5), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ), oversending = true ) @@ -415,10 +415,10 @@ class BitrateControllerTest : ShouldSpec() { 660.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd7_5), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = hd7_5), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ), oversending = false ) @@ -427,10 +427,10 @@ class BitrateControllerTest : ShouldSpec() { 1320.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd15), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = hd15), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -438,10 +438,10 @@ class BitrateControllerTest : ShouldSpec() { 2000.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd30), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -449,10 +449,10 @@ class BitrateControllerTest : ShouldSpec() { 2050.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd30), - SingleAllocation(B, targetLayer = ld7_5), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -460,10 +460,10 @@ class BitrateControllerTest : ShouldSpec() { 2100.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd30), - SingleAllocation(B, targetLayer = ld7_5), - SingleAllocation(C, targetLayer = ld7_5), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -471,10 +471,10 @@ class BitrateControllerTest : ShouldSpec() { 2150.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd30), - SingleAllocation(B, targetLayer = ld7_5), - SingleAllocation(C, targetLayer = ld7_5), - SingleAllocation(D, targetLayer = ld7_5) + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) ) ) ), @@ -482,10 +482,10 @@ class BitrateControllerTest : ShouldSpec() { 2200.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd30), - SingleAllocation(B, targetLayer = ld15), - SingleAllocation(C, targetLayer = ld7_5), - SingleAllocation(D, targetLayer = ld7_5) + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) ) ) ), @@ -493,10 +493,10 @@ class BitrateControllerTest : ShouldSpec() { 2250.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd30), - SingleAllocation(B, targetLayer = ld15), - SingleAllocation(C, targetLayer = ld15), - SingleAllocation(D, targetLayer = ld7_5) + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld7_5) ) ) ), @@ -504,10 +504,10 @@ class BitrateControllerTest : ShouldSpec() { 2300.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd30), - SingleAllocation(B, targetLayer = ld15), - SingleAllocation(C, targetLayer = ld15), - SingleAllocation(D, targetLayer = ld15) + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) ) ) ), @@ -515,10 +515,10 @@ class BitrateControllerTest : ShouldSpec() { 2350.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd30), - SingleAllocation(B, targetLayer = ld30), - SingleAllocation(C, targetLayer = ld15), - SingleAllocation(D, targetLayer = ld15) + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) ) ) ), @@ -526,10 +526,10 @@ class BitrateControllerTest : ShouldSpec() { 2400.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd30), - SingleAllocation(B, targetLayer = ld30), - SingleAllocation(C, targetLayer = ld30), - SingleAllocation(D, targetLayer = ld15) + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld15) ) ) ), @@ -537,10 +537,10 @@ class BitrateControllerTest : ShouldSpec() { 2460.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd30), - SingleAllocation(B, targetLayer = ld30), - SingleAllocation(C, targetLayer = ld30), - SingleAllocation(D, targetLayer = ld30) + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld30) ) ) ) @@ -582,10 +582,10 @@ class BitrateControllerTest : ShouldSpec() { 50.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld7_5), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ), oversending = false ) @@ -594,10 +594,10 @@ class BitrateControllerTest : ShouldSpec() { 100.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld15), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -605,10 +605,10 @@ class BitrateControllerTest : ShouldSpec() { 150.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld30), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -616,10 +616,10 @@ class BitrateControllerTest : ShouldSpec() { 500.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = sd30), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -627,10 +627,10 @@ class BitrateControllerTest : ShouldSpec() { 550.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = sd30), - SingleAllocation(B, targetLayer = ld7_5), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -638,10 +638,10 @@ class BitrateControllerTest : ShouldSpec() { 600.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = sd30), - SingleAllocation(B, targetLayer = ld7_5), - SingleAllocation(C, targetLayer = ld7_5), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -649,10 +649,10 @@ class BitrateControllerTest : ShouldSpec() { 650.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = sd30), - SingleAllocation(B, targetLayer = ld7_5), - SingleAllocation(C, targetLayer = ld7_5), - SingleAllocation(D, targetLayer = ld7_5) + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) ) ) ), @@ -660,10 +660,10 @@ class BitrateControllerTest : ShouldSpec() { 700.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = sd30), - SingleAllocation(B, targetLayer = ld15), - SingleAllocation(C, targetLayer = ld7_5), - SingleAllocation(D, targetLayer = ld7_5) + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) ) ) ), @@ -671,10 +671,10 @@ class BitrateControllerTest : ShouldSpec() { 750.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = sd30), - SingleAllocation(B, targetLayer = ld15), - SingleAllocation(C, targetLayer = ld15), - SingleAllocation(D, targetLayer = ld7_5) + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld7_5) ) ) ), @@ -682,10 +682,10 @@ class BitrateControllerTest : ShouldSpec() { 800.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = sd30), - SingleAllocation(B, targetLayer = ld15), - SingleAllocation(C, targetLayer = ld15), - SingleAllocation(D, targetLayer = ld15) + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) ) ) ), @@ -693,10 +693,10 @@ class BitrateControllerTest : ShouldSpec() { 850.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = sd30), - SingleAllocation(B, targetLayer = ld30), - SingleAllocation(C, targetLayer = ld15), - SingleAllocation(D, targetLayer = ld15) + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) ) ) ), @@ -704,10 +704,10 @@ class BitrateControllerTest : ShouldSpec() { 900.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = sd30), - SingleAllocation(B, targetLayer = ld30), - SingleAllocation(C, targetLayer = ld30), - SingleAllocation(D, targetLayer = ld15) + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld15) ) ) ), @@ -715,10 +715,10 @@ class BitrateControllerTest : ShouldSpec() { 960.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = sd30), - SingleAllocation(B, targetLayer = ld30), - SingleAllocation(C, targetLayer = ld30), - SingleAllocation(D, targetLayer = ld30) + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld30) ) ) ), @@ -726,10 +726,10 @@ class BitrateControllerTest : ShouldSpec() { 2150.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd30), - SingleAllocation(B, targetLayer = ld7_5), - SingleAllocation(C, targetLayer = ld7_5), - SingleAllocation(D, targetLayer = ld7_5) + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) ) ) ), @@ -737,10 +737,10 @@ class BitrateControllerTest : ShouldSpec() { 2200.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd30), - SingleAllocation(B, targetLayer = ld15), - SingleAllocation(C, targetLayer = ld7_5), - SingleAllocation(D, targetLayer = ld7_5) + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) ) ) ), @@ -748,10 +748,10 @@ class BitrateControllerTest : ShouldSpec() { 2250.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd30), - SingleAllocation(B, targetLayer = ld15), - SingleAllocation(C, targetLayer = ld15), - SingleAllocation(D, targetLayer = ld7_5) + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld7_5) ) ) ), @@ -759,10 +759,10 @@ class BitrateControllerTest : ShouldSpec() { 2300.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd30), - SingleAllocation(B, targetLayer = ld15), - SingleAllocation(C, targetLayer = ld15), - SingleAllocation(D, targetLayer = ld15) + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) ) ) ), @@ -770,10 +770,10 @@ class BitrateControllerTest : ShouldSpec() { 2350.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd30), - SingleAllocation(B, targetLayer = ld30), - SingleAllocation(C, targetLayer = ld15), - SingleAllocation(D, targetLayer = ld15) + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) ) ) ), @@ -781,10 +781,10 @@ class BitrateControllerTest : ShouldSpec() { 2400.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd30), - SingleAllocation(B, targetLayer = ld30), - SingleAllocation(C, targetLayer = ld30), - SingleAllocation(D, targetLayer = ld15) + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld15) ) ) ), @@ -792,10 +792,10 @@ class BitrateControllerTest : ShouldSpec() { 2460.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd30), - SingleAllocation(B, targetLayer = ld30), - SingleAllocation(C, targetLayer = ld30), - SingleAllocation(D, targetLayer = ld30) + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld30) ) ) ) @@ -817,10 +817,10 @@ class BitrateControllerTest : ShouldSpec() { bc.allocationHistory.last().event.shouldMatch( BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = noVideo), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = noVideo), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ) @@ -851,10 +851,10 @@ class BitrateControllerTest : ShouldSpec() { 50.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld7_5), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -862,10 +862,10 @@ class BitrateControllerTest : ShouldSpec() { 100.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld15), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -873,10 +873,10 @@ class BitrateControllerTest : ShouldSpec() { 150.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld30), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -884,10 +884,10 @@ class BitrateControllerTest : ShouldSpec() { 500.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = sd30), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -895,10 +895,10 @@ class BitrateControllerTest : ShouldSpec() { 2010.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = hd30), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ) @@ -921,10 +921,10 @@ class BitrateControllerTest : ShouldSpec() { (-1).bps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = noVideo), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = noVideo), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -932,10 +932,10 @@ class BitrateControllerTest : ShouldSpec() { 50.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld7_5), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -943,10 +943,10 @@ class BitrateControllerTest : ShouldSpec() { 100.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld7_5), - SingleAllocation(B, targetLayer = ld7_5), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -954,10 +954,10 @@ class BitrateControllerTest : ShouldSpec() { 150.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld7_5), - SingleAllocation(B, targetLayer = ld7_5), - SingleAllocation(C, targetLayer = ld7_5), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -965,10 +965,10 @@ class BitrateControllerTest : ShouldSpec() { 200.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld7_5), - SingleAllocation(B, targetLayer = ld7_5), - SingleAllocation(C, targetLayer = ld7_5), - SingleAllocation(D, targetLayer = ld7_5) + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) ) ) ), @@ -976,10 +976,10 @@ class BitrateControllerTest : ShouldSpec() { 250.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld15), - SingleAllocation(B, targetLayer = ld7_5), - SingleAllocation(C, targetLayer = ld7_5), - SingleAllocation(D, targetLayer = ld7_5) + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) ) ) ), @@ -987,10 +987,10 @@ class BitrateControllerTest : ShouldSpec() { 300.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld15), - SingleAllocation(B, targetLayer = ld15), - SingleAllocation(C, targetLayer = ld7_5), - SingleAllocation(D, targetLayer = ld7_5) + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) ) ) ), @@ -998,10 +998,10 @@ class BitrateControllerTest : ShouldSpec() { 350.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld15), - SingleAllocation(B, targetLayer = ld15), - SingleAllocation(C, targetLayer = ld15), - SingleAllocation(D, targetLayer = ld7_5) + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld7_5) ) ) ), @@ -1009,10 +1009,10 @@ class BitrateControllerTest : ShouldSpec() { 400.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld15), - SingleAllocation(B, targetLayer = ld15), - SingleAllocation(C, targetLayer = ld15), - SingleAllocation(D, targetLayer = ld15) + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) ) ) ), @@ -1020,10 +1020,10 @@ class BitrateControllerTest : ShouldSpec() { 450.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld30), - SingleAllocation(B, targetLayer = ld15), - SingleAllocation(C, targetLayer = ld15), - SingleAllocation(D, targetLayer = ld15) + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) ) ) ), @@ -1031,10 +1031,10 @@ class BitrateControllerTest : ShouldSpec() { 500.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld30), - SingleAllocation(B, targetLayer = ld30), - SingleAllocation(C, targetLayer = ld15), - SingleAllocation(D, targetLayer = ld15) + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) ) ) ), @@ -1042,10 +1042,10 @@ class BitrateControllerTest : ShouldSpec() { 550.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld30), - SingleAllocation(B, targetLayer = ld30), - SingleAllocation(C, targetLayer = ld30), - SingleAllocation(D, targetLayer = ld15) + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld15) ) ) ), @@ -1053,10 +1053,10 @@ class BitrateControllerTest : ShouldSpec() { 610.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld30), - SingleAllocation(B, targetLayer = ld30), - SingleAllocation(C, targetLayer = ld30), - SingleAllocation(D, targetLayer = ld30) + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld30) ) ) ) @@ -1079,10 +1079,10 @@ class BitrateControllerTest : ShouldSpec() { (-1).bps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = noVideo), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = noVideo), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -1090,10 +1090,10 @@ class BitrateControllerTest : ShouldSpec() { 50.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld7_5), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ), oversending = false ) @@ -1102,10 +1102,10 @@ class BitrateControllerTest : ShouldSpec() { 100.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld7_5), - SingleAllocation(B, targetLayer = ld7_5), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -1113,10 +1113,10 @@ class BitrateControllerTest : ShouldSpec() { 150.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld7_5), - SingleAllocation(B, targetLayer = ld7_5), - SingleAllocation(C, targetLayer = ld7_5), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -1124,10 +1124,10 @@ class BitrateControllerTest : ShouldSpec() { 200.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld7_5), - SingleAllocation(B, targetLayer = ld7_5), - SingleAllocation(C, targetLayer = ld7_5), - SingleAllocation(D, targetLayer = ld7_5) + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) ) ) ), @@ -1135,10 +1135,10 @@ class BitrateControllerTest : ShouldSpec() { 250.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld15), - SingleAllocation(B, targetLayer = ld7_5), - SingleAllocation(C, targetLayer = ld7_5), - SingleAllocation(D, targetLayer = ld7_5) + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) ) ) ), @@ -1146,10 +1146,10 @@ class BitrateControllerTest : ShouldSpec() { 300.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld15), - SingleAllocation(B, targetLayer = ld15), - SingleAllocation(C, targetLayer = ld7_5), - SingleAllocation(D, targetLayer = ld7_5) + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) ) ) ), @@ -1157,10 +1157,10 @@ class BitrateControllerTest : ShouldSpec() { 350.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld15), - SingleAllocation(B, targetLayer = ld15), - SingleAllocation(C, targetLayer = ld15), - SingleAllocation(D, targetLayer = ld7_5) + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld7_5) ) ) ), @@ -1168,10 +1168,10 @@ class BitrateControllerTest : ShouldSpec() { 400.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld15), - SingleAllocation(B, targetLayer = ld15), - SingleAllocation(C, targetLayer = ld15), - SingleAllocation(D, targetLayer = ld15) + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) ) ) ), @@ -1179,10 +1179,10 @@ class BitrateControllerTest : ShouldSpec() { 450.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld30), - SingleAllocation(B, targetLayer = ld15), - SingleAllocation(C, targetLayer = ld15), - SingleAllocation(D, targetLayer = ld15) + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) ) ) ), @@ -1190,10 +1190,10 @@ class BitrateControllerTest : ShouldSpec() { 500.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld30), - SingleAllocation(B, targetLayer = ld30), - SingleAllocation(C, targetLayer = ld15), - SingleAllocation(D, targetLayer = ld15) + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) ) ) ), @@ -1201,10 +1201,10 @@ class BitrateControllerTest : ShouldSpec() { 550.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld30), - SingleAllocation(B, targetLayer = ld30), - SingleAllocation(C, targetLayer = ld30), - SingleAllocation(D, targetLayer = ld15) + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld15) ) ) ), @@ -1212,10 +1212,10 @@ class BitrateControllerTest : ShouldSpec() { 610.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld30), - SingleAllocation(B, targetLayer = ld30), - SingleAllocation(C, targetLayer = ld30), - SingleAllocation(D, targetLayer = ld30) + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld30) ) ) ), @@ -1223,10 +1223,10 @@ class BitrateControllerTest : ShouldSpec() { 960.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = sd30), - SingleAllocation(B, targetLayer = ld30), - SingleAllocation(C, targetLayer = ld30), - SingleAllocation(D, targetLayer = ld30) + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld30) ) ) ), @@ -1234,10 +1234,10 @@ class BitrateControllerTest : ShouldSpec() { 1310.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = sd30), - SingleAllocation(B, targetLayer = sd30), - SingleAllocation(C, targetLayer = ld30), - SingleAllocation(D, targetLayer = ld30) + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = sd30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld30) ) ) ), @@ -1245,10 +1245,10 @@ class BitrateControllerTest : ShouldSpec() { 1660.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = sd30), - SingleAllocation(B, targetLayer = sd30), - SingleAllocation(C, targetLayer = sd30), - SingleAllocation(D, targetLayer = ld30) + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = sd30), + SingleAllocation(c, targetLayer = sd30), + SingleAllocation(d, targetLayer = ld30) ) ) ), @@ -1256,10 +1256,10 @@ class BitrateControllerTest : ShouldSpec() { 2010.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = sd30), - SingleAllocation(B, targetLayer = sd30), - SingleAllocation(C, targetLayer = sd30), - SingleAllocation(D, targetLayer = sd30) + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = sd30), + SingleAllocation(c, targetLayer = sd30), + SingleAllocation(d, targetLayer = sd30) ) ) ) @@ -1287,10 +1287,10 @@ class BitrateControllerTest : ShouldSpec() { (-1).bps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = noVideo), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = noVideo), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ), oversending = true ) @@ -1299,10 +1299,10 @@ class BitrateControllerTest : ShouldSpec() { 50.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld7_5), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ), @@ -1310,21 +1310,22 @@ class BitrateControllerTest : ShouldSpec() { 100.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld15), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ), Event( - 160.kbps, // TODO: why 160 instead of 150? weird. + // TODO: why 160 instead of 150? weird. + 160.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = ld30), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ) @@ -1336,10 +1337,10 @@ class BitrateControllerTest : ShouldSpec() { 510.kbps, BandwidthAllocation( setOf( - SingleAllocation(A, targetLayer = sd30), - SingleAllocation(B, targetLayer = noVideo), - SingleAllocation(C, targetLayer = noVideo), - SingleAllocation(D, targetLayer = noVideo) + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) ) ) ) @@ -1403,7 +1404,8 @@ class BitrateControllerWrapper(initialEndpoints: List, val Supplier { endpoints }, DiagnosticContext(), logger, - true, // TODO merge BitrateControllerNewTest with old and use this flag + // TODO merge BitrateControllerNewTest with old and use this flag + true, clock ) @@ -1423,11 +1425,7 @@ class BitrateControllerWrapper(initialEndpoints: List, val ) } - fun setTileView( - vararg selectedSources: String, - maxFrameHeight: Int = 180, - lastN: Int? = null - ) { + fun setTileView(vararg selectedSources: String, maxFrameHeight: Int = 180, lastN: Int? = null) { bc.setBandwidthAllocationSettings( ReceiverVideoConstraintsMessage( lastN = lastN, @@ -1481,21 +1479,16 @@ fun createSources(vararg ids: String): MutableList { } } -fun createSourceDesc( - ssrc1: Int, - ssrc2: Int, - ssrc3: Int, - sourceName: String, - owner: String -): MediaSourceDesc = MediaSourceDesc( - arrayOf( - RtpEncodingDesc(ssrc1.toLong(), arrayOf(ld7_5, ld15, ld30)), - RtpEncodingDesc(ssrc2.toLong(), arrayOf(sd7_5, sd15, sd30)), - RtpEncodingDesc(ssrc3.toLong(), arrayOf(hd7_5, hd15, hd30)) - ), - sourceName = sourceName, - owner = owner -) +fun createSourceDesc(ssrc1: Int, ssrc2: Int, ssrc3: Int, sourceName: String, owner: String): MediaSourceDesc = + MediaSourceDesc( + arrayOf( + RtpEncodingDesc(ssrc1.toLong(), arrayOf(ld7_5, ld15, ld30)), + RtpEncodingDesc(ssrc2.toLong(), arrayOf(sd7_5, sd15, sd30)), + RtpEncodingDesc(ssrc3.toLong(), arrayOf(hd7_5, hd15, hd30)) + ), + sourceName = sourceName, + owner = owner + ) val bitrateLd = 150.kbps val bitrateSd = 500.kbps diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTraceTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTraceTest.kt index f4a0d20cdf..88b8003aef 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTraceTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTraceTest.kt @@ -38,13 +38,13 @@ class BitrateControllerTraceTest : ShouldSpec() { override fun isolationMode() = IsolationMode.InstancePerLeaf private val clock = FakeClock() - private val A = Endpoint("A") - private val B = Endpoint("B") - private val C = Endpoint("C") - private val D = Endpoint("D") - private val E = Endpoint("E") - private val F = Endpoint("F") - private val bc = BitrateControllerWrapper(listOf(A, B, C, D, E, F), clock = clock).apply { + private val a = Endpoint("A") + private val b = Endpoint("B") + private val c = Endpoint("C") + private val d = Endpoint("D") + private val e = Endpoint("E") + private val f = Endpoint("F") + private val bc = BitrateControllerWrapper(listOf(a, b, c, d, e, f), clock = clock).apply { bc.endpointOrderingChanged() } @@ -57,18 +57,18 @@ class BitrateControllerTraceTest : ShouldSpec() { println("Read ${parsedLines.size} events.") parsedLines.forEach { line -> clock.setTime(line.time) - A.layer7.bitrate = line.bps_a_7.bps - A.layer30.bitrate = line.bps_a_30.bps - B.layer7.bitrate = line.bps_b_7.bps - B.layer30.bitrate = line.bps_b_30.bps - C.layer7.bitrate = line.bps_c_7.bps - C.layer30.bitrate = line.bps_c_30.bps - D.layer7.bitrate = line.bps_d_7.bps - D.layer30.bitrate = line.bps_d_30.bps - E.layer7.bitrate = line.bps_e_7.bps - E.layer30.bitrate = line.bps_e_30.bps - F.layer7.bitrate = line.bps_f_7.bps - F.layer30.bitrate = line.bps_f_30.bps + a.layer7.bitrate = line.bps_a_7.bps + a.layer30.bitrate = line.bps_a_30.bps + b.layer7.bitrate = line.bps_b_7.bps + b.layer30.bitrate = line.bps_b_30.bps + c.layer7.bitrate = line.bps_c_7.bps + c.layer30.bitrate = line.bps_c_30.bps + d.layer7.bitrate = line.bps_d_7.bps + d.layer30.bitrate = line.bps_d_30.bps + e.layer7.bitrate = line.bps_e_7.bps + e.layer30.bitrate = line.bps_e_30.bps + f.layer7.bitrate = line.bps_f_7.bps + f.layer30.bitrate = line.bps_f_30.bps bc.bwe = line.bwe.bps } diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/EffectiveConstraintsTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/EffectiveConstraintsTest.kt index b149d33672..0e572c3dfd 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/EffectiveConstraintsTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/EffectiveConstraintsTest.kt @@ -23,11 +23,7 @@ import org.jitsi.nlj.MediaSourceDesc import org.jitsi.nlj.VideoType import org.jitsi.videobridge.cc.config.BitrateControllerConfig -fun testSource( - endpointId: String, - sourceName: String, - videoType: VideoType = VideoType.CAMERA -): MediaSourceDesc { +fun testSource(endpointId: String, sourceName: String, videoType: VideoType = VideoType.CAMERA): MediaSourceDesc { return MediaSourceDesc( emptyArray(), endpointId, diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocationTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocationTest.kt index 4868548329..209cff377c 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocationTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocationTest.kt @@ -35,15 +35,18 @@ class SingleSourceAllocationTest : ShouldSpec() { private val clock = FakeClock() private val diagnosticContext = DiagnosticContext() - private val ld7_5 = MockRtpLayerDesc(tid = 0, eid = 0, height = 180, frameRate = 7.5, bitrate = bitrateLd * 0.33) + private val ld7point5 = + MockRtpLayerDesc(tid = 0, eid = 0, height = 180, frameRate = 7.5, bitrate = bitrateLd * 0.33) private val ld15 = MockRtpLayerDesc(tid = 1, eid = 0, height = 180, frameRate = 15.0, bitrate = bitrateLd * 0.66) private val ld30 = MockRtpLayerDesc(tid = 2, eid = 0, height = 180, frameRate = 30.0, bitrate = bitrateLd) - private val sd7_5 = MockRtpLayerDesc(tid = 0, eid = 1, height = 360, frameRate = 7.5, bitrate = bitrateSd * 0.33) + private val sd7point5 = + MockRtpLayerDesc(tid = 0, eid = 1, height = 360, frameRate = 7.5, bitrate = bitrateSd * 0.33) private val sd15 = MockRtpLayerDesc(tid = 1, eid = 1, height = 360, frameRate = 15.0, bitrate = bitrateSd * 0.66) private val sd30 = MockRtpLayerDesc(tid = 2, eid = 1, height = 360, frameRate = 30.0, bitrate = bitrateSd) - private val hd7_5 = MockRtpLayerDesc(tid = 0, eid = 2, height = 720, frameRate = 7.5, bitrate = bitrateHd * 0.33) + private val hd7point5 = + MockRtpLayerDesc(tid = 0, eid = 2, height = 720, frameRate = 7.5, bitrate = bitrateHd * 0.33) private val hd15 = MockRtpLayerDesc(tid = 1, eid = 2, height = 720, frameRate = 15.0, bitrate = bitrateHd * 0.66) private val hd30 = MockRtpLayerDesc(tid = 2, eid = 2, height = 720, frameRate = 30.0, bitrate = bitrateHd) @@ -53,12 +56,12 @@ class SingleSourceAllocationTest : ShouldSpec() { val endpointId = "A" val mediaSource = MediaSourceDesc( arrayOf( - RtpEncodingDesc(1L, arrayOf(ld7_5, ld15, ld30)), - RtpEncodingDesc(1L, arrayOf(sd7_5, sd15, sd30)), - RtpEncodingDesc(1L, arrayOf(hd7_5, hd15, hd30)) + RtpEncodingDesc(1L, arrayOf(ld7point5, ld15, ld30)), + RtpEncodingDesc(1L, arrayOf(sd7point5, sd15, sd30)), + RtpEncodingDesc(1L, arrayOf(hd7point5, hd15, hd30)) ), - sourceName = sourceName, - owner = owner, + sourceName = SOURCE_NAME, + owner = OWNER, videoType = VideoType.CAMERA ) @@ -77,7 +80,7 @@ class SingleSourceAllocationTest : ShouldSpec() { // "preferred FPS") layers for higher resolutions. allocation.preferredLayer shouldBe sd30 allocation.oversendLayer shouldBe null - allocation.layers.map { it.layer } shouldBe listOf(ld7_5, ld15, ld30, sd30, hd30) + allocation.layers.map { it.layer } shouldBe listOf(ld7point5, ld15, ld30, sd30, hd30) } context("With constraints") { val allocation = @@ -94,16 +97,16 @@ class SingleSourceAllocationTest : ShouldSpec() { // "preferred FPS") layers for higher resolutions. allocation.preferredLayer shouldBe sd30 allocation.oversendLayer shouldBe null - allocation.layers.map { it.layer } shouldBe listOf(ld7_5, ld15, ld30, sd30) + allocation.layers.map { it.layer } shouldBe listOf(ld7point5, ld15, ld30, sd30) } context("With constraints unmet by any layer") { // Single high-res stream with 3 temporal layers. val endpointId = "A" val mediaSource = MediaSourceDesc( // No simulcast. - arrayOf(RtpEncodingDesc(1L, arrayOf(hd7_5, hd15, hd30))), - sourceName = sourceName, - owner = owner, + arrayOf(RtpEncodingDesc(1L, arrayOf(hd7point5, hd15, hd30))), + sourceName = SOURCE_NAME, + owner = OWNER, videoType = VideoType.CAMERA ) @@ -121,7 +124,7 @@ class SingleSourceAllocationTest : ShouldSpec() { // The receiver set 360p constraints, but we only have a 720p stream. allocation.preferredLayer shouldBe hd30 allocation.oversendLayer shouldBe null - allocation.layers.map { it.layer } shouldBe listOf(hd7_5, hd15, hd30) + allocation.layers.map { it.layer } shouldBe listOf(hd7point5, hd15, hd30) } context("Zero constraints") { val allocation = @@ -144,18 +147,18 @@ class SingleSourceAllocationTest : ShouldSpec() { context("When some layers are inactive") { // Override layers with bitrate=0. Simulate only up to 360p/15 being active. val sd30 = MockRtpLayerDesc(tid = 2, eid = 1, height = 360, frameRate = 30.0, bitrate = 0.bps) - val hd7_5 = MockRtpLayerDesc(tid = 0, eid = 2, height = 720, frameRate = 7.5, bitrate = 0.bps) + val hd7point5 = MockRtpLayerDesc(tid = 0, eid = 2, height = 720, frameRate = 7.5, bitrate = 0.bps) val hd15 = MockRtpLayerDesc(tid = 1, eid = 2, height = 720, frameRate = 15.0, bitrate = 0.bps) val hd30 = MockRtpLayerDesc(tid = 2, eid = 2, height = 720, frameRate = 30.0, bitrate = 0.bps) val endpointId = "A" val mediaSource = MediaSourceDesc( arrayOf( - RtpEncodingDesc(1L, arrayOf(ld7_5, ld15, ld30)), - RtpEncodingDesc(1L, arrayOf(sd7_5, sd15, sd30)), - RtpEncodingDesc(1L, arrayOf(hd7_5, hd15, hd30)) + RtpEncodingDesc(1L, arrayOf(ld7point5, ld15, ld30)), + RtpEncodingDesc(1L, arrayOf(sd7point5, sd15, sd30)), + RtpEncodingDesc(1L, arrayOf(hd7point5, hd15, hd30)) ), - sourceName = sourceName, - owner = owner, + sourceName = SOURCE_NAME, + owner = OWNER, videoType = VideoType.CAMERA ) @@ -173,7 +176,7 @@ class SingleSourceAllocationTest : ShouldSpec() { // "preferred FPS") layers for higher resolutions. allocation.preferredLayer shouldBe ld30 allocation.oversendLayer shouldBe null - allocation.layers.map { it.layer } shouldBe listOf(ld7_5, ld15, ld30) + allocation.layers.map { it.layer } shouldBe listOf(ld7point5, ld15, ld30) } } context("Screensharing") { @@ -181,12 +184,12 @@ class SingleSourceAllocationTest : ShouldSpec() { val endpointId = "A" val mediaSource = MediaSourceDesc( arrayOf( - RtpEncodingDesc(1L, arrayOf(ld7_5, ld15, ld30)), - RtpEncodingDesc(1L, arrayOf(sd7_5, sd15, sd30)), - RtpEncodingDesc(1L, arrayOf(hd7_5, hd15, hd30)) + RtpEncodingDesc(1L, arrayOf(ld7point5, ld15, ld30)), + RtpEncodingDesc(1L, arrayOf(sd7point5, sd15, sd30)), + RtpEncodingDesc(1L, arrayOf(hd7point5, hd15, hd30)) ), - sourceName = sourceName, - owner = owner, + sourceName = SOURCE_NAME, + owner = OWNER, videoType = VideoType.DESKTOP ) @@ -204,9 +207,9 @@ class SingleSourceAllocationTest : ShouldSpec() { // For screensharing the "preferred" layer should be the highest -- always prioritized over other // endpoints. allocation.preferredLayer shouldBe hd30 - allocation.oversendLayer shouldBe hd7_5 + allocation.oversendLayer shouldBe hd7point5 allocation.layers.map { it.layer } shouldBe - listOf(ld7_5, ld15, ld30, sd7_5, sd15, sd30, hd7_5, hd15, hd30) + listOf(ld7point5, ld15, ld30, sd7point5, sd15, sd30, hd7point5, hd15, hd30) } context("With 360p constraints") { val allocation = @@ -220,23 +223,23 @@ class SingleSourceAllocationTest : ShouldSpec() { ) allocation.preferredLayer shouldBe sd30 - allocation.oversendLayer shouldBe sd7_5 - allocation.layers.map { it.layer } shouldBe listOf(ld7_5, ld15, ld30, sd7_5, sd15, sd30) + allocation.oversendLayer shouldBe sd7point5 + allocation.layers.map { it.layer } shouldBe listOf(ld7point5, ld15, ld30, sd7point5, sd15, sd30) } } context("The high layers are inactive (send-side bwe restrictions)") { // Override layers with bitrate=0. Simulate only up to 360p/30 being active. - val hd7_5 = MockRtpLayerDesc(tid = 0, eid = 2, height = 720, frameRate = 7.5, bitrate = 0.bps) + val hd7point5 = MockRtpLayerDesc(tid = 0, eid = 2, height = 720, frameRate = 7.5, bitrate = 0.bps) val hd15 = MockRtpLayerDesc(tid = 1, eid = 2, height = 720, frameRate = 15.0, bitrate = 0.bps) val hd30 = MockRtpLayerDesc(tid = 2, eid = 2, height = 720, frameRate = 30.0, bitrate = 0.bps) val mediaSource = MediaSourceDesc( arrayOf( - RtpEncodingDesc(1L, arrayOf(ld7_5, ld15, ld30)), - RtpEncodingDesc(1L, arrayOf(sd7_5, sd15, sd30)), - RtpEncodingDesc(1L, arrayOf(hd7_5, hd15, hd30)) + RtpEncodingDesc(1L, arrayOf(ld7point5, ld15, ld30)), + RtpEncodingDesc(1L, arrayOf(sd7point5, sd15, sd30)), + RtpEncodingDesc(1L, arrayOf(hd7point5, hd15, hd30)) ), - sourceName = sourceName, - owner = owner, + sourceName = SOURCE_NAME, + owner = OWNER, videoType = VideoType.DESKTOP ) @@ -253,25 +256,25 @@ class SingleSourceAllocationTest : ShouldSpec() { // For screensharing the "preferred" layer should be the highest -- always prioritized over other // endpoints. allocation.preferredLayer shouldBe sd30 - allocation.oversendLayer shouldBe sd7_5 - allocation.layers.map { it.layer } shouldBe listOf(ld7_5, ld15, ld30, sd7_5, sd15, sd30) + allocation.oversendLayer shouldBe sd7point5 + allocation.layers.map { it.layer } shouldBe listOf(ld7point5, ld15, ld30, sd7point5, sd15, sd30) } context("The low layers are inactive (simulcast signaled but not used)") { // Override layers with bitrate=0. Simulate simulcast being signaled but effectively disabled. - val ld7_5 = MockRtpLayerDesc(tid = 0, eid = 2, height = 720, frameRate = 7.5, bitrate = 0.bps) + val ld7point5 = MockRtpLayerDesc(tid = 0, eid = 2, height = 720, frameRate = 7.5, bitrate = 0.bps) val ld15 = MockRtpLayerDesc(tid = 1, eid = 2, height = 720, frameRate = 15.0, bitrate = 0.bps) val ld30 = MockRtpLayerDesc(tid = 2, eid = 2, height = 720, frameRate = 30.0, bitrate = 0.bps) - val sd7_5 = MockRtpLayerDesc(tid = 0, eid = 1, height = 360, frameRate = 7.5, bitrate = 0.bps) + val sd7point5 = MockRtpLayerDesc(tid = 0, eid = 1, height = 360, frameRate = 7.5, bitrate = 0.bps) val sd15 = MockRtpLayerDesc(tid = 1, eid = 1, height = 360, frameRate = 15.0, bitrate = 0.bps) val sd30 = MockRtpLayerDesc(tid = 2, eid = 1, height = 360, frameRate = 30.0, bitrate = 0.bps) val mediaSource = MediaSourceDesc( arrayOf( - RtpEncodingDesc(1L, arrayOf(ld7_5, ld15, ld30)), - RtpEncodingDesc(1L, arrayOf(sd7_5, sd15, sd30)), - RtpEncodingDesc(1L, arrayOf(hd7_5, hd15, hd30)) + RtpEncodingDesc(1L, arrayOf(ld7point5, ld15, ld30)), + RtpEncodingDesc(1L, arrayOf(sd7point5, sd15, sd30)), + RtpEncodingDesc(1L, arrayOf(hd7point5, hd15, hd30)) ), - sourceName = sourceName, - owner = owner, + sourceName = SOURCE_NAME, + owner = OWNER, videoType = VideoType.DESKTOP ) @@ -289,8 +292,8 @@ class SingleSourceAllocationTest : ShouldSpec() { // For screensharing the "preferred" layer should be the highest -- always prioritized over other // endpoints. allocation.preferredLayer shouldBe hd30 - allocation.oversendLayer shouldBe hd7_5 - allocation.layers.map { it.layer } shouldBe listOf(hd7_5, hd15, hd30) + allocation.oversendLayer shouldBe hd7point5 + allocation.layers.map { it.layer } shouldBe listOf(hd7point5, hd15, hd30) } context("With 180p constraints") { val allocation = @@ -307,8 +310,8 @@ class SingleSourceAllocationTest : ShouldSpec() { // endpoints. Since no layers satisfy the resolution constraints, we consider layers from the // lowest available resolution (which is high). allocation.preferredLayer shouldBe hd30 - allocation.oversendLayer shouldBe hd7_5 - allocation.layers.map { it.layer } shouldBe listOf(hd7_5, hd15, hd30) + allocation.oversendLayer shouldBe hd7point5 + allocation.layers.map { it.layer } shouldBe listOf(hd7point5, hd15, hd30) } } context("VP9") { @@ -322,8 +325,8 @@ class SingleSourceAllocationTest : ShouldSpec() { RtpEncodingDesc(1L, arrayOf(l2)), RtpEncodingDesc(1L, arrayOf(l3)) ), - sourceName = sourceName, - owner = owner, + sourceName = SOURCE_NAME, + owner = OWNER, videoType = VideoType.DESKTOP ) @@ -381,5 +384,5 @@ class SingleSourceAllocationTest : ShouldSpec() { } } -private val sourceName = "sourceName" -private val owner = "owner" +private const val SOURCE_NAME = "sourceName" +private const val OWNER = "owner" diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionTest.kt index 174c814445..5c89fd6c36 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionTest.kt @@ -1373,12 +1373,18 @@ class Vp9AdaptiveSourceProjectionTest { } } companion object { - private val vp9PacketTemplate = DatatypeConverter.parseHexBinary( // RTP Header - "80" + // V, P, X, CC - "60" + // M, PT - "0000" + // Seq - "00000000" + // TS - "cafebabe" + // SSRC + private val vp9PacketTemplate = DatatypeConverter.parseHexBinary( + // RTP Header + // V, P, X, CC + "80" + + // M, PT + "60" + + // Seq + "0000" + + // TS + "00000000" + + // SSRC + "cafebabe" + // VP9 Payload descriptor // I=1,P=0,L=0,F=0,B=1,E=0,V=0,Z=0 "88" + @@ -1563,12 +1569,18 @@ class Vp9AdaptiveSourceProjectionTest { } companion object { - private val vp9SvcPacketTemplate = DatatypeConverter.parseHexBinary( // RTP Header - "80" + // V, P, X, CC - "60" + // M, PT - "0000" + // Seq - "00000000" + // TS - "cafebabe" + // SSRC + private val vp9SvcPacketTemplate = DatatypeConverter.parseHexBinary( + // RTP Header + // V, P, X, CC + "80" + + // M, PT + "60" + + // Seq + "0000" + + // TS + "00000000" + + // SSRC + "cafebabe" + // VP9 Payload descriptor // I=1,P=0,L=1,F=0,B=1,E=0,V=0,Z=0 "a8" + @@ -1584,8 +1596,9 @@ class Vp9AdaptiveSourceProjectionTest { ) /* TODO: move this to jitsi-rtp */ + const val JAVA_TO_NTP_EPOCH_OFFSET_SECS = 2208988800L + fun setSIBuilderNtp(siBuilder: SenderInfoBuilder, wallTime: Long) { - val JAVA_TO_NTP_EPOCH_OFFSET_SECS = 2208988800L val wallSecs = wallTime / 1000 val wallMs = wallTime % 1000 siBuilder.ntpTimestampMsw = wallSecs + JAVA_TO_NTP_EPOCH_OFFSET_SECS diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilterTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilterTest.kt index 07bb90c2d0..d32cb07ed5 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilterTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilterTest.kt @@ -416,7 +416,11 @@ internal class Vp9QualityFilterTest : ShouldSpec() { while (g.hasNext() && frames < numFrames) { val f = g.next() - ms = if (f.timestamp != lastTs) { f.timestamp / 90 } else { ms + 1 } + ms = if (f.timestamp != lastTs) { + f.timestamp / 90 + } else { + ms + 1 + } lastTs = f.timestamp val packetIndex = RtpLayerDesc.getIndex(f.ssrc.toInt(), f.spatialLayer, f.temporalLayer) @@ -645,7 +649,8 @@ private class SimulcastFrameGenerator : FrameGenerator() { val keyframePicture = (pictureCount % 48) == 0 val f = Vp9Frame( - ssrc = enc.toLong(), // Use the encoding ID as the SSRC to make testing easier. + // Use the encoding ID as the SSRC to make testing easier. + ssrc = enc.toLong(), timestamp = pictureCount * 3000L, earliestKnownSequenceNumber = pictureCount + (enc * 10000), latestKnownSequenceNumber = pictureCount + (enc * 10000), diff --git a/pom.xml b/pom.xml index 1d9a16b7f2..d6c757d636 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ 1.0-127-g6c65524 1.1-127-gf49982f 1.13.8 - 2.0.0 + 3.0.0 3.5.1 4.6.0 3.0.10 diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/UnparsedPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/UnparsedPacket.kt index 16bd5f738f..d839c4e14e 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/UnparsedPacket.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/UnparsedPacket.kt @@ -30,10 +30,9 @@ class UnparsedPacket( * Note that we leave the same space at the start as for RTP packets, because an [UnparsedPacket]'s buffer * might be used directly to create an [RtpPacket]. */ - override fun clone(): UnparsedPacket = - UnparsedPacket( - cloneBuffer(RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET), - RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET, - length - ) + override fun clone(): UnparsedPacket = UnparsedPacket( + cloneBuffer(RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET), + RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET, + length + ) } diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/extensions/ByteBuffer.kt b/rtp/src/main/kotlin/org/jitsi/rtp/extensions/ByteBuffer.kt index 629f4eb2d4..3b396176b9 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/extensions/ByteBuffer.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/extensions/ByteBuffer.kt @@ -151,8 +151,7 @@ fun ByteBuffer.subBuffer(startPosition: Int, size: Int): ByteBuffer { * and capacity will be the amount of bytes between [startPosition] and * the current buffer's [limit()] */ -fun ByteBuffer.subBuffer(startPosition: Int): ByteBuffer = - subBuffer(startPosition, limit() - startPosition) +fun ByteBuffer.subBuffer(startPosition: Int): ByteBuffer = subBuffer(startPosition, limit() - startPosition) /** * Put [buf] into this buffer starting at [index] diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/extensions/bytearray/ByteArrayExtensions.kt b/rtp/src/main/kotlin/org/jitsi/rtp/extensions/bytearray/ByteArrayExtensions.kt index c353b9baed..8c4e191e78 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/extensions/bytearray/ByteArrayExtensions.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/extensions/bytearray/ByteArrayExtensions.kt @@ -38,8 +38,7 @@ fun ByteArray.putBits(byteIndex: Int, destBitPos: Int, src: Byte, numBits: Int) set(byteIndex, byte) } -fun ByteArray.getBitAsBool(byteOffset: Int, bitOffset: Int): Boolean = - get(byteOffset).getBitAsBool(bitOffset) +fun ByteArray.getBitAsBool(byteOffset: Int, bitOffset: Int): Boolean = get(byteOffset).getBitAsBool(bitOffset) fun ByteArray.putBitAsBoolean(byteIndex: Int, destBitPos: Int, isSet: Boolean) { var byte = get(byteIndex) diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpReportBlock.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpReportBlock.kt index 3f1ed76522..85d66f8f02 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpReportBlock.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpReportBlock.kt @@ -115,13 +115,10 @@ class RtcpReportBlock( ) } - fun getSsrc(buffer: ByteArray, offset: Int): Long = - buffer.getIntAsLong(offset + SSRC_OFFSET) - fun setSsrc(buf: ByteArray, baseOffset: Int, value: Long) = - buf.putInt(baseOffset + SSRC_OFFSET, value.toInt()) + fun getSsrc(buffer: ByteArray, offset: Int): Long = buffer.getIntAsLong(offset + SSRC_OFFSET) + fun setSsrc(buf: ByteArray, baseOffset: Int, value: Long) = buf.putInt(baseOffset + SSRC_OFFSET, value.toInt()) - fun getFractionLost(buffer: ByteArray, offset: Int): Int = - buffer.getByteAsInt(offset + FRACTION_LOST_OFFSET) + fun getFractionLost(buffer: ByteArray, offset: Int): Int = buffer.getByteAsInt(offset + FRACTION_LOST_OFFSET) fun setFractionLost(buf: ByteArray, baseOffset: Int, value: Int) = buf.set(baseOffset + FRACTION_LOST_OFFSET, value.toByte()) diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpRrPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpRrPacket.kt index 6f20d7dd93..2e76ab655f 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpRrPacket.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpRrPacket.kt @@ -76,8 +76,7 @@ data class RtcpRrPacketBuilder( require(reportBlocks.size <= 31) { "Too many report blocks ${reportBlocks.size}: RR can contain at most 31" } } - private fun getLengthValue(): Int = - RtpUtils.calculateRtcpLengthFieldValue(sizeBytes) + private fun getLengthValue(): Int = RtpUtils.calculateRtcpLengthFieldValue(sizeBytes) private val sizeBytes: Int get() = RtcpHeader.SIZE_BYTES + reportBlocks.size * RtcpReportBlock.SIZE_BYTES diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpSdesPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpSdesPacket.kt index 30cca023a1..a400ce89ce 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpSdesPacket.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpSdesPacket.kt @@ -92,8 +92,7 @@ class SdesChunk( const val SSRC_OFFSET = 0 const val SDES_ITEMS_OFFSET = 4 - fun getSsrc(buf: ByteArray, baseOffset: Int): Long = - buf.getIntAsLong(baseOffset + SSRC_OFFSET) + fun getSsrc(buf: ByteArray, baseOffset: Int): Long = buf.getIntAsLong(baseOffset + SSRC_OFFSET) fun getSdesItems(buf: ByteArray, baseOffset: Int): List { var currOffset = baseOffset + SDES_ITEMS_OFFSET @@ -142,11 +141,9 @@ abstract class SdesItem( const val LENGTH_OFFSET = 1 const val DATA_OFFSET = 2 - fun getType(buf: ByteArray, baseOffset: Int): Int = - buf.getByteAsInt(baseOffset + TYPE_OFFSET) + fun getType(buf: ByteArray, baseOffset: Int): Int = buf.getByteAsInt(baseOffset + TYPE_OFFSET) - fun getLength(buf: ByteArray, baseOffset: Int): Int = - buf.getByteAsInt(baseOffset + LENGTH_OFFSET) + fun getLength(buf: ByteArray, baseOffset: Int): Int = buf.getByteAsInt(baseOffset + LENGTH_OFFSET) fun copyData(buf: ByteArray, baseOffset: Int, dataLength: Int): ByteArray { if (dataLength <= 0) { diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpSrPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpSrPacket.kt index 7b2d08b167..750f92e590 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpSrPacket.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpSrPacket.kt @@ -46,18 +46,15 @@ class SenderInfoParser { const val SENDERS_PACKET_COUNT_OFFSET = 12 const val SENDERS_OCTET_COUNT_OFFSET = 16 - fun getNtpTimestampMsw(buf: ByteArray, baseOffset: Int): Long = - buf.getIntAsLong(baseOffset + NTP_TS_MSW_OFFSET) + fun getNtpTimestampMsw(buf: ByteArray, baseOffset: Int): Long = buf.getIntAsLong(baseOffset + NTP_TS_MSW_OFFSET) fun setNtpTimestampMsw(buf: ByteArray, baseOffset: Int, value: Long) = buf.putInt(baseOffset + NTP_TS_MSW_OFFSET, value.toInt()) - fun getNtpTimestampLsw(buf: ByteArray, baseOffset: Int): Long = - buf.getIntAsLong(baseOffset + NTP_TS_LSW_OFFSET) + fun getNtpTimestampLsw(buf: ByteArray, baseOffset: Int): Long = buf.getIntAsLong(baseOffset + NTP_TS_LSW_OFFSET) fun setNtpTimestampLsw(buf: ByteArray, baseOffset: Int, value: Long) = buf.putInt(baseOffset + NTP_TS_LSW_OFFSET, value.toInt()) - fun getRtpTimestamp(buf: ByteArray, baseOffset: Int): Long = - buf.getIntAsLong(baseOffset + RTP_TS_OFFSET) + fun getRtpTimestamp(buf: ByteArray, baseOffset: Int): Long = buf.getIntAsLong(baseOffset + RTP_TS_OFFSET) fun setRtpTimestamp(buf: ByteArray, baseOffset: Int, value: Long) = buf.putInt(baseOffset + RTP_TS_OFFSET, value.toInt()) diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/UnsupportedRtcpPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/UnsupportedRtcpPacket.kt index 9b60438085..8a2f56e03e 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/UnsupportedRtcpPacket.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/UnsupportedRtcpPacket.kt @@ -25,6 +25,5 @@ class UnsupportedRtcpPacket( offset: Int, packetLengthBytes: Int ) : RtcpPacket(buf, offset, packetLengthBytes) { - override fun clone(): UnsupportedRtcpPacket = - UnsupportedRtcpPacket(cloneBuffer(0), 0, length) + override fun clone(): UnsupportedRtcpPacket = UnsupportedRtcpPacket(cloneBuffer(0), 0, length) } diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/RtcpFbPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/RtcpFbPacket.kt index 9c97c521ff..99a2395557 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/RtcpFbPacket.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/RtcpFbPacket.kt @@ -62,8 +62,7 @@ abstract class RtcpFbPacket( const val HEADER_SIZE = RtcpHeader.SIZE_BYTES + 4 const val FCI_OFFSET = HEADER_SIZE - fun getFmt(buf: ByteArray, baseOffset: Int): Int = - RtcpHeader.getReportCount(buf, baseOffset) + fun getFmt(buf: ByteArray, baseOffset: Int): Int = RtcpHeader.getReportCount(buf, baseOffset) fun getMediaSourceSsrc(buf: ByteArray, baseOffset: Int): Long = buf.getInt(baseOffset + MEDIA_SOURCE_SSRC_OFFSET).toPositiveLong() fun setMediaSourceSsrc(buf: ByteArray, baseOffset: Int, value: Long) = diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/UnsupportedRtcpFbPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/UnsupportedRtcpFbPacket.kt index 1fbededa16..b35c2e5fef 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/UnsupportedRtcpFbPacket.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/UnsupportedRtcpFbPacket.kt @@ -25,6 +25,5 @@ class UnsupportedRtcpFbPacket( offset: Int, packetLengthBytes: Int ) : RtcpFbPacket(buf, offset, packetLengthBytes) { - override fun clone(): UnsupportedRtcpFbPacket = - UnsupportedRtcpFbPacket(cloneBuffer(0), 0, length) + override fun clone(): UnsupportedRtcpFbPacket = UnsupportedRtcpFbPacket(cloneBuffer(0), 0, length) } diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbFirPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbFirPacket.kt index a68dbed81c..51e97b7f5c 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbFirPacket.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbFirPacket.kt @@ -87,8 +87,7 @@ class RtcpFbFirPacket( fun setMediaSenderSsrc(buf: ByteArray, baseOffset: Int, value: Long) = buf.putInt(baseOffset + MEDIA_SENDER_SSRC_OFFSET, value.toInt()) - fun getSeqNum(buf: ByteArray, baseOffset: Int): Int = - buf.get(baseOffset + SEQ_NUM_OFFSET).toPositiveInt() + fun getSeqNum(buf: ByteArray, baseOffset: Int): Int = buf.get(baseOffset + SEQ_NUM_OFFSET).toPositiveInt() fun setSeqNum(buf: ByteArray, baseOffset: Int, value: Int) = buf.set(baseOffset + SEQ_NUM_OFFSET, value.toByte()) } diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbRembPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbRembPacket.kt index e3cad2cc72..cc69313964 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbRembPacket.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbRembPacket.kt @@ -87,8 +87,7 @@ class RtcpFbRembPacket( const val BR_LEN = 3 const val SSRCS_OFF = BR_OFF + BR_LEN - fun getBrExp(buf: ByteArray, baseOffset: Int): Int = - buf.getBitsAsInt(baseOffset + BR_OFF, 0, 6) + fun getBrExp(buf: ByteArray, baseOffset: Int): Int = buf.getBitsAsInt(baseOffset + BR_OFF, 0, 6) fun getBrMantissa(buf: ByteArray, baseOffset: Int): Int = (buf.getBitsAsInt(baseOffset + BR_OFF, 6, 2) shl 16) + buf.getShortAsInt(baseOffset + BR_OFF + 1) fun getBitrate(buf: ByteArray, baseOffset: Int): Long { @@ -104,8 +103,7 @@ class RtcpFbRembPacket( return brBps } - fun getNumSsrc(buf: ByteArray, baseOffset: Int): Int = - buf.getByteAsInt(baseOffset + NUM_SSRC_OFF) + fun getNumSsrc(buf: ByteArray, baseOffset: Int): Int = buf.getByteAsInt(baseOffset + NUM_SSRC_OFF) fun getSsrc(buf: ByteArray, baseOffset: Int, ssrcIndex: Int) = buf.getIntAsLong(baseOffset + SSRCS_OFF + ssrcIndex * 4) diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/LastChunk.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/LastChunk.kt index 3522780c0a..9d23e0732b 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/LastChunk.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/LastChunk.kt @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +@file:Suppress("ktlint:standard:property-naming", "ktlint:standard:function-naming") package org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc @@ -145,8 +146,7 @@ class LastChunk { * S = symbol * Run Length = Unsigned integer denoting the run length of the symbol */ - private fun EncodeRunLength(): Chunk = - ((delta_sizes_[0] shl 13) or size_) + private fun EncodeRunLength(): Chunk = ((delta_sizes_[0] shl 13) or size_) private fun DecodeRunLength(chunk: Chunk, max_count: Int) { size_ = min(chunk and 0x1fff, max_count) diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacket.kt index bf44629206..8566f34421 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacket.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacket.kt @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +@file:Suppress("ktlint:standard:property-naming", "ktlint:standard:function-naming") package org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc @@ -142,8 +143,7 @@ class RtcpFbTccPacketBuilder( return true } - fun GetBaseTimeUs(): Long = - base_time_ticks_ * kBaseScaleFactor + fun GetBaseTimeUs(): Long = base_time_ticks_ * kBaseScaleFactor private fun AddDeltaSize(deltaSize: DeltaSize): Boolean { if (num_seq_no_ == kMaxReportedPackets) { @@ -416,8 +416,7 @@ class RtcpFbTccPacket( val feedbackSeqNum: Int = getFeedbackPacketCount(buffer, offset) - fun GetBaseTimeUs(): Long = - base_time_ticks_ * kBaseScaleFactor + fun GetBaseTimeUs(): Long = base_time_ticks_ * kBaseScaleFactor override fun iterator(): Iterator = packets_.iterator() @@ -456,8 +455,7 @@ class RtcpFbTccPacket( const val PACKET_CHUNKS_OFFSET = RtcpFbPacket.HEADER_SIZE + 8 // baseOffset in all of these refers to the start of the entire RTCP TCC packet - fun getBaseSeqNum(buf: ByteArray, baseOffset: Int): Int = - buf.getShortAsInt(baseOffset + BASE_SEQ_NUM_OFFSET) + fun getBaseSeqNum(buf: ByteArray, baseOffset: Int): Int = buf.getShortAsInt(baseOffset + BASE_SEQ_NUM_OFFSET) fun setBaseSeqNum(buf: ByteArray, baseOffset: Int, value: Int) = buf.putShort(baseOffset + BASE_SEQ_NUM_OFFSET, value.toShort()) diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RedPacketParser.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RedPacketParser.kt index 3e50868f45..6b7baafcdb 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RedPacketParser.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RedPacketParser.kt @@ -37,10 +37,7 @@ class RedPacketParser( * @param parseRedundancy whether to parse redundancy packets * @return The list of parsed redundancy packets. */ - fun decapsulate( - rtpPacket: RtpPacket, - parseRedundancy: Boolean - ): List = with(rtpPacket) { + fun decapsulate(rtpPacket: RtpPacket, parseRedundancy: Boolean): List = with(rtpPacket) { var currentOffset = payloadOffset val redundancyBlockHeaders = mutableListOf() @@ -214,12 +211,11 @@ class RedundancyBlockHeader( } internal class RtpRedPacket(buffer: ByteArray, offset: Int, length: Int) : RtpPacket(buffer, offset, length) { - override fun clone(): RtpRedPacket = - RtpRedPacket( - cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET), - BYTES_TO_LEAVE_AT_START_OF_PACKET, - length - ) + override fun clone(): RtpRedPacket = RtpRedPacket( + cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET), + BYTES_TO_LEAVE_AT_START_OF_PACKET, + length + ) companion object { val parser = RedPacketParser { b, o, l -> RtpPacket(b, o, l) } diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpHeader.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpHeader.kt index afa1b71c87..8f9477c07c 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpHeader.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpHeader.kt @@ -59,15 +59,13 @@ class RtpHeader { const val EXT_HEADER_SIZE_BYTES = 4 const val VERSION = 2 - fun getVersion(buf: ByteArray, baseOffset: Int): Int = - (buf[baseOffset].toInt() and 0xC0) ushr 6 + fun getVersion(buf: ByteArray, baseOffset: Int): Int = (buf[baseOffset].toInt() and 0xC0) ushr 6 fun setVersion(buf: ByteArray, baseOffset: Int, version: Int) { buf[baseOffset] = ((buf[baseOffset].toInt() and 0xC0.inv()) or ((version shl 6) and 0xC0)).toByte() } - fun hasPadding(buf: ByteArray, baseOffset: Int): Boolean = - (buf[baseOffset].toInt() and 0x20) == 0x20 + fun hasPadding(buf: ByteArray, baseOffset: Int): Boolean = (buf[baseOffset].toInt() and 0x20) == 0x20 fun setPadding(buf: ByteArray, baseOffset: Int, hasPadding: Boolean) { buf[baseOffset] = when (hasPadding) { true -> (buf[baseOffset].toInt() or 0x20).toByte() @@ -75,8 +73,7 @@ class RtpHeader { } } - fun hasExtensions(buf: ByteArray, baseOffset: Int): Boolean = - (buf[baseOffset].toInt() and 0x10) == 0x10 + fun hasExtensions(buf: ByteArray, baseOffset: Int): Boolean = (buf[baseOffset].toInt() and 0x10) == 0x10 fun setHasExtensions(buf: ByteArray, baseOffset: Int, hasExtension: Boolean) { buf[baseOffset] = when (hasExtension) { true -> (buf[baseOffset].toInt() or 0x10).toByte() @@ -84,14 +81,12 @@ class RtpHeader { } } - fun getCsrcCount(buf: ByteArray, baseOffset: Int): Int = - buf[baseOffset].toInt() and 0x0F + fun getCsrcCount(buf: ByteArray, baseOffset: Int): Int = buf[baseOffset].toInt() and 0x0F fun setCsrcCount(buf: ByteArray, baseOffset: Int, csrcCount: Int) { buf[baseOffset] = ((buf[baseOffset].toInt() and 0xF0) or ((csrcCount and 0x0F))).toByte() } - fun getMarker(buf: ByteArray, baseOffset: Int): Boolean = - (buf[baseOffset + 1].toInt() and 0x80) == 0x80 + fun getMarker(buf: ByteArray, baseOffset: Int): Boolean = (buf[baseOffset + 1].toInt() and 0x80) == 0x80 fun setMarker(buf: ByteArray, baseOffset: Int, isSet: Boolean) { buf[baseOffset + 1] = when (isSet) { true -> (buf[baseOffset + 1].toInt() or 0x80).toByte() @@ -105,20 +100,17 @@ class RtpHeader { buf[baseOffset + 1] = (buf[baseOffset + 1] and 0x80.toByte()) or (payloadType and 0x7F).toByte() } - fun getSequenceNumber(buf: ByteArray, baseOffset: Int): Int = - buf.getShortAsInt(baseOffset + 2) + fun getSequenceNumber(buf: ByteArray, baseOffset: Int): Int = buf.getShortAsInt(baseOffset + 2) fun setSequenceNumber(buf: ByteArray, baseOffset: Int, sequenceNumber: Int) { buf.putShort(baseOffset + 2, sequenceNumber.toShort()) } - fun getTimestamp(buf: ByteArray, baseOffset: Int): Long = - buf.getIntAsLong(baseOffset + 4) + fun getTimestamp(buf: ByteArray, baseOffset: Int): Long = buf.getIntAsLong(baseOffset + 4) fun setTimestamp(buf: ByteArray, baseOffset: Int, timestamp: Long) { buf.putInt(baseOffset + 4, timestamp.toInt()) } - fun getSsrc(buf: ByteArray, baseOffset: Int): Long = - buf.getIntAsLong(baseOffset + 8) + fun getSsrc(buf: ByteArray, baseOffset: Int): Long = buf.getIntAsLong(baseOffset + 8) fun setSsrc(buf: ByteArray, baseOffset: Int, ssrc: Long) { buf.putInt(baseOffset + 8, ssrc.toInt()) } @@ -162,12 +154,11 @@ class RtpHeader { * The "defined by profile" header extension field. Only valid if hasExtensions is true, otherwise * returns an invalid value (-1) */ - fun getExtensionsProfileType(buf: ByteArray, baseOffset: Int): Int = - if (hasExtensions(buf, baseOffset)) { - val extHeaderOffset = getFixedHeaderAndCcLength(buf, baseOffset) - HeaderExtensionHelpers.getExtensionsProfileType(buf, baseOffset + extHeaderOffset) - } else { - -1 - } + fun getExtensionsProfileType(buf: ByteArray, baseOffset: Int): Int = if (hasExtensions(buf, baseOffset)) { + val extHeaderOffset = getFixedHeaderAndCcLength(buf, baseOffset) + HeaderExtensionHelpers.getExtensionsProfileType(buf, baseOffset + extHeaderOffset) + } else { + -1 + } } } diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpSequenceNumber.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpSequenceNumber.kt index 1240c995ed..69f305e0ad 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpSequenceNumber.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpSequenceNumber.kt @@ -51,8 +51,7 @@ value class RtpSequenceNumber internal constructor(val value: Int) : Comparable< operator fun minus(num: Int): RtpSequenceNumber = plus(-num) operator fun minus(seqNum: RtpSequenceNumber): RtpSequenceNumber = plus(-seqNum.value) - override operator fun compareTo(other: RtpSequenceNumber): Int = - RtpUtils.getSequenceNumberDelta(value, other.value) + override operator fun compareTo(other: RtpSequenceNumber): Int = RtpUtils.getSequenceNumberDelta(value, other.value) operator fun rangeTo(other: RtpSequenceNumber) = RtpSequenceNumberProgression(this, other) diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/AbsSendTimeHeaderExtension.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/AbsSendTimeHeaderExtension.kt index 60de3e3264..f66eba0c31 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/AbsSendTimeHeaderExtension.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/AbsSendTimeHeaderExtension.kt @@ -38,14 +38,14 @@ class AbsSendTimeHeaderExtension { /** * One billion. */ - private const val b = 1_000_000_000 + private const val B = 1_000_000_000 fun setTime(ext: RtpPacket.HeaderExtension, timestampNanos: Long) = setTime(ext.buffer, ext.dataOffset, timestampNanos) private fun setTime(buf: ByteArray, offset: Int, timestampNanos: Long) { - val fraction = ((timestampNanos % b) * (1 shl 18) / b) - val seconds = ((timestampNanos / b) % 64) // 6 bits only + val fraction = ((timestampNanos % B) * (1 shl 18) / B) + val seconds = ((timestampNanos / B) % 64) // 6 bits only val timestamp = ((seconds shl 18) or fraction) and 0x00FFFFFF @@ -65,7 +65,7 @@ class AbsSendTimeHeaderExtension { ).toDouble() / 0x03ffff val instantMillis = Instant.ofEpochSecond(seconds.toLong()) - return instantMillis.plusNanos((fraction * b).toLong()) + return instantMillis.plusNanos((fraction * B).toLong()) } } } diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/AudioLevelHeaderExtension.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/AudioLevelHeaderExtension.kt index ab39a6e71d..807698df98 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/AudioLevelHeaderExtension.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/AudioLevelHeaderExtension.kt @@ -36,11 +36,9 @@ class AudioLevelHeaderExtension { fun getAudioLevel(ext: RtpPacket.HeaderExtension): Int = getAudioLevel(ext.buffer, ext.dataOffset) - private fun getAudioLevel(buf: ByteArray, offset: Int): Int = - (buf[offset] and AUDIO_LEVEL_MASK).toPositiveInt() + private fun getAudioLevel(buf: ByteArray, offset: Int): Int = (buf[offset] and AUDIO_LEVEL_MASK).toPositiveInt() fun getVad(ext: RtpPacket.HeaderExtension): Boolean = getVad(ext.buffer, ext.dataOffset) - private fun getVad(buf: ByteArray, offset: Int): Boolean = - (buf[offset].toInt() and 0x80) != 0 + private fun getVad(buf: ByteArray, offset: Int): Boolean = (buf[offset].toInt() and 0x80) != 0 } } diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/HeaderExtensionHelpers.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/HeaderExtensionHelpers.kt index f307182068..4cfe8a2b4a 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/HeaderExtensionHelpers.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/HeaderExtensionHelpers.kt @@ -44,8 +44,7 @@ abstract class HeaderExtensionParser { * Return the entire size, in bytes, of the extension in [buf] whose header * starts at [offset] */ - fun getEntireLengthBytes(buf: ByteArray, offset: Int): Int = - getDataLengthBytes(buf, offset) + extHeaderSizeBytes + fun getEntireLengthBytes(buf: ByteArray, offset: Int): Int = getDataLengthBytes(buf, offset) + extHeaderSizeBytes /** * Return the data size, in bytes, of the extension in [buf] whose header @@ -60,11 +59,9 @@ object OneByteHeaderExtensionParser : HeaderExtensionParser() { override val extHeaderSizeBytes = 1 override val minimumExtSizeBytes = 2 - override fun isMatchingType(profileField: Int): Boolean = - profileField == headerExtensionLabel + override fun isMatchingType(profileField: Int): Boolean = profileField == headerExtensionLabel - override fun getId(buf: ByteArray, offset: Int): Int = - (buf[offset].toInt() ushr 4) and 0x0F + override fun getId(buf: ByteArray, offset: Int): Int = (buf[offset].toInt() ushr 4) and 0x0F override fun writeIdAndLength(id: Int, dataLength: Int, buf: ByteArray, offset: Int) { require(id in 1..14) @@ -79,16 +76,15 @@ object OneByteHeaderExtensionParser : HeaderExtensionParser() { object TwoByteHeaderExtensionParser : HeaderExtensionParser() { /* We don't support "value 256", in the low four bits of the "defined by profile" field. */ override val headerExtensionLabel = 0x1000 - private const val headerExtensionMask = 0xFFF0 + private const val HEADER_EXTENSION_MASK = 0xFFF0 override val extHeaderSizeBytes = 2 override val minimumExtSizeBytes = 2 override fun isMatchingType(profileField: Int): Boolean = - (profileField and headerExtensionMask) == headerExtensionLabel + (profileField and HEADER_EXTENSION_MASK) == headerExtensionLabel - override fun getId(buf: ByteArray, offset: Int): Int = - buf[offset].toInt() + override fun getId(buf: ByteArray, offset: Int): Int = buf[offset].toInt() override fun writeIdAndLength(id: Int, dataLength: Int, buf: ByteArray, offset: Int) { require(id in 1..255) @@ -98,8 +94,7 @@ object TwoByteHeaderExtensionParser : HeaderExtensionParser() { buf[offset + 1] = dataLength.toByte() } - override fun getDataLengthBytes(buf: ByteArray, offset: Int): Int = - buf[offset + 1].toPositiveInt() + override fun getDataLengthBytes(buf: ByteArray, offset: Int): Int = buf[offset + 1].toPositiveInt() } private val headerExtensionParsers = arrayOf(OneByteHeaderExtensionParser, TwoByteHeaderExtensionParser) @@ -116,8 +111,7 @@ class HeaderExtensionHelpers { * only be called if it's been verified that the header held in [buf] * actually contains extensions */ - fun getExtensionsProfileType(buf: ByteArray, offset: Int) = - buf.getShortAsInt(offset) + fun getExtensionsProfileType(buf: ByteArray, offset: Int) = buf.getShortAsInt(offset) /** * Return the length of the entire header extensions block, including diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/TccHeaderExtension.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/TccHeaderExtension.kt index 5c2188f99a..a271129763 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/TccHeaderExtension.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/TccHeaderExtension.kt @@ -32,14 +32,12 @@ class TccHeaderExtension { companion object { const val DATA_SIZE_BYTES = 2 - fun getSequenceNumber(ext: RtpPacket.HeaderExtension): Int = - getSequenceNumber(ext.buffer, ext.dataOffset) + fun getSequenceNumber(ext: RtpPacket.HeaderExtension): Int = getSequenceNumber(ext.buffer, ext.dataOffset) fun setSequenceNumber(ext: RtpPacket.HeaderExtension, tccSeqNum: Int) { setSequenceNumber(ext.buffer, ext.dataOffset, tccSeqNum) } - private fun getSequenceNumber(buf: ByteArray, offset: Int): Int = - buf.getShortAsInt(offset) + private fun getSequenceNumber(buf: ByteArray, offset: Int): Int = buf.getShortAsInt(offset) private fun setSequenceNumber(buf: ByteArray, offset: Int, seqNum: Int) { buf.putShort(offset, seqNum.toShort()) } diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/util/FieldParsers.kt b/rtp/src/main/kotlin/org/jitsi/rtp/util/FieldParsers.kt index 5712e998cd..eef031e49e 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/util/FieldParsers.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/util/FieldParsers.kt @@ -33,14 +33,10 @@ fun ByteArray.getBitsAsInt(byteOffset: Int, bitStartPos: Int, numBits: Int): Int fun ByteArray.putNumberAsBits(byteOffset: Int, bitOffset: Int, numBits: Int, value: Number) { putBits(byteOffset, bitOffset, value.toByte(), numBits) } -fun ByteArray.getByteAsInt(offset: Int): Int = - get(offset).toPositiveInt() +fun ByteArray.getByteAsInt(offset: Int): Int = get(offset).toPositiveInt() -fun ByteArray.getShortAsInt(offset: Int): Int = - getShort(offset).toPositiveInt() +fun ByteArray.getShortAsInt(offset: Int): Int = getShort(offset).toPositiveInt() -fun ByteArray.get3BytesAsInt(offset: Int): Int = - get3Bytes(offset).toPositiveInt() +fun ByteArray.get3BytesAsInt(offset: Int): Int = get3Bytes(offset).toPositiveInt() -fun ByteArray.getIntAsLong(offset: Int): Long = - getInt(offset).toPositiveLong() +fun ByteArray.getIntAsLong(offset: Int): Long = getInt(offset).toPositiveLong() diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/util/RtpUtils.kt b/rtp/src/main/kotlin/org/jitsi/rtp/util/RtpUtils.kt index e4f01bf373..c088ad7b2c 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/util/RtpUtils.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/util/RtpUtils.kt @@ -34,14 +34,13 @@ class RtpUtils { /** * Get the number of bytes needed to pad [dataSizeBytes] bytes to a 4-byte word boundary. */ - fun getNumPaddingBytes(dataSizeBytes: Int): Int = - when (dataSizeBytes % 4) { - 0 -> 0 - 1 -> 3 - 2 -> 2 - 3 -> 1 - else -> 0 // The above is exhaustive. - } + fun getNumPaddingBytes(dataSizeBytes: Int): Int = when (dataSizeBytes % 4) { + 0 -> 0 + 1 -> 3 + 2 -> 2 + 3 -> 1 + else -> 0 // The above is exhaustive. + } /** * Returns the delta between two RTP sequence numbers, taking into account @@ -52,8 +51,7 @@ class RtpUtils { * @return the delta between two RTP sequence numbers (modulo 2^16). */ @JvmStatic - fun getSequenceNumberDelta(a: Int, b: Int): Int = - getSequenceNumberDeltaAsShort(a, b).toInt() + fun getSequenceNumberDelta(a: Int, b: Int): Int = getSequenceNumberDeltaAsShort(a, b).toInt() /** * Like [getSequenceNumberDelta], but returning the delta as a [Short]. @@ -74,8 +72,7 @@ class RtpUtils { * @return the sequence number resulting from doing "start + delta" */ @JvmStatic - fun applySequenceNumberDelta(start: Int, delta: Int): Int = - (start + delta) and 0xffff + fun applySequenceNumberDelta(start: Int, delta: Int): Int = (start + delta) and 0xffff /** * Apply a delta to a given RTP timestamp and return the result (taking @@ -85,32 +82,26 @@ class RtpUtils { * @return the timestamp result from doing "start + delta" */ @JvmStatic - fun applyTimestampDelta(start: Long, delta: Long): Long = - (start + delta) and 0xffff_ffffL + fun applyTimestampDelta(start: Long, delta: Long): Long = (start + delta) and 0xffff_ffffL @JvmStatic - fun isNewerSequenceNumberThan(a: Int, b: Int): Boolean = - getSequenceNumberDeltaAsShort(a, b) > 0 + fun isNewerSequenceNumberThan(a: Int, b: Int): Boolean = getSequenceNumberDeltaAsShort(a, b) > 0 @JvmStatic - fun isOlderSequenceNumberThan(a: Int, b: Int): Boolean = - getSequenceNumberDeltaAsShort(a, b) < 0 + fun isOlderSequenceNumberThan(a: Int, b: Int): Boolean = getSequenceNumberDeltaAsShort(a, b) < 0 @JvmStatic - fun isNewerTimestampThan(a: Long, b: Long): Boolean = - getTimestampDiffAsInt(a, b) > 0 + fun isNewerTimestampThan(a: Long, b: Long): Boolean = getTimestampDiffAsInt(a, b) > 0 @JvmStatic - fun isOlderTimestampThan(a: Long, b: Long): Boolean = - getTimestampDiffAsInt(a, b) < 0 + fun isOlderTimestampThan(a: Long, b: Long): Boolean = getTimestampDiffAsInt(a, b) < 0 /** * Returns the difference between two RTP timestamps. * @return the difference between two RTP timestamps. */ @JvmStatic - fun getTimestampDiff(a: Long, b: Long): Long = - getTimestampDiffAsInt(a, b).toLong() + fun getTimestampDiff(a: Long, b: Long): Long = getTimestampDiffAsInt(a, b).toLong() /** * Returns the difference between two RTP timestamps as an [Int]. @@ -163,27 +154,21 @@ fun Byte.isPadding(): Boolean = this == 0x00.toByte() * Returns true if the RTP sequence number represented by [this] represents a more recent RTP packet than the one * represented by [otherSeqNum] */ -infix fun Int.isNewerThan(otherSeqNum: Int): Boolean = - RtpUtils.isNewerSequenceNumberThan(this, otherSeqNum) +infix fun Int.isNewerThan(otherSeqNum: Int): Boolean = RtpUtils.isNewerSequenceNumberThan(this, otherSeqNum) -infix fun Int.isOlderThan(otherSeqNum: Int): Boolean = - RtpUtils.isOlderSequenceNumberThan(this, otherSeqNum) +infix fun Int.isOlderThan(otherSeqNum: Int): Boolean = RtpUtils.isOlderSequenceNumberThan(this, otherSeqNum) -infix fun Long.isNewerTimestampThan(otherTimestamp: Long): Boolean = - RtpUtils.isNewerTimestampThan(this, otherTimestamp) +infix fun Long.isNewerTimestampThan(otherTimestamp: Long): Boolean = RtpUtils.isNewerTimestampThan(this, otherTimestamp) -infix fun Long.isOlderTimestampThan(otherTimestamp: Long): Boolean = - RtpUtils.isOlderTimestampThan(this, otherTimestamp) +infix fun Long.isOlderTimestampThan(otherTimestamp: Long): Boolean = RtpUtils.isOlderTimestampThan(this, otherTimestamp) /** * Returns true if getting to [otherSeqNum] from the current sequence number involves wrapping around */ infix fun Int.rolledOverTo(otherSeqNum: Int): Boolean = - /** - * If, according to [isOlderThan], [this] is older than [otherSeqNum] and - * yet [otherSeqNum] is less than [this], then we wrapped around to get from [this] to - * [otherSeqNum] - */ + // If, according to [isOlderThan], [this] is older than [otherSeqNum] and + // yet [otherSeqNum] is less than [this], then we wrapped around to get from [this] to + // [otherSeqNum] this isOlderThan otherSeqNum && otherSeqNum < this /** diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/extensions/ByteBufferExtensionsTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/extensions/ByteBufferExtensionsTest.kt index ebfe0c3f82..47abbc2c02 100644 --- a/rtp/src/test/kotlin/org/jitsi/rtp/extensions/ByteBufferExtensionsTest.kt +++ b/rtp/src/test/kotlin/org/jitsi/rtp/extensions/ByteBufferExtensionsTest.kt @@ -191,7 +191,7 @@ class ByteBufferExtensionsTest : ShouldSpec() { } context("moving the data past the capacity") { val buf = byteBufferOf(0x01, 0x02, 0x03, 0x04, 0x00, 0x0, 0x00, 0x00) - shouldThrow() { + shouldThrow { buf.shiftDataRight(4, 6, 10) } } @@ -206,7 +206,7 @@ class ByteBufferExtensionsTest : ShouldSpec() { } context("moving the data past the start") { val buf = byteBufferOf(0x01, 0x02, 0x03, 0x04, 0x00, 0x0, 0x00, 0x00) - shouldThrow() { + shouldThrow { buf.shiftDataLeft(1, 2, 2) } } diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacketTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacketTest.kt index 191a300f35..6144aa2230 100644 --- a/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacketTest.kt +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacketTest.kt @@ -53,7 +53,7 @@ class RtcpFbTccPacketTest : ShouldSpec() { /** * These correspond to the Deltas section above. */ - val expectedTccRlePacketInfo = mapOf ( + val expectedTccRlePacketInfo = mapOf( 0xfffa to 0xd8, 0xfffb to 0x00, 0xfffc to 0x18, @@ -94,7 +94,7 @@ class RtcpFbTccPacketTest : ShouldSpec() { // Recv delta padding 0x00, 0x00, 0x00 ) - val expectedTccMixedChunkTypePacketInfo = mapOf ( + val expectedTccMixedChunkTypePacketInfo = mapOf( 5376 to 2.toTicks(), 5377 to 0.toTicks(), 5378 to 0.toTicks(), @@ -129,7 +129,7 @@ class RtcpFbTccPacketTest : ShouldSpec() { // Recv delta padding 0x00 ) - val expectedTccSvChunkPacketInfo = mapOf ( + val expectedTccSvChunkPacketInfo = mapOf( 6227 to -1, 6228 to 107784064 + 27 ) diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpPacketTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpPacketTest.kt index 2d5f96e00b..623154fda9 100644 --- a/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpPacketTest.kt +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpPacketTest.kt @@ -942,8 +942,7 @@ class PaddingOnlyPacket private constructor( length: Int ) : RtpPacket(buffer, offset, length) { - override fun clone(): PaddingOnlyPacket = - throw NotImplementedError("clone() not supported for padding packets.") + override fun clone(): PaddingOnlyPacket = throw NotImplementedError("clone() not supported for padding packets.") companion object { /** From 63849ed73d69bf5bd26c78676acb4abfc03c5703 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Tue, 3 Oct 2023 16:39:38 -0500 Subject: [PATCH 054/189] fix: Fix ICE consent checks. (#2058) chore: Update ice4j kotlin to 1.9 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d6c757d636..93d6d43a41 100644 --- a/pom.xml +++ b/pom.xml @@ -106,7 +106,7 @@ ${project.groupId} ice4j - 3.0-63-g9fe2d69 + 3.0-66-g1c60acc ${project.groupId} From b09e7fcaa58cfe20c5fc412e01b2a761e39dd94d Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 5 Oct 2023 17:03:11 -0500 Subject: [PATCH 055/189] feat: Add an option to use "CPU usage" for stress. (#2060) --- .../org/jitsi/videobridge/Videobridge.java | 39 +----- .../load_management/CpuLoadSampler.kt | 26 ++++ .../load_management/CpuMeasurement.kt | 45 +++++++ .../load_management/JvbLoadManager.kt | 113 +++++++++++++++--- jvb/src/main/resources/reference.conf | 7 ++ 5 files changed, 181 insertions(+), 49 deletions(-) create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/load_management/CpuLoadSampler.kt create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/load_management/CpuMeasurement.kt diff --git a/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java b/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java index 4160002ad9..8b4ee46be0 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java @@ -49,7 +49,6 @@ import java.time.*; import java.util.*; -import java.util.concurrent.*; import java.util.concurrent.atomic.*; import java.util.stream.*; @@ -119,13 +118,8 @@ public class Videobridge /** * The {@link JvbLoadManager} instance used for this bridge. */ - private final JvbLoadManager jvbLoadManager; - - /** - * The task which manages the recurring load sampling and updating of - * {@link Videobridge#jvbLoadManager}. - */ - private final ScheduledFuture loadSamplerTask; + @NotNull + private final JvbLoadManager jvbLoadManager; @NotNull private final Version version; @Nullable private final String releaseId; @@ -158,29 +152,7 @@ public Videobridge( { this.clock = clock; videobridgeExpireThread = new VideobridgeExpireThread(this); - jvbLoadManager = new JvbLoadManager<>( - PacketRateMeasurement.getLoadedThreshold(), - PacketRateMeasurement.getRecoveryThreshold(), - new LastNReducer( - this::getConferences, - JvbLastNKt.jvbLastNSingleton - ) - ); - loadSamplerTask = TaskPools.SCHEDULED_POOL.scheduleAtFixedRate( - new PacketRateLoadSampler( - this, - (loadMeasurement) -> { - // Update the load manager with the latest measurement - jvbLoadManager.loadUpdate(loadMeasurement); - // Update the stats with the latest stress level - getStatistics().stressLevel = jvbLoadManager.getCurrentStressLevel(); - return Unit.INSTANCE; - } - ), - 0, - 10, - TimeUnit.SECONDS - ); + jvbLoadManager = JvbLoadManager.create(this); if (xmppConnection != null) { xmppConnection.setEventHandler(new XmppConnectionEventHandler()); @@ -610,10 +582,7 @@ public void start() public void stop() { videobridgeExpireThread.stop(); - if (loadSamplerTask != null) - { - loadSamplerTask.cancel(true); - } + jvbLoadManager.stop(); } /** diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/CpuLoadSampler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/CpuLoadSampler.kt new file mode 100644 index 0000000000..11b98aa453 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/CpuLoadSampler.kt @@ -0,0 +1,26 @@ +/* + * Copyright @ 2023 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.load_management + +import com.sun.management.OperatingSystemMXBean +import java.lang.management.ManagementFactory + +class CpuLoadSampler(private val newMeasurementHandler: (CpuMeasurement) -> Unit) : Runnable { + override fun run() { + val osBean = ManagementFactory.getPlatformMXBean(OperatingSystemMXBean::class.java) + newMeasurementHandler(CpuMeasurement(osBean.systemCpuLoad)) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/CpuMeasurement.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/CpuMeasurement.kt new file mode 100644 index 0000000000..6b021c2af0 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/CpuMeasurement.kt @@ -0,0 +1,45 @@ +/* + * Copyright @ 2023 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.load_management + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config + +class CpuMeasurement(private val value: Double) : JvbLoadMeasurement { + override fun getLoad(): Double = value + + override fun div(other: JvbLoadMeasurement): Double { + if (other !is CpuMeasurement) { + throw UnsupportedOperationException("Can only divide load measurements of same type") + } + return value / other.value + } + + override fun toString(): String = "CPU usage ${String.format("%.2f", value * 100)}%" + + companion object { + val loadThreshold: CpuMeasurement by config { + "${JvbLoadMeasurement.CONFIG_BASE}.cpu-usage.load-threshold" + .from(JitsiConfig.newConfig) + .convertFrom(::CpuMeasurement) + } + val recoverThreshold: CpuMeasurement by config { + "${JvbLoadMeasurement.CONFIG_BASE}.cpu-usage.recovery-threshold" + .from(JitsiConfig.newConfig) + .convertFrom(::CpuMeasurement) + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadManager.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadManager.kt index 991b0ca5b6..1961faf03a 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadManager.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadManager.kt @@ -23,20 +23,23 @@ import org.jitsi.nlj.util.NEVER import org.jitsi.utils.OrderedJsonObject import org.jitsi.utils.logging2.cdebug import org.jitsi.utils.logging2.createLogger +import org.jitsi.videobridge.Videobridge +import org.jitsi.videobridge.jvbLastNSingleton +import org.jitsi.videobridge.util.TaskPools import java.time.Clock import java.time.Duration import java.time.Instant +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit import java.util.logging.Level -class JvbLoadManager @JvmOverloads constructor( +open class JvbLoadManager @JvmOverloads constructor( private val jvbLoadThreshold: T, private val jvbRecoveryThreshold: T, - private val loadReducer: JvbLoadReducer, + private val loadReducer: JvbLoadReducer?, private val clock: Clock = Clock.systemUTC() ) { - private val logger = createLogger(minLogLevel = Level.ALL) - - val reducerEnabled: Boolean by config("videobridge.load-management.reducer-enabled".from(JitsiConfig.newConfig)) + protected val logger = createLogger(minLogLevel = Level.ALL) private var lastReducerTime: Instant = NEVER @@ -44,31 +47,42 @@ class JvbLoadManager @JvmOverloads constructor( private var mostRecentLoadMeasurement: T? = null + private var loadSamplerTask: ScheduledFuture<*>? = null + + protected fun startSampler(sampler: Runnable) { + loadSamplerTask = TaskPools.SCHEDULED_POOL.scheduleAtFixedRate(sampler, 0, 10, TimeUnit.SECONDS) + } + + fun stop() { + loadSamplerTask?.cancel(true) + loadSamplerTask = null + } + fun loadUpdate(loadMeasurement: T) { logger.cdebug { "Got a load measurement of $loadMeasurement" } mostRecentLoadMeasurement = loadMeasurement val now = clock.instant() if (loadMeasurement.getLoad() >= jvbLoadThreshold.getLoad()) { state = State.OVERLOADED - if (reducerEnabled) { + loadReducer?.let { logger.info("Load measurement $loadMeasurement is above threshold of $jvbLoadThreshold") if (canRunReducer(now)) { logger.info("Running load reducer") - loadReducer.reduceLoad() + it.reduceLoad() lastReducerTime = now } else { logger.info( "Load reducer ran at $lastReducerTime, which is within " + - "${loadReducer.impactTime()} of now, not running reduce" + "${it.impactTime()} of now, not running reduce" ) } } } else { state = State.NOT_OVERLOADED - if (reducerEnabled) { + loadReducer?.let { if (loadMeasurement.getLoad() < jvbRecoveryThreshold.getLoad()) { if (canRunReducer(now)) { - if (loadReducer.recover()) { + if (it.recover()) { logger.info( "Recovery ran after a load measurement of $loadMeasurement (which was " + "below threshold of $jvbRecoveryThreshold) was received" @@ -80,7 +94,7 @@ class JvbLoadManager @JvmOverloads constructor( } else { logger.cdebug { "Load measurement $loadMeasurement is below recovery threshold, but load reducer " + - "ran at $lastReducerTime, which is within ${loadReducer.impactTime()} of now, " + + "ran at $lastReducerTime, which is within ${it.impactTime()} of now, " + "not running recover" } } @@ -95,11 +109,17 @@ class JvbLoadManager @JvmOverloads constructor( put("state", state.toString()) put("stress", getCurrentStressLevel().toString()) put("reducer_enabled", reducerEnabled.toString()) - put("reducer", loadReducer.getStats()) + loadReducer?.let { + put("reducer", it.getStats()) + } } - private fun canRunReducer(now: Instant): Boolean = - Duration.between(lastReducerTime, now) >= loadReducer.impactTime() + private fun canRunReducer(now: Instant): Boolean { + loadReducer?.let { + return Duration.between(lastReducerTime, now) >= it.impactTime() + } + return false + } enum class State { OVERLOADED, @@ -110,5 +130,70 @@ class JvbLoadManager @JvmOverloads constructor( val averageParticipantStress: Double by config { "videobridge.load-management.average-participant-stress".from(JitsiConfig.newConfig) } + + val loadMeasurement: String by config { + "videobridge.load-management.load-measurements.load-measurement".from(JitsiConfig.newConfig) + } + + val reducerEnabled: Boolean by config("videobridge.load-management.reducer-enabled".from(JitsiConfig.newConfig)) + + const val PACKET_RATE_MEASUREMENT = "packet-rate" + const val CPU_USAGE_MEASUREMENT = "cpu-usage" + + @JvmStatic + fun create(videobridge: Videobridge): JvbLoadManager<*> { + val reducer = if (reducerEnabled) LastNReducer({ videobridge.conferences }, jvbLastNSingleton) else null + + return when (loadMeasurement) { + PACKET_RATE_MEASUREMENT -> PacketRateLoadManager( + PacketRateMeasurement.loadedThreshold, + PacketRateMeasurement.recoveryThreshold, + reducer, + videobridge + ) + CPU_USAGE_MEASUREMENT -> CpuUsageLoadManager( + CpuMeasurement.loadThreshold, + CpuMeasurement.recoverThreshold, + reducer, + videobridge + ) + else -> throw IllegalArgumentException( + "Invalid configuration for load measurement type: $loadMeasurement" + ) + } + } + } +} + +class PacketRateLoadManager( + loadThreshold: PacketRateMeasurement, + recoveryThreshold: PacketRateMeasurement, + loadReducer: JvbLoadReducer?, + videobridge: Videobridge +) : JvbLoadManager(loadThreshold, recoveryThreshold, loadReducer) { + + init { + val sampler = PacketRateLoadSampler(videobridge) { loadMeasurement -> + loadUpdate(loadMeasurement) + videobridge.statistics.stressLevel = loadMeasurement.getLoad() + } + + startSampler(sampler) + } +} + +class CpuUsageLoadManager( + loadThreshold: CpuMeasurement, + recoveryThreshold: CpuMeasurement, + loadReducer: JvbLoadReducer?, + videobridge: Videobridge +) : JvbLoadManager(loadThreshold, recoveryThreshold, loadReducer) { + init { + val sampler = CpuLoadSampler { loadMeasurement -> + loadUpdate(loadMeasurement) + videobridge.statistics.stressLevel = loadMeasurement.getLoad() + } + + startSampler(sampler) } } diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index 3e93c45ff7..16dab152ca 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -165,6 +165,13 @@ videobridge { # to start recovery recovery-threshold = 40000 } + cpu-usage { + load-threshold = 0.9 + recovery-threshold = 0.72 + } + + # Which of the available measurements to use, either "packet-rate" or "cpu-usage". + load-measurement = "packet-rate" } load-reducers { last-n { From fe8725051a426a620b347525bd1aa0b2cd082553 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Mon, 9 Oct 2023 17:14:41 -0500 Subject: [PATCH 056/189] fix: Fix setting the stress level. (#2062) --- .../org/jitsi/videobridge/load_management/JvbLoadManager.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadManager.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadManager.kt index 1961faf03a..fb8d542a94 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadManager.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadManager.kt @@ -175,7 +175,7 @@ class PacketRateLoadManager( init { val sampler = PacketRateLoadSampler(videobridge) { loadMeasurement -> loadUpdate(loadMeasurement) - videobridge.statistics.stressLevel = loadMeasurement.getLoad() + videobridge.statistics.stressLevel = getCurrentStressLevel() } startSampler(sampler) @@ -191,7 +191,7 @@ class CpuUsageLoadManager( init { val sampler = CpuLoadSampler { loadMeasurement -> loadUpdate(loadMeasurement) - videobridge.statistics.stressLevel = loadMeasurement.getLoad() + videobridge.statistics.stressLevel = getCurrentStressLevel() } startSampler(sampler) From c19677628b3edc1ff5bfb7c7cacc92abaa903222 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 26 Oct 2023 10:54:37 -0700 Subject: [PATCH 057/189] fix: Avoid calling IQ.toXML().toString() (#2064) * chore: Update jitsi-xmpp-extensions. * fix: Avoid using XmlStringBuilder.toString(). --- .../java/org/jitsi/videobridge/Conference.java | 7 ++++--- .../videobridge/xmpp/MediaSourceFactory.java | 3 ++- .../kotlin/org/jitsi/videobridge/Endpoint.kt | 5 +++-- .../colibri2/Colibri2ConferenceHandler.kt | 17 ++++++++++------- .../kotlin/org/jitsi/videobridge/relay/Relay.kt | 5 +++-- .../videobridge/transport/ice/IceTransport.kt | 3 ++- .../jitsi/videobridge/util/PayloadTypeUtil.kt | 3 ++- .../jitsi/videobridge/xmpp/XmppConnection.kt | 9 +++++---- .../videobridge/colibri2/Colibri2UtilTest.kt | 5 +++-- pom.xml | 2 +- 10 files changed, 35 insertions(+), 24 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/Conference.java b/jvb/src/main/java/org/jitsi/videobridge/Conference.java index cd2cfae219..a7afe7ee6f 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Conference.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Conference.java @@ -34,6 +34,7 @@ import org.jitsi.videobridge.util.*; import org.jitsi.videobridge.xmpp.*; import org.jitsi.xmpp.extensions.colibri2.*; +import org.jitsi.xmpp.util.*; import org.jivesoftware.smack.packet.*; import org.json.simple.*; import org.jxmpp.jid.*; @@ -230,7 +231,7 @@ public Conference(Videobridge videobridge, { try { - logger.info("RECV colibri2 request: " + request.getRequest().toXML()); + logger.info("RECV colibri2 request: " + XmlStringBuilderUtil.toStringOpt(request.getRequest())); long start = System.currentTimeMillis(); Pair p = colibri2Handler.handleConferenceModifyIQ(request.getRequest()); IQ response = p.getFirst(); @@ -243,9 +244,9 @@ public Conference(Videobridge videobridge, if (processingDelay > 100) { logger.warn("Took " + processingDelay + " ms to process an IQ (total delay " - + totalDelay + " ms): " + request.getRequest().toXML()); + + totalDelay + " ms): " + XmlStringBuilderUtil.toStringOpt(request.getRequest())); } - logger.info("SENT colibri2 response: " + response.toXML()); + logger.info("SENT colibri2 response: " + XmlStringBuilderUtil.toStringOpt(response)); request.getCallback().invoke(response); if (expire) videobridge.expireConference(this); } diff --git a/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java b/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java index 943b66f5da..44d1b78051 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java +++ b/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java @@ -21,6 +21,7 @@ import org.jitsi.xmpp.extensions.colibri.*; import org.jitsi.xmpp.extensions.jingle.*; import org.jitsi.xmpp.extensions.jitsimeet.*; +import org.jitsi.xmpp.util.*; import org.jivesoftware.smack.packet.*; import org.jxmpp.jid.parts.*; import org.jxmpp.util.*; @@ -402,7 +403,7 @@ private static List getSourceSsrcs( logger.warn( "Unprocessed source groups: " + sourceGroupsCopy.stream() - .map(e -> e.toXML(XmlEnvironment.EMPTY).toString()) + .map(XmlStringBuilderUtil::toStringOpt) .reduce(String::concat)); } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index 011146144e..01cb09af3f 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -82,6 +82,7 @@ import org.jitsi.videobridge.websocket.colibriWebSocketServiceSupplier import org.jitsi.xmpp.extensions.colibri.WebSocketPacketExtension import org.jitsi.xmpp.extensions.jingle.DtlsFingerprintPacketExtension import org.jitsi.xmpp.extensions.jingle.IceUdpTransportPacketExtension +import org.jitsi.xmpp.util.XmlStringBuilderUtil.Companion.toStringOpt import org.jitsi_modified.sctp4j.SctpDataCallback import org.jitsi_modified.sctp4j.SctpServerSocket import org.jitsi_modified.sctp4j.SctpSocket @@ -781,7 +782,7 @@ class Endpoint @JvmOverloads constructor( if (fingerprintExtension.hash != null && fingerprintExtension.fingerprint != null) { remoteFingerprints[fingerprintExtension.hash] = fingerprintExtension.fingerprint } else { - logger.info("Ignoring empty DtlsFingerprint extension: ${transportInfo.toXML()}") + logger.info("Ignoring empty DtlsFingerprint extension: ${transportInfo.toStringOpt()}") } if (CryptexConfig.endpoint) { cryptex = cryptex && fingerprintExtension.cryptex @@ -810,7 +811,7 @@ class Endpoint @JvmOverloads constructor( } } - logger.cdebug { "Transport description:\n${iceUdpTransportPacketExtension.toXML()}" } + logger.cdebug { "Transport description:\n${iceUdpTransportPacketExtension.toStringOpt()}" } return iceUdpTransportPacketExtension } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt index ad51657edd..8e65fb4f1b 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt @@ -47,6 +47,7 @@ import org.jitsi.xmpp.extensions.colibri2.Sources import org.jitsi.xmpp.extensions.colibri2.Transport import org.jitsi.xmpp.extensions.jingle.RTPHdrExtPacketExtension import org.jitsi.xmpp.extensions.jingle.SourceGroupPacketExtension +import org.jitsi.xmpp.util.XmlStringBuilderUtil.Companion.toStringOpt import org.jitsi.xmpp.util.createError import org.jivesoftware.smack.packet.IQ import org.jivesoftware.smack.packet.StanzaError.Condition @@ -109,7 +110,7 @@ class Colibri2ConferenceHandler( val error = createEndpointNotFoundError(conferenceModifyIQ, e.endpointId) Pair(error, false) } catch (e: FeatureNotImplementedException) { - logger.warn("Unsupported request (${e.message}): ${conferenceModifyIQ.toXML()}") + logger.warn("Unsupported request (${e.message}): ${conferenceModifyIQ.toStringOpt()}") Pair(createFeatureNotImplementedError(conferenceModifyIQ, e.message), false) } catch (e: IqProcessingException) { // Item not found conditions are assumed to be less critical, as they often happen in case a request @@ -226,12 +227,12 @@ class Colibri2ConferenceHandler( // TODO: support removing payload types/header extensions media.payloadTypes.forEach { ptExt -> create(ptExt, media.type)?.let { endpoint.addPayloadType(it) } - ?: logger.warn("Ignoring unrecognized payload type extension: ${ptExt.toXML()}") + ?: logger.warn("Ignoring unrecognized payload type extension: ${ptExt.toStringOpt()}") } media.rtpHdrExts.forEach { rtpHdrExt -> rtpHdrExt.toRtpExtension()?.let { endpoint.addRtpExtension(it) } - ?: logger.warn("Ignoring unrecognized RTP header extension: ${rtpHdrExt.toXML()}") + ?: logger.warn("Ignoring unrecognized RTP header extension: ${rtpHdrExt.toStringOpt()}") } endpoint.setExtmapAllowMixed(media.extmapAllowMixed != null) @@ -317,7 +318,7 @@ class Colibri2ConferenceHandler( private fun SourceGroupPacketExtension.toSsrcAssociation(): SsrcAssociation? { if (sources.size < 2) { - logger.warn("Ignoring source group with <2 sources: ${toXML()}") + logger.warn("Ignoring source group with <2 sources: ${toStringOpt()}") return null } @@ -423,12 +424,12 @@ class Colibri2ConferenceHandler( // TODO: support removing payload types/header extensions media.payloadTypes.forEach { ptExt -> create(ptExt, media.type)?.let { relay.addPayloadType(it) } - ?: logger.warn("Ignoring unrecognized payload type extension: ${ptExt.toXML()}") + ?: logger.warn("Ignoring unrecognized payload type extension: ${ptExt.toStringOpt()}") } media.rtpHdrExts.forEach { rtpHdrExt -> rtpHdrExt.toRtpExtension()?.let { relay.addRtpExtension(it) } - ?: logger.warn("Ignoring unrecognized RTP header extension: ${rtpHdrExt.toXML()}") + ?: logger.warn("Ignoring unrecognized RTP header extension: ${rtpHdrExt.toStringOpt()}") } relay.setExtmapAllowMixed(media.extmapAllowMixed != null) @@ -473,7 +474,9 @@ class Colibri2ConferenceHandler( it.mediaSources.forEach { m -> if (m.type == MediaType.AUDIO) { if (m.sources.isEmpty()) { - logger.warn("Ignoring audio source ${m.id} in endpoint $id of a relay (no SSRCs): ${toXML()}") + logger.warn( + "Ignoring audio source ${m.id} in endpoint $id of a relay (no SSRCs): ${toStringOpt()}" + ) } else { m.sources.forEach { audioSources.add(AudioSourceDesc(it.ssrc, id, m.id)) } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt index 8018bec71f..9e4ba6dd11 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt @@ -91,6 +91,7 @@ import org.jitsi.xmpp.extensions.colibri.WebSocketPacketExtension import org.jitsi.xmpp.extensions.colibri2.Sctp import org.jitsi.xmpp.extensions.jingle.DtlsFingerprintPacketExtension import org.jitsi.xmpp.extensions.jingle.IceUdpTransportPacketExtension +import org.jitsi.xmpp.util.XmlStringBuilderUtil.Companion.toStringOpt import org.jitsi_modified.sctp4j.SctpClientSocket import org.jitsi_modified.sctp4j.SctpDataCallback import org.jitsi_modified.sctp4j.SctpServerSocket @@ -563,7 +564,7 @@ class Relay @JvmOverloads constructor( if (fingerprintExtension.hash != null && fingerprintExtension.fingerprint != null) { remoteFingerprints[fingerprintExtension.hash] = fingerprintExtension.fingerprint } else { - logger.info("Ignoring empty DtlsFingerprint extension: ${transportInfo.toXML()}") + logger.info("Ignoring empty DtlsFingerprint extension: ${transportInfo.toStringOpt()}") } if (CryptexConfig.relay) { @@ -615,7 +616,7 @@ class Relay @JvmOverloads constructor( } } - logger.cdebug { "Transport description:\n${iceUdpTransportPacketExtension.toXML()}" } + logger.cdebug { "Transport description:\n${iceUdpTransportPacketExtension.toStringOpt()}" } return iceUdpTransportPacketExtension } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt index 328e775991..f67d86bc9a 100755 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt @@ -40,6 +40,7 @@ import org.jitsi.xmpp.extensions.jingle.CandidatePacketExtension import org.jitsi.xmpp.extensions.jingle.IceCandidatePacketExtension import org.jitsi.xmpp.extensions.jingle.IceRtcpmuxPacketExtension import org.jitsi.xmpp.extensions.jingle.IceUdpTransportPacketExtension +import org.jitsi.xmpp.util.XmlStringBuilderUtil.Companion.toStringOpt import java.beans.PropertyChangeEvent import java.beans.PropertyChangeListener import java.io.IOException @@ -204,7 +205,7 @@ class IceTransport @JvmOverloads constructor( logger.info("Starting the Agent without remote candidates.") iceAgent.startConnectivityEstablishment() } else { - logger.cdebug { "Not starting ICE, no ufrag and pwd yet. ${transportPacketExtension.toXML()}" } + logger.cdebug { "Not starting ICE, no ufrag and pwd yet. ${transportPacketExtension.toStringOpt()}" } } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/util/PayloadTypeUtil.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/util/PayloadTypeUtil.kt index 026a07980c..c0459a5b5a 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/util/PayloadTypeUtil.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/util/PayloadTypeUtil.kt @@ -39,6 +39,7 @@ import org.jitsi.utils.MediaType.VIDEO import org.jitsi.utils.logging2.Logger import org.jitsi.utils.logging2.LoggerImpl import org.jitsi.xmpp.extensions.jingle.PayloadTypePacketExtension +import org.jitsi.xmpp.util.XmlStringBuilderUtil.Companion.toStringOpt import java.util.concurrent.ConcurrentHashMap /** @@ -71,7 +72,7 @@ class PayloadTypeUtil { if (parameter.name != null) { parameters[parameter.name] = parameter.value } else { - logger.warn("Ignoring a format parameter with no name: " + parameter.toXML()) + logger.warn("Ignoring a format parameter with no name: " + parameter.toStringOpt()) } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt index 7833a4794d..7aa792c40a 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt @@ -29,6 +29,7 @@ import org.jitsi.xmpp.mucclient.IQListener import org.jitsi.xmpp.mucclient.MucClient import org.jitsi.xmpp.mucclient.MucClientConfiguration import org.jitsi.xmpp.mucclient.MucClientManager +import org.jitsi.xmpp.util.XmlStringBuilderUtil.Companion.toStringOpt import org.jitsi.xmpp.util.createError import org.jivesoftware.smack.packet.ExtensionElement import org.jivesoftware.smack.packet.IQ @@ -184,12 +185,12 @@ class XmppConnection : IQListener { } // colibri2 requests are logged at the conference level. if (iq !is ConferenceModifyIQ) { - logger.cdebug { "RECV: ${iq.toXML()}" } + logger.cdebug { "RECV: ${iq.toStringOpt()}" } } return when (iq.type) { IQ.Type.get, IQ.Type.set -> handleIqRequest(iq, mucClient)?.also { - logger.cdebug { "SENT: ${it.toXML()}" } + logger.cdebug { "SENT: ${it.toStringOpt()}" } } else -> null } @@ -202,7 +203,7 @@ class XmppConnection : IQListener { "Service unavailable" ) val response = when (iq) { - is Version -> measureDelay(versionDelayStats, { iq.toXML() }) { + is Version -> measureDelay(versionDelayStats, { iq.toStringOpt() }) { handler.versionIqReceived(iq) } is ConferenceModifyIQ -> { @@ -215,7 +216,7 @@ class XmppConnection : IQListener { ) null } - is HealthCheckIQ -> measureDelay(healthDelayStats, { iq.toXML() }) { + is HealthCheckIQ -> measureDelay(healthDelayStats, { iq.toStringOpt() }) { handler.healthCheckIqReceived(iq) } else -> createError( diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/colibri2/Colibri2UtilTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/colibri2/Colibri2UtilTest.kt index 3f1c0e7ace..60f38b9aa5 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/colibri2/Colibri2UtilTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/colibri2/Colibri2UtilTest.kt @@ -21,6 +21,7 @@ import org.jitsi.videobridge.colibri2.createConferenceNotFoundError import org.jitsi.xmpp.extensions.colibri2.Colibri2Error import org.jitsi.xmpp.extensions.colibri2.ConferenceModifyIQ import org.jitsi.xmpp.extensions.colibri2.IqProviderUtils +import org.jitsi.xmpp.util.XmlStringBuilderUtil.Companion.toStringOpt import org.jivesoftware.smack.util.PacketParserUtils class Colibri2UtilTest : ShouldSpec({ @@ -29,7 +30,7 @@ class Colibri2UtilTest : ShouldSpec({ context("createConferenceAlreadyExistsError") { val error = createConferenceAlreadyExistsError(iq, "i") - val parsedIq = parseIQ(error.toXML().toString()) + val parsedIq = parseIQ(error.toStringOpt()) val colibri2ErrorExtension = parsedIq.error.getExtension(Colibri2Error.ELEMENT, Colibri2Error.NAMESPACE) colibri2ErrorExtension shouldNotBe null @@ -39,7 +40,7 @@ class Colibri2UtilTest : ShouldSpec({ context("createConferenceNotFoundError") { val error = createConferenceNotFoundError(iq, "i") - val parsedIq = parseIQ(error.toXML().toString()) + val parsedIq = parseIQ(error.toStringOpt()) val colibri2ErrorExtension = parsedIq.error.getExtension(Colibri2Error.ELEMENT, Colibri2Error.NAMESPACE) colibri2ErrorExtension shouldNotBe null diff --git a/pom.xml b/pom.xml index 93d6d43a41..fdbec3db58 100644 --- a/pom.xml +++ b/pom.xml @@ -111,7 +111,7 @@ ${project.groupId} jitsi-xmpp-extensions - 1.0-76-ge98f8af + 1.0-78-g62d03d4 From b88544ebaadcff7cc82bf8afd4260a183566cf97 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 26 Oct 2023 14:28:43 -0700 Subject: [PATCH 058/189] doc: Fix address in debug docs. (#2063) --- doc/debugging.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/debugging.md b/doc/debugging.md index 512be8b590..057b2fe8dd 100644 --- a/doc/debugging.md +++ b/doc/debugging.md @@ -197,10 +197,10 @@ allow it by setting `jmt.debug.pcap.enabled=true` in `/etc/jitsi/videobridge/jvb Enable: ``` -POST /debug/stats/endpoint/CONFERENCE_ID/ENDPOINT_ID/pcap-dump/true +POST /debug/features/endpoint/CONFERENCE_ID/ENDPOINT_ID/pcap-dump/true ``` Disable: ``` -POST /debug/stats/endpoint/CONFERENCE_ID/ENDPOINT_ID/pcap-dump/false +POST /debug/features/endpoint/CONFERENCE_ID/ENDPOINT_ID/pcap-dump/false ``` From 7a4dc8b8dfa3c5f3c36255d1011b94cc2bb53001 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Thu, 2 Nov 2023 17:54:45 -0400 Subject: [PATCH 059/189] Support for video described by the AV1 Dependency Descriptor (#1998) --- .../kotlin/org/jitsi/nlj/MediaSourceDesc.kt | 18 +- .../kotlin/org/jitsi/nlj/RtpEncodingDesc.kt | 39 +- .../main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt | 131 +- .../kotlin/org/jitsi/nlj/RtpReceiverImpl.kt | 2 +- .../org/jitsi/nlj/rtp/ParsedVideoPacket.kt | 4 +- .../kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt | 9 +- .../org/jitsi/nlj/rtp/VideoRtpPacket.kt | 24 +- .../jitsi/nlj/rtp/codec/VideoCodecParser.kt | 5 +- .../jitsi/nlj/rtp/codec/av1/Av1DDPacket.kt | 226 +++ .../jitsi/nlj/rtp/codec/av1/Av1DDParser.kt | 179 ++ .../nlj/rtp/codec/av1/Av1DDRtpLayerDesc.kt | 120 ++ .../org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt | 16 +- .../org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt | 29 +- .../org/jitsi/nlj/rtp/codec/vp9/Vp9Parser.kt | 8 +- .../nlj/rtp/codec/vpx/VpxRtpLayerDesc.kt | 190 ++ .../node/incoming/BitrateCalculator.kt | 6 +- .../transform/node/incoming/VideoParser.kt | 37 +- .../node/incoming/VideoQualityLayerLookup.kt | 8 +- .../node/outgoing/HeaderExtStripper.kt | 17 +- .../org/jitsi/nlj/util/Rfc3711IndexTracker.kt | 13 + .../kotlin/org/jitsi/nlj/util/TreeCache.kt | 62 + .../org/jitsi/nlj/MediaSourceDescTest.kt | 15 +- .../kotlin/org/jitsi/nlj/RtpLayerDescTest.kt | 37 +- .../nlj/rtp/codec/av1/Av1DDPacketTest.kt | 211 +++ .../jitsi/nlj/rtp/codec/vp9/Vp9PacketTest.kt | 50 +- .../org/jitsi/nlj/util/TreeCacheTest.kt | 100 ++ .../cc/AdaptiveSourceProjection.java | 110 +- .../cc/AdaptiveSourceProjectionContext.java | 13 +- ...enericAdaptiveSourceProjectionContext.java | 16 +- .../VP8AdaptiveSourceProjectionContext.java | 26 +- .../videobridge/cc/vp8/VP8QualityFilter.java | 24 +- .../videobridge/xmpp/MediaSourceFactory.java | 19 +- .../kotlin/org/jitsi/videobridge/SsrcCache.kt | 58 + .../cc/allocation/BandwidthAllocation.kt | 13 + .../cc/allocation/BitrateController.kt | 2 - .../cc/allocation/PacketHandler.kt | 7 - .../cc/allocation/SingleSourceAllocation.kt | 3 +- .../Av1DDAdaptiveSourceProjectionContext.kt | 659 +++++++ .../jitsi/videobridge/cc/av1/Av1DDFrame.kt | 309 ++++ .../jitsi/videobridge/cc/av1/Av1DDFrameMap.kt | 254 +++ .../cc/av1/Av1DDFrameProjection.kt | 240 +++ .../videobridge/cc/av1/Av1DDQualityFilter.kt | 460 +++++ .../vp9/Vp9AdaptiveSourceProjectionContext.kt | 17 +- .../jitsi/videobridge/cc/vp9/Vp9PictureMap.kt | 2 +- .../videobridge/cc/vp9/Vp9QualityFilter.kt | 62 +- .../vp8/VP8AdaptiveSourceProjectionTest.java | 82 +- .../cc/allocation/BitrateControllerTest.kt | 7 + .../av1/Av1DDAdaptiveSourceProjectionTest.kt | 1571 +++++++++++++++++ .../cc/av1/Av1DDQualityFilterTest.kt | 896 ++++++++++ .../cc/vp9/Vp9AdaptiveSourceProjectionTest.kt | 245 +-- .../cc/vp9/Vp9QualityFilterTest.kt | 12 +- rtp/pom.xml | 6 + rtp/spotbugs-exclude.xml | 2 + .../kotlin/org/jitsi/rtp/rtcp/RtcpSrPacket.kt | 10 + .../kotlin/org/jitsi/rtp/rtp/RtpPacket.kt | 18 +- .../Av1DependencyDescriptorHeaderExtension.kt | 902 ++++++++++ .../kotlin/org/jitsi/rtp/util/BitReader.kt | 105 ++ .../kotlin/org/jitsi/rtp/util/BitWriter.kt | 73 + ...DependencyDescriptorHeaderExtensionTest.kt | 549 ++++++ .../DumpAv1DependencyDescriptor.kt | 36 + 60 files changed, 7695 insertions(+), 669 deletions(-) create mode 100644 jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacket.kt create mode 100644 jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDParser.kt create mode 100644 jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDRtpLayerDesc.kt create mode 100644 jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vpx/VpxRtpLayerDesc.kt create mode 100644 jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/TreeCache.kt create mode 100644 jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacketTest.kt create mode 100644 jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/TreeCacheTest.kt create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrame.kt create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameMap.kt create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameProjection.kt create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt create mode 100644 jvb/src/test/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionTest.kt create mode 100644 jvb/src/test/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilterTest.kt create mode 100644 rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtension.kt create mode 100644 rtp/src/main/kotlin/org/jitsi/rtp/util/BitReader.kt create mode 100644 rtp/src/main/kotlin/org/jitsi/rtp/util/BitWriter.kt create mode 100644 rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtensionTest.kt create mode 100644 rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/DumpAv1DependencyDescriptor.kt diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt index e09c7f2ee7..7256df7659 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt @@ -137,13 +137,11 @@ class MediaSourceDesc fun getRtpLayerByQualityIdx(idx: Int): RtpLayerDesc? = layersByIndex[idx] @Synchronized - fun findRtpLayerDesc(videoRtpPacket: VideoRtpPacket): RtpLayerDesc? { + fun findRtpLayerDescs(videoRtpPacket: VideoRtpPacket): Collection { if (ArrayUtils.isNullOrEmpty(rtpEncodings)) { - return null + return emptyList() } - val encodingId = videoRtpPacket.getEncodingId() - val desc = layersById[encodingId] - return desc + return videoRtpPacket.getEncodingIds().mapNotNull { layersById[it] } } @Synchronized @@ -188,10 +186,14 @@ class MediaSourceDesc */ fun Array.copy() = Array(this.size) { i -> this[i].copy() } -fun Array.findRtpLayerDesc(packet: VideoRtpPacket): RtpLayerDesc? { +fun Array.findRtpLayerDescs(packet: VideoRtpPacket): Collection { + return this.flatMap { it.findRtpLayerDescs(packet) } +} + +fun Array.findRtpEncodingId(packet: VideoRtpPacket): Int? { for (source in this) { - source.findRtpLayerDesc(packet)?.let { - return it + source.findRtpEncodingDesc(packet.ssrc)?.let { + return it.eid } } return null diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt index 8cfee2f807..570c5dde53 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt @@ -73,9 +73,17 @@ constructor( validateLayerEids(initialLayers) } + private var nominalHeight = initialLayers.getNominalHeight() + internal var layers = initialLayers set(newLayers) { validateLayerEids(newLayers) + /* Check if the new layer set is a single spatial layer that doesn't specify a height - if so, we + * want to apply the nominal height to them. + */ + val useNominalHeight = nominalHeight != RtpLayerDesc.NO_HEIGHT && + newLayers.all { it.sid == 0 } && + newLayers.all { it.height == RtpLayerDesc.NO_HEIGHT } /* Copy the rate statistics objects from the old layers to the new layers * with matching layer IDs. */ @@ -89,6 +97,15 @@ constructor( oldLayerMap[newLayer.layerId]?.let { newLayer.inheritFrom(it) } + if (useNominalHeight) { + newLayer.height = nominalHeight + } + } + if (!useNominalHeight) { + val newNominalHeight = newLayers.getNominalHeight() + if (newNominalHeight != RtpLayerDesc.NO_HEIGHT) { + nominalHeight = newNominalHeight + } } field = newLayers } @@ -157,6 +174,7 @@ constructor( addNumber("rtx_ssrc", getSecondarySsrc(SsrcAssociationType.RTX)) addNumber("fec_ssrc", getSecondarySsrc(SsrcAssociationType.FEC)) addNumber("eid", eid) + addNumber("nominal_height", nominalHeight) for (layer in layers) { addBlock(layer.getNodeStats()) } @@ -167,6 +185,23 @@ constructor( } } -fun VideoRtpPacket.getEncodingId(): Long { - return RtpEncodingDesc.calcEncodingId(ssrc, this.layerId) +fun VideoRtpPacket.getEncodingIds(): Collection { + return this.layerIds.map { RtpEncodingDesc.calcEncodingId(ssrc, it) } +} + +/** + * Get the "nominal" height of a set of layers - if they all indicate the same spatial layer and same height. + */ +private fun Array.getNominalHeight(): Int { + if (isEmpty()) { + return RtpLayerDesc.NO_HEIGHT + } + val firstHeight = first().height + if (!(all { it.sid == 0 } || all { it.sid == -1 })) { + return RtpLayerDesc.NO_HEIGHT + } + if (any { it.height != firstHeight }) { + return RtpLayerDesc.NO_HEIGHT + } + return firstHeight } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt index 2f7019b06d..9cd30d9383 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt @@ -20,106 +20,60 @@ import org.jitsi.nlj.transform.node.incoming.BitrateCalculator import org.jitsi.nlj.util.Bandwidth import org.jitsi.nlj.util.BitrateTracker import org.jitsi.nlj.util.DataSize -import org.jitsi.nlj.util.sum +import org.jitsi.utils.OrderedJsonObject /** * Keeps track of its subjective quality index, * its last stable bitrate and other useful things for adaptivity/routing. * - * Note: this class and [getBitrate] are only open to allow to be overridden for testing. We found that mocking has - * severe overhead and is not suitable for performance tests. - * * @author George Politis */ -open class RtpLayerDesc -@JvmOverloads +abstract class RtpLayerDesc constructor( /** * The index of this instance's encoding in the source encoding array. */ val eid: Int, /** - * The temporal layer ID of this instance, or negative for unknown. + * The temporal layer ID of this instance. */ val tid: Int, /** - * The spatial layer ID of this instance, or negative for unknown. + * The spatial layer ID of this instance. */ val sid: Int, /** * The max height of the bitstream that this instance represents. The actual - * height may be less due to bad network or system load. + * height may be less due to bad network or system load. [NO_HEIGHT] for unknown. * * XXX we should be able to sniff the actual height from the RTP packets. */ - val height: Int, + var height: Int, /** * The max frame rate (in fps) of the bitstream that this instance * represents. The actual frame rate may be less due to bad network or - * system load. + * system load. [NO_FRAME_RATE] for unknown. */ val frameRate: Double, - /** - * The [RtpLayerDesc]s on which this layer definitely depends. - */ - private val dependencyLayers: Array = emptyArray(), - /** - * The [RtpLayerDesc]s on which this layer possibly depends. - * (The intended use case is K-SVC mode.) - */ - private val softDependencyLayers: Array = emptyArray() ) { - init { - require(tid < 8) { "Invalid temporal ID $tid" } - require(sid < 8) { "Invalid spatial ID $sid" } - } - - /** - * Clone an existing layer desc, inheriting its statistics, - * modifying only specific values. - */ - fun copy( - eid: Int = this.eid, - tid: Int = this.tid, - sid: Int = this.sid, - height: Int = this.height, - frameRate: Double = this.frameRate, - dependencyLayers: Array = this.dependencyLayers, - softDependencyLayers: Array = this.softDependencyLayers - ) = RtpLayerDesc(eid, tid, sid, height, frameRate, dependencyLayers, softDependencyLayers).also { - it.inheritFrom(this) - } - - /** - * Whether softDependencyLayers are to be used. - */ - var useSoftDependencies = true + abstract fun copy(height: Int = this.height): RtpLayerDesc /** * The [BitrateTracker] instance used to calculate the receiving bitrate of this RTP layer. */ - private var bitrateTracker = BitrateCalculator.createBitrateTracker() + protected var bitrateTracker = BitrateCalculator.createBitrateTracker() /** * @return the "id" of this layer within this encoding. This is a server-side id and should * not be confused with any encoding id defined in the client (such as the * rid). */ - val layerId = getIndex(0, sid, tid) + abstract val layerId: Int /** * A local index of this track. */ - val index = getIndex(eid, sid, tid) - - /** - * {@inheritDoc} - */ - override fun toString(): String { - return "subjective_quality=" + index + - ",temporal_id=" + tid + - ",spatial_id=" + sid - } + abstract val index: Int /** * Inherit a [BitrateTracker] object @@ -131,9 +85,8 @@ constructor( /** * Inherit another layer description's [BitrateTracker] object. */ - internal fun inheritFrom(other: RtpLayerDesc) { + internal open fun inheritFrom(other: RtpLayerDesc) { inheritStatistics(other.bitrateTracker) - useSoftDependencies = other.useSoftDependencies } /** @@ -152,12 +105,10 @@ constructor( /** * Gets the cumulative bitrate (in bps) of this [RtpLayerDesc] and its dependencies. * - * This is left open for use in testing. - * * @param nowMs * @return the cumulative bitrate (in bps) of this [RtpLayerDesc] and its dependencies. */ - open fun getBitrate(nowMs: Long): Bandwidth = calcBitrate(nowMs).values.sum() + abstract fun getBitrate(nowMs: Long): Bandwidth /** * Expose [getBitrate] as a [Double] in order to make it accessible from java (since [Bandwidth] is an inline @@ -165,67 +116,27 @@ constructor( */ fun getBitrateBps(nowMs: Long): Double = getBitrate(nowMs).bps - /** - * Recursively adds the bitrate (in bps) of this [RTPLayerDesc] and - * its dependencies in the map passed in as an argument. - * - * This is necessary to ensure we don't double-count layers in cases - * of multiple dependencies. - * - * @param nowMs - */ - private fun calcBitrate(nowMs: Long, rates: MutableMap = HashMap()): MutableMap { - if (rates.containsKey(index)) { - return rates - } - rates[index] = bitrateTracker.getRate(nowMs) - - dependencyLayers.forEach { it.calcBitrate(nowMs, rates) } - - if (useSoftDependencies) { - softDependencyLayers.forEach { it.calcBitrate(nowMs, rates) } - } - - return rates - } - - /** - * Returns true if this layer, alone, has a zero bitrate. - */ - private fun layerHasZeroBitrate(nowMs: Long) = bitrateTracker.getAccumulatedSize(nowMs).bits == 0L - /** * Recursively checks this layer and its dependencies to see if the bitrate is zero. * Note that unlike [calcBitrate] this does not avoid double-visiting layers; the overhead * of the hash table is usually more than the cost of any double-visits. - * - * This is left open for use in testing. */ - open fun hasZeroBitrate(nowMs: Long): Boolean { - if (!layerHasZeroBitrate(nowMs)) { - return false - } - if (dependencyLayers.any { !it.layerHasZeroBitrate(nowMs) }) { - return false - } - if (useSoftDependencies && softDependencyLayers.any { !it.layerHasZeroBitrate(nowMs) }) { - return false - } - return true - } + abstract fun hasZeroBitrate(nowMs: Long): Boolean /** * Extracts a [NodeStatsBlock] from an [RtpLayerDesc]. */ - fun getNodeStats() = NodeStatsBlock(indexString(index)).apply { + open fun getNodeStats() = NodeStatsBlock(indexString()).apply { addNumber("frameRate", frameRate) addNumber("height", height) addNumber("index", index) addNumber("bitrate_bps", getBitrate(System.currentTimeMillis()).bps) - addNumber("tid", tid) - addNumber("sid", sid) } + fun debugState(): OrderedJsonObject = getNodeStats().toJson().apply { put("indexString", indexString()) } + + abstract fun indexString(): String + companion object { /** * The index value that is used to represent that forwarding is suspended. @@ -270,14 +181,14 @@ constructor( fun getEidFromIndex(index: Int) = index shr 6 /** - * Get an spatial ID from a layer index. If the index is [SUSPENDED_INDEX], + * Get a spatial ID from a layer index. If the index is [SUSPENDED_INDEX], * the value is unspecified. */ @JvmStatic fun getSidFromIndex(index: Int) = (index and 0x38) shr 3 /** - * Get an temporal ID from a layer index. If the index is [SUSPENDED_INDEX], + * Get a temporal ID from a layer index. If the index is [SUSPENDED_INDEX], * the value is unspecified. */ @JvmStatic diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt index edd15e2be6..8f749fef38 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt @@ -143,7 +143,7 @@ class RtpReceiverImpl @JvmOverloads constructor( private val videoBitrateCalculator = VideoBitrateCalculator(parentLogger) private val audioBitrateCalculator = BitrateCalculator("Audio bitrate calculator") - private val videoParser = VideoParser(streamInformationStore, logger) + private val videoParser = VideoParser(streamInformationStore, logger, diagnosticContext) override fun isReceivingAudio() = audioBitrateCalculator.active override fun isReceivingVideo() = videoBitrateCalculator.active diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ParsedVideoPacket.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ParsedVideoPacket.kt index bc9de8c01a..d4928295b8 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ParsedVideoPacket.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ParsedVideoPacket.kt @@ -25,8 +25,8 @@ abstract class ParsedVideoPacket( buffer: ByteArray, offset: Int, length: Int, - encodingIndex: Int? -) : VideoRtpPacket(buffer, offset, length, encodingIndex) { + encodingId: Int +) : VideoRtpPacket(buffer, offset, length, encodingId) { abstract val isKeyframe: Boolean abstract val isStartOfFrame: Boolean diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt index 1e869642ec..90799e11a8 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt @@ -96,7 +96,14 @@ enum class RtpExtensionType(val uri: String) { /** * The URN which identifies the RTP Header Extension for Video Orientation. */ - VIDEO_ORIENTATION("urn:3gpp:video-orientation"); + VIDEO_ORIENTATION("urn:3gpp:video-orientation"), + + /** + * The URN which identifies the AV1 Dependency Descriptor RTP Header Extension + */ + AV1_DEPENDENCY_DESCRIPTOR( + "https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension" + ); companion object { private val uriMap = RtpExtensionType.values().associateBy(RtpExtensionType::uri) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/VideoRtpPacket.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/VideoRtpPacket.kt index dc4835e3c1..19dc0716b5 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/VideoRtpPacket.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/VideoRtpPacket.kt @@ -15,6 +15,7 @@ */ package org.jitsi.nlj.rtp +import org.jitsi.nlj.RtpLayerDesc import org.jitsi.rtp.rtp.RtpPacket /** @@ -22,35 +23,22 @@ import org.jitsi.rtp.rtp.RtpPacket * parsed (i.e. we don't know information gained from * parsing codec-specific data). */ -open class VideoRtpPacket protected constructor( +open class VideoRtpPacket @JvmOverloads constructor( buffer: ByteArray, offset: Int, length: Int, - qualityIndex: Int? + /** The encoding ID of this packet. */ + var encodingId: Int = RtpLayerDesc.SUSPENDED_ENCODING_ID ) : RtpPacket(buffer, offset, length) { - constructor( - buffer: ByteArray, - offset: Int, - length: Int - ) : this( - buffer, - offset, - length, - qualityIndex = null - ) - - /** The index of this packet relative to its source's RtpLayers. */ - var qualityIndex: Int = qualityIndex ?: -1 - - open val layerId = 0 + open val layerIds: Collection = listOf(0) override fun clone(): VideoRtpPacket { return VideoRtpPacket( cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET), BYTES_TO_LEAVE_AT_START_OF_PACKET, length, - qualityIndex = qualityIndex + encodingId = encodingId ) } } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/VideoCodecParser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/VideoCodecParser.kt index 2ffe3e977a..28fad3f251 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/VideoCodecParser.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/VideoCodecParser.kt @@ -19,8 +19,7 @@ package org.jitsi.nlj.rtp.codec import org.jitsi.nlj.MediaSourceDesc import org.jitsi.nlj.PacketInfo import org.jitsi.nlj.RtpEncodingDesc -import org.jitsi.nlj.RtpLayerDesc -import org.jitsi.nlj.findRtpLayerDesc +import org.jitsi.nlj.findRtpLayerDescs import org.jitsi.nlj.rtp.VideoRtpPacket /** @@ -52,5 +51,5 @@ abstract class VideoCodecParser( return null } - protected fun findRtpLayerDesc(packet: VideoRtpPacket): RtpLayerDesc? = sources.findRtpLayerDesc(packet) + protected fun findRtpLayerDescs(packet: VideoRtpPacket) = sources.findRtpLayerDescs(packet) } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacket.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacket.kt new file mode 100644 index 0000000000..476beca682 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacket.kt @@ -0,0 +1,226 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp.codec.av1 + +import org.jitsi.nlj.RtpEncodingDesc +import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.nlj.rtp.ParsedVideoPacket +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.Av1DependencyDescriptorHeaderExtension +import org.jitsi.rtp.rtp.header_extensions.Av1DependencyDescriptorReader +import org.jitsi.rtp.rtp.header_extensions.Av1DependencyDescriptorStatelessSubset +import org.jitsi.rtp.rtp.header_extensions.Av1DependencyException +import org.jitsi.rtp.rtp.header_extensions.Av1TemplateDependencyStructure +import org.jitsi.rtp.rtp.header_extensions.FrameInfo +import org.jitsi.utils.logging2.Logger + +/** A video packet carrying an AV1 Dependency Descriptor. Note that this may or may not be an actual AV1 packet; + * other video codecs can also carry the AV1 DD. + */ +class Av1DDPacket : ParsedVideoPacket { + var descriptor: Av1DependencyDescriptorHeaderExtension? + val statelessDescriptor: Av1DependencyDescriptorStatelessSubset + val frameInfo: FrameInfo? + val av1DDHeaderExtensionId: Int + + private constructor( + buffer: ByteArray, + offset: Int, + length: Int, + av1DDHeaderExtensionId: Int, + encodingId: Int, + descriptor: Av1DependencyDescriptorHeaderExtension?, + statelessDescriptor: Av1DependencyDescriptorStatelessSubset, + frameInfo: FrameInfo? + ) : super(buffer, offset, length, encodingId) { + this.descriptor = descriptor + this.statelessDescriptor = statelessDescriptor + this.frameInfo = frameInfo + this.av1DDHeaderExtensionId = av1DDHeaderExtensionId + } + + constructor( + packet: RtpPacket, + av1DDHeaderExtensionId: Int, + templateDependencyStructure: Av1TemplateDependencyStructure?, + logger: Logger + ) : super(packet.buffer, packet.offset, packet.length, RtpLayerDesc.SUSPENDED_ENCODING_ID) { + this.av1DDHeaderExtensionId = av1DDHeaderExtensionId + val ddExt = packet.getHeaderExtension(av1DDHeaderExtensionId) + requireNotNull(ddExt) { + "Packet did not have Dependency Descriptor" + } + val parser = Av1DependencyDescriptorReader(ddExt) + descriptor = try { + parser.parse(templateDependencyStructure) + } catch (e: Av1DependencyException) { + logger.warn( + "Could not parse AV1 Dependency Descriptor for ssrc ${packet.ssrc} seq ${packet.sequenceNumber}: " + + e.message + ) + null + } + statelessDescriptor = descriptor ?: parser.parseStateless() + frameInfo = try { + descriptor?.frameInfo + } catch (e: Av1DependencyException) { + logger.warn( + "Could not extract frame info from AV1 Dependency Descriptor for " + + "ssrc ${packet.ssrc} seq ${packet.sequenceNumber}: ${e.message}" + ) + null + } + } + + /* "template_dependency_structure_present_flag MUST be set to 1 for the first packet of a coded video sequence, + * and MUST be set to 0 otherwise" + */ + override val isKeyframe: Boolean + get() = statelessDescriptor.newTemplateDependencyStructure != null + + override val isStartOfFrame: Boolean + get() = statelessDescriptor.startOfFrame + + override val isEndOfFrame: Boolean + get() = statelessDescriptor.endOfFrame + + override val layerIds: Collection + get() = frameInfo?.dtisPresent + ?: run { super.layerIds } + + val frameNumber + get() = statelessDescriptor.frameNumber + + val activeDecodeTargets + get() = descriptor?.activeDecodeTargetsBitmask + + override fun toString(): String = buildString { + append(super.toString()) + append(", DTIs=${frameInfo?.dtisPresent}") + activeDecodeTargets?.let { append(", ActiveTargets=$it") } + } + + override fun clone(): Av1DDPacket { + val descriptor = descriptor?.clone() + val statelessDescriptor = descriptor ?: statelessDescriptor.clone() + return Av1DDPacket( + cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET), + BYTES_TO_LEAVE_AT_START_OF_PACKET, + length, + av1DDHeaderExtensionId = av1DDHeaderExtensionId, + encodingId = encodingId, + descriptor = descriptor, + statelessDescriptor = statelessDescriptor, + frameInfo = frameInfo + ) + } + + fun getScalabilityStructure(eid: Int = 0, baseFrameRate: Double = 30.0): RtpEncodingDesc? { + val descriptor = this.descriptor + requireNotNull(descriptor) { + "Can't get scalability structure from packet without a descriptor" + } + return descriptor.getScalabilityStructure(ssrc, eid, baseFrameRate) + } + + /** Re-encode the current descriptor to the header extension. For use after modifying it. */ + fun reencodeDdExt() { + val descriptor = this.descriptor + requireNotNull(descriptor) { + "Can't re-encode extension from a packet without a descriptor" + } + + var ext = getHeaderExtension(av1DDHeaderExtensionId) + if (ext == null || ext.dataLengthBytes != descriptor.encodedLength) { + removeHeaderExtension(av1DDHeaderExtensionId) + ext = addHeaderExtension(av1DDHeaderExtensionId, descriptor.encodedLength) + } + descriptor.write(ext) + } +} + +fun Av1DependencyDescriptorHeaderExtension.getScalabilityStructure( + ssrc: Long, + eid: Int = 0, + baseFrameRate: Double = 30.0 +): RtpEncodingDesc? { + val activeDecodeTargetsBitmask = this.activeDecodeTargetsBitmask + ?: // Can't get scalability structure from dependency descriptor that doesn't specify decode targets + return null + + val layerCounts = Array(structure.maxSpatialId + 1) { + IntArray(structure.maxTemporalId + 1) + } + + // Figure out the frame rates per spatial/temporal layer. + structure.templateInfo.forEach { t -> + if (!t.hasInterPictureDependency()) { + // This is a template that doesn't reference any previous frames, so is probably a key frame or + // part of the same temporal picture with one, i.e. not part of the regular structure. + return@forEach + } + layerCounts[t.spatialId][t.temporalId]++ + } + + // Sum up counts per spatial layer + layerCounts.forEach { a -> + var total = 0 + for (i in a.indices) { + val entry = a[i] + a[i] += total + total += entry + } + } + + val maxFrameGroup = layerCounts.maxOf { it.maxOrNull()!! } + + val layers = ArrayList() + + structure.decodeTargetInfo.forEachIndexed { i, dt -> + if (!activeDecodeTargetsBitmask.containsDecodeTarget(i)) { + return@forEachIndexed + } + val height = structure.maxRenderResolutions.getOrNull(dt.spatialId)?.height ?: -1 + + // Calculate the fraction of this spatial layer's framerate this DT comprises. + val frameRate = baseFrameRate * layerCounts[dt.spatialId][dt.temporalId] / maxFrameGroup + + layers.add(Av1DDRtpLayerDesc(eid, i, dt.temporalId, dt.spatialId, height, frameRate)) + } + return RtpEncodingDesc(ssrc, layers.toArray(arrayOf()), eid) +} + +/** Check whether an activeDecodeTargetsBitmask contains a specific decode target. */ +fun Int.containsDecodeTarget(dt: Int) = ((1 shl dt) and this) != 0 + +/** + * Returns the delta between two AV1 templateID values, taking into account + * rollover. This will return the 'positive' delta between the two + * picture IDs in the form of the number you'd add to b to get a. e.g.: + * getTl0PicIdxDelta(1, 10) -> 55 (10 + 55 = 1) + * getTl0PicIdxDelta(1, 58) -> 7 (58 + 7 = 1) + */ +fun getTemplateIdDelta(a: Int, b: Int): Int = (a - b + 64) % 64 + +/** + * Apply a delta to a given templateID and return the result (taking + * rollover into account) + * @param start the starting templateID + * @param delta the delta to be applied + * @return the templateID resulting from doing "start + delta" + */ +fun applyTemplateIdDelta(start: Int, delta: Int): Int = (start + delta) % 64 diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDParser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDParser.kt new file mode 100644 index 0000000000..db169f1a76 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDParser.kt @@ -0,0 +1,179 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp.codec.av1 + +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.rtp.codec.VideoCodecParser +import org.jitsi.nlj.util.Rfc3711IndexTracker +import org.jitsi.nlj.util.TreeCache +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.Av1TemplateDependencyStructure +import org.jitsi.utils.LRUCache +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger + +/** + * Some [Av1DDPacket] fields are not able to be determined by looking at a single packet with an AV1 DD + * (for example the template dependency structure is only carried in keyframes). This class updates the layer + * descriptions with information from frames, and also diagnoses packet format variants that the Jitsi videobridge + * won't be able to route. + */ +class Av1DDParser( + sources: Array, + parentLogger: Logger, + private val diagnosticContext: DiagnosticContext +) : VideoCodecParser(sources) { + private val logger = createChildLogger(parentLogger) + + /** History of AV1 templates. */ + private val ddStateHistory = LRUCache(STATE_HISTORY_SIZE, true) + + fun createFrom(packet: RtpPacket, av1DdExtId: Int): Av1DDPacket { + val history = ddStateHistory.getOrPut(packet.ssrc) { + TemplateHistory(TEMPLATE_HISTORY_SIZE) + } + + val priorEntry = history.get(packet.sequenceNumber) + + val priorStructure = priorEntry?.value?.structure?.clone() + + val av1Packet = Av1DDPacket(packet, av1DdExtId, priorStructure, logger) + + val newStructure = av1Packet.descriptor?.newTemplateDependencyStructure + if (newStructure != null) { + val structureChanged = newStructure.templateIdOffset != priorStructure?.templateIdOffset + history.insert(packet.sequenceNumber, Av1DdInfo(newStructure.clone(), structureChanged)) + logger.debug { + "Inserting new structure with templates ${newStructure.templateIdOffset} .. " + + "${(newStructure.templateIdOffset + newStructure.templateCount - 1) % 64} " + + "for RTP packet ssrc ${packet.ssrc} seq ${packet.sequenceNumber}. " + + "Changed from previous: $structureChanged." + } + } + + if (timeSeriesLogger.isTraceEnabled) { + val point = diagnosticContext + .makeTimeSeriesPoint("av1_parser") + .addField("rtp.ssrc", packet.ssrc) + .addField("rtp.seq", packet.sequenceNumber) + .addField("rtp.timestamp", packet.timestamp) + .addField("av1_parser.key", priorEntry?.key) + .addField("av1.startOfFrame", av1Packet.statelessDescriptor.startOfFrame) + .addField("av1.endOfFrame", av1Packet.statelessDescriptor.endOfFrame) + .addField("av1.templateId", av1Packet.statelessDescriptor.frameDependencyTemplateId) + .addField("av1.frameNum", av1Packet.statelessDescriptor.frameNumber) + .addField("av1.frameInfo", av1Packet.frameInfo?.toString()) + .addField("av1.structure", newStructure != null) + .addField("av1.activeTargets", av1Packet.descriptor?.activeDecodeTargetsBitmask) + val packetStructure = av1Packet.descriptor?.structure + if (packetStructure != null) { + point.addField("av1.structureIdOffset", packetStructure.templateIdOffset) + .addField("av1.templateCount", packetStructure.templateCount) + .addField("av1.structureId", System.identityHashCode(packetStructure)) + } + if (newStructure != null) { + point.addField("av1.newStructureIdOffset", newStructure.templateIdOffset) + .addField("av1.newTemplateCount", newStructure.templateCount) + .addField("av1.newStructureId", System.identityHashCode(newStructure)) + } + timeSeriesLogger.trace(point) + } + + return av1Packet + } + + override fun parse(packetInfo: PacketInfo) { + val av1Packet = packetInfo.packetAs() + val history = ddStateHistory[av1Packet.ssrc] + + if (history == null) { + /** Probably getting spammed with SSRCs? */ + logger.warn("History for ${av1Packet.ssrc} disappeared between createFrom and parse!") + return + } + + val activeDecodeTargets = av1Packet.activeDecodeTargets + + if (activeDecodeTargets != null) { + val changed = history.updateDecodeTargets(av1Packet.sequenceNumber, activeDecodeTargets) + + if (changed) { + packetInfo.layeringChanged = true + logger.debug { + "Decode targets for ${av1Packet.ssrc} changed in seq ${av1Packet.sequenceNumber}: " + + "now 0x${Integer.toHexString(activeDecodeTargets)}. Updating layering." + } + + findSourceDescAndRtpEncodingDesc(av1Packet)?.let { (src, enc) -> + av1Packet.getScalabilityStructure(eid = enc.eid)?.let { + src.setEncodingLayers(it.layers, av1Packet.ssrc) + } + for (otherEnc in src.rtpEncodings) { + if (!ddStateHistory.keys.contains(otherEnc.primarySSRC)) { + src.setEncodingLayers(emptyArray(), otherEnc.primarySSRC) + } + } + } + } + } + } + + companion object { + const val STATE_HISTORY_SIZE = 500 + const val TEMPLATE_HISTORY_SIZE = 500 + + private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(Av1DDParser::class.java) + } +} + +class TemplateHistory(minHistory: Int) { + private val indexTracker = Rfc3711IndexTracker() + private val history = TreeCache(minHistory) + private var latestDecodeTargets = -1 + private var latestDecodeTargetIndex = -1 + + fun get(seqNo: Int): Map.Entry? { + val index = indexTracker.update(seqNo) + return history.getEntryBefore(index) + } + + fun insert(seqNo: Int, value: Av1DdInfo) { + val index = indexTracker.update(seqNo) + return history.insert(index, value) + } + + /** Update the current decode targets. + * Return true if the decode target set or the template structure has changed. */ + fun updateDecodeTargets(seqNo: Int, decodeTargets: Int): Boolean { + val index = indexTracker.update(seqNo) + if (index < latestDecodeTargetIndex) { + return false + } + val changed = decodeTargets != latestDecodeTargets || history.get(index)?.changed == true + latestDecodeTargetIndex = index + latestDecodeTargets = decodeTargets + return changed + } +} + +data class Av1DdInfo( + val structure: Av1TemplateDependencyStructure, + val changed: Boolean +) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDRtpLayerDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDRtpLayerDesc.kt new file mode 100644 index 0000000000..38952a2415 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDRtpLayerDesc.kt @@ -0,0 +1,120 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp.codec.av1 + +import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.nlj.stats.NodeStatsBlock + +/** + * An RtpLayerDesc of the type needed to describe AV1 DD scalability. + */ + +class Av1DDRtpLayerDesc( + /** + * The index of this instance's encoding in the source encoding array. + */ + eid: Int, + /** + * The decoding target of this instance, or negative for unknown. + */ + val dt: Int, + /** + * The temporal layer ID of this instance,. + */ + tid: Int, + /** + * The spatial layer ID of this instance. + */ + sid: Int, + /** + * The max height of the bitstream that this instance represents. The actual + * height may be less due to bad network or system load. + */ + + height: Int, + /** + * The max frame rate (in fps) of the bitstream that this instance + * represents. The actual frame rate may be less due to bad network or + * system load. + */ + frameRate: Double, +) : RtpLayerDesc(eid, tid, sid, height, frameRate) { + override fun copy(height: Int): RtpLayerDesc = Av1DDRtpLayerDesc(eid, dt, tid, sid, height, frameRate) + + override val layerId = dt + override val index = getIndex(eid, dt) + + override fun getBitrate(nowMs: Long) = bitrateTracker.getRate(nowMs) + + override fun hasZeroBitrate(nowMs: Long) = bitrateTracker.getAccumulatedSize(nowMs).bits == 0L + + /** + * Extracts a [NodeStatsBlock] from an [RtpLayerDesc]. + */ + override fun getNodeStats() = super.getNodeStats().apply { + addNumber("dt", dt) + } + + override fun indexString(): String = indexString(index) + + /** + * {@inheritDoc} + */ + override fun toString(): String { + return "subjective_quality=" + index + + ",DT=" + dt + } + + companion object { + /** + * The index value that is used to represent that forwarding is suspended. + */ + const val SUSPENDED_INDEX = -1 + + const val SUSPENDED_DT = -1 + + /** + * Calculate the "index" of a layer based on its encoding and decode target. + * This is a server-side id and should not be confused with any encoding id defined + * in the client (such as the rid) or the encodingId. This is used by the videobridge's + * adaptive source projection for filtering. + */ + @JvmStatic + fun getIndex(eid: Int, dt: Int): Int { + val e = if (eid < 0) 0 else eid + val d = if (dt < 0) 0 else dt + + return (e shl 6) or d + } + + /** + * Get a decode target ID from a layer index. If the index is [SUSPENDED_INDEX], + * the value is unspecified. + */ + @JvmStatic + fun getDtFromIndex(index: Int) = if (index == SUSPENDED_INDEX) SUSPENDED_DT else index and 0x3f + + /** + * Get a string description of a layer index. + */ + @JvmStatic + fun indexString(index: Int): String = if (index == SUSPENDED_INDEX) { + "SUSP" + } else { + "E${getEidFromIndex(index)}DT${getDtFromIndex(index)}" + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt index 93ea7b741a..2798125d7e 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt @@ -38,11 +38,11 @@ class Vp8Packet private constructor( length: Int, isKeyframe: Boolean?, isStartOfFrame: Boolean?, - encodingIndex: Int?, + encodingId: Int, height: Int?, pictureId: Int?, TL0PICIDX: Int? -) : ParsedVideoPacket(buffer, offset, length, encodingIndex) { +) : ParsedVideoPacket(buffer, offset, length, encodingId) { constructor( buffer: ByteArray, @@ -52,7 +52,7 @@ class Vp8Packet private constructor( buffer, offset, length, isKeyframe = null, isStartOfFrame = null, - encodingIndex = null, + encodingId = RtpLayerDesc.SUSPENDED_ENCODING_ID, height = null, pictureId = null, TL0PICIDX = null @@ -116,8 +116,12 @@ class Vp8Packet private constructor( val temporalLayerIndex: Int = Vp8Utils.getTemporalLayerIdOfFrame(this) - override val layerId: Int - get() = if (hasTemporalLayerIndex) RtpLayerDesc.getIndex(0, 0, temporalLayerIndex) else super.layerId + override val layerIds: Collection + get() = if (hasTemporalLayerIndex) { + listOf(RtpLayerDesc.getIndex(0, 0, temporalLayerIndex)) + } else { + super.layerIds + } /** * This is currently used as an overall spatial index, not an in-band spatial quality index a la vp9. That is, @@ -154,7 +158,7 @@ class Vp8Packet private constructor( length, isKeyframe = isKeyframe, isStartOfFrame = isStartOfFrame, - encodingIndex = qualityIndex, + encodingId = encodingId, height = height, pictureId = pictureId, TL0PICIDX = TL0PICIDX diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt index bacff0b341..ef26c01f89 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt @@ -19,6 +19,7 @@ package org.jitsi.nlj.rtp.codec.vp9 import org.jitsi.nlj.RtpEncodingDesc import org.jitsi.nlj.RtpLayerDesc import org.jitsi.nlj.rtp.ParsedVideoPacket +import org.jitsi.nlj.rtp.codec.vpx.VpxRtpLayerDesc import org.jitsi.rtp.extensions.bytearray.hashCodeOfSegment import org.jitsi.utils.logging2.createLogger import org.jitsi.utils.logging2.cwarn @@ -39,10 +40,10 @@ class Vp9Packet private constructor( isKeyframe: Boolean?, isStartOfFrame: Boolean?, isEndOfFrame: Boolean?, - encodingIndex: Int?, + encodingId: Int, pictureId: Int?, TL0PICIDX: Int? -) : ParsedVideoPacket(buffer, offset, length, encodingIndex) { +) : ParsedVideoPacket(buffer, offset, length, encodingId) { constructor( buffer: ByteArray, @@ -53,7 +54,7 @@ class Vp9Packet private constructor( isKeyframe = null, isStartOfFrame = null, isEndOfFrame = null, - encodingIndex = null, + encodingId = RtpLayerDesc.SUSPENDED_ENCODING_ID, pictureId = null, TL0PICIDX = null ) @@ -67,8 +68,12 @@ class Vp9Packet private constructor( override val isEndOfFrame: Boolean = isEndOfFrame ?: DePacketizer.VP9PayloadDescriptor.isEndOfFrame(buffer, payloadOffset, payloadLength) - override val layerId: Int - get() = if (hasLayerIndices) RtpLayerDesc.getIndex(0, spatialLayerIndex, temporalLayerIndex) else super.layerId + override val layerIds: Collection + get() = if (hasLayerIndices) { + listOf(RtpLayerDesc.getIndex(0, spatialLayerIndex, temporalLayerIndex)) + } else { + super.layerIds + } /** End of VP9 picture is the marker bit. Note frame/picture distinction. */ /* TODO: not sure this should be the override from [ParsedVideoPacket] */ @@ -194,7 +199,7 @@ class Vp9Packet private constructor( isKeyframe = isKeyframe, isStartOfFrame = isStartOfFrame, isEndOfFrame = isEndOfFrame, - encodingIndex = qualityIndex, + encodingId = encodingId, pictureId = pictureId, TL0PICIDX = TL0PICIDX ) @@ -301,12 +306,12 @@ class Vp9Packet private constructor( tlCounts[t] += tlCounts[t - 1] } - val layers = ArrayList() + val layers = ArrayList() for (s in 0 until numSpatial) { for (t in 0 until numTemporal) { - val dependencies = ArrayList() - val softDependencies = ArrayList() + val dependencies = ArrayList() + val softDependencies = ArrayList() if (s > 0) { /* Because of K-SVC, spatial layer dependencies are soft */ layers.find { it.sid == s - 1 && it.tid == t }?.let { softDependencies.add(it) } @@ -314,7 +319,7 @@ class Vp9Packet private constructor( if (t > 0) { layers.find { it.sid == s && it.tid == t - 1 }?.let { dependencies.add(it) } } - val layerDesc = RtpLayerDesc( + val layerDesc = VpxRtpLayerDesc( eid = eid, tid = t, sid = s, @@ -324,8 +329,8 @@ class Vp9Packet private constructor( } else { RtpLayerDesc.NO_FRAME_RATE }, - dependencyLayers = dependencies.toArray(arrayOf()), - softDependencyLayers = softDependencies.toArray(arrayOf()) + dependencyLayers = dependencies.toArray(arrayOf()), + softDependencyLayers = softDependencies.toArray(arrayOf()) ) layers.add(layerDesc) } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Parser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Parser.kt index 8d0f50db3a..04518ddbf6 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Parser.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Parser.kt @@ -18,8 +18,8 @@ package org.jitsi.nlj.rtp.codec.vp9 import org.jitsi.nlj.MediaSourceDesc import org.jitsi.nlj.PacketInfo -import org.jitsi.nlj.findRtpLayerDesc import org.jitsi.nlj.rtp.codec.VideoCodecParser +import org.jitsi.nlj.rtp.codec.vpx.VpxRtpLayerDesc import org.jitsi.nlj.util.StateChangeLogger import org.jitsi.rtp.extensions.toHex import org.jitsi.utils.logging2.Logger @@ -75,7 +75,11 @@ class Vp9Parser( * when calculating layers' bitrates. These values are small enough this is probably * fine, but revisit this if it turns out to be a problem. */ - findRtpLayerDesc(vp9Packet)?.useSoftDependencies = vp9Packet.usesInterLayerDependency + findRtpLayerDescs(vp9Packet).forEach { + if (it is VpxRtpLayerDesc) { + it.useSoftDependencies = vp9Packet.usesInterLayerDependency + } + } } pictureIdState.setState(vp9Packet.hasPictureId, vp9Packet) { diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vpx/VpxRtpLayerDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vpx/VpxRtpLayerDesc.kt new file mode 100644 index 0000000000..9c42b14e04 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vpx/VpxRtpLayerDesc.kt @@ -0,0 +1,190 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp.codec.vpx + +import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.BitrateTracker +import org.jitsi.nlj.util.sum + +/** + * An RtpLayerDesc of the type needed to describe VP8 and VP9 scalability. + */ +class VpxRtpLayerDesc +@JvmOverloads +constructor( + /** + * The index of this instance's encoding in the source encoding array. + */ + eid: Int, + /** + * The temporal layer ID of this instance, or negative for unknown. + */ + tid: Int, + /** + * The spatial layer ID of this instance, or negative for unknown. + */ + sid: Int, + /** + * The max height of the bitstream that this instance represents. The actual + * height may be less due to bad network or system load. [RtpLayerDesc.NO_HEIGHT] for unknown. + * + * XXX we should be able to sniff the actual height from the RTP + * packets. + */ + height: Int, + /** + * The max frame rate (in fps) of the bitstream that this instance + * represents. The actual frame rate may be less due to bad network or + * system load. [RtpLayerDesc.NO_FRAME_RATE] for unknown. + */ + frameRate: Double, + /** + * The [RtpLayerDesc]s on which this layer definitely depends. + */ + val dependencyLayers: Array = emptyArray(), + /** + * The [RtpLayerDesc]s on which this layer possibly depends. + * (The intended use case is K-SVC mode.) + */ + val softDependencyLayers: Array = emptyArray() +) : RtpLayerDesc(eid, tid, sid, height, frameRate) { + init { + require(tid < 8) { "Invalid temporal ID $tid" } + require(sid < 8) { "Invalid spatial ID $sid" } + } + + /** + * Clone an existing layer desc, inheriting its statistics, + * modifying only specific values. + */ + override fun copy(height: Int) = VpxRtpLayerDesc( + eid = this.eid, + tid = this.tid, + sid = this.sid, + height = height, + frameRate = this.frameRate, + dependencyLayers = this.dependencyLayers, + softDependencyLayers = this.softDependencyLayers + ).also { + it.inheritFrom(this) + } + + /** + * Whether softDependencyLayers are to be used. + */ + var useSoftDependencies = true + + /** + * @return the "id" of this layer within this encoding. This is a server-side id and should + * not be confused with any encoding id defined in the client (such as the + * rid). + */ + override val layerId = getIndex(0, sid, tid) + + /** + * A local index of this track. + */ + override val index = getIndex(eid, sid, tid) + + /** + * Inherit another layer description's [BitrateTracker] object. + */ + override fun inheritFrom(other: RtpLayerDesc) { + super.inheritFrom(other) + if (other is VpxRtpLayerDesc) { + useSoftDependencies = other.useSoftDependencies + } + } + + /** + * {@inheritDoc} + */ + override fun toString(): String { + return "subjective_quality=$index,temporal_id=$tid,spatial_id=$sid,height=$height" + } + + /** + * Gets the cumulative bitrate (in bps) of this [RtpLayerDesc] and its dependencies. + * + * This is left open for use in testing. + * + * @param nowMs + * @return the cumulative bitrate (in bps) of this [RtpLayerDesc] and its dependencies. + */ + override fun getBitrate(nowMs: Long): Bandwidth = calcBitrate(nowMs).values.sum() + + /** + * Recursively adds the bitrate (in bps) of this [RtpLayerDesc] and + * its dependencies in the map passed in as an argument. + * + * This is necessary to ensure we don't double-count layers in cases + * of multiple dependencies. + * + * @param nowMs + */ + private fun calcBitrate(nowMs: Long, rates: MutableMap = HashMap()): MutableMap { + if (rates.containsKey(index)) { + return rates + } + rates[index] = bitrateTracker.getRate(nowMs) + + dependencyLayers.forEach { it.calcBitrate(nowMs, rates) } + + if (useSoftDependencies) { + softDependencyLayers.forEach { it.calcBitrate(nowMs, rates) } + } + + return rates + } + + /** + * Returns true if this layer, alone, has a zero bitrate. + */ + private fun layerHasZeroBitrate(nowMs: Long) = bitrateTracker.getAccumulatedSize(nowMs).bits == 0L + + /** + * Recursively checks this layer and its dependencies to see if the bitrate is zero. + * Note that unlike [calcBitrate] this does not avoid double-visiting layers; the overhead + * of the hash table is usually more than the cost of any double-visits. + * + * This is left open for use in testing. + */ + override fun hasZeroBitrate(nowMs: Long): Boolean { + if (!layerHasZeroBitrate(nowMs)) { + return false + } + if (dependencyLayers.any { !it.layerHasZeroBitrate(nowMs) }) { + return false + } + if (useSoftDependencies && softDependencyLayers.any { !it.layerHasZeroBitrate(nowMs) }) { + return false + } + return true + } + + /** + * Extracts a [NodeStatsBlock] from an [RtpLayerDesc]. + */ + override fun getNodeStats() = super.getNodeStats().apply { + addNumber("tid", tid) + addNumber("sid", sid) + } + + override fun indexString(): String = indexString(index) +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/BitrateCalculator.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/BitrateCalculator.kt index 62105d6994..2daf4cc1da 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/BitrateCalculator.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/BitrateCalculator.kt @@ -22,7 +22,7 @@ import org.jitsi.nlj.Event import org.jitsi.nlj.MediaSourceDesc import org.jitsi.nlj.PacketInfo import org.jitsi.nlj.SetMediaSourcesEvent -import org.jitsi.nlj.findRtpLayerDesc +import org.jitsi.nlj.findRtpLayerDescs import org.jitsi.nlj.rtp.VideoRtpPacket import org.jitsi.nlj.stats.NodeStatsBlock import org.jitsi.nlj.transform.node.ObserverNode @@ -55,8 +55,8 @@ class VideoBitrateCalculator( super.observe(packetInfo) val videoRtpPacket: VideoRtpPacket = packetInfo.packet as VideoRtpPacket - mediaSourceDescs.findRtpLayerDesc(videoRtpPacket)?.let { - val now = clock.millis() + val now = clock.millis() + mediaSourceDescs.findRtpLayerDescs(videoRtpPacket).forEach { if (it.updateBitrate(videoRtpPacket.length.bytes, now)) { /* When a layer is started when it was previously inactive, * we want to recalculate bandwidth allocation. diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoParser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoParser.kt index a31326f595..685c70639a 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoParser.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoParser.kt @@ -21,7 +21,9 @@ import org.jitsi.nlj.PacketInfo import org.jitsi.nlj.SetMediaSourcesEvent import org.jitsi.nlj.format.Vp8PayloadType import org.jitsi.nlj.format.Vp9PayloadType +import org.jitsi.nlj.rtp.RtpExtensionType import org.jitsi.nlj.rtp.codec.VideoCodecParser +import org.jitsi.nlj.rtp.codec.av1.Av1DDParser import org.jitsi.nlj.rtp.codec.vp8.Vp8Packet import org.jitsi.nlj.rtp.codec.vp8.Vp8Parser import org.jitsi.nlj.rtp.codec.vp9.Vp9Packet @@ -32,6 +34,7 @@ import org.jitsi.nlj.util.ReadOnlyStreamInformationStore import org.jitsi.rtp.extensions.bytearray.toHex import org.jitsi.rtp.rtp.RtpPacket import org.jitsi.utils.OrderedJsonObject +import org.jitsi.utils.logging.DiagnosticContext import org.jitsi.utils.logging2.Logger import org.jitsi.utils.logging2.cdebug import org.jitsi.utils.logging2.createChildLogger @@ -41,7 +44,8 @@ import org.jitsi.utils.logging2.createChildLogger */ class VideoParser( private val streamInformationStore: ReadOnlyStreamInformationStore, - parentLogger: Logger + parentLogger: Logger, + private val diagnosticContext: DiagnosticContext ) : TransformerNode("Video parser") { private val logger = createChildLogger(parentLogger) private val stats = Stats() @@ -49,18 +53,27 @@ class VideoParser( private var sources: Array = arrayOf() private var signaledSources: Array = sources + private var av1DDExtId: Int? = null + private var videoCodecParser: VideoCodecParser? = null + init { + streamInformationStore.onRtpExtensionMapping(RtpExtensionType.AV1_DEPENDENCY_DESCRIPTOR) { + av1DDExtId = it + } + } + override fun transform(packetInfo: PacketInfo): PacketInfo? { val packet = packetInfo.packetAs() + val av1DDExtId = this.av1DDExtId // So null checks work val payloadType = streamInformationStore.rtpPayloadTypes[packet.payloadType.toByte()] ?: run { logger.error("Unrecognized video payload type ${packet.payloadType}, cannot parse video information") stats.numPacketsDroppedUnknownPt++ return null } val parsedPacket = try { - when (payloadType) { - is Vp8PayloadType -> { + when { + payloadType is Vp8PayloadType -> { val vp8Packet = packetInfo.packet.toOtherType(::Vp8Packet) packetInfo.packet = vp8Packet packetInfo.resetPayloadVerification() @@ -75,7 +88,7 @@ class VideoParser( } vp8Packet } - is Vp9PayloadType -> { + payloadType is Vp9PayloadType -> { val vp9Packet = packetInfo.packet.toOtherType(::Vp9Packet) packetInfo.packet = vp9Packet packetInfo.resetPayloadVerification() @@ -90,6 +103,22 @@ class VideoParser( } vp9Packet } + av1DDExtId != null && packet.getHeaderExtension(av1DDExtId) != null -> { + if (videoCodecParser !is Av1DDParser) { + logger.cdebug { + "Creating new Av1DDParser, current videoCodecParser is ${videoCodecParser?.javaClass}" + } + resetSources() + packetInfo.layeringChanged = true + videoCodecParser = Av1DDParser(sources, logger, diagnosticContext) + } + + val av1DDPacket = (videoCodecParser as Av1DDParser).createFrom(packet, av1DDExtId) + packetInfo.packet = av1DDPacket + packetInfo.resetPayloadVerification() + + av1DDPacket + } else -> { if (videoCodecParser != null) { logger.cdebug { diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoQualityLayerLookup.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoQualityLayerLookup.kt index 6b2935d41b..399dc79fdc 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoQualityLayerLookup.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoQualityLayerLookup.kt @@ -19,7 +19,7 @@ import org.jitsi.nlj.Event import org.jitsi.nlj.MediaSourceDesc import org.jitsi.nlj.PacketInfo import org.jitsi.nlj.SetMediaSourcesEvent -import org.jitsi.nlj.findRtpLayerDesc +import org.jitsi.nlj.findRtpEncodingId import org.jitsi.nlj.rtp.VideoRtpPacket import org.jitsi.nlj.stats.NodeStatsBlock import org.jitsi.nlj.transform.node.TransformerNode @@ -37,10 +37,10 @@ class VideoQualityLayerLookup( private var sources: Array = arrayOf() private val numPacketsDroppedNoEncoding = AtomicInteger() - /* TODO: combine this with VideoBitrateCalculator? They both do findRtpLayerDesc. */ override fun transform(packetInfo: PacketInfo): PacketInfo? { val videoPacket = packetInfo.packetAs() - val encodingDesc = sources.findRtpLayerDesc(videoPacket) ?: run { + val encodingId = sources.findRtpEncodingId(videoPacket) + if (encodingId == null) { logger.warn( "Unable to find encoding matching packet! packet=$videoPacket; " + "sources=${sources.joinToString(separator = "\n")}" @@ -48,7 +48,7 @@ class VideoQualityLayerLookup( numPacketsDroppedNoEncoding.incrementAndGet() return null } - videoPacket.qualityIndex = encodingDesc.index + videoPacket.encodingId = encodingId return packetInfo } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/HeaderExtStripper.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/HeaderExtStripper.kt index 009bf1288e..b697d9a763 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/HeaderExtStripper.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/HeaderExtStripper.kt @@ -17,30 +17,41 @@ package org.jitsi.nlj.transform.node.outgoing import org.jitsi.nlj.PacketInfo import org.jitsi.nlj.rtp.RtpExtensionType +import org.jitsi.nlj.rtp.codec.av1.Av1DDPacket import org.jitsi.nlj.transform.node.ModifierNode import org.jitsi.nlj.util.ReadOnlyStreamInformationStore import org.jitsi.rtp.rtp.RtpPacket /** - * Strip all hop-by-hop header extensions. Currently this leaves only ssrc-audio-level and video-orientation. + * Strip all hop-by-hop header extensions. Currently this leaves ssrc-audio-level and video-orientation, + * plus the AV1 dependency descriptor if the packet is an Av1DDPacket. */ class HeaderExtStripper( streamInformationStore: ReadOnlyStreamInformationStore ) : ModifierNode("Strip header extensions") { private var retainedExts: Set = emptySet() + private var retainedExtsWithAv1DD: Set = emptySet() init { retainedExtTypes.forEach { rtpExtensionType -> streamInformationStore.onRtpExtensionMapping(rtpExtensionType) { - it?.let { retainedExts = retainedExts.plus(it) } + it?.let { + retainedExts = retainedExts.plus(it) + retainedExtsWithAv1DD = retainedExtsWithAv1DD.plus(it) + } } } + streamInformationStore.onRtpExtensionMapping(RtpExtensionType.AV1_DEPENDENCY_DESCRIPTOR) { + it?.let { retainedExtsWithAv1DD = retainedExtsWithAv1DD.plus(it) } + } } override fun modify(packetInfo: PacketInfo): PacketInfo { val rtpPacket = packetInfo.packetAs() - rtpPacket.removeHeaderExtensionsExcept(retainedExts) + val retained = if (rtpPacket is Av1DDPacket) retainedExtsWithAv1DD else retainedExts + + rtpPacket.removeHeaderExtensionsExcept(retained) return packetInfo } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/Rfc3711IndexTracker.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/Rfc3711IndexTracker.kt index 65928583cf..fbe7fa679f 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/Rfc3711IndexTracker.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/Rfc3711IndexTracker.kt @@ -16,6 +16,7 @@ package org.jitsi.nlj.util +import org.jitsi.rtp.util.RtpUtils import org.jitsi.rtp.util.isNewerThan import org.jitsi.rtp.util.rolledOverTo @@ -78,4 +79,16 @@ class Rfc3711IndexTracker { fun interpret(seqNum: Int): Int { return getIndex(seqNum, false) } + + /** Force this sequence number to be interpreted as the new highest, regardless + * of its rollover state. + */ + fun resetAt(seq: Int) { + val delta = RtpUtils.getSequenceNumberDelta(seq, highestSeqNumReceived) + if (delta < 0) { + roc++ + highestSeqNumReceived = seq + } + getIndex(seq, true) + } } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/TreeCache.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/TreeCache.kt new file mode 100644 index 0000000000..7288eb420a --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/TreeCache.kt @@ -0,0 +1,62 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import java.util.* + +/** + * Implements a cache based on integer values, optimized for sparse values, such that you can find the most + * recent cached value before a specific value. + * + * The intended use case is AV1 Dependency Descriptor history. + */ +open class TreeCache( + private val minSize: Int +) { + private val map = TreeMap() + + private var highestIndex = -1 + + fun insert(index: Int, value: T) { + map[index] = value + + updateState(index) + } + + fun getEntryBefore(index: Int): Map.Entry? { + updateState(index) + return map.floorEntry(index) + } + + fun get(index: Int): T? = map[index] + + private fun updateState(index: Int) { + if (highestIndex < index) { + highestIndex = index + } + + /* Keep at most one entry older than highestIndex - minSize. */ + val headMap = map.headMap(highestIndex - minSize) + if (headMap.size > 1) { + val last = headMap.keys.last() + headMap.keys.removeIf { it < last } + } + } + + val size + get() = map.size +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/MediaSourceDescTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/MediaSourceDescTest.kt index b1526978bc..a5eb419ff3 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/MediaSourceDescTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/MediaSourceDescTest.kt @@ -17,7 +17,10 @@ package org.jitsi.nlj import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.should import io.kotest.matchers.shouldBe +import io.kotest.matchers.types.beInstanceOf +import org.jitsi.nlj.rtp.codec.vpx.VpxRtpLayerDesc import org.jitsi.nlj.util.Bandwidth import org.jitsi.nlj.util.BitrateTracker import org.jitsi.nlj.util.bps @@ -56,6 +59,9 @@ class MediaSourceDescTest : ShouldSpec() { e.layers.size shouldBe 3 for (j in e.layers.indices) { val l = e.layers[j] + l should beInstanceOf() + l as VpxRtpLayerDesc + l.eid shouldBe i l.tid shouldBe j l.sid shouldBe -1 @@ -124,13 +130,12 @@ private fun idx(spatialIdx: Int, temporalIdx: Int, temporalLen: Int) = spatialId * @return an array that holds the layer descriptions. */ private fun createRTPLayerDescs(spatialLen: Int, temporalLen: Int, encodingIdx: Int, height: Int): Array { - val rtpLayers = arrayOfNulls(spatialLen * temporalLen) + val rtpLayers = arrayOfNulls(spatialLen * temporalLen) for (spatialIdx in 0 until spatialLen) { var frameRate = 30.toDouble() / (1 shl temporalLen - 1) for (temporalIdx in 0 until temporalLen) { val idx: Int = idx(spatialIdx, temporalIdx, temporalLen) - var dependencies: Array - dependencies = if (spatialIdx > 0 && temporalIdx > 0) { + val dependencies: Array = if (spatialIdx > 0 && temporalIdx > 0) { // this layer depends on spatialIdx-1 and temporalIdx-1. arrayOf( rtpLayers[ @@ -176,7 +181,7 @@ private fun createRTPLayerDescs(spatialLen: Int, temporalLen: Int, encodingIdx: } val temporalId = if (temporalLen > 1) temporalIdx else -1 val spatialId = if (spatialLen > 1) spatialIdx else -1 - rtpLayers[idx] = RtpLayerDesc( + rtpLayers[idx] = VpxRtpLayerDesc( encodingIdx, temporalId, spatialId, @@ -226,7 +231,7 @@ private fun createSource( name: String, videoType: VideoType, ): MediaSourceDesc { - var height = 720 + var height = 180 val encodings = Array(primarySsrcs.size) { encodingIdx -> val primarySsrc: Long = primarySsrcs[encodingIdx] diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/RtpLayerDescTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/RtpLayerDescTest.kt index 31fbf55972..7333842171 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/RtpLayerDescTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/RtpLayerDescTest.kt @@ -17,21 +17,22 @@ package org.jitsi.nlj import io.kotest.core.spec.style.FunSpec import io.kotest.matchers.shouldBe +import org.jitsi.nlj.rtp.codec.vpx.VpxRtpLayerDesc class RtpLayerDescTest : FunSpec({ test("VP8 layer ids") { // mostly for documenting the encoding -> index mapping. val vp8Layers = arrayOf( - RtpLayerDesc(0, 0, 0, 180, 7.5), - RtpLayerDesc(0, 1, 0, 180, 15.0), - RtpLayerDesc(0, 2, 0, 180, 30.0), - RtpLayerDesc(1, 0, 0, 360, 7.5), - RtpLayerDesc(1, 1, 0, 360, 15.0), - RtpLayerDesc(1, 2, 0, 360, 30.0), - RtpLayerDesc(2, 0, 0, 720, 7.5), - RtpLayerDesc(2, 1, 0, 720, 15.0), - RtpLayerDesc(2, 2, 0, 720, 30.0) + VpxRtpLayerDesc(0, 0, 0, 180, 7.5), + VpxRtpLayerDesc(0, 1, 0, 180, 15.0), + VpxRtpLayerDesc(0, 2, 0, 180, 30.0), + VpxRtpLayerDesc(1, 0, 0, 360, 7.5), + VpxRtpLayerDesc(1, 1, 0, 360, 15.0), + VpxRtpLayerDesc(1, 2, 0, 360, 30.0), + VpxRtpLayerDesc(2, 0, 0, 720, 7.5), + VpxRtpLayerDesc(2, 1, 0, 720, 15.0), + VpxRtpLayerDesc(2, 2, 0, 720, 30.0) ) vp8Layers[0].index shouldBe 0 @@ -58,15 +59,15 @@ class RtpLayerDescTest : FunSpec({ test("VP9 layer ids") { // same here, mostly for documenting the encoding -> index mapping. val vp9Layers = arrayOf( - RtpLayerDesc(0, 0, 0, 180, 7.5), - RtpLayerDesc(0, 1, 0, 180, 15.0), - RtpLayerDesc(0, 2, 0, 180, 30.0), - RtpLayerDesc(0, 0, 1, 360, 7.5), - RtpLayerDesc(0, 1, 1, 360, 15.0), - RtpLayerDesc(0, 2, 1, 360, 30.0), - RtpLayerDesc(0, 0, 2, 720, 7.5), - RtpLayerDesc(0, 1, 2, 720, 15.0), - RtpLayerDesc(0, 2, 2, 720, 30.0) + VpxRtpLayerDesc(0, 0, 0, 180, 7.5), + VpxRtpLayerDesc(0, 1, 0, 180, 15.0), + VpxRtpLayerDesc(0, 2, 0, 180, 30.0), + VpxRtpLayerDesc(0, 0, 1, 360, 7.5), + VpxRtpLayerDesc(0, 1, 1, 360, 15.0), + VpxRtpLayerDesc(0, 2, 1, 360, 30.0), + VpxRtpLayerDesc(0, 0, 2, 720, 7.5), + VpxRtpLayerDesc(0, 1, 2, 720, 15.0), + VpxRtpLayerDesc(0, 2, 2, 720, 30.0) ) vp9Layers[0].index shouldBe 0 diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacketTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacketTest.kt new file mode 100644 index 0000000000..f123be9038 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacketTest.kt @@ -0,0 +1,211 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp.codec.av1 + +import io.kotest.assertions.withClue +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import io.kotest.matchers.types.beInstanceOf +import org.jitsi.nlj.RtpEncodingDesc +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.logging2.LoggerImpl +import javax.xml.bind.DatatypeConverter + +class Av1DDPacketTest : ShouldSpec() { + val logger = LoggerImpl(javaClass.name) + + private data class SampleAv1DDPacket( + val description: String, + val data: ByteArray, + val structureSource: SampleAv1DDPacket?, + // ... + val scalabilityStructure: RtpEncodingDesc? = null + + ) { + constructor( + description: String, + hexData: String, + structureSource: SampleAv1DDPacket?, + // ... + scalabilityStructure: RtpEncodingDesc? = null + ) : this( + description = description, + data = DatatypeConverter.parseHexBinary(hexData), + structureSource = structureSource, + // ... + scalabilityStructure = scalabilityStructure + ) + } + + /* Packets captured from AV1 test calls */ + + private val nonScalableKeyframe = + SampleAv1DDPacket( + "non-scalable keyframe", + // RTP header + "902939b91ff6a695114ed316" + + // Header extension header + "bede0006" + + // Other header extensions + "3248c482" + + "510002" + + // AV1 DD + "bc80000180003a410180ef808680" + + // Padding + "000000" + + // AV1 Media payload, truncated. + "680b0800000004477e1a00d004301061", + null, + RtpEncodingDesc( + 0x114ed316L, + arrayOf( + Av1DDRtpLayerDesc(0, 0, 0, 0, 270, 30.0) + ) + ) + ) + + private val scalableKeyframe = + SampleAv1DDPacket( + "scalable keyframe", + // RTP header + "906519aed780a8f32ab2873c" + + // Header extension header (2-byte) + "1000001a" + + // Other header extensions + "030302228e" + + "050219f9" + + // AV1 DD + "0b5a8003ca80081485214eaaaaa8000600004000100002aa80a80006" + + "00004000100002a000a800060000400016d549241b5524906d549231" + + "57e001974ca864330e222396eca8655304224390eca87753013f00b3" + + "027f016704ff02cf" + + // Padding + "000000" + + // AV1 Media payload, truncated. + "aabbdc101a58014000b4028001680500", + null, + RtpEncodingDesc( + 0x2ab2873cL, + arrayOf( + Av1DDRtpLayerDesc(0, 0, 0, 0, 180, 7.5), + Av1DDRtpLayerDesc(0, 1, 1, 0, 180, 15.0), + Av1DDRtpLayerDesc(0, 2, 2, 0, 180, 30.0), + Av1DDRtpLayerDesc(0, 3, 1, 0, 360, 7.5), + Av1DDRtpLayerDesc(0, 4, 1, 1, 360, 15.0), + Av1DDRtpLayerDesc(0, 5, 1, 2, 360, 30.0), + Av1DDRtpLayerDesc(0, 6, 2, 0, 720, 7.5), + Av1DDRtpLayerDesc(0, 7, 2, 1, 720, 15.0), + Av1DDRtpLayerDesc(0, 8, 2, 2, 720, 30.0) + ) + ) + ) + + private val testPackets = arrayOf( + nonScalableKeyframe, + SampleAv1DDPacket( + "non-scalable following packet", + // RTP header + "90a939ba1ff6a695114ed316" + + // Header extension header + "bede0003" + + // Other header extensions + "3248d02a" + + "510003" + + // AV1 DD + "b2400001" + + // Padding + "00" + + // AV1 Media payload, truncated. + "9057f7c3f51ba803b0c397e938589750", + nonScalableKeyframe + ), + scalableKeyframe, + SampleAv1DDPacket( + "scalable following packet changing available DTs", + // RTP header + "90e519c2d780b1bd2ab2873c" + + // Header extension header + "bede0004" + + // Other header extensions + "3202f824" + + "511a19" + + // AV1 DD + "b4c303cd401c" + + // Padding + "000000" + + // AV1 Media payload, truncated. + "edbbdd501a87000000027e016704ff02", + scalableKeyframe, + RtpEncodingDesc( + 0x2ab2873cL, + arrayOf( + Av1DDRtpLayerDesc(0, 0, 0, 0, 180, 7.5), + Av1DDRtpLayerDesc(0, 1, 0, 1, 180, 15.0), + Av1DDRtpLayerDesc(0, 2, 0, 2, 180, 30.0), + ) + ) + ) + ) + + init { + context("AV1 packets") { + should("be parsed correctly") { + for (t in testPackets) { + withClue(t.description) { + val structure = if (t.structureSource != null) { + val sourceR = + RtpPacket(t.structureSource.data, 0, t.structureSource.data.size) + val sourceP = Av1DDPacket(sourceR, AV1_DD_HEADER_EXTENSION_ID, null, logger) + sourceP.descriptor?.structure + } else { + null + } + + val r = RtpPacket(t.data, 0, t.data.size) + val p = Av1DDPacket(r, AV1_DD_HEADER_EXTENSION_ID, structure, logger) + + if (t.scalabilityStructure != null) { + val tss = t.scalabilityStructure + val ss = p.getScalabilityStructure() + ss shouldNotBe null + ss!!.primarySSRC shouldBe tss.primarySSRC + ss.layers.size shouldBe tss.layers.size + for ((index, layer) in ss.layers.withIndex()) { + val tLayer = tss.layers[index] + layer.layerId shouldBe tLayer.layerId + layer.index shouldBe tLayer.index + layer should beInstanceOf() + layer as Av1DDRtpLayerDesc + tLayer as Av1DDRtpLayerDesc + layer.dt shouldBe tLayer.dt + layer.height shouldBe tLayer.height + layer.frameRate shouldBe tLayer.frameRate + } + } else { + p.getScalabilityStructure() shouldBe null + } + } + } + } + } + } + + companion object { + const val AV1_DD_HEADER_EXTENSION_ID = 11 + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9PacketTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9PacketTest.kt index 3aee52fc19..93b08517bf 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9PacketTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9PacketTest.kt @@ -1,11 +1,28 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.jitsi.nlj.rtp.codec.vp9 import io.kotest.assertions.withClue import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.should import io.kotest.matchers.shouldBe import io.kotest.matchers.shouldNotBe +import io.kotest.matchers.types.beInstanceOf import org.jitsi.nlj.RtpEncodingDesc -import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.nlj.rtp.codec.vpx.VpxRtpLayerDesc import org.jitsi_modified.impl.neomedia.codec.video.vp9.DePacketizer import javax.xml.bind.DatatypeConverter @@ -136,16 +153,16 @@ class Vp9PacketTest : ShouldSpec() { scalabilityStructure = RtpEncodingDesc( 0x6098017bL, arrayOf( - RtpLayerDesc(0, 0, 0, 180, 7.5), + VpxRtpLayerDesc(0, 0, 0, 180, 7.5), /* TODO: dependencies */ - RtpLayerDesc(0, 1, 0, 180, 15.0), - RtpLayerDesc(0, 2, 0, 180, 30.0), - RtpLayerDesc(0, 0, 1, 360, 7.5), - RtpLayerDesc(0, 1, 1, 360, 15.0), - RtpLayerDesc(0, 2, 1, 360, 30.0), - RtpLayerDesc(0, 0, 2, 720, 7.5), - RtpLayerDesc(0, 1, 2, 720, 15.0), - RtpLayerDesc(0, 2, 2, 720, 30.0) + VpxRtpLayerDesc(0, 1, 0, 180, 15.0), + VpxRtpLayerDesc(0, 2, 0, 180, 30.0), + VpxRtpLayerDesc(0, 0, 1, 360, 7.5), + VpxRtpLayerDesc(0, 1, 1, 360, 15.0), + VpxRtpLayerDesc(0, 2, 1, 360, 30.0), + VpxRtpLayerDesc(0, 0, 2, 720, 7.5), + VpxRtpLayerDesc(0, 1, 2, 720, 15.0), + VpxRtpLayerDesc(0, 2, 2, 720, 30.0) ) ) ), @@ -480,7 +497,7 @@ class Vp9PacketTest : ShouldSpec() { scalabilityStructure = RtpEncodingDesc( 0x184b0cc4L, arrayOf( - RtpLayerDesc(0, 0, 0, 1158, 30.0) + VpxRtpLayerDesc(0, 0, 0, 1158, 30.0) ) ) ), @@ -600,7 +617,7 @@ class Vp9PacketTest : ShouldSpec() { scalabilityStructure = RtpEncodingDesc( 0x6538459eL, arrayOf( - RtpLayerDesc(0, 0, 0, 720, 30.0) + VpxRtpLayerDesc(0, 0, 0, 720, 30.0) ) ) ), @@ -663,9 +680,9 @@ class Vp9PacketTest : ShouldSpec() { scalabilityStructure = RtpEncodingDesc( 0xa4d04528L, arrayOf( - RtpLayerDesc(0, 0, 0, 720, 7.5), - RtpLayerDesc(0, 1, 0, 720, 15.0), - RtpLayerDesc(0, 2, 0, 720, 30.0) + VpxRtpLayerDesc(0, 0, 0, 720, 7.5), + VpxRtpLayerDesc(0, 1, 0, 720, 15.0), + VpxRtpLayerDesc(0, 2, 0, 720, 30.0) ) ) ), @@ -773,6 +790,9 @@ class Vp9PacketTest : ShouldSpec() { val tLayer = tss.layers[index] layer.layerId shouldBe tLayer.layerId layer.index shouldBe tLayer.index + layer should beInstanceOf() + layer as VpxRtpLayerDesc + tLayer as VpxRtpLayerDesc layer.sid shouldBe tLayer.sid layer.tid shouldBe tLayer.tid layer.height shouldBe tLayer.height diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/TreeCacheTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/TreeCacheTest.kt new file mode 100644 index 0000000000..95e3113515 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/TreeCacheTest.kt @@ -0,0 +1,100 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.util + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import java.util.AbstractMap.SimpleImmutableEntry + +class TreeCacheTest : ShouldSpec() { + override fun isolationMode() = IsolationMode.InstancePerLeaf + + data class Dummy(val data: String) + + private val treeCache = TreeCache(16) + + /** Shorthand for a Map.Entry mapping [key] to a Dummy containing [dummyVal] */ + private fun ed(key: Int, dummyVal: String) = SimpleImmutableEntry(key, Dummy(dummyVal)) + + init { + context("Reading from an empty TreeCache") { + should("return null") { + treeCache.getEntryBefore(10) shouldBe null + } + should("have size 0") { + treeCache.size shouldBe 0 + } + } + context("An entry in a TreeCache") { + treeCache.insert(5, Dummy("A")) + should("be found looking up values after it") { + treeCache.getEntryBefore(10) shouldBe ed(5, "A") + } + should("be found looking up the same value") { + treeCache.getEntryBefore(5) shouldBe ed(5, "A") + } + should("not be found looking up values before it") { + treeCache.getEntryBefore(3) shouldBe null + } + should("not be expired even if values long after it are looked up") { + treeCache.getEntryBefore(10000) shouldBe ed(5, "A") + } + should("cause the tree to have size 1") { + treeCache.size shouldBe 1 + } + } + context("Multiple values in a TreeCache") { + treeCache.insert(5, Dummy("A")) + treeCache.insert(10, Dummy("B")) + should("Be looked up properly") { + treeCache.getEntryBefore(13) shouldBe ed(10, "B") + treeCache.size shouldBe 2 + } + should("Persist within the cache window") { + treeCache.getEntryBefore(8) shouldBe ed(5, "A") + treeCache.size shouldBe 2 + } + should("Not expire an older one if it is the only value outside the cache window") { + treeCache.getEntryBefore(25) shouldBe ed(10, "B") + treeCache.getEntryBefore(8) shouldBe ed(5, "A") + treeCache.size shouldBe 2 + } + should("Expire older ones when newer ones are outside the cache window") { + treeCache.getEntryBefore(30) shouldBe ed(10, "B") + treeCache.getEntryBefore(8) shouldBe null + treeCache.size shouldBe 1 + } + should("Expire only older ones when later values are inserted") { + treeCache.insert(40, Dummy("C")) + treeCache.getEntryBefore(13) shouldBe ed(10, "B") + treeCache.getEntryBefore(8) shouldBe null + treeCache.size shouldBe 2 + } + should("Persist values within the window while expiring values outside it") { + treeCache.insert(15, Dummy("C")) + treeCache.getEntryBefore(8) shouldBe ed(5, "A") + treeCache.getEntryBefore(25) shouldBe ed(15, "C") + treeCache.getEntryBefore(13) shouldBe ed(10, "B") + treeCache.getEntryBefore(8) shouldBe ed(5, "A") + treeCache.size shouldBe 3 + treeCache.insert(30, Dummy("D")) + treeCache.getEntryBefore(8) shouldBe null + treeCache.size shouldBe 3 + } + } + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjection.java b/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjection.java index a0f545f364..c59ec62c9f 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjection.java +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjection.java @@ -17,19 +17,21 @@ import org.jetbrains.annotations.*; import org.jitsi.nlj.*; -import org.jitsi.nlj.format.*; import org.jitsi.nlj.rtp.*; +import org.jitsi.nlj.rtp.codec.av1.*; import org.jitsi.nlj.rtp.codec.vp8.*; +import org.jitsi.nlj.rtp.codec.vp9.*; import org.jitsi.rtp.rtcp.*; -import org.jitsi.utils.collections.*; import org.jitsi.utils.logging.*; import org.jitsi.utils.logging2.Logger; +import org.jitsi.videobridge.cc.av1.*; import org.jitsi.videobridge.cc.vp8.*; import org.jitsi.videobridge.cc.vp9.*; import org.json.simple.*; import java.lang.*; import java.util.*; +import java.util.stream.*; /** * Filters the packets coming from a specific {@link MediaSourceDesc} @@ -88,38 +90,26 @@ public class AdaptiveSourceProjection */ private AdaptiveSourceProjectionContext context; - /** - * The payload type that was used to determine the {@link #context} type. - */ - private int contextPayloadType = -1; - /** * The target quality index for this source projection. */ private int targetIndex = RtpLayerDesc.SUSPENDED_INDEX; - private final Map payloadTypes; - /** * Ctor. * * @param source the {@link MediaSourceDesc} that owns the packets * that this instance filters. - * - * @param payloadTypes a reference to a map of payload types. This map - * should be updated as the payload types change. */ public AdaptiveSourceProjection( @NotNull DiagnosticContext diagnosticContext, @NotNull MediaSourceDesc source, Runnable keyframeRequester, - Map payloadTypes, Logger parentLogger ) { targetSsrc = source.getPrimarySSRC(); this.diagnosticContext = diagnosticContext; - this.payloadTypes = payloadTypes; this.parentLogger = parentLogger; this.logger = parentLogger.createChildLogger(AdaptiveSourceProjection.class.getName(), Map.of("targetSsrc", Long.toString(targetSsrc), @@ -160,7 +150,8 @@ public boolean accept(@NotNull PacketInfo packetInfo) // suspended so that it can raise the needsKeyframe flag and also allow // it to compute a sequence number delta when the target becomes > -1. - if (videoRtpPacket.getQualityIndex() < 0) + int encodingId = videoRtpPacket.getEncodingId(); + if (encodingId == RtpLayerDesc.SUSPENDED_ENCODING_ID) { logger.warn( "Dropping an RTP packet, because egress was unable to find " + @@ -169,8 +160,7 @@ public boolean accept(@NotNull PacketInfo packetInfo) } int targetIndexCopy = targetIndex; - boolean accept = contextCopy.accept( - packetInfo, videoRtpPacket.getQualityIndex(), targetIndexCopy); + boolean accept = contextCopy.accept(packetInfo, encodingId, targetIndexCopy); // We check if the context needs a keyframe regardless of whether or not // the packet was accepted. @@ -190,8 +180,8 @@ public boolean accept(@NotNull PacketInfo packetInfo) /** * Gets or creates the adaptive source projection context that corresponds to - * the payload type of the RTP packet that is specified as a parameter. If - * the payload type is different from {@link #contextPayloadType}, then a + * the parsed class of the RTP packet that is specified as a parameter. If + * the payload type is different from the appropriate one for the context, then a * new adaptive source projection context is created that is appropriate for * the new payload type. * @@ -204,26 +194,9 @@ public boolean accept(@NotNull PacketInfo packetInfo) */ private AdaptiveSourceProjectionContext getContext(@NotNull VideoRtpPacket rtpPacket) { - PayloadType payloadTypeObject; int payloadType = rtpPacket.getPayloadType(); - if (context == null || contextPayloadType != payloadType) - { - payloadTypeObject = payloadTypes.get((byte)payloadType); - if (payloadTypeObject == null) - { - logger.error("No payload type object signalled for payload type " + payloadType + " yet, " + - "cannot create source projection context"); - return null; - } - } - else - { - // No need to call the expensive getDynamicRTPPayloadTypes. - payloadTypeObject = context.getPayloadType(); - } - - if (payloadTypeObject instanceof Vp8PayloadType) + if (rtpPacket instanceof Vp8Packet) { // Context switch between VP8 simulcast and VP8 non-simulcast (sort // of pretend that they're different codecs). @@ -233,8 +206,7 @@ private AdaptiveSourceProjectionContext getContext(@NotNull VideoRtpPacket rtpPa // then simulcast is disabled. /* Check whether this stream is projectable by the VP8AdaptiveSourceProjectionContext. */ - boolean projectable = rtpPacket instanceof Vp8Packet && - ((Vp8Packet)rtpPacket).getHasTemporalLayerIndex() && + boolean projectable = ((Vp8Packet)rtpPacket).getHasTemporalLayerIndex() && ((Vp8Packet)rtpPacket).getHasPictureId(); if (projectable @@ -244,39 +216,33 @@ private AdaptiveSourceProjectionContext getContext(@NotNull VideoRtpPacket rtpPa RtpState rtpState = getRtpState(); logger.debug(() -> "adaptive source projection " + (context == null ? "creating new" : "changing to") + - " VP8 context for payload type " - + payloadType + - ", source packet ssrc " + rtpPacket.getSsrc()); + " VP8 context for source packet ssrc " + rtpPacket.getSsrc()); context = new VP8AdaptiveSourceProjectionContext( - diagnosticContext, payloadTypeObject, rtpState, parentLogger); - contextPayloadType = payloadType; + diagnosticContext, rtpState, parentLogger); } else if (!projectable - && !(context instanceof GenericAdaptiveSourceProjectionContext)) + && (!(context instanceof GenericAdaptiveSourceProjectionContext) || + ((GenericAdaptiveSourceProjectionContext)context).getPayloadType() != payloadType)) { RtpState rtpState = getRtpState(); // context switch logger.debug(() -> { - boolean hasTemporalLayer = rtpPacket instanceof Vp8Packet && - ((Vp8Packet)rtpPacket).getHasTemporalLayerIndex(); - boolean hasPictureId = rtpPacket instanceof Vp8Packet && - ((Vp8Packet)rtpPacket).getHasPictureId(); + boolean hasTemporalLayer = ((Vp8Packet)rtpPacket).getHasTemporalLayerIndex(); + boolean hasPictureId = ((Vp8Packet)rtpPacket).getHasPictureId(); return "adaptive source projection " + (context == null ? "creating new" : "changing to") + - " generic context for non-scalable VP8 payload type " - + payloadType + + " generic context for non-scalable VP8 payload " + " (packet is " + rtpPacket.getClass().getSimpleName() + ", ssrc " + rtpPacket.getSsrc() + ", hasTL=" + hasTemporalLayer + ", hasPID=" + hasPictureId + ")"; }); - context = new GenericAdaptiveSourceProjectionContext(payloadTypeObject, rtpState, parentLogger); - contextPayloadType = payloadType; + context = new GenericAdaptiveSourceProjectionContext(payloadType, rtpState, parentLogger); } // no context switch return context; } - else if (payloadTypeObject instanceof Vp9PayloadType) + else if (rtpPacket instanceof Vp9Packet) { if (!(context instanceof Vp9AdaptiveSourceProjectionContext)) { @@ -284,28 +250,39 @@ else if (payloadTypeObject instanceof Vp9PayloadType) RtpState rtpState = getRtpState(); logger.debug(() -> "adaptive source projection " + (context == null ? "creating new" : "changing to") + - " VP9 context for payload type " - + payloadType + - ", source packet ssrc " + rtpPacket.getSsrc()); + " VP9 context for source packet ssrc " + rtpPacket.getSsrc()); context = new Vp9AdaptiveSourceProjectionContext( - diagnosticContext, payloadTypeObject, rtpState, parentLogger); - contextPayloadType = payloadType; + diagnosticContext, rtpState, parentLogger); } return context; } - else if (context == null || contextPayloadType != payloadType) + else if (rtpPacket instanceof Av1DDPacket) { - RtpState rtpState = getRtpState(); - logger.debug(() -> "adaptive source projection " + - (context == null ? "creating new" : "changing to") + - " generic context for payload type " + payloadType); - context = new GenericAdaptiveSourceProjectionContext(payloadTypeObject, rtpState, parentLogger); - contextPayloadType = payloadType; + if (!(context instanceof Av1DDAdaptiveSourceProjectionContext)) + { + // context switch + RtpState rtpState = getRtpState(); + logger.debug(() -> "adaptive source projection " + + (context == null ? "creating new" : "changing to") + + " AV1 DD context for source packet ssrc " + rtpPacket.getSsrc()); + context = new Av1DDAdaptiveSourceProjectionContext( + diagnosticContext, rtpState, parentLogger); + } + return context; } else { + if (!(context instanceof GenericAdaptiveSourceProjectionContext) || + ((GenericAdaptiveSourceProjectionContext)context).getPayloadType() != payloadType) + { + RtpState rtpState = getRtpState(); + logger.debug(() -> "adaptive source projection " + + (context == null ? "creating new" : "changing to") + + " generic context for payload type " + rtpPacket.getPayloadType()); + context = new GenericAdaptiveSourceProjectionContext(payloadType, rtpState, parentLogger); + } return context; } } @@ -384,7 +361,6 @@ public JSONObject getDebugState() debugState.put( "context", contextCopy == null ? null : contextCopy.getDebugState()); - debugState.put("contextPayloadType", contextPayloadType); debugState.put("targetIndex", targetIndex); return debugState; diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjectionContext.java b/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjectionContext.java index 72629e9f1a..af56947d67 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjectionContext.java +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjectionContext.java @@ -16,10 +16,11 @@ package org.jitsi.videobridge.cc; import org.jitsi.nlj.*; -import org.jitsi.nlj.format.*; import org.jitsi.rtp.rtcp.*; import org.json.simple.*; +import java.util.*; + /** * Implementations of this interface are responsible for projecting a specific * video source of a specific payload type. @@ -40,11 +41,11 @@ public interface AdaptiveSourceProjectionContext * Determines whether an RTP packet should be accepted or not. * * @param packetInfo the RTP packet to determine whether to accept or not. - * @param incomingIndex the quality index of the incoming RTP packet. + * @param incomingEncoding The encoding of the incoming packet. * @param targetIndex the target quality index * @return true if the packet should be accepted, false otherwise. */ - boolean accept(PacketInfo packetInfo, int incomingIndex, int targetIndex); + boolean accept(PacketInfo packetInfo, int incomingEncoding, int targetIndex); /** * @return true if this stream context needs a keyframe in order to either @@ -81,12 +82,6 @@ void rewriteRtp(PacketInfo packetInfo) */ RtpState getRtpState(); - /** - * @return the {@link PayloadType} of the RTP packets that this context - * processes. - */ - PayloadType getPayloadType(); - /** * Gets a JSON representation of the parts of this object's state that * are deemed useful for debugging. diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/GenericAdaptiveSourceProjectionContext.java b/jvb/src/main/java/org/jitsi/videobridge/cc/GenericAdaptiveSourceProjectionContext.java index 6a6e6d9987..f1561bab66 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/cc/GenericAdaptiveSourceProjectionContext.java +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/GenericAdaptiveSourceProjectionContext.java @@ -17,13 +17,14 @@ import org.jetbrains.annotations.*; import org.jitsi.nlj.*; -import org.jitsi.nlj.format.*; import org.jitsi.nlj.rtp.*; import org.jitsi.rtp.rtcp.*; import org.jitsi.rtp.util.*; import org.jitsi.utils.logging2.*; import org.json.simple.*; +import java.util.*; + /** * A generic implementation of an adaptive source projection context that can be * used with non-SVC codecs or when simulcast is not enabled/used or when @@ -69,9 +70,9 @@ class GenericAdaptiveSourceProjectionContext private boolean needsKeyframe = true; /** - * Useful to determine whether a packet is a "keyframe". + * Useful to determine when we need to change generic projection contexts. */ - private final PayloadType payloadType; + private final int payloadType; /** * The maximum sequence number that we have sent. @@ -109,7 +110,7 @@ class GenericAdaptiveSourceProjectionContext * @param rtpState the RTP state (i.e. seqnum, timestamp to start with, etc). */ GenericAdaptiveSourceProjectionContext( - @NotNull PayloadType payloadType, + int payloadType, @NotNull RtpState rtpState, @NotNull Logger parentLogger) { @@ -131,13 +132,13 @@ class GenericAdaptiveSourceProjectionContext * thread) accessing this method at a time. * * @param packetInfo the RTP packet to determine whether to accept or not. - * @param incomingIndex the quality index of the + * @param incomingEncoding The encoding index of the packet * @param targetIndex the target quality index * @return true if the packet should be accepted, false otherwise. */ @Override public synchronized boolean - accept(@NotNull PacketInfo packetInfo, int incomingIndex, int targetIndex) + accept(@NotNull PacketInfo packetInfo, int incomingEncoding, int targetIndex) { VideoRtpPacket rtpPacket = packetInfo.packetAs(); if (targetIndex == RtpLayerDesc.SUSPENDED_INDEX) @@ -327,8 +328,7 @@ public RtpState getRtpState() ssrc, maxDestinationSequenceNumber, maxDestinationTimestamp); } - @Override - public PayloadType getPayloadType() + public int getPayloadType() { return payloadType; } diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionContext.java b/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionContext.java index da073e321c..97be97d72a 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionContext.java +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionContext.java @@ -18,11 +18,9 @@ import org.jetbrains.annotations.*; import org.jitsi.nlj.*; import org.jitsi.nlj.codec.vpx.*; -import org.jitsi.nlj.format.*; import org.jitsi.nlj.rtp.codec.vp8.*; import org.jitsi.rtp.rtcp.*; import org.jitsi.rtp.util.*; -import org.jitsi.utils.*; import org.jitsi.utils.logging.*; import org.jitsi.utils.logging2.Logger; import org.jitsi.videobridge.cc.*; @@ -67,30 +65,19 @@ public class VP8AdaptiveSourceProjectionContext */ private VP8FrameProjection lastVP8FrameProjection; - /** - * The VP8 media format. No essential functionality relies on this field, - * it's only used as a cache of the {@link PayloadType} instance for VP8 in - * case we have to do a context switch (see {@link AdaptiveSourceProjection}), - * in order to avoid having to resolve the format. - */ - private final PayloadType payloadType; - /** * Ctor. * - * @param payloadType the VP8 media format. * @param rtpState the RTP state to begin with. */ public VP8AdaptiveSourceProjectionContext( @NotNull DiagnosticContext diagnosticContext, - @NotNull PayloadType payloadType, @NotNull RtpState rtpState, @NotNull Logger parentLogger) { this.diagnosticContext = diagnosticContext; this.logger = parentLogger.createChildLogger( VP8AdaptiveSourceProjectionContext.class.getName()); - this.payloadType = payloadType; this.vp8QualityFilter = new VP8QualityFilter(parentLogger); lastVP8FrameProjection = new VP8FrameProjection(diagnosticContext, @@ -265,13 +252,13 @@ private boolean frameIsNewSsrc(VP8Frame frame) * Determines whether a packet should be accepted or not. * * @param packetInfo the RTP packet to determine whether to project or not. - * @param incomingIndex the quality index of the incoming RTP packet + * @param incomingEncoding the encoding of the incoming RTP packet * @param targetIndex the target quality index we want to achieve * @return true if the packet should be accepted, false otherwise. */ @Override public synchronized boolean accept( - @NotNull PacketInfo packetInfo, int incomingIndex, int targetIndex) + @NotNull PacketInfo packetInfo, int incomingEncoding, int targetIndex) { if (!(packetInfo.getPacket() instanceof Vp8Packet)) { @@ -309,7 +296,7 @@ public synchronized boolean accept( Instant receivedTime = packetInfo.getReceivedTime(); boolean accepted = vp8QualityFilter - .acceptFrame(frame, incomingIndex, targetIndex, receivedTime); + .acceptFrame(frame, incomingEncoding, targetIndex, receivedTime); if (accepted) { @@ -672,12 +659,6 @@ public RtpState getRtpState() lastVP8FrameProjection.getTimestamp()); } - @Override - public PayloadType getPayloadType() - { - return payloadType; - } - /** * Rewrites the RTP packet that is specified as an argument. * @@ -744,7 +725,6 @@ public synchronized JSONObject getDebugState() debugState.put( "vp8FrameMaps", mapSizes); debugState.put("vp8QualityFilter", vp8QualityFilter.getDebugState()); - debugState.put("payloadType", payloadType.toString()); return debugState; } diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8QualityFilter.java b/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8QualityFilter.java index e3b635d515..9461b37a91 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8QualityFilter.java +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8QualityFilter.java @@ -106,7 +106,7 @@ boolean needsKeyframe() * method at a time. * * @param frame the VP8 frame. - * @param incomingIndex the quality index of the incoming RTP packet + * @param incomingEncoding the encoding index of the incoming RTP packet * @param externalTargetIndex the target quality index that the user of this * instance wants to achieve. * @param receivedTime the current time @@ -114,7 +114,7 @@ boolean needsKeyframe() */ synchronized boolean acceptFrame( @NotNull VP8Frame frame, - int incomingIndex, + int incomingEncoding, int externalTargetIndex, Instant receivedTime) { // We make local copies of the externalTemporalLayerIdTarget and the @@ -154,16 +154,15 @@ synchronized boolean acceptFrame( temporalLayerIdOfFrame = 0; } - int encodingId = RtpLayerDesc.getEidFromIndex(incomingIndex); if (frame.isKeyframe()) { logger.debug(() -> "Quality filter got keyframe for stream " + frame.getSsrc()); - return acceptKeyframe(encodingId, receivedTime); + return acceptKeyframe(incomingEncoding, receivedTime); } else if (currentEncodingId > SUSPENDED_ENCODING_ID) { - if (isOutOfSwitchingPhase(receivedTime) && isPossibleToSwitch(encodingId)) + if (isOutOfSwitchingPhase(receivedTime) && isPossibleToSwitch(incomingEncoding)) { // XXX(george) i've noticed some "rogue" base layer keyframes // that trigger this. what happens is the client sends a base @@ -176,7 +175,7 @@ else if (currentEncodingId > SUSPENDED_ENCODING_ID) needsKeyframe = true; } - if (encodingId != currentEncodingId) + if (incomingEncoding != currentEncodingId) { // for non-keyframes, we can't route anything but the current encoding return false; @@ -206,13 +205,12 @@ else if (currentEncodingId < externalEncodingIdTarget) else { // In this branch we're not processing a keyframe and the - // currentSpatialLayerId is in suspended state, which means we need - // a keyframe to start streaming again. Reaching this point also - // means that we want to forward something (because both - // externalEncodingIdTarget and externalTemporalLayerIdTarget - // are greater than 0) so we set the request keyframe flag. + // currentEncodingId is in suspended state, which means we need + // a keyframe to start streaming again. + + // We should have already requested a keyframe, either above or when the + // internal target encoding was first moved off SUSPENDED_ENCODING. - // assert needsKeyframe == true; return false; } } @@ -222,7 +220,7 @@ else if (currentEncodingId < externalEncodingIdTarget) * or not. * * @param receivedTime the time the latest frame was received - * @return true if we're in layer switching phase, false otherwise. + * @return false if we're in layer switching phase, true otherwise. */ private synchronized boolean isOutOfSwitchingPhase(@Nullable Instant receivedTime) { diff --git a/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java b/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java index 44d1b78051..3d9c777131 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java +++ b/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java @@ -17,6 +17,7 @@ import org.jitsi.nlj.*; import org.jitsi.nlj.rtp.*; +import org.jitsi.nlj.rtp.codec.vpx.*; import org.jitsi.utils.logging2.*; import org.jitsi.xmpp.extensions.colibri.*; import org.jitsi.xmpp.extensions.jingle.*; @@ -80,7 +81,7 @@ Map getSecondarySsrcTypeMap() return secondarySsrcTypeMap; } - private static final RtpLayerDesc[] noDependencies = new RtpLayerDesc[0]; + private static final VpxRtpLayerDesc[] noDependencies = new VpxRtpLayerDesc[0]; /* * Creates layers for an encoding. @@ -93,8 +94,8 @@ Map getSecondarySsrcTypeMap() private static RtpLayerDesc[] createRTPLayerDescs( int spatialLen, int temporalLen, int encodingIdx, int height) { - RtpLayerDesc[] rtpLayers - = new RtpLayerDesc[spatialLen * temporalLen]; + VpxRtpLayerDesc[] rtpLayers + = new VpxRtpLayerDesc[spatialLen * temporalLen]; for (int spatialIdx = 0; spatialIdx < spatialLen; spatialIdx++) { @@ -105,11 +106,11 @@ private static RtpLayerDesc[] createRTPLayerDescs( int idx = idx(spatialIdx, temporalIdx, temporalLen); - RtpLayerDesc[] dependencies; + VpxRtpLayerDesc[] dependencies; if (spatialIdx > 0 && temporalIdx > 0) { // this layer depends on spatialIdx-1 and temporalIdx-1. - dependencies = new RtpLayerDesc[]{ + dependencies = new VpxRtpLayerDesc[]{ rtpLayers[ idx(spatialIdx, temporalIdx - 1, temporalLen)], @@ -121,7 +122,7 @@ private static RtpLayerDesc[] createRTPLayerDescs( else if (spatialIdx > 0) { // this layer depends on spatialIdx-1. - dependencies = new RtpLayerDesc[] + dependencies = new VpxRtpLayerDesc[] {rtpLayers[ idx(spatialIdx - 1, temporalIdx, temporalLen)]}; @@ -129,7 +130,7 @@ else if (spatialIdx > 0) else if (temporalIdx > 0) { // this layer depends on temporalIdx-1. - dependencies = new RtpLayerDesc[] + dependencies = new VpxRtpLayerDesc[] {rtpLayers[ idx(spatialIdx, temporalIdx - 1, temporalLen)]}; @@ -144,13 +145,11 @@ else if (temporalIdx > 0) int spatialId = spatialLen > 1 ? spatialIdx : -1; rtpLayers[idx] - = new RtpLayerDesc(encodingIdx, + = new VpxRtpLayerDesc(encodingIdx, temporalId, spatialId, height, frameRate, dependencies); frameRate *= 2; } - - } return rtpLayers; } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt index 5189d7f391..1754ab6aa0 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt @@ -20,6 +20,9 @@ import org.jitsi.nlj.MediaSourceDesc import org.jitsi.nlj.VideoType import org.jitsi.nlj.codec.vpx.VpxUtils import org.jitsi.nlj.rtp.SsrcAssociationType +import org.jitsi.nlj.rtp.codec.av1.Av1DDPacket +import org.jitsi.nlj.rtp.codec.av1.applyTemplateIdDelta +import org.jitsi.nlj.rtp.codec.av1.getTemplateIdDelta import org.jitsi.nlj.rtp.codec.vp8.Vp8Packet import org.jitsi.nlj.rtp.codec.vp9.Vp9Packet import org.jitsi.rtp.rtcp.RtcpPacket @@ -660,10 +663,65 @@ private class Vp9CodecDeltas(val tl0IndexDelta: Int) : CodecDeltas { override fun toString() = "[VP9 TL0Idx]$tl0IndexDelta" } +private class Av1DDCodecState : CodecState { + val lastFrameNum: Int + val lastTemplateIdx: Int + constructor(lastFrameNum: Int, lastTemplateIdx: Int) { + this.lastFrameNum = lastFrameNum + this.lastTemplateIdx = lastTemplateIdx + } + + constructor(packet: Av1DDPacket) { + val descriptor = packet.descriptor + requireNotNull(descriptor) { "AV1 Packet being routed must have non-null descriptor" } + this.lastFrameNum = packet.frameNumber + this.lastTemplateIdx = descriptor.structure.templateIdOffset + descriptor.structure.templateCount + } + + override fun getDeltas(otherState: CodecState?): CodecDeltas? { + if (otherState !is Av1DDCodecState) { + return null + } + val frameNumDelta = RtpUtils.getSequenceNumberDelta(lastFrameNum, otherState.lastFrameNum) + val templateIdDelta = getTemplateIdDelta(lastTemplateIdx, otherState.lastTemplateIdx) + return Av1DDCodecDeltas(frameNumDelta, templateIdDelta) + } + + override fun getDeltas(packet: RtpPacket): CodecDeltas? { + if (packet !is Av1DDPacket) { + return null + } + val descriptor = packet.descriptor ?: return null + val frameNumDelta = RtpUtils.getSequenceNumberDelta(lastFrameNum, packet.frameNumber - 1) + val packetLastTemplateIdx = descriptor.structure.templateIdOffset + descriptor.structure.templateCount + val templateIdDelta = getTemplateIdDelta(lastTemplateIdx, packetLastTemplateIdx - 1) + return Av1DDCodecDeltas(frameNumDelta, templateIdDelta) + } +} + +private class Av1DDCodecDeltas(val frameNumDelta: Int, val templateIdDelta: Int) : CodecDeltas { + override fun rewritePacket(packet: RtpPacket) { + require(packet is Av1DDPacket) + val descriptor = packet.descriptor + requireNotNull(descriptor) + + descriptor.frameNumber = RtpUtils.applySequenceNumberDelta(descriptor.frameNumber, frameNumDelta) + descriptor.frameDependencyTemplateId = + applyTemplateIdDelta(descriptor.frameDependencyTemplateId, templateIdDelta) + descriptor.structure.templateIdOffset = + applyTemplateIdDelta(descriptor.structure.templateIdOffset, templateIdDelta) + + packet.reencodeDdExt() + } + + override fun toString() = "[AV1DD FrameNum]$frameNumDelta [Av1DD templateId]$templateIdDelta" +} + private fun RtpPacket.getCodecState(): CodecState? { return when (this) { is Vp8Packet -> Vp8CodecState(this) is Vp9Packet -> Vp9CodecState(this) + is Av1DDPacket -> Av1DDCodecState(this) else -> null } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt index ebca3fd157..16681fac83 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt @@ -60,6 +60,13 @@ class BandwidthAllocation @JvmOverloads constructor( put("oversending", oversending) put("has_suspended_sources", hasSuspendedSources) put("suspended_sources", suspendedSources) + val allocations = JSONObject().apply { + allocations.forEach { + val name = it.mediaSource?.sourceName ?: it.endpointId + put(name, it.debugState) + } + } + put("allocations", allocations) } } @@ -93,4 +100,10 @@ data class SingleAllocation( override fun toString(): String = "[id=$endpointId target=${targetLayer?.height}/${targetLayer?.frameRate} " + "ideal=${idealLayer?.height}/${idealLayer?.frameRate}]" + + val debugState: JSONObject + get() = JSONObject().apply { + put("target", targetLayer?.debugState()) + put("ideal", idealLayer?.debugState()) + } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt index 851468a01c..de2e61400d 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt @@ -163,8 +163,6 @@ class BitrateController @JvmOverloads constructor( } fun addPayloadType(payloadType: PayloadType) { - packetHandler.addPayloadType(payloadType) - if (payloadType.encoding == PayloadTypeEncoding.RTX) { supportsRtx = true } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/PacketHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/PacketHandler.kt index ed0d2c040c..3cff98225c 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/PacketHandler.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/PacketHandler.kt @@ -19,7 +19,6 @@ import org.jitsi.nlj.MediaSourceDesc import org.jitsi.nlj.PacketInfo import org.jitsi.nlj.PacketInfo.Companion.ENABLE_PAYLOAD_VERIFICATION import org.jitsi.nlj.RtpLayerDesc -import org.jitsi.nlj.format.PayloadType import org.jitsi.nlj.rtp.VideoRtpPacket import org.jitsi.rtp.rtcp.RtcpSrPacket import org.jitsi.utils.event.EventEmitter @@ -59,7 +58,6 @@ internal class PacketHandler( private var firstMedia: Instant? = null private val numDroppedPacketsUnknownSsrc = AtomicInteger(0) - private val payloadTypes: MutableMap = ConcurrentHashMap() /** * The [AdaptiveSourceProjection]s that this instance is managing, keyed @@ -158,7 +156,6 @@ internal class PacketHandler( { eventEmitter.fireEvent { keyframeNeeded(endpointID, source.primarySSRC) } }, - payloadTypes, logger ) logger.debug { "new source projection for $source" } @@ -173,10 +170,6 @@ internal class PacketHandler( fun timeSinceFirstMedia(): Duration = firstMedia?.let { Duration.between(it, clock.instant()) } ?: Duration.ZERO - fun addPayloadType(payloadType: PayloadType) { - payloadTypes[payloadType.pt] = payloadType - } - val debugState: JSONObject get() = JSONObject().apply { this["numDroppedPacketsUnknownSsrc"] = numDroppedPacketsUnknownSsrc.toInt() diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt index 509b42616d..82d139f860 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt @@ -18,7 +18,6 @@ package org.jitsi.videobridge.cc.allocation import org.jitsi.nlj.MediaSourceDesc import org.jitsi.nlj.RtpLayerDesc -import org.jitsi.nlj.RtpLayerDesc.Companion.indexString import org.jitsi.nlj.VideoType import org.jitsi.utils.logging.DiagnosticContext import org.jitsi.utils.logging.TimeSeriesLogger @@ -63,7 +62,7 @@ internal class SingleSourceAllocation( .addField("remote_endpoint_id", endpointId) for ((l, bitrate) in layers.layers) { ratesTimeSeriesPoint.addField( - "${indexString(l.index)}_${l.height}p_${l.frameRate}fps_bps", + "${l.indexString()}_${l.height}p_${l.frameRate}fps_bps", bitrate ) } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt new file mode 100644 index 0000000000..78159c4f0c --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt @@ -0,0 +1,659 @@ +/* + * Copyright @ 2019-present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.av1 + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.RtpLayerDesc.Companion.getEidFromIndex +import org.jitsi.nlj.rtp.codec.av1.Av1DDPacket +import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc +import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc.Companion.getDtFromIndex +import org.jitsi.nlj.rtp.codec.av1.getTemplateIdDelta +import org.jitsi.rtp.rtcp.RtcpSrPacket +import org.jitsi.rtp.rtp.header_extensions.DTI +import org.jitsi.rtp.rtp.header_extensions.toShortString +import org.jitsi.rtp.util.RtpUtils +import org.jitsi.rtp.util.isNewerThan +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.videobridge.cc.AdaptiveSourceProjectionContext +import org.jitsi.videobridge.cc.RewriteException +import org.jitsi.videobridge.cc.RtpState +import org.json.simple.JSONArray +import org.json.simple.JSONObject +import java.time.Duration +import java.time.Instant + +class Av1DDAdaptiveSourceProjectionContext( + private val diagnosticContext: DiagnosticContext, + rtpState: RtpState, + parentLogger: Logger +) : AdaptiveSourceProjectionContext { + private val logger: Logger = createChildLogger(parentLogger) + + /** + * A map that stores the per-encoding AV1 frame maps. + */ + private val av1FrameMaps = HashMap() + + /** + * The [Av1DDQualityFilter] instance that does quality filtering on the + * incoming pictures, to choose encodings and layers to forward. + */ + private val av1QualityFilter = Av1DDQualityFilter(av1FrameMaps, logger) + + private var lastAv1FrameProjection = Av1DDFrameProjection( + diagnosticContext, + rtpState.ssrc, + rtpState.maxSequenceNumber, + rtpState.maxTimestamp + ) + + /** + * The frame number index that started the latest stream resumption. + * We can't send frames with frame number less than this, because we don't have + * space in the projected sequence number/frame number counts. + */ + private var lastFrameNumberIndexResumption = -1 + + override fun accept(packetInfo: PacketInfo, incomingEncoding: Int, targetIndex: Int): Boolean { + val packet = packetInfo.packet + + if (packet !is Av1DDPacket) { + logger.warn("Packet is not AV1 DD Packet") + return false + } + + /* If insertPacketInMap returns null, this is a very old picture, more than Av1FrameMap.PICTURE_MAP_SIZE old, + or something is wrong with the stream. */ + val result = insertPacketInMap(packet) ?: return false + + val frame = result.frame + + if (result.isNewFrame) { + if (packet.isKeyframe && frameIsNewSsrc(frame)) { + /* If we're not currently projecting this SSRC, check if we've + * already decided to drop a subsequent required frame of this SSRC for the DT. + * If we have, we can't turn on the encoding starting from this + * packet, so treat this frame as though it weren't a keyframe. + * Note that this may mean re-analyzing the packets with a now-available template dependency structure. + */ + if (haveSubsequentNonAcceptedChain(frame, incomingEncoding, targetIndex)) { + frame.isKeyframe = false + } + } + val receivedTime = packetInfo.receivedTime + val acceptResult = av1QualityFilter + .acceptFrame(frame, incomingEncoding, targetIndex, receivedTime) + frame.isAccepted = acceptResult.accept && frame.index >= lastFrameNumberIndexResumption + if (frame.isAccepted) { + val projection: Av1DDFrameProjection + try { + projection = createProjection( + frame = frame, + initialPacket = packet, + isResumption = acceptResult.isResumption, + isReset = result.isReset, + mark = acceptResult.mark, + newDt = acceptResult.newDt, + receivedTime = receivedTime + ) + } catch (e: Exception) { + logger.warn("Failed to create frame projection", e) + /* Make sure we don't have an accepted frame without a projection in the map. */ + frame.isAccepted = false + return false + } + frame.projection = projection + if (projection.earliestProjectedSeqNum isNewerThan lastAv1FrameProjection.latestProjectedSeqNum) { + lastAv1FrameProjection = projection + } + } + } + + val accept = frame.isAccepted && frame.projection?.accept(packet) == true + + if (timeSeriesLogger.isTraceEnabled) { + val pt = diagnosticContext.makeTimeSeriesPoint("rtp_av1") + .addField("ssrc", packet.ssrc) + .addField("timestamp", packet.timestamp) + .addField("seq", packet.sequenceNumber) + .addField("frameNumber", packet.frameNumber) + .addField("templateId", packet.statelessDescriptor.frameDependencyTemplateId) + .addField("hasStructure", packet.descriptor?.newTemplateDependencyStructure != null) + .addField("spatialLayer", packet.frameInfo?.spatialId) + .addField("temporalLayer", packet.frameInfo?.temporalId) + .addField("dti", packet.frameInfo?.dti?.toShortString()) + .addField("hasInterPictureDependency", packet.frameInfo?.hasInterPictureDependency()) + // TODO add more relevant fields from AV1 DD for debugging + .addField("targetIndex", Av1DDRtpLayerDesc.indexString(targetIndex)) + .addField("new_frame", result.isNewFrame) + .addField("accept", accept) + av1QualityFilter.addDiagnosticContext(pt) + timeSeriesLogger.trace(pt) + } + + return accept + } + + /** Look up an Av1DDFrame for a packet. */ + private fun lookupAv1Frame(av1Packet: Av1DDPacket): Av1DDFrame? = av1FrameMaps[av1Packet.ssrc]?.findFrame(av1Packet) + + /** + * Insert a packet in the appropriate [Av1DDFrameMap]. + */ + private fun insertPacketInMap(av1Packet: Av1DDPacket) = + av1FrameMaps.getOrPut(av1Packet.ssrc) { Av1DDFrameMap(logger) }.insertPacket(av1Packet) + + private fun haveSubsequentNonAcceptedChain(frame: Av1DDFrame, incomingEncoding: Int, targetIndex: Int): Boolean { + val map = av1FrameMaps[frame.ssrc] ?: return false + val structure = frame.structure ?: return false + val dtsToCheck = if (incomingEncoding == getEidFromIndex(targetIndex)) { + setOf(getDtFromIndex(targetIndex)) + } else { + frame.frameInfo?.dtisPresent ?: emptySet() + } + val chainsToCheck = dtsToCheck.map { structure.decodeTargetInfo[it].protectedBy }.toSet() + return map.nextFrameWith(frame) { + if (it.isAccepted) return@nextFrameWith false + if (it.frameInfo == null) { + it.updateParse(structure, logger) + } + return@nextFrameWith chainsToCheck.any { chainIdx -> + it.partOfActiveChain(chainIdx) + } + } != null + } + + private fun Av1DDFrame.partOfActiveChain(chainIdx: Int): Boolean { + val structure = structure ?: return false + val frameInfo = frameInfo ?: return false + for (i in structure.decodeTargetInfo.indices) { + if (structure.decodeTargetInfo[i].protectedBy != chainIdx) { + continue + } + if (frameInfo.dti[i] == DTI.NOT_PRESENT || frameInfo.dti[i] == DTI.DISCARDABLE) { + return false + } + } + return true + } + + /** + * Calculate the projected sequence number gap between two frames (of the same encoding), + * allowing collapsing for unaccepted frames. + */ + private fun seqGap(frame1: Av1DDFrame, frame2: Av1DDFrame): Int { + var seqGap = RtpUtils.getSequenceNumberDelta( + frame2.earliestKnownSequenceNumber, + frame1.latestKnownSequenceNumber + ) + if (!frame1.isAccepted && !frame2.isAccepted && frame2.isImmediatelyAfter(frame1)) { + /* If neither frame is being projected, and they have consecutive + frame numbers, we don't need to leave any gap. */ + seqGap = 0 + } else { + /* If the earlier frame wasn't projected, and we haven't seen its + * final packet, we know it has to consume at least one more sequence number. */ + if (!frame1.isAccepted && !frame1.seenEndOfFrame && seqGap > 1) { + seqGap-- + } + /* Similarly, if the later frame wasn't projected and we haven't seen + * its first packet. */ + if (!frame2.isAccepted && !frame2.seenStartOfFrame && seqGap > 1) { + seqGap-- + } + /* If the frame wasn't accepted, it has to have consumed at least one sequence number, + * which we can collapse out. */ + if (!frame1.isAccepted && seqGap > 0) { + seqGap-- + } + } + return seqGap + } + + private fun frameIsNewSsrc(frame: Av1DDFrame): Boolean = lastAv1FrameProjection.av1Frame?.matchesSSRC(frame) != true + + /** + * Find the previous frame before the given one. + */ + @Synchronized + private fun prevFrame(frame: Av1DDFrame) = av1FrameMaps[frame.ssrc]?.prevFrame(frame) + + /** + * Find the next frame after the given one. + */ + @Synchronized + private fun nextFrame(frame: Av1DDFrame) = av1FrameMaps[frame.ssrc]?.nextFrame(frame) + + /** + * Find the previous accepted frame before the given one. + */ + private fun findPrevAcceptedFrame(frame: Av1DDFrame) = av1FrameMaps[frame.ssrc]?.findPrevAcceptedFrame(frame) + + /** + * Find the next accepted frame after the given one. + */ + private fun findNextAcceptedFrame(frame: Av1DDFrame) = av1FrameMaps[frame.ssrc]?.findNextAcceptedFrame(frame) + + /** + * Create a projection for this frame. + */ + private fun createProjection( + frame: Av1DDFrame, + initialPacket: Av1DDPacket, + mark: Boolean, + isResumption: Boolean, + isReset: Boolean, + newDt: Int?, + receivedTime: Instant? + ): Av1DDFrameProjection { + if (frameIsNewSsrc(frame)) { + return createEncodingSwitchProjection(frame, initialPacket, mark, newDt, receivedTime) + } else if (isResumption) { + return createResumptionProjection(frame, initialPacket, mark, newDt, receivedTime) + } else if (isReset) { + return createResetProjection(frame, initialPacket, mark, newDt, receivedTime) + } + + return createInEncodingProjection(frame, initialPacket, mark, newDt, receivedTime) + } + + /** + * Create an projection for the first frame after an encoding switch. + */ + private fun createEncodingSwitchProjection( + frame: Av1DDFrame, + initialPacket: Av1DDPacket, + mark: Boolean, + newDt: Int?, + receivedTime: Instant? + ): Av1DDFrameProjection { + // We can only switch on packets that carry a scalability structure, which is the first packet of a keyframe + assert(frame.isKeyframe) + assert(initialPacket.isStartOfFrame) + + var projectedSeqGap = 1 + + if (lastAv1FrameProjection.av1Frame?.seenEndOfFrame == false) { + /* Leave a gap to signal to the decoder that the previously routed + frame was incomplete. */ + projectedSeqGap++ + + /* Make sure subsequent packets of the previous projection won't + overlap the new one. (This means the gap, above, will never be + filled in.) + */ + lastAv1FrameProjection.close() + } + + val projectedSeq = + RtpUtils.applySequenceNumberDelta(lastAv1FrameProjection.latestProjectedSeqNum, projectedSeqGap) + + // this is a simulcast switch. The typical incremental value = + // 90kHz / 30 = 90,000Hz / 30 = 3000 per frame or per 33ms + val tsDelta = lastAv1FrameProjection.created?.let { created -> + receivedTime?.let { + 3000 * Duration.between(created, receivedTime).dividedBy(33).seconds.coerceAtLeast(1L) + } + } ?: 3000 + val projectedTs = RtpUtils.applyTimestampDelta(lastAv1FrameProjection.timestamp, tsDelta) + + val frameNumber: Int + val templateIdDelta: Int + if (lastAv1FrameProjection.av1Frame != null) { + frameNumber = RtpUtils.applySequenceNumberDelta( + lastAv1FrameProjection.frameNumber, + 1 + ) + val nextTemplateId = lastAv1FrameProjection.getNextTemplateId() + templateIdDelta = if (nextTemplateId != null) { + val structure = frame.structure + check(structure != null) + getTemplateIdDelta(nextTemplateId, structure.templateIdOffset) + } else { + 0 + } + } else { + frameNumber = frame.frameNumber + templateIdDelta = 0 + } + + return Av1DDFrameProjection( + diagnosticContext = diagnosticContext, + av1Frame = frame, + ssrc = lastAv1FrameProjection.ssrc, + timestamp = projectedTs, + sequenceNumberDelta = RtpUtils.getSequenceNumberDelta(projectedSeq, initialPacket.sequenceNumber), + frameNumber = frameNumber, + templateIdDelta = templateIdDelta, + dti = newDt?.let { frame.structure?.getDtBitmaskForDt(it) }, + mark = mark, + created = receivedTime + ) + } + + /** + * Create a projection for the first frame after a resumption, i.e. when a source is turned back on. + */ + private fun createResumptionProjection( + frame: Av1DDFrame, + initialPacket: Av1DDPacket, + mark: Boolean, + newDt: Int?, + receivedTime: Instant? + ): Av1DDFrameProjection { + lastFrameNumberIndexResumption = frame.index + + /* These must be non-null because we don't execute this function unless + frameIsNewSsrc has returned false. + */ + val lastFrame = prevFrame(frame)!! + val lastProjectedFrame = lastAv1FrameProjection.av1Frame!! + + /* Project timestamps linearly. */ + val tsDelta = RtpUtils.getTimestampDiff( + lastAv1FrameProjection.timestamp, + lastProjectedFrame.timestamp + ) + val projectedTs = RtpUtils.applyTimestampDelta(frame.timestamp, tsDelta) + + /* Increment frameNumber by 1 from the last projected frame. */ + val projectedFrameNumber = RtpUtils.applySequenceNumberDelta(lastAv1FrameProjection.frameNumber, 1) + + /** If this packet has a template structure, rewrite it to follow after the pre-resumption structure. + * (We could check to see if the structure is unchanged, but that's an unnecessary optimization.) + */ + val templateIdDelta = if (frame.structure != null) { + val nextTemplateId = lastAv1FrameProjection.getNextTemplateId() + + if (nextTemplateId != null) { + val structure = frame.structure + check(structure != null) + getTemplateIdDelta(nextTemplateId, structure.templateIdOffset) + } else { + 0 + } + } else { + 0 + } + + /* Increment sequence numbers based on the last projected frame, but leave a gap + * for packet reordering in case this isn't the first packet of the keyframe. + */ + val seqGap = RtpUtils.getSequenceNumberDelta(initialPacket.sequenceNumber, lastFrame.latestKnownSequenceNumber) + val newSeq = RtpUtils.applySequenceNumberDelta(lastAv1FrameProjection.latestProjectedSeqNum, seqGap) + val seqDelta = RtpUtils.getSequenceNumberDelta(newSeq, initialPacket.sequenceNumber) + + return Av1DDFrameProjection( + diagnosticContext = diagnosticContext, + av1Frame = frame, + ssrc = lastAv1FrameProjection.ssrc, + timestamp = projectedTs, + sequenceNumberDelta = seqDelta, + frameNumber = projectedFrameNumber, + templateIdDelta = templateIdDelta, + dti = newDt?.let { frame.structure?.getDtBitmaskForDt(it) }, + mark = mark, + created = receivedTime + ) + } + + /** + * Create a projection for the first frame after a frame reset, i.e. after a large gap in sequence numbers. + */ + private fun createResetProjection( + frame: Av1DDFrame, + initialPacket: Av1DDPacket, + mark: Boolean, + newDt: Int?, + receivedTime: Instant? + ): Av1DDFrameProjection { + /* This must be non-null because we don't execute this function unless + frameIsNewSsrc has returned false. + */ + val lastFrame = lastAv1FrameProjection.av1Frame!! + + /* Apply the latest projected frame's projections out, linearly. */ + val seqDelta = RtpUtils.getSequenceNumberDelta( + lastAv1FrameProjection.latestProjectedSeqNum, + lastFrame.latestKnownSequenceNumber + ) + val tsDelta = RtpUtils.getTimestampDiff( + lastAv1FrameProjection.timestamp, + lastFrame.timestamp + ) + val frameNumberDelta = RtpUtils.applySequenceNumberDelta( + lastAv1FrameProjection.frameNumber, + lastFrame.frameNumber + ) + + val projectedTs = RtpUtils.applyTimestampDelta(frame.timestamp, tsDelta) + val projectedFrameNumber = RtpUtils.applySequenceNumberDelta(frame.frameNumber, frameNumberDelta) + + /** If this packet has a template structure, rewrite it to follow after the pre-reset structure. + * (We could check to see if the structure is unchanged, but that's an unnecessary optimization.) + */ + val templateIdDelta = if (frame.structure != null) { + val nextTemplateId = lastAv1FrameProjection.getNextTemplateId() + + if (nextTemplateId != null) { + val structure = frame.structure + check(structure != null) + getTemplateIdDelta(nextTemplateId, structure.templateIdOffset) + } else { + 0 + } + } else { + 0 + } + return Av1DDFrameProjection( + diagnosticContext = diagnosticContext, + av1Frame = frame, + ssrc = lastAv1FrameProjection.ssrc, + timestamp = projectedTs, + sequenceNumberDelta = seqDelta, + frameNumber = projectedFrameNumber, + templateIdDelta = templateIdDelta, + dti = newDt?.let { frame.structure?.getDtBitmaskForDt(it) }, + mark = mark, + created = receivedTime + ) + } + + /** + * Create a frame projection for the normal case, i.e. as part of the same encoding as the + * previously-projected frame. + */ + private fun createInEncodingProjection( + frame: Av1DDFrame, + initialPacket: Av1DDPacket, + mark: Boolean, + newDt: Int?, + receivedTime: Instant? + ): Av1DDFrameProjection { + val prevFrame = findPrevAcceptedFrame(frame) + if (prevFrame != null) { + return createInEncodingProjection(frame, prevFrame, initialPacket, mark, newDt, receivedTime) + } + + /* prev frame has rolled off beginning of frame map, try next frame */ + val nextFrame = findNextAcceptedFrame(frame) + if (nextFrame != null) { + return createInEncodingProjection(frame, nextFrame, initialPacket, mark, newDt, receivedTime) + } + + /* Neither previous or next is found. Very big frame? Use previous projected. + (This must be valid because we don't execute this function unless + frameIsNewSsrc has returned false.) + */ + return createInEncodingProjection( + frame, + lastAv1FrameProjection.av1Frame!!, + initialPacket, + mark, + newDt, + receivedTime + ) + } + + /** + * Create a frame projection for the normal case, i.e. as part of the same encoding as the + * previously-projected frame, based on a specific chosen previously-projected frame. + */ + private fun createInEncodingProjection( + frame: Av1DDFrame, + refFrame: Av1DDFrame, + initialPacket: Av1DDPacket, + mark: Boolean, + newDt: Int?, + receivedTime: Instant? + ): Av1DDFrameProjection { + val tsGap = RtpUtils.getTimestampDiff(frame.timestamp, refFrame.timestamp) + val frameNumGap = RtpUtils.getSequenceNumberDelta(frame.frameNumber, refFrame.frameNumber) + var seqGap = 0 + + var f1 = refFrame + var f2: Av1DDFrame? + val refSeq: Int + if (frameNumGap > 0) { + /* refFrame is earlier than frame in decode order. */ + do { + f2 = nextFrame(f1) + checkNotNull(f2) { + "No next frame found after frame with frame number ${f1.frameNumber}, " + + "even though refFrame ${refFrame.frameNumber} is before " + + "frame ${frame.frameNumber}!" + } + seqGap += seqGap(f1, f2) + f1 = f2 + } while (f2 !== frame) + /* refFrame is a projected frame, so it has a projection. */ + refSeq = refFrame.projection!!.latestProjectedSeqNum + } else { + /* refFrame is later than frame in decode order. */ + do { + f2 = prevFrame(f1) + checkNotNull(f2) { + "No previous frame found before frame with frame number ${f1.frameNumber}, " + + "even though refFrame ${refFrame.frameNumber} is after " + + "frame ${frame.frameNumber}!" + } + seqGap += -seqGap(f2, f1) + f1 = f2 + } while (f2 !== frame) + refSeq = refFrame.projection!!.earliestProjectedSeqNum + } + + val projectedSeq = RtpUtils.applySequenceNumberDelta(refSeq, seqGap) + val projectedTs = RtpUtils.applyTimestampDelta(refFrame.projection!!.timestamp, tsGap) + val projectedFrameNumber = RtpUtils.applySequenceNumberDelta(refFrame.projection!!.frameNumber, frameNumGap) + + return Av1DDFrameProjection( + diagnosticContext = diagnosticContext, + av1Frame = frame, + ssrc = lastAv1FrameProjection.ssrc, + timestamp = projectedTs, + sequenceNumberDelta = RtpUtils.getSequenceNumberDelta(projectedSeq, initialPacket.sequenceNumber), + frameNumber = projectedFrameNumber, + templateIdDelta = lastAv1FrameProjection.templateIdDelta, + dti = newDt?.let { frame.structure?.getDtBitmaskForDt(it) }, + mark = mark, + created = receivedTime + ) + } + + override fun needsKeyframe(): Boolean { + if (av1QualityFilter.needsKeyframe) { + return true + } + + return lastAv1FrameProjection.av1Frame == null + } + + override fun rewriteRtp(packetInfo: PacketInfo) { + if (packetInfo.packet !is Av1DDPacket) { + logger.info("Got a non-AV1 DD packet.") + throw RewriteException("Non-AV1 DD packet in AV1 DD source projection") + } + val av1Packet = packetInfo.packetAs() + + val av1Frame = lookupAv1Frame(av1Packet) + ?: throw RewriteException("Frame not in tracker (aged off?)") + + val av1Projection = av1Frame.projection + ?: throw RewriteException("Frame does not have projection?") + /* Shouldn't happen for an accepted packet whose frame is still known? */ + + logger.trace { "Rewriting packet with structure ${System.identityHashCode(av1Packet.descriptor?.structure)}" } + av1Projection.rewriteRtp(av1Packet) + } + + override fun rewriteRtcp(rtcpSrPacket: RtcpSrPacket): Boolean { + val lastAv1FrameProjectionCopy: Av1DDFrameProjection = lastAv1FrameProjection + if (rtcpSrPacket.senderSsrc != lastAv1FrameProjectionCopy.av1Frame?.ssrc) { + return false + } + + rtcpSrPacket.senderSsrc = lastAv1FrameProjectionCopy.ssrc + + val srcTs = rtcpSrPacket.senderInfo.rtpTimestamp + val delta = RtpUtils.getTimestampDiff( + lastAv1FrameProjectionCopy.timestamp, + lastAv1FrameProjectionCopy.av1Frame.timestamp + ) + + val dstTs = RtpUtils.applyTimestampDelta(srcTs, delta) + + if (srcTs != dstTs) { + rtcpSrPacket.senderInfo.rtpTimestamp = dstTs + } + + return true + } + + override fun getRtpState() = RtpState( + lastAv1FrameProjection.ssrc, + lastAv1FrameProjection.latestProjectedSeqNum, + lastAv1FrameProjection.timestamp + ) + + override fun getDebugState(): JSONObject { + val debugState = JSONObject() + debugState["class"] = Av1DDAdaptiveSourceProjectionContext::class.java.simpleName + + val mapSizes = JSONArray() + for ((key, value) in av1FrameMaps.entries) { + val sizeInfo = JSONObject() + sizeInfo["ssrc"] = key + sizeInfo["size"] = value.size() + mapSizes.add(sizeInfo) + } + debugState["av1FrameMaps"] = mapSizes + debugState["av1QualityFilter"] = av1QualityFilter.debugState + + return debugState + } + + companion object { + /** + * The time series logger for this class. + */ + private val timeSeriesLogger = + TimeSeriesLogger.getTimeSeriesLogger(Av1DDAdaptiveSourceProjectionContext::class.java) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrame.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrame.kt new file mode 100644 index 0000000000..fcf82cf027 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrame.kt @@ -0,0 +1,309 @@ +/* + * Copyright @ 2019 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.av1 + +import org.jitsi.nlj.rtp.codec.av1.Av1DDPacket +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.Av1DependencyDescriptorReader +import org.jitsi.rtp.rtp.header_extensions.Av1DependencyException +import org.jitsi.rtp.rtp.header_extensions.Av1TemplateDependencyStructure +import org.jitsi.rtp.rtp.header_extensions.FrameInfo +import org.jitsi.rtp.util.RtpUtils.Companion.applySequenceNumberDelta +import org.jitsi.rtp.util.isNewerThan +import org.jitsi.rtp.util.isOlderThan +import org.jitsi.utils.logging2.Logger + +class Av1DDFrame internal constructor( + /** + * The RTP SSRC of the incoming frame that this instance refers to + * (RFC3550). + */ + val ssrc: Long, + + /** + * The RTP timestamp of the incoming frame that this instance refers to + * (RFC3550). + */ + val timestamp: Long, + + /** + * The earliest RTP sequence number seen of the incoming frame that this instance + * refers to (RFC3550). + */ + earliestKnownSequenceNumber: Int, + + /** + * The latest RTP sequence number seen of the incoming frame that this instance + * refers to (RFC3550). + */ + latestKnownSequenceNumber: Int, + + /** + * A boolean that indicates whether we've seen the first packet of the frame. + * If so, its sequence is earliestKnownSequenceNumber. + */ + seenStartOfFrame: Boolean, + + /** + * A boolean that indicates whether we've seen the last packet of the frame. + * If so, its sequence is latestKnownSequenceNumber. + */ + seenEndOfFrame: Boolean, + + /** + * A boolean that indicates whether we've seen a packet with the marker bit set. + */ + seenMarker: Boolean, + + /** + * AV1 FrameInfo for the frame + */ + frameInfo: FrameInfo?, + + /** + * The AV1 DD Frame Number of this frame. + */ + val frameNumber: Int, + + /** + * The FrameID index (FrameID plus cycles) of this frame. + */ + val index: Int, + + /** + * The template ID of this frame + */ + val templateId: Int, + + /** + * The AV1 Template Dependency Structure in effect for this frame, if known + */ + structure: Av1TemplateDependencyStructure?, + + /** + * A new activeDecodeTargets specified for this frame, if any. + * TODO: is this always specified in all packets of the frame? + */ + val activeDecodeTargets: Int?, + + /** + * A boolean that indicates whether the incoming AV1 frame that this + * instance refers to is a keyframe. + */ + var isKeyframe: Boolean, + + /** + * The raw dependency descriptor included in the packet. Stored if it could not be parsed initially. + */ + val rawDependencyDescriptor: RtpPacket.HeaderExtension? +) { + /** + * AV1 FrameInfo for the frame + */ + var frameInfo = frameInfo + private set + + /** + * The AV1 Template Dependency Structure in effect for this frame, if known + */ + var structure = structure + private set + + /** + * The earliest RTP sequence number seen of the incoming frame that this instance + * refers to (RFC3550). + */ + var earliestKnownSequenceNumber = earliestKnownSequenceNumber + private set + + /** + * The latest RTP sequence number seen of the incoming frame that this instance + * refers to (RFC3550). + */ + var latestKnownSequenceNumber: Int = latestKnownSequenceNumber + private set + + /** + * A boolean that indicates whether we've seen the first packet of the frame. + * If so, its sequence is earliestKnownSequenceNumber. + */ + var seenStartOfFrame: Boolean = seenStartOfFrame + private set + + /** + * A boolean that indicates whether we've seen the last packet of the frame. + * If so, its sequence is latestKnownSequenceNumber. + */ + var seenEndOfFrame: Boolean = seenEndOfFrame + private set + + /** + * A boolean that indicates whether we've seen a packet with the marker bit set. + */ + var seenMarker: Boolean = seenMarker + private set + + /** + * A record of how this frame was projected, or null if not. + */ + var projection: Av1DDFrameProjection? = null + + /** + * A boolean that records whether this frame was accepted, i.e. should be forwarded to the receiver + * given the decoding target currently being forwarded. + */ + var isAccepted = false + + // Validate that the index matches the pictureId + init { + assert((index and 0xffff) == frameNumber) + } + + constructor(packet: Av1DDPacket, index: Int) : this( + ssrc = packet.ssrc, + timestamp = packet.timestamp, + earliestKnownSequenceNumber = packet.sequenceNumber, + latestKnownSequenceNumber = packet.sequenceNumber, + seenStartOfFrame = packet.isStartOfFrame, + seenEndOfFrame = packet.isEndOfFrame, + seenMarker = packet.isMarked, + frameInfo = packet.frameInfo, + frameNumber = packet.statelessDescriptor.frameNumber, + index = index, + templateId = packet.statelessDescriptor.frameDependencyTemplateId, + structure = packet.descriptor?.structure, + activeDecodeTargets = packet.activeDecodeTargets, + isKeyframe = packet.isKeyframe, + rawDependencyDescriptor = if (packet.frameInfo == null) { + packet.getHeaderExtension(packet.av1DDHeaderExtensionId)?.clone() + } else { + null + } + ) + + /** + * Remember another packet of this frame. + * Note: this assumes every packet is received only once, i.e. a filter + * like [org.jitsi.nlj.transform.node.incoming.PaddingTermination] is in use. + * @param packet The packet to remember. This should be a packet which + * has tested true with [matchesFrame]. + */ + fun addPacket(packet: Av1DDPacket) { + require(matchesFrame(packet)) { "Non-matching packet added to frame" } + val seq = packet.sequenceNumber + if (seq isOlderThan earliestKnownSequenceNumber) { + earliestKnownSequenceNumber = seq + } + if (seq isNewerThan latestKnownSequenceNumber) { + latestKnownSequenceNumber = seq + } + if (packet.isStartOfFrame) { + seenStartOfFrame = true + } + if (packet.isEndOfFrame) { + seenEndOfFrame = true + } + if (packet.isMarked) { + seenMarker = true + } + + if (structure == null && packet.descriptor?.structure != null) { + structure = packet.descriptor?.structure + } + + if (frameInfo == null && packet.frameInfo != null) { + frameInfo = packet.frameInfo + } + } + + fun updateParse(templateDependencyStructure: Av1TemplateDependencyStructure, logger: Logger) { + if (rawDependencyDescriptor == null) { + return + } + val parser = Av1DependencyDescriptorReader(rawDependencyDescriptor) + val descriptor = try { + parser.parse(templateDependencyStructure) + } catch (e: Av1DependencyException) { + logger.warn("Could not parse updated AV1 Dependency Descriptor: ${e.message}") + return + } + structure = descriptor.structure + frameInfo = try { + descriptor.frameInfo + } catch (e: Av1DependencyException) { + logger.warn("Could not extract frame info from updated AV1 Dependency Descriptor: ${e.message}") + null + } + } + + /** + * Small utility method that checks whether the [Av1DDFrame] that is + * specified as a parameter belongs to the same RTP stream as the frame that + * this instance refers to. + * + * @param av1Frame the [Av1DDFrame] to check whether it belongs to the + * same RTP stream as the frame that this instance refers to. + * @return true if the [Av1DDFrame] that is specified as a parameter + * belongs to the same RTP stream as the frame that this instance refers to, + * false otherwise. + */ + fun matchesSSRC(av1Frame: Av1DDFrame): Boolean { + return ssrc == av1Frame.ssrc + } + + /** + * Checks whether the specified RTP packet is part of this frame. + * + * @param pkt the RTP packet to check whether it's part of this frame. + * @return true if the specified RTP packet is part of this frame, false + * otherwise. + */ + fun matchesFrame(pkt: Av1DDPacket): Boolean { + return ssrc == pkt.ssrc && timestamp == pkt.timestamp && + frameNumber == pkt.frameNumber + } + + fun validateConsistency(pkt: Av1DDPacket) { + if (frameInfo == null) { + return + } + if (frameInfo == pkt.frameInfo) { + return + } + + throw RuntimeException( + buildString { + with(pkt) { + append("Packet ssrc $ssrc, seq $sequenceNumber, frame number $frameNumber, timestamp $timestamp ") + append("packet template ${statelessDescriptor.frameDependencyTemplateId} ") + append("frame info $frameInfo ") + } + append("is not consistent with frame ${this@Av1DDFrame}") + } + ) + } + fun isImmediatelyAfter(otherFrame: Av1DDFrame): Boolean { + return frameNumber == + applySequenceNumberDelta(otherFrame.frameNumber, 1) + } + + override fun toString() = buildString { + append("$ssrc, ") + append("seq $earliestKnownSequenceNumber-$latestKnownSequenceNumber ") + append("frame number $frameNumber, timestamp $timestamp: ") + append("frame template $templateId info $frameInfo") + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameMap.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameMap.kt new file mode 100644 index 0000000000..e3722e1e2f --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameMap.kt @@ -0,0 +1,254 @@ +package org.jitsi.videobridge.cc.av1 + +import org.jitsi.nlj.rtp.codec.av1.Av1DDPacket +import org.jitsi.nlj.util.ArrayCache +import org.jitsi.nlj.util.Rfc3711IndexTracker +import org.jitsi.rtp.util.RtpUtils +import org.jitsi.rtp.util.isNewerThan +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger + +/** + * A history of recent frames on a Av1 stream. + */ +class Av1DDFrameMap( + parentLogger: Logger +) { + /** Cache mapping frame IDs to frames. */ + private val frameHistory = FrameHistory(FRAME_MAP_SIZE) + private val logger: Logger = createChildLogger(parentLogger) + + /** Find a frame in the frame map, based on a packet. */ + @Synchronized + fun findFrame(packet: Av1DDPacket): Av1DDFrame? { + return frameHistory[packet.frameNumber] + } + + /** Get the current size of the map. */ + fun size(): Int { + return frameHistory.numCached + } + + /** Check whether this is a large jump from previous state, so the map should be reset. */ + private fun isLargeJump(packet: Av1DDPacket): Boolean { + val latestFrame: Av1DDFrame = frameHistory.latestFrame ?: return false + val picDelta = RtpUtils.getSequenceNumberDelta(packet.frameNumber, latestFrame.frameNumber) + if (picDelta > FRAME_MAP_SIZE) { + return true + } + val tsDelta: Long = RtpUtils.getTimestampDiff(packet.timestamp, latestFrame.timestamp) + if (picDelta < 0) { + /* if picDelta is negative but timestamp or sequence delta is positive, we've cycled. */ + if (tsDelta > 0) { + return true + } + if (packet.sequenceNumber isNewerThan latestFrame.latestKnownSequenceNumber) { + return true + } + } + + /* If tsDelta is more than twice the frame map size at 1 fps, we've cycled. */ + return tsDelta > FRAME_MAP_SIZE * 90000 * 2 + } + + /** Insert a packet into the frame map. Return a frameInsertionResult + * describing what happened. + * @param packet The packet to insert. + * @return What happened. null if insertion failed. + */ + @Synchronized + fun insertPacket(packet: Av1DDPacket): PacketInsertionResult? { + val frameNumber = packet.frameNumber + + if (isLargeJump(packet)) { + frameHistory.indexTracker.resetAt(frameNumber) + val frame = frameHistory.insert(packet) ?: return null + + return PacketInsertionResult(frame, true, isReset = true) + } + val frame = frameHistory[frameNumber] + if (frame != null) { + if (!frame.matchesFrame(packet)) { + check(frame.frameNumber == frameNumber) { + "frame map returned frame with frame number ${frame.frameNumber} " + "when asked for frame with frame ID $frameNumber" + } + logger.warn( + "Cannot insert packet in frame map: " + + with(frame) { + "frame with ssrc $ssrc, timestamp $timestamp, " + + "and sequence number range $earliestKnownSequenceNumber-$latestKnownSequenceNumber, " + } + + with(packet) { + "and packet $sequenceNumber with ssrc $ssrc, timestamp $timestamp, " + + "and sequence number $sequenceNumber" + } + + " both have frame ID $frameNumber" + ) + return null + } + try { + frame.validateConsistency(packet) + } catch (e: Exception) { + logger.warn(e) + } + + frame.addPacket(packet) + return PacketInsertionResult(frame, isNewFrame = false) + } + + val newframe = frameHistory.insert(packet) ?: return null + + return PacketInsertionResult(newframe, true) + } + + /** Insert a frame. Only used for unit testing. */ + @Synchronized + internal fun insertFrame(frame: Av1DDFrame) { + frameHistory.insert(frame) + } + + @Synchronized + fun getIndex(frameIndex: Int) = frameHistory.getIndex(frameIndex) + + @Synchronized + fun nextFrame(frame: Av1DDFrame): Av1DDFrame? { + return frameHistory.findAfter(frame) { true } + } + + @Synchronized + fun nextFrameWith(frame: Av1DDFrame, pred: (Av1DDFrame) -> Boolean): Av1DDFrame? { + return frameHistory.findAfter(frame, pred) + } + + @Synchronized + fun prevFrame(frame: Av1DDFrame): Av1DDFrame? { + return frameHistory.findBefore(frame) { true } + } + + @Synchronized + fun prevFrameWith(frame: Av1DDFrame, pred: (Av1DDFrame) -> Boolean): Av1DDFrame? { + return frameHistory.findBefore(frame, pred) + } + + fun findPrevAcceptedFrame(frame: Av1DDFrame): Av1DDFrame? { + return prevFrameWith(frame) { it.isAccepted } + } + + fun findNextAcceptedFrame(frame: Av1DDFrame): Av1DDFrame? { + return nextFrameWith(frame) { it.isAccepted } + } + + companion object { + const val FRAME_MAP_SIZE = 500 // Matches PacketCache default size. + } +} + +internal class FrameHistory(size: Int) : + ArrayCache( + size, + cloneItem = { k -> k }, + synchronize = false + ) { + var numCached = 0 + var firstIndex = -1 + var indexTracker = Rfc3711IndexTracker() + + /** + * Gets a frame with a given frame number from the cache. + */ + operator fun get(frameNumber: Int): Av1DDFrame? { + val index = indexTracker.interpret(frameNumber) + return getIndex(index) + } + + /** + * Gets a frame with a given frame number index from the cache. + */ + fun getIndex(index: Int): Av1DDFrame? { + if (index <= lastIndex - size) { + /* We don't want to remember old frames even if they're still + tracked; their neighboring frames may have been evicted, + so findBefore / findAfter will return bogus data. */ + return null + } + val c = getContainer(index) ?: return null + return c.item + } + + /** Get the latest frame in the tracker. */ + val latestFrame: Av1DDFrame? + get() = getIndex(lastIndex) + + fun insert(frame: Av1DDFrame): Boolean { + val ret = super.insertItem(frame, frame.index) + if (ret) { + numCached++ + if (firstIndex == -1 || frame.index < firstIndex) { + firstIndex = frame.index + } + } + return ret + } + + fun insert(packet: Av1DDPacket): Av1DDFrame? { + val index = indexTracker.update(packet.frameNumber) + val frame = Av1DDFrame(packet, index) + return if (insert(frame)) frame else null + } + + /** + * Called when an item in the cache is replaced/discarded. + */ + override fun discardItem(item: Av1DDFrame) { + numCached-- + } + + fun findBefore(frame: Av1DDFrame, pred: (Av1DDFrame) -> Boolean): Av1DDFrame? { + val lastIndex = lastIndex + if (lastIndex == -1) { + return null + } + val index = frame.index + val searchStartIndex = Integer.min(index - 1, lastIndex) + val searchEndIndex = Integer.max(lastIndex - size, firstIndex - 1) + return doFind(pred, searchStartIndex, searchEndIndex, -1) + } + + fun findAfter(frame: Av1DDFrame, pred: (Av1DDFrame) -> Boolean): Av1DDFrame? { + val lastIndex = lastIndex + if (lastIndex == -1) { + return null + } + val index = frame.index + if (index >= lastIndex) { + return null + } + val searchStartIndex = Integer.max(index + 1, Integer.max(lastIndex - size + 1, firstIndex)) + return doFind(pred, searchStartIndex, lastIndex + 1, 1) + } + + private fun doFind(pred: (Av1DDFrame) -> Boolean, startIndex: Int, endIndex: Int, increment: Int): Av1DDFrame? { + var index = startIndex + while (index != endIndex) { + val frame = getIndex(index) + if (frame != null && pred(frame)) { + return frame + } + index += increment + } + return null + } +} + +/** + * The result of calling [insertPacket] + */ +class PacketInsertionResult( + /** The frame corresponding to the packet that was inserted. */ + val frame: Av1DDFrame, + /** Whether inserting the packet created a new frame. */ + val isNewFrame: Boolean, + /** Whether inserting the packet caused a reset */ + val isReset: Boolean = false +) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameProjection.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameProjection.kt new file mode 100644 index 0000000000..88e34cb2b7 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameProjection.kt @@ -0,0 +1,240 @@ +/* + * Copyright @ 2019 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.av1 + +import org.jitsi.nlj.rtp.codec.av1.Av1DDPacket +import org.jitsi.nlj.rtp.codec.av1.applyTemplateIdDelta +import org.jitsi.rtp.rtp.header_extensions.toShortString +import org.jitsi.rtp.util.RtpUtils.Companion.applySequenceNumberDelta +import org.jitsi.rtp.util.isOlderThan +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import java.time.Instant + +/** + * Represents an AV1 DD frame projection. It puts together all the necessary bits + * and pieces that are useful when projecting an accepted AV1 DD frame. A + * projection is responsible for rewriting a AV1 DD packet. Instances of this class + * are thread-safe. + */ +class Av1DDFrameProjection internal constructor( + /** + * The diagnostic context for this instance. + */ + private val diagnosticContext: DiagnosticContext, + /** + * The projected [Av1DDFrame]. + */ + val av1Frame: Av1DDFrame?, + /** + * The RTP SSRC of the projection (RFC7667, RFC3550). + */ + val ssrc: Long, + /** + * The RTP timestamp of the projection (RFC7667, RFC3550). + */ + val timestamp: Long, + /** + * The sequence number delta for packets of this frame. + */ + private val sequenceNumberDelta: Int, + /** + * The AV1 frame number of the projection. + */ + val frameNumber: Int, + /** + * The template ID delta for this frame. This applies both to the frame's own template ID, and to + * the template ID offset in the dependency structure, if present. + */ + val templateIdDelta: Int, + /** + * The decode target indication to set on this frame, if any. + */ + val dti: Int?, + /** + * Whether to add a marker bit to the last packet of this frame. + * (Note this will not clear already-existing marker bits.) + */ + val mark: Boolean, + /** + * A timestamp of when this instance was created. It's used to calculate + * RTP timestamps when we switch encodings. + */ + val created: Instant? +) { + + /** + * -1 if this projection is still "open" for new, later packets. + * Projections can be closed when we switch away from their encodings. + */ + private var closedSeq = -1 + + /** + * Ctor. + * + * @param ssrc the SSRC of the destination AV1 frame. + * @param timestamp The RTP timestamp of the projected frame that this + * instance refers to (RFC3550). + * @param sequenceNumberDelta The starting RTP sequence number of the + * projected frame that this instance refers to (RFC3550). + */ + internal constructor( + diagnosticContext: DiagnosticContext, + ssrc: Long, + sequenceNumberDelta: Int, + timestamp: Long + ) : this( + diagnosticContext = diagnosticContext, + av1Frame = null, + ssrc = ssrc, + timestamp = timestamp, + sequenceNumberDelta = sequenceNumberDelta, + frameNumber = 0, + templateIdDelta = 0, + dti = null, + mark = false, + created = null + ) + + fun rewriteSeqNo(seq: Int): Int = applySequenceNumberDelta(seq, sequenceNumberDelta) + + fun rewriteTemplateId(id: Int): Int = applyTemplateIdDelta(id, templateIdDelta) + + /** + * Rewrites an RTP packet. + * + * @param pkt the RTP packet to rewrite. + */ + fun rewriteRtp(pkt: Av1DDPacket) { + val sequenceNumber = rewriteSeqNo(pkt.sequenceNumber) + val templateId = rewriteTemplateId(pkt.statelessDescriptor.frameDependencyTemplateId) + if (timeSeriesLogger.isTraceEnabled) { + timeSeriesLogger.trace( + diagnosticContext + .makeTimeSeriesPoint("rtp_av1_rewrite") + .addField("orig.rtp.ssrc", pkt.ssrc) + .addField("orig.rtp.timestamp", pkt.timestamp) + .addField("orig.rtp.seq", pkt.sequenceNumber) + .addField("orig.av1.framenum", pkt.frameNumber) + .addField("orig.av1.dti", pkt.frameInfo?.dti?.toShortString() ?: "-") + .addField("orig.av1.templateid", pkt.statelessDescriptor.frameDependencyTemplateId) + .addField("proj.rtp.ssrc", ssrc) + .addField("proj.rtp.timestamp", timestamp) + .addField("proj.rtp.seq", sequenceNumber) + .addField("proj.av1.framenum", frameNumber) + .addField("proj.av1.dti", dti ?: -1) + .addField("proj.av1.templateid", templateId) + .addField("proj.rtp.mark", mark) + ) + } + + // update ssrc, sequence number, timestamp, frameNumber, and templateID + pkt.ssrc = ssrc + pkt.timestamp = timestamp + pkt.sequenceNumber = sequenceNumber + if (mark && pkt.isEndOfFrame) pkt.isMarked = true + + val descriptor = pkt.descriptor + if (descriptor != null && ( + frameNumber != pkt.frameNumber || templateId != descriptor.frameDependencyTemplateId || + dti != null + ) + ) { + descriptor.frameNumber = frameNumber + descriptor.frameDependencyTemplateId = templateId + val structure = descriptor.structure + check( + descriptor.newTemplateDependencyStructure == null || + descriptor.newTemplateDependencyStructure === descriptor.structure + ) + + structure.templateIdOffset = rewriteTemplateId(structure.templateIdOffset) + if (dti != null && ( + descriptor.newTemplateDependencyStructure == null || + dti != (1 shl structure.decodeTargetCount) - 1 + ) + ) { + descriptor.activeDecodeTargetsBitmask = dti + } + + pkt.descriptor = descriptor + pkt.reencodeDdExt() + } + } + + /** + * Determines whether a packet can be forwarded as part of this + * [Av1DDFrameProjection] instance. The check is based on the sequence + * of the incoming packet and whether or not the [Av1DDFrameProjection] + * has been "closed" or not. + * + * @param rtpPacket the [Av1DDPacket] that will be examined. + * @return true if the packet can be forwarded as part of this + * [Av1DDFrameProjection], false otherwise. + */ + fun accept(rtpPacket: Av1DDPacket): Boolean { + if (av1Frame?.matchesFrame(rtpPacket) != true) { + // The packet does not belong to this AV1 picture. + return false + } + synchronized(av1Frame) { + return if (closedSeq < 0) { + true + } else { + rtpPacket.sequenceNumber isOlderThan closedSeq + } + } + } + + val earliestProjectedSeqNum: Int + get() { + if (av1Frame == null) { + return sequenceNumberDelta + } + synchronized(av1Frame) { return rewriteSeqNo(av1Frame.earliestKnownSequenceNumber) } + } + + val latestProjectedSeqNum: Int + get() { + if (av1Frame == null) { + return sequenceNumberDelta + } + synchronized(av1Frame) { return rewriteSeqNo(av1Frame.latestKnownSequenceNumber) } + } + + /** + * Prevents the max sequence number of this frame to grow any further. + */ + fun close() { + if (av1Frame != null) { + synchronized(av1Frame) { closedSeq = av1Frame.latestKnownSequenceNumber } + } + } + + /** + * Get the next template ID that would come after the template IDs in this projection's structure + */ + fun getNextTemplateId(): Int? { + return av1Frame?.structure?.let { rewriteTemplateId(it.templateIdOffset + it.templateCount) } + } + + companion object { + /** + * The time series logger for this class. + */ + private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(Av1DDFrameProjection::class.java) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt new file mode 100644 index 0000000000..331a80faa9 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt @@ -0,0 +1,460 @@ +/* + * Copyright @ 2019 - present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.av1 + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import org.jitsi.nlj.RtpLayerDesc.Companion.SUSPENDED_ENCODING_ID +import org.jitsi.nlj.RtpLayerDesc.Companion.SUSPENDED_INDEX +import org.jitsi.nlj.RtpLayerDesc.Companion.getEidFromIndex +import org.jitsi.nlj.RtpLayerDesc.Companion.indexString +import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc +import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc.Companion.SUSPENDED_DT +import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc.Companion.getDtFromIndex +import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc.Companion.getIndex +import org.jitsi.nlj.rtp.codec.av1.containsDecodeTarget +import org.jitsi.rtp.rtp.header_extensions.DTI +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import org.json.simple.JSONObject +import java.time.Duration +import java.time.Instant + +/** + * This class is responsible for dropping AV1 simulcast/svc packets based on + * their quality, i.e. packets that correspond to qualities that are above a + * given quality target. Instances of this class are thread-safe. + */ +internal class Av1DDQualityFilter( + val av1FrameMap: Map, + parentLogger: Logger +) { + /** + * The [Logger] to be used by this instance to print debug + * information. + */ + private val logger: Logger = createChildLogger(parentLogger) + + /** + * Holds the arrival time of the most recent keyframe group. + * Reading/writing of this field is synchronized on this instance. + */ + private var mostRecentKeyframeGroupArrivalTime: Instant? = null + + /** + * A boolean flag that indicates whether a keyframe is needed, due to an + * encoding or (in some cases) a decode target switch. + */ + var needsKeyframe = false + private set + + /** + * The encoding ID that this instance tries to achieve. Upon + * receipt of a packet, we check whether encoding in the externalTargetIndex + * (that's specified as an argument to the + * [acceptFrame] method) is set to something different, + * in which case we set [needsKeyframe] equal to true and + * update. + */ + private var internalTargetEncoding = SUSPENDED_ENCODING_ID + + /** + * The decode target that this instance tries to achieve. + */ + private var internalTargetDt = SUSPENDED_DT + + /** + * The layer index that we're currently forwarding. [SUSPENDED_INDEX] + * indicates that we're not forwarding anything. Reading/writing of this + * field is synchronized on this instance. + */ + private var currentIndex = SUSPENDED_INDEX + + /** + * Determines whether to accept or drop an AV1 frame. + * + * Note that, at the time of this writing, there's no practical need for a + * synchronized keyword because there's only one thread accessing this + * method at a time. + * + * @param frame the AV1 frame. + * @param incomingEncoding The encoding ID of the incoming packet + * @param externalTargetIndex the target quality index that the user of this + * instance wants to achieve. + * @param receivedTime the current time (as an Instant) + * @return true to accept the AV1 frame, otherwise false. + */ + @Synchronized + fun acceptFrame( + frame: Av1DDFrame, + incomingEncoding: Int, + externalTargetIndex: Int, + receivedTime: Instant? + ): AcceptResult { + val prevIndex = currentIndex + val accept = doAcceptFrame(frame, incomingEncoding, externalTargetIndex, receivedTime) + val currentDt = getDtFromIndex(currentIndex) + val mark = currentDt != SUSPENDED_DT && + (frame.frameInfo?.spatialId == frame.structure?.decodeTargetInfo?.get(currentDt)?.spatialId) + val isResumption = (prevIndex == SUSPENDED_INDEX && currentIndex != SUSPENDED_INDEX) + if (isResumption) { + check(accept) { + // Every code path that can turn off SUSPENDED_INDEX also accepts + "isResumption=$isResumption but accept=$accept for frame ${frame.frameNumber}, " + + "frameInfo=${frame.frameInfo}" + } + } + val dtChanged = (prevIndex != currentIndex) + if (dtChanged && currentDt != SUSPENDED_DT) { + check(accept) { + // Every code path that changes DT also accepts + "dtChanged=$dtChanged but accept=$accept for frame ${frame.frameNumber}, frameInfo=${frame.frameInfo}" + } + } + val newDt = if (dtChanged || frame.activeDecodeTargets != null) currentDt else null + return AcceptResult(accept = accept, isResumption = isResumption, mark = mark, newDt = newDt) + } + + private fun doAcceptFrame( + frame: Av1DDFrame, + incomingEncoding: Int, + externalTargetIndex: Int, + receivedTime: Instant? + ): Boolean { + val externalTargetEncoding = getEidFromIndex(externalTargetIndex) + val currentEncoding = getEidFromIndex(currentIndex) + + if (externalTargetEncoding != internalTargetEncoding) { + // The externalEncodingIdTarget has changed since accept last + // ran; perhaps we should request a keyframe. + internalTargetEncoding = externalTargetEncoding + if (externalTargetEncoding != SUSPENDED_ENCODING_ID && + externalTargetEncoding != currentEncoding + ) { + needsKeyframe = true + } + } + if (externalTargetEncoding == SUSPENDED_ENCODING_ID) { + // We stop forwarding immediately. We will need a keyframe in order + // to resume. + currentIndex = SUSPENDED_INDEX + return false + } + return if (frame.isKeyframe) { + logger.debug { + "Quality filter got keyframe for stream ${frame.ssrc}" + } + acceptKeyframe(frame, incomingEncoding, externalTargetIndex, receivedTime) + } else if (currentEncoding != SUSPENDED_ENCODING_ID) { + if (isOutOfSwitchingPhase(receivedTime) && isPossibleToSwitch(incomingEncoding)) { + // XXX(george) i've noticed some "rogue" base layer keyframes + // that trigger this. what happens is the client sends a base + // layer key frame, the bridge switches to that layer because + // for all it knows it may be the only keyframe sent by the + // client engine. then the bridge notices that packets from the + // higher quality streams are flowing and execution ends-up + // here. it is a mystery why the engine is "leaking" base layer + // key frames + needsKeyframe = true + } + if (incomingEncoding != currentEncoding) { + // for non-keyframes, we can't route anything but the current encoding + return false + } + + /** Logic to forward a non-keyframe: + * If the frame does not have FrameInfo, reject and set needsKeyframe (we couldn't decode its templates). + * If we're trying to switch DTs in the current encoding, check the template structure to ensure that + * there is at least one template which is SWITCH for the target DT and not NOT_PRESENT for the current DT. + * If there is not, request a keyframe. + * If the current frame is SWITCH for the target DT, and we've forwarded all the frames on which (by + * its fdiffs) it depends, forward it, and change the current DT to the target DT. + * In normal circumstances (when the target index == the current index), or when we're trying to switch + * *up* encodings, forward all frames whose DT for the current DT is not NOT_PRESENT. + * If we're trying to switch *down* encodings, only forward frames which are REQUIRED or SWITCH for the + * current DT. + */ + val frameInfo = frame.frameInfo ?: run { + needsKeyframe = true + return@doAcceptFrame false + } + var currentDt = getDtFromIndex(currentIndex) + val externalTargetDt = if (currentEncoding == externalTargetEncoding) { + getDtFromIndex(externalTargetIndex) + } else { + currentDt + } + + if ( + frame.activeDecodeTargets != null && + !frame.activeDecodeTargets.containsDecodeTarget(externalTargetDt) + ) { + /* This shouldn't happen, because we should have set layeringChanged for this packet. */ + logger.warn { + "External target DT $externalTargetDt not present in current decode targets 0x" + + Integer.toHexString(frame.activeDecodeTargets) + " for frame $frame." + } + return false + } + + if (currentDt != externalTargetDt) { + val frameMap = av1FrameMap[frame.ssrc] + if (frameInfo.dti.getOrNull(externalTargetDt) == null) { + logger.warn { "Target DT $externalTargetDt not present for frame $frame [frameInfo $frameInfo]" } + } + if (frameInfo.dti[externalTargetDt] == DTI.SWITCH && + frameMap != null && + frameInfo.fdiff.all { + frameMap.getIndex(frame.index - it)?.isAccepted == true + } + ) { + logger.debug { "Switching to DT $externalTargetDt from $currentDt" } + currentDt = externalTargetDt + currentIndex = externalTargetIndex + } else { + if (frame.structure?.canSwitchWithoutKeyframe(currentDt, externalTargetDt) != true) { + logger.debug { + "Want to switch to DT $externalTargetDt from $currentDt, requesting keyframe" + } + needsKeyframe = true + } + } + } + + val currentFrameDti = frameInfo.dti[currentDt] + if (currentEncoding > externalTargetEncoding) { + (currentFrameDti == DTI.SWITCH || currentFrameDti == DTI.REQUIRED) + } else { + (currentFrameDti != DTI.NOT_PRESENT) + } + } else { + // In this branch we're not processing a keyframe and the + // currentEncoding is in suspended state, which means we need + // a keyframe to start streaming again. + + // We should have already requested a keyframe, either above or when the + // internal target encoding was first moved off SUSPENDED_ENCODING. + false + } + } + + /** + * Returns a boolean that indicates whether we are in layer switching phase + * or not. + * + * @param receivedTime the time the latest frame was received + * @return false if we're in layer switching phase, true otherwise. + */ + @Synchronized + private fun isOutOfSwitchingPhase(receivedTime: Instant?): Boolean { + if (receivedTime == null) { + return false + } + if (mostRecentKeyframeGroupArrivalTime == null) { + return true + } + val delta = Duration.between(mostRecentKeyframeGroupArrivalTime, receivedTime) + return delta > MIN_KEY_FRAME_WAIT + } + + /** + * @return true if it looks like we can re-scale (see implementation of + * method for specific details). + */ + @Synchronized + private fun isPossibleToSwitch(incomingEncoding: Int): Boolean { + val currentEncoding = getEidFromIndex(currentIndex) + + if (incomingEncoding == SUSPENDED_ENCODING_ID) { + // We failed to resolve the spatial/quality layer of the packet. + return false + } + return when { + incomingEncoding > currentEncoding && currentEncoding < internalTargetEncoding -> + // It looks like upscaling is possible + true + incomingEncoding < currentEncoding && currentEncoding > internalTargetEncoding -> + // It looks like downscaling is possible. + true + else -> + false + } + } + + /** + * Determines whether to accept or drop an AV1 keyframe. This method updates + * the encoding id. + * + * Note that, at the time of this writing, there's no practical need for a + * synchronized keyword because there's only one thread accessing this + * method at a time. + * + * @param receivedTime the time the frame was received + * @return true to accept the AV1 keyframe, otherwise false. + */ + @Synchronized + private fun acceptKeyframe( + frame: Av1DDFrame, + incomingEncoding: Int, + externalTargetIndex: Int, + receivedTime: Instant? + ): Boolean { + // This branch writes the {@link #currentSpatialLayerId} and it + // determines whether or not we should switch to another simulcast + // stream. + if (incomingEncoding < 0) { + // something went terribly wrong, normally we should be able to + // extract the layer id from a keyframe. + logger.error("unable to get layer id from keyframe") + return false + } + val frameInfo = frame.frameInfo ?: run { + // something went terribly wrong, normally we should be able to + // extract the frame info from a keyframe. + logger.error("unable to get frame info from keyframe") + return@acceptKeyframe false + } + logger.debug { + "Received a keyframe of encoding: $incomingEncoding" + } + + val currentEncoding = getEidFromIndex(currentIndex) + val externalTargetEncoding = getEidFromIndex(externalTargetIndex) + + val indexIfSwitched = when { + incomingEncoding == externalTargetEncoding -> externalTargetIndex + incomingEncoding == internalTargetEncoding && internalTargetDt != -1 -> + getIndex(currentEncoding, internalTargetDt) + else -> frameInfo.dtisPresent.maxOrNull()!! + } + val dtIfSwitched = getDtFromIndex(indexIfSwitched) + val dtiIfSwitched = frameInfo.dti[dtIfSwitched] + val acceptIfSwitched = dtiIfSwitched != DTI.NOT_PRESENT + + // The keyframe request has been fulfilled at this point, regardless of + // whether we'll be able to achieve the internalEncodingIdTarget. + needsKeyframe = false + return if (isOutOfSwitchingPhase(receivedTime)) { + // During the switching phase we always project the first + // keyframe because it may very well be the only one that we + // receive (i.e. the endpoint is sending low quality only). Then + // we try to approach the target. + mostRecentKeyframeGroupArrivalTime = receivedTime + logger.debug { + "First keyframe in this kf group " + + "currentEncodingId: $incomingEncoding. " + + "Target is $internalTargetEncoding" + } + if (incomingEncoding <= internalTargetEncoding) { + // If the target is 180p and the first keyframe of a group of + // keyframes is a 720p keyframe we don't project it. If we + // receive a 720p keyframe, we know that there MUST be a 180p + // keyframe shortly after. + if (acceptIfSwitched) { + currentIndex = indexIfSwitched + } + acceptIfSwitched + } else { + false + } + } else { + // We're within the 300ms window since the reception of the + // first key frame of a key frame group, let's check whether an + // upscale/downscale is possible. + when { + currentEncoding <= incomingEncoding && + incomingEncoding <= internalTargetEncoding -> { + // upscale or current quality case + if (acceptIfSwitched) { + currentIndex = indexIfSwitched + logger.debug { + "Upscaling to encoding $incomingEncoding. " + + "The target is $internalTargetEncoding" + } + } + acceptIfSwitched + } + incomingEncoding <= internalTargetEncoding && + internalTargetEncoding < currentEncoding -> { + // downscale case + if (acceptIfSwitched) { + currentIndex = indexIfSwitched + logger.debug { + "Downscaling to encoding $incomingEncoding. " + + "The target is $internalTargetEncoding" + } + } + acceptIfSwitched + } + else -> { + false + } + } + } + } + + /** + * Adds internal state to a diagnostic context time series point. + */ + @SuppressFBWarnings( + value = ["IS2_INCONSISTENT_SYNC"], + justification = "We intentionally avoid synchronizing while reading fields only used in debug output." + ) + internal fun addDiagnosticContext(pt: DiagnosticContext.TimeSeriesPoint) { + pt.addField("qf.currentIndex", Av1DDRtpLayerDesc.indexString(currentIndex)) + .addField("qf.internalTargetEncoding", internalTargetEncoding) + .addField("qf.needsKeyframe", needsKeyframe) + .addField( + "qf.mostRecentKeyframeGroupArrivalTimeMs", + mostRecentKeyframeGroupArrivalTime?.toEpochMilli() ?: -1 + ) + /* TODO any other fields necessary */ + } + + /** + * Gets a JSON representation of the parts of this object's state that + * are deemed useful for debugging. + */ + @get:SuppressFBWarnings( + value = ["IS2_INCONSISTENT_SYNC"], + justification = "We intentionally avoid synchronizing while reading fields only used in debug output." + ) + val debugState: JSONObject + get() { + val debugState = JSONObject() + debugState["mostRecentKeyframeGroupArrivalTimeMs"] = + mostRecentKeyframeGroupArrivalTime?.toEpochMilli() ?: -1 + debugState["needsKeyframe"] = needsKeyframe + debugState["internalTargetEncoding"] = internalTargetEncoding + debugState["currentIndex"] = Av1DDRtpLayerDesc.indexString(currentIndex) + return debugState + } + + data class AcceptResult( + val accept: Boolean, + val isResumption: Boolean, + val mark: Boolean, + val newDt: Int? + ) + + companion object { + /** + * The default maximum frequency at which the media engine + * generates key frame. + */ + private val MIN_KEY_FRAME_WAIT = Duration.ofMillis(300) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt index 1fe7d7b1e8..1417f82c34 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt @@ -21,7 +21,6 @@ import org.jitsi.nlj.codec.vpx.VpxUtils.Companion.applyExtendedPictureIdDelta import org.jitsi.nlj.codec.vpx.VpxUtils.Companion.applyTl0PicIdxDelta import org.jitsi.nlj.codec.vpx.VpxUtils.Companion.getExtendedPictureIdDelta import org.jitsi.nlj.codec.vpx.VpxUtils.Companion.getTl0PicIdxDelta -import org.jitsi.nlj.format.PayloadType import org.jitsi.nlj.rtp.codec.vp9.Vp9Packet import org.jitsi.rtp.rtcp.RtcpSrPacket import org.jitsi.rtp.util.RtpUtils.Companion.applySequenceNumberDelta @@ -33,7 +32,6 @@ import org.jitsi.utils.logging.DiagnosticContext import org.jitsi.utils.logging.TimeSeriesLogger import org.jitsi.utils.logging2.Logger import org.jitsi.utils.logging2.createChildLogger -import org.jitsi.utils.times import org.jitsi.videobridge.cc.AdaptiveSourceProjectionContext import org.jitsi.videobridge.cc.RewriteException import org.jitsi.videobridge.cc.RtpState @@ -49,7 +47,6 @@ import java.time.Instant */ class Vp9AdaptiveSourceProjectionContext( private val diagnosticContext: DiagnosticContext, - private val payloadType: PayloadType, rtpState: RtpState, parentLogger: Logger ) : AdaptiveSourceProjectionContext { @@ -81,7 +78,7 @@ class Vp9AdaptiveSourceProjectionContext( private var lastPicIdIndexResumption = -1 @Synchronized - override fun accept(packetInfo: PacketInfo, incomingIndex: Int, targetIndex: Int): Boolean { + override fun accept(packetInfo: PacketInfo, incomingEncoding: Int, targetIndex: Int): Boolean { val packet = packetInfo.packet if (packet !is Vp9Packet) { logger.warn("Packet is not Vp9 packet") @@ -108,7 +105,7 @@ class Vp9AdaptiveSourceProjectionContext( } val receivedTime = packetInfo.receivedTime val acceptResult = vp9QualityFilter - .acceptFrame(frame, incomingIndex, targetIndex, receivedTime) + .acceptFrame(frame, incomingEncoding, targetIndex, receivedTime) frame.isAccepted = acceptResult.accept && frame.index >= lastPicIdIndexResumption if (frame.isAccepted) { val projection: Vp9FrameProjection @@ -142,7 +139,9 @@ class Vp9AdaptiveSourceProjectionContext( .addField("timestamp", packet.timestamp) .addField("seq", packet.sequenceNumber) .addField("pictureId", packet.pictureId) - .addField("index", indexString(incomingIndex)) + .addField("encoding", incomingEncoding) + .addField("spatialLayer", packet.spatialLayerIndex) + .addField("temporalLayer", packet.temporalLayerIndex) .addField("isInterPicturePredicted", packet.isInterPicturePredicted) .addField("usesInterLayerDependency", packet.usesInterLayerDependency) .addField("isUpperLevelReference", packet.isUpperLevelReference) @@ -584,10 +583,6 @@ class Vp9AdaptiveSourceProjectionContext( lastVp9FrameProjection.timestamp ) - override fun getPayloadType(): PayloadType { - return payloadType - } - @Synchronized override fun getDebugState(): JSONObject { val debugState = JSONObject() @@ -603,8 +598,6 @@ class Vp9AdaptiveSourceProjectionContext( debugState["vp9FrameMaps"] = mapSizes debugState["vp9QualityFilter"] = vp9QualityFilter.debugState - debugState["payloadType"] = payloadType.toString() - return debugState } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9PictureMap.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9PictureMap.kt index 2913590852..b7a65b63af 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9PictureMap.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9PictureMap.kt @@ -165,7 +165,7 @@ constructor(size: Int) : ArrayCache( ) { var numCached = 0 var firstIndex = -1 - var indexTracker = PictureIdIndexTracker() + val indexTracker = PictureIdIndexTracker() /** * Gets a picture with a given VP9 picture ID from the cache. diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilter.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilter.kt index c17e94665b..863b4c8786 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilter.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilter.kt @@ -90,7 +90,7 @@ internal class Vp9QualityFilter(parentLogger: Logger) { * method at a time. * * @param frame the VP9 frame. - * @param incomingIndex the quality index of the incoming RTP packet + * @param incomingEncoding the encoding of the incoming RTP packet * @param externalTargetIndex the target quality index that the user of this * instance wants to achieve. * @param receivedTime the current time (as an Instant) @@ -99,19 +99,19 @@ internal class Vp9QualityFilter(parentLogger: Logger) { @Synchronized fun acceptFrame( frame: Vp9Frame, - incomingIndex: Int, + incomingEncoding: Int, externalTargetIndex: Int, receivedTime: Instant? ): AcceptResult { val prevIndex = currentIndex - val accept = doAcceptFrame(frame, incomingIndex, externalTargetIndex, receivedTime) + val accept = doAcceptFrame(frame, incomingEncoding, externalTargetIndex, receivedTime) val mark = if (frame.isInterPicturePredicted) { - getSidFromIndex(incomingIndex) == getSidFromIndex(currentIndex) + frame.spatialLayer.coerceAtLeast(0) == getSidFromIndex(currentIndex) } else { /* This is wrong if the stream isn't actually currently encoding the target index's spatial layer */ /* However, in that case the final (lower) spatial layer should have the marker bit set by the encoder, so I think this shouldn't be a problem? */ - getSidFromIndex(incomingIndex) == getSidFromIndex(externalTargetIndex) + frame.spatialLayer.coerceAtLeast(0) == getSidFromIndex(externalTargetIndex) } val isResumption = (prevIndex == SUSPENDED_INDEX && currentIndex != SUSPENDED_INDEX) if (isResumption) assert(accept) // Every code path that can turn off SUSPENDED_INDEX also accepts @@ -120,7 +120,7 @@ internal class Vp9QualityFilter(parentLogger: Logger) { private fun doAcceptFrame( frame: Vp9Frame, - incomingIndex: Int, + incomingEncoding: Int, externalTargetIndex: Int, receivedTime: Instant? ): Boolean { @@ -145,12 +145,11 @@ internal class Vp9QualityFilter(parentLogger: Logger) { } // If temporal scalability is not enabled, pretend that this is the base temporal layer. val temporalLayerIdOfFrame = frame.temporalLayer.coerceAtLeast(0) - val incomingEncoding = getEidFromIndex(incomingIndex) return if (frame.isKeyframe) { logger.debug { "Quality filter got keyframe for stream ${frame.ssrc}" } - val accept = acceptKeyframe(incomingIndex, receivedTime) + val accept = acceptKeyframe(frame, incomingEncoding, receivedTime) if (accept) { // Keyframes reset layer forwarding, whether or not they're an encoding switch for (i in layers.indices) { @@ -189,7 +188,7 @@ internal class Vp9QualityFilter(parentLogger: Logger) { * return accept */ - val spatialLayerOfFrame = getSidFromIndex(incomingIndex) + val spatialLayerOfFrame = frame.spatialLayer.coerceAtLeast(0) var externalTargetSpatialId = getSidFromIndex(externalTargetIndex) var currentSpatialLayer = getSidFromIndex(currentIndex) @@ -211,7 +210,7 @@ internal class Vp9QualityFilter(parentLogger: Logger) { if (wantToSwitch) { if (canForwardLayer) { logger.debug { "Switching to spatial layer $externalTargetSpatialId from $currentSpatialLayer" } - currentIndex = incomingIndex + currentIndex = RtpLayerDesc.getIndex(incomingEncoding, frame.spatialLayer, frame.temporalLayer) currentSpatialLayer = spatialLayerOfFrame } else { if (internalTargetSpatialId != externalTargetSpatialId) { @@ -273,11 +272,11 @@ internal class Vp9QualityFilter(parentLogger: Logger) { } else { // In this branch we're not processing a keyframe and the // currentEncoding is in suspended state, which means we need - // a keyframe to start streaming again. Reaching this point also - // means that we want to forward something (because both - // externalEncodingTarget is not suspended) so we set the request keyframe flag. + // a keyframe to start streaming again. + + // We should have already requested a keyframe, either above or when the + // internal target encoding was first moved off SUSPENDED_ENCODING. - // assert needsKeyframe == true; false } } @@ -287,7 +286,7 @@ internal class Vp9QualityFilter(parentLogger: Logger) { * or not. * * @param receivedTime the time the latest frame was received - * @return true if we're in layer switching phase, false otherwise. + * @return false if we're in layer switching phase, true otherwise. */ @Synchronized private fun isOutOfSwitchingPhase(receivedTime: Instant?): Boolean { @@ -337,24 +336,25 @@ internal class Vp9QualityFilter(parentLogger: Logger) { * @return true to accept the VP9 keyframe, otherwise false. */ @Synchronized - private fun acceptKeyframe(incomingIndex: Int, receivedTime: Instant?): Boolean { - val encodingIdOfKeyframe = getEidFromIndex(incomingIndex) + private fun acceptKeyframe(frame: Vp9Frame, incomingEncoding: Int, receivedTime: Instant?): Boolean { // This branch writes the {@link #currentSpatialLayerId} and it // determines whether or not we should switch to another simulcast // stream. - if (encodingIdOfKeyframe < 0) { + if (incomingEncoding < 0) { // something went terribly wrong, normally we should be able to // extract the layer id from a keyframe. - logger.error("unable to get layer id from keyframe") + logger.error("invalid encoding id for keyframe") return false } // Keyframes have to be sid 0, tid 0, unless something screwy is going on. - if (getSidFromIndex(incomingIndex) != 0 || getTidFromIndex(incomingIndex) != 0) { - logger.warn("Surprising index ${indexString(incomingIndex)} on keyframe") + // The layers can also be -1 if the layers aren't known + if (frame.spatialLayer > 0 || frame.temporalLayer > 0) { + logger.warn("Surprising layers S${frame.spatialLayer}T${frame.temporalLayer} on keyframe") } logger.debug { - "Received a keyframe of encoding: $encodingIdOfKeyframe" + "Received a keyframe of encoding: $incomingEncoding" } + val incomingIndex = RtpLayerDesc.getIndex(incomingEncoding, frame.spatialLayer, frame.temporalLayer) // The keyframe request has been fulfilled at this point, regardless of // whether we'll be able to achieve the internalEncodingIdTarget. @@ -367,16 +367,16 @@ internal class Vp9QualityFilter(parentLogger: Logger) { mostRecentKeyframeGroupArrivalTime = receivedTime logger.debug { "First keyframe in this kf group " + - "currentEncodingId: $encodingIdOfKeyframe. " + + "currentEncodingId: $incomingEncoding. " + "Target is $internalTargetEncoding" } - if (encodingIdOfKeyframe <= internalTargetEncoding) { + if (incomingEncoding <= internalTargetEncoding) { val currentEncoding = getEidFromIndex(currentIndex) // If the target is 180p and the first keyframe of a group of // keyframes is a 720p keyframe we don't project it. If we // receive a 720p keyframe, we know that there MUST be a 180p // keyframe shortly after. - if (currentEncoding != encodingIdOfKeyframe) { + if (currentEncoding != incomingEncoding) { currentIndex = incomingIndex } true @@ -389,24 +389,24 @@ internal class Vp9QualityFilter(parentLogger: Logger) { // upscale/downscale is possible. val currentEncoding = getEidFromIndex(currentIndex) when { - currentEncoding <= encodingIdOfKeyframe && - encodingIdOfKeyframe <= internalTargetEncoding -> { + currentEncoding <= incomingEncoding && + incomingEncoding <= internalTargetEncoding -> { // upscale or current quality case - if (currentEncoding != encodingIdOfKeyframe) { + if (currentEncoding != incomingEncoding) { currentIndex = incomingIndex } logger.debug { - "Upscaling to encoding $encodingIdOfKeyframe. " + + "Upscaling to encoding $incomingEncoding. " + "The target is $internalTargetEncoding" } true } - encodingIdOfKeyframe <= internalTargetEncoding && + incomingEncoding <= internalTargetEncoding && internalTargetEncoding < currentEncoding -> { // downscale case currentIndex = incomingIndex logger.debug { - "Downscaling to encoding $encodingIdOfKeyframe. " + + "Downscaling to encoding $incomingEncoding. " + "The target is $internalTargetEncoding" } true diff --git a/jvb/src/test/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionTest.java b/jvb/src/test/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionTest.java index 4257733130..56e9782943 100644 --- a/jvb/src/test/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionTest.java +++ b/jvb/src/test/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionTest.java @@ -39,8 +39,6 @@ public class VP8AdaptiveSourceProjectionTest { private final Logger logger = new LoggerImpl(getClass().getName()); - private final PayloadType payloadType = new Vp8PayloadType((byte)96, - new ConcurrentHashMap<>(), new CopyOnWriteArraySet<>()); @Test public void singlePacketProjectionTest() throws RewriteException @@ -52,8 +50,7 @@ public void singlePacketProjectionTest() throws RewriteException new RtpState(1, 10000, 1000000); VP8AdaptiveSourceProjectionContext context = - new VP8AdaptiveSourceProjectionContext(diagnosticContext, payloadType, - initialState, logger); + new VP8AdaptiveSourceProjectionContext(diagnosticContext, initialState, logger); Vp8PacketGenerator generator = new Vp8PacketGenerator(1); @@ -62,7 +59,7 @@ public void singlePacketProjectionTest() throws RewriteException int targetIndex = RtpLayerDesc.getIndex(0, 0, 0); - assertTrue(context.accept(packetInfo, packet.getTemporalLayerIndex(), targetIndex)); + assertTrue(context.accept(packetInfo, 0, targetIndex)); context.rewriteRtp(packetInfo); @@ -84,7 +81,7 @@ private void runInOrderTest(Vp8PacketGenerator generator, int targetTid) int targetIndex = RtpLayerDesc.getIndex(0, 0, targetTid); VP8AdaptiveSourceProjectionContext context = - new VP8AdaptiveSourceProjectionContext(diagnosticContext, payloadType, + new VP8AdaptiveSourceProjectionContext(diagnosticContext, initialState, logger); int expectedSeq = 10001; @@ -97,7 +94,7 @@ private void runInOrderTest(Vp8PacketGenerator generator, int targetTid) PacketInfo packetInfo = generator.nextPacket(); Vp8Packet packet = packetInfo.packetAs(); - boolean accepted = context.accept(packetInfo, packet.getTemporalLayerIndex(), targetIndex); + boolean accepted = context.accept(packetInfo, 0, targetIndex); if (packet.isStartOfFrame() && packet.getTemporalLayerIndex() == 0) { @@ -176,7 +173,6 @@ private void doRunOutOfOrderTest(Vp8PacketGenerator generator, int targetTid, VP8AdaptiveSourceProjectionContext context = new VP8AdaptiveSourceProjectionContext(diagnosticContext, - payloadType, initialState, logger); int latestSeq = buffer.get(0).packetAs().getSequenceNumber(); @@ -198,7 +194,7 @@ private void doRunOutOfOrderTest(Vp8PacketGenerator generator, int targetTid, { latestSeq = origSeq; } - boolean accepted = context.accept(packetInfo, packet.getTemporalLayerIndex(), targetIndex); + boolean accepted = context.accept(packetInfo, 0, targetIndex); int oldestValidSeq = RtpUtils.applySequenceNumberDelta(latestSeq, -((VP8FrameMap.FRAME_MAP_SIZE - 1) * generator.packetsPerFrame)); @@ -385,8 +381,7 @@ public void slightlyDelayedKeyframeTest() throws RewriteException new RtpState(1, 10000, 1000000); VP8AdaptiveSourceProjectionContext context = - new VP8AdaptiveSourceProjectionContext(diagnosticContext, payloadType, - initialState, logger); + new VP8AdaptiveSourceProjectionContext(diagnosticContext, initialState, logger); PacketInfo firstPacketInfo = generator.nextPacket(); Vp8Packet firstPacket = firstPacketInfo.packetAs(); @@ -398,10 +393,10 @@ public void slightlyDelayedKeyframeTest() throws RewriteException PacketInfo packetInfo = generator.nextPacket(); Vp8Packet packet = packetInfo.packetAs(); - assertFalse(context.accept(packetInfo, RtpLayerDesc.getIndex(0, 0, packet.getTemporalLayerIndex()), targetIndex)); + assertFalse(context.accept(packetInfo, 0, targetIndex)); } - assertTrue(context.accept(firstPacketInfo, RtpLayerDesc.getIndex(0, 0, firstPacket.getTemporalLayerIndex()), targetIndex)); + assertTrue(context.accept(firstPacketInfo, 0, targetIndex)); context.rewriteRtp(firstPacketInfo); for (int i = 0; i < 9996; i++) @@ -409,7 +404,7 @@ public void slightlyDelayedKeyframeTest() throws RewriteException PacketInfo packetInfo = generator.nextPacket(); Vp8Packet packet = packetInfo.packetAs(); - assertTrue(context.accept(packetInfo, RtpLayerDesc.getIndex(0, 0, packet.getTemporalLayerIndex()), targetIndex)); + assertTrue(context.accept(packetInfo, 0, targetIndex)); context.rewriteRtp(packetInfo); } } @@ -426,8 +421,7 @@ public void veryDelayedKeyframeTest() throws RewriteException new RtpState(1, 10000, 1000000); VP8AdaptiveSourceProjectionContext context = - new VP8AdaptiveSourceProjectionContext(diagnosticContext, payloadType, - initialState, logger); + new VP8AdaptiveSourceProjectionContext(diagnosticContext, initialState, logger); PacketInfo firstPacketInfo = generator.nextPacket(); Vp8Packet firstPacket = firstPacketInfo.packetAs(); @@ -439,17 +433,17 @@ public void veryDelayedKeyframeTest() throws RewriteException PacketInfo packetInfo = generator.nextPacket(); Vp8Packet packet = packetInfo.packetAs(); - assertFalse(context.accept(packetInfo, RtpLayerDesc.getIndex(0, 0, packet.getTemporalLayerIndex()), targetIndex)); + assertFalse(context.accept(packetInfo, 0, targetIndex)); } - assertFalse(context.accept(firstPacketInfo, RtpLayerDesc.getIndex(0, 0, firstPacket.getTemporalLayerIndex()), targetIndex)); + assertFalse(context.accept(firstPacketInfo, 0, targetIndex)); for (int i = 0; i < 10; i++) { PacketInfo packetInfo = generator.nextPacket(); Vp8Packet packet = packetInfo.packetAs(); - assertFalse(context.accept(packetInfo, RtpLayerDesc.getIndex(0, 0, packet.getTemporalLayerIndex()), targetIndex)); + assertFalse(context.accept(packetInfo, 0, targetIndex)); } generator.requestKeyframe(); @@ -459,7 +453,7 @@ public void veryDelayedKeyframeTest() throws RewriteException PacketInfo packetInfo = generator.nextPacket(); Vp8Packet packet = packetInfo.packetAs(); - assertTrue(context.accept(packetInfo, RtpLayerDesc.getIndex(0, 0, packet.getTemporalLayerIndex()), targetIndex)); + assertTrue(context.accept(packetInfo, 0, targetIndex)); context.rewriteRtp(packetInfo); } } @@ -476,8 +470,7 @@ public void delayedPartialKeyframeTest() throws RewriteException new RtpState(1, 10000, 1000000); VP8AdaptiveSourceProjectionContext context = - new VP8AdaptiveSourceProjectionContext(diagnosticContext, payloadType, - initialState, logger); + new VP8AdaptiveSourceProjectionContext(diagnosticContext, initialState, logger); PacketInfo firstPacketInfo = generator.nextPacket(); Vp8Packet firstPacket = firstPacketInfo.packetAs(); @@ -489,17 +482,17 @@ public void delayedPartialKeyframeTest() throws RewriteException PacketInfo packetInfo = generator.nextPacket(); Vp8Packet packet = packetInfo.packetAs(); - assertFalse(context.accept(packetInfo, RtpLayerDesc.getIndex(0, 0, packet.getTemporalLayerIndex()), targetIndex)); + assertFalse(context.accept(packetInfo, 0, targetIndex)); } - assertFalse(context.accept(firstPacketInfo, firstPacket.getTemporalLayerIndex(), 2)); + assertFalse(context.accept(firstPacketInfo, 0, 2)); for (int i = 0; i < 30; i++) { PacketInfo packetInfo = generator.nextPacket(); Vp8Packet packet = packetInfo.packetAs(); - assertFalse(context.accept(packetInfo, RtpLayerDesc.getIndex(0, 0, packet.getTemporalLayerIndex()), targetIndex)); + assertFalse(context.accept(packetInfo, 0, targetIndex)); } generator.requestKeyframe(); @@ -509,7 +502,7 @@ public void delayedPartialKeyframeTest() throws RewriteException PacketInfo packetInfo = generator.nextPacket(); Vp8Packet packet = packetInfo.packetAs(); - assertTrue(context.accept(packetInfo, RtpLayerDesc.getIndex(0, 0, packet.getTemporalLayerIndex()), targetIndex)); + assertTrue(context.accept(packetInfo, 0, targetIndex)); context.rewriteRtp(packetInfo); } } @@ -528,8 +521,7 @@ public void twoStreamsNoSwitchingTest() throws RewriteException new RtpState(1, 10000, 1000000); VP8AdaptiveSourceProjectionContext context = - new VP8AdaptiveSourceProjectionContext(diagnosticContext, payloadType, - initialState, logger); + new VP8AdaptiveSourceProjectionContext(diagnosticContext, initialState, logger); int targetIndex = RtpLayerDesc.getIndex(1, 0, 2); @@ -540,12 +532,12 @@ public void twoStreamsNoSwitchingTest() throws RewriteException PacketInfo packetInfo1 = generator1.nextPacket(); Vp8Packet packet1 = packetInfo1.packetAs(); - assertTrue(context.accept(packetInfo1, RtpLayerDesc.getIndex(1, 0, packet1.getTemporalLayerIndex()), targetIndex)); + assertTrue(context.accept(packetInfo1, 1, targetIndex)); PacketInfo packetInfo2 = generator2.nextPacket(); Vp8Packet packet2 = packetInfo2.packetAs(); - assertFalse(context.accept(packetInfo2, RtpLayerDesc.getIndex(0, 0, packet2.getTemporalLayerIndex()), targetIndex)); + assertFalse(context.accept(packetInfo2, 0, targetIndex)); context.rewriteRtp(packetInfo1); @@ -574,8 +566,7 @@ public void twoStreamsSwitchingTest() throws RewriteException new RtpState(1, 10000, 1000000); VP8AdaptiveSourceProjectionContext context = - new VP8AdaptiveSourceProjectionContext(diagnosticContext, payloadType, - initialState, logger); + new VP8AdaptiveSourceProjectionContext(diagnosticContext, initialState, logger); int expectedSeq = 10001; long expectedTs = 1003000; @@ -596,7 +587,7 @@ public void twoStreamsSwitchingTest() throws RewriteException expectedTl0PicIdx = VpxUtils.applyTl0PicIdxDelta(expectedTl0PicIdx, 1); } - assertTrue(context.accept(packetInfo1, RtpLayerDesc.getIndex(0, 0, packet1.getTemporalLayerIndex()), targetIndex)); + assertTrue(context.accept(packetInfo1, 0, targetIndex)); context.rewriteRtp(packetInfo1); @@ -608,7 +599,7 @@ public void twoStreamsSwitchingTest() throws RewriteException PacketInfo packetInfo2 = generator2.nextPacket(); Vp8Packet packet2 = packetInfo2.packetAs(); - assertFalse(context.accept(packetInfo2, RtpLayerDesc.getIndex(1, 0, packet2.getTemporalLayerIndex()), targetIndex)); + assertFalse(context.accept(packetInfo2, 1, targetIndex)); assertFalse(context.rewriteRtcp(srPacket2)); assertEquals(expectedSeq, packet1.getSequenceNumber()); @@ -637,7 +628,7 @@ public void twoStreamsSwitchingTest() throws RewriteException expectedTl0PicIdx = VpxUtils.applyTl0PicIdxDelta(expectedTl0PicIdx, 1); } - assertTrue(context.accept(packetInfo1, RtpLayerDesc.getIndex(0, 0, packet1.getTemporalLayerIndex()), targetIndex)); + assertTrue(context.accept(packetInfo1, 0, targetIndex)); context.rewriteRtp(packetInfo1); @@ -649,7 +640,7 @@ public void twoStreamsSwitchingTest() throws RewriteException PacketInfo packetInfo2 = generator2.nextPacket(); Vp8Packet packet2 = packetInfo2.packetAs(); - assertFalse(context.accept(packetInfo2, RtpLayerDesc.getIndex(1, 0, packet2.getTemporalLayerIndex()), targetIndex)); + assertFalse(context.accept(packetInfo2, 1, targetIndex)); assertFalse(context.rewriteRtcp(srPacket2)); assertEquals(expectedSeq, packet1.getSequenceNumber()); @@ -682,7 +673,7 @@ public void twoStreamsSwitchingTest() throws RewriteException } /* We will cut off the layer 0 keyframe after 1 packet, once we see the layer 1 keyframe. */ - assertEquals(i == 0, context.accept(packetInfo1, RtpLayerDesc.getIndex(0, 0, packet1.getTemporalLayerIndex()), targetIndex)); + assertEquals(i == 0, context.accept(packetInfo1, 0, targetIndex)); assertEquals(i == 0, context.rewriteRtcp(srPacket1)); if (i == 0) @@ -701,7 +692,7 @@ public void twoStreamsSwitchingTest() throws RewriteException expectedTl0PicIdx = VpxUtils.applyTl0PicIdxDelta(expectedTl0PicIdx, 1); } - assertTrue(context.accept(packetInfo2, RtpLayerDesc.getIndex(1, 0, packet2.getTemporalLayerIndex()), targetIndex)); + assertTrue(context.accept(packetInfo2, 1, targetIndex)); context.rewriteRtp(packetInfo2); @@ -745,8 +736,7 @@ public void temporalLayerSwitchingTest() throws RewriteException new RtpState(1, 10000, 1000000); VP8AdaptiveSourceProjectionContext context = - new VP8AdaptiveSourceProjectionContext(diagnosticContext, payloadType, - initialState, logger); + new VP8AdaptiveSourceProjectionContext(diagnosticContext, initialState, logger); int targetTid = 0; int decodableTid = 0; @@ -763,7 +753,7 @@ public void temporalLayerSwitchingTest() throws RewriteException PacketInfo packetInfo = generator.nextPacket(); Vp8Packet packet = packetInfo.packetAs(); - boolean accepted = context.accept(packetInfo, RtpLayerDesc.getIndex(0, 0, packet.getTemporalLayerIndex()), targetIndex); + boolean accepted = context.accept(packetInfo, 0, targetIndex); if (packet.isStartOfFrame() && packet.getTemporalLayerIndex() == 0) { @@ -830,9 +820,7 @@ private void runLargeDropoutTest(Vp8PacketGenerator generator, new RtpState(1, 10000, 1000000); VP8AdaptiveSourceProjectionContext context = - new VP8AdaptiveSourceProjectionContext(diagnosticContext, - payloadType, - initialState, logger); + new VP8AdaptiveSourceProjectionContext(diagnosticContext, initialState, logger); int expectedSeq = 10001; long expectedTs = 1003000; @@ -845,7 +833,7 @@ private void runLargeDropoutTest(Vp8PacketGenerator generator, Vp8Packet packet = packetInfo.packetAs(); boolean accepted = - context.accept(packetInfo, RtpLayerDesc.getIndex(0, 0, packet.getTemporalLayerIndex()), targetIndex); + context.accept(packetInfo, 0, targetIndex); if (packet.isStartOfFrame() && packet.getTemporalLayerIndex() == 0) { @@ -898,7 +886,7 @@ private void runLargeDropoutTest(Vp8PacketGenerator generator, } while (packet.getTemporalLayerIndex() > targetIndex); - assertTrue(context.accept(packetInfo, RtpLayerDesc.getIndex(0, 0, packet.getTemporalLayerIndex()), targetIndex)); + assertTrue(context.accept(packetInfo, 0, targetIndex)); context.rewriteRtp(packetInfo); /* Allow any values after a gap. */ @@ -921,7 +909,7 @@ private void runLargeDropoutTest(Vp8PacketGenerator generator, packet = packetInfo.packetAs(); boolean accepted = context - .accept(packetInfo, packet.getTemporalLayerIndex(), targetIndex); + .accept(packetInfo, 0, targetIndex); if (packet.isStartOfFrame() && packet.getTemporalLayerIndex() == 0) diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt index bd9435c513..e50c0f7197 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt @@ -1532,9 +1532,16 @@ class MockRtpLayerDesc( var bitrate: Bandwidth, sid: Int = -1 ) : RtpLayerDesc(eid, tid, sid, height, frameRate) { + override fun copy(height: Int): RtpLayerDesc { + TODO("Not yet implemented") + } + + override val layerId = getIndex(0, sid, tid) + override val index = getIndex(eid, sid, tid) override fun getBitrate(nowMs: Long): Bandwidth = bitrate override fun hasZeroBitrate(nowMs: Long): Boolean = bitrate == 0.bps + override fun indexString() = indexString(index) } typealias History = MutableList> diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionTest.kt new file mode 100644 index 0000000000..e3bbead0ef --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionTest.kt @@ -0,0 +1,1571 @@ +/* + * Copyright @ 2019 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.av1 + +import jakarta.xml.bind.DatatypeConverter.parseHexBinary +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.nlj.rtp.codec.av1.Av1DDPacket +import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc.Companion.getIndex +import org.jitsi.nlj.util.Rfc3711IndexTracker +import org.jitsi.rtp.rtcp.RtcpSrPacket +import org.jitsi.rtp.rtcp.RtcpSrPacketBuilder +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.Av1DependencyDescriptorHeaderExtension +import org.jitsi.rtp.rtp.header_extensions.Av1DependencyDescriptorReader +import org.jitsi.rtp.rtp.header_extensions.Av1TemplateDependencyStructure +import org.jitsi.rtp.rtp.header_extensions.FrameInfo +import org.jitsi.rtp.util.RtpUtils +import org.jitsi.rtp.util.isNewerThan +import org.jitsi.rtp.util.isOlderThan +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.LoggerImpl +import org.jitsi.videobridge.cc.RtpState +import org.junit.Assert +import org.junit.Test +import java.time.Duration +import java.time.Instant +import java.util.* +import javax.xml.bind.DatatypeConverter + +class Av1DDAdaptiveSourceProjectionTest { + private val logger: Logger = LoggerImpl(javaClass.name) + + @Test + fun singlePacketProjectionTest() { + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "singlePacketProjectionTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Av1DDAdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + val generator = ScalableAv1PacketGenerator(1) + val packetInfo = generator.nextPacket() + val packet = packetInfo.packetAs() + val targetIndex = getIndex(eid = 0, dt = 0) + Assert.assertTrue(context.accept(packetInfo, 0, targetIndex)) + context.rewriteRtp(packetInfo) + Assert.assertEquals(10001, packet.sequenceNumber) + Assert.assertEquals(1003000, packet.timestamp) + Assert.assertEquals(0, packet.frameNumber) + Assert.assertEquals(0, packet.frameInfo?.spatialId) + Assert.assertEquals(0, packet.frameInfo?.temporalId) + } + + private fun runInOrderTest(generator: Av1PacketGenerator, targetIndex: Int, expectAccept: (FrameInfo) -> Boolean) { + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = Thread.currentThread().stackTrace[2].methodName + val initialState = RtpState(1, 10000, 1000000) + val context = Av1DDAdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + var expectedSeq = 10001 + var expectedTs: Long = 1003000 + var expectedFrameNumber = 0 + for (i in 0..99999) { + val packetInfo = generator.nextPacket() + val packet = packetInfo.packetAs() + val frameInfo = packet.frameInfo!! + + val accepted = context.accept(packetInfo, 0, targetIndex) + val endOfFrame = packet.isEndOfFrame + val endOfPicture = packet.isMarked // Save this before rewriteRtp + if (expectAccept(frameInfo)) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + Assert.assertEquals(expectedSeq, packet.sequenceNumber) + Assert.assertEquals(expectedTs, packet.timestamp) + Assert.assertEquals(expectedFrameNumber, packet.frameNumber) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + } else { + Assert.assertFalse(accepted) + } + if (endOfFrame) { + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + } + + @Test + fun simpleNonScalableTest() { + val generator = NonScalableAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 0)) { + true + } + } + + @Test + fun simpleTemporalProjectionTest() { + val generator = TemporallyScaledPacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 2)) { + true + } + } + + @Test + fun filteredTemporalProjectionTest() { + val generator = TemporallyScaledPacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.temporalId == 0 + } + } + + @Test + fun largerFrameTemporalProjectionTest() { + val generator = TemporallyScaledPacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 2)) { + true + } + } + + @Test + fun largerFrameTemporalFilteredTest() { + val generator = TemporallyScaledPacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.temporalId == 0 + } + } + + @Test + fun hugeFrameTest() { + val generator = TemporallyScaledPacketGenerator(200) + runInOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.temporalId == 0 + } + } + + @Test + fun simpleSvcTest() { + val generator = ScalableAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2)) { + true + } + } + + @Test + fun filteredSvcTest() { + val generator = ScalableAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 2)) { + it.spatialId == 0 + } + } + + @Test + fun temporalFilteredSvcTest() { + val generator = ScalableAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2)) { + it.temporalId == 0 + } + } + + @Test + fun spatialAndTemporalFilteredSvcTest() { + val generator = ScalableAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun largerSvcTest() { + val generator = ScalableAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2)) { + true + } + } + + @Test + fun largerFilteredSvcTest() { + val generator = ScalableAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 2)) { + it.spatialId == 0 + } + } + + @Test + fun largerTemporalFilteredSvcTest() { + val generator = ScalableAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2)) { + it.temporalId == 0 + } + } + + @Test + fun largerSpatialAndTemporalFilteredSvcTest() { + val generator = ScalableAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun simpleKSvcTest() { + val generator = KeyScalableAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2)) { + it.spatialId == 2 || !it.hasInterPictureDependency() + } + } + + @Test + fun filteredKSvcTest() { + val generator = KeyScalableAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 2)) { + it.spatialId == 0 + } + } + + @Test + fun temporalFilteredKSvcTest() { + val generator = KeyScalableAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2)) { + it.temporalId == 0 && (it.spatialId == 2 || !it.hasInterPictureDependency()) + } + } + + @Test + fun spatialAndTemporalFilteredKSvcTest() { + val generator = KeyScalableAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun largerKSvcTest() { + val generator = KeyScalableAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2)) { + it.spatialId == 2 || !it.hasInterPictureDependency() + } + } + + @Test + fun largerFilteredKSvcTest() { + val generator = KeyScalableAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 2)) { + it.spatialId == 0 + } + } + + @Test + fun largerTemporalFilteredKSvcTest() { + val generator = KeyScalableAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2)) { + it.temporalId == 0 && (it.spatialId == 2 || !it.hasInterPictureDependency()) + } + } + + @Test + fun largerSpatialAndTemporalFilteredKSvcTest() { + val generator = KeyScalableAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun simpleSingleEncodingSimulcastTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2)) { + it.spatialId == 2 + } + } + + @Test + fun filteredSingleEncodingSimulcastTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 2)) { + it.spatialId == 0 + } + } + + @Test + fun temporalFilteredSingleEncodingSimulcastTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2)) { + it.temporalId == 0 && it.spatialId == 2 + } + } + + @Test + fun spatialAndTemporalFilteredSingleEncodingSimulcastTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun largerSingleEncodingSimulcastTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2)) { + it.spatialId == 2 + } + } + + @Test + fun largerFilteredSingleEncodingSimulcastTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 2)) { + it.spatialId == 0 + } + } + + @Test + fun largerTemporalFilteredSingleEncodingSimulcastTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2)) { + it.temporalId == 0 && it.spatialId == 2 + } + } + + @Test + fun largerSpatialAndTemporalFilteredSingleEncodingSimulcastTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + private class ProjectedPacket constructor( + val packet: Av1DDPacket, + val origSeq: Int, + val extOrigSeq: Int, + val extFrameNum: Int, + ) + + /** Run an out-of-order test on a single stream, randomized order except for the first + * [initialOrderedCount] packets. */ + private fun doRunOutOfOrderTest( + generator: Av1PacketGenerator, + targetIndex: Int, + initialOrderedCount: Int, + seed: Long, + expectAccept: (FrameInfo) -> Boolean + ) { + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = Thread.currentThread().stackTrace[2].methodName + val initialState = RtpState(1, 10000, 1000000) + val expectedInitialTs: Long = RtpUtils.applyTimestampDelta(initialState.maxTimestamp, 3000) + val expectedTsOffset: Long = RtpUtils.getTimestampDiff(expectedInitialTs, generator.ts) + val reorderSize = 64 + val buffer = ArrayList(reorderSize) + for (i in 0 until reorderSize) { + buffer.add(generator.nextPacket()) + } + val random = Random(seed) + var orderedCount = initialOrderedCount - 1 + val context = Av1DDAdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + var latestSeq = buffer[0].packetAs().sequenceNumber + val projectedPackets = TreeMap() + val origSeqIdxTracker = Rfc3711IndexTracker() + val newSeqIdxTracker = Rfc3711IndexTracker() + val frameNumsDropped = HashSet() + val frameNumsIndexTracker = Rfc3711IndexTracker() + for (i in 0..99999) { + val packetInfo = buffer[0] + val packet = packetInfo.packetAs() + val origSeq = packet.sequenceNumber + val origTs = packet.timestamp + if (latestSeq isOlderThan origSeq) { + latestSeq = origSeq + } + val frameInfo = packet.frameInfo!! + + val accepted = context.accept(packetInfo, 0, targetIndex) + val oldestValidSeq: Int = + RtpUtils.applySequenceNumberDelta( + latestSeq, + -((Av1DDFrameMap.FRAME_MAP_SIZE - 1) * generator.packetsPerFrame) + ) + if (origSeq isOlderThan oldestValidSeq && !accepted) { + /* This is fine; packets that are too old get ignored. */ + /* Note we don't want assertFalse(accepted) here because slightly-too-old packets + * that are part of an existing accepted frame will be accepted. + */ + val extFrameNum = frameNumsIndexTracker.update(packet.frameNumber) + frameNumsDropped.add(extFrameNum) + } else if (expectAccept(frameInfo) + ) { + Assert.assertTrue(accepted) + + context.rewriteRtp(packetInfo) + Assert.assertEquals(RtpUtils.applyTimestampDelta(origTs, expectedTsOffset), packet.timestamp) + val newSeq = packet.sequenceNumber + val extNewSeq = newSeqIdxTracker.update(newSeq) + val extOrigSeq = origSeqIdxTracker.update(origSeq) + Assert.assertFalse(projectedPackets.containsKey(extNewSeq)) + val extFrameNum = frameNumsIndexTracker.update(packet.frameNumber) + projectedPackets[extNewSeq] = ProjectedPacket(packet, origSeq, extOrigSeq, extFrameNum) + } else { + Assert.assertFalse(accepted) + } + if (orderedCount > 0) { + buffer.removeAt(0) + buffer.add(generator.nextPacket()) + orderedCount-- + } else { + buffer[0] = generator.nextPacket() + buffer.shuffle(random) + } + } + val frameNumsSeen = HashSet() + + /* Add packets that weren't added yet, or that were dropped for being too old, to frameNumsSeen. */ + frameNumsSeen.addAll(frameNumsDropped) + buffer.forEach { + frameNumsSeen.add(frameNumsIndexTracker.update(it.packetAs().frameNumber)) + } + + val iter = projectedPackets.keys.iterator() + var prevPacket = projectedPackets[iter.next()]!! + frameNumsSeen.add(prevPacket.extFrameNum) + while (iter.hasNext()) { + val packet = projectedPackets[iter.next()] + Assert.assertTrue(packet!!.origSeq isNewerThan prevPacket.origSeq) + frameNumsSeen.add(packet.extFrameNum) + Assert.assertTrue( + RtpUtils.getSequenceNumberDelta( + prevPacket.packet.frameNumber, + packet.packet.frameNumber + ) <= 0 + ) + if (packet.packet.isStartOfFrame) { + Assert.assertTrue( + RtpUtils.getSequenceNumberDelta( + prevPacket.packet.frameNumber, + packet.packet.frameNumber + ) < 0 + ) + if (prevPacket.packet.sequenceNumber == + RtpUtils.applySequenceNumberDelta(packet.packet.sequenceNumber, -1) + ) { + Assert.assertTrue(prevPacket.packet.isEndOfFrame) + } + } else { + if (prevPacket.packet.sequenceNumber == + RtpUtils.applySequenceNumberDelta(packet.packet.sequenceNumber, -1) + ) { + Assert.assertEquals(prevPacket.packet.frameNumber, packet.packet.frameNumber) + Assert.assertEquals(prevPacket.packet.timestamp, packet.packet.timestamp) + } + } + packet.packet.frameInfo?.fdiff?.forEach { + Assert.assertTrue(frameNumsSeen.contains(packet.extFrameNum - it)) + } + prevPacket = packet + } + + /* Overall, we should not have expanded sequence numbers. */ + val firstPacket = projectedPackets.firstEntry().value + val lastPacket = projectedPackets.lastEntry().value + val origDelta = lastPacket!!.extOrigSeq - firstPacket!!.extOrigSeq + val projDelta = projectedPackets.lastKey() - projectedPackets.firstKey() + Assert.assertTrue(projDelta <= origDelta) + } + + /** Run multiple instances of out-of-order test on a single stream, with different + * random seeds. */ + private fun runOutOfOrderTest( + generator: Av1PacketGenerator, + targetIndex: Int, + initialOrderedCount: Int = 1, + expectAccept: (FrameInfo) -> Boolean + ) { + /* Seeds that have triggered problems in the past for this or VP8/VP9, plus a random one. */ + val seeds = longArrayOf( + 1576267371838L, + 1578347926155L, + 1579620018479L, + 5786714086792432950L, + 5929140296748347521L, + -8226056792707023108L, + System.currentTimeMillis() + ) + for (seed in seeds) { + try { + doRunOutOfOrderTest(generator, targetIndex, initialOrderedCount, seed, expectAccept) + } catch (e: Throwable) { + logger.error( + "Exception thrown in randomized test, seed = $seed", + e + ) + throw e + } + generator.reset() + } + } + + @Test + fun simpleOutOfOrderNonScalableTest() { + val generator = NonScalableAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 0)) { + true + } + } + + @Test + fun simpleOutOfOrderTemporalProjectionTest() { + val generator = TemporallyScaledPacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 2)) { + true + } + } + + @Test + fun filteredOutOfOrderTemporalProjectionTest() { + val generator = TemporallyScaledPacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.temporalId == 0 + } + } + + @Test + fun largerFrameOutOfOrderTemporalProjectionTest() { + val generator = TemporallyScaledPacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 2)) { + true + } + } + + @Test + fun largerFrameOutOfOrderTemporalFilteredTest() { + val generator = TemporallyScaledPacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.temporalId == 0 + } + } + + @Test + fun hugeFrameOutOfOrderTest() { + val generator = TemporallyScaledPacketGenerator(200) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.temporalId == 0 + } + } + + @Test + fun simpleSvcOutOfOrderTest() { + val generator = ScalableAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2)) { + true + } + } + + @Test + fun filteredSvcOutOfOrderTest() { + val generator = ScalableAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 2)) { + it.spatialId == 0 + } + } + + @Test + fun temporalFilteredOutOfOrderSvcOutOfOrderTest() { + val generator = ScalableAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2)) { + it.temporalId == 0 + } + } + + @Test + fun spatialAndTemporalFilteredSvcOutOfOrderTest() { + val generator = ScalableAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun largerSvcOutOfOrderTest() { + val generator = ScalableAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2)) { + true + } + } + + @Test + fun largerFilteredSvcOutOfOrderTest() { + val generator = ScalableAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 2)) { + it.spatialId == 0 + } + } + + @Test + fun largerTemporalFilteredSvcOutOfOrderTest() { + val generator = ScalableAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2)) { + it.temporalId == 0 + } + } + + @Test + fun largerSpatialAndTemporalFilteredSvcOutOfOrderTest() { + val generator = ScalableAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun simpleKSvcOutOfOrderTest() { + val generator = KeyScalableAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2)) { + it.spatialId == 2 || !it.hasInterPictureDependency() + } + } + + @Test + fun filteredKSvcOutOfOrderTest() { + val generator = KeyScalableAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 2)) { + it.spatialId == 0 + } + } + + @Test + fun temporalFilteredKSvcOutOfOrderTest() { + val generator = KeyScalableAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2)) { + it.temporalId == 0 && (it.spatialId == 2 || !it.hasInterPictureDependency()) + } + } + + @Test + fun spatialAndTemporalFilteredKSvcOutOfOrderTest() { + val generator = KeyScalableAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun largerKSvcOutOfOrderTest() { + val generator = KeyScalableAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2)) { + it.spatialId == 2 || !it.hasInterPictureDependency() + } + } + + @Test + fun largerFilteredKSvcOutOfOrderTest() { + val generator = KeyScalableAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 2)) { + it.spatialId == 0 + } + } + + @Test + fun largerTemporalFilteredKSvcOutOfOrderTest() { + val generator = KeyScalableAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2)) { + it.temporalId == 0 && (it.spatialId == 2 || !it.hasInterPictureDependency()) + } + } + + @Test + fun largerSpatialAndTemporalFilteredKSvcOutOfOrderTest() { + val generator = KeyScalableAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun simpleSingleEncodingSimulcastOutOfOrderTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2), 3) { + it.spatialId == 2 + } + } + + @Test + fun filteredSingleEncodingSimulcastOutOfOrderTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 2), 3) { + it.spatialId == 0 + } + } + + @Test + fun temporalFilteredSingleEncodingSimulcastOutOfOrderTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2), 3) { + it.temporalId == 0 && it.spatialId == 2 + } + } + + @Test + fun spatialAndTemporalFilteredSingleEncodingSimulcastOutOfOrderTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 0), 3) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun largerSingleEncodingSimulcastOutOfOrderTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2), 7) { + it.spatialId == 2 + } + } + + @Test + fun largerFilteredSingleEncodingSimulcastOutOfOrderTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 2), 7) { + it.spatialId == 0 + } + } + + @Test + fun largerTemporalFilteredSingleEncodingSimulcastOutOfOrderTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2), 7) { + it.temporalId == 0 && it.spatialId == 2 + } + } + + @Test + fun largerSpatialAndTemporalFilteredSingleEncodingSimulcastOutOfOrderTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 0), 7) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun slightlyDelayedKeyframeTest() { + val generator = TemporallyScaledPacketGenerator(1) + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "slightlyDelayedKeyframeTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Av1DDAdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + val firstPacketInfo = generator.nextPacket() + val targetIndex = getIndex(eid = 0, dt = 2) + for (i in 0..2) { + val packetInfo = generator.nextPacket() + + Assert.assertFalse(context.accept(packetInfo, 0, targetIndex)) + } + Assert.assertTrue(context.accept(firstPacketInfo, 0, targetIndex)) + context.rewriteRtp(firstPacketInfo) + for (i in 0..9995) { + val packetInfo = generator.nextPacket() + Assert.assertTrue(context.accept(packetInfo, 0, targetIndex)) + context.rewriteRtp(packetInfo) + } + } + + @Test + fun veryDelayedKeyframeTest() { + val generator = TemporallyScaledPacketGenerator(1) + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "veryDelayedKeyframeTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Av1DDAdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + val firstPacketInfo = generator.nextPacket() + val targetIndex = getIndex(eid = 0, dt = 2) + for (i in 0..3) { + val packetInfo = generator.nextPacket(missedStructure = true) + Assert.assertFalse(context.accept(packetInfo, 0, targetIndex)) + } + Assert.assertFalse(context.accept(firstPacketInfo, 0, targetIndex)) + for (i in 0..9) { + val packetInfo = generator.nextPacket() + Assert.assertFalse(context.accept(packetInfo, 0, targetIndex)) + } + generator.requestKeyframe() + for (i in 0..9995) { + val packetInfo = generator.nextPacket() + Assert.assertTrue(context.accept(packetInfo, 0, targetIndex)) + context.rewriteRtp(packetInfo) + } + } + + @Test + fun twoStreamsNoSwitchingTest() { + val generator1 = TemporallyScaledPacketGenerator(3) + val generator2 = TemporallyScaledPacketGenerator(3) + generator2.ssrc = 0xdeadbeefL + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "twoStreamsNoSwitchingTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Av1DDAdaptiveSourceProjectionContext(diagnosticContext, initialState, logger) + val targetIndex = getIndex(eid = 1, dt = 2) + var expectedSeq = 10001 + var expectedTs: Long = 1003000 + for (i in 0..9999) { + val packetInfo1 = generator1.nextPacket() + val packet1 = packetInfo1.packetAs() + + Assert.assertTrue(context.accept(packetInfo1, 1, targetIndex)) + val packetInfo2 = generator2.nextPacket() + Assert.assertFalse(context.accept(packetInfo2, 0, targetIndex)) + context.rewriteRtp(packetInfo1) + Assert.assertEquals(expectedSeq, packet1.sequenceNumber) + Assert.assertEquals(expectedTs, packet1.timestamp) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + if (packet1.isEndOfFrame) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + } + + @Test + fun twoStreamsSwitchingTest() { + val generator1 = TemporallyScaledPacketGenerator(3) + val generator2 = TemporallyScaledPacketGenerator(3) + generator2.ssrc = 0xdeadbeefL + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "twoStreamsSwitchingTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Av1DDAdaptiveSourceProjectionContext(diagnosticContext, initialState, logger) + var expectedSeq = 10001 + var expectedTs: Long = 1003000 + var expectedFrameNumber = 0 + var expectedTemplateOffset = 0 + var targetIndex = getIndex(eid = 0, dt = 2) + + /* Start by wanting encoding 0 */ + for (i in 0..899) { + val srPacket1 = generator1.srPacket + val packetInfo1 = generator1.nextPacket() + val packet1 = packetInfo1.packetAs() + if (i == 0) { + expectedTemplateOffset = packet1.descriptor!!.structure.templateIdOffset + } + Assert.assertTrue(context.accept(packetInfo1, 0, targetIndex)) + context.rewriteRtp(packetInfo1) + Assert.assertTrue(context.rewriteRtcp(srPacket1)) + Assert.assertEquals(packet1.ssrc, srPacket1.senderSsrc) + Assert.assertEquals(packet1.timestamp, srPacket1.senderInfo.rtpTimestamp) + val srPacket2 = generator2.srPacket + val packetInfo2 = generator2.nextPacket() + Assert.assertFalse(context.accept(packetInfo2, 1, targetIndex)) + Assert.assertFalse(context.rewriteRtcp(srPacket2)) + Assert.assertEquals(expectedSeq, packet1.sequenceNumber) + Assert.assertEquals(expectedTs, packet1.timestamp) + Assert.assertEquals(expectedFrameNumber, packet1.frameNumber) + Assert.assertEquals(expectedTemplateOffset, packet1.descriptor?.structure?.templateIdOffset) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + if (packet1.isEndOfFrame) { + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + if (packet1.isMarked) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + + /* Switch to wanting encoding 1, but don't send a keyframe. We should stay at the first encoding. */ + targetIndex = getIndex(eid = 1, dt = 2) + for (i in 0..89) { + val srPacket1 = generator1.srPacket + val packetInfo1 = generator1.nextPacket() + val packet1 = packetInfo1.packetAs() + Assert.assertTrue(context.accept(packetInfo1, 0, targetIndex)) + context.rewriteRtp(packetInfo1) + Assert.assertTrue(context.rewriteRtcp(srPacket1)) + Assert.assertEquals(packet1.ssrc, srPacket1.senderSsrc) + Assert.assertEquals(packet1.timestamp, srPacket1.senderInfo.rtpTimestamp) + val srPacket2 = generator2.srPacket + val packetInfo2 = generator2.nextPacket() + Assert.assertFalse(context.accept(packetInfo2, 1, targetIndex)) + Assert.assertFalse(context.rewriteRtcp(srPacket2)) + Assert.assertEquals(expectedSeq, packet1.sequenceNumber) + Assert.assertEquals(expectedTs, packet1.timestamp) + Assert.assertEquals(expectedFrameNumber, packet1.frameNumber) + Assert.assertEquals(expectedTemplateOffset, packet1.descriptor?.structure?.templateIdOffset) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + if (packet1.isEndOfFrame) { + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + if (packet1.isMarked) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + generator1.requestKeyframe() + generator2.requestKeyframe() + + /* After a keyframe we should accept spatial layer 1 */ + for (i in 0..8999) { + val srPacket1 = generator1.srPacket + val packetInfo1 = generator1.nextPacket() + val packet1 = packetInfo1.packetAs() + + /* We will cut off the layer 0 keyframe after 1 packet, once we see the layer 1 keyframe. */ + Assert.assertEquals(i == 0, context.accept(packetInfo1, 0, targetIndex)) + Assert.assertEquals(i == 0, context.rewriteRtcp(srPacket1)) + if (i == 0) { + context.rewriteRtp(packetInfo1) + Assert.assertEquals(packet1.ssrc, srPacket1.senderSsrc) + Assert.assertEquals(packet1.timestamp, srPacket1.senderInfo.rtpTimestamp) + expectedTemplateOffset += packet1.descriptor!!.structure.templateCount + } + val srPacket2 = generator2.srPacket + val packetInfo2 = generator2.nextPacket() + val packet2 = packetInfo2.packetAs() + Assert.assertTrue(context.accept(packetInfo2, 1, targetIndex)) + val expectedTemplateId = (packet2.descriptor!!.frameDependencyTemplateId + expectedTemplateOffset) % 64 + context.rewriteRtp(packetInfo2) + Assert.assertTrue(context.rewriteRtcp(srPacket2)) + Assert.assertEquals(packet2.ssrc, srPacket2.senderSsrc) + Assert.assertEquals(packet2.timestamp, srPacket2.senderInfo.rtpTimestamp) + if (i == 0) { + /* We leave a 1-packet gap for the layer 0 keyframe. */ + expectedSeq += 2 + /* ts will advance by an extra 3000 samples for the extra frame. */ + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + /* frame number will advance by 1 for the extra keyframe. */ + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + Assert.assertEquals(expectedSeq, packet2.sequenceNumber) + Assert.assertEquals(expectedTs, packet2.timestamp) + Assert.assertEquals(expectedFrameNumber, packet2.frameNumber) + Assert.assertEquals(expectedTemplateId, packet2.descriptor?.frameDependencyTemplateId) + if (packet2.descriptor?.newTemplateDependencyStructure != null) { + Assert.assertEquals( + expectedTemplateOffset, + packet2.descriptor?.newTemplateDependencyStructure?.templateIdOffset + ) + } + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + if (packet2.isEndOfFrame) { + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + if (packet2.isMarked) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + } + + @Test + fun temporalLayerSwitchingTest() { + val generator = TemporallyScaledPacketGenerator(3) + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "temporalLayerSwitchingTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Av1DDAdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + var targetTid = 0 + var decodableTid = 0 + var targetIndex = getIndex(0, targetTid) + var expectedSeq = 10001 + var expectedTs: Long = 1003000 + var expectedFrameNumber = 0 + for (i in 0..9999) { + val packetInfo = generator.nextPacket() + val packet = packetInfo.packetAs() + val accepted = context.accept(packetInfo, 0, targetIndex) + if (accepted) { + if (decodableTid < packet.frameInfo!!.temporalId) { + decodableTid = packet.frameInfo!!.temporalId + } + } else { + if (decodableTid > packet.frameInfo!!.temporalId - 1) { + decodableTid = packet.frameInfo!!.temporalId - 1 + } + } + if (packet.frameInfo!!.temporalId <= decodableTid) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + Assert.assertEquals(expectedSeq, packet.sequenceNumber) + Assert.assertEquals(expectedTs, packet.timestamp) + Assert.assertEquals(expectedFrameNumber, packet.frameNumber) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + } else { + Assert.assertFalse(accepted) + } + if (packet.isEndOfFrame) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + if (i % 97 == 0) { // Prime number so it's out of sync with packet cycles. + targetTid = (targetTid + 2) % 3 + targetIndex = getIndex(0, targetTid) + } + } + } + } + + private fun runLargeDropoutTest( + generator: Av1PacketGenerator, + targetIndex: Int, + expectAccept: (FrameInfo) -> Boolean + ) { + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = Thread.currentThread().stackTrace[2].methodName + val initialState = RtpState(1, 10000, 1000000) + val context = Av1DDAdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + var expectedSeq = 10001 + var expectedTs: Long = 1003000 + var expectedFrameNumber = 0 + for (i in 0..999) { + val packetInfo = generator.nextPacket() + val packet = packetInfo.packetAs() + + val accepted = context.accept(packetInfo, 0, targetIndex) + val frameInfo = packet.frameInfo!! + val endOfPicture = packet.isMarked + if (expectAccept(frameInfo)) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + Assert.assertEquals(expectedSeq, packet.sequenceNumber) + Assert.assertEquals(expectedTs, packet.timestamp) + Assert.assertEquals(expectedFrameNumber, packet.frameNumber) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + } else { + Assert.assertFalse(accepted) + } + if (packet.isEndOfFrame) { + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + for (gap in 64..65536 step { it * 2 }) { + for (i in 0 until gap) { + generator.nextPacket() + } + var packetInfo: PacketInfo + var packet: Av1DDPacket + var frameInfo: FrameInfo + do { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + frameInfo = packet.frameInfo!! + } while (!expectAccept(frameInfo)) + var endOfPicture = packet.isMarked + Assert.assertTrue(context.accept(packetInfo, 0, targetIndex)) + context.rewriteRtp(packetInfo) + + /* Allow any values after a gap. */ + expectedSeq = RtpUtils.applySequenceNumberDelta(packet.sequenceNumber, 1) + expectedTs = packet.timestamp + expectedFrameNumber = packet.frameNumber + if (packet.isEndOfFrame) { + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + for (i in 0..999) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + val accepted = context.accept(packetInfo, 0, targetIndex) + endOfPicture = packet.isMarked + frameInfo = packet.frameInfo!! + if (expectAccept(frameInfo)) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + Assert.assertEquals(expectedSeq, packet.sequenceNumber) + Assert.assertEquals(expectedTs, packet.timestamp) + Assert.assertEquals(expectedFrameNumber, packet.frameNumber) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + } else { + Assert.assertFalse(accepted) + } + if (packet.isEndOfFrame) { + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + } + } + + @Test + fun largeDropoutTest() { + val generator = TemporallyScaledPacketGenerator(1) + runLargeDropoutTest(generator, getIndex(eid = 0, dt = 2)) { + true + } + } + + @Test + fun filteredDropoutTest() { + val generator = TemporallyScaledPacketGenerator(1) + runLargeDropoutTest(generator, getIndex(eid = 0, dt = 0)) { + it.temporalId == 0 + } + } + + @Test + fun largeFrameDropoutTest() { + val generator = TemporallyScaledPacketGenerator(3) + runLargeDropoutTest(generator, getIndex(eid = 0, dt = 2)) { + true + } + } + + @Test + fun largeFrameFilteredDropoutTest() { + val generator = TemporallyScaledPacketGenerator(3) + runLargeDropoutTest(generator, getIndex(eid = 0, dt = 0)) { + it.temporalId == 0 + } + } + + private fun runSourceSuspensionTest( + generator: Av1PacketGenerator, + targetIndex: Int, + expectAccept: (FrameInfo) -> Boolean + ) { + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = Thread.currentThread().stackTrace[2].methodName + val initialState = RtpState(1, 10000, 1000000) + val context = Av1DDAdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + var expectedSeq = 10001 + var expectedTs: Long = 1003000 + var expectedFrameNumber = 0 + + var packetInfo: PacketInfo + var packet: Av1DDPacket + var frameInfo: FrameInfo + + var lastPacketAccepted = false + var lastFrameAccepted = -1 + + for (i in 0..999) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + frameInfo = packet.frameInfo!! + val accepted = context.accept(packetInfo, 0, targetIndex) + val endOfPicture = packet.isMarked + if (expectAccept(frameInfo)) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + Assert.assertEquals(expectedSeq, packet.sequenceNumber) + Assert.assertEquals(expectedTs, packet.timestamp) + Assert.assertEquals(expectedFrameNumber, packet.frameNumber) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + lastPacketAccepted = true + lastFrameAccepted = packet.frameNumber + } else { + Assert.assertFalse(accepted) + lastPacketAccepted = false + } + if (packet.isEndOfFrame) { + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + for (suspended in 64..65536 step { it * 2 }) { + /* If the last frame was accepted, finish the current frame if this generator is creating multi-packet + frames. */ + if (lastPacketAccepted) { + while (generator.packetOfFrame != 0) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + + val accepted = context.accept(packetInfo, 0, targetIndex) + val endOfPicture = packet.isMarked + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + if (packet.isEndOfFrame) { + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + } + /* Turn the source off for a time. */ + for (i in 0 until suspended) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + + val accepted = context.accept(packetInfo, 0, RtpLayerDesc.SUSPENDED_INDEX) + Assert.assertFalse(accepted) + val endOfPicture = packet.isMarked + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + + /* Switch back to wanting [targetIndex], but don't send a keyframe for a while. + * Should still be dropped. */ + for (i in 0 until 30) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + + val accepted = context.accept(packetInfo, 0, targetIndex) + val endOfPicture = packet.isMarked + Assert.assertFalse(accepted) + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + + /* Request a keyframe. Will be sent as of the next frame. */ + generator.requestKeyframe() + /* If this generator is creating multi-packet frames, finish the previous frame. */ + while (generator.packetOfFrame != 0) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + val accepted = context.accept(packetInfo, 0, targetIndex) + val endOfPicture = packet.isMarked + Assert.assertFalse(accepted) + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(lastFrameAccepted, 1) + + for (i in 0..999) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + frameInfo = packet.frameInfo!! + val accepted = context.accept(packetInfo, 0, targetIndex) + val endOfPicture = packet.isMarked + if (expectAccept(frameInfo)) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + Assert.assertEquals(expectedSeq, packet.sequenceNumber) + Assert.assertEquals(expectedTs, packet.timestamp) + Assert.assertEquals(expectedFrameNumber, packet.frameNumber) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + lastPacketAccepted = true + lastFrameAccepted = packet.frameNumber + } else { + Assert.assertFalse(accepted) + lastPacketAccepted = false + } + if (packet.isEndOfFrame) { + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + } + } + + @Test + fun sourceSuspensionTest() { + val generator = TemporallyScaledPacketGenerator(1) + runSourceSuspensionTest(generator, getIndex(eid = 0, dt = 2)) { + true + } + } + + @Test + fun filteredSourceSuspensionTest() { + val generator = TemporallyScaledPacketGenerator(1) + runSourceSuspensionTest(generator, getIndex(eid = 0, dt = 0)) { + it.temporalId == 0 + } + } + + @Test + fun largeFrameSourceSuspensionTest() { + val generator = TemporallyScaledPacketGenerator(3) + runSourceSuspensionTest(generator, getIndex(eid = 0, dt = 2)) { + true + } + } + + @Test + fun largeFrameFilteredSourceSuspensionTest() { + val generator = TemporallyScaledPacketGenerator(3) + runSourceSuspensionTest(generator, getIndex(eid = 0, dt = 0)) { + it.temporalId == 0 + } + } +} + +private open class Av1PacketGenerator( + val packetsPerFrame: Int, + val keyframeTemplates: Array, + val normalTemplates: Array, + // Equivalent to number of layers + val framesPerTimestamp: Int, + templateDdHex: String, + val allKeyframesGetStructure: Boolean = false +) { + private val logger: Logger = LoggerImpl(javaClass.name) + + var packetOfFrame = 0 + private set + private var frameOfPicture = 0 + + private var seq = 0 + var ts: Long = 0L + private set + var ssrc: Long = 0xcafebabeL + private var frameNumber = 0 + private var keyframePicture = false + private var keyframeRequested = false + private var pictureCount = 0 + private var receivedTime = baseReceivedTime + private var templateIdx = 0 + private var packetCount = 0 + private var octetCount = 0 + + private val structure: Av1TemplateDependencyStructure + + init { + val dd = parseHexBinary(templateDdHex) + structure = Av1DependencyDescriptorReader(dd, 0, dd.size).parse(null).structure + } + + fun reset() { + val useRandom = true // switch off to ease debugging + val seed = System.currentTimeMillis() + val random = Random(seed) + seq = if (useRandom) random.nextInt() % 0x10000 else 0 + ts = if (useRandom) random.nextLong() % 0x100000000L else 0 + frameNumber = 0 + packetOfFrame = 0 + frameOfPicture = 0 + keyframePicture = true + keyframeRequested = false + ssrc = 0xcafebabeL + pictureCount = 0 + receivedTime = baseReceivedTime + templateIdx = 0 + packetCount = 0 + octetCount = 0 + } + + fun nextPacket(missedStructure: Boolean = false): PacketInfo { + val startOfFrame = packetOfFrame == 0 + val endOfFrame = packetOfFrame == packetsPerFrame - 1 + val startOfPicture = startOfFrame && frameOfPicture == 0 + val endOfPicture = endOfFrame && frameOfPicture == framesPerTimestamp - 1 + + val templateId = ( + (if (keyframePicture) keyframeTemplates[templateIdx] else normalTemplates[templateIdx]) + + structure.templateIdOffset + ) % 64 + + val buffer = packetTemplate.clone() + val rtpPacket = RtpPacket(buffer, 0, buffer.size) + rtpPacket.ssrc = ssrc + rtpPacket.sequenceNumber = seq + rtpPacket.timestamp = ts + rtpPacket.isMarked = endOfPicture + + val dd = Av1DependencyDescriptorHeaderExtension( + startOfFrame = startOfFrame, + endOfFrame = endOfFrame, + frameDependencyTemplateId = templateId, + frameNumber = frameNumber, + newTemplateDependencyStructure = + if (keyframePicture && startOfFrame && (startOfPicture || allKeyframesGetStructure)) { + structure + } else { + null + }, + activeDecodeTargetsBitmask = null, + customDtis = null, + customFdiffs = null, + customChains = null, + structure = structure + ) + + val ext = rtpPacket.addHeaderExtension(AV1_DD_HEADER_EXTENSION_ID, dd.encodedLength) + dd.write(ext) + rtpPacket.encodeHeaderExtensions() + + val av1Packet = Av1DDPacket( + rtpPacket, + AV1_DD_HEADER_EXTENSION_ID, + if (missedStructure) null else structure, + logger + ) + + val info = PacketInfo(av1Packet) + info.receivedTime = receivedTime + + seq = RtpUtils.applySequenceNumberDelta(seq, 1) + packetCount++ + octetCount += av1Packet.length + + if (endOfFrame) { + packetOfFrame = 0 + if (endOfPicture) { + frameOfPicture = 0 + } else { + frameOfPicture++ + } + templateIdx++ + if (keyframeRequested) { + keyframePicture = true + templateIdx = 0 + } else if (keyframePicture) { + if (templateIdx >= keyframeTemplates.size) { + keyframePicture = false + } + } + frameNumber = RtpUtils.applySequenceNumberDelta(frameNumber, 1) + } else { + packetOfFrame++ + } + + if (endOfPicture) { + ts = RtpUtils.applyTimestampDelta(ts, 3000) + + keyframeRequested = false + if (templateIdx >= normalTemplates.size) { + templateIdx = 0 + } + pictureCount++ + receivedTime = baseReceivedTime + Duration.ofMillis(pictureCount * 100L / 3) + } + + return info + } + + fun requestKeyframe() { + if (packetOfFrame == 0) { + keyframePicture = true + templateIdx = 0 + } else { + keyframeRequested = true + } + } + + val srPacket: RtcpSrPacket + get() { + val srPacketBuilder = RtcpSrPacketBuilder() + srPacketBuilder.rtcpHeader.senderSsrc = ssrc + val siBuilder = srPacketBuilder.senderInfo + siBuilder.setNtpFromJavaTime(receivedTime.toEpochMilli()) + siBuilder.rtpTimestamp = ts + siBuilder.sendersOctetCount = packetCount.toLong() + siBuilder.sendersOctetCount = octetCount.toLong() + return srPacketBuilder.build() + } + + init { + reset() + } + + companion object { + val baseReceivedTime: Instant = Instant.ofEpochMilli(1577836800000L) // 2020-01-01 00:00:00 UTC + + const val AV1_DD_HEADER_EXTENSION_ID = 11 + + private val packetTemplate = DatatypeConverter.parseHexBinary( + // RTP Header + "80" + // V, P, X, CC + "29" + // M, PT + "0000" + // Seq + "00000000" + // TS + "cafebabe" + // SSRC + // Header extension will be added dynamically + // Dummy payload data + "0000000000000000000000" + ) + } +} + +private class NonScalableAv1PacketGenerator( + packetsPerFrame: Int +) : + Av1PacketGenerator( + packetsPerFrame, + arrayOf(0), + arrayOf(1), + 1, + "80000180003a410180ef808680" + ) + +private class TemporallyScaledPacketGenerator(packetsPerFrame: Int) : Av1PacketGenerator( + packetsPerFrame, + arrayOf(0), + arrayOf(1, 3, 2, 4), + 1, + "800001800214eaa860414d141020842701df010d" +) + +private class ScalableAv1PacketGenerator( + packetsPerFrame: Int +) : + Av1PacketGenerator( + packetsPerFrame, + arrayOf(1, 6, 11), + arrayOf(0, 5, 10, 3, 8, 13, 2, 7, 12, 4, 9, 14), + 3, + "d0013481e81485214eafffaaaa863cf0430c10c302afc0aaa0063c00430010c002a000a800060000" + + "40001d954926e082b04a0941b820ac1282503157f974000ca864330e222222eca8655304224230ec" + + "a87753013f00b3027f016704ff02cf" + ) + +private class KeyScalableAv1PacketGenerator( + packetsPerFrame: Int +) : + Av1PacketGenerator( + packetsPerFrame, + arrayOf(0, 5, 10), + arrayOf(1, 6, 11, 3, 8, 13, 2, 7, 12, 4, 9, 14), + 3, + "8f008581e81485214eaaaaa8000600004000100002aa80a8000600004000100002a000a80006000040" + + "0016d549241b5524906d54923157e001974ca864330e222396eca8655304224390eca87753013f00b3027f016704ff02cf" + ) + +private class SingleEncodingSimulcastAv1PacketGenerator( + packetsPerFrame: Int +) : + Av1PacketGenerator( + packetsPerFrame, + arrayOf(1, 6, 11), + arrayOf(0, 5, 10, 3, 8, 13, 2, 7, 12, 4, 9, 14), + 3, + "c1000180081485214ea000a8000600004000100002a000a8000600004000100002a000a8000600004" + + "0001d954926caa493655248c55fe5d00032a190cc38e58803b2a1954c10e10843b2a1dd4c01dc010803bc0218077c0434", + allKeyframesGetStructure = true + ) + +private infix fun IntRange.step(next: (Int) -> Int) = + generateSequence(first, next).takeWhile { if (first < last) it <= last else it >= last } diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilterTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilterTest.kt new file mode 100644 index 0000000000..0ab04c8ec5 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilterTest.kt @@ -0,0 +1,896 @@ +/* + * Copyright @ 2019 - present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.cc.av1 + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import jakarta.xml.bind.DatatypeConverter +import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc +import org.jitsi.rtp.rtp.header_extensions.Av1DependencyDescriptorReader +import org.jitsi.rtp.rtp.header_extensions.Av1TemplateDependencyStructure +import org.jitsi.utils.logging2.LoggerImpl +import org.jitsi.utils.logging2.getClassForLogging +import java.time.Instant + +internal class Av1DDQualityFilterTest : ShouldSpec() { + init { + context("A non-scalable stream") { + should("be entirely projected") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SingleLayerFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 0) + + testGenerator(generator, filter, targetIndex) { _, result -> + result.accept shouldBe true + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + context("A temporally scalable stream") { + should("be entirely projected when TL2 is requested") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = TemporallyScaledFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 2) + + testGenerator(generator, filter, targetIndex) { _, result -> + result.accept shouldBe true + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + should("project only the base temporal layer when targeted") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = TemporallyScaledFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 0) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.temporalId == 0) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("project only the intermediate temporal layer when targeted") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = TemporallyScaledFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 1) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.temporalId <= 1) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to switch the targeted layers, without a keyframe") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = TemporallyScaledFrameGenerator(av1FrameMaps) + val targetIndex1 = Av1DDRtpLayerDesc.getIndex(0, 0) + + testGenerator(generator, filter, targetIndex1, numFrames = 500) { f, result -> + result.accept shouldBe (f.frameInfo!!.temporalId == 0) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + val targetIndex2 = Av1DDRtpLayerDesc.getIndex(0, 2) + + testGenerator(generator, filter, targetIndex2) { _, result -> + result.accept shouldBe true + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + context("A spatially scalable stream") { + should("be entirely projected when SL2/TL2 is requested (L3T3)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SVCFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 3 * 2 + 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe true + result.mark shouldBe (f.frameInfo!!.spatialId == 2) + filter.needsKeyframe shouldBe false + } + } + should("be able to be shaped to SL0/TL2 (L3T3)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SVCFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 0) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 0) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to SL1/TL2 (L3T3)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SVCFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 3 * 1 + 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId <= 1) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 1) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to SL2/TL0 (L3T3)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SVCFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 3 * 2 + 0) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.temporalId == 0) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 2) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to switch spatial layers (L3T3)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SVCFrameGenerator(av1FrameMaps) + + /* Start by sending spatial layer 0. */ + val targetIndex1 = Av1DDRtpLayerDesc.getIndex(0, 2) + + testGenerator(generator, filter, targetIndex1, numFrames = 1200) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 0) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 0) + filter.needsKeyframe shouldBe false + } + } + + /* Switch to spatial layer 2. Need a keyframe. */ + val targetIndex2 = Av1DDRtpLayerDesc.getIndex(0, 3 * 2 + 2) + var sawKeyframe = false + testGenerator(generator, filter, targetIndex2, numFrames = 1200) { f, result -> + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) (f.frameInfo!!.spatialId == 0) else true + if (result.accept) { + result.mark shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 0) + } else { + (f.frameInfo!!.spatialId == 2) + } + filter.needsKeyframe shouldBe (!sawKeyframe) + } + } + + /* Switch to spatial layer 1. For SVC, dropping down in spatial layers can happen immediately. */ + val targetIndex3 = Av1DDRtpLayerDesc.getIndex(0, 3 * 1 + 2) + testGenerator(generator, filter, targetIndex3) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId <= 1) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 1) + filter.needsKeyframe shouldBe false + } + } + } + } + context("A K-SVC spatially scalable stream") { + should("be able to be shaped to SL2/TL2 (L3T3_KEY)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = KSVCFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 3 * 2 + 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe ( + f.frameInfo!!.spatialId == 2 || !f.frameInfo!!.hasInterPictureDependency() + ) + result.mark shouldBe (f.frameInfo!!.spatialId == 2) + filter.needsKeyframe shouldBe false + } + } + should("be able to be shaped to SL0/TL2 (L3T3_KEY)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = KSVCFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 0) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 0) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to SL1/TL2 (L3T3_KEY)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = KSVCFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 3 * 1 + 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe ( + f.frameInfo!!.spatialId == 1 || ( + f.frameInfo!!.spatialId == 0 && !f.frameInfo!!.hasInterPictureDependency() + ) + ) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 1) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to SL2/TL0 (L3T3_KEY)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = KSVCFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 3 * 2 + 0) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe ( + f.frameInfo!!.temporalId == 0 && ( + f.frameInfo!!.spatialId == 2 || !f.frameInfo!!.hasInterPictureDependency() + ) + ) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 2) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to switch spatial layers (L3T3_KEY)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = KSVCFrameGenerator(av1FrameMaps) + + /* Start by sending spatial layer 0. */ + val targetIndex1 = Av1DDRtpLayerDesc.getIndex(0, 2) + + testGenerator(generator, filter, targetIndex1, numFrames = 1200) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 0) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 0) + filter.needsKeyframe shouldBe false + } + } + + /* Switch to spatial layer 2. Need a keyframe. */ + val targetIndex2 = Av1DDRtpLayerDesc.getIndex(0, 3 * 2 + 2) + var sawKeyframe = false + testGenerator(generator, filter, targetIndex2, numFrames = 1200) { f, result -> + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 0) + } else { + (f.frameInfo!!.spatialId == 2 || !f.frameInfo!!.hasInterPictureDependency()) + } + if (result.accept) { + result.mark shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 0) + } else { + (f.frameInfo!!.spatialId == 2) + } + filter.needsKeyframe shouldBe (!sawKeyframe) + } + } + + /* Switch to spatial layer 1. For K-SVC, dropping down in spatial layers needs a keyframe. */ + val targetIndex3 = Av1DDRtpLayerDesc.getIndex(0, 3 * 1 + 2) + sawKeyframe = false + testGenerator(generator, filter, targetIndex3) { f, result -> + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 2 || !f.frameInfo!!.hasInterPictureDependency()) + } else { + ( + f.frameInfo!!.spatialId == 1 || ( + f.frameInfo!!.spatialId == 0 && !f.frameInfo!!.hasInterPictureDependency() + ) + ) + } + if (result.accept) { + result.mark shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 2) + } else { + (f.frameInfo!!.spatialId == 1) + } + filter.needsKeyframe shouldBe (!sawKeyframe) + } + } + } + } + context("A K-SVC spatially scalable stream with a temporal shift") { + should("be able to be shaped to SL1/TL1 (L2S2_KEY_SHIFT)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = KSVCShiftFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 2 * 1 + 1) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe ( + f.frameInfo!!.spatialId == 1 || !f.frameInfo!!.hasInterPictureDependency() + ) + result.mark shouldBe (f.frameInfo!!.spatialId == 1) + filter.needsKeyframe shouldBe false + } + } + should("be able to be shaped to SL0/TL1 (L2S2_KEY_SHIFT)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = KSVCShiftFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 1) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 0) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 0) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to SL1/TL0 (L2S2_KEY_SHIFT)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = KSVCShiftFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 2 * 1 + 0) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe ( + f.frameInfo!!.temporalId == 0 && ( + f.frameInfo!!.spatialId == 1 || !f.frameInfo!!.hasInterPictureDependency() + ) + ) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 1) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to switch spatial layers (L2S2_KEY_SHIFT)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = KSVCShiftFrameGenerator(av1FrameMaps) + + /* Start by sending spatial layer 0. */ + val targetIndex1 = Av1DDRtpLayerDesc.getIndex(0, 1) + + testGenerator(generator, filter, targetIndex1, numFrames = 1200) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 0) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 0) + filter.needsKeyframe shouldBe false + } + } + + /* Switch to spatial layer 1. Need a keyframe. */ + val targetIndex2 = Av1DDRtpLayerDesc.getIndex(0, 2 * 1 + 1) + var sawKeyframe = false + testGenerator(generator, filter, targetIndex2, numFrames = 1200) { f, result -> + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 0) + } else { + (f.frameInfo!!.spatialId == 1 || !f.frameInfo!!.hasInterPictureDependency()) + } + if (result.accept) { + result.mark shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 0) + } else { + (f.frameInfo!!.spatialId == 1) + } + filter.needsKeyframe shouldBe (!sawKeyframe) + } + } + + /* Switch back to spatial layer 0. For K-SVC, dropping down in spatial layers needs a keyframe. */ + val targetIndex3 = Av1DDRtpLayerDesc.getIndex(0, 1) + sawKeyframe = false + testGenerator(generator, filter, targetIndex3) { f, result -> + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 1 || !f.frameInfo!!.hasInterPictureDependency()) + } else { + (f.frameInfo!!.spatialId == 0) + } + if (result.accept) { + result.mark shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 1) + } else { + (f.frameInfo!!.spatialId == 0) + } + filter.needsKeyframe shouldBe (!sawKeyframe) + } + } + } + } + context("A single-encoding simulcast stream") { + should("project all of layer 2 when when SL2/TL2 is requested (S3T3)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SingleEncodingSimulcastGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 3 * 2 + 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 2) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to SL0/TL2 (S3T3)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SingleEncodingSimulcastGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 0) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to SL1/TL2 (S3T3)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SingleEncodingSimulcastGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 3 * 1 + 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 1) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to SL2/TL0 (S3T3)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SingleEncodingSimulcastGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 3 * 2 + 0) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 2 && f.frameInfo!!.temporalId == 0) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to switch spatial layers (S3T3)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SingleEncodingSimulcastGenerator(av1FrameMaps) + + /* Start by sending spatial layer 0. */ + val targetIndex1 = Av1DDRtpLayerDesc.getIndex(0, 2) + + testGenerator(generator, filter, targetIndex1, numFrames = 1200) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 0) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + + /* Switch to spatial layer 2. Need a keyframe. */ + val targetIndex2 = Av1DDRtpLayerDesc.getIndex(0, 3 * 2 + 2) + var sawKeyframe = false + testGenerator(generator, filter, targetIndex2, numFrames = 1200) { f, result -> + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 0) + } else { + (f.frameInfo!!.spatialId == 2) + } + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe (!sawKeyframe) + } + } + + /* Switch to spatial layer 1. Need a keyframe. */ + val targetIndex3 = Av1DDRtpLayerDesc.getIndex(0, 3 * 1 + 2) + sawKeyframe = false + testGenerator(generator, filter, targetIndex3) { f, result -> + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 2) + } else { + (f.frameInfo!!.spatialId == 1) + } + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe (!sawKeyframe) + } + } + } + } + context("A multi-encoding simulcast stream") { + should("project all of encoding 2 when when Enc 2/TL2 is requested") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = MultiEncodingSimulcastGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(2, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.ssrc == 2L || f.isKeyframe) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to Enc 0/TL2") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = MultiEncodingSimulcastGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.ssrc == 0L) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to Enc 1/TL2") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = MultiEncodingSimulcastGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(1, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.ssrc == 1L || (f.isKeyframe && f.ssrc == 0L)) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to Enc 2/TL0") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = MultiEncodingSimulcastGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(2, 0) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe ((f.ssrc == 2L || f.isKeyframe) && f.frameInfo!!.temporalId == 0) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to switch encodings") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = MultiEncodingSimulcastGenerator(av1FrameMaps) + + /* Start by sending encoding 0. */ + val targetIndex1 = Av1DDRtpLayerDesc.getIndex(0, 2) + + testGenerator(generator, filter, targetIndex1, numFrames = 1200) { f, result -> + result.accept shouldBe (f.ssrc == 0L) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + + /* Switch to encoding 2. Need a keyframe. */ + val targetIndex2 = Av1DDRtpLayerDesc.getIndex(2, 2) + var sawKeyframe = false + testGenerator(generator, filter, targetIndex2, numFrames = 1200) { f, result -> + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) { + (f.ssrc == 0L) + } else { + (f.ssrc == 2L || f.isKeyframe) + } + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe (!sawKeyframe) + } + } + + /* Switch to encoding 1. Need a keyframe. */ + val targetIndex3 = Av1DDRtpLayerDesc.getIndex(1, 2) + sawKeyframe = false + testGenerator(generator, filter, targetIndex3) { f, result -> + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) { + // We don't send discardable frames for the DT while there's a pending encoding downswitch + (f.ssrc == 2L && f.frameInfo!!.temporalId != 2) + } else { + (f.ssrc == 1L || (f.ssrc == 0L && f.isKeyframe)) + } + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe (!sawKeyframe) + } + } + } + } + } + + private fun testGenerator( + g: FrameGenerator, + filter: Av1DDQualityFilter, + targetIndex: Int, + numFrames: Int = Int.MAX_VALUE, + evaluator: (Av1DDFrame, Av1DDQualityFilter.AcceptResult) -> Unit + ) { + var lastTs = -1L + var ms = -1L + var frames = 0 + while (g.hasNext() && frames < numFrames) { + val f = g.next() + + ms = if (f.timestamp != lastTs) { + f.timestamp / 90 + } else { + ms + 1 + } + lastTs = f.timestamp + + val result = filter.acceptFrame( + frame = f, + externalTargetIndex = targetIndex, + incomingEncoding = f.ssrc.toInt(), + receivedTime = Instant.ofEpochMilli(ms) + ) + f.isAccepted = result.accept + evaluator(f, result) + frames++ + } + } + + companion object { + val logger = LoggerImpl(getClassForLogging(this::class.java).name) + } +} + +private abstract class FrameGenerator : Iterator + +private open class DDBasedGenerator( + val av1FrameMaps: HashMap, + val keyframeInterval: Int, + val keyframeTemplates: Array, + val normalTemplates: Array, + ddHex: String +) : FrameGenerator() { + private var frameCount = 0 + private val structure: Av1TemplateDependencyStructure + + init { + val dd = DatatypeConverter.parseHexBinary(ddHex) + structure = Av1DependencyDescriptorReader(dd, 0, dd.size).parse(null).structure + } + + override fun hasNext(): Boolean = frameCount < TOTAL_FRAMES + + protected open fun isKeyframe(keyCycle: Int) = keyCycle == 0 + + override fun next(): Av1DDFrame { + val tCycle = frameCount % normalTemplates.size + val keyCycle = frameCount % keyframeInterval + + val templateId = if (keyCycle < keyframeTemplates.size) { + keyframeTemplates[tCycle] + } else { + normalTemplates[tCycle] + } + + val f = Av1DDFrame( + ssrc = 0, + timestamp = frameCount * 3000L, + earliestKnownSequenceNumber = frameCount, + latestKnownSequenceNumber = frameCount, + seenStartOfFrame = true, + seenEndOfFrame = true, + seenMarker = true, + frameInfo = structure.templateInfo[templateId], + // Will be less than 0xffff + frameNumber = frameCount, + index = frameCount, + templateId = templateId, + structure = structure, + activeDecodeTargets = null, + isKeyframe = isKeyframe(keyCycle), + rawDependencyDescriptor = null + ) + av1FrameMaps.getOrPut(f.ssrc) { Av1DDFrameMap(Av1DDQualityFilterTest.logger) }.insertFrame(f) + frameCount++ + return f + } + + companion object { + private const val TOTAL_FRAMES = 10000 + } +} + +/** Generate a non-scalable AV1 stream, with a single keyframe at the start. */ +private class SingleLayerFrameGenerator(av1FrameMaps: HashMap) : DDBasedGenerator( + av1FrameMaps, + 10000, + arrayOf(0), + arrayOf(1), + "80000180003a410180ef808680" +) + +/** Generate a temporally-scaled series of AV1 frames, with a single keyframe at the start. */ +private class TemporallyScaledFrameGenerator(av1FrameMaps: HashMap) : DDBasedGenerator( + av1FrameMaps, + 10000, + arrayOf(0), + arrayOf(1, 3, 2, 4), + "800001800214eaa860414d141020842701df010d" +) + +/** Generate a spatially-scaled series of AV1 frames (L3T3), with full spatial dependencies and periodic keyframes. */ +private class SVCFrameGenerator(av1FrameMaps: HashMap) : DDBasedGenerator( + av1FrameMaps, + 144, + arrayOf(1, 6, 11), + arrayOf(0, 5, 10, 3, 8, 13, 2, 7, 12, 4, 9, 14), + "d0013481e81485214eafffaaaa863cf0430c10c302afc0aaa0063c00430010c002a000a800060000" + + "40001d954926e082b04a0941b820ac1282503157f974000ca864330e222222eca8655304224230ec" + + "a87753013f00b3027f016704ff02cf" +) + +/** Generate a spatially-scaled series of AV1 frames (L3T3), with keyframe spatial dependencies and periodic + * keyframes. */ +private class KSVCFrameGenerator(av1FrameMaps: HashMap) : DDBasedGenerator( + av1FrameMaps, + 144, + arrayOf(0, 5, 10), + arrayOf(1, 6, 11, 3, 8, 13, 2, 7, 12, 4, 9, 14), + "8f008581e81485214eaaaaa8000600004000100002aa80a8000600004000100002a000a80006000040" + + "0016d549241b5524906d54923157e001974ca864330e222396eca8655304224390eca87753013f00b3027f016704ff02cf" +) + +/** Generate a spatially-scaled series of AV1 frames (L2T2), with keyframe spatial dependencies and periodic + * keyframes, with temporal structures shifted. */ +/* Note that as of Chrome 111, L3T3_KEY_SHIFT is not supported yet, so we're testing L2T2_KEY_SHIFT instead. */ +private class KSVCShiftFrameGenerator(av1FrameMaps: HashMap) : DDBasedGenerator( + av1FrameMaps, + 144, + arrayOf(0, 4, 1), + arrayOf(2, 6, 3, 5), + "8700ed80e3061eaa82804028280514d14134518010a091889a09409fc059c13fc0b3c0" +) + +/** Generate a single-stream temporally-scaled simulcast (S3T3) series of AV1 frames, with periodic keyframes. */ +private class SingleEncodingSimulcastGenerator(av1FrameMaps: HashMap) : DDBasedGenerator( + av1FrameMaps, + 144, + arrayOf(1, 6, 11), + arrayOf(0, 5, 10, 3, 8, 13, 2, 7, 12, 4, 9, 14), + "c1000180081485214ea000a8000600004000100002a000a8000600004000100002a000a8000600004" + + "0001d954926caa493655248c55fe5d00032a190cc38e58803b2a1954c10e10843b2a1dd4c01dc010803bc0218077c0434" +) { + // All frames of the initial picture get the DD structure attached + override fun isKeyframe(keyCycle: Int) = keyCycle < keyframeTemplates.size +} + +private class MultiEncodingSimulcastGenerator(val av1FrameMaps: HashMap) : FrameGenerator() { + private var frameCount = 0 + + override fun hasNext(): Boolean = frameCount < TOTAL_FRAMES + + override fun next(): Av1DDFrame { + val pictureCount = frameCount / NUM_ENCODINGS + val encoding = frameCount % NUM_ENCODINGS + val tCycle = pictureCount % normalTemplates.size + val keyCycle = pictureCount % KEYFRAME_INTERVAL + + val templateId = if (keyCycle < keyframeTemplates.size) { + keyframeTemplates[tCycle] + } else { + normalTemplates[tCycle] + } + + val keyframePicture = keyCycle == 0 + + val f = Av1DDFrame( + ssrc = encoding.toLong(), + timestamp = pictureCount * 3000L, + earliestKnownSequenceNumber = pictureCount, + latestKnownSequenceNumber = pictureCount, + seenStartOfFrame = true, + seenEndOfFrame = true, + seenMarker = true, + frameInfo = structure.templateInfo[templateId], + // Will be less than 0xffff + frameNumber = pictureCount, + index = pictureCount, + templateId = templateId, + structure = structure, + activeDecodeTargets = null, + isKeyframe = keyframePicture, + rawDependencyDescriptor = null + ) + av1FrameMaps.getOrPut(f.ssrc) { Av1DDFrameMap(Av1DDQualityFilterTest.logger) }.insertFrame(f) + frameCount++ + return f + } + + companion object { + private const val TOTAL_FRAMES = 10000 + private const val KEYFRAME_INTERVAL = 144 + private const val NUM_ENCODINGS = 3 + private val keyframeTemplates = arrayOf(0) + private val normalTemplates = arrayOf(1, 3, 2, 4) + + private val structure: Av1TemplateDependencyStructure + + init { + val dd = DatatypeConverter.parseHexBinary("800001800214eaa860414d141020842701df010d") + structure = Av1DependencyDescriptorReader(dd, 0, dd.size).parse(null).structure + } + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionTest.kt index 5c89fd6c36..1eaac25854 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionTest.kt @@ -23,13 +23,10 @@ import org.jitsi.nlj.RtpLayerDesc.Companion.getTidFromIndex import org.jitsi.nlj.codec.vpx.VpxUtils.Companion.applyExtendedPictureIdDelta import org.jitsi.nlj.codec.vpx.VpxUtils.Companion.applyTl0PicIdxDelta import org.jitsi.nlj.codec.vpx.VpxUtils.Companion.getExtendedPictureIdDelta -import org.jitsi.nlj.format.PayloadType -import org.jitsi.nlj.format.Vp9PayloadType import org.jitsi.nlj.rtp.codec.vp9.Vp9Packet import org.jitsi.nlj.util.Rfc3711IndexTracker import org.jitsi.rtp.rtcp.RtcpSrPacket import org.jitsi.rtp.rtcp.RtcpSrPacketBuilder -import org.jitsi.rtp.rtcp.SenderInfoBuilder import org.jitsi.rtp.rtp.RtpPacket import org.jitsi.rtp.util.RtpUtils import org.jitsi.rtp.util.isNewerThan @@ -46,18 +43,11 @@ import java.time.Duration import java.time.Instant import java.util.Random import java.util.TreeMap -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.CopyOnWriteArraySet import javax.xml.bind.DatatypeConverter import kotlin.collections.ArrayList class Vp9AdaptiveSourceProjectionTest { private val logger: Logger = LoggerImpl(javaClass.name) - private val payloadType: PayloadType = Vp9PayloadType( - 96.toByte(), - ConcurrentHashMap(), - CopyOnWriteArraySet() - ) @Test fun singlePacketProjectionTest() { @@ -66,7 +56,6 @@ class Vp9AdaptiveSourceProjectionTest { val initialState = RtpState(1, 10000, 1000000) val context = Vp9AdaptiveSourceProjectionContext( diagnosticContext, - payloadType, initialState, logger ) @@ -74,13 +63,7 @@ class Vp9AdaptiveSourceProjectionTest { val packetInfo = generator.nextPacket() val packet = packetInfo.packetAs() val targetIndex = getIndex(eid = 0, sid = 0, tid = 0) - Assert.assertTrue( - context.accept( - packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), - targetIndex - ) - ) + Assert.assertTrue(context.accept(packetInfo, 0, targetIndex)) context.rewriteRtp(packetInfo) Assert.assertEquals(10001, packet.sequenceNumber) Assert.assertEquals(1003000, packet.timestamp) @@ -95,7 +78,6 @@ class Vp9AdaptiveSourceProjectionTest { val initialState = RtpState(1, 10000, 1000000) val context = Vp9AdaptiveSourceProjectionContext( diagnosticContext, - payloadType, initialState, logger ) @@ -110,7 +92,7 @@ class Vp9AdaptiveSourceProjectionTest { val packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + 0, targetIndex ) if (!packet.hasLayerIndices) { @@ -146,7 +128,7 @@ class Vp9AdaptiveSourceProjectionTest { } } - private class ProjectedPacket internal constructor( + private class ProjectedPacket constructor( val packet: Vp9Packet, val origSeq: Int, val extOrigSeq: Int, @@ -176,7 +158,6 @@ class Vp9AdaptiveSourceProjectionTest { var orderedCount = initialOrderedCount - 1 val context = Vp9AdaptiveSourceProjectionContext( diagnosticContext, - payloadType, initialState, logger ) @@ -193,11 +174,7 @@ class Vp9AdaptiveSourceProjectionTest { if (latestSeq isOlderThan origSeq) { latestSeq = origSeq } - val accepted = context.accept( - packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), - targetIndex - ) + val accepted = context.accept(packetInfo, 0, targetIndex) val oldestValidSeq: Int = RtpUtils.applySequenceNumberDelta( latestSeq, @@ -501,42 +478,20 @@ class Vp9AdaptiveSourceProjectionTest { val initialState = RtpState(1, 10000, 1000000) val context = Vp9AdaptiveSourceProjectionContext( diagnosticContext, - payloadType, initialState, logger ) val firstPacketInfo = generator.nextPacket() - val firstPacket = firstPacketInfo.packetAs() val targetIndex = getIndex(eid = 0, sid = 0, tid = 2) for (i in 0..2) { val packetInfo = generator.nextPacket() - val packet = packetInfo.packetAs() - Assert.assertFalse( - context.accept( - packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), - targetIndex - ) - ) + Assert.assertFalse(context.accept(packetInfo, 0, targetIndex)) } - Assert.assertTrue( - context.accept( - firstPacketInfo, - getIndex(0, firstPacket.spatialLayerIndex, firstPacket.temporalLayerIndex), - targetIndex - ) - ) + Assert.assertTrue(context.accept(firstPacketInfo, 0, targetIndex)) context.rewriteRtp(firstPacketInfo) for (i in 0..9995) { val packetInfo = generator.nextPacket() - val packet = packetInfo.packetAs() - Assert.assertTrue( - context.accept( - packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), - targetIndex - ) - ) + Assert.assertTrue(context.accept(packetInfo, 0, targetIndex)) context.rewriteRtp(packetInfo) } } @@ -549,53 +504,24 @@ class Vp9AdaptiveSourceProjectionTest { val initialState = RtpState(1, 10000, 1000000) val context = Vp9AdaptiveSourceProjectionContext( diagnosticContext, - payloadType, initialState, logger ) val firstPacketInfo = generator.nextPacket() - val firstPacket = firstPacketInfo.packetAs() val targetIndex = getIndex(eid = 0, sid = 0, tid = 2) for (i in 0..3) { val packetInfo = generator.nextPacket() - val packet = packetInfo.packetAs() - Assert.assertFalse( - context.accept( - packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), - targetIndex - ) - ) + Assert.assertFalse(context.accept(packetInfo, 0, targetIndex)) } - Assert.assertFalse( - context.accept( - firstPacketInfo, - getIndex(0, firstPacket.spatialLayerIndex, firstPacket.temporalLayerIndex), - targetIndex - ) - ) + Assert.assertFalse(context.accept(firstPacketInfo, 0, targetIndex)) for (i in 0..9) { val packetInfo = generator.nextPacket() - val packet = packetInfo.packetAs() - Assert.assertFalse( - context.accept( - packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), - targetIndex - ) - ) + Assert.assertFalse(context.accept(packetInfo, 0, targetIndex)) } generator.requestKeyframe() for (i in 0..9995) { val packetInfo = generator.nextPacket() - val packet = packetInfo.packetAs() - Assert.assertTrue( - context.accept( - packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), - targetIndex - ) - ) + Assert.assertTrue(context.accept(packetInfo, 0, targetIndex)) context.rewriteRtp(packetInfo) } } @@ -608,7 +534,6 @@ class Vp9AdaptiveSourceProjectionTest { val initialState = RtpState(1, 10000, 1000000) val context = Vp9AdaptiveSourceProjectionContext( diagnosticContext, - payloadType, initialState, logger ) @@ -619,38 +544,19 @@ class Vp9AdaptiveSourceProjectionTest { for (i in 0..10) { val packetInfo = generator.nextPacket() val packet = packetInfo.packetAs() - Assert.assertTrue( - context.accept( - packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), - targetIndex - ) - ) + Assert.assertTrue(context.accept(packetInfo, 0, targetIndex)) context.rewriteRtp(packetInfo) Assert.assertTrue(packet.sequenceNumber > 10001) lowestSeq = minOf(lowestSeq, packet.sequenceNumber) } - Assert.assertTrue( - context.accept( - firstPacketInfo, - getIndex(0, firstPacket.spatialLayerIndex, firstPacket.temporalLayerIndex), - targetIndex - ) - ) + Assert.assertTrue(context.accept(firstPacketInfo, 0, targetIndex)) context.rewriteRtp(firstPacketInfo) Assert.assertEquals(lowestSeq - 1, firstPacket.sequenceNumber) for (i in 0..9980) { val packetInfo = generator.nextPacket() - val packet = packetInfo.packetAs() - Assert.assertTrue( - context.accept( - packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), - targetIndex - ) - ) + Assert.assertTrue(context.accept(packetInfo, 0, targetIndex)) context.rewriteRtp(packetInfo) } } @@ -665,7 +571,6 @@ class Vp9AdaptiveSourceProjectionTest { val initialState = RtpState(1, 10000, 1000000) val context = Vp9AdaptiveSourceProjectionContext( diagnosticContext, - payloadType, initialState, logger ) @@ -675,22 +580,9 @@ class Vp9AdaptiveSourceProjectionTest { for (i in 0..9999) { val packetInfo1 = generator1.nextPacket() val packet1 = packetInfo1.packetAs() - Assert.assertTrue( - context.accept( - packetInfo1, - getIndex(1, packet1.spatialLayerIndex, packet1.temporalLayerIndex), - targetIndex - ) - ) + Assert.assertTrue(context.accept(packetInfo1, 1, targetIndex)) val packetInfo2 = generator2.nextPacket() - val packet2 = packetInfo2.packetAs() - Assert.assertFalse( - context.accept( - packetInfo2, - getIndex(0, packet2.spatialLayerIndex, packet2.temporalLayerIndex), - targetIndex - ) - ) + Assert.assertFalse(context.accept(packetInfo2, 0, targetIndex)) context.rewriteRtp(packetInfo1) Assert.assertEquals(expectedSeq, packet1.sequenceNumber) Assert.assertEquals(expectedTs, packet1.timestamp) @@ -711,7 +603,6 @@ class Vp9AdaptiveSourceProjectionTest { val initialState = RtpState(1, 10000, 1000000) val context = Vp9AdaptiveSourceProjectionContext( diagnosticContext, - payloadType, initialState, logger ) @@ -729,35 +620,14 @@ class Vp9AdaptiveSourceProjectionTest { if (packet1.isStartOfFrame && packet1.temporalLayerIndex == 0) { expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) } - Assert.assertTrue( - context.accept( - packetInfo1, - getIndex( - 0, - packet1.spatialLayerIndex, - packet1.temporalLayerIndex - ), - targetIndex - ) - ) + Assert.assertTrue(context.accept(packetInfo1, 0, targetIndex)) context.rewriteRtp(packetInfo1) Assert.assertTrue(context.rewriteRtcp(srPacket1)) Assert.assertEquals(packet1.ssrc, srPacket1.senderSsrc) Assert.assertEquals(packet1.timestamp, srPacket1.senderInfo.rtpTimestamp) val srPacket2 = generator2.srPacket val packetInfo2 = generator2.nextPacket() - val packet2 = packetInfo2.packetAs() - Assert.assertFalse( - context.accept( - packetInfo2, - getIndex( - 1, - packet2.spatialLayerIndex, - packet2.temporalLayerIndex - ), - targetIndex - ) - ) + Assert.assertFalse(context.accept(packetInfo2, 1, targetIndex)) Assert.assertFalse(context.rewriteRtcp(srPacket2)) Assert.assertEquals(expectedSeq, packet1.sequenceNumber) Assert.assertEquals(expectedTs, packet1.timestamp) @@ -779,27 +649,14 @@ class Vp9AdaptiveSourceProjectionTest { if (packet1.isStartOfFrame && packet1.temporalLayerIndex == 0) { expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) } - Assert.assertTrue( - context.accept( - packetInfo1, - getIndex(0, packet1.spatialLayerIndex, packet1.temporalLayerIndex), - targetIndex - ) - ) + Assert.assertTrue(context.accept(packetInfo1, 0, targetIndex)) context.rewriteRtp(packetInfo1) Assert.assertTrue(context.rewriteRtcp(srPacket1)) Assert.assertEquals(packet1.ssrc, srPacket1.senderSsrc) Assert.assertEquals(packet1.timestamp, srPacket1.senderInfo.rtpTimestamp) val srPacket2 = generator2.srPacket val packetInfo2 = generator2.nextPacket() - val packet2 = packetInfo2.packetAs() - Assert.assertFalse( - context.accept( - packetInfo2, - getIndex(1, packet2.spatialLayerIndex, packet2.temporalLayerIndex), - targetIndex - ) - ) + Assert.assertFalse(context.accept(packetInfo2, 1, targetIndex)) Assert.assertFalse(context.rewriteRtcp(srPacket2)) Assert.assertEquals(expectedSeq, packet1.sequenceNumber) Assert.assertEquals(expectedTs, packet1.timestamp) @@ -824,14 +681,7 @@ class Vp9AdaptiveSourceProjectionTest { } /* We will cut off the layer 0 keyframe after 1 packet, once we see the layer 1 keyframe. */ - Assert.assertEquals( - i == 0, - context.accept( - packetInfo1, - getIndex(0, packet1.spatialLayerIndex, packet1.temporalLayerIndex), - targetIndex - ) - ) + Assert.assertEquals(i == 0, context.accept(packetInfo1, 0, targetIndex)) Assert.assertEquals(i == 0, context.rewriteRtcp(srPacket1)) if (i == 0) { context.rewriteRtp(packetInfo1) @@ -844,13 +694,7 @@ class Vp9AdaptiveSourceProjectionTest { if (packet2.isStartOfFrame && packet2.temporalLayerIndex == 0) { expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) } - Assert.assertTrue( - context.accept( - packetInfo2, - getIndex(1, packet2.spatialLayerIndex, packet2.temporalLayerIndex), - targetIndex - ) - ) + Assert.assertTrue(context.accept(packetInfo2, 1, targetIndex)) context.rewriteRtp(packetInfo2) Assert.assertTrue(context.rewriteRtcp(srPacket2)) Assert.assertEquals(packet2.ssrc, srPacket2.senderSsrc) @@ -883,7 +727,6 @@ class Vp9AdaptiveSourceProjectionTest { val initialState = RtpState(1, 10000, 1000000) val context = Vp9AdaptiveSourceProjectionContext( diagnosticContext, - payloadType, initialState, logger ) @@ -897,11 +740,7 @@ class Vp9AdaptiveSourceProjectionTest { for (i in 0..9999) { val packetInfo = generator.nextPacket() val packet = packetInfo.packetAs() - val accepted = context.accept( - packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), - targetIndex - ) + val accepted = context.accept(packetInfo, 0, targetIndex) if (packet.isStartOfFrame && packet.temporalLayerIndex == 0) { expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) } @@ -942,7 +781,6 @@ class Vp9AdaptiveSourceProjectionTest { val initialState = RtpState(1, 10000, 1000000) val context = Vp9AdaptiveSourceProjectionContext( diagnosticContext, - payloadType, initialState, logger ) @@ -957,7 +795,7 @@ class Vp9AdaptiveSourceProjectionTest { val packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + 0, targetIndex ) if (packet.isStartOfFrame && packet.temporalLayerIndex == 0) { @@ -995,13 +833,7 @@ class Vp9AdaptiveSourceProjectionTest { packetInfo = generator.nextPacket() packet = packetInfo.packetAs() } while (packet.temporalLayerIndex > targetIndex) - Assert.assertTrue( - context.accept( - packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), - targetIndex - ) - ) + Assert.assertTrue(context.accept(packetInfo, 0, targetIndex)) context.rewriteRtp(packetInfo) /* Allow any values after a gap. */ @@ -1018,7 +850,7 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + 0, targetIndex ) if (packet.isStartOfFrame && packet.temporalLayerIndex == 0) { @@ -1074,7 +906,6 @@ class Vp9AdaptiveSourceProjectionTest { val initialState = RtpState(1, 10000, 1000000) val context = Vp9AdaptiveSourceProjectionContext( diagnosticContext, - payloadType, initialState, logger ) @@ -1096,7 +927,7 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + 0, targetIndex ) if (packet.isStartOfFrame && packet.temporalLayerIndex == 0) { @@ -1136,7 +967,7 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + 0, targetIndex ) Assert.assertTrue(accepted) @@ -1154,7 +985,7 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + 0, RtpLayerDesc.SUSPENDED_INDEX ) Assert.assertFalse(accepted) @@ -1170,7 +1001,7 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + 0, targetIndex ) Assert.assertFalse(accepted) @@ -1187,7 +1018,7 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + 0, targetIndex ) Assert.assertFalse(accepted) @@ -1202,7 +1033,7 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - getIndex(0, packet.spatialLayerIndex, packet.temporalLayerIndex), + 0, targetIndex ) if (packet.isStartOfFrame && packet.temporalLayerIndex == 0) { @@ -1561,7 +1392,7 @@ class Vp9AdaptiveSourceProjectionTest { val srPacketBuilder = RtcpSrPacketBuilder() srPacketBuilder.rtcpHeader.senderSsrc = ssrc val siBuilder = srPacketBuilder.senderInfo - setSIBuilderNtp(srPacketBuilder.senderInfo, receivedTime.toEpochMilli()) + siBuilder.setNtpFromJavaTime(receivedTime.toEpochMilli()) siBuilder.rtpTimestamp = ts siBuilder.sendersOctetCount = packetCount.toLong() siBuilder.sendersOctetCount = octetCount.toLong() @@ -1594,16 +1425,6 @@ class Vp9AdaptiveSourceProjectionTest { // Dummy payload data "000000" ) - - /* TODO: move this to jitsi-rtp */ - const val JAVA_TO_NTP_EPOCH_OFFSET_SECS = 2208988800L - - fun setSIBuilderNtp(siBuilder: SenderInfoBuilder, wallTime: Long) { - val wallSecs = wallTime / 1000 - val wallMs = wallTime % 1000 - siBuilder.ntpTimestampMsw = wallSecs + JAVA_TO_NTP_EPOCH_OFFSET_SECS - siBuilder.ntpTimestampLsw = wallMs * (1L shl 32) / 1000 - } } } } diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilterTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilterTest.kt index d32cb07ed5..f2cb5a7ae7 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilterTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilterTest.kt @@ -423,12 +423,10 @@ internal class Vp9QualityFilterTest : ShouldSpec() { } lastTs = f.timestamp - val packetIndex = RtpLayerDesc.getIndex(f.ssrc.toInt(), f.spatialLayer, f.temporalLayer) - val result = filter.acceptFrame( frame = f, + incomingEncoding = f.ssrc.toInt(), externalTargetIndex = targetIndex, - incomingIndex = packetIndex, receivedTime = Instant.ofEpochMilli(ms) ) evaluator(f, result) @@ -445,7 +443,7 @@ private abstract class FrameGenerator : Iterator /** Generate a non-scalable series of VP9 frames, with a single keyframe at the start. */ private class SingleLayerFrameGenerator : FrameGenerator() { - private val totalPictures = 1000 + private val totalPictures = 10000 private var pictureCount = 0 override fun hasNext(): Boolean = pictureCount < totalPictures @@ -478,7 +476,7 @@ private class SingleLayerFrameGenerator : FrameGenerator() { /** Generate a temporally-scaled series of VP9 frames, with a single keyframe at the start. */ private class TemporallyScaledFrameGenerator : FrameGenerator() { - private val totalPictures = 1000 + private val totalPictures = 10000 private var pictureCount = 0 private var tl0Count = -1 @@ -521,7 +519,7 @@ private class TemporallyScaledFrameGenerator : FrameGenerator() { /** Generate a spatially-scaled series of VP9 frames, with full spatial dependencies and periodic keyframes. */ private class SVCFrameGenerator : FrameGenerator() { - private val totalPictures = 1000 + private val totalPictures = 10000 private var pictureCount = 0 private var frameCount = 0 private var tl0Count = -1 @@ -574,7 +572,7 @@ private class SVCFrameGenerator : FrameGenerator() { /** Generate a spatially-scaled series of VP9 frames, with K-SVC spatial dependencies and periodic keyframes. */ private class KSVCFrameGenerator : FrameGenerator() { - private val totalPictures = 1000 + private val totalPictures = 10000 private var pictureCount = 0 private var frameCount = 0 private var tl0Count = -1 diff --git a/rtp/pom.xml b/rtp/pom.xml index f03c567fca..001ead4e8c 100644 --- a/rtp/pom.xml +++ b/rtp/pom.xml @@ -55,6 +55,12 @@ 3.0.3 test + + jakarta.xml.bind + jakarta.xml.bind-api + 4.0.0 + test + diff --git a/rtp/spotbugs-exclude.xml b/rtp/spotbugs-exclude.xml index f7cf298ceb..7327eba7c3 100644 --- a/rtp/spotbugs-exclude.xml +++ b/rtp/spotbugs-exclude.xml @@ -18,6 +18,8 @@ + + diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpSrPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpSrPacket.kt index 750f92e590..b821f0250c 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpSrPacket.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpSrPacket.kt @@ -77,6 +77,12 @@ data class SenderInfoBuilder( var sendersPacketCount: Long = -1, var sendersOctetCount: Long = -1 ) { + fun setNtpFromJavaTime(javaTime: Long) { + val wallSecs = javaTime / 1000 + val wallMs = javaTime % 1000 + ntpTimestampMsw = wallSecs + JAVA_TO_NTP_EPOCH_OFFSET_SECS + ntpTimestampLsw = wallMs * (1L shl 32) / 1000 + } fun writeTo(buf: ByteArray, offset: Int) { SenderInfoParser.setNtpTimestampMsw(buf, offset, ntpTimestampMsw) @@ -85,6 +91,10 @@ data class SenderInfoBuilder( SenderInfoParser.setSendersPacketCount(buf, offset, sendersPacketCount) SenderInfoParser.setSendersOctetCount(buf, offset, sendersOctetCount) } + + companion object { + const val JAVA_TO_NTP_EPOCH_OFFSET_SECS = 2208988800L + } } /** diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpPacket.kt index 6741a65f8b..f0246eca04 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpPacket.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpPacket.kt @@ -432,8 +432,24 @@ open class RtpPacket( val dataLengthBytes: Int val totalLengthBytes: Int + + fun clone(): HeaderExtension = StandaloneHeaderExtension(this) + } + + @SuppressFBWarnings("CN_IMPLEMENTS_CLONE_BUT_NOT_CLONEABLE") + class StandaloneHeaderExtension(ext: HeaderExtension) : HeaderExtension { + override val buffer: ByteArray = ByteArray(ext.dataLengthBytes).also { + System.arraycopy(ext.buffer, ext.dataOffset, it, 0, ext.dataLengthBytes) + } + override val dataOffset = 0 + override var id = ext.id + override val dataLengthBytes: Int + get() = buffer.size + override val totalLengthBytes: Int + get() = buffer.size } + @SuppressFBWarnings("CN_IMPLEMENTS_CLONE_BUT_NOT_CLONEABLE") inner class EncodedHeaderExtension : HeaderExtension { private var currExtOffset: Int = 0 private var currExtLength: Int = 0 @@ -471,7 +487,7 @@ open class RtpPacket( } @SuppressFBWarnings( - value = ["EI_EXPOSE_REP"], + value = ["EI_EXPOSE_REP", "CN_IMPLEMENTS_CLONE_BUT_NOT_CLONEABLE"], justification = "We intentionally expose the internal buffer." ) inner class PendingHeaderExtension(override var id: Int, override val dataLengthBytes: Int) : HeaderExtension { diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtension.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtension.kt new file mode 100644 index 0000000000..679c6e3e83 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtension.kt @@ -0,0 +1,902 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.rtp.header_extensions + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.util.BitReader +import org.jitsi.rtp.util.BitWriter +import org.jitsi.utils.OrderedJsonObject +import org.json.simple.JSONAware + +/** + * The subset of the fields of an AV1 Dependency Descriptor that can be parsed statelessly. + */ +@SuppressFBWarnings("CN_IMPLEMENTS_CLONE_BUT_NOT_CLONEABLE") +open class Av1DependencyDescriptorStatelessSubset( + val startOfFrame: Boolean, + val endOfFrame: Boolean, + var frameDependencyTemplateId: Int, + var frameNumber: Int, + + val newTemplateDependencyStructure: Av1TemplateDependencyStructure?, +) { + open fun clone(): Av1DependencyDescriptorStatelessSubset { + return Av1DependencyDescriptorStatelessSubset( + startOfFrame = startOfFrame, + endOfFrame = endOfFrame, + frameDependencyTemplateId = frameDependencyTemplateId, + frameNumber = frameNumber, + newTemplateDependencyStructure = newTemplateDependencyStructure?.clone() + ) + } +} + +/** + * The AV1 Dependency Descriptor header extension, as defined in https://aomediacodec.github.io/av1-rtp-spec/#appendix + */ +@SuppressFBWarnings("CN_IMPLEMENTS_CLONE_BUT_NOT_CLONEABLE") +class Av1DependencyDescriptorHeaderExtension( + startOfFrame: Boolean, + endOfFrame: Boolean, + frameDependencyTemplateId: Int, + frameNumber: Int, + + newTemplateDependencyStructure: Av1TemplateDependencyStructure?, + + var activeDecodeTargetsBitmask: Int?, + + val customDtis: List?, + val customFdiffs: List?, + val customChains: List?, + + val structure: Av1TemplateDependencyStructure +) : Av1DependencyDescriptorStatelessSubset( + startOfFrame, + endOfFrame, + frameDependencyTemplateId, + frameNumber, + newTemplateDependencyStructure +), + JSONAware { + val frameInfo: FrameInfo by lazy { + val templateIndex = (frameDependencyTemplateId + 64 - structure.templateIdOffset) % 64 + if (templateIndex >= structure.templateCount) { + val maxTemplate = (structure.templateIdOffset + structure.templateCount - 1) % 64 + throw Av1DependencyException( + "Invalid template ID $frameDependencyTemplateId. " + + "Should be in range ${structure.templateIdOffset} .. $maxTemplate. " + + "Missed a keyframe?" + ) + } + val templateVal = structure.templateInfo[templateIndex] + + FrameInfo( + spatialId = templateVal.spatialId, + temporalId = templateVal.temporalId, + dti = customDtis ?: templateVal.dti, + fdiff = customFdiffs ?: templateVal.fdiff, + chains = customChains ?: templateVal.chains + ) + } + + val encodedLength: Int + get() = (unpaddedLengthBits + 7) / 8 + + private val unpaddedLengthBits: Int + get() { + var length = 24 + if (newTemplateDependencyStructure != null || + activeDecodeTargetsBitmask != null || + customDtis != null || + customFdiffs != null || + customChains != null + ) { + length += 5 + } + if (newTemplateDependencyStructure != null) { + length += newTemplateDependencyStructure.unpaddedLengthBits + } + if (activeDecodeTargetsBitmask != null && + ( + newTemplateDependencyStructure == null || + activeDecodeTargetsBitmask != ((1 shl newTemplateDependencyStructure.decodeTargetCount) - 1) + ) + ) { + length += structure.decodeTargetCount + } + if (customDtis != null) { + length += 2 * structure.decodeTargetCount + } + if (customFdiffs != null) { + customFdiffs.forEach { + length += 2 + it.bitsForFdiff() + } + length += 2 + } + if (customChains != null) { + length += 8 * customChains.size + } + + return length + } + + override fun clone(): Av1DependencyDescriptorHeaderExtension { + val structureCopy = structure.clone() + val newStructure = if (newTemplateDependencyStructure == null) null else structureCopy + return Av1DependencyDescriptorHeaderExtension( + startOfFrame, + endOfFrame, + frameDependencyTemplateId, + frameNumber, + newStructure, + activeDecodeTargetsBitmask, + // These values are not mutable so it's safe to copy them by reference + customDtis, + customFdiffs, + customChains, + structureCopy + ) + } + + fun write(ext: RtpPacket.HeaderExtension) = write(ext.buffer, ext.dataOffset, ext.dataLengthBytes) + + fun write(buffer: ByteArray, offset: Int, length: Int) { + check(length <= encodedLength) { + "Cannot write AV1 DD to buffer: buffer length $length must be at least $encodedLength" + } + val writer = BitWriter(buffer, offset, length) + + writeMandatoryDescriptorFields(writer) + + if (newTemplateDependencyStructure != null || + activeDecodeTargetsBitmask != null || + customDtis != null || + customFdiffs != null || + customChains != null + ) { + writeOptionalDescriptorFields(writer) + writePadding(writer) + } else { + check(length == 3) { + "AV1 DD without optional descriptors must be 3 bytes in length" + } + } + } + + private fun writeMandatoryDescriptorFields(writer: BitWriter) { + writer.writeBit(startOfFrame) + writer.writeBit(endOfFrame) + writer.writeBits(6, frameDependencyTemplateId) + writer.writeBits(16, frameNumber) + } + + private fun writeOptionalDescriptorFields(writer: BitWriter) { + val templateDependencyStructurePresent = newTemplateDependencyStructure != null + val activeDecodeTargetsPresent = activeDecodeTargetsBitmask != null && + ( + newTemplateDependencyStructure == null || + activeDecodeTargetsBitmask != ((1 shl newTemplateDependencyStructure.decodeTargetCount) - 1) + ) + + val customDtisFlag = customDtis != null + val customFdiffsFlag = customFdiffs != null + val customChainsFlag = customChains != null + + writer.writeBit(templateDependencyStructurePresent) + writer.writeBit(activeDecodeTargetsPresent) + writer.writeBit(customDtisFlag) + writer.writeBit(customFdiffsFlag) + writer.writeBit(customChainsFlag) + + if (templateDependencyStructurePresent) { + newTemplateDependencyStructure!!.write(writer) + } + + if (activeDecodeTargetsPresent) { + writeActiveDecodeTargets(writer) + } + + if (customDtisFlag) { + writeFrameDtis(writer) + } + + if (customFdiffsFlag) { + writeFrameFdiffs(writer) + } + + if (customChainsFlag) { + writeFrameChains(writer) + } + } + + private fun writeActiveDecodeTargets(writer: BitWriter) { + writer.writeBits(structure.decodeTargetCount, activeDecodeTargetsBitmask!!) + } + + private fun writeFrameDtis(writer: BitWriter) { + customDtis!!.forEach { dti -> + writer.writeBits(2, dti.dti) + } + } + + private fun writeFrameFdiffs(writer: BitWriter) { + customFdiffs!!.forEach { fdiff -> + val bits = fdiff.bitsForFdiff() + writer.writeBits(2, bits / 4) + writer.writeBits(bits, fdiff - 1) + } + writer.writeBits(2, 0) + } + + private fun writeFrameChains(writer: BitWriter) { + customChains!!.forEach { chain -> + writer.writeBits(8, chain) + } + } + + private fun writePadding(writer: BitWriter) { + writer.writeBits(writer.remainingBits, 0) + } + + override fun toJSONString(): String { + return OrderedJsonObject().apply { + put("startOfFrame", startOfFrame) + put("endOfFrame", endOfFrame) + put("frameDependencyTemplateId", frameDependencyTemplateId) + put("frameNumber", frameNumber) + newTemplateDependencyStructure?.let { put("templateStructure", it) } + customDtis?.let { put("customDTIs", it) } + customFdiffs?.let { put("customFdiffs", it) } + customChains?.let { put("customChains", it) } + }.toJSONString() + } + + override fun toString(): String = toJSONString() +} + +fun Int.bitsForFdiff() = when { + this <= 0x10 -> 4 + this <= 0x100 -> 8 + this <= 0x1000 -> 12 + else -> throw IllegalArgumentException("Invalid FDiff value $this") +} + +/** + * The template information about a stream described by AV1 dependency descriptors. This is carried in the + * first packet of a codec video sequence (i.e. the first packet of a keyframe), and is necessary to interpret + * dependency descriptors carried in subsequent packets of the sequence. + */ +@SuppressFBWarnings("CN_IMPLEMENTS_CLONE_BUT_NOT_CLONEABLE") +class Av1TemplateDependencyStructure( + var templateIdOffset: Int, + val templateInfo: List, + val decodeTargetInfo: List, + val maxRenderResolutions: List, + val maxSpatialId: Int, + val maxTemporalId: Int +) : JSONAware { + val templateCount + get() = templateInfo.size + + val decodeTargetCount + get() = decodeTargetInfo.size + + val chainCount: Int = + templateInfo.first().chains.size + + init { + check(templateInfo.all { it.chains.size == chainCount }) { + "Templates have inconsistent chain sizes" + } + check(templateInfo.all { it.temporalId <= maxTemporalId }) { + "Incorrect maxTemporalId" + } + check(maxRenderResolutions.isEmpty() || maxRenderResolutions.size == maxSpatialId + 1) { + "Non-zero number of render resolutions does not match maxSpatialId" + } + check(templateInfo.all { it.spatialId <= maxSpatialId }) { + "Incorrect maxSpatialId" + } + } + + val unpaddedLengthBits: Int + get() { + var length = 6 // Template ID offset + + length += 5 // DT Count - 1 + + length += templateCount * 2 // templateLayers + length += templateCount * decodeTargetCount * 2 // TemplateDTIs + templateInfo.forEach { + length += it.fdiffCnt * 5 + 1 // TemplateFDiffs + } + // TemplateChains + length += nsBits(decodeTargetCount + 1, chainCount) + if (chainCount > 0) { + decodeTargetInfo.forEach { + length += nsBits(chainCount, it.protectedBy) + } + length += templateCount * chainCount * 4 + } + length += 1 // ResolutionsPresent + length += maxRenderResolutions.size * 32 // RenderResolutions + + return length + } + + fun clone(): Av1TemplateDependencyStructure { + return Av1TemplateDependencyStructure( + templateIdOffset, + // These objects are not mutable so it's safe to copy them by reference + templateInfo, + decodeTargetInfo, + maxRenderResolutions, + maxSpatialId, + maxTemporalId + ) + } + + fun write(writer: BitWriter) { + writer.writeBits(6, templateIdOffset) + + writer.writeBits(5, decodeTargetCount - 1) + + writeTemplateLayers(writer) + writeTemplateDtis(writer) + writeTemplateFdiffs(writer) + writeTemplateChains(writer) + + writeRenderResolutions(writer) + } + + private fun writeTemplateLayers(writer: BitWriter) { + check(templateInfo[0].spatialId == 0 && templateInfo[0].temporalId == 0) { + "First template must have spatial and temporal IDs 0/0, but found " + + "${templateInfo[0].spatialId}/${templateInfo[0].temporalId}" + } + for (templateNum in 1 until templateInfo.size) { + val layerIdc = when { + templateInfo[templateNum].spatialId == templateInfo[templateNum - 1].spatialId && + templateInfo[templateNum].temporalId == templateInfo[templateNum - 1].temporalId -> + 0 + templateInfo[templateNum].spatialId == templateInfo[templateNum - 1].spatialId && + templateInfo[templateNum].temporalId == templateInfo[templateNum - 1].temporalId + 1 -> + 1 + templateInfo[templateNum].spatialId == templateInfo[templateNum - 1].spatialId + 1 && + templateInfo[templateNum].temporalId == 0 -> + 2 + else -> + throw IllegalStateException( + "Template $templateNum with spatial and temporal IDs " + + "${templateInfo[templateNum].spatialId}/${templateInfo[templateNum].temporalId} " + + "cannot follow template ${templateNum - 1} with spatial and temporal IDs " + + "${templateInfo[templateNum - 1].spatialId}/${templateInfo[templateNum - 1].temporalId}." + ) + } + writer.writeBits(2, layerIdc) + } + writer.writeBits(2, 3) + } + + private fun writeTemplateDtis(writer: BitWriter) { + templateInfo.forEach { t -> + t.dti.forEach { dti -> + writer.writeBits(2, dti.dti) + } + } + } + + private fun writeTemplateFdiffs(writer: BitWriter) { + templateInfo.forEach { t -> + t.fdiff.forEach { fdiff -> + writer.writeBit(true) + writer.writeBits(4, fdiff - 1) + } + writer.writeBit(false) + } + } + + private fun writeTemplateChains(writer: BitWriter) { + writer.writeNs(decodeTargetCount + 1, chainCount) + decodeTargetInfo.forEach { + writer.writeNs(chainCount, it.protectedBy) + } + templateInfo.forEach { t -> + t.chains.forEach { chain -> + writer.writeBits(4, chain) + } + } + } + + private fun writeRenderResolutions(writer: BitWriter) { + if (maxRenderResolutions.isEmpty()) { + writer.writeBit(false) + } else { + writer.writeBit(true) + maxRenderResolutions.forEach { r -> + writer.writeBits(16, r.width - 1) + writer.writeBits(16, r.height - 1) + } + } + } + + /** Return whether, in this structure, it's possible to switch from DT [fromDt] to DT [toDt] + * without a keyframe. + * Note this makes certain assumptions about the encoding structure. + */ + fun canSwitchWithoutKeyframe(fromDt: Int, toDt: Int): Boolean = templateInfo.any { + it.hasInterPictureDependency() && it.dti[fromDt] != DTI.NOT_PRESENT && it.dti[toDt] == DTI.SWITCH + } + + /** Given that we are sending packets for a given DT, return a decodeTargetBitmask corresponding to all DTs + * contained in that DT. + */ + fun getDtBitmaskForDt(dt: Int): Int { + var mask = (1 shl decodeTargetCount) - 1 + templateInfo.forEach { frameInfo -> + frameInfo.dti.forEachIndexed { i, dti -> + if (frameInfo.dti[dt] == DTI.NOT_PRESENT && dti != DTI.NOT_PRESENT) { + mask = mask and (1 shl i).inv() + } + } + } + return mask + } + + override fun toJSONString(): String { + return OrderedJsonObject().apply { + put("templateIdOffset", templateIdOffset) + put("templateInfo", templateInfo.toIndexedMap()) + put("decodeTargetInfo", decodeTargetInfo.toIndexedMap()) + if (maxRenderResolutions.isNotEmpty()) { + put("maxRenderResolutions", maxRenderResolutions.toIndexedMap()) + } + put("maxSpatialId", maxSpatialId) + put("maxTemporalId", maxTemporalId) + }.toJSONString() + } + + override fun toString() = toJSONString() +} + +fun nsBits(n: Int, v: Int): Int { + require(n > 0) + if (n == 1) return 0 + var w = 0 + var x = n + while (x != 0) { + x = x shr 1 + w++ + } + val m = (1 shl w) - n + if (v < m) return w - 1 + return w +} + +class Av1DependencyDescriptorReader( + buffer: ByteArray, + offset: Int, + val length: Int, +) { + private var startOfFrame = false + private var endOfFrame = false + private var frameDependencyTemplateId = 0 + private var frameNumber = 0 + + private var customDtis: List? = null + private var customFdiffs: List? = null + private var customChains: List? = null + + private var localTemplateDependencyStructure: Av1TemplateDependencyStructure? = null + private var templateDependencyStructure: Av1TemplateDependencyStructure? = null + + private var activeDecodeTargetsBitmask: Int? = null + + private val reader = BitReader(buffer, offset, length) + + constructor(ext: RtpPacket.HeaderExtension) : + this(ext.buffer, ext.dataOffset, ext.dataLengthBytes) + + /** Parse those parts of the dependency descriptor that can be parsed statelessly, i.e. without an external + * template dependency structure. The returned object will not be a complete representation of the + * dependency descriptor, because some fields need the external structure to be parseable. + */ + fun parseStateless(): Av1DependencyDescriptorStatelessSubset { + reset() + readMandatoryDescriptorFields() + + if (length > 3) { + val templateDependencyStructurePresent = reader.bitAsBoolean() + + /* activeDecodeTargetsPresent, customDtisFlag, customFdiffsFlag, and customChainsFlag; + * none of these fields are parseable statelessly. + */ + reader.skipBits(4) + + if (templateDependencyStructurePresent) { + localTemplateDependencyStructure = readTemplateDependencyStructure() + } + } + return Av1DependencyDescriptorStatelessSubset( + startOfFrame, + endOfFrame, + frameDependencyTemplateId, + frameNumber, + localTemplateDependencyStructure, + ) + } + + /** Parse the dependency descriptor in the context of [dep], the currently-applicable template dependency + * structure.*/ + fun parse(dep: Av1TemplateDependencyStructure?): Av1DependencyDescriptorHeaderExtension { + reset() + readMandatoryDescriptorFields() + if (length > 3) { + readExtendedDescriptorFields(dep) + } else { + if (dep == null) { + throw Av1DependencyException("No external dependency structure specified for non-first packet") + } + templateDependencyStructure = dep + } + return Av1DependencyDescriptorHeaderExtension( + startOfFrame, + endOfFrame, + frameDependencyTemplateId, + frameNumber, + localTemplateDependencyStructure, + activeDecodeTargetsBitmask, + customDtis, + customFdiffs, + customChains, + templateDependencyStructure!! + ) + } + + private fun reset() = reader.reset() + + private fun readMandatoryDescriptorFields() { + startOfFrame = reader.bitAsBoolean() + endOfFrame = reader.bitAsBoolean() + frameDependencyTemplateId = reader.bits(6) + frameNumber = reader.bits(16) + } + + private fun readExtendedDescriptorFields(dep: Av1TemplateDependencyStructure?) { + val templateDependencyStructurePresent = reader.bitAsBoolean() + val activeDecodeTargetsPresent = reader.bitAsBoolean() + val customDtisFlag = reader.bitAsBoolean() + val customFdiffsFlag = reader.bitAsBoolean() + val customChainsFlag = reader.bitAsBoolean() + + if (templateDependencyStructurePresent) { + localTemplateDependencyStructure = readTemplateDependencyStructure() + templateDependencyStructure = localTemplateDependencyStructure + } else { + if (dep == null) { + throw Av1DependencyException("No external dependency structure specified for non-first packet") + } + templateDependencyStructure = dep + } + if (activeDecodeTargetsPresent) { + activeDecodeTargetsBitmask = reader.bits(templateDependencyStructure!!.decodeTargetCount) + } else if (templateDependencyStructurePresent) { + activeDecodeTargetsBitmask = (1 shl templateDependencyStructure!!.decodeTargetCount) - 1 + } + + if (customDtisFlag) { + customDtis = readFrameDtis() + } + if (customFdiffsFlag) { + customFdiffs = readFrameFdiffs() + } + if (customChainsFlag) { + customChains = readFrameChains() + } + } + + /* Data for template dependency structure */ + private var templateIdOffset: Int = 0 + private val templateInfo = mutableListOf() + private val decodeTargetInfo = mutableListOf() + private val maxRenderResolutions = mutableListOf() + + private var dtCnt = 0 + + private fun resetDependencyStructureInfo() { + /* These fields are assembled incrementally when parsing a dependency structure; reset them + * in case we're running a parser more than once. + */ + templateCnt = 0 + templateInfo.clear() + decodeTargetInfo.clear() + maxRenderResolutions.clear() + } + + private fun readTemplateDependencyStructure(): Av1TemplateDependencyStructure { + resetDependencyStructureInfo() + + templateIdOffset = reader.bits(6) + + val dtCntMinusOne = reader.bits(5) + dtCnt = dtCntMinusOne + 1 + + readTemplateLayers() + readTemplateDtis() + readTemplateFdiffs() + readTemplateChains() + readDecodeTargetLayers() + + val resolutionsPresent = reader.bitAsBoolean() + + if (resolutionsPresent) { + readRenderResolutions() + } + + return Av1TemplateDependencyStructure( + templateIdOffset, + templateInfo.toList(), + decodeTargetInfo.toList(), + maxRenderResolutions.toList(), + maxSpatialId, + maxTemporalId + ) + } + + private var templateCnt = 0 + private var maxSpatialId = 0 + private var maxTemporalId = 0 + + @SuppressFBWarnings( + value = ["SF_SWITCH_NO_DEFAULT"], + justification = "Artifact of generated Kotlin code." + ) + private fun readTemplateLayers() { + var temporalId = 0 + var spatialId = 0 + + var nextLayerIdc: Int + do { + templateInfo.add(templateCnt, TemplateFrameInfo(spatialId, temporalId)) + templateCnt++ + nextLayerIdc = reader.bits(2) + if (nextLayerIdc == 1) { + temporalId++ + if (maxTemporalId < temporalId) { + maxTemporalId = temporalId + } + } else if (nextLayerIdc == 2) { + temporalId = 0 + spatialId++ + } + } while (nextLayerIdc != 3) + + check(templateInfo.size == templateCnt) + + maxSpatialId = spatialId + } + + private fun readTemplateDtis() { + for (templateIndex in 0 until templateCnt) { + for (dtIndex in 0 until dtCnt) { + templateInfo[templateIndex].dti.add(dtIndex, DTI.fromInt(reader.bits(2))) + } + } + } + + private fun readFrameDtis(): List { + return List(templateDependencyStructure!!.decodeTargetCount) { + DTI.fromInt(reader.bits(2)) + } + } + + private fun readTemplateFdiffs() { + for (templateIndex in 0 until templateCnt) { + var fdiffCnt = 0 + var fdiffFollowsFlag = reader.bitAsBoolean() + while (fdiffFollowsFlag) { + val fdiffMinusOne = reader.bits(4) + templateInfo[templateIndex].fdiff.add(fdiffCnt, fdiffMinusOne + 1) + fdiffCnt++ + fdiffFollowsFlag = reader.bitAsBoolean() + } + check(fdiffCnt == templateInfo[templateIndex].fdiffCnt) + } + } + + private fun readFrameFdiffs(): List { + return buildList { + var nextFdiffSize = reader.bits(2) + while (nextFdiffSize != 0) { + val fdiffMinus1 = reader.bits(4 * nextFdiffSize) + add(fdiffMinus1 + 1) + nextFdiffSize = reader.bits(2) + } + } + } + + private fun readTemplateChains() { + val chainCount = reader.ns(dtCnt + 1) + if (chainCount != 0) { + for (dtIndex in 0 until dtCnt) { + decodeTargetInfo.add(dtIndex, DecodeTargetInfo(reader.ns(chainCount))) + } + for (templateIndex in 0 until templateCnt) { + for (chainIndex in 0 until chainCount) { + templateInfo[templateIndex].chains.add(chainIndex, reader.bits(4)) + } + check(templateInfo[templateIndex].chains.size == chainCount) + } + } + } + + private fun readFrameChains(): List { + return List(templateDependencyStructure!!.chainCount) { + reader.bits(8) + } + } + + private fun readDecodeTargetLayers() { + for (dtIndex in 0 until dtCnt) { + var spatialId = 0 + var temporalId = 0 + for (templateIndex in 0 until templateCnt) { + if (templateInfo[templateIndex].dti[dtIndex] != DTI.NOT_PRESENT) { + if (templateInfo[templateIndex].spatialId > spatialId) { + spatialId = templateInfo[templateIndex].spatialId + } + if (templateInfo[templateIndex].temporalId > temporalId) { + temporalId = templateInfo[templateIndex].temporalId + } + } + } + decodeTargetInfo[dtIndex].spatialId = spatialId + decodeTargetInfo[dtIndex].temporalId = temporalId + } + check(decodeTargetInfo.size == dtCnt) + } + + private fun readRenderResolutions() { + for (spatialId in 0..maxSpatialId) { + val widthMinus1 = reader.bits(16) + val heightMinus1 = reader.bits(16) + maxRenderResolutions.add(spatialId, Resolution(widthMinus1 + 1, heightMinus1 + 1)) + } + } +} + +open class FrameInfo( + val spatialId: Int, + val temporalId: Int, + open val dti: List, + open val fdiff: List, + open val chains: List +) : JSONAware { + val fdiffCnt + get() = fdiff.size + + override fun equals(other: Any?): Boolean { + if (other !is FrameInfo) { + return false + } + return other.spatialId == spatialId && + other.temporalId == temporalId && + other.dti == dti && + other.fdiff == fdiff && + other.chains == chains + } + + override fun hashCode(): Int { + var result = spatialId + result = 31 * result + temporalId + result = 31 * result + dti.hashCode() + result = 31 * result + fdiff.hashCode() + result = 31 * result + chains.hashCode() + return result + } + + override fun toString(): String { + return "spatialId=$spatialId, temporalId=$temporalId, dti=$dti, fdiff=$fdiff, chains=$chains" + } + + override fun toJSONString(): String { + return OrderedJsonObject().apply { + put("spatialId", spatialId) + put("temporalId", temporalId) + put("dti", dti) + put("fdiff", fdiff) + put("chains", chains) + }.toJSONString() + } + + /** Whether the frame has a dependency on a frame earlier than this "picture", the other frames of this + * temporal moment. If it doesn't, it's probably part of a keyframe, and not part of the regular structure. + * Note this makes assumptions about the scalability structure. + */ + fun hasInterPictureDependency(): Boolean = fdiff.any { it > spatialId } + + val dtisPresent: List + get() = dti.withIndex().filter { (_, dti) -> dti != DTI.NOT_PRESENT }.map { (i, _) -> i } +} + +/* The only thing this changes from its parent class is to make the lists mutable, so the parent equals() is fine. */ +@SuppressFBWarnings("EQ_DOESNT_OVERRIDE_EQUALS") +class TemplateFrameInfo( + spatialId: Int, + temporalId: Int, + override val dti: MutableList = mutableListOf(), + override val fdiff: MutableList = mutableListOf(), + override val chains: MutableList = mutableListOf() +) : FrameInfo(spatialId, temporalId, dti, fdiff, chains) + +class DecodeTargetInfo( + val protectedBy: Int +) : JSONAware { + /** Todo: only want to be able to set these from the constructor */ + var spatialId: Int = -1 + var temporalId: Int = -1 + + override fun toJSONString(): String { + return OrderedJsonObject().apply { + put("protectedBy", protectedBy) + put("spatialId", spatialId) + put("temporalId", temporalId) + }.toJSONString() + } +} + +data class Resolution( + val width: Int, + val height: Int +) : JSONAware { + override fun toJSONString(): String { + return OrderedJsonObject().apply { + put("width", width) + put("height", height) + }.toJSONString() + } +} + +/** Decode target indication */ +enum class DTI(val dti: Int) { + NOT_PRESENT(0), + DISCARDABLE(1), + SWITCH(2), + REQUIRED(3); + + companion object { + private val map = DTI.values().associateBy(DTI::dti) + fun fromInt(type: Int) = map[type] ?: throw java.lang.IllegalArgumentException("Bad DTI $type") + } + + fun toShortString(): String { + return when (this) { + NOT_PRESENT -> "N" + DISCARDABLE -> "D" + SWITCH -> "S" + REQUIRED -> "R" + } + } +} + +fun List.toShortString(): String { + return joinToString(separator = "") { it.toShortString() } +} + +class Av1DependencyException(msg: String) : RuntimeException(msg) + +fun List.toIndexedMap(): Map = mapIndexed { i, t -> i to t }.toMap() diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/util/BitReader.kt b/rtp/src/main/kotlin/org/jitsi/rtp/util/BitReader.kt new file mode 100644 index 0000000000..d141b7aed4 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/util/BitReader.kt @@ -0,0 +1,105 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.util + +import kotlin.experimental.and + +/** + * Read individual bits, and unaligned sets of bits, from a [ByteArray], with an incrementing offset. + */ +/* TODO: put this in jitsi-utils? */ +class BitReader(val buf: ByteArray, private val byteOffset: Int = 0, private val byteLength: Int = buf.size) { + private var offset = byteOffset * 8 + private val byteBound = byteOffset + byteLength + + /** Read a single bit from the buffer, as a boolean, incrementing the offset. */ + fun bitAsBoolean(): Boolean { + val byteIdx = offset / 8 + val bitIdx = offset % 8 + check(byteIdx < byteBound) { + "offset $offset ($byteIdx/$bitIdx) invalid in buffer of length $byteLength after offset $byteOffset" + } + val byte = buf[byteIdx] + val mask = (1 shl (7 - bitIdx)).toByte() + offset++ + + return (byte and mask) != 0.toByte() + } + + /** Read a single bit from the buffer, as an integer, incrementing the offset. */ + fun bit() = if (bitAsBoolean()) 1 else 0 + + /** Read [n] bits from the buffer, returning them as an unsigned integer. */ + fun bits(n: Int): Int { + require(n < Int.SIZE_BITS) + + var ret = 0 + + /* TODO: optimize this */ + repeat(n) { + ret = ret shl 1 + ret = ret or bit() + } + + return ret + } + + /** Read [n] bits from the buffer, returning them as an unsigned long. */ + fun bitsLong(n: Int): Long { + require(n < Long.SIZE_BITS) + + var ret = 0L + + /* TODO: optimize this */ + repeat(n) { + ret = ret shl 1 + ret = ret or bit().toLong() + } + + return ret + } + + /** Skip forward [n] bits in the buffer. */ + fun skipBits(n: Int) { + offset += n + } + + /** Read a non-symmetric unsigned integer with max *value* [n] from the buffer. + * (Note: *not* the number of bits.) + * See https://aomediacodec.github.io/av1-rtp-spec/#a82-syntax + */ + fun ns(n: Int): Int { + var w = 0 + var x = n + while (x != 0) { + x = x shr 1 + w++ + } + val m = (1 shl w) - n + val v = bits(w - 1) + if (v < m) { + return v + } + val extraBit = bit() + return (v shl 1) - m + extraBit + } + + /** Reset the reader to the beginning of the buffer */ + fun reset() { + offset = byteOffset * 8 + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/util/BitWriter.kt b/rtp/src/main/kotlin/org/jitsi/rtp/util/BitWriter.kt new file mode 100644 index 0000000000..9601d0db6f --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/util/BitWriter.kt @@ -0,0 +1,73 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.util + +import java.util.* +import kotlin.experimental.or + +/** + * Write individual bits, and unaligned sets of bits, to a [ByteArray], with an incrementing offset. + */ +class BitWriter(val buf: ByteArray, val byteOffset: Int = 0, private val byteLength: Int = buf.size) { + private var offset = byteOffset * 8 + private val byteBound = byteOffset + byteLength + + init { + Arrays.fill(buf, byteOffset, byteBound, 0) + } + + fun writeBit(value: Boolean) { + val byteIdx = offset / 8 + val bitIdx = offset % 8 + check(byteIdx < byteBound) { + "offset $offset ($byteIdx/$bitIdx) invalid in buffer of length $byteLength after offset $byteOffset" + } + + if (value) { + buf[byteIdx] = buf[byteIdx] or (1 shl (7 - bitIdx)).toByte() + } + offset++ + } + + fun writeBits(bits: Int, value: Int) { + check(value < (1 shl bits)) { + "value $value cannot be represented in $bits bits" + } + repeat(bits) { i -> + writeBit((value and (1 shl (bits - i - 1))) != 0) + } + } + + fun writeNs(n: Int, v: Int) { + if (n == 1) return + var w = 0 + var x = n + while (x != 0) { + x = x shr 1 + w++ + } + val m = (1 shl w) - n + if (v < m) { + writeBits(w - 1, v) + } else { + writeBits(w, v + m) + } + } + + val remainingBits + get() = byteBound * 8 - offset +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtensionTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtensionTest.kt new file mode 100644 index 0000000000..4a69bb9c93 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtensionTest.kt @@ -0,0 +1,549 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.rtp.header_extensions + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.assertions.withClue +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import jakarta.xml.bind.DatatypeConverter.parseHexBinary + +@SuppressFBWarnings( + value = ["NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"], + justification = "Use of pointer after shouldNotBeNull test." +) +class Av1DependencyDescriptorHeaderExtensionTest : ShouldSpec() { + override fun isolationMode(): IsolationMode = IsolationMode.InstancePerLeaf + + /* Headers generated by Chrome 110 sending AV1 in its default configuration - L1T1 */ + val descL1T1 = parseHexBinary("80000180003a410180ef808680") + + val shortDesc = parseHexBinary("400001") + + val descL1T3 = parseHexBinary("800001800214eaa860414d141020842701df010d") + + /* Header generated by Chrome 112 sending AV1 with L3T3 set. */ + val descL3T3 = parseHexBinary( + "d0013481e81485214eafffaaaa863cf0430c10c302afc0aaa0063c00430010c002a000a800060000" + + "40001d954926e082b04a0941b820ac1282503157f974000ca864330e222222eca8655304224230ec" + + "a87753013f00b3027f016704ff02cf" + ) + + val midDescScalable = parseHexBinary("d10146401c") + + val midDescScalable2 = parseHexBinary("c203ce581d30100000") + val longForMid2 = parseHexBinary( + "8003ca80081485214eaaaaa8000600004000100002aa80a8000600004000100002a000a8000600004" + + "00016d549241b5524906d54923157e001974ca864330e222396eca8655304224390eca87753013f00b3027f016704ff02cf" + ) + + val descS3T3 = parseHexBinary( + "c1000180081485214ea000a8000600004000100002a000a8000600004000100002a000a8000600004" + + "0001d954926caa493655248c55fe5d00032a190cc38e58803b2a1954c10e10843b2a1dd4c01dc010803bc0218077c0434" + ) + + val descL3T3Key = parseHexBinary( + "8f008581e81485214eaaaaa8000600004000100002aa80a8000600004000100002a000a80006000040" + + "0016d549241b5524906d54923157e001974ca864330e222396eca8655304224390eca87753013f00b3027f016704ff02cf" + ) + + /* As of Chrome version 111, it doesn't support L3T3_KEY_SHIFT, but it does support L2T2_KEY_SHIFT, so test that. */ + val descL2T2KeyShift = parseHexBinary( + "8700ed80e3061eaa82804028280514d14134518010a091889a09409fc059c13fc0b3c0" + ) + + init { + context("AV1 Dependency Descriptors") { + context("a descriptor with a single-layer dependency structure") { + val ld1r = Av1DependencyDescriptorReader(descL1T1, 0, descL1T1.size) + val ld1 = ld1r.parse(null) + should("be parsed properly") { + ld1.startOfFrame shouldBe true + ld1.endOfFrame shouldBe false + ld1.frameNumber shouldBe 0x0001 + + val structure = ld1.newTemplateDependencyStructure + structure shouldNotBe null + structure!!.decodeTargetCount shouldBe 1 + structure.maxTemporalId shouldBe 0 + structure.maxSpatialId shouldBe 0 + } + should("be parseable statelessly") { + val ld1s = ld1r.parseStateless() + ld1s.startOfFrame shouldBe true + ld1s.endOfFrame shouldBe false + ld1s.frameNumber shouldBe 0x0001 + + val structure = ld1s.newTemplateDependencyStructure + structure shouldNotBe null + structure!!.decodeTargetCount shouldBe 1 + } + should("calculate correct frame info") { + val ld1i = ld1.frameInfo + ld1i.spatialId shouldBe 0 + ld1i.temporalId shouldBe 0 + } + should("Calculate its own length properly") { + ld1.encodedLength shouldBe descL1T1.size + } + should("Be re-encoded to the same bytes") { + val buf = ByteArray(ld1.encodedLength) + ld1.write(buf, 0, buf.size) + buf shouldBe descL1T1 + } + } + context("a descriptor with a scalable dependency structure") { + val ldsr = Av1DependencyDescriptorReader(descL3T3, 0, descL3T3.size) + val lds = ldsr.parse(null) + should("be parsed properly") { + lds.startOfFrame shouldBe true + lds.endOfFrame shouldBe true + lds.frameNumber shouldBe 0x0134 + lds.activeDecodeTargetsBitmask shouldBe 0x1ff + + val structure = lds.newTemplateDependencyStructure + structure shouldNotBe null + structure!!.decodeTargetCount shouldBe 9 + structure.maxTemporalId shouldBe 2 + structure.maxSpatialId shouldBe 2 + } + should("calculate correct frame info") { + val ldsi = lds.frameInfo + ldsi.spatialId shouldBe 0 + ldsi.temporalId shouldBe 0 + } + should("calculate correctly whether layer switching needs keyframes") { + val structure = lds.newTemplateDependencyStructure!! + for (fromS in 0..2) { + for (fromT in 0..2) { + val fromDT = 3 * fromS + fromT + for (toS in 0..2) { + for (toT in 0..2) { + val toDT = 3 * toS + toT + /* With this structure you can switch down spatial layers, or to other temporal + * layers within the same spatial layer, without a keyframe; but switching up + * spatial layers needs a keyframe. + */ + withClue("from DT $fromDT to DT $toDT") { + if (fromS >= toS) { + structure.canSwitchWithoutKeyframe( + fromDt = fromDT, + toDt = toDT + ) shouldBe true + } else { + structure.canSwitchWithoutKeyframe( + fromDt = fromDT, + toDt = toDT + ) shouldBe false + } + } + } + } + } + } + } + should("calculate DTI bitmasks corresponding to a given DT") { + val structure = lds.newTemplateDependencyStructure!! + structure.getDtBitmaskForDt(0) shouldBe 0b000000001 + structure.getDtBitmaskForDt(1) shouldBe 0b000000011 + structure.getDtBitmaskForDt(2) shouldBe 0b000000111 + structure.getDtBitmaskForDt(3) shouldBe 0b000001001 + structure.getDtBitmaskForDt(4) shouldBe 0b000011011 + structure.getDtBitmaskForDt(5) shouldBe 0b000111111 + structure.getDtBitmaskForDt(6) shouldBe 0b001001001 + structure.getDtBitmaskForDt(7) shouldBe 0b011011011 + structure.getDtBitmaskForDt(8) shouldBe 0b111111111 + } + should("Calculate its own length properly") { + lds.encodedLength shouldBe descL3T3.size + } + should("Be re-encoded to the same bytes") { + val buf = ByteArray(lds.encodedLength) + lds.write(buf, 0, buf.size) + buf shouldBe descL3T3 + } + } + context("a descriptor with a K-SVC dependency structure") { + val ldsr = Av1DependencyDescriptorReader(descL3T3Key, 0, descL3T3Key.size) + val lds = ldsr.parse(null) + should("be parsed properly") { + lds.startOfFrame shouldBe true + lds.endOfFrame shouldBe false + lds.frameNumber shouldBe 0x0085 + lds.activeDecodeTargetsBitmask shouldBe 0x1ff + + val structure = lds.newTemplateDependencyStructure + structure shouldNotBe null + structure!!.decodeTargetCount shouldBe 9 + structure.maxTemporalId shouldBe 2 + structure.maxSpatialId shouldBe 2 + } + should("calculate correct frame info") { + val ldsi = lds.frameInfo + ldsi.spatialId shouldBe 0 + ldsi.temporalId shouldBe 0 + } + should("calculate correctly whether layer switching needs keyframes") { + val structure = lds.newTemplateDependencyStructure!! + for (fromS in 0..2) { + for (fromT in 0..2) { + val fromDT = 3 * fromS + fromT + for (toS in 0..2) { + for (toT in 0..2) { + val toDT = 3 * toS + toT + /* With this structure you can switch to other temporal + * layers within the same spatial layer, without a keyframe; but switching + * spatial layers needs a keyframe. + */ + withClue("from DT $fromDT to DT $toDT") { + if (fromS == toS) { + structure.canSwitchWithoutKeyframe( + fromDt = fromDT, + toDt = toDT + ) shouldBe true + } else { + structure.canSwitchWithoutKeyframe( + fromDt = fromDT, + toDt = toDT + ) shouldBe false + } + } + } + } + } + } + } + should("calculate DTI bitmasks corresponding to a given DT") { + val structure = lds.newTemplateDependencyStructure!! + structure.getDtBitmaskForDt(0) shouldBe 0b000000001 + structure.getDtBitmaskForDt(1) shouldBe 0b000000011 + structure.getDtBitmaskForDt(2) shouldBe 0b000000111 + structure.getDtBitmaskForDt(3) shouldBe 0b000001000 + structure.getDtBitmaskForDt(4) shouldBe 0b000011000 + structure.getDtBitmaskForDt(5) shouldBe 0b000111000 + structure.getDtBitmaskForDt(6) shouldBe 0b001000000 + structure.getDtBitmaskForDt(7) shouldBe 0b011000000 + structure.getDtBitmaskForDt(8) shouldBe 0b111000000 + } + should("Calculate its own length properly") { + lds.encodedLength shouldBe descL3T3Key.size + } + should("Be re-encoded to the same bytes") { + val buf = ByteArray(lds.encodedLength) + lds.write(buf, 0, buf.size) + buf shouldBe descL3T3Key + } + } + context("a descriptor with a K-SVC dependency structure with key shift") { + val ldsr = Av1DependencyDescriptorReader(descL2T2KeyShift, 0, descL2T2KeyShift.size) + val lds = ldsr.parse(null) + should("be parsed properly") { + lds.startOfFrame shouldBe true + lds.endOfFrame shouldBe false + lds.frameNumber shouldBe 0x00ed + lds.activeDecodeTargetsBitmask shouldBe 0x0f + + val structure = lds.newTemplateDependencyStructure + structure shouldNotBe null + structure!!.decodeTargetCount shouldBe 4 + structure.maxTemporalId shouldBe 1 + structure.maxSpatialId shouldBe 1 + } + should("calculate correct frame info") { + val ldsi = lds.frameInfo + ldsi.spatialId shouldBe 0 + ldsi.temporalId shouldBe 0 + } + should("calculate correctly whether layer switching needs keyframes") { + val structure = lds.newTemplateDependencyStructure!! + for (fromS in 0..1) { + for (fromT in 0..1) { + val fromDT = 2 * fromS + fromT + for (toS in 0..1) { + for (toT in 0..1) { + val toDT = 2 * toS + toT + /* With this structure you can switch to other temporal + * layers within the same spatial layer, without a keyframe; but switching + * spatial layers needs a keyframe. + */ + withClue("from DT $fromDT to DT $toDT") { + if (fromS == toS) { + structure.canSwitchWithoutKeyframe( + fromDt = fromDT, + toDt = toDT + ) shouldBe true + } else { + structure.canSwitchWithoutKeyframe( + fromDt = fromDT, + toDt = toDT + ) shouldBe false + } + } + } + } + } + } + } + should("calculate DTI bitmasks corresponding to a given DT") { + val structure = lds.newTemplateDependencyStructure!! + structure.getDtBitmaskForDt(0) shouldBe 0b0001 + structure.getDtBitmaskForDt(1) shouldBe 0b0011 + structure.getDtBitmaskForDt(2) shouldBe 0b0100 + structure.getDtBitmaskForDt(3) shouldBe 0b1100 + } + should("Calculate its own length properly") { + lds.encodedLength shouldBe descL2T2KeyShift.size + } + should("Be re-encoded to the same bytes") { + val buf = ByteArray(lds.encodedLength) + lds.write(buf, 0, buf.size) + buf shouldBe descL2T2KeyShift + } + } + context("a descriptor following the dependency structure, specifying decode targets") { + val ldsr = Av1DependencyDescriptorReader(descL3T3, 0, descL3T3.size) + val lds = ldsr.parse(null) + val mdsr = Av1DependencyDescriptorReader(midDescScalable, 0, midDescScalable.size) + val mds = mdsr.parse(lds.newTemplateDependencyStructure) + + should("be parsed properly") { + mds.startOfFrame shouldBe true + mds.endOfFrame shouldBe true + mds.frameNumber shouldBe 0x0146 + + mds.newTemplateDependencyStructure shouldBe null + mds.activeDecodeTargetsBitmask shouldBe 0x7 + } + should("calculate correct frame info") { + val mdsi = mds.frameInfo + mdsi.spatialId shouldBe 0 + mdsi.temporalId shouldBe 1 + } + should("Calculate its own length properly") { + mds.encodedLength shouldBe midDescScalable.size + } + should("Be re-encoded to the same bytes") { + val buf = ByteArray(mds.encodedLength) + mds.write(buf, 0, buf.size) + buf shouldBe midDescScalable + } + } + context("another such descriptor") { + val ldsr = Av1DependencyDescriptorReader(longForMid2, 0, longForMid2.size) + val lds = ldsr.parse(null) + val mdsr = Av1DependencyDescriptorReader(midDescScalable2, 0, midDescScalable2.size) + val mds = mdsr.parse(lds.newTemplateDependencyStructure) + + should("be parsed properly") { + mds.startOfFrame shouldBe true + mds.endOfFrame shouldBe true + mds.frameNumber shouldBe 0x03ce + + mds.newTemplateDependencyStructure shouldBe null + mds.activeDecodeTargetsBitmask shouldBe 0x7 + } + should("calculate correct frame info") { + val mdsi = mds.frameInfo + mdsi.spatialId shouldBe 0 + mdsi.temporalId shouldBe 1 + } + should("Calculate its own length properly") { + mds.encodedLength shouldBe midDescScalable2.size + } + should("Be re-encoded to the same bytes") { + val buf = ByteArray(mds.encodedLength) + mds.write(buf, 0, buf.size) + buf shouldBe midDescScalable2 + } + } + context("a descriptor without a dependency structure") { + val mdsr = Av1DependencyDescriptorReader(midDescScalable, 0, midDescScalable.size) + should("be parseable as the stateless subset") { + val mds = mdsr.parseStateless() + + mds.startOfFrame shouldBe true + mds.endOfFrame shouldBe true + mds.frameNumber shouldBe 0x0146 + + mds.newTemplateDependencyStructure shouldBe null + } + should("fail to parse if the dependency structure is not present") { + shouldThrow { + mdsr.parse(null) + } + } + } + context("a descriptor without extended fields") { + val ld1r = Av1DependencyDescriptorReader(descL1T1, 0, descL1T1.size) + val ld1 = ld1r.parse(null) + val sd1r = Av1DependencyDescriptorReader(shortDesc, 0, shortDesc.size) + val sd1 = sd1r.parse(ld1.newTemplateDependencyStructure) + + should("be parsed properly") { + sd1.startOfFrame shouldBe false + sd1.endOfFrame shouldBe true + sd1.frameNumber shouldBe 0x0001 + + sd1.newTemplateDependencyStructure shouldBe null + sd1.activeDecodeTargetsBitmask shouldBe null + } + should("calculate correct frame info") { + val sd1i = sd1.frameInfo + sd1i.spatialId shouldBe 0 + sd1i.temporalId shouldBe 0 + } + should("Calculate its own length properly") { + sd1.encodedLength shouldBe shortDesc.size + } + should("Be re-encoded to the same bytes") { + val buf = ByteArray(sd1.encodedLength) + sd1.write(buf, 0, buf.size) + buf shouldBe shortDesc + } + } + context("A descriptor with a Temporal-only scalability structure ") { + val ldsr = Av1DependencyDescriptorReader(descL1T3, 0, descL1T3.size) + val lds = ldsr.parse(null) + should("be parsed properly") { + lds.startOfFrame shouldBe true + lds.endOfFrame shouldBe false + lds.frameNumber shouldBe 0x0001 + lds.activeDecodeTargetsBitmask shouldBe 0x7 + + val structure = lds.newTemplateDependencyStructure + structure shouldNotBe null + structure!!.decodeTargetCount shouldBe 3 + structure.maxTemporalId shouldBe 2 + structure.maxSpatialId shouldBe 0 + } + should("calculate correct frame info") { + val ldsi = lds.frameInfo + ldsi.spatialId shouldBe 0 + ldsi.temporalId shouldBe 0 + } + should("calculate correctly whether layer switching needs keyframes") { + val structure = lds.newTemplateDependencyStructure!! + val fromS = 0 + for (fromT in 0..2) { + val fromDT = 3 * fromS + fromT + val toS = 0 + for (toT in 0..2) { + val toDT = 3 * toS + toT + /* With this structure you can switch down spatial layers, or to other temporal + * layers within the same spatial layer, without a keyframe; but switching up + * spatial layers needs a keyframe. + */ + withClue("from DT $fromDT to DT $toDT") { + structure.canSwitchWithoutKeyframe( + fromDt = fromDT, + toDt = toDT + ) shouldBe true + } + } + } + } + should("calculate DTI bitmasks corresponding to a given DT") { + val structure = lds.newTemplateDependencyStructure!! + structure.getDtBitmaskForDt(0) shouldBe 0b001 + structure.getDtBitmaskForDt(1) shouldBe 0b011 + structure.getDtBitmaskForDt(2) shouldBe 0b111 + } + should("Calculate its own length properly") { + lds.encodedLength shouldBe descL1T3.size + } + should("Be re-encoded to the same bytes") { + val buf = ByteArray(lds.encodedLength) + lds.write(buf, 0, buf.size) + buf shouldBe descL1T3 + } + } + context("a descriptor with a simulcast dependency structure") { + val ldsr = Av1DependencyDescriptorReader(descS3T3, 0, descS3T3.size) + val lds = ldsr.parse(null) + should("be parsed properly") { + lds.startOfFrame shouldBe true + lds.endOfFrame shouldBe true + lds.frameNumber shouldBe 0x0001 + lds.activeDecodeTargetsBitmask shouldBe 0x1ff + + val structure = lds.newTemplateDependencyStructure + structure shouldNotBe null + structure!!.decodeTargetCount shouldBe 9 + structure.maxTemporalId shouldBe 2 + structure.maxSpatialId shouldBe 2 + } + should("calculate correct frame info") { + val ldsi = lds.frameInfo + ldsi.spatialId shouldBe 0 + ldsi.temporalId shouldBe 0 + } + should("calculate correctly whether layer switching needs keyframes") { + val structure = lds.newTemplateDependencyStructure!! + for (fromS in 0..2) { + for (fromT in 0..2) { + val fromDT = 3 * fromS + fromT + for (toS in 0..2) { + for (toT in 0..2) { + val toDT = 3 * toS + toT + /* With this structure you can switch to other temporal + * layers within the same spatial layer, without a keyframe; but switching + * spatial layers needs a keyframe. + */ + withClue("from DT $fromDT to DT $toDT") { + if (fromS == toS) { + structure.canSwitchWithoutKeyframe( + fromDt = fromDT, + toDt = toDT + ) shouldBe true + } else { + structure.canSwitchWithoutKeyframe( + fromDt = fromDT, + toDt = toDT + ) shouldBe false + } + } + } + } + } + } + } + should("calculate DTI bitmasks corresponding to a given DT") { + val structure = lds.newTemplateDependencyStructure!! + structure.getDtBitmaskForDt(0) shouldBe 0b000000001 + structure.getDtBitmaskForDt(1) shouldBe 0b000000011 + structure.getDtBitmaskForDt(2) shouldBe 0b000000111 + structure.getDtBitmaskForDt(3) shouldBe 0b000001000 + structure.getDtBitmaskForDt(4) shouldBe 0b000011000 + structure.getDtBitmaskForDt(5) shouldBe 0b000111000 + structure.getDtBitmaskForDt(6) shouldBe 0b001000000 + structure.getDtBitmaskForDt(7) shouldBe 0b011000000 + structure.getDtBitmaskForDt(8) shouldBe 0b111000000 + } + should("Calculate its own length properly") { + lds.encodedLength shouldBe descS3T3.size + } + should("Be re-encoded to the same bytes") { + val buf = ByteArray(lds.encodedLength) + lds.write(buf, 0, buf.size) + buf shouldBe descS3T3 + } + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/DumpAv1DependencyDescriptor.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/DumpAv1DependencyDescriptor.kt new file mode 100644 index 0000000000..b488ad9a20 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/DumpAv1DependencyDescriptor.kt @@ -0,0 +1,36 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.rtp.header_extensions + +import jakarta.xml.bind.DatatypeConverter.parseHexBinary + +fun main(args: Array) { + var structure: Av1TemplateDependencyStructure? = null + var line: String? + while (readLine().also { line = it } != null) { + try { + val descBinary = parseHexBinary(line) + val reader = Av1DependencyDescriptorReader(descBinary, 0, descBinary.size) + val desc = reader.parse(structure) + desc.newTemplateDependencyStructure?.let { structure = it } + println(desc.toJSONString()) + val frameInfo = desc.frameInfo + println(frameInfo) + } catch (e: Exception) { + println(e.message) + } + } +} From 5c48e421d1b2b24d9f2f06c067e1fc671b89efa8 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Mon, 6 Nov 2023 10:28:14 -0600 Subject: [PATCH 060/189] fix: Fix saving the connected WebSocket instance the first time. (#2066) --- .../org/jitsi/videobridge/relay/RelayMessageTransport.kt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt index 73c09aed94..520f048b05 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt @@ -309,9 +309,11 @@ class RelayMessageTransport( override fun webSocketConnected(ws: ColibriWebSocket) { synchronized(webSocketSyncRoot) { // If we already have a web-socket, discard it and use the new one. - if (ws != webSocket && webSocket != null) { - logger.info("Replacing an existing websocket.") - webSocket?.session?.close(CloseStatus.NORMAL, "replaced") + if (ws != webSocket) { + if (webSocket != null) { + logger.info("Replacing an existing websocket.") + webSocket?.session?.close(CloseStatus.NORMAL, "replaced") + } webSocketLastActive = true webSocket = ws sendMessage(ws, createServerHello()) From b5bdb0f405219e0120a89aba4bbacc9bdb6f667b Mon Sep 17 00:00:00 2001 From: bgrozev Date: Wed, 8 Nov 2023 10:39:52 -0600 Subject: [PATCH 061/189] chore: Update jicoco (try to fix slow XMPP reconnects). (#2068) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fdbec3db58..5a75527d62 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 5.7.2 5.10.0 1.0-127-g6c65524 - 1.1-127-gf49982f + 1.1-129-g23ac61c 1.13.8 3.0.0 3.5.1 From 814bffd6cf050001a020288e70e942fa0d505d64 Mon Sep 17 00:00:00 2001 From: dkirov-dev <147876663+dkirov-dev@users.noreply.github.com> Date: Mon, 13 Nov 2023 12:43:43 -0800 Subject: [PATCH 062/189] Fixed default file path (#2070) --- .../src/main/kotlin/org/jitsi/nlj/transform/node/PcapWriter.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PcapWriter.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PcapWriter.kt index b2b46913a2..4dbdac812c 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PcapWriter.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PcapWriter.kt @@ -36,7 +36,7 @@ import java.util.Random class PcapWriter( parentLogger: Logger, - filePath: String = "/tmp/${Random().nextLong()}.pcap}" + filePath: String = "/tmp/${Random().nextLong()}.pcap" ) : ObserverNode("PCAP writer") { private val logger = createChildLogger(parentLogger) private val lazyHandle = lazy { From 73fb0cf07e5627d1a6ed7131b6309af26afacd52 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Wed, 29 Nov 2023 09:13:00 -0600 Subject: [PATCH 063/189] fix: Reject JSON with duplicate keys (#2073) --- .../message/BridgeChannelMessage.kt | 2 + .../message/BridgeChannelMessageTest.kt | 44 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt index f7f887e84c..6a0d65f461 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.annotation.JsonSubTypes import com.fasterxml.jackson.annotation.JsonTypeInfo +import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.core.JsonProcessingException import com.fasterxml.jackson.databind.JsonMappingException import com.fasterxml.jackson.databind.MapperFeature @@ -86,6 +87,7 @@ sealed class BridgeChannelMessage { companion object { private val mapper = jacksonObjectMapper().apply { enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS) + enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION) } @JvmStatic diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/message/BridgeChannelMessageTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/message/BridgeChannelMessageTest.kt index 10cee7ffeb..9ef13769e8 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/message/BridgeChannelMessageTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/message/BridgeChannelMessageTest.kt @@ -15,6 +15,7 @@ */ package org.jitsi.videobridge.message +import com.fasterxml.jackson.core.JsonParseException import com.fasterxml.jackson.core.JsonProcessingException import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.exc.InvalidTypeIdException @@ -65,6 +66,49 @@ class BridgeChannelMessageTest : ShouldSpec() { shouldThrow { parse("""{"colibriClass": "invalid-colibri-class" }""") } + shouldThrow { + parse( + """ + { + "colibriClass": "EndpointStats", + "colibriClass": "duplicate" + } + """.trimIndent() + ) + } + shouldThrow { + parse( + """ + { + "colibriClass": "EndpointStats", + "to": "a", + "to": "b" + } + """.trimIndent() + ) + } + shouldThrow { + parse( + """ + { + "colibriClass": "EndpointStats", + "from": "a", + "from": "b" + } + """.trimIndent() + ) + } + shouldThrow { + parse( + """ + { + "colibriClass": "EndpointStats", + "non-defined-prop": "a", + "non-defined-prop": "b" + } + """.trimIndent() + ) + } context("when some of the message-specific fields are missing/invalid") { shouldThrow { From 73f6068c60936f7089543648464fd15009661d8a Mon Sep 17 00:00:00 2001 From: Aaron van Meerten Date: Tue, 19 Dec 2023 13:47:25 -0600 Subject: [PATCH 064/189] task: graceful shutdown only via REST (#2075) * task: graceful shutdown only via REST --- resources/graceful_shutdown.sh | 65 ++++------------------------------ 1 file changed, 6 insertions(+), 59 deletions(-) diff --git a/resources/graceful_shutdown.sh b/resources/graceful_shutdown.sh index a1a6084e1c..ef875aeb72 100755 --- a/resources/graceful_shutdown.sh +++ b/resources/graceful_shutdown.sh @@ -9,10 +9,7 @@ # returned and 1 otherwise. # # Arguments: -# "-p"(mandatory) the PID of jitsi Videobridge process # "-h"("http://localhost:8080" by default) REST requests host URI part -# "-t"("25" by default) number of second we we for the bridge to shutdown -# gracefully after participant count drops to 0 # "-s"(disabled by default) enable silent mode - no info output # # NOTE: script depends on the tool jq, used to parse json @@ -20,22 +17,15 @@ # Initialize arguments hostUrl="http://localhost:8080" -timeout=25 verbose=1 # Parse arguments OPTIND=1 while getopts "p:h:t:s" opt; do case "$opt" in - p) - pid=$OPTARG - ;; h) hostUrl=$OPTARG ;; - t) - timeout=$OPTARG - ;; s) verbose=0 ;; @@ -43,22 +33,6 @@ while getopts "p:h:t:s" opt; do done shift "$((OPTIND-1))" -# Try the pid file, if no pid was provided as an argument. -# for systemd we use different pid file in a subfolder -if [ "$pid" = "" ] ;then - if [ -f /var/run/jitsi-videobridge.pid ]; then - pid=`cat /var/run/jitsi-videobridge.pid` - else - pid=`cat /var/run/jitsi-videobridge/jitsi-videobridge.pid` - fi -fi - -#Check if PID is a number -re='^[0-9]+$' -if ! [[ $pid =~ $re ]] ; then - echo "error: PID is not a number" >&2; exit 1 -fi - # Returns local participant count by calling JVB REST statistics API and extracting # participant count from JSON stats text returned. function getParticipantCount { @@ -88,41 +62,14 @@ then printInfo "There are still $participantCount participants" sleep 10 participantCount=`getParticipantCount` - done - - sleep 5 - - if ps -p $pid > /dev/null 2>&1 - then - printInfo "It is still running, lets give it $timeout seconds" - sleep $timeout - if ps -p $pid > /dev/null 2>&1 - then - printError "Bridge did not exit after $timeout sec - killing $pid" - kill $pid - fi - fi - # check for 3 seconds if we managed to kill - for I in 1 2 3 - do - if ps -p $pid > /dev/null 2>&1 - then - sleep 1 + if [[ $? -gt 0 ]] ; then + printInfo "Failed to get participant count, Bridge already shutdown" + exit 0 fi done - if ps -p $pid > /dev/null 2>&1 - then - printError "Failed to kill $pid" - printError "Sending force kill to $pid" - kill -9 $pid - if ps -p $pid > /dev/null 2>&1 - then - printError "Failed to force kill $pid, giving up." - exit 1 - fi - fi - rm -f /var/run/jitsi-videobridge.pid - rm -f /var/run/jitsi-videobridge/jitsi-videobridge.pid + + echo "Waiting 60 seconds for bridge to finish shutting down" + sleep 60 printInfo "Bridge shutdown OK" exit 0 else From 719465d17ee1113ecf80030b5f81e08a7f8c0e2c Mon Sep 17 00:00:00 2001 From: Aaron van Meerten Date: Tue, 19 Dec 2023 14:02:45 -0600 Subject: [PATCH 065/189] task: wait on pid before graceful shutdown is complete (#2076) * task: wait on pid before graceful shutdown is complete --- resources/graceful_shutdown.sh | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/resources/graceful_shutdown.sh b/resources/graceful_shutdown.sh index ef875aeb72..8a99ef0e86 100755 --- a/resources/graceful_shutdown.sh +++ b/resources/graceful_shutdown.sh @@ -53,6 +53,16 @@ function printError { echo "$@" 1>&2 } +function waitForPid { + while [ -d /proc/$1 ] ;do + echo "PID $1 still exists" + sleep 10 + done + echo "PID $1 is done" +} + +JVB_PID=$(ps aux | grep java | grep jitsi-videobridge.jar | awk '{print $2}') + shutdownStatus=`curl -s -o /dev/null -H "Content-Type: application/json" -d '{ "graceful-shutdown": "true" }' -w "%{http_code}" "$hostUrl/colibri/shutdown"` if [ "$shutdownStatus" == "200" ] then @@ -63,13 +73,14 @@ then sleep 10 participantCount=`getParticipantCount` if [[ $? -gt 0 ]] ; then - printInfo "Failed to get participant count, Bridge already shutdown" + printInfo "Failed to get participant count, bridge may be already shutdown, waiting on pid $JVB_PID" + waitForPid $JVB_PID exit 0 fi done - echo "Waiting 60 seconds for bridge to finish shutting down" - sleep 60 + echo "Waiting for bridge pid $JVB_PID to finish shutting down" + waitForPid $JVB_PID printInfo "Bridge shutdown OK" exit 0 else From e41cad1e476efef0cd0efae1b44b80cb7f432b03 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 21 Dec 2023 12:19:08 -0600 Subject: [PATCH 066/189] feat: Relax the health checks for STUN (#2077) * feat: Relax the health checks for STUN By default only fail when there is no valid address for ICE. Put the option to require STUN behind a flag. * chore: Bump ice4j. --- .../videobridge/health/JvbHealthChecker.kt | 20 ++++++++++++++++++- .../videobridge/health/config/HealthConfig.kt | 8 ++++++++ jvb/src/main/resources/reference.conf | 8 ++++++++ pom.xml | 2 +- 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/health/JvbHealthChecker.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/health/JvbHealthChecker.kt index a27e832787..4ac422933a 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/health/JvbHealthChecker.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/health/JvbHealthChecker.kt @@ -22,6 +22,7 @@ import org.jitsi.health.HealthChecker import org.jitsi.health.Result import org.jitsi.videobridge.health.config.HealthConfig.Companion.config import org.jitsi.videobridge.ice.Harvesters +import java.net.InetAddress class JvbHealthChecker : HealthCheckService { private val healthChecker = HealthChecker( @@ -36,7 +37,10 @@ class JvbHealthChecker : HealthCheckService { fun stop() = healthChecker.stop() private fun check(): Result { - if (MappingCandidateHarvesters.stunDiscoveryFailed) { + if (config.requireValidAddress && !hasValidAddress()) { + return Result(success = false, message = "No valid IP addresses available for harvesting.") + } + if (config.requireStun && MappingCandidateHarvesters.stunDiscoveryFailed) { return Result(success = false, message = "Address discovery through STUN failed") } if (!Harvesters.isHealthy()) { @@ -48,6 +52,20 @@ class JvbHealthChecker : HealthCheckService { return Result(success = true) } + private fun InetAddress.isValid(): Boolean { + return !this.isSiteLocalAddress && !this.isLinkLocalAddress && !this.isLoopbackAddress + } + + private fun hasValidAddress(): Boolean { + if (Harvesters.singlePortHarvesters.any { it.localAddress.address.isValid() }) { + return true + } + if (MappingCandidateHarvesters.getHarvesters().any { it.mask?.address?.isValid() == true }) { + return true + } + return false + } + override val result: Result get() = healthChecker.result } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/health/config/HealthConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/health/config/HealthConfig.kt index 4564000274..db2a93576a 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/health/config/HealthConfig.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/health/config/HealthConfig.kt @@ -41,6 +41,14 @@ class HealthConfig private constructor() { "videobridge.health.sticky-failures".from(JitsiConfig.newConfig) } + val requireStun: Boolean by config { + "videobridge.health.require-stun".from(JitsiConfig.newConfig) + } + + val requireValidAddress: Boolean by config { + "videobridge.health.require-valid-address".from(JitsiConfig.newConfig) + } + companion object { @JvmField val config = HealthConfig() diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index 16dab152ca..5904c1f62f 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -21,6 +21,14 @@ videobridge { # (i.e. once the bridge becomes unhealthy, it will never # go back to a healthy state) sticky-failures = false + + # If enabled, health checks fail when there are no valid addresses available for ICE. Here "valid" means not + # site local, link local or loopback. + require-valid-address = true + + # If enabled, health checks will fail when a STUN mapping harvester is configured, but fails to get a mapping (even + # if there is a valid address available through other mappings or locally). + require-stun = false } ep-connection-status { # How long we'll wait for an endpoint to *start* sending diff --git a/pom.xml b/pom.xml index 5a75527d62..3cec068b10 100644 --- a/pom.xml +++ b/pom.xml @@ -106,7 +106,7 @@ ${project.groupId} ice4j - 3.0-66-g1c60acc + 3.0-68-gd289f12 ${project.groupId} From 07b97ebd4b9d5ef8ef0551191a96115c7cb9aabb Mon Sep 17 00:00:00 2001 From: bgrozev Date: Fri, 12 Jan 2024 15:02:52 -0600 Subject: [PATCH 067/189] Add null check when accesing singlePortHarvesters. (#2078) --- .../kotlin/org/jitsi/videobridge/health/JvbHealthChecker.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/health/JvbHealthChecker.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/health/JvbHealthChecker.kt index 4ac422933a..d01402dc67 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/health/JvbHealthChecker.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/health/JvbHealthChecker.kt @@ -57,7 +57,7 @@ class JvbHealthChecker : HealthCheckService { } private fun hasValidAddress(): Boolean { - if (Harvesters.singlePortHarvesters.any { it.localAddress.address.isValid() }) { + if (Harvesters.singlePortHarvesters?.any { it.localAddress.address.isValid() } == true) { return true } if (MappingCandidateHarvesters.getHarvesters().any { it.mask?.address?.isValid() == true }) { From b2d4229f8e9e49f9b2a2e5a3d0a4be195665c24f Mon Sep 17 00:00:00 2001 From: bgrozev Date: Fri, 19 Jan 2024 14:37:38 -0600 Subject: [PATCH 068/189] Initialize the udp/10000 harvester at startup instead of first conference. (#2080) --- jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt | 1 + .../kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt index 0e5416bf98..9af6988949 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt @@ -76,6 +76,7 @@ fun main() { } startIce4j() + Harvesters.initializeStaticConfiguration() XmppStringPrepUtil.setMaxCacheSizes(XmppClientConnectionConfig.config.jidCacheSize) PacketQueue.setEnableStatisticsDefault(true) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt index f67d86bc9a..e9cccd98d1 100755 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt @@ -393,7 +393,6 @@ class IceTransport @JvmOverloads constructor( companion object { fun appendHarvesters(iceAgent: Agent) { - Harvesters.initializeStaticConfiguration() Harvesters.tcpHarvester?.let { iceAgent.addCandidateHarvester(it) } From 57abe58d9f924daf32cc6cfaa75c6921f2995105 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Tue, 23 Jan 2024 14:55:38 -0600 Subject: [PATCH 069/189] ref: Move Harvesters to kotlin. (#2081) * ref: Move Harvesters to kotlin. --- .../org/jitsi/videobridge/ice/Harvesters.java | 147 ------------------ .../main/kotlin/org/jitsi/videobridge/Main.kt | 6 +- .../videobridge/health/JvbHealthChecker.kt | 4 +- .../org/jitsi/videobridge/ice/Harvesters.kt | 72 +++++++++ .../videobridge/transport/ice/IceTransport.kt | 4 +- 5 files changed, 80 insertions(+), 153 deletions(-) delete mode 100644 jvb/src/main/java/org/jitsi/videobridge/ice/Harvesters.java create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/ice/Harvesters.kt diff --git a/jvb/src/main/java/org/jitsi/videobridge/ice/Harvesters.java b/jvb/src/main/java/org/jitsi/videobridge/ice/Harvesters.java deleted file mode 100644 index b3e8a89f6e..0000000000 --- a/jvb/src/main/java/org/jitsi/videobridge/ice/Harvesters.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright @ 2018 - present 8x8, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.jitsi.videobridge.ice; - -import org.ice4j.ice.harvest.*; -import org.jitsi.utils.logging2.*; - -import java.io.*; -import java.util.*; - -public class Harvesters -{ - /** - * The flag which indicates whether application-wide harvesters, stored - * in the static fields {@link #tcpHarvester} and - * {@link #singlePortHarvesters} have been initialized. - */ - private static boolean staticConfigurationInitialized = false; - - /** - * Global variable do we consider this transport manager as healthy. - * By default we consider healthy, if we fail to bind to the single port - * port we consider the bridge as unhealthy. - */ - private static boolean healthy = true; - - public static boolean isHealthy() - { - return healthy; - } - - /** - * The {@link Logger} used by the {@link Harvesters} class to - * print debug information. - */ - private static final Logger classLogger - = new LoggerImpl(Harvesters.class.getName()); - - /** - * The single TcpHarvester instance for the - * application. - */ - public static TcpHarvester tcpHarvester = null; - - /** - * The SinglePortUdpHarvesters which will be appended to ICE - * Agents managed by IceTransport instances. - */ - public static List singlePortHarvesters = null; - - /** - * Initializes the static Harvester instances used by all - * IceTransport instances, that is - * {@link #tcpHarvester} and {@link #singlePortHarvesters}. - */ - public static void initializeStaticConfiguration() - { - synchronized (Harvesters.class) - { - if (staticConfigurationInitialized) - { - return; - } - staticConfigurationInitialized = true; - - - singlePortHarvesters - = SinglePortUdpHarvester.createHarvesters(IceConfig.config.getPort()); - if (singlePortHarvesters.isEmpty()) - { - singlePortHarvesters = null; - classLogger.info("No single-port harvesters created."); - } - - healthy = singlePortHarvesters != null; - - if (IceConfig.config.getTcpEnabled()) - { - int port = IceConfig.config.getTcpPort(); - try - { - tcpHarvester = new TcpHarvester(port, IceConfig.config.getIceSslTcp()); - classLogger.info("Initialized TCP harvester on port " - + port + ", ssltcp=" + IceConfig.config.getIceSslTcp()); - - } - catch (IOException ioe) - { - classLogger.warn( - "Failed to initialize TCP harvester on port " + port); - } - - Integer mappedPort = IceConfig.config.getTcpMappedPort(); - if (mappedPort != null) - { - tcpHarvester.addMappedPort(mappedPort); - } - } - } - } - - /** - * Stops the static Harvester instances used by all - * IceTransport instances, that is - * {@link #tcpHarvester} and {@link #singlePortHarvesters}. - */ - public static void closeStaticConfiguration() - { - synchronized (Harvesters.class) - { - if (!staticConfigurationInitialized) - { - return; - } - staticConfigurationInitialized = false; - - if (singlePortHarvesters != null) - { - singlePortHarvesters.forEach(AbstractUdpListener::close); - singlePortHarvesters = null; - } - - if (tcpHarvester != null) - { - tcpHarvester.close(); - tcpHarvester = null; - } - - // Reset the flag to initial state. - healthy = true; - } - } -} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt index 9af6988949..ce0ccd0c27 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt @@ -76,7 +76,9 @@ fun main() { } startIce4j() - Harvesters.initializeStaticConfiguration() + + // Initialize, binding on the main ICE port. + Harvesters.init() XmppStringPrepUtil.setMaxCacheSizes(XmppClientConnectionConfig.config.jidCacheSize) PacketQueue.setEnableStatisticsDefault(true) @@ -219,5 +221,5 @@ private fun startIce4j() { private fun stopIce4j() { // Shut down harvesters. - Harvesters.closeStaticConfiguration() + Harvesters.close() } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/health/JvbHealthChecker.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/health/JvbHealthChecker.kt index d01402dc67..e02a8738bb 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/health/JvbHealthChecker.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/health/JvbHealthChecker.kt @@ -43,7 +43,7 @@ class JvbHealthChecker : HealthCheckService { if (config.requireStun && MappingCandidateHarvesters.stunDiscoveryFailed) { return Result(success = false, message = "Address discovery through STUN failed") } - if (!Harvesters.isHealthy()) { + if (!Harvesters.INSTANCE.healthy) { return Result(success = false, message = "Failed to bind single-port") } @@ -57,7 +57,7 @@ class JvbHealthChecker : HealthCheckService { } private fun hasValidAddress(): Boolean { - if (Harvesters.singlePortHarvesters?.any { it.localAddress.address.isValid() } == true) { + if (Harvesters.INSTANCE.singlePortHarvesters.any { it.localAddress.address.isValid() }) { return true } if (MappingCandidateHarvesters.getHarvesters().any { it.mask?.address?.isValid() == true }) { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/ice/Harvesters.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/ice/Harvesters.kt new file mode 100644 index 0000000000..c5ea68ccaf --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/ice/Harvesters.kt @@ -0,0 +1,72 @@ +/* +* Copyright @ 2018 - present 8x8, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.jitsi.videobridge.ice + +import org.ice4j.ice.harvest.SinglePortUdpHarvester +import org.ice4j.ice.harvest.TcpHarvester +import org.jitsi.utils.logging2.createLogger +import java.io.IOException + +class Harvesters private constructor( + val tcpHarvester: TcpHarvester?, + val singlePortHarvesters: List +) { + /* We're unhealthy if there are no single port harvesters. */ + val healthy: Boolean + get() = singlePortHarvesters.isNotEmpty() + + private fun close() { + singlePortHarvesters.forEach { it.close() } + tcpHarvester?.close() + } + + companion object { + private val logger = createLogger() + + fun init() { + // Trigger the lazy init. + INSTANCE + } + + fun close() = INSTANCE.close() + + val INSTANCE: Harvesters by lazy { + val singlePortHarvesters = SinglePortUdpHarvester.createHarvesters(IceConfig.config.port) + if (singlePortHarvesters.isEmpty()) { + logger.warn("No single-port harvesters created.") + } + val tcpHarvester: TcpHarvester? = if (IceConfig.config.tcpEnabled) { + val port = IceConfig.config.tcpPort + try { + TcpHarvester(IceConfig.config.port, IceConfig.config.iceSslTcp).apply { + logger.info("Initialized TCP harvester on port $port, ssltcp=${IceConfig.config.iceSslTcp}") + IceConfig.config.tcpMappedPort?.let { mappedPort -> + logger.info("Adding mapped port $mappedPort") + addMappedPort(mappedPort) + } + } + } catch (ioe: IOException) { + logger.warn("Failed to initialize TCP harvester on port $port") + null + } + } else { + null + } + + Harvesters(tcpHarvester, singlePortHarvesters) + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt index e9cccd98d1..317c35b8aa 100755 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt @@ -393,10 +393,10 @@ class IceTransport @JvmOverloads constructor( companion object { fun appendHarvesters(iceAgent: Agent) { - Harvesters.tcpHarvester?.let { + Harvesters.INSTANCE.tcpHarvester?.let { iceAgent.addCandidateHarvester(it) } - Harvesters.singlePortHarvesters?.forEach(iceAgent::addCandidateHarvester) + Harvesters.INSTANCE.singlePortHarvesters.forEach(iceAgent::addCandidateHarvester) } /** From 9d121e4e299e04dde53fcb3b7441c9855c2d45d9 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Wed, 24 Jan 2024 12:43:26 -0500 Subject: [PATCH 070/189] Send pings periodically on idle websocket connections. (#2084) --- .../videobridge/EndpointMessageTransport.java | 9 +-- .../websocket/ColibriWebSocket.java | 75 ++++++++++++++++++- .../websocket/ColibriWebSocketServlet.java | 4 +- .../relay/RelayMessageTransport.kt | 12 +-- .../config/WebsocketServiceConfig.kt | 16 ++++ jvb/src/main/resources/reference.conf | 6 ++ 6 files changed, 106 insertions(+), 16 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java b/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java index f45918c5bc..c0223afa45 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java +++ b/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java @@ -212,14 +212,7 @@ private void sendMessage(DataChannel dst, BridgeChannelMessage message) */ private void sendMessage(ColibriWebSocket dst, BridgeChannelMessage message) { - RemoteEndpoint remote = dst.getRemote(); - if (remote != null) - { - // We'll use the async version of sendString since this may be called - // from multiple threads. It's just fire-and-forget though, so we - // don't wait on the result - remote.sendString(message.toJson(), new WriteCallback.Adaptor()); - } + dst.sendString(message.toJson()); statisticsSupplier.get().colibriWebSocketMessagesSent.inc(); } diff --git a/jvb/src/main/java/org/jitsi/videobridge/websocket/ColibriWebSocket.java b/jvb/src/main/java/org/jitsi/videobridge/websocket/ColibriWebSocket.java index 28c7e0024b..d9d7ff59ec 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/websocket/ColibriWebSocket.java +++ b/jvb/src/main/java/org/jitsi/videobridge/websocket/ColibriWebSocket.java @@ -16,11 +16,15 @@ package org.jitsi.videobridge.websocket; import org.eclipse.jetty.websocket.api.*; -import org.jitsi.utils.collections.*; import org.jitsi.utils.logging2.*; import org.jitsi.videobridge.*; +import org.jitsi.videobridge.util.*; +import org.jitsi.videobridge.websocket.config.*; +import java.nio.*; +import java.time.*; import java.util.*; +import java.util.concurrent.*; /** * @author Boris Grozev @@ -37,6 +41,17 @@ public class ColibriWebSocket extends WebSocketAdapter */ private final EventHandler eventHandler; + /** + * The clock used to compute lastSendTime. + */ + private final Clock clock = Clock.systemUTC(); + + /** The last time something was sent on this web socket */ + private Instant lastSendTime = Instant.MIN; + + /** The recurring task to send pings on the connection, if needed. */ + private ScheduledFuture pinger = null; + /** * Initializes a new {@link ColibriWebSocket} instance. */ @@ -72,6 +87,14 @@ public void onWebSocketConnect(Session sess) { super.onWebSocketConnect(sess); + if (WebsocketServiceConfig.config.getSendKeepalivePings()) + { + pinger = TaskPools.SCHEDULED_POOL.scheduleAtFixedRate(this::maybeSendPing, + WebsocketServiceConfig.config.getKeepalivePingInterval().toMillis(), + WebsocketServiceConfig.config.getKeepalivePingInterval().toMillis(), + TimeUnit.MILLISECONDS); + } + eventHandler.webSocketConnected(this); } @@ -82,6 +105,52 @@ public void onWebSocketConnect(Session sess) public void onWebSocketClose(int statusCode, String reason) { eventHandler.webSocketClosed(this, statusCode, reason); + if (pinger != null) + { + pinger.cancel(true); + } + } + + public void sendString(String message) + { + RemoteEndpoint remote = getRemote(); + if (remote != null) + { + // We'll use the async version of sendString since this may be called + // from multiple threads. It's just fire-and-forget though, so we + // don't wait on the result + + remote.sendString(message, WriteCallback.NOOP); + synchronized (this) + { + lastSendTime = clock.instant(); + } + } + } + + private void maybeSendPing() + { + try + { + Instant now = clock.instant(); + synchronized (this) + { + if (Duration.between(lastSendTime, now). + compareTo(WebsocketServiceConfig.config.getKeepalivePingInterval()) < 0) + { + RemoteEndpoint remote = getRemote(); + if (remote != null) + { + remote.sendPing(ByteBuffer.allocate(0), WriteCallback.NOOP); + lastSendTime = clock.instant(); + } + } + } + } + catch (Exception e) + { + logger.error("Error sending websocket ping", e); + } } /** @@ -91,6 +160,10 @@ public void onWebSocketClose(int statusCode, String reason) public void onWebSocketError(Throwable cause) { eventHandler.webSocketError(this, cause); + if (pinger != null) + { + pinger.cancel(true); + } } public interface EventHandler diff --git a/jvb/src/main/java/org/jitsi/videobridge/websocket/ColibriWebSocketServlet.java b/jvb/src/main/java/org/jitsi/videobridge/websocket/ColibriWebSocketServlet.java index 50a4a48b2e..153f5f48ac 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/websocket/ColibriWebSocketServlet.java +++ b/jvb/src/main/java/org/jitsi/videobridge/websocket/ColibriWebSocketServlet.java @@ -22,6 +22,8 @@ import org.jitsi.utils.logging2.*; import org.jitsi.videobridge.*; import org.jitsi.videobridge.relay.*; +import org.jitsi.videobridge.websocket.config.*; + import static org.jitsi.videobridge.websocket.config.WebsocketServiceConfig.config; import java.io.*; @@ -67,7 +69,7 @@ class ColibriWebSocketServlet public void configure(JettyWebSocketServletFactory webSocketServletFactory) { // set a timeout of 1min - webSocketServletFactory.setIdleTimeout(Duration.ofMinutes(1)); + webSocketServletFactory.setIdleTimeout(WebsocketServiceConfig.config.getIdleTimeout()); webSocketServletFactory.setCreator((request, response) -> { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt index 520f048b05..d3024b6d6c 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt @@ -15,7 +15,6 @@ */ package org.jitsi.videobridge.relay -import org.eclipse.jetty.websocket.api.WriteCallback import org.eclipse.jetty.websocket.client.ClientUpgradeRequest import org.eclipse.jetty.websocket.client.WebSocketClient import org.eclipse.jetty.websocket.core.CloseStatus @@ -37,6 +36,7 @@ import org.jitsi.videobridge.message.ServerHelloMessage import org.jitsi.videobridge.message.SourceVideoTypeMessage import org.jitsi.videobridge.message.VideoTypeMessage import org.jitsi.videobridge.websocket.ColibriWebSocket +import org.jitsi.videobridge.websocket.config.WebsocketServiceConfig import org.json.simple.JSONObject import java.lang.ref.WeakReference import java.net.URI @@ -229,10 +229,7 @@ class RelayMessageTransport( * @param message the message to send. */ private fun sendMessage(dst: ColibriWebSocket, message: BridgeChannelMessage) { - // We'll use the async version of sendString since this may be called - // from multiple threads. It's just fire-and-forget though, so we - // don't wait on the result - dst.remote?.sendString(message.toJson(), WriteCallback.Adaptor()) + dst.sendString(message.toJson()) statisticsSupplier.get().colibriWebSocketMessagesSent.inc() } @@ -496,7 +493,10 @@ class RelayMessageTransport( /** * The single [WebSocketClient] instance that all [Relay]s use to initiate a web socket connection. */ - val webSocketClient = WebSocketClient().apply { start() } + val webSocketClient = WebSocketClient().apply { + idleTimeout = WebsocketServiceConfig.config.idleTimeout + start() + } /** * Reason to use when closing a WS due to the relay being expired. diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/config/WebsocketServiceConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/config/WebsocketServiceConfig.kt index fffe083572..e5e1a0c9d7 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/config/WebsocketServiceConfig.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/config/WebsocketServiceConfig.kt @@ -19,6 +19,7 @@ package org.jitsi.videobridge.websocket.config import org.jitsi.config.JitsiConfig import org.jitsi.metaconfig.config import org.jitsi.metaconfig.optionalconfig +import java.time.Duration class WebsocketServiceConfig private constructor() { /** @@ -113,6 +114,21 @@ class WebsocketServiceConfig private constructor() { } } + /** Whether keepalive pings are enabled */ + val sendKeepalivePings: Boolean by config { + "videobridge.websockets.send-keepalive-pings".from(JitsiConfig.newConfig) + } + + /** The time interval for keepalive pings */ + val keepalivePingInterval: Duration by config { + "videobridge.websockets.keepalive-ping-interval".from(JitsiConfig.newConfig) + } + + /** The time interval for websocket timeouts */ + val idleTimeout: Duration by config { + "videobridge.websockets.idle-timeout".from(JitsiConfig.newConfig) + } + companion object { @JvmField val config = WebsocketServiceConfig() diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index 5904c1f62f..cb49a9d49a 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -225,6 +225,12 @@ videobridge { server-id = "default-id" // Whether to negotiate WebSocket compression (permessage-deflate) enable-compression = true + // Whether to send keepalive pings on idle websockets + send-keepalive-pings = true + // The keepalive ping interval + keepalive-ping-interval = 15 seconds + // The websocket idle timeout + idle-timeout = 60 seconds // Optional, even when 'enabled' is set to true #tls = true From e1ae15bd0b8e16e405b9cc2a3530aefa1f3ba62e Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Wed, 24 Jan 2024 17:13:26 -0500 Subject: [PATCH 071/189] Bump Bouncycastle to version 1.77. (#2085) --- jitsi-media-transform/pom.xml | 8 ++++---- pom.xml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jitsi-media-transform/pom.xml b/jitsi-media-transform/pom.xml index 1e54e4d958..25a4beee56 100644 --- a/jitsi-media-transform/pom.xml +++ b/jitsi-media-transform/pom.xml @@ -22,7 +22,7 @@ ${project.groupId} jitsi-srtp - 1.1-12-ga64adcc + 1.1-15-ga19c05a ${project.groupId} @@ -55,17 +55,17 @@ org.bouncycastle - bctls-jdk15on + bctls-jdk18on ${bouncycastle.version} org.bouncycastle - bcprov-jdk15on + bcprov-jdk18on ${bouncycastle.version} org.bouncycastle - bcpkix-jdk15on + bcpkix-jdk18on ${bouncycastle.version} diff --git a/pom.xml b/pom.xml index 3cec068b10..5dcade75b1 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ 4.6.0 3.0.10 2.12.4 - 1.70 + 1.77 0.16.0 UTF-8 From 573abcdf4c7b3f17493d30550811ad7a8fd27a64 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Wed, 24 Jan 2024 17:13:34 -0500 Subject: [PATCH 072/189] Fix compatibility with DTLS 1.3. (#2086) Remove lingering pieces of DTLS 1.0 support. --- .../org/jitsi/nlj/dtls/TlsClientImpl.kt | 9 ++---- .../org/jitsi/nlj/dtls/TlsServerImpl.kt | 31 +++---------------- 2 files changed, 6 insertions(+), 34 deletions(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsClientImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsClientImpl.kt index 10423a5087..3800b3dffe 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsClientImpl.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsClientImpl.kt @@ -137,6 +137,7 @@ class TlsClientImpl( override fun notifyHandshakeComplete() { super.notifyHandshakeComplete() + logger.cinfo { "Negotiated DTLS version ${context.securityParameters.negotiatedVersion}" } context.resumableSession?.let { newSession -> session?.let { existingSession -> @@ -163,13 +164,7 @@ class TlsClientImpl( ) } - override fun notifyServerVersion(serverVersion: ProtocolVersion?) { - super.notifyServerVersion(serverVersion) - - logger.cinfo { "Negotiated DTLS version $serverVersion" } - } - - override fun getSupportedVersions(): Array = arrayOf(ProtocolVersion.DTLSv12) + override fun getSupportedVersions(): Array = arrayOf(ProtocolVersion.DTLSv12) override fun notifyAlertRaised(alertLevel: Short, alertDescription: Short, message: String?, cause: Throwable?) = logger.notifyAlertRaised(alertLevel, alertDescription, message, cause) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsServerImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsServerImpl.kt index 3a9007fe24..f7d1dfd29c 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsServerImpl.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsServerImpl.kt @@ -114,37 +114,21 @@ class TlsServerImpl( (context.crypto as BcTlsCrypto), PrivateKeyFactory.createKey(certificateInfo.keyPair.private.encoded), certificateInfo.certificate, - /* For DTLS 1.0 support (needed for Jigasi) we can't set this to sha256 fixed */ - if (TlsUtils.isSignatureAlgorithmsExtensionAllowed(context.serverVersion)) { - SignatureAndHashAlgorithm( - HashAlgorithm.sha256, - SignatureAlgorithm.ecdsa - ) - } else { - null - } + SignatureAndHashAlgorithm(HashAlgorithm.sha256, SignatureAlgorithm.ecdsa) ) } override fun getCertificateRequest(): CertificateRequest { val signatureAlgorithms = Vector(1) signatureAlgorithms.add(SignatureAndHashAlgorithm(HashAlgorithm.sha256, SignatureAlgorithm.ecdsa)) - return when (context.clientVersion) { - ProtocolVersion.DTLSv12 -> { - CertificateRequest( - shortArrayOf(ClientCertificateType.ecdsa_sign), - signatureAlgorithms, - null - ) - } - else -> throw DtlsUtils.DtlsException("Unsupported version: ${context.clientVersion}") - } + return CertificateRequest(shortArrayOf(ClientCertificateType.ecdsa_sign), signatureAlgorithms, null) } override fun getHandshakeTimeoutMillis(): Int = DtlsUtils.config.handshakeTimeout.toMillis().toInt() override fun notifyHandshakeComplete() { super.notifyHandshakeComplete() + logger.cinfo { "Negotiated DTLS version ${context.securityParameters.negotiatedVersion}" } context.resumableSession?.let { newSession -> val newSessionIdHex = ByteBuffer.wrap(newSession.sessionID).toHex() @@ -182,18 +166,11 @@ class TlsServerImpl( notifyClientCertificateReceived(clientCertificate) } - override fun notifyClientVersion(clientVersion: ProtocolVersion?) { - super.notifyClientVersion(clientVersion) - - logger.cinfo { "Negotiated DTLS version $clientVersion" } - } - override fun notifyAlertRaised(alertLevel: Short, alertDescription: Short, message: String?, cause: Throwable?) = logger.notifyAlertRaised(alertLevel, alertDescription, message, cause) override fun notifyAlertReceived(alertLevel: Short, alertDescription: Short) = logger.notifyAlertReceived(alertLevel, alertDescription) - override fun getSupportedVersions(): Array = - ProtocolVersion.DTLSv12.downTo(ProtocolVersion.DTLSv10) + override fun getSupportedVersions(): Array = arrayOf(ProtocolVersion.DTLSv12) } From f9314a2370ed0e1cd6ce213b21c580b3a7c71472 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Mon, 29 Jan 2024 14:05:32 -0600 Subject: [PATCH 073/189] Remove backward compat for source names. (#2087) * Remove backward compat for source names. * Remove deprecated sendVideoConstraints. * ref: Remove SenderVideoConstraintsMessage. * ref: Remove "V2" from function name. --- doc/constraints.md | 13 ---- .../org/jitsi/videobridge/Conference.java | 4 +- .../org/jitsi/videobridge/AbstractEndpoint.kt | 15 +--- .../kotlin/org/jitsi/videobridge/Endpoint.kt | 78 +++---------------- .../cc/allocation/AllocationSettings.kt | 63 +++------------ .../cc/allocation/BitrateController.kt | 37 ++------- .../colibri2/Colibri2ConferenceHandler.kt | 8 +- .../load_management/LastNReducer.kt | 2 +- .../message/BridgeChannelMessage.kt | 54 +------------ .../videobridge/relay/RelayedEndpoint.kt | 19 +---- .../org/jitsi/videobridge/ConferenceTest.kt | 2 +- .../cc/allocation/AllocationSettingsTest.kt | 46 +---------- .../allocation/BitrateControllerPerfTest.kt | 6 +- .../cc/allocation/BitrateControllerTest.kt | 4 - .../load_management/LastNReducerTest.kt | 2 +- .../message/BridgeChannelMessageTest.kt | 38 +-------- 16 files changed, 49 insertions(+), 342 deletions(-) diff --git a/doc/constraints.md b/doc/constraints.md index 79875d8125..5cb6adb15f 100644 --- a/doc/constraints.md +++ b/doc/constraints.md @@ -12,16 +12,3 @@ higher than the specified need not be transmitted for a specific video source: "maxHeight": 180 } ``` - -The legacy format prior to the multi-stream support was endpoint scoped. This message is still sent to old clients, but -will be removed in the future. - -``` -{ - "colibriClass": "SenderVideoConstraints", - "videoConstraints": { - "idealHeight": 180 - } -} -``` - diff --git a/jvb/src/main/java/org/jitsi/videobridge/Conference.java b/jvb/src/main/java/org/jitsi/videobridge/Conference.java index a7afe7ee6f..456985d2a5 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Conference.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Conference.java @@ -718,7 +718,6 @@ public AbstractEndpoint findSourceOwner(@NotNull String sourceName) * @param iceControlling {@code true} if the ICE agent of this endpoint's * transport will be initialized to serve as a controlling ICE agent; * otherwise, {@code false} - * @param sourceNames whether this endpoint signaled the source names support. * @param doSsrcRewriting whether this endpoint signaled SSRC rewriting support. * @return an Endpoint participating in this Conference */ @@ -726,7 +725,6 @@ public AbstractEndpoint findSourceOwner(@NotNull String sourceName) public Endpoint createLocalEndpoint( String id, boolean iceControlling, - boolean sourceNames, boolean doSsrcRewriting, boolean visitor, boolean privateAddresses) @@ -738,7 +736,7 @@ public Endpoint createLocalEndpoint( } final Endpoint endpoint = new Endpoint( - id, this, logger, iceControlling, sourceNames, doSsrcRewriting, visitor, privateAddresses); + id, this, logger, iceControlling, doSsrcRewriting, visitor, privateAddresses); videobridge.localEndpointCreated(visitor); subscribeToEndpointEvents(endpoint); diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt index b52f121552..40816f7eb0 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt @@ -236,7 +236,7 @@ abstract class AbstractEndpoint protected constructor( val newReceiverMaxVideoConstraints = VideoConstraints(newMaxHeight, -1.0) if (newReceiverMaxVideoConstraints != oldReceiverMaxVideoConstraints) { maxReceiverVideoConstraints[sourceName] = newReceiverMaxVideoConstraints - sendVideoConstraintsV2(sourceName, newReceiverMaxVideoConstraints) + sendVideoConstraints(sourceName, newReceiverMaxVideoConstraints) } } @@ -265,17 +265,6 @@ abstract class AbstractEndpoint protected constructor( */ abstract fun setExtmapAllowMixed(allow: Boolean) - /** - * Notifies this instance that the max video constraints that the bridge - * needs to receive from this endpoint has changed. Each implementation - * handles this notification differently. - * - * @param maxVideoConstraints the max video constraints that the bridge - * needs to receive from this endpoint - */ - @Deprecated("use sendVideoConstraintsV2") - protected abstract fun sendVideoConstraints(maxVideoConstraints: VideoConstraints) - /** * Notifies this instance that the max video constraints that the bridge needs to receive from a source of this * endpoint has changed. Each implementation handles this notification differently. @@ -283,7 +272,7 @@ abstract class AbstractEndpoint protected constructor( * @param sourceName the name of the media source * @param maxVideoConstraints the max video constraints that the bridge needs to receive from the source */ - protected abstract fun sendVideoConstraintsV2(sourceName: String, maxVideoConstraints: VideoConstraints) + protected abstract fun sendVideoConstraints(sourceName: String, maxVideoConstraints: VideoConstraints) /** * Notifies this instance that a specified received wants to receive the specified video constraints from the media diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index 01cb09af3f..512168c948 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -61,11 +61,9 @@ import org.jitsi.videobridge.datachannel.DataChannelStack import org.jitsi.videobridge.datachannel.protocol.DataChannelPacket import org.jitsi.videobridge.datachannel.protocol.DataChannelProtocolConstants import org.jitsi.videobridge.message.BridgeChannelMessage -import org.jitsi.videobridge.message.ForwardedEndpointsMessage import org.jitsi.videobridge.message.ForwardedSourcesMessage import org.jitsi.videobridge.message.ReceiverVideoConstraintsMessage import org.jitsi.videobridge.message.SenderSourceConstraintsMessage -import org.jitsi.videobridge.message.SenderVideoConstraintsMessage import org.jitsi.videobridge.relay.AudioSourceDesc import org.jitsi.videobridge.relay.RelayedEndpoint import org.jitsi.videobridge.rest.root.debug.EndpointDebugFeatures @@ -107,7 +105,6 @@ class Endpoint @JvmOverloads constructor( * as a controlling ICE agent, false otherwise */ iceControlling: Boolean, - private val isUsingSourceNames: Boolean, private val doSsrcRewriting: Boolean, /** * Whether this endpoint is in "visitor" mode, i.e. should be invisible to other endpoints. @@ -205,9 +202,6 @@ class Endpoint @JvmOverloads constructor( // Intentional no-op } - override fun forwardedEndpointsChanged(forwardedEndpoints: Set) = - sendForwardedEndpointsMessage(forwardedEndpoints) - override fun forwardedSourcesChanged(forwardedSources: Set) { sendForwardedSourcesMessage(forwardedSources) } @@ -221,8 +215,7 @@ class Endpoint @JvmOverloads constructor( }, { getOrderedEndpoints() }, diagnosticContext, - logger, - isUsingSourceNames, + logger ) /** Whether any sources are suspended from being sent to this endpoint because of BWE. */ @@ -329,7 +322,7 @@ class Endpoint @JvmOverloads constructor( conference.videobridge.statistics.totalVisitors.inc() } - logger.info("Created new endpoint isUsingSourceNames=$isUsingSourceNames, iceControlling=$iceControlling") + logger.info("Created new endpoint, iceControlling=$iceControlling") } override var mediaSources: Array @@ -514,18 +507,14 @@ class Endpoint @JvmOverloads constructor( // TODO: this should be part of an EndpointMessageTransport.EventHandler interface fun endpointMessageTransportConnected() { sendAllVideoConstraints() - if (isUsingSourceNames) { - sendForwardedSourcesMessage(bitrateController.forwardedSources) - } else { - sendForwardedEndpointsMessage(bitrateController.forwardedEndpoints) - } + sendForwardedSourcesMessage(bitrateController.forwardedSources) videoSsrcs.sendAllMappings() audioSsrcs.sendAllMappings() } private fun sendAllVideoConstraints() { maxReceiverVideoConstraints.forEach { (sourceName, constraints) -> - sendVideoConstraintsV2(sourceName, constraints) + sendVideoConstraints(sourceName, constraints) } } @@ -566,36 +555,17 @@ class Endpoint @JvmOverloads constructor( } } - @Deprecated("use sendVideoConstraintsV2") - override fun sendVideoConstraints(maxVideoConstraints: VideoConstraints) { - // Note that it's up to the client to respect these constraints. - if (mediaSources.isEmpty()) { - logger.cdebug { "Suppressing sending a SenderVideoConstraints message, endpoint has no streams." } - } else { - val senderVideoConstraintsMessage = SenderVideoConstraintsMessage(maxVideoConstraints.maxHeight) - logger.cdebug { "Sender constraints changed: ${senderVideoConstraintsMessage.toJson()}" } - sendMessage(senderVideoConstraintsMessage) - } - } - - override fun sendVideoConstraintsV2(sourceName: String, maxVideoConstraints: VideoConstraints) { + override fun sendVideoConstraints(sourceName: String, maxVideoConstraints: VideoConstraints) { // Note that it's up to the client to respect these constraints. if (findMediaSourceDesc(sourceName) == null) { logger.warn { "Suppressing sending a SenderVideoConstraints message, endpoint has no such source: $sourceName" } } else { - if (isUsingSourceNames) { - val senderSourceConstraintsMessage = - SenderSourceConstraintsMessage(sourceName, maxVideoConstraints.maxHeight) - logger.cdebug { "Sender constraints changed: ${senderSourceConstraintsMessage.toJson()}" } - sendMessage(senderSourceConstraintsMessage) - } else { - maxReceiverVideoConstraints[sourceName]?.let { - sendVideoConstraints(it) - } - ?: logger.error("No max receiver constraints mapping found for: $sourceName") - } + val senderSourceConstraintsMessage = + SenderSourceConstraintsMessage(sourceName, maxVideoConstraints.maxHeight) + logger.cdebug { "Sender constraints changed: ${senderSourceConstraintsMessage.toJson()}" } + sendMessage(senderSourceConstraintsMessage) } } @@ -726,28 +696,6 @@ class Endpoint @JvmOverloads constructor( return true } - /** - * Sends a message to this endpoint in order to notify it that the set of endpoints for which the bridge - * is sending video has changed. - * - * @param forwardedEndpoints the collection of forwarded endpoints. - */ - @Deprecated("", ReplaceWith("sendForwardedSourcesMessage"), DeprecationLevel.WARNING) - fun sendForwardedEndpointsMessage(forwardedEndpoints: Collection) { - if (isUsingSourceNames) { - return - } - - val msg = ForwardedEndpointsMessage(forwardedEndpoints) - TaskPools.IO_POOL.execute { - try { - sendMessage(msg) - } catch (t: Throwable) { - logger.warn("Failed to send message:", t) - } - } - } - /** * Sends a message to this endpoint in order to notify it that the set of media sources for which the bridge * is sending video has changed. @@ -755,10 +703,6 @@ class Endpoint @JvmOverloads constructor( * @param forwardedSources the collection of forwarded media sources (by name). */ fun sendForwardedSourcesMessage(forwardedSources: Collection) { - if (!isUsingSourceNames) { - return - } - val msg = ForwardedSourcesMessage(forwardedSources) TaskPools.IO_POOL.execute { try { @@ -851,9 +795,9 @@ class Endpoint @JvmOverloads constructor( fun isOversending(): Boolean = bitrateController.isOversending() /** - * Returns how many endpoints this Endpoint is currently forwarding video for + * Returns how many video sources are currently forwarding to this endpoint. */ - fun numForwardedEndpoints(): Int = bitrateController.numForwardedEndpoints() + fun numForwardedSources(): Int = bitrateController.numForwardedSources() fun setBandwidthAllocationSettings(message: ReceiverVideoConstraintsMessage) { initialReceiverConstraintsReceived = true diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt index c644595069..73ca5fb400 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt @@ -23,7 +23,6 @@ import org.jitsi.utils.logging2.LoggerImpl import org.jitsi.utils.logging2.createChildLogger import org.jitsi.videobridge.cc.config.BitrateControllerConfig.Companion.config import org.jitsi.videobridge.message.ReceiverVideoConstraintsMessage -import org.jitsi.videobridge.util.endpointIdToSourceName /** * This class encapsulates all of the client-controlled settings for bandwidth allocation. @@ -60,17 +59,10 @@ data class AllocationSettings @JvmOverloads constructor( * the overall state changed. */ internal class AllocationSettingsWrapper( - private val useSourceNames: Boolean, parentLogger: Logger = LoggerImpl(AllocationSettingsWrapper::class.java.name) ) { private val logger = createChildLogger(parentLogger) - /** - * The last selected endpoints set signaled by the receiving endpoint. - */ - @Deprecated("", ReplaceWith("selectedSources"), DeprecationLevel.WARNING) - private var selectedEndpoints = emptyList() - /** * The last selected sources set signaled by the receiving endpoint. */ @@ -84,9 +76,6 @@ internal class AllocationSettingsWrapper( private var assumedBandwidthBps: Long = -1 - @Deprecated("", ReplaceWith("onStageSources"), DeprecationLevel.WARNING) - private var onStageEndpoints: List = emptyList() - private var onStageSources: List = emptyList() private var allocationSettings = create() @@ -111,35 +100,16 @@ internal class AllocationSettingsWrapper( changed = true } } - if (useSourceNames) { - message.selectedSources?.let { - if (selectedSources != it) { - selectedSources = it - changed = true - } - } - message.onStageSources?.let { - if (onStageSources != it) { - onStageSources = it - changed = true - } - } - } else { - message.selectedEndpoints?.let { - logger.warn("Setting deprecated selectedEndpoints=$it") - val newSelectedSources = it.map { endpoint -> endpointIdToSourceName(endpoint) } - if (selectedSources != newSelectedSources) { - selectedSources = newSelectedSources - changed = true - } + message.selectedSources?.let { + if (selectedSources != it) { + selectedSources = it + changed = true } - message.onStageEndpoints?.let { - logger.warn("Setting deprecated onStateEndpoints=$it") - val newOnStageSources = it.map { endpoint -> endpointIdToSourceName(endpoint) } - if (onStageSources != newOnStageSources) { - onStageSources = newOnStageSources - changed = true - } + } + message.onStageSources?.let { + if (onStageSources != it) { + onStageSources = it + changed = true } } message.defaultConstraints?.let { @@ -149,19 +119,8 @@ internal class AllocationSettingsWrapper( } } message.constraints?.let { - var newConstraints = it - - // Convert endpoint IDs to source names - if (!useSourceNames) { - newConstraints = HashMap(it.size) - it.entries.forEach { - entry -> - newConstraints[endpointIdToSourceName(entry.key)] = entry.value - } - } - - if (this.videoConstraints != newConstraints) { - this.videoConstraints = newConstraints + if (this.videoConstraints != it) { + this.videoConstraints = it changed = true } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt index de2e61400d..48b2ba21ac 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt @@ -48,20 +48,12 @@ class BitrateController @JvmOverloads constructor( endpointsSupplier: Supplier>, private val diagnosticContext: DiagnosticContext, parentLogger: Logger, - private val useSourceNames: Boolean, private val clock: Clock = Clock.systemUTC() ) { val eventEmitter = SyncEventEmitter() private val bitrateAllocatorEventHandler = BitrateAllocatorEventHandler() - /** - * Keep track of the "forwarded" endpoints, i.e. the endpoints for which we are forwarding *some* layer. - */ - @Deprecated("", ReplaceWith("forwardedSources"), DeprecationLevel.WARNING) - var forwardedEndpoints: Set = emptySet() - private set - /** * Keep track of the "forwarded" sources, i.e. the media sources for which we are forwarding *some* layer. */ @@ -99,7 +91,7 @@ class BitrateController @JvmOverloads constructor( ) fun hasSuspendedSources() = bandwidthAllocator.allocation.hasSuspendedSources - private val allocationSettingsWrapper = AllocationSettingsWrapper(useSourceNames, parentLogger) + private val allocationSettingsWrapper = AllocationSettingsWrapper(parentLogger) val allocationSettings get() = allocationSettingsWrapper.get() @@ -127,10 +119,8 @@ class BitrateController @JvmOverloads constructor( fun expire() = bandwidthAllocator.expire() - /** - * Return the number of endpoints whose streams are currently being forwarded. - */ - fun numForwardedEndpoints(): Int = forwardedEndpoints.size + /** Return the number of sources currently being forwarded. */ + fun numForwardedSources(): Int = forwardedSources.size fun getTotalOversendingTime(): Duration = oversendingTimeTracker.totalTimeOn() fun isOversending() = oversendingTimeTracker.state fun bandwidthChanged(newBandwidthBps: Long) { @@ -154,7 +144,6 @@ class BitrateController @JvmOverloads constructor( get() = JSONObject().apply { put("bitrate_allocator", bandwidthAllocator.debugState) put("packet_handler", packetHandler.debugState) - put("forwardedEndpoints", forwardedEndpoints.toString()) put("forwardedSources", forwardedSources.toString()) put("oversending", oversendingTimeTracker.state) put("total_oversending_time_secs", oversendingTimeTracker.totalTimeOn().seconds) @@ -170,7 +159,6 @@ class BitrateController @JvmOverloads constructor( fun setBandwidthAllocationSettings(message: ReceiverVideoConstraintsMessage) { if (allocationSettingsWrapper.setBandwidthAllocationSettings(message)) { - // TODO write a test for a user which uses only the endpoint based constraints bandwidthAllocator.update(allocationSettingsWrapper.get()) } } @@ -256,7 +244,6 @@ class BitrateController @JvmOverloads constructor( } interface EventHandler { - fun forwardedEndpointsChanged(forwardedEndpoints: Set) fun forwardedSourcesChanged(forwardedSources: Set) fun effectiveVideoConstraintsChanged( oldEffectiveConstraints: EffectiveConstraintsMap, @@ -276,18 +263,10 @@ class BitrateController @JvmOverloads constructor( // Actually implement the allocation (configure the packet filter to forward the chosen target layers). packetHandler.allocationChanged(allocation) - if (useSourceNames) { - val newForwardedSources = allocation.forwardedSources - if (forwardedSources != newForwardedSources) { - forwardedSources = newForwardedSources - eventEmitter.fireEvent { forwardedSourcesChanged(newForwardedSources) } - } - } else { - val newForwardedEndpoints = allocation.forwardedEndpoints - if (forwardedEndpoints != newForwardedEndpoints) { - forwardedEndpoints = newForwardedEndpoints - eventEmitter.fireEvent { forwardedEndpointsChanged(newForwardedEndpoints) } - } + val newForwardedSources = allocation.forwardedSources + if (forwardedSources != newForwardedSources) { + forwardedSources = newForwardedSources + eventEmitter.fireEvent { forwardedSourcesChanged(newForwardedSources) } } oversendingTimeTracker.setState(allocation.oversending) @@ -309,7 +288,7 @@ class BitrateController @JvmOverloads constructor( } /** - * Abstracts a source endpoint for the purposes of [BandwidthAllocator]. + * Abstracts a media source for the purposes of [BandwidthAllocator]. */ interface MediaSourceContainer { val id: String diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt index 8e65fb4f1b..0e33906f70 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt @@ -174,13 +174,15 @@ class Colibri2ConferenceHandler( Condition.bad_request, "Attempt to create endpoint ${c2endpoint.id} with no " ) - val sourceNames = c2endpoint.hasCapability(Capability.CAP_SOURCE_NAME_SUPPORT) - val ssrcRewriting = sourceNames && c2endpoint.hasCapability(Capability.CAP_SSRC_REWRITING_SUPPORT) + if (!c2endpoint.hasCapability(Capability.CAP_SOURCE_NAME_SUPPORT)) { + throw IqProcessingException(Condition.bad_request, "Source name support is mandatory.") + } + + val ssrcRewriting = c2endpoint.hasCapability(Capability.CAP_SSRC_REWRITING_SUPPORT) val privateAddresses = c2endpoint.hasCapability(Capability.CAP_PRIVATE_ADDRESS_CONNECTIVITY) conference.createLocalEndpoint( c2endpoint.id, transport.iceControlling, - sourceNames, ssrcRewriting, c2endpoint.mucRole == MUCRole.visitor, privateAddresses diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/LastNReducer.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/LastNReducer.kt index 529c4362ee..de4f0f3f32 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/LastNReducer.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/LastNReducer.kt @@ -71,7 +71,7 @@ class LastNReducer( .asSequence() .filterIsInstance() .map { - it.numForwardedEndpoints() + it.numForwardedSources() } .maxOrNull() } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt index 6a0d65f461..71daa008ae 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt @@ -20,7 +20,6 @@ import com.fasterxml.jackson.annotation.JsonAnySetter import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonInclude -import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.annotation.JsonSubTypes import com.fasterxml.jackson.annotation.JsonTypeInfo import com.fasterxml.jackson.core.JsonParser @@ -51,11 +50,9 @@ import java.util.concurrent.atomic.AtomicLong JsonSubTypes.Type(value = LastNMessage::class, name = LastNMessage.TYPE), JsonSubTypes.Type(value = DominantSpeakerMessage::class, name = DominantSpeakerMessage.TYPE), JsonSubTypes.Type(value = EndpointConnectionStatusMessage::class, name = EndpointConnectionStatusMessage.TYPE), - JsonSubTypes.Type(value = ForwardedEndpointsMessage::class, name = ForwardedEndpointsMessage.TYPE), JsonSubTypes.Type(value = ForwardedSourcesMessage::class, name = ForwardedSourcesMessage.TYPE), JsonSubTypes.Type(value = VideoSourcesMap::class, name = VideoSourcesMap.TYPE), JsonSubTypes.Type(value = AudioSourcesMap::class, name = AudioSourcesMap.TYPE), - JsonSubTypes.Type(value = SenderVideoConstraintsMessage::class, name = SenderVideoConstraintsMessage.TYPE), JsonSubTypes.Type(value = SenderSourceConstraintsMessage::class, name = SenderSourceConstraintsMessage.TYPE), JsonSubTypes.Type(value = AddReceiverMessage::class, name = AddReceiverMessage.TYPE), JsonSubTypes.Type(value = RemoveReceiverMessage::class, name = RemoveReceiverMessage.TYPE), @@ -116,11 +113,9 @@ open class MessageHandler { is LastNMessage -> lastN(message) is DominantSpeakerMessage -> dominantSpeaker(message) is EndpointConnectionStatusMessage -> endpointConnectionStatus(message) - is ForwardedEndpointsMessage -> forwardedEndpoints(message) is ForwardedSourcesMessage -> forwardedSources(message) is VideoSourcesMap -> videoSourcesMap(message) is AudioSourcesMap -> audioSourcesMap(message) - is SenderVideoConstraintsMessage -> senderVideoConstraints(message) is SenderSourceConstraintsMessage -> senderSourceConstraints(message) is AddReceiverMessage -> addReceiver(message) is RemoveReceiverMessage -> removeReceiver(message) @@ -143,11 +138,9 @@ open class MessageHandler { open fun lastN(message: LastNMessage) = unhandledMessageReturnNull(message) open fun dominantSpeaker(message: DominantSpeakerMessage) = unhandledMessageReturnNull(message) open fun endpointConnectionStatus(message: EndpointConnectionStatusMessage) = unhandledMessageReturnNull(message) - open fun forwardedEndpoints(message: ForwardedEndpointsMessage) = unhandledMessageReturnNull(message) open fun forwardedSources(message: ForwardedSourcesMessage) = unhandledMessageReturnNull(message) open fun videoSourcesMap(message: VideoSourcesMap) = unhandledMessageReturnNull(message) open fun audioSourcesMap(message: AudioSourcesMap) = unhandledMessageReturnNull(message) - open fun senderVideoConstraints(message: SenderVideoConstraintsMessage) = unhandledMessageReturnNull(message) open fun senderSourceConstraints(message: SenderSourceConstraintsMessage) = unhandledMessageReturnNull(message) open fun addReceiver(message: AddReceiverMessage) = unhandledMessageReturnNull(message) open fun removeReceiver(message: RemoveReceiverMessage) = unhandledMessageReturnNull(message) @@ -310,22 +303,6 @@ class EndpointConnectionStatusMessage( } } -/** - * A message sent from the bridge to a client, indicating the set of endpoints that are currently being forwarded. - */ -@Deprecated("Use ForwardedSourcesMessage", ReplaceWith("ForwardedSourcesMessage"), DeprecationLevel.WARNING) -class ForwardedEndpointsMessage( - /** - * The set of endpoints for which the bridge is currently sending video. - */ - @get:JsonProperty("lastNEndpoints") - val forwardedEndpoints: Collection -) : BridgeChannelMessage() { - companion object { - const val TYPE = "LastNEndpointsChangeEvent" - } -} - /** * A message sent from the bridge to a client, indicating the set of media sources that are currently being forwarded. */ @@ -392,32 +369,6 @@ class AudioSourcesMap( } } -/** - * A message sent from the bridge to a client (sender), indicating constraints for the sender's video streams. - * - * TODO: consider and adjust the format of videoConstraints. Do we need all of the VideoConstraints fields? Document. - * TODO: update https://github.com/jitsi/jitsi-videobridge/blob/master/doc/constraints.md before removing. - */ -@Deprecated("", ReplaceWith("SenderSourceConstraints"), DeprecationLevel.WARNING) -class SenderVideoConstraintsMessage(val videoConstraints: VideoConstraints) : BridgeChannelMessage() { - constructor(maxHeight: Int) : this(VideoConstraints(maxHeight)) - - /** - * Serialize manually because it's faster than Jackson. - * - * We use the "idealHeight" format that the jitsi-meet client expects. - */ - override fun createJson(): String = - """{"colibriClass":"$TYPE", "videoConstraints":{"idealHeight":${videoConstraints.idealHeight}}}""" - - @JsonIgnoreProperties(ignoreUnknown = true) - data class VideoConstraints(val idealHeight: Int) - - companion object { - const val TYPE = "SenderVideoConstraints" - } -} - /** * A message sent from the bridge to a client (sender), indicating constraints for the sender's video stream. */ @@ -482,11 +433,7 @@ class RemoveReceiverMessage( class ReceiverVideoConstraintsMessage( val lastN: Int? = null, - @Deprecated("", ReplaceWith("selectedSources"), DeprecationLevel.WARNING) - val selectedEndpoints: List? = null, val selectedSources: List? = null, - @Deprecated("", ReplaceWith("onStageSources"), DeprecationLevel.WARNING) - val onStageEndpoints: List? = null, val onStageSources: List? = null, val defaultConstraints: VideoConstraints? = null, val constraints: Map? = null, @@ -531,6 +478,7 @@ class SourceVideoTypeMessage( /** * A signaling the type of video stream an endpoint has available. */ +@Deprecated("", ReplaceWith("SourceVideoTypeMessage"), DeprecationLevel.WARNING) class VideoTypeMessage( val videoType: VideoType, /** diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt index 079fe2dd8f..37beba0136 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt @@ -124,8 +124,6 @@ class RelayedEndpoint( // Visitors are never advertised between relays, so relayed endpoints are never visitors. override val visitor = false - fun hasReceiveSsrcs(): Boolean = streamInformationStore.receiveSsrcs.isNotEmpty() - /** Relayed endpoints are not automatically expired. **/ override fun shouldExpire(): Boolean = false @@ -144,20 +142,7 @@ class RelayedEndpoint( override fun setExtmapAllowMixed(allow: Boolean) = streamInformationStore.setExtmapAllowMixed(allow) - @Deprecated("use sendVideoConstraintsV2") - override fun sendVideoConstraints(maxVideoConstraints: VideoConstraints) { - relay.sendMessage( - AddReceiverMessage( - RelayConfig.config.relayId, - id, - // source name - used in multi-stream - null, - maxVideoConstraints - ) - ) - } - - override fun sendVideoConstraintsV2(sourceName: String, maxVideoConstraints: VideoConstraints) { + override fun sendVideoConstraints(sourceName: String, maxVideoConstraints: VideoConstraints) { relay.sendMessage( AddReceiverMessage( RelayConfig.config.relayId, @@ -171,7 +156,7 @@ class RelayedEndpoint( fun relayMessageTransportConnected() { maxReceiverVideoConstraints.forEach { (sourceName, constraints) -> - sendVideoConstraintsV2(sourceName, constraints) + sendVideoConstraints(sourceName, constraints) } } diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/ConferenceTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/ConferenceTest.kt index 95aaebcb24..b0eb96f306 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/ConferenceTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/ConferenceTest.kt @@ -38,7 +38,7 @@ class ConferenceTest : ConfigTest() { with(Conference(videobridge, "id", name, null, false)) { endpointCount shouldBe 0 // TODO cover the case when they're true - createLocalEndpoint("abcdabcd", true, false, false, false, false) + createLocalEndpoint("abcdabcd", true, false, false, false) endpointCount shouldBe 1 debugState.shouldBeValidJson() } diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettingsTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettingsTest.kt index 51668dc9e6..540632c8e4 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettingsTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettingsTest.kt @@ -24,13 +24,11 @@ class AllocationSettingsTest : ShouldSpec() { context("computeVideoConstraints") { context("With client which supports source names") { context("no conversion from endpoint to source takes place") { - val allocationSettings = AllocationSettingsWrapper(true) + val allocationSettings = AllocationSettingsWrapper() allocationSettings.setBandwidthAllocationSettings( ReceiverVideoConstraintsMessage( onStageSources = listOf("S1", "S2"), - onStageEndpoints = listOf("E1", "E2"), selectedSources = listOf("S3", "S4"), - selectedEndpoints = listOf("E3", "E4"), constraints = mapOf( "S1" to VideoConstraints(720), "E1" to VideoConstraints(360) @@ -51,48 +49,6 @@ class AllocationSettingsTest : ShouldSpec() { ) } } - context("With client which doesn't support source names") { - context("Converts onStageEndpoints to onStageSources") { - val allocationSettings = AllocationSettingsWrapper(false) - allocationSettings.setBandwidthAllocationSettings( - ReceiverVideoConstraintsMessage( - onStageEndpoints = listOf("A", "C") - ) - ) - - allocationSettings.get().onStageEndpoints shouldBe emptyList() - allocationSettings.get().onStageSources shouldBe listOf("A-v0", "C-v0") - } - context("Converts selectedEndpoints to selectedSources") { - val allocationSettings = AllocationSettingsWrapper(false) - allocationSettings.setBandwidthAllocationSettings( - ReceiverVideoConstraintsMessage( - selectedEndpoints = listOf("A", "C") - ) - ) - - allocationSettings.get().selectedEndpoints shouldBe emptyList() - allocationSettings.get().selectedSources shouldBe listOf("A-v0", "C-v0") - } - context("Converts endpoints based constraints to source based ones") { - val allocationSettings = AllocationSettingsWrapper(false) - allocationSettings.setBandwidthAllocationSettings( - ReceiverVideoConstraintsMessage( - constraints = mapOf( - "A" to VideoConstraints(720, 15.0), - "B" to VideoConstraints(360, 24.0), - "C" to VideoConstraints(180, 30.0) - ) - ) - ) - - allocationSettings.get().videoConstraints shouldBe mapOf( - "A-v0" to VideoConstraints(720, 15.0), - "B-v0" to VideoConstraints(360, 24.0), - "C-v0" to VideoConstraints(180, 30.0) - ) - } - } } } } diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerPerfTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerPerfTest.kt index 6b7058fbfd..68fc0881a9 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerPerfTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerPerfTest.kt @@ -29,6 +29,7 @@ import org.jitsi.utils.nanos import org.jitsi.utils.secs import org.jitsi.utils.time.FakeClock import org.jitsi.videobridge.message.ReceiverVideoConstraintsMessage +import org.jitsi.videobridge.util.endpointIdToSourceName import java.util.function.Supplier import kotlin.random.Random import kotlin.time.ExperimentalTime @@ -67,7 +68,6 @@ class BitrateControllerPerfTest : StringSpec() { private val endpoints: MutableList = createEndpoints(*endpointIds.toTypedArray()) private val bc = BitrateController( object : BitrateController.EventHandler { - override fun forwardedEndpointsChanged(forwardedEndpoints: Set) { } override fun forwardedSourcesChanged(forwardedSources: Set) { } override fun effectiveVideoConstraintsChanged( oldEffectiveConstraints: EffectiveConstraintsMap, @@ -79,8 +79,6 @@ class BitrateControllerPerfTest : StringSpec() { Supplier { endpoints.toList() }, DiagnosticContext(), createLogger(), - // TODO cover the case for true? - false, clock, ).apply { // The BC only starts working 10 seconds after it first received media, so fake that. @@ -122,7 +120,7 @@ class BitrateControllerPerfTest : StringSpec() { bc.setBandwidthAllocationSettings( ReceiverVideoConstraintsMessage( - selectedEndpoints = selectedEndpoints, + selectedSources = selectedEndpoints.map { endpointIdToSourceName(it) }, defaultConstraints = VideoConstraints(maxFrameHeight) ) ) diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt index e50c0f7197..4649e80586 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt @@ -1369,8 +1369,6 @@ class BitrateControllerWrapper(initialEndpoints: List, val val bc = BitrateController( object : BitrateController.EventHandler { - override fun forwardedEndpointsChanged(forwardedEndpoints: Set) { } - override fun forwardedSourcesChanged(forwardedSources: Set) { Event(bwe, forwardedSources, clock.instant()).apply { logger.info("Forwarded sources changed: $this") @@ -1404,8 +1402,6 @@ class BitrateControllerWrapper(initialEndpoints: List, val Supplier { endpoints }, DiagnosticContext(), logger, - // TODO merge BitrateControllerNewTest with old and use this flag - true, clock ) diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/load_management/LastNReducerTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/load_management/LastNReducerTest.kt index 3350e4d582..1ff0a251b9 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/load_management/LastNReducerTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/load_management/LastNReducerTest.kt @@ -108,7 +108,7 @@ inline fun withLastNConfig(config: String, block: () -> T): T { */ private fun createMockConference(vararg epNumForwardedVideo: Int): Conference { val eps = epNumForwardedVideo.map { - mockk { every { numForwardedEndpoints() } returns it } + mockk { every { numForwardedSources() } returns it } }.toList() return mockk { every { endpoints } returns eps diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/message/BridgeChannelMessageTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/message/BridgeChannelMessageTest.kt index 9ef13769e8..06854a0f14 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/message/BridgeChannelMessageTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/message/BridgeChannelMessageTest.kt @@ -31,7 +31,6 @@ import io.kotest.matchers.types.shouldBeInstanceOf import org.jitsi.nlj.VideoType import org.jitsi.videobridge.cc.allocation.VideoConstraints import org.jitsi.videobridge.message.BridgeChannelMessage.Companion.parse -import org.json.simple.JSONArray import org.json.simple.JSONObject import org.json.simple.parser.JSONParser @@ -201,24 +200,6 @@ class BridgeChannelMessageTest : ShouldSpec() { parsed.active shouldBe "true" } - context("serializing and parsing ForwardedEndpointsMessage") { - val forwardedEndpoints = setOf("a", "b", "c") - - val message = ForwardedEndpointsMessage(forwardedEndpoints) - val parsed = parse(message.toJson()) - - parsed.shouldBeInstanceOf() - - parsed.forwardedEndpoints shouldContainExactly forwardedEndpoints - - // Make sure the forwardedEndpoints field is serialized as lastNEndpoints as the client (presumably) expects - val parsedJson = JSONParser().parse(message.toJson()) - parsedJson.shouldBeInstanceOf() - val parsedForwardedEndpoints = parsedJson["lastNEndpoints"] - parsedForwardedEndpoints.shouldBeInstanceOf() - parsedForwardedEndpoints.toList() shouldContainExactly forwardedEndpoints - } - context("serializing and parsing ForwardedSourcesMessage") { val forwardedSources = setOf("s1", "s2", "s3") @@ -236,18 +217,9 @@ class BridgeChannelMessageTest : ShouldSpec() { videoConstraints.maxFrameRate shouldBe 15.0 } - context("and SenderVideoConstraintsMessage") { - val senderVideoConstraintsMessage = SenderVideoConstraintsMessage(1080) - val parsed = parse(senderVideoConstraintsMessage.toJson()) - - parsed.shouldBeInstanceOf() - - parsed.videoConstraints.idealHeight shouldBe 1080 - } - context("serializing and parsing SenderSourceConstraintsMessage") { - val senderVideoConstraintsMessage = SenderSourceConstraintsMessage("s1", 1080) - val parsed = parse(senderVideoConstraintsMessage.toJson()) + val senderSourceConstraintsMessage = SenderSourceConstraintsMessage("s1", 1080) + val parsed = parse(senderSourceConstraintsMessage.toJson()) parsed.shouldBeInstanceOf() @@ -455,8 +427,6 @@ class BridgeChannelMessageTest : ShouldSpec() { parsed.shouldBeInstanceOf() parsed.lastN shouldBe 3 - parsed.onStageEndpoints shouldBe listOf("onstage1", "onstage2") - parsed.selectedEndpoints shouldBe listOf("selected1", "selected2") parsed.defaultConstraints shouldBe VideoConstraints(0) val constraints = parsed.constraints constraints.shouldNotBeNull() @@ -470,8 +440,6 @@ class BridgeChannelMessageTest : ShouldSpec() { val parsed = parse(RECEIVER_VIDEO_CONSTRAINTS_EMPTY) parsed.shouldBeInstanceOf() parsed.lastN shouldBe null - parsed.onStageEndpoints shouldBe null - parsed.selectedEndpoints shouldBe null parsed.defaultConstraints shouldBe null parsed.constraints shouldBe null } @@ -546,8 +514,6 @@ class BridgeChannelMessageTest : ShouldSpec() { { "colibriClass": "ReceiverVideoConstraints", "lastN": 3, - "selectedEndpoints": [ "selected1", "selected2" ], - "onStageEndpoints": [ "onstage1", "onstage2" ], "defaultConstraints": { "maxHeight": 0 }, "constraints": { "epOnStage": { "maxHeight": 720 }, From 443455639c88ab68073655ecdc7091e84b043784 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 30 Jan 2024 13:13:43 -0500 Subject: [PATCH 074/189] Bump jitsi-sctp version, hopefully fixing crashes. (#2088) --- jvb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/pom.xml b/jvb/pom.xml index 0f0cbb501b..bd76694748 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -108,7 +108,7 @@ ${project.groupId} sctp - 1.0-11-gcd70942 + 1.0-12-g5b45737 From a015be96887fd907a6c5579f94fb8c15659d81b2 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Wed, 31 Jan 2024 14:17:36 -0500 Subject: [PATCH 075/189] Bump jitsi-sctp version, hopefully fixing crashes, again. (#2089) * Bump jitsi-sctp version, hopefully fixing crashes, again. * Another bump, to use older arm64/Linux glibc. --- jvb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/pom.xml b/jvb/pom.xml index bd76694748..02dad82449 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -108,7 +108,7 @@ ${project.groupId} sctp - 1.0-12-g5b45737 + 1.0-14-ge26331d From 68ea2824db107afcf6a776f5f3799389c751e116 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 20 Feb 2024 17:16:35 -0500 Subject: [PATCH 076/189] Fix bug that would cause VP9/AV1 simulcast not to be routed properly after a switch. (#2101) --- .../cc/av1/Av1DDAdaptiveSourceProjectionContext.kt | 6 +++++- .../cc/vp9/Vp9AdaptiveSourceProjectionContext.kt | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt index 78159c4f0c..219f885090 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt @@ -99,7 +99,7 @@ class Av1DDAdaptiveSourceProjectionContext( val receivedTime = packetInfo.receivedTime val acceptResult = av1QualityFilter .acceptFrame(frame, incomingEncoding, targetIndex, receivedTime) - frame.isAccepted = acceptResult.accept && frame.index >= lastFrameNumberIndexResumption + frame.isAccepted = acceptResult.accept && frameIsProjectable(frame) if (frame.isAccepted) { val projection: Av1DDFrameProjection try { @@ -228,6 +228,9 @@ class Av1DDAdaptiveSourceProjectionContext( private fun frameIsNewSsrc(frame: Av1DDFrame): Boolean = lastAv1FrameProjection.av1Frame?.matchesSSRC(frame) != true + private fun frameIsProjectable(frame: Av1DDFrame): Boolean = + frameIsNewSsrc(frame) || frame.index >= lastFrameNumberIndexResumption + /** * Find the previous frame before the given one. */ @@ -286,6 +289,7 @@ class Av1DDAdaptiveSourceProjectionContext( // We can only switch on packets that carry a scalability structure, which is the first packet of a keyframe assert(frame.isKeyframe) assert(initialPacket.isStartOfFrame) + lastFrameNumberIndexResumption = frame.index var projectedSeqGap = 1 diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt index 1417f82c34..f489dc3257 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt @@ -106,7 +106,7 @@ class Vp9AdaptiveSourceProjectionContext( val receivedTime = packetInfo.receivedTime val acceptResult = vp9QualityFilter .acceptFrame(frame, incomingEncoding, targetIndex, receivedTime) - frame.isAccepted = acceptResult.accept && frame.index >= lastPicIdIndexResumption + frame.isAccepted = acceptResult.accept && frameIsProjectable(frame) if (frame.isAccepted) { val projection: Vp9FrameProjection try { @@ -200,6 +200,9 @@ class Vp9AdaptiveSourceProjectionContext( private fun frameIsNewSsrc(frame: Vp9Frame): Boolean = lastVp9FrameProjection.vp9Frame?.matchesSSRC(frame) != true + private fun frameIsProjectable(frame: Vp9Frame): Boolean = + frameIsNewSsrc(frame) || frame.index >= lastPicIdIndexResumption + /** * Find the previous frame before the given one. */ From 3e369e86047bbdf4c2091828eaeea057e3983bab Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 20 Feb 2024 17:18:16 -0500 Subject: [PATCH 077/189] Pass the appropriate log context to the adaptive source projection contexts. (#2096) This will then include the source epId and target ssrc. --- .../cc/AdaptiveSourceProjection.java | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjection.java b/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjection.java index c59ec62c9f..8cad7c69a0 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjection.java +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjection.java @@ -57,12 +57,6 @@ public class AdaptiveSourceProjection * and its instances for logging output. */ private final Logger logger; - /** - * The parent logger, so that we can pass it to the next created context - * TODO(brian): maybe we should allow a child logger to retrieve its - * parent? - */ - private final Logger parentLogger; /** * The main SSRC of the source (if simulcast is used, this is the SSRC @@ -110,7 +104,6 @@ public AdaptiveSourceProjection( { targetSsrc = source.getPrimarySSRC(); this.diagnosticContext = diagnosticContext; - this.parentLogger = parentLogger; this.logger = parentLogger.createChildLogger(AdaptiveSourceProjection.class.getName(), Map.of("targetSsrc", Long.toString(targetSsrc), "srcEpId", Objects.toString(source.getOwner(), ""))); @@ -218,7 +211,7 @@ private AdaptiveSourceProjectionContext getContext(@NotNull VideoRtpPacket rtpPa (context == null ? "creating new" : "changing to") + " VP8 context for source packet ssrc " + rtpPacket.getSsrc()); context = new VP8AdaptiveSourceProjectionContext( - diagnosticContext, rtpState, parentLogger); + diagnosticContext, rtpState, logger); } else if (!projectable && (!(context instanceof GenericAdaptiveSourceProjectionContext) || @@ -236,7 +229,7 @@ else if (!projectable ", ssrc " + rtpPacket.getSsrc() + ", hasTL=" + hasTemporalLayer + ", hasPID=" + hasPictureId + ")"; }); - context = new GenericAdaptiveSourceProjectionContext(payloadType, rtpState, parentLogger); + context = new GenericAdaptiveSourceProjectionContext(payloadType, rtpState, logger); } // no context switch @@ -252,7 +245,7 @@ else if (rtpPacket instanceof Vp9Packet) (context == null ? "creating new" : "changing to") + " VP9 context for source packet ssrc " + rtpPacket.getSsrc()); context = new Vp9AdaptiveSourceProjectionContext( - diagnosticContext, rtpState, parentLogger); + diagnosticContext, rtpState, logger); } return context; @@ -267,7 +260,7 @@ else if (rtpPacket instanceof Av1DDPacket) (context == null ? "creating new" : "changing to") + " AV1 DD context for source packet ssrc " + rtpPacket.getSsrc()); context = new Av1DDAdaptiveSourceProjectionContext( - diagnosticContext, rtpState, parentLogger); + diagnosticContext, rtpState, logger); } return context; @@ -281,7 +274,7 @@ else if (rtpPacket instanceof Av1DDPacket) logger.debug(() -> "adaptive source projection " + (context == null ? "creating new" : "changing to") + " generic context for payload type " + rtpPacket.getPayloadType()); - context = new GenericAdaptiveSourceProjectionContext(payloadType, rtpState, parentLogger); + context = new GenericAdaptiveSourceProjectionContext(payloadType, rtpState, logger); } return context; } From fbd563caaba18efaac35bac025ab17d6903d1cd2 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Tue, 20 Feb 2024 16:51:59 -0600 Subject: [PATCH 078/189] feat: Implement a Prometheus histogram of rtp transit time. (#2091) * chore: Update jicoco. * feat: Implement a Prometheus histogram of rtp transit time. --- .../kotlin/org/jitsi/nlj/stats/DelayStats.kt | 18 +--- .../videobridge/EndpointMessageTransport.java | 2 +- .../org/jitsi/videobridge/Videobridge.java | 7 +- .../videobridge/stats/PacketTransitStats.kt | 102 ++++++++++++++++-- jvb/src/main/resources/reference.conf | 13 +++ pom.xml | 2 +- 6 files changed, 116 insertions(+), 28 deletions(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/DelayStats.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/DelayStats.kt index f2db58b804..4a9aeb7534 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/DelayStats.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/DelayStats.kt @@ -16,19 +16,13 @@ package org.jitsi.nlj.stats -import org.jitsi.nlj.PacketInfo import org.jitsi.utils.OrderedJsonObject import org.jitsi.utils.stats.BucketStats -import java.time.Clock -import java.time.Duration import java.util.concurrent.atomic.LongAdder open class DelayStats(thresholds: List = defaultThresholds) : BucketStats(thresholds, "_delay_ms", "_ms") { - fun addDelay(delay: Duration) { - addDelay(delay.toMillis()) - } fun addDelay(delayMs: Long) = addValue(delayMs) companion object { @@ -36,16 +30,10 @@ open class DelayStats(thresholds: List = defaultThresholds) : } } -class PacketDelayStats( - thresholds: List = defaultThresholds, - private val clock: Clock = Clock.systemUTC() -) : DelayStats(thresholds) { +class PacketDelayStats(thresholds: List = defaultThresholds) : DelayStats(thresholds) { private val numPacketsWithoutTimestamps = LongAdder() - fun addPacket(packetInfo: PacketInfo) { - packetInfo.receivedTime?.let { addDelay(Duration.between(it, clock.instant())) } ?: run { - numPacketsWithoutTimestamps.increment() - } - } + + fun addUnknown() = numPacketsWithoutTimestamps.increment() override fun toJson(format: Format): OrderedJsonObject { return super.toJson(format).apply { diff --git a/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java b/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java index c0223afa45..7b6f2a28d2 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java +++ b/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java @@ -562,7 +562,7 @@ public BridgeChannelMessage endpointStats(@NotNull EndpointStats message) Conference conference = endpoint.getConference(); - if (conference == null || conference.isExpired()) + if (conference.isExpired()) { getLogger().warn("Unable to send EndpointStats, conference is null or expired"); return null; diff --git a/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java b/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java index 8b4ee46be0..2fa1088058 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java @@ -607,7 +607,12 @@ public OrderedJsonObject getDebugState(String conferenceId, String endpointId, b debugState.put("time", System.currentTimeMillis()); debugState.put("load-management", jvbLoadManager.getStats()); - debugState.put("overall_bridge_jitter", PacketTransitStats.getBridgeJitter()); + + Double jitter = PacketTransitStats.getBridgeJitter(); + if (jitter != null) + { + debugState.put("overall_bridge_jitter", jitter); + } JSONObject conferences = new JSONObject(); debugState.put("conferences", conferences); diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/PacketTransitStats.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/PacketTransitStats.kt index d779ba2ffe..a8001b7bef 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/PacketTransitStats.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/PacketTransitStats.kt @@ -15,14 +15,20 @@ */ package org.jitsi.videobridge.stats +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config import org.jitsi.nlj.PacketInfo import org.jitsi.nlj.stats.BridgeJitterStats import org.jitsi.nlj.stats.PacketDelayStats import org.jitsi.rtp.extensions.looksLikeRtcp import org.jitsi.rtp.extensions.looksLikeRtp import org.jitsi.utils.OrderedJsonObject +import org.jitsi.utils.logging2.createLogger import org.jitsi.utils.stats.BucketStats import org.jitsi.videobridge.Endpoint +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer +import java.time.Clock +import java.time.Duration /** * Track how long it takes for all RTP and RTCP packets to make their way through the bridge. @@ -31,18 +37,66 @@ import org.jitsi.videobridge.Endpoint * for packets going out to all endpoints. */ object PacketTransitStats { - private val rtpPacketDelayStats = PacketDelayStats() - private val rtcpPacketDelayStats = PacketDelayStats() + private val jsonEnabled: Boolean by config { + "videobridge.stats.transit-time.enable-json".from(JitsiConfig.newConfig) + } + private val prometheusEnabled: Boolean by config { + "videobridge.stats.transit-time.enable-prometheus".from(JitsiConfig.newConfig) + } + private val jitterEnabled: Boolean by config { + "videobridge.stats.transit-time.enable-jitter".from(JitsiConfig.newConfig) + } + private val enabled = jsonEnabled || prometheusEnabled || jitterEnabled + + private val logger = createLogger() + private val clock: Clock = Clock.systemUTC() + + init { + logger.info( + "Initializing, jsonEnabled=$jsonEnabled, prometheusEnabled=$prometheusEnabled, " + + "jitterEnabled=$jitterEnabled" + ) + } + + private val rtpPacketDelayStats = if (jsonEnabled) PacketDelayStats() else null + private val rtcpPacketDelayStats = if (jsonEnabled) PacketDelayStats() else null + private val prometheusRtpDelayStats = if (prometheusEnabled) { + PrometheusPacketDelayStats("rtp_transit_time") + } else { + null + } + private val prometheusRtcpDelayStats = if (prometheusEnabled) { + PrometheusPacketDelayStats("rtcp_transit_time") + } else { + null + } - private val bridgeJitterStats = BridgeJitterStats() + private val bridgeJitterStats = if (jitterEnabled) BridgeJitterStats() else null @JvmStatic fun packetSent(packetInfo: PacketInfo) { + if (!enabled) { + return + } + + val delayMs = packetInfo.receivedTime?.let { Duration.between(it, clock.instant()).toMillis() } if (packetInfo.packet.looksLikeRtp()) { - rtpPacketDelayStats.addPacket(packetInfo) - bridgeJitterStats.packetSent(packetInfo) + if (delayMs != null) { + rtpPacketDelayStats?.addDelay(delayMs) + prometheusRtpDelayStats?.addDelay(delayMs) + bridgeJitterStats?.packetSent(packetInfo) + } else { + rtpPacketDelayStats?.addUnknown() + prometheusRtpDelayStats?.addUnknown() + } } else if (packetInfo.packet.looksLikeRtcp()) { - rtcpPacketDelayStats.addPacket(packetInfo) + if (delayMs != null) { + rtcpPacketDelayStats?.addDelay(delayMs) + prometheusRtcpDelayStats?.addDelay(delayMs) + } else { + rtcpPacketDelayStats?.addUnknown() + prometheusRtcpDelayStats?.addUnknown() + } } } @@ -51,16 +105,44 @@ object PacketTransitStats { get() { val stats = OrderedJsonObject() stats["e2e_packet_delay"] = getPacketDelayStats() - stats["overall_bridge_jitter"] = bridgeJitterStats.jitter + bridgeJitterStats?.let { + stats["overall_bridge_jitter"] = it.jitter + } return stats } @JvmStatic val bridgeJitter - get() = bridgeJitterStats.jitter + get() = bridgeJitterStats?.jitter private fun getPacketDelayStats() = OrderedJsonObject().apply { - put("rtp", rtpPacketDelayStats.toJson(format = BucketStats.Format.CumulativeRight)) - put("rtcp", rtcpPacketDelayStats.toJson(format = BucketStats.Format.CumulativeRight)) + rtpPacketDelayStats?.let { + put("rtp", it.toJson(format = BucketStats.Format.CumulativeRight)) + } + rtcpPacketDelayStats?.let { + put("rtcp", it.toJson(format = BucketStats.Format.CumulativeRight)) + } + } +} + +class PrometheusPacketDelayStats(name: String) { + private val histogram = VideobridgeMetricsContainer.instance.registerHistogram( + name, + "Packet delay stats for $name", + 0.0, + 5.0, + 50.0, + 500.0 + ) + private val numPacketsWithoutTimestamps = VideobridgeMetricsContainer.instance.registerCounter( + "${name}_unknown_delay", + "Number of packets without an unknown delay ($name)" + ) + + fun addUnknown() { + numPacketsWithoutTimestamps.inc() + } + fun addDelay(delayMs: Long) { + histogram.histogram.observe(delayMs.toDouble()) } } diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index cb49a9d49a..0972caec80 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -219,6 +219,19 @@ videobridge { stats { // The interval at which stats are gathered. interval = 5 seconds + + // Statistics about the transit time of RTP/RTCP packets. Note that the collection code for the JSON and Prometheus + // outputs is different, and each has a slight performance impact, so the format(s) that are not needed should be + // kept disabled. + transit-time { + // Enable collection of transit time stats in JSON format. Available through /debug/jvb/stats/transit-time + enable-json = true + // Enable collection of transit time stats in Prometheus. Available through /metrics. + enable-prometheus = false + // Enable collection of internal jitter (difference in processing time). Available through + // /debug/jvb/stats/transit-time + enable-jitter = false + } } websockets { enabled = false diff --git a/pom.xml b/pom.xml index 5dcade75b1..b6152eb6a0 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 5.7.2 5.10.0 1.0-127-g6c65524 - 1.1-129-g23ac61c + 1.1-132-g906f995 1.13.8 3.0.0 3.5.1 From 7bd488850d1d4fa3dcfe4435bdec0de20c610dd2 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Tue, 20 Feb 2024 16:54:58 -0600 Subject: [PATCH 079/189] feat: Add a config option for default initial last n. (#2095) * feat: Add a config option for default initial last n. --- .../videobridge/cc/allocation/AllocationSettings.kt | 12 ++++++++++-- jvb/src/main/resources/reference.conf | 4 ++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt index 73ca5fb400..c9a1762e65 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt @@ -16,6 +16,8 @@ */ package org.jitsi.videobridge.cc.allocation +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config import org.jitsi.nlj.util.bps import org.jitsi.utils.OrderedJsonObject import org.jitsi.utils.logging2.Logger @@ -35,7 +37,7 @@ data class AllocationSettings @JvmOverloads constructor( val onStageSources: List = emptyList(), val selectedSources: List = emptyList(), val videoConstraints: Map = emptyMap(), - val lastN: Int = -1, + val lastN: Int = defaultInitialLastN, val defaultConstraints: VideoConstraints, /** A non-negative value is assumed as the available bandwidth in bps. A negative value is ignored. */ val assumedBandwidthBps: Long = -1 @@ -52,6 +54,12 @@ data class AllocationSettings @JvmOverloads constructor( override fun toString(): String = toJson().toJSONString() fun getConstraints(endpointId: String) = videoConstraints.getOrDefault(endpointId, defaultConstraints) + + companion object { + val defaultInitialLastN: Int by config { + "videobridge.cc.initial-last-n".from(JitsiConfig.newConfig) + } + } } /** @@ -68,7 +76,7 @@ internal class AllocationSettingsWrapper( */ private var selectedSources = emptyList() - internal var lastN: Int = -1 + internal var lastN: Int = AllocationSettings.defaultInitialLastN private var videoConstraints: Map = emptyMap() diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index 0972caec80..94eaeddb0e 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -67,6 +67,10 @@ videobridge { # no last-n limit) jvb-last-n = -1 + // The initial value for the client-configured last-n setting. This applies before it is overriden by colibri or + // data channel signaling (-1 implies no last-n limit). + initial-last-n = -1 + # If set allows receivers to override bandwidth estimation (BWE) with a specific value signaled over the bridge # channel (limited to the configured value). If not set, receivers are not allowed to override BWE. // assumed-bandwidth-limit = 10 Mbps From af071219c2599ba6980bfa2a8e02684fd9a3f2e8 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 20 Feb 2024 18:53:45 -0500 Subject: [PATCH 080/189] Improve debug output of VP9 and AV1 quality filters. (#2093) * Improve debug output of VP9 and AV1 quality filters. * More improvements to Vp9 rewrite debugging. * Copy relevant fixes to AV1 projection. --- .../Av1DDAdaptiveSourceProjectionContext.kt | 18 ++++++++++++++++-- .../cc/av1/Av1DDFrameProjection.kt | 3 ++- .../videobridge/cc/av1/Av1DDQualityFilter.kt | 2 ++ .../vp9/Vp9AdaptiveSourceProjectionContext.kt | 19 +++++++++++++++++-- .../videobridge/cc/vp9/Vp9FrameProjection.kt | 3 ++- .../videobridge/cc/vp9/Vp9QualityFilter.kt | 7 +++++-- 6 files changed, 44 insertions(+), 8 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt index 219f885090..b060238a3b 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt @@ -125,7 +125,21 @@ class Av1DDAdaptiveSourceProjectionContext( } } - val accept = frame.isAccepted && frame.projection?.accept(packet) == true + val accept = frame.isAccepted && + if (frame.projection?.accept(packet) == true) { + true + } else { + if (frame.projection != null && frame.projection?.closedSeq != -1) { + logger.debug( + "Not accepting $packet: frame projection is closed at ${frame.projection?.closedSeq}" + ) + } else if (frame.projection == null) { + logger.warn("Not accepting $packet: frame has no projection, even though QF accepted it") + } else { + logger.warn("Not accepting $packet, even though frame projection is not closed") + } + false + } if (timeSeriesLogger.isTraceEnabled) { val pt = diagnosticContext.makeTimeSeriesPoint("rtp_av1") @@ -277,7 +291,7 @@ class Av1DDAdaptiveSourceProjectionContext( } /** - * Create an projection for the first frame after an encoding switch. + * Create a projection for the first frame after an encoding switch. */ private fun createEncodingSwitchProjection( frame: Av1DDFrame, diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameProjection.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameProjection.kt index 88e34cb2b7..db4b784056 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameProjection.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameProjection.kt @@ -80,7 +80,8 @@ class Av1DDFrameProjection internal constructor( * -1 if this projection is still "open" for new, later packets. * Projections can be closed when we switch away from their encodings. */ - private var closedSeq = -1 + var closedSeq = -1 + private set /** * Ctor. diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt index 331a80faa9..01808bd5a2 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt @@ -416,6 +416,7 @@ internal class Av1DDQualityFilter( internal fun addDiagnosticContext(pt: DiagnosticContext.TimeSeriesPoint) { pt.addField("qf.currentIndex", Av1DDRtpLayerDesc.indexString(currentIndex)) .addField("qf.internalTargetEncoding", internalTargetEncoding) + .addField("qf.internalTargetDt", internalTargetDt) .addField("qf.needsKeyframe", needsKeyframe) .addField( "qf.mostRecentKeyframeGroupArrivalTimeMs", @@ -439,6 +440,7 @@ internal class Av1DDQualityFilter( mostRecentKeyframeGroupArrivalTime?.toEpochMilli() ?: -1 debugState["needsKeyframe"] = needsKeyframe debugState["internalTargetEncoding"] = internalTargetEncoding + debugState["internalTargetDt"] = internalTargetDt debugState["currentIndex"] = Av1DDRtpLayerDesc.indexString(currentIndex) return debugState } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt index f489dc3257..166760a728 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt @@ -131,7 +131,21 @@ class Vp9AdaptiveSourceProjectionContext( } } - val accept = frame.isAccepted && frame.projection?.accept(packet) == true + val accept = frame.isAccepted && + if (frame.projection?.accept(packet) == true) { + true + } else { + if (frame.projection != null && frame.projection?.closedSeq != -1) { + logger.debug( + "Not accepting $packet: frame projection is closed at ${frame.projection?.closedSeq}" + ) + } else if (frame.projection == null) { + logger.warn("Not accepting $packet: frame has no projection, even though QF accepted it") + } else { + logger.warn("Not accepting $packet, even though frame projection is not closed") + } + false + } if (timeSeriesLogger.isTraceEnabled) { val pt = diagnosticContext.makeTimeSeriesPoint("rtp_vp9") @@ -139,6 +153,7 @@ class Vp9AdaptiveSourceProjectionContext( .addField("timestamp", packet.timestamp) .addField("seq", packet.sequenceNumber) .addField("pictureId", packet.pictureId) + .addField("pictureIdIndex", frame.index) .addField("encoding", incomingEncoding) .addField("spatialLayer", packet.spatialLayerIndex) .addField("temporalLayer", packet.temporalLayerIndex) @@ -255,7 +270,7 @@ class Vp9AdaptiveSourceProjectionContext( } /** - * Create an projection for the first frame after an encoding switch. + * Create a projection for the first frame after an encoding switch. */ private fun createEncodingSwitchProjection( frame: Vp9Frame, diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9FrameProjection.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9FrameProjection.kt index 87598ee961..f9d08fb49f 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9FrameProjection.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9FrameProjection.kt @@ -89,7 +89,8 @@ internal constructor( * -1 if this projection is still "open" for new, later packets. * Projections can be closed when we switch away from their encodings. */ - private var closedSeq = -1 + var closedSeq = -1 + private set /** * Ctor. diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilter.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilter.kt index 863b4c8786..2111eda04d 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilter.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilter.kt @@ -80,7 +80,7 @@ internal class Vp9QualityFilter(parentLogger: Logger) { /** * Which spatial layers are currently being forwarded. */ - private val layers: Array = Array(MAX_VP9_LAYERS) { false } + private val layers = BooleanArray(MAX_VP9_LAYERS) /** * Determines whether to accept or drop a VP9 frame. @@ -428,6 +428,7 @@ internal class Vp9QualityFilter(parentLogger: Logger) { internal fun addDiagnosticContext(pt: DiagnosticContext.TimeSeriesPoint) { pt.addField("qf.currentIndex", indexString(currentIndex)) .addField("qf.internalTargetEncoding", internalTargetEncoding) + .addField("qf.internalTargetSpatialId", internalTargetSpatialId) .addField("qf.needsKeyframe", needsKeyframe) .addField( "qf.mostRecentKeyframeGroupArrivalTimeMs", @@ -452,8 +453,10 @@ internal class Vp9QualityFilter(parentLogger: Logger) { debugState["mostRecentKeyframeGroupArrivalTimeMs"] = mostRecentKeyframeGroupArrivalTime?.toEpochMilli() ?: -1 debugState["needsKeyframe"] = needsKeyframe - debugState["internalTargetIndex"] = internalTargetEncoding + debugState["internalTargetEncoding"] = internalTargetEncoding + debugState["internalTargetSpatialId"] = internalTargetSpatialId debugState["currentIndex"] = indexString(currentIndex) + debugState["layersForwarded"] = layers.map { toString().first() }.joinToString(separator = "") return debugState } From 4ee14d99aa2d087a94aa79b91419bcf2a5f3b7ed Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 20 Feb 2024 18:53:59 -0500 Subject: [PATCH 081/189] Recommend libpcap0.8 for the JVB debian package. (#2094) Needed by pcap4j, but pcap capture is not enabled by default. --- debian/control | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/control b/debian/control index c4fe18fa8d..949114bf16 100644 --- a/debian/control +++ b/debian/control @@ -13,6 +13,7 @@ Conflicts: jitsi-videobridge (<= 1400-1) Architecture: all Pre-Depends: openjdk-11-jre-headless | openjdk-11-jre | java11-runtime-headless | java11-runtime, libssl3 | libssl1.1 Depends: ${misc:Depends}, procps, uuid-runtime, ruby-hocon +Recommends: libpcap0.8 Description: WebRTC compatible Selective Forwarding Unit (SFU) Jitsi Videobridge is a WebRTC compatible Selective Forwarding Unit (SFU) for multiuser video communication. It is an essential part From 58eadf661c1e1312fbdefafcfd8ab1e26a17fd5e Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 20 Feb 2024 18:54:09 -0500 Subject: [PATCH 082/189] Include layers' index strings in BandwidthAllocation.toString(). (#2098) --- .../org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt index 16681fac83..db791bc68e 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt @@ -99,7 +99,8 @@ data class SingleAllocation( fun isForwarded(): Boolean = targetIndex > -1 override fun toString(): String = "[id=$endpointId target=${targetLayer?.height}/${targetLayer?.frameRate} " + - "ideal=${idealLayer?.height}/${idealLayer?.frameRate}]" + "(${targetLayer?.indexString()}) " + + "ideal=${idealLayer?.height}/${idealLayer?.frameRate} (${idealLayer?.indexString()})]" val debugState: JSONObject get() = JSONObject().apply { From ef51846540e72b2d0bdc89ff2cba8fe3dc88f293 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 20 Feb 2024 18:54:22 -0500 Subject: [PATCH 083/189] Simplify source projection accept() and add more unit tests. (#2099) * Remove encodingId from AdaptiveSourceProjectionContext.accept(). It's set in the packet. Adapt source projection unit tests to set the encodingId in the packet. * Add unit tests for Vp9 simulcast. * Add Vp9 simulcast-to-svc switch unit test; fix some bugs in other tests. --- .../org/jitsi/nlj/rtp/VideoRtpPacket.kt | 4 + .../cc/AdaptiveSourceProjection.java | 6 +- .../cc/AdaptiveSourceProjectionContext.java | 7 +- ...enericAdaptiveSourceProjectionContext.java | 9 +- .../VP8AdaptiveSourceProjectionContext.java | 6 +- .../Av1DDAdaptiveSourceProjectionContext.kt | 3 +- .../vp9/Vp9AdaptiveSourceProjectionContext.kt | 3 +- .../vp8/VP8AdaptiveSourceProjectionTest.java | 72 +-- .../av1/Av1DDAdaptiveSourceProjectionTest.kt | 132 +++-- .../cc/vp9/Vp9AdaptiveSourceProjectionTest.kt | 522 ++++++++++++++++-- 10 files changed, 595 insertions(+), 169 deletions(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/VideoRtpPacket.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/VideoRtpPacket.kt index 19dc0716b5..f4453b280d 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/VideoRtpPacket.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/VideoRtpPacket.kt @@ -33,6 +33,10 @@ open class VideoRtpPacket @JvmOverloads constructor( open val layerIds: Collection = listOf(0) + override fun toString(): String { + return super.toString() + ", EncID=$encodingId" + } + override fun clone(): VideoRtpPacket { return VideoRtpPacket( cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET), diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjection.java b/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjection.java index 8cad7c69a0..3cf093ea26 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjection.java +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjection.java @@ -31,7 +31,6 @@ import java.lang.*; import java.util.*; -import java.util.stream.*; /** * Filters the packets coming from a specific {@link MediaSourceDesc} @@ -143,8 +142,7 @@ public boolean accept(@NotNull PacketInfo packetInfo) // suspended so that it can raise the needsKeyframe flag and also allow // it to compute a sequence number delta when the target becomes > -1. - int encodingId = videoRtpPacket.getEncodingId(); - if (encodingId == RtpLayerDesc.SUSPENDED_ENCODING_ID) + if (videoRtpPacket.getEncodingId() == RtpLayerDesc.SUSPENDED_ENCODING_ID) { logger.warn( "Dropping an RTP packet, because egress was unable to find " + @@ -153,7 +151,7 @@ public boolean accept(@NotNull PacketInfo packetInfo) } int targetIndexCopy = targetIndex; - boolean accept = contextCopy.accept(packetInfo, encodingId, targetIndexCopy); + boolean accept = contextCopy.accept(packetInfo, targetIndexCopy); // We check if the context needs a keyframe regardless of whether or not // the packet was accepted. diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjectionContext.java b/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjectionContext.java index af56947d67..9c766175fe 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjectionContext.java +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjectionContext.java @@ -19,8 +19,6 @@ import org.jitsi.rtp.rtcp.*; import org.json.simple.*; -import java.util.*; - /** * Implementations of this interface are responsible for projecting a specific * video source of a specific payload type. @@ -40,12 +38,11 @@ public interface AdaptiveSourceProjectionContext /** * Determines whether an RTP packet should be accepted or not. * - * @param packetInfo the RTP packet to determine whether to accept or not. - * @param incomingEncoding The encoding of the incoming packet. + * @param packetInfo the RTP packet to determine whether to accept or not. * @param targetIndex the target quality index * @return true if the packet should be accepted, false otherwise. */ - boolean accept(PacketInfo packetInfo, int incomingEncoding, int targetIndex); + boolean accept(PacketInfo packetInfo, int targetIndex); /** * @return true if this stream context needs a keyframe in order to either diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/GenericAdaptiveSourceProjectionContext.java b/jvb/src/main/java/org/jitsi/videobridge/cc/GenericAdaptiveSourceProjectionContext.java index f1561bab66..fb4e6155e2 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/cc/GenericAdaptiveSourceProjectionContext.java +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/GenericAdaptiveSourceProjectionContext.java @@ -23,8 +23,6 @@ import org.jitsi.utils.logging2.*; import org.json.simple.*; -import java.util.*; - /** * A generic implementation of an adaptive source projection context that can be * used with non-SVC codecs or when simulcast is not enabled/used or when @@ -126,19 +124,18 @@ class GenericAdaptiveSourceProjectionContext * Determines whether an RTP packet from the incoming source should be accepted * or not. If the source is currently suspended, a key frame is necessary to * start accepting packets again. - * + *

* Note that, at the time of this writing, there's no practical need for a * synchronized keyword because there's only one thread (the translator * thread) accessing this method at a time. * - * @param packetInfo the RTP packet to determine whether to accept or not. - * @param incomingEncoding The encoding index of the packet + * @param packetInfo the RTP packet to determine whether to accept or not. * @param targetIndex the target quality index * @return true if the packet should be accepted, false otherwise. */ @Override public synchronized boolean - accept(@NotNull PacketInfo packetInfo, int incomingEncoding, int targetIndex) + accept(@NotNull PacketInfo packetInfo, int targetIndex) { VideoRtpPacket rtpPacket = packetInfo.packetAs(); if (targetIndex == RtpLayerDesc.SUSPENDED_INDEX) diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionContext.java b/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionContext.java index 97be97d72a..26ba81dd78 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionContext.java +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionContext.java @@ -251,14 +251,13 @@ private boolean frameIsNewSsrc(VP8Frame frame) /** * Determines whether a packet should be accepted or not. * - * @param packetInfo the RTP packet to determine whether to project or not. - * @param incomingEncoding the encoding of the incoming RTP packet + * @param packetInfo the RTP packet to determine whether to project or not. * @param targetIndex the target quality index we want to achieve * @return true if the packet should be accepted, false otherwise. */ @Override public synchronized boolean accept( - @NotNull PacketInfo packetInfo, int incomingEncoding, int targetIndex) + @NotNull PacketInfo packetInfo, int targetIndex) { if (!(packetInfo.getPacket() instanceof Vp8Packet)) { @@ -266,6 +265,7 @@ public synchronized boolean accept( return false; } Vp8Packet vp8Packet = packetInfo.packetAs(); + int incomingEncoding = vp8Packet.getEncodingId(); VP8FrameMap.FrameInsertionResult result = insertPacketInMap(vp8Packet); diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt index b060238a3b..6388dc3f04 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt @@ -70,13 +70,14 @@ class Av1DDAdaptiveSourceProjectionContext( */ private var lastFrameNumberIndexResumption = -1 - override fun accept(packetInfo: PacketInfo, incomingEncoding: Int, targetIndex: Int): Boolean { + override fun accept(packetInfo: PacketInfo, targetIndex: Int): Boolean { val packet = packetInfo.packet if (packet !is Av1DDPacket) { logger.warn("Packet is not AV1 DD Packet") return false } + val incomingEncoding = packet.encodingId /* If insertPacketInMap returns null, this is a very old picture, more than Av1FrameMap.PICTURE_MAP_SIZE old, or something is wrong with the stream. */ diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt index 166760a728..d8783baee1 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt @@ -78,12 +78,13 @@ class Vp9AdaptiveSourceProjectionContext( private var lastPicIdIndexResumption = -1 @Synchronized - override fun accept(packetInfo: PacketInfo, incomingEncoding: Int, targetIndex: Int): Boolean { + override fun accept(packetInfo: PacketInfo, targetIndex: Int): Boolean { val packet = packetInfo.packet if (packet !is Vp9Packet) { logger.warn("Packet is not Vp9 packet") return false } + val incomingEncoding = packet.encodingId /* If insertPacketInMap returns null, this is a very old picture, more than Vp9PictureMap.PICTURE_MAP_SIZE old, or something is wrong with the stream. */ diff --git a/jvb/src/test/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionTest.java b/jvb/src/test/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionTest.java index 56e9782943..feaf1d3dd9 100644 --- a/jvb/src/test/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionTest.java +++ b/jvb/src/test/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionTest.java @@ -17,7 +17,6 @@ import org.jitsi.nlj.*; import org.jitsi.nlj.codec.vpx.*; -import org.jitsi.nlj.format.*; import org.jitsi.nlj.rtp.codec.vp8.*; import org.jitsi.nlj.util.*; import org.jitsi.rtp.rtcp.*; @@ -32,7 +31,6 @@ import javax.xml.bind.*; import java.time.*; import java.util.*; -import java.util.concurrent.*; import static org.junit.Assert.*; @@ -59,7 +57,7 @@ public void singlePacketProjectionTest() throws RewriteException int targetIndex = RtpLayerDesc.getIndex(0, 0, 0); - assertTrue(context.accept(packetInfo, 0, targetIndex)); + assertTrue(context.accept(packetInfo, targetIndex)); context.rewriteRtp(packetInfo); @@ -94,7 +92,7 @@ private void runInOrderTest(Vp8PacketGenerator generator, int targetTid) PacketInfo packetInfo = generator.nextPacket(); Vp8Packet packet = packetInfo.packetAs(); - boolean accepted = context.accept(packetInfo, 0, targetIndex); + boolean accepted = context.accept(packetInfo, targetIndex); if (packet.isStartOfFrame() && packet.getTemporalLayerIndex() == 0) { @@ -194,7 +192,7 @@ private void doRunOutOfOrderTest(Vp8PacketGenerator generator, int targetTid, { latestSeq = origSeq; } - boolean accepted = context.accept(packetInfo, 0, targetIndex); + boolean accepted = context.accept(packetInfo, targetIndex); int oldestValidSeq = RtpUtils.applySequenceNumberDelta(latestSeq, -((VP8FrameMap.FRAME_MAP_SIZE - 1) * generator.packetsPerFrame)); @@ -393,10 +391,10 @@ public void slightlyDelayedKeyframeTest() throws RewriteException PacketInfo packetInfo = generator.nextPacket(); Vp8Packet packet = packetInfo.packetAs(); - assertFalse(context.accept(packetInfo, 0, targetIndex)); + assertFalse(context.accept(packetInfo, targetIndex)); } - assertTrue(context.accept(firstPacketInfo, 0, targetIndex)); + assertTrue(context.accept(firstPacketInfo, targetIndex)); context.rewriteRtp(firstPacketInfo); for (int i = 0; i < 9996; i++) @@ -404,7 +402,7 @@ public void slightlyDelayedKeyframeTest() throws RewriteException PacketInfo packetInfo = generator.nextPacket(); Vp8Packet packet = packetInfo.packetAs(); - assertTrue(context.accept(packetInfo, 0, targetIndex)); + assertTrue(context.accept(packetInfo, targetIndex)); context.rewriteRtp(packetInfo); } } @@ -433,17 +431,17 @@ public void veryDelayedKeyframeTest() throws RewriteException PacketInfo packetInfo = generator.nextPacket(); Vp8Packet packet = packetInfo.packetAs(); - assertFalse(context.accept(packetInfo, 0, targetIndex)); + assertFalse(context.accept(packetInfo, targetIndex)); } - assertFalse(context.accept(firstPacketInfo, 0, targetIndex)); + assertFalse(context.accept(firstPacketInfo, targetIndex)); for (int i = 0; i < 10; i++) { PacketInfo packetInfo = generator.nextPacket(); Vp8Packet packet = packetInfo.packetAs(); - assertFalse(context.accept(packetInfo, 0, targetIndex)); + assertFalse(context.accept(packetInfo, targetIndex)); } generator.requestKeyframe(); @@ -453,7 +451,7 @@ public void veryDelayedKeyframeTest() throws RewriteException PacketInfo packetInfo = generator.nextPacket(); Vp8Packet packet = packetInfo.packetAs(); - assertTrue(context.accept(packetInfo, 0, targetIndex)); + assertTrue(context.accept(packetInfo, targetIndex)); context.rewriteRtp(packetInfo); } } @@ -482,17 +480,17 @@ public void delayedPartialKeyframeTest() throws RewriteException PacketInfo packetInfo = generator.nextPacket(); Vp8Packet packet = packetInfo.packetAs(); - assertFalse(context.accept(packetInfo, 0, targetIndex)); + assertFalse(context.accept(packetInfo, targetIndex)); } - assertFalse(context.accept(firstPacketInfo, 0, 2)); + assertFalse(context.accept(firstPacketInfo, 2)); for (int i = 0; i < 30; i++) { PacketInfo packetInfo = generator.nextPacket(); Vp8Packet packet = packetInfo.packetAs(); - assertFalse(context.accept(packetInfo, 0, targetIndex)); + assertFalse(context.accept(packetInfo, targetIndex)); } generator.requestKeyframe(); @@ -502,7 +500,7 @@ public void delayedPartialKeyframeTest() throws RewriteException PacketInfo packetInfo = generator.nextPacket(); Vp8Packet packet = packetInfo.packetAs(); - assertTrue(context.accept(packetInfo, 0, targetIndex)); + assertTrue(context.accept(packetInfo, targetIndex)); context.rewriteRtp(packetInfo); } } @@ -510,8 +508,8 @@ public void delayedPartialKeyframeTest() throws RewriteException @Test public void twoStreamsNoSwitchingTest() throws RewriteException { - Vp8PacketGenerator generator1 = new Vp8PacketGenerator(3); - Vp8PacketGenerator generator2 = new Vp8PacketGenerator(3); + Vp8PacketGenerator generator1 = new Vp8PacketGenerator(3, 1); + Vp8PacketGenerator generator2 = new Vp8PacketGenerator(3, 0); generator2.setSsrc(0xdeadbeefL); DiagnosticContext diagnosticContext = new DiagnosticContext(); @@ -532,12 +530,12 @@ public void twoStreamsNoSwitchingTest() throws RewriteException PacketInfo packetInfo1 = generator1.nextPacket(); Vp8Packet packet1 = packetInfo1.packetAs(); - assertTrue(context.accept(packetInfo1, 1, targetIndex)); + assertTrue(context.accept(packetInfo1, targetIndex)); PacketInfo packetInfo2 = generator2.nextPacket(); Vp8Packet packet2 = packetInfo2.packetAs(); - assertFalse(context.accept(packetInfo2, 0, targetIndex)); + assertFalse(context.accept(packetInfo2, targetIndex)); context.rewriteRtp(packetInfo1); @@ -555,8 +553,8 @@ public void twoStreamsNoSwitchingTest() throws RewriteException @Test public void twoStreamsSwitchingTest() throws RewriteException { - Vp8PacketGenerator generator1 = new Vp8PacketGenerator(3); - Vp8PacketGenerator generator2 = new Vp8PacketGenerator(3); + Vp8PacketGenerator generator1 = new Vp8PacketGenerator(3, 0); + Vp8PacketGenerator generator2 = new Vp8PacketGenerator(3, 1); generator2.setSsrc(0xdeadbeefL); DiagnosticContext diagnosticContext = new DiagnosticContext(); @@ -587,7 +585,7 @@ public void twoStreamsSwitchingTest() throws RewriteException expectedTl0PicIdx = VpxUtils.applyTl0PicIdxDelta(expectedTl0PicIdx, 1); } - assertTrue(context.accept(packetInfo1, 0, targetIndex)); + assertTrue(context.accept(packetInfo1, targetIndex)); context.rewriteRtp(packetInfo1); @@ -599,7 +597,7 @@ public void twoStreamsSwitchingTest() throws RewriteException PacketInfo packetInfo2 = generator2.nextPacket(); Vp8Packet packet2 = packetInfo2.packetAs(); - assertFalse(context.accept(packetInfo2, 1, targetIndex)); + assertFalse(context.accept(packetInfo2, targetIndex)); assertFalse(context.rewriteRtcp(srPacket2)); assertEquals(expectedSeq, packet1.getSequenceNumber()); @@ -628,7 +626,7 @@ public void twoStreamsSwitchingTest() throws RewriteException expectedTl0PicIdx = VpxUtils.applyTl0PicIdxDelta(expectedTl0PicIdx, 1); } - assertTrue(context.accept(packetInfo1, 0, targetIndex)); + assertTrue(context.accept(packetInfo1, targetIndex)); context.rewriteRtp(packetInfo1); @@ -640,7 +638,7 @@ public void twoStreamsSwitchingTest() throws RewriteException PacketInfo packetInfo2 = generator2.nextPacket(); Vp8Packet packet2 = packetInfo2.packetAs(); - assertFalse(context.accept(packetInfo2, 1, targetIndex)); + assertFalse(context.accept(packetInfo2, targetIndex)); assertFalse(context.rewriteRtcp(srPacket2)); assertEquals(expectedSeq, packet1.getSequenceNumber()); @@ -673,7 +671,7 @@ public void twoStreamsSwitchingTest() throws RewriteException } /* We will cut off the layer 0 keyframe after 1 packet, once we see the layer 1 keyframe. */ - assertEquals(i == 0, context.accept(packetInfo1, 0, targetIndex)); + assertEquals(i == 0, context.accept(packetInfo1, targetIndex)); assertEquals(i == 0, context.rewriteRtcp(srPacket1)); if (i == 0) @@ -692,7 +690,7 @@ public void twoStreamsSwitchingTest() throws RewriteException expectedTl0PicIdx = VpxUtils.applyTl0PicIdxDelta(expectedTl0PicIdx, 1); } - assertTrue(context.accept(packetInfo2, 1, targetIndex)); + assertTrue(context.accept(packetInfo2, targetIndex)); context.rewriteRtp(packetInfo2); @@ -753,7 +751,7 @@ public void temporalLayerSwitchingTest() throws RewriteException PacketInfo packetInfo = generator.nextPacket(); Vp8Packet packet = packetInfo.packetAs(); - boolean accepted = context.accept(packetInfo, 0, targetIndex); + boolean accepted = context.accept(packetInfo, targetIndex); if (packet.isStartOfFrame() && packet.getTemporalLayerIndex() == 0) { @@ -833,7 +831,7 @@ private void runLargeDropoutTest(Vp8PacketGenerator generator, Vp8Packet packet = packetInfo.packetAs(); boolean accepted = - context.accept(packetInfo, 0, targetIndex); + context.accept(packetInfo, targetIndex); if (packet.isStartOfFrame() && packet.getTemporalLayerIndex() == 0) { @@ -886,7 +884,7 @@ private void runLargeDropoutTest(Vp8PacketGenerator generator, } while (packet.getTemporalLayerIndex() > targetIndex); - assertTrue(context.accept(packetInfo, 0, targetIndex)); + assertTrue(context.accept(packetInfo, targetIndex)); context.rewriteRtp(packetInfo); /* Allow any values after a gap. */ @@ -909,7 +907,7 @@ private void runLargeDropoutTest(Vp8PacketGenerator generator, packet = packetInfo.packetAs(); boolean accepted = context - .accept(packetInfo, 0, targetIndex); + .accept(packetInfo, targetIndex); if (packet.isStartOfFrame() && packet.getTemporalLayerIndex() == 0) @@ -1009,13 +1007,20 @@ private static class Vp8PacketGenerator { final int packetsPerFrame; - Vp8PacketGenerator(int packetsPerFrame) + final int encodingId; + + Vp8PacketGenerator(int packetsPerFrame, int encodingId) { this.packetsPerFrame = packetsPerFrame; + this.encodingId = encodingId; reset(); } + Vp8PacketGenerator(int packetsPerFrame) { + this(packetsPerFrame, 0); + } + private int seq; private long ts; private int picId ; @@ -1116,6 +1121,7 @@ public PacketInfo nextPacket() rtpPacket.setMarked(endOfFrame); Vp8Packet vp8Packet = rtpPacket.toOtherType(Vp8Packet::new); + vp8Packet.setEncodingId(encodingId); /* Make sure our manipulations of the raw buffer were correct. */ assertEquals(startOfFrame, vp8Packet.isStartOfFrame()); diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionTest.kt index e3bbead0ef..2ff4bff81e 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionTest.kt @@ -59,7 +59,7 @@ class Av1DDAdaptiveSourceProjectionTest { val packetInfo = generator.nextPacket() val packet = packetInfo.packetAs() val targetIndex = getIndex(eid = 0, dt = 0) - Assert.assertTrue(context.accept(packetInfo, 0, targetIndex)) + Assert.assertTrue(context.accept(packetInfo, targetIndex)) context.rewriteRtp(packetInfo) Assert.assertEquals(10001, packet.sequenceNumber) Assert.assertEquals(1003000, packet.timestamp) @@ -85,7 +85,7 @@ class Av1DDAdaptiveSourceProjectionTest { val packet = packetInfo.packetAs() val frameInfo = packet.frameInfo!! - val accepted = context.accept(packetInfo, 0, targetIndex) + val accepted = context.accept(packetInfo, targetIndex) val endOfFrame = packet.isEndOfFrame val endOfPicture = packet.isMarked // Save this before rewriteRtp if (expectAccept(frameInfo)) { @@ -396,7 +396,7 @@ class Av1DDAdaptiveSourceProjectionTest { } val frameInfo = packet.frameInfo!! - val accepted = context.accept(packetInfo, 0, targetIndex) + val accepted = context.accept(packetInfo, targetIndex) val oldestValidSeq: Int = RtpUtils.applySequenceNumberDelta( latestSeq, @@ -776,13 +776,13 @@ class Av1DDAdaptiveSourceProjectionTest { for (i in 0..2) { val packetInfo = generator.nextPacket() - Assert.assertFalse(context.accept(packetInfo, 0, targetIndex)) + Assert.assertFalse(context.accept(packetInfo, targetIndex)) } - Assert.assertTrue(context.accept(firstPacketInfo, 0, targetIndex)) + Assert.assertTrue(context.accept(firstPacketInfo, targetIndex)) context.rewriteRtp(firstPacketInfo) for (i in 0..9995) { val packetInfo = generator.nextPacket() - Assert.assertTrue(context.accept(packetInfo, 0, targetIndex)) + Assert.assertTrue(context.accept(packetInfo, targetIndex)) context.rewriteRtp(packetInfo) } } @@ -802,25 +802,25 @@ class Av1DDAdaptiveSourceProjectionTest { val targetIndex = getIndex(eid = 0, dt = 2) for (i in 0..3) { val packetInfo = generator.nextPacket(missedStructure = true) - Assert.assertFalse(context.accept(packetInfo, 0, targetIndex)) + Assert.assertFalse(context.accept(packetInfo, targetIndex)) } - Assert.assertFalse(context.accept(firstPacketInfo, 0, targetIndex)) + Assert.assertFalse(context.accept(firstPacketInfo, targetIndex)) for (i in 0..9) { val packetInfo = generator.nextPacket() - Assert.assertFalse(context.accept(packetInfo, 0, targetIndex)) + Assert.assertFalse(context.accept(packetInfo, targetIndex)) } generator.requestKeyframe() for (i in 0..9995) { val packetInfo = generator.nextPacket() - Assert.assertTrue(context.accept(packetInfo, 0, targetIndex)) + Assert.assertTrue(context.accept(packetInfo, targetIndex)) context.rewriteRtp(packetInfo) } } @Test fun twoStreamsNoSwitchingTest() { - val generator1 = TemporallyScaledPacketGenerator(3) - val generator2 = TemporallyScaledPacketGenerator(3) + val generator1 = TemporallyScaledPacketGenerator(packetsPerFrame = 3, encodingId = 1) + val generator2 = TemporallyScaledPacketGenerator(packetsPerFrame = 3, encodingId = 0) generator2.ssrc = 0xdeadbeefL val diagnosticContext = DiagnosticContext() diagnosticContext["test"] = "twoStreamsNoSwitchingTest" @@ -833,9 +833,9 @@ class Av1DDAdaptiveSourceProjectionTest { val packetInfo1 = generator1.nextPacket() val packet1 = packetInfo1.packetAs() - Assert.assertTrue(context.accept(packetInfo1, 1, targetIndex)) + Assert.assertTrue(context.accept(packetInfo1, targetIndex)) val packetInfo2 = generator2.nextPacket() - Assert.assertFalse(context.accept(packetInfo2, 0, targetIndex)) + Assert.assertFalse(context.accept(packetInfo2, targetIndex)) context.rewriteRtp(packetInfo1) Assert.assertEquals(expectedSeq, packet1.sequenceNumber) Assert.assertEquals(expectedTs, packet1.timestamp) @@ -848,8 +848,8 @@ class Av1DDAdaptiveSourceProjectionTest { @Test fun twoStreamsSwitchingTest() { - val generator1 = TemporallyScaledPacketGenerator(3) - val generator2 = TemporallyScaledPacketGenerator(3) + val generator1 = TemporallyScaledPacketGenerator(3, encodingId = 0) + val generator2 = TemporallyScaledPacketGenerator(3, encodingId = 1) generator2.ssrc = 0xdeadbeefL val diagnosticContext = DiagnosticContext() diagnosticContext["test"] = "twoStreamsSwitchingTest" @@ -869,14 +869,14 @@ class Av1DDAdaptiveSourceProjectionTest { if (i == 0) { expectedTemplateOffset = packet1.descriptor!!.structure.templateIdOffset } - Assert.assertTrue(context.accept(packetInfo1, 0, targetIndex)) + Assert.assertTrue(context.accept(packetInfo1, targetIndex)) context.rewriteRtp(packetInfo1) Assert.assertTrue(context.rewriteRtcp(srPacket1)) Assert.assertEquals(packet1.ssrc, srPacket1.senderSsrc) Assert.assertEquals(packet1.timestamp, srPacket1.senderInfo.rtpTimestamp) val srPacket2 = generator2.srPacket val packetInfo2 = generator2.nextPacket() - Assert.assertFalse(context.accept(packetInfo2, 1, targetIndex)) + Assert.assertFalse(context.accept(packetInfo2, targetIndex)) Assert.assertFalse(context.rewriteRtcp(srPacket2)) Assert.assertEquals(expectedSeq, packet1.sequenceNumber) Assert.assertEquals(expectedTs, packet1.timestamp) @@ -897,14 +897,14 @@ class Av1DDAdaptiveSourceProjectionTest { val srPacket1 = generator1.srPacket val packetInfo1 = generator1.nextPacket() val packet1 = packetInfo1.packetAs() - Assert.assertTrue(context.accept(packetInfo1, 0, targetIndex)) + Assert.assertTrue(context.accept(packetInfo1, targetIndex)) context.rewriteRtp(packetInfo1) Assert.assertTrue(context.rewriteRtcp(srPacket1)) Assert.assertEquals(packet1.ssrc, srPacket1.senderSsrc) Assert.assertEquals(packet1.timestamp, srPacket1.senderInfo.rtpTimestamp) val srPacket2 = generator2.srPacket val packetInfo2 = generator2.nextPacket() - Assert.assertFalse(context.accept(packetInfo2, 1, targetIndex)) + Assert.assertFalse(context.accept(packetInfo2, targetIndex)) Assert.assertFalse(context.rewriteRtcp(srPacket2)) Assert.assertEquals(expectedSeq, packet1.sequenceNumber) Assert.assertEquals(expectedTs, packet1.timestamp) @@ -928,7 +928,7 @@ class Av1DDAdaptiveSourceProjectionTest { val packet1 = packetInfo1.packetAs() /* We will cut off the layer 0 keyframe after 1 packet, once we see the layer 1 keyframe. */ - Assert.assertEquals(i == 0, context.accept(packetInfo1, 0, targetIndex)) + Assert.assertEquals(i == 0, context.accept(packetInfo1, targetIndex)) Assert.assertEquals(i == 0, context.rewriteRtcp(srPacket1)) if (i == 0) { context.rewriteRtp(packetInfo1) @@ -939,7 +939,7 @@ class Av1DDAdaptiveSourceProjectionTest { val srPacket2 = generator2.srPacket val packetInfo2 = generator2.nextPacket() val packet2 = packetInfo2.packetAs() - Assert.assertTrue(context.accept(packetInfo2, 1, targetIndex)) + Assert.assertTrue(context.accept(packetInfo2, targetIndex)) val expectedTemplateId = (packet2.descriptor!!.frameDependencyTemplateId + expectedTemplateOffset) % 64 context.rewriteRtp(packetInfo2) Assert.assertTrue(context.rewriteRtcp(srPacket2)) @@ -993,7 +993,7 @@ class Av1DDAdaptiveSourceProjectionTest { for (i in 0..9999) { val packetInfo = generator.nextPacket() val packet = packetInfo.packetAs() - val accepted = context.accept(packetInfo, 0, targetIndex) + val accepted = context.accept(packetInfo, targetIndex) if (accepted) { if (decodableTid < packet.frameInfo!!.temporalId) { decodableTid = packet.frameInfo!!.temporalId @@ -1044,7 +1044,7 @@ class Av1DDAdaptiveSourceProjectionTest { val packetInfo = generator.nextPacket() val packet = packetInfo.packetAs() - val accepted = context.accept(packetInfo, 0, targetIndex) + val accepted = context.accept(packetInfo, targetIndex) val frameInfo = packet.frameInfo!! val endOfPicture = packet.isMarked if (expectAccept(frameInfo)) { @@ -1077,7 +1077,7 @@ class Av1DDAdaptiveSourceProjectionTest { frameInfo = packet.frameInfo!! } while (!expectAccept(frameInfo)) var endOfPicture = packet.isMarked - Assert.assertTrue(context.accept(packetInfo, 0, targetIndex)) + Assert.assertTrue(context.accept(packetInfo, targetIndex)) context.rewriteRtp(packetInfo) /* Allow any values after a gap. */ @@ -1093,7 +1093,7 @@ class Av1DDAdaptiveSourceProjectionTest { for (i in 0..999) { packetInfo = generator.nextPacket() packet = packetInfo.packetAs() - val accepted = context.accept(packetInfo, 0, targetIndex) + val accepted = context.accept(packetInfo, targetIndex) endOfPicture = packet.isMarked frameInfo = packet.frameInfo!! if (expectAccept(frameInfo)) { @@ -1176,7 +1176,7 @@ class Av1DDAdaptiveSourceProjectionTest { packetInfo = generator.nextPacket() packet = packetInfo.packetAs() frameInfo = packet.frameInfo!! - val accepted = context.accept(packetInfo, 0, targetIndex) + val accepted = context.accept(packetInfo, targetIndex) val endOfPicture = packet.isMarked if (expectAccept(frameInfo)) { Assert.assertTrue(accepted) @@ -1206,7 +1206,7 @@ class Av1DDAdaptiveSourceProjectionTest { packetInfo = generator.nextPacket() packet = packetInfo.packetAs() - val accepted = context.accept(packetInfo, 0, targetIndex) + val accepted = context.accept(packetInfo, targetIndex) val endOfPicture = packet.isMarked Assert.assertTrue(accepted) context.rewriteRtp(packetInfo) @@ -1224,7 +1224,7 @@ class Av1DDAdaptiveSourceProjectionTest { packetInfo = generator.nextPacket() packet = packetInfo.packetAs() - val accepted = context.accept(packetInfo, 0, RtpLayerDesc.SUSPENDED_INDEX) + val accepted = context.accept(packetInfo, RtpLayerDesc.SUSPENDED_INDEX) Assert.assertFalse(accepted) val endOfPicture = packet.isMarked if (endOfPicture) { @@ -1238,7 +1238,7 @@ class Av1DDAdaptiveSourceProjectionTest { packetInfo = generator.nextPacket() packet = packetInfo.packetAs() - val accepted = context.accept(packetInfo, 0, targetIndex) + val accepted = context.accept(packetInfo, targetIndex) val endOfPicture = packet.isMarked Assert.assertFalse(accepted) if (endOfPicture) { @@ -1252,7 +1252,7 @@ class Av1DDAdaptiveSourceProjectionTest { while (generator.packetOfFrame != 0) { packetInfo = generator.nextPacket() packet = packetInfo.packetAs() - val accepted = context.accept(packetInfo, 0, targetIndex) + val accepted = context.accept(packetInfo, targetIndex) val endOfPicture = packet.isMarked Assert.assertFalse(accepted) if (endOfPicture) { @@ -1265,7 +1265,7 @@ class Av1DDAdaptiveSourceProjectionTest { packetInfo = generator.nextPacket() packet = packetInfo.packetAs() frameInfo = packet.frameInfo!! - val accepted = context.accept(packetInfo, 0, targetIndex) + val accepted = context.accept(packetInfo, targetIndex) val endOfPicture = packet.isMarked if (expectAccept(frameInfo)) { Assert.assertTrue(accepted) @@ -1330,7 +1330,8 @@ private open class Av1PacketGenerator( // Equivalent to number of layers val framesPerTimestamp: Int, templateDdHex: String, - val allKeyframesGetStructure: Boolean = false + val allKeyframesGetStructure: Boolean = false, + val encodingId: Int = 0 ) { private val logger: Logger = LoggerImpl(javaClass.name) @@ -1423,6 +1424,7 @@ private open class Av1PacketGenerator( if (missedStructure) null else structure, logger ) + av1Packet.encodingId = encodingId val info = PacketInfo(av1Packet) info.receivedTime = receivedTime @@ -1521,50 +1523,60 @@ private class NonScalableAv1PacketGenerator( "80000180003a410180ef808680" ) -private class TemporallyScaledPacketGenerator(packetsPerFrame: Int) : Av1PacketGenerator( - packetsPerFrame, - arrayOf(0), - arrayOf(1, 3, 2, 4), - 1, - "800001800214eaa860414d141020842701df010d" +private class TemporallyScaledPacketGenerator( + packetsPerFrame: Int, + encodingId: Int = 0 +) : Av1PacketGenerator( + packetsPerFrame = packetsPerFrame, + keyframeTemplates = arrayOf(0), + normalTemplates = arrayOf(1, 3, 2, 4), + framesPerTimestamp = 1, + templateDdHex = "800001800214eaa860414d141020842701df010d", + encodingId = encodingId ) private class ScalableAv1PacketGenerator( - packetsPerFrame: Int + packetsPerFrame: Int, + encodingId: Int = 0 ) : Av1PacketGenerator( - packetsPerFrame, - arrayOf(1, 6, 11), - arrayOf(0, 5, 10, 3, 8, 13, 2, 7, 12, 4, 9, 14), - 3, - "d0013481e81485214eafffaaaa863cf0430c10c302afc0aaa0063c00430010c002a000a800060000" + + packetsPerFrame = packetsPerFrame, + keyframeTemplates = arrayOf(1, 6, 11), + normalTemplates = arrayOf(0, 5, 10, 3, 8, 13, 2, 7, 12, 4, 9, 14), + framesPerTimestamp = 3, + templateDdHex = "d0013481e81485214eafffaaaa863cf0430c10c302afc0aaa0063c00430010c002a000a800060000" + "40001d954926e082b04a0941b820ac1282503157f974000ca864330e222222eca8655304224230ec" + - "a87753013f00b3027f016704ff02cf" + "a87753013f00b3027f016704ff02cf", + encodingId = encodingId ) private class KeyScalableAv1PacketGenerator( - packetsPerFrame: Int + packetsPerFrame: Int, + encodingId: Int = 0 ) : Av1PacketGenerator( - packetsPerFrame, - arrayOf(0, 5, 10), - arrayOf(1, 6, 11, 3, 8, 13, 2, 7, 12, 4, 9, 14), - 3, - "8f008581e81485214eaaaaa8000600004000100002aa80a8000600004000100002a000a80006000040" + - "0016d549241b5524906d54923157e001974ca864330e222396eca8655304224390eca87753013f00b3027f016704ff02cf" + packetsPerFrame = packetsPerFrame, + keyframeTemplates = arrayOf(0, 5, 10), + normalTemplates = arrayOf(1, 6, 11, 3, 8, 13, 2, 7, 12, 4, 9, 14), + framesPerTimestamp = 3, + templateDdHex = "8f008581e81485214eaaaaa8000600004000100002aa80a8000600004000100002a000a80006000040" + + "0016d549241b5524906d54923157e001974ca864330e222396eca8655304224390eca87753013f00b3027f016704ff02cf", + encodingId = encodingId ) private class SingleEncodingSimulcastAv1PacketGenerator( - packetsPerFrame: Int + packetsPerFrame: Int, + encodingId: Int = 0 ) : Av1PacketGenerator( - packetsPerFrame, - arrayOf(1, 6, 11), - arrayOf(0, 5, 10, 3, 8, 13, 2, 7, 12, 4, 9, 14), - 3, - "c1000180081485214ea000a8000600004000100002a000a8000600004000100002a000a8000600004" + + packetsPerFrame = packetsPerFrame, + keyframeTemplates = arrayOf(1, 6, 11), + normalTemplates = arrayOf(0, 5, 10, 3, 8, 13, 2, 7, 12, 4, 9, 14), + framesPerTimestamp = 3, + templateDdHex = "c1000180081485214ea000a8000600004000100002a000a8000600004000100002a000a8000600004" + "0001d954926caa493655248c55fe5d00032a190cc38e58803b2a1954c10e10843b2a1dd4c01dc010803bc0218077c0434", - allKeyframesGetStructure = true + allKeyframesGetStructure = true, + encodingId = encodingId ) private infix fun IntRange.step(next: (Int) -> Int) = diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionTest.kt index 1eaac25854..25d874d656 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionTest.kt @@ -17,6 +17,7 @@ package org.jitsi.videobridge.cc.vp9 import org.jitsi.nlj.PacketInfo import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.nlj.RtpLayerDesc.Companion.getEidFromIndex import org.jitsi.nlj.RtpLayerDesc.Companion.getIndex import org.jitsi.nlj.RtpLayerDesc.Companion.getSidFromIndex import org.jitsi.nlj.RtpLayerDesc.Companion.getTidFromIndex @@ -63,7 +64,7 @@ class Vp9AdaptiveSourceProjectionTest { val packetInfo = generator.nextPacket() val packet = packetInfo.packetAs() val targetIndex = getIndex(eid = 0, sid = 0, tid = 0) - Assert.assertTrue(context.accept(packetInfo, 0, targetIndex)) + Assert.assertTrue(context.accept(packetInfo, targetIndex)) context.rewriteRtp(packetInfo) Assert.assertEquals(10001, packet.sequenceNumber) Assert.assertEquals(1003000, packet.timestamp) @@ -85,23 +86,57 @@ class Vp9AdaptiveSourceProjectionTest { var expectedTs: Long = 1003000 var expectedPicId = 0 var expectedTl0PicIdx = 0 + var maybeStartOfPicture = false + var prevEncodingId = -1 + val targetEid = getEidFromIndex(targetIndex) val targetSid = getSidFromIndex(targetIndex) val targetTid = getTidFromIndex(targetIndex) for (i in 0..99999) { val packetInfo = generator.nextPacket() val packet = packetInfo.packetAs() + val bumpedTsAndPic: Boolean + if (maybeStartOfPicture && packet.encodingId == 0) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + expectedPicId = applyExtendedPictureIdDelta(expectedPicId, 1) + bumpedTsAndPic = true + } else { + bumpedTsAndPic = false + } val accepted = context.accept( packetInfo, - 0, targetIndex ) + val bumpedTl0PicIdx: Boolean if (!packet.hasLayerIndices) { expectedTl0PicIdx = -1 - } else if (packet.isStartOfFrame && packet.spatialLayerIndex == 0 && packet.temporalLayerIndex == 0) { + bumpedTl0PicIdx = false + } else if (packet.isStartOfFrame && + packet.spatialLayerIndex == 0 && + packet.temporalLayerIndex == 0 && + packet.encodingId == 0 + ) { expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) + bumpedTl0PicIdx = true + } else { + bumpedTl0PicIdx = false + } + /* When we switch encodings we always bump the TS, picID, and tl0picidx, + * even if the source packets had them the same. */ + if (accepted && packet.isKeyframe && prevEncodingId != -1 && prevEncodingId != packet.encodingId) { + if (!bumpedTsAndPic) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + expectedPicId = applyExtendedPictureIdDelta(expectedPicId, 1) + } + if (!bumpedTl0PicIdx) { + expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) + } } val endOfPicture = packet.isEndOfPicture // Save this before rewriteRtp if (packet.temporalLayerIndex <= targetTid && + ( + packet.encodingId == targetEid || + packet.isKeyframe && packet.encodingId < targetEid + ) && ( packet.spatialLayerIndex == targetSid || (packet.isUpperLevelReference && packet.spatialLayerIndex < targetSid) @@ -118,13 +153,12 @@ class Vp9AdaptiveSourceProjectionTest { packet.isMarked ) expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + prevEncodingId = packet.encodingId } else { Assert.assertFalse(accepted) + prevEncodingId = -1 } - if (endOfPicture) { - expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) - expectedPicId = applyExtendedPictureIdDelta(expectedPicId, 1) - } + maybeStartOfPicture = endOfPicture } } @@ -174,7 +208,7 @@ class Vp9AdaptiveSourceProjectionTest { if (latestSeq isOlderThan origSeq) { latestSeq = origSeq } - val accepted = context.accept(packetInfo, 0, targetIndex) + val accepted = context.accept(packetInfo, targetIndex) val oldestValidSeq: Int = RtpUtils.applySequenceNumberDelta( latestSeq, @@ -398,6 +432,54 @@ class Vp9AdaptiveSourceProjectionTest { runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 0)) } + @Test + fun simpleSimulcastTest() { + val generator = SimulcastVp9PacketGenerator(1, 3) + runInOrderTest(generator, getIndex(eid = 2, sid = 0, tid = 2)) + } + + @Test + fun filteredSimulcastTest() { + val generator = SimulcastVp9PacketGenerator(1, 3) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 2)) + } + + @Test + fun temporalFilteredSimulcastTest() { + val generator = SimulcastVp9PacketGenerator(1, 3) + runInOrderTest(generator, getIndex(eid = 2, sid = 0, tid = 0)) + } + + @Test + fun spatialAndTemporalFilteredSimulcastTest() { + val generator = SimulcastVp9PacketGenerator(1, 3) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 0)) + } + + @Test + fun largerSimulcastTest() { + val generator = SimulcastVp9PacketGenerator(3, 3) + runInOrderTest(generator, getIndex(eid = 2, sid = 0, tid = 2)) + } + + @Test + fun largerFilteredSimulcastTest() { + val generator = SimulcastVp9PacketGenerator(3, 3) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 2)) + } + + @Test + fun largerTemporalFilteredSimulcastTest() { + val generator = SimulcastVp9PacketGenerator(3, 3) + runInOrderTest(generator, getIndex(eid = 2, sid = 0, tid = 0)) + } + + @Test + fun largerSpatialAndTemporalFilteredSimulcastTest() { + val generator = SimulcastVp9PacketGenerator(3, 3) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 0)) + } + @Test fun simpleOutOfOrderTest() { val generator = ScalableVp9PacketGenerator(1) @@ -485,13 +567,13 @@ class Vp9AdaptiveSourceProjectionTest { val targetIndex = getIndex(eid = 0, sid = 0, tid = 2) for (i in 0..2) { val packetInfo = generator.nextPacket() - Assert.assertFalse(context.accept(packetInfo, 0, targetIndex)) + Assert.assertFalse(context.accept(packetInfo, targetIndex)) } - Assert.assertTrue(context.accept(firstPacketInfo, 0, targetIndex)) + Assert.assertTrue(context.accept(firstPacketInfo, targetIndex)) context.rewriteRtp(firstPacketInfo) for (i in 0..9995) { val packetInfo = generator.nextPacket() - Assert.assertTrue(context.accept(packetInfo, 0, targetIndex)) + Assert.assertTrue(context.accept(packetInfo, targetIndex)) context.rewriteRtp(packetInfo) } } @@ -511,17 +593,17 @@ class Vp9AdaptiveSourceProjectionTest { val targetIndex = getIndex(eid = 0, sid = 0, tid = 2) for (i in 0..3) { val packetInfo = generator.nextPacket() - Assert.assertFalse(context.accept(packetInfo, 0, targetIndex)) + Assert.assertFalse(context.accept(packetInfo, targetIndex)) } - Assert.assertFalse(context.accept(firstPacketInfo, 0, targetIndex)) + Assert.assertFalse(context.accept(firstPacketInfo, targetIndex)) for (i in 0..9) { val packetInfo = generator.nextPacket() - Assert.assertFalse(context.accept(packetInfo, 0, targetIndex)) + Assert.assertFalse(context.accept(packetInfo, targetIndex)) } generator.requestKeyframe() for (i in 0..9995) { val packetInfo = generator.nextPacket() - Assert.assertTrue(context.accept(packetInfo, 0, targetIndex)) + Assert.assertTrue(context.accept(packetInfo, targetIndex)) context.rewriteRtp(packetInfo) } } @@ -544,27 +626,27 @@ class Vp9AdaptiveSourceProjectionTest { for (i in 0..10) { val packetInfo = generator.nextPacket() val packet = packetInfo.packetAs() - Assert.assertTrue(context.accept(packetInfo, 0, targetIndex)) + Assert.assertTrue(context.accept(packetInfo, targetIndex)) context.rewriteRtp(packetInfo) Assert.assertTrue(packet.sequenceNumber > 10001) lowestSeq = minOf(lowestSeq, packet.sequenceNumber) } - Assert.assertTrue(context.accept(firstPacketInfo, 0, targetIndex)) + Assert.assertTrue(context.accept(firstPacketInfo, targetIndex)) context.rewriteRtp(firstPacketInfo) Assert.assertEquals(lowestSeq - 1, firstPacket.sequenceNumber) for (i in 0..9980) { val packetInfo = generator.nextPacket() - Assert.assertTrue(context.accept(packetInfo, 0, targetIndex)) + Assert.assertTrue(context.accept(packetInfo, targetIndex)) context.rewriteRtp(packetInfo) } } @Test fun twoStreamsNoSwitchingTest() { - val generator1 = ScalableVp9PacketGenerator(3) - val generator2 = ScalableVp9PacketGenerator(3) + val generator1 = ScalableVp9PacketGenerator(packetsPerFrame = 3, encodingId = 1) + val generator2 = ScalableVp9PacketGenerator(packetsPerFrame = 3, encodingId = 0) generator2.ssrc = 0xdeadbeefL val diagnosticContext = DiagnosticContext() diagnosticContext["test"] = "twoStreamsNoSwitchingTest" @@ -580,9 +662,9 @@ class Vp9AdaptiveSourceProjectionTest { for (i in 0..9999) { val packetInfo1 = generator1.nextPacket() val packet1 = packetInfo1.packetAs() - Assert.assertTrue(context.accept(packetInfo1, 1, targetIndex)) + Assert.assertTrue(context.accept(packetInfo1, targetIndex)) val packetInfo2 = generator2.nextPacket() - Assert.assertFalse(context.accept(packetInfo2, 0, targetIndex)) + Assert.assertFalse(context.accept(packetInfo2, targetIndex)) context.rewriteRtp(packetInfo1) Assert.assertEquals(expectedSeq, packet1.sequenceNumber) Assert.assertEquals(expectedTs, packet1.timestamp) @@ -595,8 +677,8 @@ class Vp9AdaptiveSourceProjectionTest { @Test fun twoStreamsSwitchingTest() { - val generator1 = ScalableVp9PacketGenerator(3) - val generator2 = ScalableVp9PacketGenerator(3) + val generator1 = ScalableVp9PacketGenerator(packetsPerFrame = 3, encodingId = 0) + val generator2 = ScalableVp9PacketGenerator(packetsPerFrame = 3, encodingId = 1) generator2.ssrc = 0xdeadbeefL val diagnosticContext = DiagnosticContext() diagnosticContext["test"] = "twoStreamsSwitchingTest" @@ -620,14 +702,14 @@ class Vp9AdaptiveSourceProjectionTest { if (packet1.isStartOfFrame && packet1.temporalLayerIndex == 0) { expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) } - Assert.assertTrue(context.accept(packetInfo1, 0, targetIndex)) + Assert.assertTrue(context.accept(packetInfo1, targetIndex)) context.rewriteRtp(packetInfo1) Assert.assertTrue(context.rewriteRtcp(srPacket1)) Assert.assertEquals(packet1.ssrc, srPacket1.senderSsrc) Assert.assertEquals(packet1.timestamp, srPacket1.senderInfo.rtpTimestamp) val srPacket2 = generator2.srPacket val packetInfo2 = generator2.nextPacket() - Assert.assertFalse(context.accept(packetInfo2, 1, targetIndex)) + Assert.assertFalse(context.accept(packetInfo2, targetIndex)) Assert.assertFalse(context.rewriteRtcp(srPacket2)) Assert.assertEquals(expectedSeq, packet1.sequenceNumber) Assert.assertEquals(expectedTs, packet1.timestamp) @@ -649,14 +731,14 @@ class Vp9AdaptiveSourceProjectionTest { if (packet1.isStartOfFrame && packet1.temporalLayerIndex == 0) { expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) } - Assert.assertTrue(context.accept(packetInfo1, 0, targetIndex)) + Assert.assertTrue(context.accept(packetInfo1, targetIndex)) context.rewriteRtp(packetInfo1) Assert.assertTrue(context.rewriteRtcp(srPacket1)) Assert.assertEquals(packet1.ssrc, srPacket1.senderSsrc) Assert.assertEquals(packet1.timestamp, srPacket1.senderInfo.rtpTimestamp) val srPacket2 = generator2.srPacket val packetInfo2 = generator2.nextPacket() - Assert.assertFalse(context.accept(packetInfo2, 1, targetIndex)) + Assert.assertFalse(context.accept(packetInfo2, targetIndex)) Assert.assertFalse(context.rewriteRtcp(srPacket2)) Assert.assertEquals(expectedSeq, packet1.sequenceNumber) Assert.assertEquals(expectedTs, packet1.timestamp) @@ -681,7 +763,7 @@ class Vp9AdaptiveSourceProjectionTest { } /* We will cut off the layer 0 keyframe after 1 packet, once we see the layer 1 keyframe. */ - Assert.assertEquals(i == 0, context.accept(packetInfo1, 0, targetIndex)) + Assert.assertEquals(i == 0, context.accept(packetInfo1, targetIndex)) Assert.assertEquals(i == 0, context.rewriteRtcp(srPacket1)) if (i == 0) { context.rewriteRtp(packetInfo1) @@ -694,7 +776,7 @@ class Vp9AdaptiveSourceProjectionTest { if (packet2.isStartOfFrame && packet2.temporalLayerIndex == 0) { expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) } - Assert.assertTrue(context.accept(packetInfo2, 1, targetIndex)) + Assert.assertTrue(context.accept(packetInfo2, targetIndex)) context.rewriteRtp(packetInfo2) Assert.assertTrue(context.rewriteRtcp(srPacket2)) Assert.assertEquals(packet2.ssrc, srPacket2.senderSsrc) @@ -740,7 +822,7 @@ class Vp9AdaptiveSourceProjectionTest { for (i in 0..9999) { val packetInfo = generator.nextPacket() val packet = packetInfo.packetAs() - val accepted = context.accept(packetInfo, 0, targetIndex) + val accepted = context.accept(packetInfo, targetIndex) if (packet.isStartOfFrame && packet.temporalLayerIndex == 0) { expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) } @@ -775,6 +857,52 @@ class Vp9AdaptiveSourceProjectionTest { } } + @Test + fun simulcastToSvcSwitchTest() { + val simulcastGenerator = SimulcastVp9PacketGenerator(packetsPerFrame = 3, numEncodings = 3) + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "twoStreamsSwitchingTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Vp9AdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + val simulcastTargetIndex = getIndex(1, 0, 2) + for (i in 0..999) { + val packetInfo = simulcastGenerator.nextPacket() + val packet = packetInfo.packetAs() + val accepted = context.accept(packetInfo, simulcastTargetIndex) + Assert.assertTrue(packet.spatialLayerIndex == 0) + if (packet.encodingId == 1 || packet.isKeyframe && packet.encodingId < 1) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + } else { + Assert.assertFalse(accepted) + } + } + val ksvcGenerator = + ScalableVp9PacketGenerator( + packetsPerFrame = 3, + numLayers = 3, + initialRtpState = simulcastGenerator.getRtpState() + ) + val svcTargetIndex = getIndex(0, 1, 2) + for (i in 0..9999) { + val packetInfo = ksvcGenerator.nextPacket() + val packet = packetInfo.packetAs() + val accepted = context.accept(packetInfo, svcTargetIndex) + if (packet.encodingId == 0 && + (packet.spatialLayerIndex == 1 || (packet.isKeyframe && packet.spatialLayerIndex < 1)) + ) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + } else { + Assert.assertFalse(accepted) + } + } + } + private fun runLargeDropoutTest(generator: Vp9PacketGenerator, targetIndex: Int) { val diagnosticContext = DiagnosticContext() diagnosticContext["test"] = Thread.currentThread().stackTrace[2].methodName @@ -795,7 +923,6 @@ class Vp9AdaptiveSourceProjectionTest { val packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - 0, targetIndex ) if (packet.isStartOfFrame && packet.temporalLayerIndex == 0) { @@ -833,7 +960,7 @@ class Vp9AdaptiveSourceProjectionTest { packetInfo = generator.nextPacket() packet = packetInfo.packetAs() } while (packet.temporalLayerIndex > targetIndex) - Assert.assertTrue(context.accept(packetInfo, 0, targetIndex)) + Assert.assertTrue(context.accept(packetInfo, targetIndex)) context.rewriteRtp(packetInfo) /* Allow any values after a gap. */ @@ -850,7 +977,6 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - 0, targetIndex ) if (packet.isStartOfFrame && packet.temporalLayerIndex == 0) { @@ -927,7 +1053,6 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - 0, targetIndex ) if (packet.isStartOfFrame && packet.temporalLayerIndex == 0) { @@ -967,7 +1092,6 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - 0, targetIndex ) Assert.assertTrue(accepted) @@ -985,7 +1109,6 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - 0, RtpLayerDesc.SUSPENDED_INDEX ) Assert.assertFalse(accepted) @@ -1001,7 +1124,6 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - 0, targetIndex ) Assert.assertFalse(accepted) @@ -1018,7 +1140,6 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - 0, targetIndex ) Assert.assertFalse(accepted) @@ -1033,7 +1154,6 @@ class Vp9AdaptiveSourceProjectionTest { packet = packetInfo.packetAs() val accepted = context.accept( packetInfo, - 0, targetIndex ) if (packet.isStartOfFrame && packet.temporalLayerIndex == 0) { @@ -1096,22 +1216,24 @@ class Vp9AdaptiveSourceProjectionTest { abstract val ts: Long var ssrc: Long = 0xcafebabeL + abstract fun getRtpState(): RtpState + abstract fun reset() abstract fun nextPacket(): PacketInfo abstract fun requestKeyframe() abstract val packetOfFrame: Int - init { - reset() - } - companion object { val baseReceivedTime = Instant.ofEpochMilli(1577836800000L) // 2020-01-01 00:00:00 UTC } } - private class NonScalableVp9PacketGenerator() : Vp9PacketGenerator() { + private class NonScalableVp9PacketGenerator( + val initialRtpState: RtpState? = null, + val encodingId: Int = 0 + ) : Vp9PacketGenerator() { private var seq = 0 + private set override var ts: Long = 0 private set private var picId = 0 @@ -1121,12 +1243,31 @@ class Vp9AdaptiveSourceProjectionTest { private var frameCount = 0 private var receivedTime = baseReceivedTime + private val useRandom = initialRtpState == null // or switch off to ease debugging + + override fun getRtpState(): RtpState = RtpState(ssrc, seq, ts) + + init { + reset() + } + override fun reset() { - val useRandom = true // switch off to ease debugging val seed = System.currentTimeMillis() val random = Random(seed) - seq = if (useRandom) random.nextInt() % 0x10000 else 0 - ts = if (useRandom) random.nextLong() % 0x100000000L else 0 + seq = if (initialRtpState != null) { + RtpUtils.applySequenceNumberDelta(initialRtpState.maxSequenceNumber, 1) + } else if (useRandom) { + random.nextInt() % 0x10000 + } else { + 0 + } + ts = if (initialRtpState != null) { + RtpUtils.applyTimestampDelta(initialRtpState.maxTimestamp, 3000) + } else if (useRandom) { + random.nextLong() % 0x100000000L + } else { + 0 + } picId = 0 packetOfFrame = 0 keyframePicture = true @@ -1178,6 +1319,7 @@ class Vp9AdaptiveSourceProjectionTest { Assert.assertEquals(false, vp9Packet.usesInterLayerDependency) Assert.assertEquals(keyframePicture, vp9Packet.isKeyframe) + vp9Packet.encodingId = encodingId vp9Packet.pictureId = picId val info = PacketInfo(vp9Packet) info.receivedTime = receivedTime @@ -1231,10 +1373,13 @@ class Vp9AdaptiveSourceProjectionTest { private class ScalableVp9PacketGenerator( override val packetsPerFrame: Int, val numLayers: Int = 1, - val isKsvc: Boolean = true + val isKsvc: Boolean = true, + val initialRtpState: RtpState? = null, + val encodingId: Int = 0 ) : Vp9PacketGenerator() { private var seq = 0 + private set override var ts: Long = 0 private set private var picId = 0 @@ -1248,12 +1393,32 @@ class Vp9AdaptiveSourceProjectionTest { private var octetCount = 0 private var frameCount = 0 private var receivedTime = baseReceivedTime + + private val useRandom = initialRtpState == null // or switch off to ease debugging + + override fun getRtpState(): RtpState = RtpState(ssrc, seq, ts) + + init { + reset() + } + override fun reset() { - val useRandom = true // switch off to ease debugging val seed = System.currentTimeMillis() val random = Random(seed) - seq = if (useRandom) random.nextInt() % 0x10000 else 0 - ts = if (useRandom) random.nextLong() % 0x100000000L else 0 + seq = if (initialRtpState != null) { + RtpUtils.applySequenceNumberDelta(initialRtpState.maxSequenceNumber, 1) + } else if (useRandom) { + random.nextInt() % 0x10000 + } else { + 0 + } + ts = if (initialRtpState != null) { + RtpUtils.applyTimestampDelta(initialRtpState.maxTimestamp, 3000) + } else if (useRandom) { + random.nextLong() % 0x100000000L + } else { + 0 + } picId = 0 tl0picidx = 0 packetOfFrame = 0 @@ -1315,7 +1480,11 @@ class Vp9AdaptiveSourceProjectionTest { rtpPacket.buffer, rtpPacket.payloadOffset, rtpPacket.payloadLength, - sid != numLayers - 1 + if (!isKsvc) { + sid != numLayers - 1 + } else { + keyframePicture + } ) Assert.assertTrue( @@ -1326,7 +1495,7 @@ class Vp9AdaptiveSourceProjectionTest { sid, tid, tid > 0, - sid > 0 && (isKsvc || keyframePicture) + sid > 0 && (!isKsvc || keyframePicture) ) ) @@ -1338,13 +1507,21 @@ class Vp9AdaptiveSourceProjectionTest { Assert.assertEquals(endOfFrame, vp9Packet.isEndOfFrame) Assert.assertEquals(endOfPicture, vp9Packet.isEndOfPicture) Assert.assertEquals(!keyframePicture, vp9Packet.isInterPicturePredicted) - Assert.assertEquals(sid != numLayers - 1, vp9Packet.isUpperLevelReference) + Assert.assertEquals( + if (!isKsvc) { + sid != numLayers - 1 + } else { + keyframePicture + }, + vp9Packet.isUpperLevelReference + ) Assert.assertEquals(sid, vp9Packet.spatialLayerIndex) Assert.assertEquals(tid, vp9Packet.temporalLayerIndex) Assert.assertEquals(tid > 0, vp9Packet.isSwitchingUpPoint) - Assert.assertEquals(sid > 0 && (isKsvc || keyframePicture), vp9Packet.usesInterLayerDependency) + Assert.assertEquals(sid > 0 && (!isKsvc || keyframePicture), vp9Packet.usesInterLayerDependency) Assert.assertEquals(keyframePicture && sid == 0, vp9Packet.isKeyframe) + vp9Packet.encodingId = encodingId vp9Packet.pictureId = picId vp9Packet.TL0PICIDX = tl0picidx val info = PacketInfo(vp9Packet) @@ -1427,6 +1604,239 @@ class Vp9AdaptiveSourceProjectionTest { ) } } + + private class SimulcastVp9PacketGenerator( + override val packetsPerFrame: Int, + val numEncodings: Int = 1, + val initialRtpState: RtpState? = null + ) : + Vp9PacketGenerator() { + private var seq = IntArray(numEncodings) { 0 } + override var ts: Long = 0 + private set + private var picId = 0 + private var tl0picidx = 0 + override var packetOfFrame = 0 + private var keyframePicture = false + private var keyframeRequested = false + private var enc = 0 + private var tidCycle = 0 + private var packetCount = 0 + private var octetCount = 0 + private var frameCount = 0 + private var receivedTime = baseReceivedTime + + private val ssrcs = arrayOf(ssrc, 0xdeadbeefL, 0xc001d00dL) + init { + require(numEncodings <= ssrcs.size) + } + + private val useRandom = initialRtpState == null // or switch off to ease debugging + + override fun getRtpState(): RtpState = RtpState(ssrc, seq[0], ts) + + init { + reset() + } + + override fun reset() { + val seed = System.currentTimeMillis() + val random = Random(seed) + seq[0] = if (initialRtpState != null) { + RtpUtils.applySequenceNumberDelta(initialRtpState.maxSequenceNumber, 1) + } else if (useRandom) { + random.nextInt() % 0x10000 + } else { + 0 + } + for (i in 1 until numEncodings) { + seq[i] = if (useRandom) { + random.nextInt() % 0x10000 + } else { + seq[0] + } + } + ts = if (initialRtpState != null) { + RtpUtils.applyTimestampDelta(initialRtpState.maxTimestamp, 3000) + } else if (useRandom) { + random.nextLong() % 0x100000000L + } else { + 0 + } + picId = 0 + tl0picidx = 0 + packetOfFrame = 0 + keyframePicture = true + keyframeRequested = false + enc = 0 + tidCycle = 0 + ssrc = 0xcafebabeL + packetCount = 0 + octetCount = 0 + frameCount = 0 + receivedTime = baseReceivedTime + } + + override fun nextPacket(): PacketInfo { + val tid = when (tidCycle % 4) { + 0 -> 0 + 2 -> 1 + 1, 3 -> 2 + else -> { + assert(false) // Math is broken + -1 + } + } + val startOfFrame = packetOfFrame == 0 + val endOfFrame = packetOfFrame == packetsPerFrame - 1 + val startOfPicture = startOfFrame && enc == 0 + val endOfPicture = endOfFrame && enc == numEncodings - 1 + if (startOfPicture && tid == 0) { + tl0picidx = applyTl0PicIdxDelta(tl0picidx, 1) + } + val buffer = vp9SvcPacketTemplate.clone() + val rtpPacket = RtpPacket(buffer, 0, buffer.size) + rtpPacket.ssrc = ssrcs[enc] + rtpPacket.sequenceNumber = seq[enc] + rtpPacket.timestamp = ts + + /* Do VP9 manipulations on buffer before constructing Vp9Packet, because + Vp9Packet computes values at construct-time. */ + DePacketizer.VP9PayloadDescriptor.setStartOfFrame( + rtpPacket.buffer, + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + startOfFrame + ) + DePacketizer.VP9PayloadDescriptor.setEndOfFrame( + rtpPacket.buffer, + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + endOfFrame + ) + DePacketizer.VP9PayloadDescriptor.setInterPicturePredicted( + rtpPacket.buffer, + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + !keyframePicture + ) + DePacketizer.VP9PayloadDescriptor.setUpperLevelReference( + rtpPacket.buffer, + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + false + ) + + Assert.assertTrue( + DePacketizer.VP9PayloadDescriptor.setLayerIndices( + rtpPacket.buffer, + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + 0, + tid, + tid > 0, + false + ) + ) + + rtpPacket.isMarked = endOfFrame + val vp9Packet = rtpPacket.toOtherType(::Vp9Packet) + + /* Make sure our manipulations of the raw buffer were correct. */ + Assert.assertEquals(startOfFrame, vp9Packet.isStartOfFrame) + Assert.assertEquals(endOfFrame, vp9Packet.isEndOfFrame) + Assert.assertEquals(endOfFrame, vp9Packet.isEndOfPicture) + Assert.assertEquals(!keyframePicture, vp9Packet.isInterPicturePredicted) + Assert.assertFalse(vp9Packet.isUpperLevelReference) + Assert.assertEquals(0, vp9Packet.spatialLayerIndex) + Assert.assertEquals(tid, vp9Packet.temporalLayerIndex) + Assert.assertEquals(tid > 0, vp9Packet.isSwitchingUpPoint) + Assert.assertFalse(vp9Packet.usesInterLayerDependency) + Assert.assertEquals(keyframePicture, vp9Packet.isKeyframe) + + vp9Packet.encodingId = enc + vp9Packet.pictureId = picId + vp9Packet.TL0PICIDX = tl0picidx + val info = PacketInfo(vp9Packet) + info.receivedTime = receivedTime + seq[enc] = RtpUtils.applySequenceNumberDelta(seq[enc], 1) + packetCount++ + octetCount += vp9Packet.length + if (endOfFrame) { + packetOfFrame = 0 + if (endOfPicture) { + enc = 0 + } else { + enc++ + } + } else { + packetOfFrame++ + } + if (endOfPicture) { + ts = RtpUtils.applyTimestampDelta(ts, 3000) + picId = applyExtendedPictureIdDelta(picId, 1) + tidCycle++ + keyframePicture = keyframeRequested + keyframeRequested = false + if (keyframePicture) { + tidCycle = 0 + } + frameCount++ + receivedTime = baseReceivedTime + Duration.ofMillis(frameCount * 100L / 3) + } + return info + } + + override fun requestKeyframe() { + if (packetOfFrame == 0) { + keyframePicture = true + keyframeRequested = false + tidCycle = 0 + } else { + keyframeRequested = true + } + } + + val srPacket: RtcpSrPacket + get() { + val srPacketBuilder = RtcpSrPacketBuilder() + srPacketBuilder.rtcpHeader.senderSsrc = ssrc + val siBuilder = srPacketBuilder.senderInfo + siBuilder.setNtpFromJavaTime(receivedTime.toEpochMilli()) + siBuilder.rtpTimestamp = ts + siBuilder.sendersOctetCount = packetCount.toLong() + siBuilder.sendersOctetCount = octetCount.toLong() + return srPacketBuilder.build() + } + + companion object { + private val vp9SvcPacketTemplate = DatatypeConverter.parseHexBinary( + // RTP Header + // V, P, X, CC + "80" + + // M, PT + "60" + + // Seq + "0000" + + // TS + "00000000" + + // SSRC + "cafebabe" + + // VP9 Payload descriptor + // I=1,P=0,L=1,F=0,B=1,E=0,V=0,Z=0 + "a8" + + // M=1,PID=0x653e=25918 + "e53e" + + // TID=0,U=0,SID=0,D=0 + "00" + + // TL0PICIDX=0x5b=91 + "5b" + + /* TODO: Add SS if necessary. Not currently parsed by the source projection context. */ + // Dummy payload data + "000000" + ) + } + } } private infix fun IntRange.step(next: (Int) -> Int) = From b227786f27ea1abd0052afae9afa1e66e70c3ad8 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Wed, 21 Feb 2024 09:36:53 -0500 Subject: [PATCH 084/189] Add more fields to the VP9 timeseries trace. (#2100) --- .../videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt index d8783baee1..13475f819b 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt @@ -156,13 +156,18 @@ class Vp9AdaptiveSourceProjectionContext( .addField("pictureId", packet.pictureId) .addField("pictureIdIndex", frame.index) .addField("encoding", incomingEncoding) + .addField("keyframe", packet.isKeyframe) .addField("spatialLayer", packet.spatialLayerIndex) .addField("temporalLayer", packet.temporalLayerIndex) .addField("isInterPicturePredicted", packet.isInterPicturePredicted) .addField("usesInterLayerDependency", packet.usesInterLayerDependency) .addField("isUpperLevelReference", packet.isUpperLevelReference) + .addField("startOfFrame", packet.isStartOfFrame) + .addField("endOfFrame", packet.isEndOfFrame) + .addField("mark", packet.isMarked) .addField("targetIndex", indexString(targetIndex)) .addField("new_frame", result.isNewFrame) + .addField("reset", result.isReset) .addField("accept", accept) vp9QualityFilter.addDiagnosticContext(pt) timeSeriesLogger.trace(pt) From f8f2f8131bf190680e3f936188f79593bf754282 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Wed, 21 Feb 2024 13:39:39 -0500 Subject: [PATCH 085/189] Make PcapWriter's output directory configurable. (#2103) --- .../org/jitsi/nlj/transform/node/PcapWriter.kt | 12 ++++++++++-- .../jitsi/nlj/transform/node/ToggleablePcapWriter.kt | 3 ++- .../src/main/resources/reference.conf | 2 ++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PcapWriter.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PcapWriter.kt index 4dbdac812c..47507b693c 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PcapWriter.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PcapWriter.kt @@ -15,6 +15,9 @@ */ package org.jitsi.nlj.transform.node +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.from import org.jitsi.nlj.PacketInfo import org.jitsi.utils.logging2.Logger import org.jitsi.utils.logging2.cinfo @@ -32,12 +35,16 @@ import org.pcap4j.packet.namednumber.IpVersion import org.pcap4j.packet.namednumber.UdpPort import org.pcap4j.util.MacAddress import java.net.Inet4Address +import java.nio.file.Path import java.util.Random +import kotlin.io.path.Path class PcapWriter( parentLogger: Logger, - filePath: String = "/tmp/${Random().nextLong()}.pcap" + filePath: Path = Path(directory, "${Random().nextLong()}.pcap") ) : ObserverNode("PCAP writer") { + constructor(parentLogger: Logger, filePath: String) : this(parentLogger, Path(filePath)) + private val logger = createChildLogger(parentLogger) private val lazyHandle = lazy { Pcaps.openDead(DataLinkType.EN10MB, 65536) @@ -46,13 +53,14 @@ class PcapWriter( private val lazyWriter = lazy { logger.cinfo { "Pcap writer writing to file $filePath" } - handle.dumpOpen(filePath) + handle.dumpOpen(filePath.toString()) } private val writer by lazyWriter companion object { private val localhost = Inet4Address.getByName("127.0.0.1") as Inet4Address + val directory: String by config("jmt.debug.pcap.directory".from(JitsiConfig.newConfig)) } override fun observe(packetInfo: PacketInfo) { diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/ToggleablePcapWriter.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/ToggleablePcapWriter.kt index aae7dd0407..46403f6365 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/ToggleablePcapWriter.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/ToggleablePcapWriter.kt @@ -21,6 +21,7 @@ import org.jitsi.metaconfig.from import org.jitsi.nlj.PacketInfo import org.jitsi.utils.logging2.Logger import java.util.Date +import kotlin.io.path.Path class ToggleablePcapWriter( private val parentLogger: Logger, @@ -36,7 +37,7 @@ class ToggleablePcapWriter( synchronized(pcapLock) { if (pcapWriter == null) { - pcapWriter = PcapWriter(parentLogger, "/tmp/$prefix-${Date().toInstant()}.pcap") + pcapWriter = PcapWriter(parentLogger, Path(PcapWriter.directory, "$prefix-${Date().toInstant()}.pcap")) } } } diff --git a/jitsi-media-transform/src/main/resources/reference.conf b/jitsi-media-transform/src/main/resources/reference.conf index 9f7a5e252a..3c3e1438cc 100644 --- a/jitsi-media-transform/src/main/resources/reference.conf +++ b/jitsi-media-transform/src/main/resources/reference.conf @@ -138,6 +138,8 @@ jmt { // Whether to permit the API to dynamically enable the capture of // unencrypted PCAP files of media traffic. enabled=false + // The directory in which to place captured PCAP files. + directory="/tmp" } packet-loss { // Artificial loss to introduce in the receive pipeline. From 7db96701dcaf2a6cea25e985395200f9f6a7de55 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Thu, 22 Feb 2024 14:42:52 -0500 Subject: [PATCH 086/189] Don't do per-packet debug logging in SsrcCache. (#2104) We have debug logs turned on during PR testing and it blows up the log. --- jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt index 1754ab6aa0..81f8174527 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt @@ -424,16 +424,6 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: val ss = getSendSource(rs.props.ssrc1, rs.props, allowCreateOnPacket, remappings) if (ss != null) { send = ss.rewriteRtp(packet, start, rs) - logger.debug { this.toString() } - logger.debug { - if (send) { - "Sending packet: ${debugInfo(packet)} source=${rs.props.name} start=$start" - } else { - "Dropping packet from ${rs.props.name}/${packet.ssrc}. waiting for key frame." - } - } - } else { - logger.debug { "Dropping packet from ${rs.props.name}/${packet.ssrc}. source not active." } } } From 328fef7994cbc5ab6d0a91ff537dbbb3da12e0ef Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Thu, 22 Feb 2024 17:39:43 -0500 Subject: [PATCH 087/189] Don't try to rewrite TL0PICIDX on VP8 packets that don't have one. (#2105) --- jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt index 81f8174527..95663e85df 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt @@ -708,10 +708,12 @@ private class Av1DDCodecDeltas(val frameNumDelta: Int, val templateIdDelta: Int) } private fun RtpPacket.getCodecState(): CodecState? { - return when (this) { - is Vp8Packet -> Vp8CodecState(this) - is Vp9Packet -> Vp9CodecState(this) - is Av1DDPacket -> Av1DDCodecState(this) + return when { + this is Vp8Packet && isRewritable() -> Vp8CodecState(this) + this is Vp9Packet -> Vp9CodecState(this) + this is Av1DDPacket -> Av1DDCodecState(this) else -> null } } + +private fun Vp8Packet.isRewritable(): Boolean = hasTL0PICIDX From f0e4f8514083e5a46d4cad5dc6f58764e7923e8d Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Thu, 22 Feb 2024 18:13:57 -0500 Subject: [PATCH 088/189] Remove some more excessively-chatty SsrcCache logs. (#2106) --- jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt index 95663e85df..2e691ea36e 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt @@ -410,8 +410,6 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: val remappings = mutableListOf() var send = false - logger.debug { "Received packet: ${debugInfo(packet)}" } - synchronized(sendSources) { var rs = receivedSsrcs.get(packet.ssrc) if (rs == null) { @@ -447,9 +445,6 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: val rs = receivedSsrcs.get(packet.senderSsrc) ?: return false val ss = getSendSource(rs.props.ssrc1, rs.props, allowCreate = false, remappings) ?: return false ss.rewriteRtcp(packet) - logger.debug { - "Received RTCP packet. Translated receive SSRC $senderSsrc to send SSRC ${packet.senderSsrc}." - } return true } } From 0c54ff7c1c872a057bb1426419621c0562ba3f99 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Fri, 23 Feb 2024 12:37:00 -0500 Subject: [PATCH 089/189] Bump jitsi-sctp. (#2107) This version's native libraries are auto-built by GitHub actions, and now include ppc64el libraries. --- jvb/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jvb/pom.xml b/jvb/pom.xml index 02dad82449..e3023729bc 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -107,8 +107,8 @@ ${project.groupId} - sctp - 1.0-14-ge26331d + jitsi-sctp + 1.0-20-gcf25585 From 74e504033d25bf24ab6ee284b990798ff34277b6 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Thu, 29 Feb 2024 10:20:31 -0500 Subject: [PATCH 090/189] Update jitsi-sctp to 1.0-21-gfe0d028. (#2108) --- jvb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/pom.xml b/jvb/pom.xml index e3023729bc..7cf98e8893 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -108,7 +108,7 @@ ${project.groupId} jitsi-sctp - 1.0-20-gcf25585 + 1.0-21-gfe0d028 From 03b8c3b28096a5b42fa044c6b2d5cb60e0af965b Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 5 Mar 2024 16:49:43 -0500 Subject: [PATCH 091/189] Add a Java option to work around a major performance regression in JDK 17 (#2110) There was a rewrite of the Java networking stack in Java 13 which made it impossible to concurrently send UDP from multiple threads on the same DatagramSocket. See https://bugs.openjdk.org/browse/JDK-8303616. This Java option causes Java to use its old DatagramSocket implementation. Note that unfortunately this workaround does not work in Java 18 and later. --- jvb/resources/jvb.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/resources/jvb.sh b/jvb/resources/jvb.sh index fb460a67c3..97821b128b 100755 --- a/jvb/resources/jvb.sh +++ b/jvb/resources/jvb.sh @@ -19,4 +19,4 @@ fi if [ -z "$VIDEOBRIDGE_MAX_MEMORY" ]; then VIDEOBRIDGE_MAX_MEMORY=3072m; fi if [ -z "$VIDEOBRIDGE_GC_TYPE" ]; then VIDEOBRIDGE_GC_TYPE=G1GC; fi -exec java -Xmx$VIDEOBRIDGE_MAX_MEMORY $VIDEOBRIDGE_DEBUG_OPTIONS -XX:+Use$VIDEOBRIDGE_GC_TYPE -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp -Djdk.tls.ephemeralDHKeySize=2048 $LOGGING_CONFIG_PARAM $JAVA_SYS_PROPS -cp $cp $mainClass $@ +exec java -Xmx$VIDEOBRIDGE_MAX_MEMORY $VIDEOBRIDGE_DEBUG_OPTIONS -XX:+Use$VIDEOBRIDGE_GC_TYPE -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp -Djdk.tls.ephemeralDHKeySize=2048 -Djdk.net.usePlainDatagramSocketImpl=true $LOGGING_CONFIG_PARAM $JAVA_SYS_PROPS -cp $cp $mainClass $@ From 499c312fb5864ca8b2dee2502ca012b755aadfdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sa=C3=BAl=20Ibarra=20Corretg=C3=A9?= Date: Thu, 1 Feb 2024 16:04:32 +0100 Subject: [PATCH 092/189] Update Java Debian dependencies Allows for running with Java 17 without any additional install. --- debian/control | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index 949114bf16..2f944c7fa4 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Section: net Priority: extra Maintainer: Jitsi Team Uploaders: Emil Ivov , Damian Minkov -Build-Depends: debhelper (>= 9), dh-systemd, java8-runtime-headless | java8-runtime | java11-runtime-headless | java11-runtime, maven +Build-Depends: debhelper (>= 9), dh-systemd, java11-sdk, maven Standards-Version: 3.9.3 Homepage: https://jitsi.org/videobridge @@ -11,7 +11,7 @@ Package: jitsi-videobridge2 Replaces: jitsi-videobridge Conflicts: jitsi-videobridge (<= 1400-1) Architecture: all -Pre-Depends: openjdk-11-jre-headless | openjdk-11-jre | java11-runtime-headless | java11-runtime, libssl3 | libssl1.1 +Pre-Depends: java11-runtime-headless | java11-runtime, libssl3 | libssl1.1 Depends: ${misc:Depends}, procps, uuid-runtime, ruby-hocon Recommends: libpcap0.8 Description: WebRTC compatible Selective Forwarding Unit (SFU) From 64f9f34fe42679baf43aacddb58b972fd57f2115 Mon Sep 17 00:00:00 2001 From: damencho Date: Thu, 7 Mar 2024 15:00:05 -0600 Subject: [PATCH 093/189] feat: Updates java dependencies to include java17. Reverts 499c312 and adds java17. --- debian/control | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index 2f944c7fa4..5e492c55e1 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Section: net Priority: extra Maintainer: Jitsi Team Uploaders: Emil Ivov , Damian Minkov -Build-Depends: debhelper (>= 9), dh-systemd, java11-sdk, maven +Build-Depends: debhelper (>= 9), dh-systemd, openjdk-11-jdk | openjdk-17-jdk, maven Standards-Version: 3.9.3 Homepage: https://jitsi.org/videobridge @@ -11,7 +11,7 @@ Package: jitsi-videobridge2 Replaces: jitsi-videobridge Conflicts: jitsi-videobridge (<= 1400-1) Architecture: all -Pre-Depends: java11-runtime-headless | java11-runtime, libssl3 | libssl1.1 +Pre-Depends: openjdk-11-jre-headless | openjdk-11-jre | openjdk-17-jre-headless | openjdk-17-jre, libssl3 | libssl1.1 Depends: ${misc:Depends}, procps, uuid-runtime, ruby-hocon Recommends: libpcap0.8 Description: WebRTC compatible Selective Forwarding Unit (SFU) From 6bc6d4602d90c0608177881b3c33d5e7afe1b00b Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Mon, 11 Mar 2024 16:22:42 -0400 Subject: [PATCH 094/189] Update analyze-timeline.pl for timeline change to Duration. (#2109) --- resources/analyze-timeline.pl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/resources/analyze-timeline.pl b/resources/analyze-timeline.pl index 7800395ae2..a31cc390ff 100755 --- a/resources/analyze-timeline.pl +++ b/resources/analyze-timeline.pl @@ -9,7 +9,7 @@ while (<>) { if (/Reference time: /) { my $prev_time; - while (/; \(([^)]*), ([0-9]*)/g) { + while (/; \(([^)]*), PT([0-9.]*)/g) { my $name = $1; my $time = $2; @@ -30,13 +30,13 @@ $stats{$name} = Statistics::Descriptive::Full->new(); push(@stat_names, $name); } - $stats{$name}->add_data($delta_time); + $stats{$name}->add_data($delta_time * 1e3); } } } for my $name (@stat_names) { my $s = $stats{$name}; - printf("%s: min %d ms, mean %d ms, median %d ms, 90%% %d ms, 99%% %d ms, max %d ms\n", + printf("%s: min %.3f ms, mean %.3f ms, median %.3f ms, 90%% %.3f ms, 99%% %.3f ms, max %.3f ms\n", $name, $s->min(), $s->mean(), $s->median(), scalar($s->percentile(90)), scalar($s->percentile(99)), $s->max()); } From e80e868326797087904d6cfff934ab563b843ca7 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Fri, 29 Mar 2024 08:56:29 -0500 Subject: [PATCH 095/189] fix: Fix ICE TCP port. (#2114) --- jvb/src/main/kotlin/org/jitsi/videobridge/ice/Harvesters.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/ice/Harvesters.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/ice/Harvesters.kt index c5ea68ccaf..7ce6de52c0 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/ice/Harvesters.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/ice/Harvesters.kt @@ -51,7 +51,7 @@ class Harvesters private constructor( val tcpHarvester: TcpHarvester? = if (IceConfig.config.tcpEnabled) { val port = IceConfig.config.tcpPort try { - TcpHarvester(IceConfig.config.port, IceConfig.config.iceSslTcp).apply { + TcpHarvester(port, IceConfig.config.iceSslTcp).apply { logger.info("Initialized TCP harvester on port $port, ssltcp=${IceConfig.config.iceSslTcp}") IceConfig.config.tcpMappedPort?.let { mappedPort -> logger.info("Adding mapped port $mappedPort") From fb16f25d31eb307478500584185db9ae4a3ec36d Mon Sep 17 00:00:00 2001 From: bgrozev Date: Tue, 2 Apr 2024 08:52:22 -0500 Subject: [PATCH 096/189] Port all metrics to Prometheus (#2113) * feat: Add Prometheus metrics for all existing stats * ref: Remove some unused stats/metrics (jitter, bwe expiration/degraded/limited/controlled, failed conferences) * ref: Remove conference size buckets stats and audio and video sender buckets stats from /colibri/stats. They are now available as prometheus histograms. * ref: Extract periodic metric update into a separate class. * ref: Remove Conference.Statistics and Videobridge.Statistics, update metrics directly. * feat: Add a lock around updating and accessing periodic metrics. * ref: Refactor how the legacy stats interfaces (/colibri/stats and MUC presence) are fed. They read from underlying Prometheus metrics via VideobridgeStatisticsShim --- .../AimdRateControl.java | 15 - .../RemoteBitrateEstimatorAbsSendTime.java | 11 - .../bandwidthestimation/GoogleCcEstimator.kt | 1 - .../org/jitsi/videobridge/Conference.java | 184 +---- .../videobridge/EndpointMessageTransport.java | 18 +- .../org/jitsi/videobridge/Videobridge.java | 317 +-------- .../videobridge/rest/root/Application.java | 2 - .../rest/root/colibri/stats/Stats.java | 15 +- .../videobridge/rest/root/debug/Debug.java | 3 +- .../videobridge/stats/MucStatsTransport.java | 69 -- .../jitsi/videobridge/stats/Statistics.java | 325 --------- .../videobridge/stats/StatsTransport.java | 34 - .../stats/VideobridgeStatistics.java | 659 ------------------ .../org/jitsi/videobridge/AbstractEndpoint.kt | 2 - .../kotlin/org/jitsi/videobridge/Endpoint.kt | 67 +- .../main/kotlin/org/jitsi/videobridge/Main.kt | 32 +- .../videobridge/health/JvbHealthChecker.kt | 15 +- .../load_management/JvbLoadManager.kt | 5 +- .../org/jitsi/videobridge/metrics/Metrics.kt | 48 ++ .../videobridge/metrics/ThreadsMetric.kt | 29 + .../videobridge/metrics/VideobridgeMetrics.kt | 253 +++++++ .../metrics/VideobridgeMetricsContainer.kt | 16 +- .../metrics/VideobridgePeriodicMetrics.kt | 362 ++++++++++ .../org/jitsi/videobridge/relay/Relay.kt | 45 +- .../relay/RelayMessageTransport.kt | 12 +- .../videobridge/rest/binders/ServiceBinder.kt | 7 - .../videobridge/shutdown/ShutdownManager.kt | 3 + .../stats/ConferencePacketStats.kt | 4 +- .../jitsi/videobridge/stats/MucPublisher.kt | 60 ++ .../jitsi/videobridge/stats/StatsCollector.kt | 151 ---- .../stats/VideobridgeStatisticsShim.kt | 236 +++++++ .../stats/config/StatsManagerConfig.kt | 110 --- .../videobridge/version/JvbVersionService.kt | 1 + .../jitsi/videobridge/xmpp/XmppConnection.kt | 26 + .../xmpp/config/XmppClientConnectionConfig.kt | 14 +- .../org/jitsi/videobridge/ConferenceTest.kt | 5 +- .../org/jitsi/videobridge/VideobridgeTest.kt | 3 +- .../rest/root/colibri/stats/StatsTest.kt | 24 +- .../stats/config/StatsManagerConfigTest.kt | 141 ---- pom.xml | 2 +- 40 files changed, 1146 insertions(+), 2180 deletions(-) delete mode 100644 jvb/src/main/java/org/jitsi/videobridge/stats/MucStatsTransport.java delete mode 100644 jvb/src/main/java/org/jitsi/videobridge/stats/Statistics.java delete mode 100644 jvb/src/main/java/org/jitsi/videobridge/stats/StatsTransport.java delete mode 100644 jvb/src/main/java/org/jitsi/videobridge/stats/VideobridgeStatistics.java create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/metrics/Metrics.kt create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/metrics/ThreadsMetric.kt create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetrics.kt create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgePeriodicMetrics.kt create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/stats/MucPublisher.kt delete mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/stats/StatsCollector.kt create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/stats/VideobridgeStatisticsShim.kt delete mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/stats/config/StatsManagerConfig.kt delete mode 100644 jvb/src/test/kotlin/org/jitsi/videobridge/stats/config/StatsManagerConfigTest.kt diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/AimdRateControl.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/AimdRateControl.java index f343333362..3de8db7d8f 100644 --- a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/AimdRateControl.java +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/AimdRateControl.java @@ -64,11 +64,6 @@ class AimdRateControl private boolean bitrateIsInitialized; - /** - * the number of time we've expired the initial incoming estimate. - */ - private int incomingBitrateExpirations = 0; - private long currentBitrateBps; private final RateControlInput currentInput @@ -351,14 +346,6 @@ public boolean isTimeToReduceFurther(long timeNow, long incomingBitrateBps) return false; } - /** - * @return the number of time we've expired the initial incoming estimate. - */ - public int getIncomingEstimateExpirations() - { - return incomingBitrateExpirations; - } - /** * Returns true if there is a valid estimate of the incoming * bitrate, false otherwise. @@ -465,8 +452,6 @@ public void update(RateControlInput input, long nowMs) { timeFirstIncomingEstimate = -1L; } - - incomingBitrateExpirations++; } else if (timeSinceFirstIncomingEstimate > kInitializationTimeMs && input.incomingBitRate > 0L) diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RemoteBitrateEstimatorAbsSendTime.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RemoteBitrateEstimatorAbsSendTime.java index 071e9649a9..6a8f6da1b4 100644 --- a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RemoteBitrateEstimatorAbsSendTime.java +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RemoteBitrateEstimatorAbsSendTime.java @@ -155,17 +155,6 @@ public class RemoteBitrateEstimatorAbsSendTime */ private final Logger logger; - /** - * The number of expirations of the initial estimate of the underlying AIMD. - * - * @return the number of expirations of the initial estimate of the - * underlying AIMD. - */ - public int getIncomingEstimateExpirations() - { - return remoteRate.getIncomingEstimateExpirations(); - } - /** * Ctor. * diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/GoogleCcEstimator.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/GoogleCcEstimator.kt index c5ccc64f86..e051fcf7b5 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/GoogleCcEstimator.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/GoogleCcEstimator.kt @@ -110,7 +110,6 @@ class GoogleCcEstimator(diagnosticContext: DiagnosticContext, parentLogger: Logg "GoogleCcEstimator", getCurrentBw(now) ).apply { - addNumber("incomingEstimateExpirations", bitrateEstimatorAbsSendTime.incomingEstimateExpirations) bitrateEstimatorAbsSendTime.statistics?.run { addNumber("delayBasedEstimatorOffset", offset) addNumber("delayBasedEstimatorThreshold", threshold) diff --git a/jvb/src/main/java/org/jitsi/videobridge/Conference.java b/jvb/src/main/java/org/jitsi/videobridge/Conference.java index 456985d2a5..4e8c24d5ff 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Conference.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Conference.java @@ -30,6 +30,7 @@ import org.jitsi.utils.queue.*; import org.jitsi.videobridge.colibri2.*; import org.jitsi.videobridge.message.*; +import org.jitsi.videobridge.metrics.*; import org.jitsi.videobridge.relay.*; import org.jitsi.videobridge.util.*; import org.jitsi.videobridge.xmpp.*; @@ -134,11 +135,6 @@ public long getLocalVideoSsrc() */ private final Videobridge videobridge; - /** - * Holds conference statistics. - */ - private final Statistics statistics = new Statistics(); - /** * The {@link Logger} to be used by this instance to print debug * information. @@ -283,8 +279,7 @@ public Conference(Videobridge videobridge, }, 3, 3, TimeUnit.SECONDS); - Videobridge.Statistics videobridgeStatistics = videobridge.getStatistics(); - videobridgeStatistics.conferencesCreated.inc(); + VideobridgeMetrics.conferencesCreated.inc(); epConnectionStatusMonitor = new EndpointConnectionStatusMonitor(this, TaskPools.SCHEDULED_POOL, logger); epConnectionStatusMonitor.start(); } @@ -315,16 +310,6 @@ public DiagnosticContext newDiagnosticContext() } } - /** - * Gets the statistics of this {@link Conference}. - * - * @return the statistics of this {@link Conference}. - */ - public Statistics getStatistics() - { - return statistics; - } - /** * Sends a message to a subset of endpoints in the call, primary use * case being a message that has originated from an endpoint (as opposed to @@ -478,7 +463,7 @@ private void recentSpeakersChanged( if (dominantSpeakerChanged && !silence) { - getVideobridge().getStatistics().dominantSpeakerChanges.inc(); + VideobridgeMetrics.dominantSpeakerChanges.inc(); if (getEndpointCount() > 2) { maybeSendKeyframeRequest(recentSpeakers.get(0)); @@ -527,10 +512,10 @@ private void maybeSendKeyframeRequest(AbstractEndpoint dominantSpeaker) { // If all other endpoints are in tile view, there is no switch to anticipate. Don't trigger an unnecessary // keyframe. - getVideobridge().getStatistics().preemptiveKeyframeRequestsSuppressed.inc(); + VideobridgeMetrics.preemptiveKeyframeRequestsSuppressed.inc(); return; } - getVideobridge().getStatistics().preemptiveKeyframeRequestsSent.inc(); + VideobridgeMetrics.preemptiveKeyframeRequestsSent.inc(); double senderRtt = getRtt(dominantSpeaker); double maxReceiveRtt = getMaxReceiverRtt(dominantSpeaker.getId()); @@ -619,42 +604,10 @@ private void updateStatisticsOnExpire() { long durationSeconds = Math.round((System.currentTimeMillis() - creationTime) / 1000d); - Videobridge.Statistics videobridgeStatistics = getVideobridge().getStatistics(); - - videobridgeStatistics.conferencesCompleted.incAndGet(); - videobridgeStatistics.totalConferenceSeconds.addAndGet(durationSeconds); - - videobridgeStatistics.totalBytesReceived.addAndGet(statistics.totalBytesReceived.get()); - videobridgeStatistics.totalBytesSent.addAndGet(statistics.totalBytesSent.get()); - videobridgeStatistics.packetsReceived.addAndGet(statistics.totalPacketsReceived.get()); - videobridgeStatistics.packetsSent.addAndGet(statistics.totalPacketsSent.get()); - - videobridgeStatistics.totalRelayBytesReceived.addAndGet(statistics.totalRelayBytesReceived.get()); - videobridgeStatistics.totalRelayBytesSent.addAndGet(statistics.totalRelayBytesSent.get()); - videobridgeStatistics.relayPacketsReceived.addAndGet(statistics.totalRelayPacketsReceived.get()); - videobridgeStatistics.relayPacketsSent.addAndGet(statistics.totalRelayPacketsSent.get()); - - boolean hasFailed = statistics.hasIceFailedEndpoint && !statistics.hasIceSucceededEndpoint; - boolean hasPartiallyFailed = statistics.hasIceFailedEndpoint && statistics.hasIceSucceededEndpoint; - - videobridgeStatistics.endpointsDtlsFailed.addAndGet(statistics.dtlsFailedEndpoints.get()); - - if (hasPartiallyFailed) - { - videobridgeStatistics.partiallyFailedConferences.incAndGet(); - } - - if (hasFailed) - { - videobridgeStatistics.failedConferences.incAndGet(); - } + VideobridgeMetrics.conferencesCompleted.inc(); + VideobridgeMetrics.totalConferenceSeconds.add(durationSeconds); - if (logger.isInfoEnabled()) - { - logger.info("expire_conf,duration=" + durationSeconds + - ",has_failed=" + hasFailed + - ",has_partially_failed=" + hasPartiallyFailed); - } + logger.info("expire_conf,duration=" + durationSeconds); } /** @@ -739,7 +692,7 @@ public Endpoint createLocalEndpoint( id, this, logger, iceControlling, doSsrcRewriting, visitor, privateAddresses); videobridge.localEndpointCreated(visitor); - subscribeToEndpointEvents(endpoint); + endpoint.addEventHandler(() -> endpointSourcesChanged(endpoint)); addEndpoints(Collections.singleton(endpoint)); @@ -762,30 +715,6 @@ public Relay createRelay(String id, @Nullable String meshId, boolean iceControll return relay; } - private void subscribeToEndpointEvents(Endpoint endpoint) - { - endpoint.addEventHandler(new AbstractEndpoint.EventHandler() - { - @Override - public void iceSucceeded() - { - getStatistics().hasIceSucceededEndpoint = true; - } - - @Override - public void iceFailed() - { - getStatistics().hasIceFailedEndpoint = true; - } - - @Override - public void sourcesChanged() - { - endpointSourcesChanged(endpoint); - } - }); - } - /** * One or more endpoints was added or removed. * @param includesNonVisitors Whether any of the endpoints changed was not a visitor. @@ -1289,7 +1218,7 @@ public boolean levelChanged(@NotNull AbstractEndpoint endpoint, long level) return false; if (ranking.energyRanking < LoudestConfig.Companion.getNumLoudest()) return false; - videobridge.getStatistics().tossedPacketsEnergy.addValue(ranking.energyScore); + VideobridgeMetrics.tossedPacketsEnergy.getHistogram().observe(ranking.energyScore); return true; } @@ -1346,7 +1275,6 @@ public JSONObject getDebugState(boolean full, String endpointId) debugState.put("expired", expired.get()); debugState.put("creationTime", creationTime); debugState.put("speechActivity", speechActivity.getDebugState()); - debugState.put("statistics", statistics.getJson()); //debugState.put("encodingsManager", encodingsManager.getDebugState()); } @@ -1396,98 +1324,6 @@ public boolean isInactive() return encodingsManager; } - /** - * Holds conference statistics. - */ - public static class Statistics - { - /** - * The total number of bytes received in RTP packets in channels in this - * conference. Note that this is only updated when channels expire. - */ - public AtomicLong totalBytesReceived = new AtomicLong(); - - /** - * The total number of bytes sent in RTP packets in channels in this - * conference. Note that this is only updated when channels expire. - */ - public AtomicLong totalBytesSent = new AtomicLong(); - - /** - * The total number of RTP packets received in channels in this - * conference. Note that this is only updated when channels expire. - */ - public AtomicLong totalPacketsReceived = new AtomicLong(); - - /** - * The total number of RTP packets received in channels in this - * conference. Note that this is only updated when channels expire. - */ - public AtomicLong totalPacketsSent = new AtomicLong(); - - /** - * The total number of bytes received in RTP packets in relays in this - * conference. Note that this is only updated when relays expire. - */ - public AtomicLong totalRelayBytesReceived = new AtomicLong(); - - /** - * The total number of bytes sent in RTP packets in relays in this - * conference. Note that this is only updated when relays expire. - */ - public AtomicLong totalRelayBytesSent = new AtomicLong(); - - /** - * The total number of RTP packets received in relays in this - * conference. Note that this is only updated when relays expire. - */ - public AtomicLong totalRelayPacketsReceived = new AtomicLong(); - - /** - * The total number of RTP packets received in relays in this - * conference. Note that this is only updated when relays expire. - */ - public AtomicLong totalRelayPacketsSent = new AtomicLong(); - - /** - * Whether at least one endpoint in this conference failed ICE. - */ - public boolean hasIceFailedEndpoint = false; - - /** - * Whether at least one endpoint in this conference completed ICE - * successfully. - */ - public boolean hasIceSucceededEndpoint = false; - - /** - * Number of endpoints whose ICE connection was established, but DTLS - * wasn't (at the time of expiration). - */ - public AtomicInteger dtlsFailedEndpoints = new AtomicInteger(); - - /** - * Gets a snapshot of this object's state as JSON. - */ - @SuppressWarnings("unchecked") - private JSONObject getJson() - { - JSONObject jsonObject = new JSONObject(); - jsonObject.put("total_bytes_received", totalBytesReceived.get()); - jsonObject.put("total_bytes_sent", totalBytesSent.get()); - jsonObject.put("total_packets_received", totalPacketsReceived.get()); - jsonObject.put("total_packets_sent", totalPacketsSent.get()); - jsonObject.put("total_relay_bytes_received", totalRelayBytesReceived.get()); - jsonObject.put("total_relay_bytes_sent", totalRelayBytesSent.get()); - jsonObject.put("total_relay_packets_received", totalRelayPacketsReceived.get()); - jsonObject.put("total_relay_packets_sent", totalRelayPacketsSent.get()); - jsonObject.put("has_failed_endpoint", hasIceFailedEndpoint); - jsonObject.put("has_succeeded_endpoint", hasIceSucceededEndpoint); - jsonObject.put("dtls_failed_endpoints", dtlsFailedEndpoints.get()); - return jsonObject; - } - } - /** * This is a no-op diagnostic context (one that will record nothing) meant * to disable logging of time-series for health checks. diff --git a/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java b/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java index 7b6f2a28d2..4ab0df86be 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java +++ b/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java @@ -22,6 +22,7 @@ import org.jitsi.videobridge.datachannel.*; import org.jitsi.videobridge.datachannel.protocol.*; import org.jitsi.videobridge.message.*; +import org.jitsi.videobridge.metrics.*; import org.jitsi.videobridge.relay.*; import org.jitsi.videobridge.websocket.*; import org.json.simple.*; @@ -30,7 +31,6 @@ import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.*; -import java.util.function.*; import java.util.stream.*; import static org.jitsi.videobridge.VersionConfig.config; @@ -67,8 +67,6 @@ public class EndpointMessageTransport private WeakReference dataChannel = new WeakReference<>(null); - private final Supplier statisticsSupplier; - private final EndpointMessageTransportEventHandler eventHandler; private final AtomicInteger numOutgoingMessagesDropped = new AtomicInteger(0); @@ -84,20 +82,14 @@ public class EndpointMessageTransport /** * Initializes a new {@link EndpointMessageTransport} instance. * @param endpoint the associated {@link Endpoint}. - * @param statisticsSupplier a {@link Supplier} which returns an instance - * of {@link Videobridge.Statistics} which will - * be used to update any stats generated by this - * class */ EndpointMessageTransport( @NotNull Endpoint endpoint, - Supplier statisticsSupplier, EndpointMessageTransportEventHandler eventHandler, Logger parentLogger) { super(parentLogger); this.endpoint = endpoint; - this.statisticsSupplier = statisticsSupplier; this.eventHandler = eventHandler; } @@ -202,7 +194,7 @@ else if (dst instanceof DataChannel) private void sendMessage(DataChannel dst, BridgeChannelMessage message) { dst.sendString(message.toJson()); - statisticsSupplier.get().dataChannelMessagesSent.inc(); + VideobridgeMetrics.dataChannelMessagesSent.inc(); } /** @@ -213,14 +205,14 @@ private void sendMessage(DataChannel dst, BridgeChannelMessage message) private void sendMessage(ColibriWebSocket dst, BridgeChannelMessage message) { dst.sendString(message.toJson()); - statisticsSupplier.get().colibriWebSocketMessagesSent.inc(); + VideobridgeMetrics.colibriWebSocketMessagesSent.inc(); } @Override public void onDataChannelMessage(DataChannelMessage dataChannelMessage) { webSocketLastActive = false; - statisticsSupplier.get().dataChannelMessagesReceived.inc(); + VideobridgeMetrics.dataChannelMessagesReceived.inc(); if (dataChannelMessage instanceof DataChannelStringMessage) { @@ -406,7 +398,7 @@ public void webSocketTextReceived(ColibriWebSocket ws, String message) return; } - statisticsSupplier.get().colibriWebSocketMessagesReceived.inc(); + VideobridgeMetrics.colibriWebSocketMessagesReceived.inc(); webSocketLastActive = true; onMessage(ws, message); diff --git a/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java b/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java index 2fa1088058..9dbaa6da77 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java @@ -19,13 +19,11 @@ import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.*; import org.jitsi.health.Result; -import org.jitsi.metrics.*; import org.jitsi.nlj.*; import org.jitsi.shutdown.*; import org.jitsi.utils.*; import org.jitsi.utils.logging2.*; import org.jitsi.utils.queue.*; -import org.jitsi.utils.stats.*; import org.jitsi.utils.version.*; import org.jitsi.videobridge.health.*; import org.jitsi.videobridge.load_management.*; @@ -49,8 +47,6 @@ import java.time.*; import java.util.*; -import java.util.concurrent.atomic.*; -import java.util.stream.*; import static org.jitsi.videobridge.colibri2.Colibri2UtilKt.*; import static org.jitsi.xmpp.util.ErrorUtilKt.createError; @@ -104,11 +100,6 @@ public class Videobridge @NotNull private final Clock clock; - /** - * A class that holds some instance statistics. - */ - private final Statistics statistics = new Statistics(); - /** * Thread that checks expiration for conferences, contents, channels and * execute expire procedure for any of them. @@ -194,7 +185,7 @@ public JvbHealthChecker getJvbHealthChecker() { conference = new Conference(this, id, name, meetingId, isRtcStatsEnabled); conferencesById.put(id, conference); - statistics.currentConferences.inc(); + VideobridgeMetrics.currentConferences.inc(); if (meetingId != null) { @@ -210,19 +201,19 @@ public JvbHealthChecker getJvbHealthChecker() void localEndpointCreated(boolean visitor) { - statistics.currentLocalEndpoints.inc(); + VideobridgeMetrics.currentLocalEndpoints.inc(); if (visitor) { - statistics.currentVisitors.inc(); + VideobridgeMetrics.currentVisitors.inc(); } } void localEndpointExpired(boolean visitor) { - long remainingEndpoints = statistics.currentLocalEndpoints.decAndGet(); + long remainingEndpoints = VideobridgeMetrics.currentLocalEndpoints.decAndGet(); if (visitor) { - statistics.currentVisitors.dec(); + VideobridgeMetrics.currentVisitors.dec(); } if (remainingEndpoints < 0) @@ -275,7 +266,7 @@ public void expireConference(Conference conference) if (conference.equals(conferencesById.get(id))) { conferencesById.remove(id); - statistics.currentConferences.dec(); + VideobridgeMetrics.currentConferences.dec(); if (meetingId != null) { @@ -300,16 +291,6 @@ private String generateConferenceID() return Long.toHexString(System.currentTimeMillis() + RANDOM.nextLong()); } - /** - * Gets the statistics of this instance. - * - * @return the statistics of this instance. - */ - public Statistics getStatistics() - { - return statistics; - } - /** * Gets an existing {@link Conference} with a specific ID. * @@ -471,7 +452,7 @@ private void handleColibriRequest(XmppConnection.ColibriRequest request) public void shutdown(boolean graceful) { shutdownManager.initiateShutdown(graceful); - shutdownManager.maybeShutdown(statistics.currentLocalEndpoints.get()); + shutdownManager.maybeShutdown(VideobridgeMetrics.currentLocalEndpoints.get()); } /** @@ -486,6 +467,7 @@ public void shutdown(boolean graceful) public void setDrainMode(boolean enable) { logger.info("Received drain request. enable=" + enable); + VideobridgeMetrics.INSTANCE.getDrainMode().set(enable); drainMode = enable; } @@ -736,10 +718,6 @@ public IQ versionIqReceived(@NotNull org.jivesoftware.smackx.iqversion.packet.Ve System.getProperty("os.name") ); - // to, from and packetId are set by the caller. - // versionResult.setTo(versionRequest.getFrom()); - // versionResult.setFrom(versionRequest.getTo()); - // versionResult.setPacketID(versionRequest.getPacketID()); versionResult.setType(IQ.Type.result); return versionResult; @@ -770,285 +748,6 @@ public IQ healthCheckIqReceived(@NotNull HealthCheckIQ iq) */ public static class Statistics { - /** - * The total number of times our AIMDs have expired the incoming bitrate - * (and which would otherwise result in video suspension). - * (see {@link AimdRateControl#incomingBitrateExpirations}). - */ - public CounterMetric incomingBitrateExpirations = VideobridgeMetricsContainer.getInstance().registerCounter( - "incoming_bitrate_expirations", - "Number of times our AIMDs have expired the incoming bitrate."); - - /** - * The cumulative/total number of conferences in which ALL of the endpoints failed ICE. - */ - public CounterMetric failedConferences = VideobridgeMetricsContainer.getInstance().registerCounter( - "failed_conferences", - "Number of conferences in which ALL of the endpoints failed ICE."); - - /** - * The cumulative/total number of conferences in which SOME of the endpoints failed ICE. - */ - public CounterMetric partiallyFailedConferences = VideobridgeMetricsContainer.getInstance().registerCounter( - "partially_failed_conferences", - "Number of conferences in which SOME of the endpoints failed ICE."); - - /** - * The cumulative/total number of conferences completed/expired on this - * {@link Videobridge}. - */ - public CounterMetric conferencesCompleted = VideobridgeMetricsContainer.getInstance().registerCounter( - "conferences_completed", - "The total number of conferences completed/expired on the Videobridge."); - - /** - * The cumulative/total number of conferences created on this - * {@link Videobridge}. - */ - public CounterMetric conferencesCreated = VideobridgeMetricsContainer.getInstance().registerCounter( - "conferences_created", - "The total number of conferences created on the Videobridge."); - - /** - * The total duration in seconds of all completed conferences on this - * {@link Videobridge}. - */ - public AtomicLong totalConferenceSeconds = new AtomicLong(); - - /** - * The total number of participant-milliseconds that are loss-controlled - * (i.e. the sum of the lengths is seconds) on this {@link Videobridge}. - */ - public AtomicLong totalLossControlledParticipantMs = new AtomicLong(); - - /** - * The total number of participant-milliseconds that are loss-limited - * on this {@link Videobridge}. - */ - public AtomicLong totalLossLimitedParticipantMs = new AtomicLong(); - - /** - * The total number of participant-milliseconds that are loss-degraded - * on this {@link Videobridge}. We chose the unit to be millis because - * we expect that a lot of our calls spend very few ms (<500) in the - * lossDegraded state for example, and they might get cut to 0. - */ - public AtomicLong totalLossDegradedParticipantMs = new AtomicLong(); - - /** - * The total number of messages received from the data channels of - * the endpoints of this conference. - */ - public CounterMetric dataChannelMessagesReceived = VideobridgeMetricsContainer.getInstance().registerCounter( - "data_channel_messages_received", - "Number of messages received from the data channels of the endpoints of this conference."); - - /** - * The total number of messages sent via the data channels of the - * endpoints of this conference. - */ - public CounterMetric dataChannelMessagesSent = VideobridgeMetricsContainer.getInstance().registerCounter( - "data_channel_messages_sent", - "Number of messages sent via the data channels of the endpoints of this conference."); - - /** - * The total number of messages received from the data channels of - * the endpoints of this conference. - */ - public CounterMetric colibriWebSocketMessagesReceived = VideobridgeMetricsContainer.getInstance() - .registerCounter("colibri_web_socket_messages_received", - "Number of messages received from the data channels of the endpoints of this conference."); - - /** - * The total number of messages sent via the data channels of the - * endpoints of this conference. - */ - public CounterMetric colibriWebSocketMessagesSent = VideobridgeMetricsContainer.getInstance().registerCounter( - "colibri_web_socket_messages_sent", - "Number of messages sent via the data channels of the endpoints of this conference."); - - /** - * The total number of bytes received in RTP packets in conferences on - * this videobridge. Note that this is only updated when conferences - * expire. - */ - public AtomicLong totalBytesReceived = new AtomicLong(); - - /** - * The total number of bytes sent in RTP packets in conferences on - * this videobridge. Note that this is only updated when conferences - * expire. - */ - public AtomicLong totalBytesSent = new AtomicLong(); - - /** - * The total number of RTP packets received in conferences on this - * videobridge. Note that this is only updated when conferences - * expire. - */ - public CounterMetric packetsReceived = VideobridgeMetricsContainer.getInstance().registerCounter( - "packets_received", - "Number of RTP packets received in conferences on this videobridge."); - - /** - * The total number of RTP packets sent in conferences on this - * videobridge. Note that this is only updated when conferences - * expire. - */ - public CounterMetric packetsSent = VideobridgeMetricsContainer.getInstance().registerCounter( - "packets_sent", - "Number of RTP packets sent in conferences on this videobridge."); - - /** - * The total number of bytes received by relays in RTP packets in conferences on - * this videobridge. Note that this is only updated when conferences - * expire. - */ - public AtomicLong totalRelayBytesReceived = new AtomicLong(); - - /** - * The total number of bytes sent by relays in RTP packets in conferences on - * this videobridge. Note that this is only updated when conferences - * expire. - */ - public AtomicLong totalRelayBytesSent = new AtomicLong(); - - /** - * The total number of RTP packets received by relays in conferences on this - * videobridge. Note that this is only updated when conferences - * expire. - */ - public CounterMetric relayPacketsReceived = VideobridgeMetricsContainer.getInstance().registerCounter( - "relay_packets_received", - "Number of RTP packets received by relays in conferences on this videobridge."); - - /** - * The total number of RTP packets sent by relays in conferences on this - * videobridge. Note that this is only updated when conferences - * expire. - */ - public CounterMetric relayPacketsSent = VideobridgeMetricsContainer.getInstance().registerCounter( - "relay_packets_sent", - "Number of RTP packets sent by relays in conferences on this videobridge."); - /** - * The total number of endpoints created. - */ - public CounterMetric totalEndpoints = VideobridgeMetricsContainer.getInstance().registerCounter( - "endpoints", - "The total number of endpoints created."); - - /** - * The total number of visitor endpoints. - */ - public CounterMetric totalVisitors = VideobridgeMetricsContainer.getInstance().registerCounter( - "visitors", - "The total number of visitor endpoints created."); - - /** - * The number of endpoints which had not established an endpoint - * message transport even after some delay. - */ - public CounterMetric numEndpointsNoMessageTransportAfterDelay = VideobridgeMetricsContainer.getInstance() - .registerCounter("endpoints_no_message_transport_after_delay", - "Number of endpoints which had not established a relay message transport even after some delay."); - - /** - * The total number of relays created. - */ - public CounterMetric totalRelays = VideobridgeMetricsContainer.getInstance().registerCounter( - "relays", - "The total number of relays created."); - - /** - * The number of relays which had not established a relay - * message transport even after some delay. - */ - public CounterMetric numRelaysNoMessageTransportAfterDelay = VideobridgeMetricsContainer.getInstance() - .registerCounter("relays_no_message_transport_after_delay", - "Number of relays which had not established a relay message transport even after some delay."); - - /** - * The total number of times the dominant speaker in any conference - * changed. - */ - public CounterMetric dominantSpeakerChanges = VideobridgeMetricsContainer.getInstance().registerCounter( - "dominant_speaker_changes", - "Number of times the dominant speaker in any conference changed."); - - /** - * Number of endpoints whose ICE connection was established, but DTLS - * wasn't (at the time of expiration). - */ - public CounterMetric endpointsDtlsFailed = VideobridgeMetricsContainer.getInstance().registerCounter( - "endpoints_dtls_failed", - "Number of endpoints whose ICE connection was established, but DTLS wasn't (at time of expiration)."); - - /** - * The stress level for this bridge - */ - public Double stressLevel = 0.0; - - /** Distribution of energy scores for discarded audio packets */ - public BucketStats tossedPacketsEnergy = new BucketStats( - Stream.iterate(0L, n -> n + 1).limit(17) - .map(w -> Math.max(8 * w - 1, 0)) - .collect(Collectors.toList()), - "", ""); - - /** Number of preemptive keyframe requests that were sent. */ - public CounterMetric preemptiveKeyframeRequestsSent = VideobridgeMetricsContainer.getInstance().registerCounter( - "preemptive_keyframe_requests_sent", - "Number of preemptive keyframe requests that were sent."); - - /** Number of preemptive keyframe requests that were not sent because no endpoints were in stage view. */ - public CounterMetric preemptiveKeyframeRequestsSuppressed = VideobridgeMetricsContainer.getInstance() - .registerCounter("preemptive_keyframe_requests_suppressed", - "Number of preemptive keyframe requests that were not sent because no endpoints were in stage view."); - - /** The total number of keyframes that were received (updated on endpoint expiration). */ - public CounterMetric keyframesReceived = VideobridgeMetricsContainer.getInstance().registerCounter( - "keyframes_received", - "Number of keyframes that were received (updated on endpoint expiration)."); - - /** - * The total number of times the layering of an incoming video stream changed (updated on endpoint expiration). - */ - public CounterMetric layeringChangesReceived = VideobridgeMetricsContainer.getInstance().registerCounter( - "layering_changes_received", - "Number of times the layering of an incoming video stream changed (updated on endpoint expiration)."); - - /** - * The total duration, in milliseconds, of video streams (SSRCs) that were received. For example, if an - * endpoint sends simulcast with 3 SSRCs for 1 minute it would contribute a total of 3 minutes. Suspended - * streams do not contribute to this duration. - * - * This is updated on endpoint expiration. - */ - public AtomicLong totalVideoStreamMillisecondsReceived = new AtomicLong(); - - /** - * Number of local endpoints that exist currently. - */ - public LongGaugeMetric currentLocalEndpoints = VideobridgeMetricsContainer.getInstance().registerLongGauge( - "local_endpoints", - "Number of local endpoints that exist currently." - ); - - /** - * Number of visitor endpoints that exist currently. - */ - public LongGaugeMetric currentVisitors = VideobridgeMetricsContainer.getInstance().registerLongGauge( - "current_visitors", - "Number of visitor endpoints." - ); - - /** - * Current number of conferences. - */ - public LongGaugeMetric currentConferences = VideobridgeMetricsContainer.getInstance().registerLongGauge( - "conferences", - "Current number of conferences." - ); } private static class ConferenceNotFoundException extends Exception {} diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/root/Application.java b/jvb/src/main/java/org/jitsi/videobridge/rest/root/Application.java index d73c3ddc6e..0dfa4e2462 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/rest/root/Application.java +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/root/Application.java @@ -35,7 +35,6 @@ public class Application extends ResourceConfig public Application( Videobridge videobridge, XmppConnection xmppConnection, - StatsCollector statsCollector, @NotNull Version version, @NotNull JvbHealthChecker healthChecker) @@ -44,7 +43,6 @@ public Application( new ServiceBinder( videobridge, xmppConnection, - statsCollector, healthChecker ) ); diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/stats/Stats.java b/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/stats/Stats.java index 67ed1f62f7..73f1df49b0 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/stats/Stats.java +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/stats/Stats.java @@ -19,10 +19,7 @@ import org.jitsi.videobridge.rest.*; import org.jitsi.videobridge.rest.annotations.*; import org.jitsi.videobridge.stats.*; -import org.json.simple.*; -import org.jvnet.hk2.annotations.*; -import jakarta.inject.*; import jakarta.ws.rs.*; import jakarta.ws.rs.core.*; @@ -30,20 +27,10 @@ @EnabledByConfig(RestApis.COLIBRI) public class Stats { - @Inject - @Optional - protected StatsCollector statsManager; - @GET @Produces(MediaType.APPLICATION_JSON) public String getStats() { - StatsCollector statsManager = this.statsManager; - - if (this.statsManager != null) - { - return new JSONObject(statsManager.getStatistics().getStats()).toJSONString(); - } - return new JSONObject().toJSONString(); + return VideobridgeStatisticsShim.INSTANCE.getStatsJson().toJSONString(); } } diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/Debug.java b/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/Debug.java index 8a670e0c1d..7a82d812ac 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/Debug.java +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/Debug.java @@ -24,6 +24,7 @@ import org.jitsi.utils.logging2.Logger; import org.jitsi.utils.queue.*; import org.jitsi.videobridge.*; +import org.jitsi.videobridge.metrics.*; import org.jitsi.videobridge.relay.*; import org.jitsi.videobridge.rest.*; import org.jitsi.videobridge.rest.annotations.*; @@ -390,7 +391,7 @@ public String getJvbFeatureStats(@PathParam("feature") DebugFeatures feature) return ConferencePacketStats.stats.toJson().toJSONString(); } case TOSSED_PACKET_STATS: { - return videobridge.getStatistics().tossedPacketsEnergy.toJson().toJSONString(); + return VideobridgeMetrics.tossedPacketsEnergy.get().toJSONString(); } default: { throw new NotFoundException(); diff --git a/jvb/src/main/java/org/jitsi/videobridge/stats/MucStatsTransport.java b/jvb/src/main/java/org/jitsi/videobridge/stats/MucStatsTransport.java deleted file mode 100644 index 5b7e32d2f6..0000000000 --- a/jvb/src/main/java/org/jitsi/videobridge/stats/MucStatsTransport.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright @ 2018 - Present, 8x8 Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.jitsi.videobridge.stats; - -import org.jitsi.utils.logging2.*; -import org.jitsi.videobridge.xmpp.*; -import org.jitsi.videobridge.xmpp.config.*; -import org.jitsi.xmpp.extensions.colibri.*; -import java.util.List; - -/** - * Implements a {@link StatsTransport} which publishes via Presence in an XMPP MUC. - * - * @author Boris Grozev - */ -public class MucStatsTransport - implements StatsTransport -{ - /** - * The Logger used by the MucStatsTransport class and - * its instances to print debug information. - */ - private static final Logger logger = new LoggerImpl(MucStatsTransport.class.getName()); - - private final XmppConnection xmppConnection; - - public MucStatsTransport(XmppConnection xmppConnection) - { - this.xmppConnection = xmppConnection; - } - - /** - * {@inheritDoc} - */ - @Override - public void publishStatistics(Statistics stats, long measurementInterval) - { - logger.debug(() -> "Publishing statistics through MUC: " + stats); - - ColibriStatsExtension statsExt; - - if (XmppClientConnectionConfig.config.getStatsFilterEnabled()) - { - List whitelist = XmppClientConnectionConfig.config.getStatsWhitelist(); - logger.debug(() -> "Statistics filter applied: " + whitelist); - statsExt = Statistics.toXmppExtensionElementFiltered(stats, whitelist); - } - else - { - statsExt = Statistics.toXmppExtensionElement(stats); - } - - xmppConnection.setPresenceExtension(statsExt); - } -} - diff --git a/jvb/src/main/java/org/jitsi/videobridge/stats/Statistics.java b/jvb/src/main/java/org/jitsi/videobridge/stats/Statistics.java deleted file mode 100644 index de341aa86a..0000000000 --- a/jvb/src/main/java/org/jitsi/videobridge/stats/Statistics.java +++ /dev/null @@ -1,325 +0,0 @@ -/* - * Copyright @ 2015 - Present, 8x8 Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.jitsi.videobridge.stats; - -import java.util.*; -import java.util.concurrent.atomic.*; -import java.util.concurrent.locks.*; - -import org.jitsi.utils.logging2.*; -import org.jitsi.xmpp.extensions.colibri.*; - -/** - * Abstract class that defines common interface for a collection of statistics. - * - * @author Hristo Terezov - * @author Lyubomir Marinov - */ -public abstract class Statistics -{ - /** - * The {@link Logger} used by the {@link Endpoint} class to print debug - * information. - */ - private static final Logger logger = new LoggerImpl(Statistics.class.getName()); - - /** - * Formats statistics in ColibriStatsExtension object - * @param statistics the statistics instance - * @return the ColibriStatsExtension instance. - */ - public static ColibriStatsExtension toXmppExtensionElement( - Statistics statistics) - { - ColibriStatsExtension ext = new ColibriStatsExtension(); - - for (Map.Entry e : statistics.getStats().entrySet()) - { - ext.addStat( - new ColibriStatsExtension.Stat(e.getKey(), e.getValue())); - } - return ext; - } - - /** - * Formats statistics in ColibriStatsExtension object - * @param statistics the statistics instance - * @param whitelist which of the statistics to use - * @return the ColibriStatsExtension instance. - */ - public static ColibriStatsExtension toXmppExtensionElementFiltered( - Statistics statistics, List whitelist) - { - ColibriStatsExtension ext = new ColibriStatsExtension(); - Map m = statistics.getStats(); - whitelist.forEach(k -> { - Object v = m.get(k); - if (v != null) - { - ext.addStat(new ColibriStatsExtension.Stat(k, v)); - } - }); - return ext; - } - - /** - * The ReadWriteLock which synchronizes the access to and/or - * modification of the state of this instance. Replaces - * synchronized blocks in order to reduce the number of exclusive - * locks and, therefore, the risks of superfluous waiting. - */ - protected final ReadWriteLock lock = new ReentrantReadWriteLock(); - - /** - * Map of the names of the statistics and their values. - */ - private final Map stats = new HashMap<>(); - - /** - * Generates/updates the statistics represented by this instance. - */ - public abstract void generate(); - - /** - * Returns the value of the statistic. - * - * @param stat the name of the statistic. - * @return the value. - */ - public Object getStat(String stat) - { - Lock lock = this.lock.readLock(); - Object value; - - lock.lock(); - try - { - value = stats.get(stat); - } - finally - { - lock.unlock(); - } - return value; - } - - /** - * Gets the value of a specific piece of statistic as a {@code double} - * value. - * - * @param stat the name of the piece of statistics to return - * @return the value of {@code stat} as a {@code double} value - */ - public double getStatAsDouble(String stat) - { - Object o = getStat(stat); - double d; - double defaultValue = 0.0d; - - if (o == null) - { - d = defaultValue; - } - else if (o instanceof Number) - { - d = ((Number) o).floatValue(); - } - else - { - String s = o.toString(); - - if (s == null || s.length() == 0) - { - d = defaultValue; - } - else - { - try - { - d = Double.parseDouble(s); - } - catch (NumberFormatException nfe) - { - d = defaultValue; - } - } - } - return d; - } - - /** - * Gets the value of a specific piece of statistic as a {@code float} value. - * - * @param stat the name of the piece of statistics to return - * @return the value of {@code stat} as a {@code float} value - */ - public float getStatAsFloat(String stat) - { - Object o = getStat(stat); - float f; - float defaultValue = 0.0f; - - if (o == null) - { - f = defaultValue; - } - else if (o instanceof Number) - { - f = ((Number) o).floatValue(); - } - else - { - String s = o.toString(); - - if (s == null || s.length() == 0) - { - f = defaultValue; - } - else - { - try - { - f = Float.parseFloat(s); - } - catch (NumberFormatException nfe) - { - f = defaultValue; - } - } - } - return f; - } - - /** - * Gets the value of a specific piece of statistic as an {@code int} value. - * - * @param stat the name of the piece of statistics to return - * @return the value of {@code stat} as an {@code int} value - */ - public int getStatAsInt(String stat) - { - Object o = getStat(stat); - int i; - int defaultValue = 0; - - if (o == null) - { - i = defaultValue; - } - else if (o instanceof Number) - { - i = ((Number) o).intValue(); - } - else - { - String s = o.toString(); - - if (s == null || s.length() == 0) - { - i = defaultValue; - } - else - { - try - { - i = Integer.parseInt(s); - } - catch (NumberFormatException nfe) - { - i = defaultValue; - } - } - } - return i; - } - - /** - * Returns the map with the names of the statistics and their values. - * - * @return the map with the names of the statistics and their values. - */ - public Map getStats() - { - Lock lock = this.lock.readLock(); - Map stats; - - lock.lock(); - try - { - stats = new HashMap<>(this.stats); - } - finally - { - lock.unlock(); - } - return stats; - } - - /** - * Sets the value of statistic - * @param stat the name of the statistic - * @param value the value of the statistic - */ - public void setStat(String stat, Object value) - { - Lock lock = this.lock.writeLock(); - - lock.lock(); - try - { - unlockedSetStat(stat, value); - } - finally - { - lock.unlock(); - } - } - - /** - * {@inheritDoc} - */ - @Override - public String toString() - { - StringBuilder s = new StringBuilder(); - - for (Map.Entry e : getStats().entrySet()) - { - s.append(e.getKey()).append(":").append(e.getValue()).append("\n"); - } - return s.toString(); - } - - /** - * Sets the value of a specific piece of statistics. The method assumes that - * the caller has acquired the write lock of {@link #lock} and, thus, allows - * the optimization of batch updates to multiple pieces of statistics. - * - * @param stat the piece of statistics to set - * @param value the value of the piece of statistics to set - */ - protected void unlockedSetStat(String stat, Object value) - { - if (value instanceof AtomicLong || value instanceof AtomicInteger) - { - logger.warn(() -> "Using an Atomic number as a stat for " + stat + ", probably not what we want"); - } - if (value == null) - stats.remove(stat); - else - stats.put(stat, value); - } -} diff --git a/jvb/src/main/java/org/jitsi/videobridge/stats/StatsTransport.java b/jvb/src/main/java/org/jitsi/videobridge/stats/StatsTransport.java deleted file mode 100644 index 9e3dde283c..0000000000 --- a/jvb/src/main/java/org/jitsi/videobridge/stats/StatsTransport.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright @ 2015 - Present, 8x8 Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.jitsi.videobridge.stats; - -/** - * Defines an interface for classes that will send statistics. - * - * @author Hristo Terezov - * @author Lyubomir Marinov - */ -public interface StatsTransport -{ - /** - * Publishes a specific (set of) {@link Statistics} through this {@link StatsTransport}. - * - * @param statistics the {@link Statistics} to be published. - * @param measurementInterval the interval of time in milliseconds covered by the measurements carried by the - * specified {@link Statistics}. - */ - void publishStatistics(Statistics statistics, long measurementInterval); -} diff --git a/jvb/src/main/java/org/jitsi/videobridge/stats/VideobridgeStatistics.java b/jvb/src/main/java/org/jitsi/videobridge/stats/VideobridgeStatistics.java deleted file mode 100644 index 2613d033b6..0000000000 --- a/jvb/src/main/java/org/jitsi/videobridge/stats/VideobridgeStatistics.java +++ /dev/null @@ -1,659 +0,0 @@ -/* - * Copyright @ 2015 - Present, 8x8 Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.jitsi.videobridge.stats; - -import org.jetbrains.annotations.*; -import org.jitsi.metrics.*; -import org.jitsi.nlj.rtcp.*; -import org.jitsi.nlj.stats.*; -import org.jitsi.nlj.transform.node.incoming.*; -import org.jitsi.videobridge.*; -import org.jitsi.videobridge.load_management.*; -import org.jitsi.videobridge.metrics.*; -import org.jitsi.videobridge.relay.*; -import org.jitsi.videobridge.shutdown.*; -import org.jitsi.videobridge.transport.ice.*; -import org.jitsi.videobridge.xmpp.*; -import org.json.simple.*; - -import java.lang.management.*; -import java.text.*; -import java.util.*; -import java.util.concurrent.locks.*; - -import static org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.*; - -/** - * Implements statistics that are collected by the Videobridge. - * - * @author Hristo Terezov - * @author Lyubomir Marinov - */ -public class VideobridgeStatistics - extends Statistics -{ - /** - * The DateFormat to be utilized by VideobridgeStatistics - * in order to represent time and date as String. - */ - private final DateFormat timestampFormat; - - /** - * The number of buckets to use for conference sizes. - */ - private static final int CONFERENCE_SIZE_BUCKETS = 22; - - /** - * The currently configured region. - */ - private static final InfoMetric regionInfo = RelayConfig.config.getRegion() != null ? - VideobridgeMetricsContainer.getInstance() - .registerInfo(REGION, "The currently configured region.", RelayConfig.config.getRegion()) : null; - - private static final String relayId = RelayConfig.config.getEnabled() ? RelayConfig.config.getRelayId() : null; - - public static final String EPS_NO_MSG_TRANSPORT_AFTER_DELAY = "num_eps_no_msg_transport_after_delay"; - public static final String TOTAL_ICE_SUCCEEDED_RELAYED = "total_ice_succeeded_relayed"; - - /** - * Number of configured MUC clients. - */ - public static final String MUC_CLIENTS_CONFIGURED = "muc_clients_configured"; - - /** - * Number of configured MUC clients that are connected to XMPP. - */ - public static final String MUC_CLIENTS_CONNECTED = "muc_clients_connected"; - - /** - * Number of MUCs that are configured - */ - public static final String MUCS_CONFIGURED = "mucs_configured"; - - /** - * Number of MUCs that are joined. - */ - public static final String MUCS_JOINED = "mucs_joined"; - - /** - * Fraction of incoming packets that were lost. - */ - public static final String INCOMING_LOSS = "incoming_loss"; - - /** - * Fraction of outgoing packets that were lost. - */ - public static final String OUTGOING_LOSS = "outgoing_loss"; - - /** - * The name of the stat that tracks the total number of times our AIMDs have - * expired the incoming bitrate (and which would otherwise result in video - * suspension). - */ - private static final String TOTAL_AIMD_BWE_EXPIRATIONS = "total_aimd_bwe_expirations"; - - /** - * Fraction of incoming and outgoing packets that were lost. - */ - public static final String OVERALL_LOSS = "overall_loss"; - - /** - * The indicator which determines whether {@link #generate()} is executing - * on this VideobridgeStatistics. If true, invocations of - * generate() will do nothing. Introduced in order to mitigate an - * issue in which a blocking in generate() will cause a multiple of - * threads to be initialized and blocked. - */ - private boolean inGenerate = false; - - private final @NotNull Videobridge videobridge; - private final @NotNull XmppConnection xmppConnection; - - private final BooleanMetric healthy = VideobridgeMetricsContainer.getInstance() - .registerBooleanMetric("healthy", "Whether the Videobridge instance is healthy or not.", true); - - /** - * Creates instance of VideobridgeStatistics. - */ - public VideobridgeStatistics( - @NotNull Videobridge videobridge, - @NotNull XmppConnection xmppConnection - ) - { - this.videobridge = videobridge; - this.xmppConnection = xmppConnection; - - timestampFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); - timestampFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - - // Is it necessary to set initial values for all of these? - unlockedSetStat(BITRATE_DOWNLOAD, 0); - unlockedSetStat(BITRATE_UPLOAD, 0); - unlockedSetStat(CONFERENCES, 0); - unlockedSetStat(PARTICIPANTS, 0); - unlockedSetStat(THREADS, 0); - unlockedSetStat(VIDEO_CHANNELS, 0); - unlockedSetStat(JITTER_AGGREGATE, 0d); - unlockedSetStat(RTT_AGGREGATE, 0d); - unlockedSetStat(LARGEST_CONFERENCE, 0); - unlockedSetStat(CONFERENCE_SIZES, "[]"); - unlockedSetStat(TIMESTAMP, timestampFormat.format(new Date())); - unlockedSetStat("healthy", - healthy.setAndGet(videobridge.getJvbHealthChecker().getResult().getSuccess())); - - // Set these once, they won't change. - unlockedSetStat(VERSION, videobridge.getVersion().toString()); - - String releaseId = videobridge.getReleaseId(); - if (releaseId != null) - { - unlockedSetStat(RELEASE, releaseId); - } - if (regionInfo != null) - { - unlockedSetStat(REGION, regionInfo.get()); - } - } - - /** - * {@inheritDoc} - */ - @Override - public void generate() - { - // If a thread is already executing generate and has potentially - // blocked, do not allow other threads to fall into the same trap. - Lock lock = this.lock.writeLock(); - boolean inGenerate; - - lock.lock(); - try - { - if (this.inGenerate) - { - inGenerate = true; - } - else - { - // Enter the generate method. - inGenerate = false; - this.inGenerate = true; - } - } - finally - { - lock.unlock(); - } - if (!inGenerate) - { - try - { - generate0(); - } - finally - { - // Exit the generate method. - lock.lock(); - try - { - this.inGenerate = false; - } - finally - { - lock.unlock(); - } - } - } - } - - /** - * Generates/updates the statistics represented by this instance outside a - * synchronized block. - */ - @SuppressWarnings("unchecked") - private void generate0() - { - Videobridge.Statistics jvbStats = videobridge.getStatistics(); - - int videoChannels = 0; - int octoConferences = 0; - int endpoints = 0; - int localEndpoints = 0; - int octoEndpoints = 0; - double bitrateDownloadBps = 0; - double bitrateUploadBps = 0; - long packetRateUpload = 0; - long packetRateDownload = 0; - double relayBitrateIncomingBps = 0; - double relayBitrateOutgoingBps = 0; - long relayPacketRateOutgoing = 0; - long relayPacketRateIncoming = 0; - - // Packets we received - long incomingPacketsReceived = 0; - // Packets we should have received but were lost - long incomingPacketsLost = 0; - // Packets we sent that were reported received - long outgoingPacketsReceived = 0; - // Packets we sent that were reported lost - long outgoingPacketsLost = 0; - - // Average jitter and RTT across MediaStreams which report a valid value. - double jitterSumMs = 0; // TODO verify - int jitterCount = 0; - double rttSumMs = 0; - long rttCount = 0; - int largestConferenceSize = 0; - int[] conferenceSizes = new int[CONFERENCE_SIZE_BUCKETS]; - int[] audioSendersBuckets = new int[CONFERENCE_SIZE_BUCKETS]; - int[] videoSendersBuckets = new int[CONFERENCE_SIZE_BUCKETS]; - int inactiveConferences = 0; - int p2pConferences = 0; - int inactiveEndpoints = 0; - int receiveOnlyEndpoints = 0; - int numAudioSenders = 0; - int numVideoSenders = 0; - // The number of endpoints to which we're "oversending" (which can occur when - // enableOnstageVideoSuspend is false) - int numOversending = 0; - int endpointsWithHighOutgoingLoss = 0; - int numLocalActiveEndpoints = 0; - int endpointsWithSuspendedSources = 0; - - for (Conference conference : videobridge.getConferences()) - { - long conferenceBitrate = 0; - long conferencePacketRate = 0; - if (conference.isP2p()) - { - p2pConferences++; - } - - boolean inactive = conference.isInactive(); - if (inactive) - { - inactiveConferences++; - inactiveEndpoints += conference.getEndpointCount(); - } - else - { - numLocalActiveEndpoints += conference.getLocalEndpointCount(); - } - - if (conference.hasRelays()) - { - octoConferences++; - } - int numConferenceEndpoints = conference.getEndpointCount(); - int numLocalEndpoints = conference.getLocalEndpointCount(); - localEndpoints += numLocalEndpoints; - if (numConferenceEndpoints > largestConferenceSize) - { - largestConferenceSize = numConferenceEndpoints; - } - - updateBuckets(conferenceSizes, numConferenceEndpoints); - endpoints += numConferenceEndpoints; - octoEndpoints += (numConferenceEndpoints - numLocalEndpoints); - - int conferenceAudioSenders = 0; - int conferenceVideoSenders = 0; - - for (Endpoint endpoint : conference.getLocalEndpoints()) - { - if (endpoint.getAcceptVideo()) - { - videoChannels++; - } - - if (endpoint.isOversending()) - { - numOversending++; - } - boolean sendingAudio = endpoint.isSendingAudio(); - boolean sendingVideo = endpoint.isSendingVideo(); - if (sendingAudio) - { - conferenceAudioSenders++; - } - if (sendingVideo) - { - conferenceVideoSenders++; - } - if (!sendingAudio && !sendingVideo && !inactive) - { - receiveOnlyEndpoints++; - } - if (endpoint.hasSuspendedSources()) - { - endpointsWithSuspendedSources++; - } - TransceiverStats transceiverStats = endpoint.getTransceiver().getTransceiverStats(); - IncomingStatisticsSnapshot incomingStats = transceiverStats.getRtpReceiverStats().getIncomingStats(); - PacketStreamStats.Snapshot incomingPacketStreamStats - = transceiverStats.getRtpReceiverStats().getPacketStreamStats(); - bitrateDownloadBps += incomingPacketStreamStats.getBitrateBps(); - packetRateDownload += incomingPacketStreamStats.getPacketRate(); - conferenceBitrate += incomingPacketStreamStats.getBitrateBps(); - conferencePacketRate += incomingPacketStreamStats.getPacketRate(); - for (IncomingSsrcStats.Snapshot ssrcStats : incomingStats.getSsrcStats().values()) - { - double ssrcJitter = ssrcStats.getJitter(); - if (ssrcJitter != 0) - { - // We take the abs because otherwise the - // aggregate makes no sense. - jitterSumMs += Math.abs(ssrcJitter); - jitterCount++; - } - } - - PacketStreamStats.Snapshot outgoingStats = transceiverStats.getOutgoingPacketStreamStats(); - bitrateUploadBps += outgoingStats.getBitrateBps(); - packetRateUpload += outgoingStats.getPacketRate(); - conferenceBitrate += outgoingStats.getBitrateBps(); - conferencePacketRate += outgoingStats.getPacketRate(); - - EndpointConnectionStats.Snapshot endpointConnectionStats - = transceiverStats.getEndpointConnectionStats(); - double endpointRtt = endpointConnectionStats.getRtt(); - if (endpointRtt > 0) - { - rttSumMs += endpointRtt; - rttCount++; - } - - incomingPacketsReceived += endpointConnectionStats.getIncomingLossStats().getPacketsReceived(); - incomingPacketsLost += endpointConnectionStats.getIncomingLossStats().getPacketsLost(); - - long endpointOutgoingPacketsReceived - = endpointConnectionStats.getOutgoingLossStats().getPacketsReceived(); - long endpointOutgoingPacketsLost = endpointConnectionStats.getOutgoingLossStats().getPacketsLost(); - outgoingPacketsReceived += endpointOutgoingPacketsReceived; - outgoingPacketsLost += endpointOutgoingPacketsLost; - - if (!inactive && endpointOutgoingPacketsLost + endpointOutgoingPacketsReceived > 0) - { - double endpointOutgoingFractionLost = ((double) endpointOutgoingPacketsLost) - / (endpointOutgoingPacketsLost + endpointOutgoingPacketsReceived); - if (endpointOutgoingFractionLost > 0.1) - { - endpointsWithHighOutgoingLoss++; - } - } - } - - for (Relay relay : conference.getRelays()) - { - relayBitrateIncomingBps += relay.getIncomingBitrateBps(); - relayPacketRateIncoming += relay.getIncomingPacketRate(); - conferenceBitrate += relay.getIncomingBitrateBps(); - conferencePacketRate += relay.getIncomingPacketRate(); - - relayBitrateOutgoingBps += relay.getOutgoingBitrateBps(); - relayPacketRateOutgoing += relay.getOutgoingPacketRate(); - conferenceBitrate += relay.getOutgoingBitrateBps(); - conferencePacketRate += relay.getOutgoingPacketRate(); - - /* TODO: report Relay RTT and loss, like we do for Endpoints? */ - } - - updateBuckets(audioSendersBuckets, conferenceAudioSenders); - numAudioSenders += conferenceAudioSenders; - updateBuckets(videoSendersBuckets, conferenceVideoSenders); - numVideoSenders += conferenceVideoSenders; - ConferencePacketStats.stats.addValue(numConferenceEndpoints, conferencePacketRate, conferenceBitrate); - } - - // JITTER_AGGREGATE - double jitterAggregate - = jitterCount > 0 - ? jitterSumMs / jitterCount - : 0; - - // RTT_AGGREGATE - double rttAggregate - = rttCount > 0 - ? rttSumMs / rttCount - : 0; - - // CONFERENCE_SIZES - JSONArray conferenceSizesJson = new JSONArray(); - for (int size : conferenceSizes) - conferenceSizesJson.add(size); - - JSONArray audioSendersJson = new JSONArray(); - for (int n : audioSendersBuckets) - { - audioSendersJson.add(n); - } - JSONArray videoSendersJson = new JSONArray(); - for (int n : videoSendersBuckets) - { - videoSendersJson.add(n); - } - - // THREADS - int threadCount = ManagementFactory.getThreadMXBean().getThreadCount(); - - double incomingLoss = 0; - if (incomingPacketsReceived + incomingPacketsLost > 0) - { - incomingLoss = ((double) incomingPacketsLost) / (incomingPacketsReceived + incomingPacketsLost); - } - - double outgoingLoss = 0; - if (outgoingPacketsReceived + outgoingPacketsLost > 0) - { - outgoingLoss = ((double) outgoingPacketsLost) / (outgoingPacketsReceived + outgoingPacketsLost); - } - - double overallLoss = 0; - if (incomingPacketsReceived + incomingPacketsLost + outgoingPacketsReceived + outgoingPacketsLost > 0) - { - overallLoss = ((double) (outgoingPacketsLost + incomingPacketsLost)) - / (incomingPacketsReceived + incomingPacketsLost + outgoingPacketsReceived + outgoingPacketsLost); - } - - // Now that (the new values of) the statistics have been calculated and - // the risks of the current thread hanging have been reduced as much as - // possible, commit (the new values of) the statistics. - Lock lock = this.lock.writeLock(); - - lock.lock(); - try - { - unlockedSetStat(INCOMING_LOSS, incomingLoss); - unlockedSetStat(OUTGOING_LOSS, outgoingLoss); - - unlockedSetStat(OVERALL_LOSS, overallLoss); - // The number of active endpoints that have more than 10% loss in the bridge->endpoint direction. - unlockedSetStat("endpoints_with_high_outgoing_loss", endpointsWithHighOutgoingLoss); - // The number of local (non-octo) active (in a conference where at least one endpoint sends audio or video) - // endpoints. - unlockedSetStat("local_active_endpoints", numLocalActiveEndpoints); - unlockedSetStat( - BITRATE_DOWNLOAD, - bitrateDownloadBps / 1000 /* kbps */); - unlockedSetStat( - BITRATE_UPLOAD, - bitrateUploadBps / 1000 /* kbps */); - unlockedSetStat(PACKET_RATE_DOWNLOAD, packetRateDownload); - unlockedSetStat(PACKET_RATE_UPLOAD, packetRateUpload); - unlockedSetStat( - TOTAL_AIMD_BWE_EXPIRATIONS, - jvbStats.incomingBitrateExpirations.get()); - // TODO seems broken (I see values of > 11 seconds) - unlockedSetStat(JITTER_AGGREGATE, jitterAggregate); - unlockedSetStat(RTT_AGGREGATE, rttAggregate); - unlockedSetStat( - TOTAL_FAILED_CONFERENCES, - jvbStats.failedConferences.get()); - unlockedSetStat( - TOTAL_PARTIALLY_FAILED_CONFERENCES, - jvbStats.partiallyFailedConferences.get()); - unlockedSetStat( - TOTAL_CONFERENCES_CREATED, - jvbStats.conferencesCreated.get()); - unlockedSetStat( - TOTAL_CONFERENCES_COMPLETED, - jvbStats.conferencesCompleted.get()); - unlockedSetStat( - TOTAL_ICE_FAILED, - IceTransport.Companion.getIceFailed().get()); - unlockedSetStat( - TOTAL_ICE_SUCCEEDED, - IceTransport.Companion.getIceSucceeded().get()); - unlockedSetStat( - TOTAL_ICE_SUCCEEDED_TCP, - IceTransport.Companion.getIceSucceededTcp().get()); - unlockedSetStat( - TOTAL_ICE_SUCCEEDED_RELAYED, - IceTransport.Companion.getIceSucceededRelayed().get()); - unlockedSetStat( - TOTAL_CONFERENCE_SECONDS, - jvbStats.totalConferenceSeconds.get()); - - unlockedSetStat( - TOTAL_LOSS_CONTROLLED_PARTICIPANT_SECONDS, - jvbStats.totalLossControlledParticipantMs.get() / 1000); - unlockedSetStat( - TOTAL_LOSS_LIMITED_PARTICIPANT_SECONDS, - jvbStats.totalLossLimitedParticipantMs.get() / 1000); - unlockedSetStat( - TOTAL_LOSS_DEGRADED_PARTICIPANT_SECONDS, - jvbStats.totalLossDegradedParticipantMs.get() / 1000); - unlockedSetStat(TOTAL_PARTICIPANTS, jvbStats.totalEndpoints.get()); - unlockedSetStat("total_visitors", jvbStats.totalVisitors.get()); - unlockedSetStat( - EPS_NO_MSG_TRANSPORT_AFTER_DELAY, - jvbStats.numEndpointsNoMessageTransportAfterDelay.get() - ); - unlockedSetStat("total_relays", jvbStats.totalRelays.get()); - unlockedSetStat( - "num_relays_no_msg_transport_after_delay", - jvbStats.numRelaysNoMessageTransportAfterDelay.get() - ); - unlockedSetStat("total_keyframes_received", jvbStats.keyframesReceived.get()); - unlockedSetStat("total_layering_changes_received", jvbStats.layeringChangesReceived.get()); - unlockedSetStat( - "total_video_stream_milliseconds_received", - jvbStats.totalVideoStreamMillisecondsReceived.get()); - unlockedSetStat( - "stress_level", - jvbStats.stressLevel - ); - unlockedSetStat( - "average_participant_stress", - JvbLoadManager.Companion.getAverageParticipantStress() - ); - unlockedSetStat("num_eps_oversending", numOversending); - unlockedSetStat(CONFERENCES, jvbStats.currentConferences.get()); - unlockedSetStat(OCTO_CONFERENCES, octoConferences); - unlockedSetStat(INACTIVE_CONFERENCES, inactiveConferences); - unlockedSetStat(P2P_CONFERENCES, p2pConferences); - unlockedSetStat("endpoints", endpoints); - unlockedSetStat("visitors", jvbStats.currentVisitors.get()); - unlockedSetStat(PARTICIPANTS, endpoints); - unlockedSetStat("local_endpoints", localEndpoints); - unlockedSetStat(RECEIVE_ONLY_ENDPOINTS, receiveOnlyEndpoints); - unlockedSetStat(INACTIVE_ENDPOINTS, inactiveEndpoints); - unlockedSetStat(OCTO_ENDPOINTS, octoEndpoints); - unlockedSetStat(ENDPOINTS_SENDING_AUDIO, numAudioSenders); - unlockedSetStat(ENDPOINTS_SENDING_VIDEO, numVideoSenders); - unlockedSetStat(VIDEO_CHANNELS, videoChannels); - unlockedSetStat(LARGEST_CONFERENCE, largestConferenceSize); - unlockedSetStat(CONFERENCE_SIZES, conferenceSizesJson); - unlockedSetStat(CONFERENCES_BY_AUDIO_SENDERS, audioSendersJson); - unlockedSetStat(CONFERENCES_BY_VIDEO_SENDERS, videoSendersJson); - unlockedSetStat(THREADS, threadCount); - unlockedSetStat( - SHUTDOWN_IN_PROGRESS, - videobridge.isInGracefulShutdown()); - if (videobridge.getShutdownState() == ShutdownState.SHUTTING_DOWN) - { - unlockedSetStat("shutting_down", true); - } - unlockedSetStat(DRAIN, videobridge.getDrainMode()); - unlockedSetStat(TOTAL_DATA_CHANNEL_MESSAGES_RECEIVED, - jvbStats.dataChannelMessagesReceived.get()); - unlockedSetStat(TOTAL_DATA_CHANNEL_MESSAGES_SENT, - jvbStats.dataChannelMessagesSent.get()); - unlockedSetStat(TOTAL_COLIBRI_WEB_SOCKET_MESSAGES_RECEIVED, - jvbStats.colibriWebSocketMessagesReceived.get()); - unlockedSetStat(TOTAL_COLIBRI_WEB_SOCKET_MESSAGES_SENT, - jvbStats.colibriWebSocketMessagesSent.get()); - unlockedSetStat( - TOTAL_BYTES_RECEIVED, jvbStats.totalBytesReceived.get()); - unlockedSetStat("dtls_failed_endpoints", jvbStats.endpointsDtlsFailed.get()); - unlockedSetStat(TOTAL_BYTES_SENT, jvbStats.totalBytesSent.get()); - unlockedSetStat( - TOTAL_PACKETS_RECEIVED, jvbStats.packetsReceived.get()); - unlockedSetStat(TOTAL_PACKETS_SENT, jvbStats.packetsSent.get()); - - unlockedSetStat("colibri2", true); - - unlockedSetStat(TOTAL_BYTES_RECEIVED_OCTO, jvbStats.totalRelayBytesReceived.get()); - unlockedSetStat(TOTAL_BYTES_SENT_OCTO, jvbStats.totalRelayBytesSent.get()); - unlockedSetStat(TOTAL_PACKETS_RECEIVED_OCTO, jvbStats.relayPacketsReceived.get()); - unlockedSetStat(TOTAL_PACKETS_SENT_OCTO, jvbStats.relayPacketsSent.get()); - unlockedSetStat(OCTO_RECEIVE_BITRATE, relayBitrateIncomingBps); - unlockedSetStat(OCTO_RECEIVE_PACKET_RATE, relayPacketRateIncoming); - unlockedSetStat(OCTO_SEND_BITRATE, relayBitrateOutgoingBps); - unlockedSetStat(OCTO_SEND_PACKET_RATE, relayPacketRateOutgoing); - unlockedSetStat(TOTAL_DOMINANT_SPEAKER_CHANGES, jvbStats.dominantSpeakerChanges.get()); - unlockedSetStat("endpoints_with_suspended_sources", endpointsWithSuspendedSources); - - unlockedSetStat(TIMESTAMP, timestampFormat.format(new Date())); - if (relayId != null) - { - unlockedSetStat(RELAY_ID, relayId); - } - - // TODO(brian): expose these stats in a `getStats` call in XmppConnection - // rather than calling xmppConnection.getMucClientManager? - unlockedSetStat( - MUC_CLIENTS_CONFIGURED, - xmppConnection.getMucClientManager().getClientCount()); - unlockedSetStat( - MUC_CLIENTS_CONNECTED, - xmppConnection.getMucClientManager().getClientConnectedCount()); - unlockedSetStat( - MUCS_CONFIGURED, - xmppConnection.getMucClientManager().getMucCount()); - unlockedSetStat( - MUCS_JOINED, - xmppConnection.getMucClientManager().getMucJoinedCount()); - unlockedSetStat("preemptive_kfr_sent", jvbStats.preemptiveKeyframeRequestsSent.get()); - unlockedSetStat("preemptive_kfr_suppressed", jvbStats.preemptiveKeyframeRequestsSuppressed.get()); - unlockedSetStat("endpoints_with_spurious_remb", RembHandler.Companion.endpointsWithSpuriousRemb()); - unlockedSetStat("healthy", - healthy.setAndGet(videobridge.getJvbHealthChecker().getResult().getSuccess())); - unlockedSetStat("endpoints_disconnected", EndpointConnectionStatusMonitor.endpointsDisconnected.get()); - unlockedSetStat("endpoints_reconnected", EndpointConnectionStatusMonitor.endpointsReconnected.get()); - } - finally - { - lock.unlock(); - } - } - - private static void updateBuckets(int[] buckets, int n) - { - int index = Math.min(n, buckets.length - 1); - buckets[index]++; - } -} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt index 40816f7eb0..cc78dcb859 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt @@ -329,8 +329,6 @@ abstract class AbstractEndpoint protected constructor( } interface EventHandler { - fun iceSucceeded() - fun iceFailed() fun sourcesChanged() } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index 512168c948..c1c3694a54 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -64,6 +64,7 @@ import org.jitsi.videobridge.message.BridgeChannelMessage import org.jitsi.videobridge.message.ForwardedSourcesMessage import org.jitsi.videobridge.message.ReceiverVideoConstraintsMessage import org.jitsi.videobridge.message.SenderSourceConstraintsMessage +import org.jitsi.videobridge.metrics.VideobridgeMetrics import org.jitsi.videobridge.relay.AudioSourceDesc import org.jitsi.videobridge.relay.RelayedEndpoint import org.jitsi.videobridge.rest.root.debug.EndpointDebugFeatures @@ -227,7 +228,6 @@ class Endpoint @JvmOverloads constructor( */ override val messageTransport = EndpointMessageTransport( this, - { conference.videobridge.statistics }, conference, logger ) @@ -317,9 +317,9 @@ class Endpoint @JvmOverloads constructor( setupIceTransport() setupDtlsTransport() - conference.videobridge.statistics.totalEndpoints.inc() + VideobridgeMetrics.totalEndpoints.inc() if (visitor) { - conference.videobridge.statistics.totalVisitors.inc() + VideobridgeMetrics.totalVisitors.inc() } logger.info("Created new endpoint, iceControlling=$iceControlling") @@ -370,7 +370,6 @@ class Endpoint @JvmOverloads constructor( iceTransport.eventHandler = object : IceTransport.EventHandler { override fun connected() { logger.info("ICE connected") - eventEmitter.fireEvent { iceSucceeded() } transceiver.setOutgoingPacketHandler(object : PacketHandler { override fun processPacket(packetInfo: PacketInfo) { packetInfo.addEvent(SRTP_QUEUE_ENTRY_EVENT) @@ -382,7 +381,6 @@ class Endpoint @JvmOverloads constructor( } override fun failed() { - eventEmitter.fireEvent { iceFailed() } } override fun consentUpdated(time: Instant) { @@ -670,7 +668,7 @@ class Endpoint @JvmOverloads constructor( if (!isExpired) { if (!messageTransport.isConnected) { logger.error("EndpointMessageTransport still not connected.") - conference.videobridge.statistics.numEndpointsNoMessageTransportAfterDelay.inc() + VideobridgeMetrics.numEndpointsNoMessageTransportAfterDelay.inc() } } }, @@ -998,51 +996,28 @@ class Endpoint @JvmOverloads constructor( * expires. */ private fun updateStatsOnExpire() { - val conferenceStats = conference.statistics val transceiverStats = transceiver.getTransceiverStats() - conferenceStats.apply { - val incomingStats = transceiverStats.rtpReceiverStats.packetStreamStats - val outgoingStats = transceiverStats.outgoingPacketStreamStats - totalBytesReceived.addAndGet(incomingStats.bytes) - totalPacketsReceived.addAndGet(incomingStats.packets) - totalBytesSent.addAndGet(outgoingStats.bytes) - totalPacketsSent.addAndGet(outgoingStats.packets) - } - - conference.videobridge.statistics.apply { - val bweStats = transceiverStats.bandwidthEstimatorStats - bweStats.getNumber("incomingEstimateExpirations")?.toLong()?.let { - incomingBitrateExpirations.addAndGet(it) - } - keyframesReceived.addAndGet(transceiverStats.rtpReceiverStats.videoParserStats.numKeyframes.toLong()) - layeringChangesReceived.addAndGet( - transceiverStats.rtpReceiverStats.videoParserStats.numLayeringChanges.toLong() - ) - - val durationActiveVideo = transceiverStats.rtpReceiverStats.incomingStats.ssrcStats.values.filter { - it.mediaType == MediaType.VIDEO - }.sumOf { it.durationActive } - totalVideoStreamMillisecondsReceived.addAndGet(durationActiveVideo.toMillis()) - } - - run { - val bweStats = transceiverStats.bandwidthEstimatorStats - val lossLimitedMs = bweStats.getNumber("lossLimitedMs")?.toLong() ?: return@run - val lossDegradedMs = bweStats.getNumber("lossDegradedMs")?.toLong() ?: return@run - val lossFreeMs = bweStats.getNumber("lossFreeMs")?.toLong() ?: return@run - - val participantMs = lossFreeMs + lossDegradedMs + lossLimitedMs - conference.videobridge.statistics.apply { - totalLossControlledParticipantMs.addAndGet(participantMs) - totalLossLimitedParticipantMs.addAndGet(lossLimitedMs) - totalLossDegradedParticipantMs.addAndGet(lossDegradedMs) - } - } + val incomingStats = transceiverStats.rtpReceiverStats.packetStreamStats + val outgoingStats = transceiverStats.outgoingPacketStreamStats + VideobridgeMetrics.totalBytesReceived.add(incomingStats.bytes) + VideobridgeMetrics.totalBytesSent.add(outgoingStats.bytes) + VideobridgeMetrics.packetsReceived.addAndGet(incomingStats.packets) + VideobridgeMetrics.packetsSent.addAndGet(outgoingStats.packets) + VideobridgeMetrics.keyframesReceived.addAndGet( + transceiverStats.rtpReceiverStats.videoParserStats.numKeyframes.toLong() + ) + VideobridgeMetrics.layeringChangesReceived.addAndGet( + transceiverStats.rtpReceiverStats.videoParserStats.numLayeringChanges.toLong() + ) + val durationActiveVideo = transceiverStats.rtpReceiverStats.incomingStats.ssrcStats.values.filter { + it.mediaType == MediaType.VIDEO + }.sumOf { it.durationActive } + VideobridgeMetrics.totalVideoStreamMillisecondsReceived.add(durationActiveVideo.toMillis()) if (iceTransport.isConnected() && !dtlsTransport.isConnected) { logger.info("Expiring an endpoint with ICE connected, but not DTLS.") - conferenceStats.dtlsFailedEndpoints.incrementAndGet() + VideobridgeMetrics.endpointsDtlsFailed.inc() } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt index ce0ccd0c27..7df9dd6cff 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt @@ -33,10 +33,10 @@ import org.jitsi.shutdown.ShutdownServiceImpl import org.jitsi.utils.logging2.LoggerImpl import org.jitsi.utils.queue.PacketQueue import org.jitsi.videobridge.ice.Harvesters +import org.jitsi.videobridge.metrics.Metrics +import org.jitsi.videobridge.metrics.VideobridgePeriodicMetrics import org.jitsi.videobridge.rest.root.Application -import org.jitsi.videobridge.stats.MucStatsTransport -import org.jitsi.videobridge.stats.StatsCollector -import org.jitsi.videobridge.stats.VideobridgeStatistics +import org.jitsi.videobridge.stats.MucPublisher import org.jitsi.videobridge.util.TaskPools import org.jitsi.videobridge.version.JvbVersionService import org.jitsi.videobridge.websocket.ColibriWebSocketService @@ -67,13 +67,13 @@ fun main() { // to be passed. System.setProperty("org.eclipse.jetty.util.log.class", "org.eclipse.jetty.util.log.JavaUtilLog") + Metrics.start() + // Reload the Typesafe config used by ice4j, because the original was initialized before the new system // properties were set. JitsiConfig.reloadNewConfig() - val versionService = JvbVersionService().also { - logger.info("Starting jitsi-videobridge version ${it.currentVersion}") - } + logger.info("Starting jitsi-videobridge version ${JvbVersionService.instance.currentVersion}") startIce4j() @@ -98,15 +98,19 @@ fun main() { val videobridge = Videobridge( xmppConnection, shutdownService, - versionService.currentVersion, + JvbVersionService.instance.currentVersion, VersionConfig.config.release, Clock.systemUTC() ).apply { start() } - val healthChecker = videobridge.jvbHealthChecker - val statsCollector = StatsCollector(VideobridgeStatistics(videobridge, xmppConnection)).apply { - start() - addTransport(MucStatsTransport(xmppConnection), XmppClientConnectionConfig.config.presenceInterval.toMillis()) + Metrics.metricsUpdater.addUpdateTask { + VideobridgePeriodicMetrics.update(videobridge) } + val healthChecker = videobridge.jvbHealthChecker + val presencePublisher = MucPublisher( + TaskPools.SCHEDULED_POOL, + XmppClientConnectionConfig.config.presenceInterval, + xmppConnection + ).apply { start() } val publicServerConfig = JettyBundleActivatorConfig( "org.jitsi.videobridge.rest", @@ -135,8 +139,7 @@ fun main() { val restApp = Application( videobridge, xmppConnection, - statsCollector, - versionService.currentVersion, + JvbVersionService.instance.currentVersion, healthChecker ) createServer(privateServerConfig).also { @@ -157,8 +160,8 @@ fun main() { logger.info("Bridge shutting down") healthChecker.stop() + presencePublisher.stop() xmppConnection.stop() - statsCollector.stop() try { publicHttpServer?.stop() @@ -168,6 +171,7 @@ fun main() { } videobridge.stop() stopIce4j() + Metrics.stop() TaskPools.SCHEDULED_POOL.shutdownNow() TaskPools.CPU_POOL.shutdownNow() diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/health/JvbHealthChecker.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/health/JvbHealthChecker.kt index e02a8738bb..8ead98af33 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/health/JvbHealthChecker.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/health/JvbHealthChecker.kt @@ -22,6 +22,7 @@ import org.jitsi.health.HealthChecker import org.jitsi.health.Result import org.jitsi.videobridge.health.config.HealthConfig.Companion.config import org.jitsi.videobridge.ice.Harvesters +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer import java.net.InetAddress class JvbHealthChecker : HealthCheckService { @@ -30,12 +31,16 @@ class JvbHealthChecker : HealthCheckService { config.timeout, config.maxCheckDuration, config.stickyFailures, - healthCheckFunc = ::check + healthCheckFunc = ::checkAndUpdateMetric ) fun start() = healthChecker.start() fun stop() = healthChecker.stop() + private fun checkAndUpdateMetric(): Result = check().also { + healthyMetric.set(it.success) + } + private fun check(): Result { if (config.requireValidAddress && !hasValidAddress()) { return Result(success = false, message = "No valid IP addresses available for harvesting.") @@ -68,4 +73,12 @@ class JvbHealthChecker : HealthCheckService { override val result: Result get() = healthChecker.result + + companion object { + val healthyMetric = VideobridgeMetricsContainer.instance.registerBooleanMetric( + "healthy", + "Whether the Videobridge instance is healthy or not.", + true + ) + } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadManager.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadManager.kt index fb8d542a94..912cd10be1 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadManager.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadManager.kt @@ -25,6 +25,7 @@ import org.jitsi.utils.logging2.cdebug import org.jitsi.utils.logging2.createLogger import org.jitsi.videobridge.Videobridge import org.jitsi.videobridge.jvbLastNSingleton +import org.jitsi.videobridge.metrics.VideobridgeMetrics import org.jitsi.videobridge.util.TaskPools import java.time.Clock import java.time.Duration @@ -175,7 +176,7 @@ class PacketRateLoadManager( init { val sampler = PacketRateLoadSampler(videobridge) { loadMeasurement -> loadUpdate(loadMeasurement) - videobridge.statistics.stressLevel = getCurrentStressLevel() + VideobridgeMetrics.stressLevel.set(getCurrentStressLevel()) } startSampler(sampler) @@ -191,7 +192,7 @@ class CpuUsageLoadManager( init { val sampler = CpuLoadSampler { loadMeasurement -> loadUpdate(loadMeasurement) - videobridge.statistics.stressLevel = getCurrentStressLevel() + VideobridgeMetrics.stressLevel.set(getCurrentStressLevel()) } startSampler(sampler) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/Metrics.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/Metrics.kt new file mode 100644 index 0000000000..61944800b3 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/Metrics.kt @@ -0,0 +1,48 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.metrics + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metrics.MetricsUpdater +import org.jitsi.utils.concurrent.CustomizableThreadFactory +import java.time.Duration +import java.util.concurrent.Executors + +object Metrics { + val interval: Duration by config { + "videobridge.stats.interval".from(JitsiConfig.newConfig) + } + + /** Updating the metrics shouldn't block anywhere, but use a separate executor just in case. */ + private val executor = Executors.newSingleThreadScheduledExecutor( + CustomizableThreadFactory("MetricsUpdater-scheduled", false) + ) + val metricsUpdater = MetricsUpdater(executor, interval) + + /** + * The lock which is used when metrics are updated or queried. The [MetricsUpdater] internally uses itself as the + * lock, so we reuse it here. + */ + val lock: Any + get() = metricsUpdater + + fun start() = metricsUpdater.addUpdateTask { ThreadsMetric.update() } + fun stop() { + metricsUpdater.stop() + executor.shutdown() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/ThreadsMetric.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/ThreadsMetric.kt new file mode 100644 index 0000000000..978d290bbd --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/ThreadsMetric.kt @@ -0,0 +1,29 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.metrics + +import java.lang.management.ManagementFactory + +object ThreadsMetric { + fun update() { + threadCount.set(ManagementFactory.getThreadMXBean().threadCount.toLong()) + } + + val threadCount = VideobridgeMetricsContainer.instance.registerLongGauge( + "thread_count", + "Current number of JVM threads." + ) +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetrics.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetrics.kt new file mode 100644 index 0000000000..e16ace9c2f --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetrics.kt @@ -0,0 +1,253 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.metrics + +import org.jitsi.metrics.CounterMetric +import org.jitsi.videobridge.VideobridgeConfig +import org.jitsi.videobridge.relay.RelayConfig +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer.Companion.instance as metricsContainer + +object VideobridgeMetrics { + val gracefulShutdown = metricsContainer.registerBooleanMetric( + "graceful_shutdown", + "Whether the bridge is in graceful shutdown mode (not accepting new conferences)." + ) + val shuttingDown = metricsContainer.registerBooleanMetric( + "shutting_down", + "Whether the bridge is shutting down." + ) + val drainMode = VideobridgeMetricsContainer.instance.registerBooleanMetric( + "drain_mode", + "Whether the bridge is in drain shutdown mode.", + VideobridgeConfig.initialDrainMode + ) + + @JvmField + val conferencesCompleted = metricsContainer.registerCounter( + "conferences_completed", + "The total number of conferences completed/expired on the Videobridge." + ) + + @JvmField + val conferencesCreated = metricsContainer.registerCounter( + "conferences_created", + "The total number of conferences created on the Videobridge." + ) + + @JvmField + val dataChannelMessagesReceived = metricsContainer.registerCounter( + "data_channel_messages_received", + "Number of messages received from the data channels of the endpoints of this conference." + ) + + @JvmField + val dataChannelMessagesSent = metricsContainer.registerCounter( + "data_channel_messages_sent", + "Number of messages sent via the data channels of the endpoints of this conference." + ) + + @JvmField + val colibriWebSocketMessagesReceived: CounterMetric = metricsContainer.registerCounter( + "colibri_web_socket_messages_received", + "Number of messages received from the data channels of the endpoints of this conference." + ) + + @JvmField + val colibriWebSocketMessagesSent = metricsContainer.registerCounter( + "colibri_web_socket_messages_sent", + "Number of messages sent via the data channels of the endpoints of this conference." + ) + + @JvmField + val packetsReceived = metricsContainer.registerCounter( + "packets_received", + "Number of RTP packets received in conferences on this videobridge." + ) + + @JvmField + val packetsSent = metricsContainer.registerCounter( + "packets_sent", + "Number of RTP packets sent in conferences on this videobridge." + ) + + @JvmField + val relayPacketsReceived = metricsContainer.registerCounter( + "relay_packets_received", + "Number of RTP packets received by relays in conferences on this videobridge." + ) + + @JvmField + val relayPacketsSent = metricsContainer.registerCounter( + "relay_packets_sent", + "Number of RTP packets sent by relays in conferences on this videobridge." + ) + + @JvmField + val totalEndpoints = metricsContainer.registerCounter( + "endpoints", + "The total number of endpoints created." + ) + + @JvmField + val totalVisitors = metricsContainer.registerCounter( + "visitors", + "The total number of visitor endpoints created." + ) + + @JvmField + val numEndpointsNoMessageTransportAfterDelay = metricsContainer.registerCounter( + "endpoints_no_message_transport_after_delay", + "Number of endpoints which had not established a relay message transport even after some delay." + ) + + @JvmField + val totalRelays = metricsContainer.registerCounter( + "relays", + "The total number of relays created." + ) + + @JvmField + val numRelaysNoMessageTransportAfterDelay = metricsContainer.registerCounter( + "relays_no_message_transport_after_delay", + "Number of relays which had not established a relay message transport even after some delay." + ) + + @JvmField + val dominantSpeakerChanges = metricsContainer.registerCounter( + "dominant_speaker_changes", + "Number of times the dominant speaker in any conference changed." + ) + + @JvmField + val endpointsDtlsFailed = metricsContainer.registerCounter( + "endpoints_dtls_failed", + "Number of endpoints whose ICE connection was established, but DTLS wasn't (at time of expiration)." + ) + + @JvmField + val stressLevel = metricsContainer.registerDoubleGauge( + "stress", + "Current stress (between 0 and 1)." + ) + + @JvmField + val preemptiveKeyframeRequestsSent = metricsContainer.registerCounter( + "preemptive_keyframe_requests_sent", + "Number of preemptive keyframe requests that were sent." + ) + + @JvmField + val preemptiveKeyframeRequestsSuppressed = metricsContainer.registerCounter( + "preemptive_keyframe_requests_suppressed", + "Number of preemptive keyframe requests that were not sent because no endpoints were in stage view." + ) + + @JvmField + val keyframesReceived = metricsContainer.registerCounter( + "keyframes_received", + "Number of keyframes that were received (updated on endpoint expiration)." + ) + + @JvmField + val layeringChangesReceived = metricsContainer.registerCounter( + "layering_changes_received", + "Number of times the layering of an incoming video stream changed (updated on endpoint expiration)." + ) + + @JvmField + val currentLocalEndpoints = metricsContainer.registerLongGauge( + "local_endpoints", + "Number of local endpoints that exist currently." + ) + + @JvmField + val currentVisitors = metricsContainer.registerLongGauge( + "current_visitors", + "Number of visitor endpoints." + ) + + @JvmField + val currentConferences = metricsContainer.registerLongGauge( + "conferences", + "Current number of conferences." + ) + + @JvmField + val totalConferenceSeconds = metricsContainer.registerCounter( + "conference_seconds", + "The total duration in seconds of all completed conferences." + ) + + @JvmField + val totalBytesReceived = metricsContainer.registerCounter( + "bytes_received", + "The total number of bytes received in RTP packets." + ) + + @JvmField + val totalBytesSent = metricsContainer.registerCounter( + "bytes_sent", + "The total number of bytes sent in RTP packets." + ) + + @JvmField + val totalRelayBytesReceived = metricsContainer.registerCounter( + "relay_bytes_received", + "The total number of bytes received by relays in RTP packets." + ) + + @JvmField + val totalRelayBytesSent = metricsContainer.registerCounter( + "relay_bytes_sent", + "The total number of bytes sent to relays in RTP packets." + ) + + /** + * The total duration, in milliseconds, of video streams (SSRCs) that were received. For example, if an + * endpoint sends simulcast with 3 SSRCs for 1 minute it would contribute a total of 3 minutes. Suspended + * streams do not contribute to this duration. + * + * This is updated on endpoint expiration. + */ + @JvmField + val totalVideoStreamMillisecondsReceived = metricsContainer.registerCounter( + "video_milliseconds_received", + "Total duration of video received, in milliseconds (each SSRC counts separately)." + ) + + private val tossedPacketsEnergyBuckets = + listOf(0, 7, 15, 23, 31, 39, 47, 55, 63, 71, 79, 87, 95, 103, 111, 119, 127).map { it.toDouble() } + .toDoubleArray() + + @JvmField + val tossedPacketsEnergy = metricsContainer.registerHistogram( + "tossed_packets_energy", + "Distribution of energy scores for discarded audio packets.", + *tossedPacketsEnergyBuckets + ) + + /** The currently configured region, if any. */ + val regionInfo = if (RelayConfig.config.region != null) { + metricsContainer.registerInfo( + ColibriStatsExtension.REGION, + "The currently configured region.", + RelayConfig.config.region!! + ) + } else { + null + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetricsContainer.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetricsContainer.kt index d7e58f9d42..e13c0b1f96 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetricsContainer.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetricsContainer.kt @@ -18,11 +18,23 @@ package org.jitsi.videobridge.metrics import org.jitsi.metrics.MetricsContainer /** - * `VideobridgeMetricsContainer` gathers and exports metrics - * from a [Videobridge][org.jitsi.videobridge.Videobridge] instance. + * The [MetricsContainer] instance for jitsi-videobridge where all Prometheus metrics are registered. */ class VideobridgeMetricsContainer private constructor() : MetricsContainer(namespace = "jitsi_jvb") { + /** Acquire Metrics.lock to make sure we don't race with the updater. */ + override fun getPrometheusMetrics(contentType: String): String { + synchronized(Metrics.lock) { + return super.getPrometheusMetrics(contentType) + } + } + + /** Acquire Metrics.lock to make sure we don't race with the updater. */ + override val jsonString: String + get() = synchronized(Metrics.lock) { + super.jsonString + } + companion object { /** * The singleton instance of `MetricsContainer`. diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgePeriodicMetrics.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgePeriodicMetrics.kt new file mode 100644 index 0000000000..a3243e064a --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgePeriodicMetrics.kt @@ -0,0 +1,362 @@ +/* + * Copyright @ 2024 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.metrics + +import org.jitsi.nlj.rtcp.RembHandler.Companion.endpointsWithSpuriousRemb +import org.jitsi.videobridge.Videobridge +import org.jitsi.videobridge.stats.ConferencePacketStats +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer.Companion.instance as metricsContainer + +/** + * Holds gauge metrics that need to be updated periodically. + */ +object VideobridgePeriodicMetrics { + val incomingLoss = metricsContainer.registerDoubleGauge( + "incoming_loss_fraction", + "Fraction of incoming RTP packets that are lost." + ) + val outgoingLoss = metricsContainer.registerDoubleGauge( + "outgoing_loss_fraction", + "Fraction of outgoing RTP packets that are lost." + ) + val loss = metricsContainer.registerDoubleGauge( + "loss_fraction", + "Fraction of RTP packets that are lost (incoming and outgoing combined)." + ) + + val relayIncomingBitrate = metricsContainer.registerLongGauge( + "relay_incoming_bitrate", + "Incoming RTP/RTCP bitrate from relays in bps." + ) + val relayOutgoingBitrate = metricsContainer.registerLongGauge( + "relay_outgoing_bitrate", + "Outgoing RTP/RTCP bitrate to relays in bps." + ) + val relayIncomingPacketRate = metricsContainer.registerLongGauge( + "relay_incoming_packet_rate", + "Incoming RTP/RTCP packet rate from relays in pps." + ) + val relayOutgoingPacketRate = metricsContainer.registerLongGauge( + "relay_outgoing_packet_rate", + "Outgoing RTP/RTCP packet rate to relays in pps." + ) + val incomingBitrate = metricsContainer.registerLongGauge( + "incoming_bitrate", + "Incoming RTP/RTCP bitrate in bps." + ) + val outgoingBitrate = metricsContainer.registerLongGauge( + "outgoing_bitrate", + "Outgoing RTP/RTCP bitrate in bps." + ) + val incomingPacketRate = metricsContainer.registerLongGauge( + "incoming_packet_rate", + "Incoming RTP/RTCP packet rate in pps." + ) + val outgoingPacketRate = metricsContainer.registerLongGauge( + "outgoing_packet_rate", + "Outgoing RTP/RTCP packet rate in pps." + ) + + val averageRtt = metricsContainer.registerDoubleGauge( + "average_rtt", + "Average RTT across all local endpoints in ms." + ) + + val largestConference = metricsContainer.registerLongGauge( + "largest_conference", + "The size of the largest conference (number of endpoints)." + ) + + val endpoints = metricsContainer.registerLongGauge( + "current_endpoints", + "Number of current endpoints (local and relayed)." + ) + val endpointsWithHighOutgoingLoss = metricsContainer.registerLongGauge( + "endpoints_with_high_outgoing_loss", + "Number of endpoints that have high outgoing loss (>10%)." + ) + val activeEndpoints = metricsContainer.registerLongGauge( + "active_endpoints", + "The number of active local endpoints (in a conference where at least one endpoint sends audio or video)." + ) + val endpointsSendingAudio = metricsContainer.registerLongGauge( + "endpoints_sending_audio", + "The number of local endpoints sending audio." + ) + val endpointsSendingVideo = metricsContainer.registerLongGauge( + "endpoints_sending_video", + "The number of local endpoints sending video." + ) + val endpointsRelayed = metricsContainer.registerLongGauge( + "endpoints_relayed", + "Number of relayed endpoints." + ) + val endpointsOversending = metricsContainer.registerLongGauge( + "endpoints_oversending", + "Number of endpoints that we are oversending to." + ) + val endpointsReceiveOnly = metricsContainer.registerLongGauge( + "endpoints_recvonly", + "Number of endpoints that are not sending audio or video (but are receiveing)." + ) + val endpointsInactive = metricsContainer.registerLongGauge( + "endpoints_inactive", + "Number of endpoints in inactive conferences (where no endpoint sends audio or video)." + ) + val endpointsWithSpuriousRemb = metricsContainer.registerLongGauge( + "endpoints_with_spurious_remb", + "Number of endpoints that have send a REMB packet even though REMB wasn't configured." + ) + val endpointsWithSuspendedSources = metricsContainer.registerLongGauge( + "endpoints_with_suspended_sources", + "Number of endpoints that that we have suspended sending some video streams to because of bwe." + ) + + val conferencesInactive = metricsContainer.registerLongGauge( + "conferences_inactive", + "Number of inactive conferences (no endpoint is sending audio or video)." + ) + val conferencesP2p = metricsContainer.registerLongGauge( + "conferences_p2p", + "Number of p2p conferences (inactive with 2 endpoints)." + ) + val conferencesWithRelay = metricsContainer.registerLongGauge( + "conferences_with_relay", + "Number of conferences with one or more relays." + ) + + private val conferenceSizeBuckets = (0..20).map { it.toDouble() }.toList().toDoubleArray() + val conferencesBySize = metricsContainer.registerHistogram( + "conferences_by_size", + "Histogram of conferences by total number of endpoints.", + *conferenceSizeBuckets + ) + val conferencesByAudioSender = metricsContainer.registerHistogram( + "conferences_by_audio_sender", + "Histogram of conferences by number of local endpoints sending audio.", + *conferenceSizeBuckets + ) + val conferencesByVideoSender = metricsContainer.registerHistogram( + "conferences_by_video_sender", + "Histogram of conferences by number of local endpoints sending video.", + *conferenceSizeBuckets + ) + + fun update(videobridge: Videobridge) { + var endpoints = 0L + var localEndpoints = 0L + var octoEndpoints = 0L + + var octoConferences = 0L + + var bitrateDownloadBps = 0.0 + var bitrateUploadBps = 0.0 + var packetRateUpload: Long = 0 + var packetRateDownload: Long = 0 + + var relayBitrateIncomingBps = 0.0 + var relayBitrateOutgoingBps = 0.0 + var relayPacketRateOutgoing: Long = 0 + var relayPacketRateIncoming: Long = 0 + + // Packets we received + var incomingPacketsReceived: Long = 0 + // Packets we should have received but were lost + var incomingPacketsLost: Long = 0 + // Packets we sent that were reported received + var outgoingPacketsReceived: Long = 0 + // Packets we sent that were reported lost + var outgoingPacketsLost: Long = 0 + + var rttSumMs = 0.0 + var rttCount: Long = 0 + var largestConferenceSize = 0L + var inactiveConferences = 0L + var p2pConferences = 0L + var inactiveEndpoints = 0L + var receiveOnlyEndpoints = 0L + var numAudioSenders = 0L + var numVideoSenders = 0L + // The number of endpoints to which we're "oversending" (which can occur when + // enableOnstageVideoSuspend is false) + var numOversending = 0L + var endpointsWithHighOutgoingLoss = 0L + var numLocalActiveEndpoints = 0L + var endpointsWithSuspendedSources = 0L + + val conferences = videobridge.conferences + val conferenceSizesList = ArrayList(conferences.size) + val audioSendersList = ArrayList(conferences.size) + val videoSendersList = ArrayList(conferences.size) + + for (conference in conferences) { + var conferenceBitrate: Long = 0 + var conferencePacketRate: Long = 0 + if (conference.isP2p) { + p2pConferences++ + } + val inactive = conference.isInactive + if (inactive) { + inactiveConferences++ + inactiveEndpoints += conference.endpointCount + } else { + numLocalActiveEndpoints += conference.localEndpointCount + } + if (conference.hasRelays()) { + octoConferences++ + } + val numConferenceEndpoints = conference.endpointCount.toLong() + val numLocalEndpoints = conference.localEndpointCount + localEndpoints += numLocalEndpoints + if (numConferenceEndpoints > largestConferenceSize) { + largestConferenceSize = numConferenceEndpoints + } + conferenceSizesList.add(numConferenceEndpoints) + endpoints += numConferenceEndpoints + octoEndpoints += numConferenceEndpoints - numLocalEndpoints + var conferenceAudioSenders = 0L + var conferenceVideoSenders = 0L + for (endpoint in conference.localEndpoints) { + if (endpoint.isOversending()) { + numOversending++ + } + val sendingAudio = endpoint.isSendingAudio + val sendingVideo = endpoint.isSendingVideo + if (sendingAudio) { + conferenceAudioSenders++ + } + if (sendingVideo) { + conferenceVideoSenders++ + } + if (!sendingAudio && !sendingVideo && !inactive) { + receiveOnlyEndpoints++ + } + if (endpoint.hasSuspendedSources()) { + endpointsWithSuspendedSources++ + } + val (endpointConnectionStats, rtpReceiverStats, _, outgoingStats) = + endpoint.transceiver.getTransceiverStats() + val incomingStats = rtpReceiverStats.incomingStats + val incomingPacketStreamStats = rtpReceiverStats.packetStreamStats + bitrateDownloadBps += incomingPacketStreamStats.getBitrateBps() + packetRateDownload += incomingPacketStreamStats.packetRate + conferenceBitrate = (conferenceBitrate + incomingPacketStreamStats.getBitrateBps()).toLong() + conferencePacketRate += incomingPacketStreamStats.packetRate + bitrateUploadBps += outgoingStats.getBitrateBps() + packetRateUpload += outgoingStats.packetRate + conferenceBitrate = (conferenceBitrate + outgoingStats.getBitrateBps()).toLong() + conferencePacketRate += outgoingStats.packetRate + val endpointRtt = endpointConnectionStats.rtt + if (endpointRtt > 0) { + rttSumMs += endpointRtt + rttCount++ + } + incomingPacketsReceived += endpointConnectionStats.incomingLossStats.packetsReceived + incomingPacketsLost += endpointConnectionStats.incomingLossStats.packetsLost + val endpointOutgoingPacketsReceived = endpointConnectionStats.outgoingLossStats.packetsReceived + val endpointOutgoingPacketsLost = endpointConnectionStats.outgoingLossStats.packetsLost + outgoingPacketsReceived += endpointOutgoingPacketsReceived + outgoingPacketsLost += endpointOutgoingPacketsLost + if (!inactive && endpointOutgoingPacketsLost + endpointOutgoingPacketsReceived > 0) { + val endpointOutgoingFractionLost = ( + endpointOutgoingPacketsLost.toDouble() / + (endpointOutgoingPacketsLost + endpointOutgoingPacketsReceived) + ) + if (endpointOutgoingFractionLost > 0.1) { + endpointsWithHighOutgoingLoss++ + } + } + } + for (relay in conference.relays) { + relayBitrateIncomingBps += relay.incomingBitrateBps + relayPacketRateIncoming += relay.incomingPacketRate + conferenceBitrate = (conferenceBitrate + relay.incomingBitrateBps).toLong() + conferencePacketRate += relay.incomingPacketRate + relayBitrateOutgoingBps += relay.outgoingBitrateBps + relayPacketRateOutgoing += relay.outgoingPacketRate + conferenceBitrate = (conferenceBitrate + relay.outgoingBitrateBps).toLong() + conferencePacketRate += relay.outgoingPacketRate + + /* TODO: report Relay RTT and loss, like we do for Endpoints? */ + } + audioSendersList.add(conferenceAudioSenders) + numAudioSenders += conferenceAudioSenders + videoSendersList.add(conferenceVideoSenders) + numVideoSenders += conferenceVideoSenders + ConferencePacketStats.stats.addValue( + numConferenceEndpoints.toInt(), + conferencePacketRate, + conferenceBitrate + ) + } + + // RTT_AGGREGATE + val rttAggregate: Double = if (rttCount > 0) rttSumMs / rttCount else 0.0 + + var incomingLoss = 0.0 + if (incomingPacketsReceived + incomingPacketsLost > 0) { + incomingLoss = incomingPacketsLost.toDouble() / (incomingPacketsReceived + incomingPacketsLost) + } + var outgoingLoss = 0.0 + if (outgoingPacketsReceived + outgoingPacketsLost > 0) { + outgoingLoss = outgoingPacketsLost.toDouble() / (outgoingPacketsReceived + outgoingPacketsLost) + } + var overallLoss = 0.0 + if (incomingPacketsReceived + incomingPacketsLost + outgoingPacketsReceived + outgoingPacketsLost > 0) { + overallLoss = ( + (outgoingPacketsLost + incomingPacketsLost).toDouble() / + (incomingPacketsReceived + incomingPacketsLost + outgoingPacketsReceived + outgoingPacketsLost) + ) + } + + VideobridgePeriodicMetrics.incomingLoss.set(incomingLoss) + VideobridgePeriodicMetrics.outgoingLoss.set(outgoingLoss) + VideobridgePeriodicMetrics.loss.set(overallLoss) + VideobridgePeriodicMetrics.endpoints.set(endpoints) + VideobridgePeriodicMetrics.endpointsWithHighOutgoingLoss.set(endpointsWithHighOutgoingLoss) + VideobridgePeriodicMetrics.endpointsWithSpuriousRemb.set(endpointsWithSpuriousRemb().toLong()) + VideobridgePeriodicMetrics.activeEndpoints.set(numLocalActiveEndpoints) + VideobridgePeriodicMetrics.incomingBitrate.set(bitrateDownloadBps.toLong()) + VideobridgePeriodicMetrics.outgoingBitrate.set(bitrateUploadBps.toLong()) + VideobridgePeriodicMetrics.incomingPacketRate.set(packetRateDownload) + VideobridgePeriodicMetrics.outgoingPacketRate.set(packetRateUpload) + VideobridgePeriodicMetrics.averageRtt.set(rttAggregate) + VideobridgePeriodicMetrics.largestConference.set(largestConferenceSize) + VideobridgePeriodicMetrics.endpointsSendingAudio.set(numAudioSenders) + VideobridgePeriodicMetrics.endpointsSendingVideo.set(numVideoSenders) + VideobridgePeriodicMetrics.endpointsRelayed.set(octoEndpoints) + VideobridgePeriodicMetrics.endpointsOversending.set(numOversending) + VideobridgePeriodicMetrics.endpointsReceiveOnly.set(receiveOnlyEndpoints) + VideobridgePeriodicMetrics.endpointsInactive.set(inactiveEndpoints) + VideobridgePeriodicMetrics.endpointsWithSuspendedSources.set(endpointsWithSuspendedSources) + VideobridgePeriodicMetrics.conferencesInactive.set(inactiveConferences) + VideobridgePeriodicMetrics.conferencesP2p.set(p2pConferences) + VideobridgePeriodicMetrics.conferencesWithRelay.set(octoConferences) + VideobridgePeriodicMetrics.relayIncomingBitrate.set(relayBitrateIncomingBps.toLong()) + VideobridgePeriodicMetrics.relayOutgoingBitrate.set(relayBitrateOutgoingBps.toLong()) + VideobridgePeriodicMetrics.relayIncomingPacketRate.set(relayPacketRateIncoming) + VideobridgePeriodicMetrics.relayOutgoingPacketRate.set(relayPacketRateOutgoing) + + conferencesBySize.histogram.clear() + conferenceSizesList.forEach { conferencesBySize.histogram.observe(it.toDouble()) } + + conferencesByAudioSender.histogram.clear() + audioSendersList.forEach { conferencesByAudioSender.histogram.observe(it.toDouble()) } + + conferencesByVideoSender.histogram.clear() + videoSendersList.forEach { conferencesByVideoSender.histogram.observe(it.toDouble()) } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt index 9e4ba6dd11..1b69be4e31 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt @@ -76,6 +76,7 @@ import org.jitsi.videobridge.datachannel.protocol.DataChannelPacket import org.jitsi.videobridge.datachannel.protocol.DataChannelProtocolConstants import org.jitsi.videobridge.message.BridgeChannelMessage import org.jitsi.videobridge.message.SourceVideoTypeMessage +import org.jitsi.videobridge.metrics.VideobridgeMetrics import org.jitsi.videobridge.rest.root.debug.EndpointDebugFeatures import org.jitsi.videobridge.sctp.DataChannelHandler import org.jitsi.videobridge.sctp.SctpHandler @@ -103,7 +104,6 @@ import java.util.* import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicLong -import java.util.function.Supplier import kotlin.collections.ArrayList import kotlin.collections.HashMap import kotlin.collections.HashSet @@ -263,7 +263,6 @@ class Relay @JvmOverloads constructor( */ private val messageTransport = RelayMessageTransport( this, - Supplier { conference.videobridge.statistics }, conference, logger ) @@ -273,7 +272,7 @@ class Relay @JvmOverloads constructor( setupIceTransport() setupDtlsTransport() - conference.videobridge.statistics.totalRelays.inc() + VideobridgeMetrics.totalRelays.inc() } fun getMessageTransport(): RelayMessageTransport = messageTransport @@ -362,7 +361,6 @@ class Relay @JvmOverloads constructor( iceTransport.eventHandler = object : IceTransport.EventHandler { override fun connected() { logger.info("ICE connected") - eventEmitter.fireEvent { iceSucceeded() } transceiver.setOutgoingPacketHandler(object : PacketHandler { override fun processPacket(packetInfo: PacketInfo) { packetInfo.addEvent(SRTP_QUEUE_ENTRY_EVENT) @@ -373,9 +371,7 @@ class Relay @JvmOverloads constructor( TaskPools.IO_POOL.execute(dtlsTransport::startDtlsHandshake) } - override fun failed() { - eventEmitter.fireEvent { iceFailed() } - } + override fun failed() {} override fun consentUpdated(time: Instant) { transceiver.packetIOActivity.lastIceActivityInstant = time @@ -911,7 +907,7 @@ class Relay @JvmOverloads constructor( if (!expired) { if (!messageTransport.isConnected) { logger.error("RelayMessageTransport still not connected.") - conference.videobridge.statistics.numRelaysNoMessageTransportAfterDelay.inc() + VideobridgeMetrics.numRelaysNoMessageTransportAfterDelay.inc() } } }, @@ -1057,7 +1053,6 @@ class Relay @JvmOverloads constructor( * expires. */ private fun updateStatsOnExpire() { - val conferenceStats = conference.statistics val transceiverStats = transceiver.getTransceiverStats() // Add stats from the local transceiver @@ -1069,25 +1064,21 @@ class Relay @JvmOverloads constructor( statistics.bytesSent.getAndAdd(outgoingStats.bytes) statistics.packetsSent.getAndAdd(outgoingStats.packets) - conferenceStats.apply { - totalRelayBytesReceived.addAndGet(statistics.bytesReceived.get()) - totalRelayPacketsReceived.addAndGet(statistics.packetsReceived.get()) - totalRelayBytesSent.addAndGet(statistics.bytesSent.get()) - totalRelayPacketsSent.addAndGet(statistics.packetsSent.get()) - } - - conference.videobridge.statistics.apply { - /* TODO: should these be separate stats from the endpoint stats? */ - keyframesReceived.addAndGet(transceiverStats.rtpReceiverStats.videoParserStats.numKeyframes.toLong()) - layeringChangesReceived.addAndGet( - transceiverStats.rtpReceiverStats.videoParserStats.numLayeringChanges.toLong() - ) + VideobridgeMetrics.totalRelayBytesReceived.add(statistics.bytesReceived.get()) + VideobridgeMetrics.totalRelayBytesSent.add(statistics.bytesSent.get()) + VideobridgeMetrics.relayPacketsReceived.add(statistics.packetsReceived.get()) + VideobridgeMetrics.relayPacketsSent.add(statistics.packetsSent.get()) - val durationActiveVideo = transceiverStats.rtpReceiverStats.incomingStats.ssrcStats.values.filter { - it.mediaType == MediaType.VIDEO - }.sumOf { it.durationActive } - totalVideoStreamMillisecondsReceived.addAndGet(durationActiveVideo.toMillis()) - } + VideobridgeMetrics.keyframesReceived.addAndGet( + transceiverStats.rtpReceiverStats.videoParserStats.numKeyframes.toLong() + ) + VideobridgeMetrics.layeringChangesReceived.addAndGet( + transceiverStats.rtpReceiverStats.videoParserStats.numLayeringChanges.toLong() + ) + val durationActiveVideo = transceiverStats.rtpReceiverStats.incomingStats.ssrcStats.values.filter { + it.mediaType == MediaType.VIDEO + }.sumOf { it.durationActive } + VideobridgeMetrics.totalVideoStreamMillisecondsReceived.add(durationActiveVideo.toMillis()) } fun expire() { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt index d3024b6d6c..24540b14c7 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt @@ -21,7 +21,6 @@ import org.eclipse.jetty.websocket.core.CloseStatus import org.jitsi.utils.logging2.Logger import org.jitsi.videobridge.AbstractEndpointMessageTransport import org.jitsi.videobridge.VersionConfig -import org.jitsi.videobridge.Videobridge import org.jitsi.videobridge.datachannel.DataChannel import org.jitsi.videobridge.datachannel.DataChannelStack.DataChannelMessageListener import org.jitsi.videobridge.datachannel.protocol.DataChannelMessage @@ -35,6 +34,7 @@ import org.jitsi.videobridge.message.EndpointStats import org.jitsi.videobridge.message.ServerHelloMessage import org.jitsi.videobridge.message.SourceVideoTypeMessage import org.jitsi.videobridge.message.VideoTypeMessage +import org.jitsi.videobridge.metrics.VideobridgeMetrics import org.jitsi.videobridge.websocket.ColibriWebSocket import org.jitsi.videobridge.websocket.config.WebsocketServiceConfig import org.json.simple.JSONObject @@ -43,7 +43,6 @@ import java.net.URI import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicLong -import java.util.function.Supplier /** * Handles the functionality related to sending and receiving COLIBRI messages @@ -52,7 +51,6 @@ import java.util.function.Supplier */ class RelayMessageTransport( private val relay: Relay, - private val statisticsSupplier: Supplier, private val eventHandler: EndpointMessageTransportEventHandler, parentLogger: Logger ) : AbstractEndpointMessageTransport(parentLogger), ColibriWebSocket.EventHandler, DataChannelMessageListener { @@ -220,7 +218,7 @@ class RelayMessageTransport( */ private fun sendMessage(dst: DataChannel, message: BridgeChannelMessage) { dst.sendString(message.toJson()) - statisticsSupplier.get().dataChannelMessagesSent.inc() + VideobridgeMetrics.dataChannelMessagesSent.inc() } /** @@ -230,12 +228,12 @@ class RelayMessageTransport( */ private fun sendMessage(dst: ColibriWebSocket, message: BridgeChannelMessage) { dst.sendString(message.toJson()) - statisticsSupplier.get().colibriWebSocketMessagesSent.inc() + VideobridgeMetrics.colibriWebSocketMessagesSent.inc() } override fun onDataChannelMessage(dataChannelMessage: DataChannelMessage?) { webSocketLastActive = false - statisticsSupplier.get().dataChannelMessagesReceived.inc() + VideobridgeMetrics.dataChannelMessagesReceived.inc() if (dataChannelMessage is DataChannelStringMessage) { onMessage(dataChannel.get(), dataChannelMessage.data) } @@ -378,7 +376,7 @@ class RelayMessageTransport( logger.warn("Received text from an unknown web socket.") return } - statisticsSupplier.get().colibriWebSocketMessagesReceived.inc() + VideobridgeMetrics.colibriWebSocketMessagesReceived.inc() webSocketLastActive = true onMessage(ws, message) } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/rest/binders/ServiceBinder.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/rest/binders/ServiceBinder.kt index 9121b1eb1b..6a41558899 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/rest/binders/ServiceBinder.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/rest/binders/ServiceBinder.kt @@ -19,22 +19,15 @@ package org.jitsi.videobridge.rest.binders import org.glassfish.hk2.utilities.binding.AbstractBinder import org.jitsi.health.HealthCheckService import org.jitsi.videobridge.Videobridge -import org.jitsi.videobridge.stats.StatsCollector import org.jitsi.videobridge.xmpp.XmppConnection class ServiceBinder( private val videobridge: Videobridge, private val xmppConnection: XmppConnection, - private val statsCollector: StatsCollector?, private val healthChecker: HealthCheckService ) : AbstractBinder() { override fun configure() { bind(videobridge).to(Videobridge::class.java) - // We have to test this, because the nullablle 'StatsCollector?' type doesn't play - // nicely in hk2 since we're binding to 'StatsCollector' - if (statsCollector != null) { - bind(statsCollector).to(StatsCollector::class.java) - } bind(xmppConnection).to(XmppConnection::class.java) bind(healthChecker).to(HealthCheckService::class.java) } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/shutdown/ShutdownManager.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/shutdown/ShutdownManager.kt index 7088ac66fe..1ff6601867 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/shutdown/ShutdownManager.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/shutdown/ShutdownManager.kt @@ -18,6 +18,7 @@ package org.jitsi.videobridge.shutdown import org.jitsi.meet.ShutdownService import org.jitsi.utils.logging2.Logger import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.videobridge.metrics.VideobridgeMetrics import org.jitsi.videobridge.shutdown.ShutdownConfig.Companion.config import org.jitsi.videobridge.shutdown.ShutdownState.GRACEFUL_SHUTDOWN import org.jitsi.videobridge.shutdown.ShutdownState.RUNNING @@ -52,6 +53,7 @@ class ShutdownManager( if (graceful) { if (state == RUNNING) { state = GRACEFUL_SHUTDOWN + VideobridgeMetrics.gracefulShutdown.set(true) logger.info( "Entered graceful shutdown mode, will stay in this mode for up to " + config.gracefulShutdownMaxDuration @@ -94,6 +96,7 @@ class ShutdownManager( logger.info("Will shut down in ${config.shuttingDownDelay}") state = SHUTTING_DOWN + VideobridgeMetrics.shuttingDown.set(true) TaskPools.SCHEDULED_POOL.schedule( { logger.info("Videobridge is shutting down NOW") diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/ConferencePacketStats.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/ConferencePacketStats.kt index 6f71b9c4ca..a6384ee9db 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/ConferencePacketStats.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/ConferencePacketStats.kt @@ -15,7 +15,7 @@ */ package org.jitsi.videobridge.stats -import org.jitsi.videobridge.stats.config.StatsManagerConfig +import org.jitsi.videobridge.metrics.Metrics import org.json.simple.JSONArray import org.json.simple.JSONObject import java.util.concurrent.atomic.AtomicLong @@ -27,7 +27,7 @@ import java.util.concurrent.atomic.AtomicLong * the rates passed to [addValue] to number of packets and bytes. */ class ConferencePacketStats private constructor() { - private val periodSeconds = StatsManagerConfig.config.interval.toMillis().toDouble() / 1000 + private val periodSeconds = Metrics.interval.toMillis().toDouble() / 1000 /** Maps a conference size to a [Stats] instance that sums the added packet and bit rates. */ private val stats: Map = mutableMapOf().apply { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/MucPublisher.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/MucPublisher.kt new file mode 100644 index 0000000000..be20d347e8 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/MucPublisher.kt @@ -0,0 +1,60 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.stats + +import org.jitsi.utils.logging2.createLogger +import org.jitsi.videobridge.xmpp.XmppConnection +import org.jitsi.videobridge.xmpp.config.XmppClientConnectionConfig +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension +import java.time.Duration +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit + +class MucPublisher( + val executor: ScheduledExecutorService, + val interval: Duration, + val xmppConnection: XmppConnection +) { + val logger = createLogger() + + var task: ScheduledFuture<*>? = null + fun start() { + logger.info("Starting with interval $interval.") + // We read the stats from the prometheus metrics and send updated presence. None of these are blocking. + task = executor.scheduleAtFixedRate( + { publishPresence() }, + 0, + interval.toMillis(), + TimeUnit.MILLISECONDS + ) + } + fun stop() { + task?.cancel(true) + task = null + } + + private fun publishPresence() { + val statsExt: ColibriStatsExtension = if (XmppClientConnectionConfig.config.statsFilterEnabled) { + VideobridgeStatisticsShim.getColibriStatsExtension(XmppClientConnectionConfig.config.statsWhitelist) + } else { + VideobridgeStatisticsShim.getColibriStatsExtension() + } + logger.debug { "Publishing statistics in presence: $statsExt" } + + xmppConnection.setPresenceExtension(statsExt) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/StatsCollector.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/StatsCollector.kt deleted file mode 100644 index 1356730b32..0000000000 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/StatsCollector.kt +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright @ 2015 - Present, 8x8 Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.jitsi.videobridge.stats - -import org.jitsi.utils.concurrent.PeriodicRunnableWithObject -import org.jitsi.utils.concurrent.RecurringRunnableExecutor -import org.jitsi.videobridge.stats.config.StatsManagerConfig.Companion.config -import java.util.concurrent.CopyOnWriteArrayList -import java.util.concurrent.atomic.AtomicBoolean - -/** - * [StatsCollector] periodically collects statistics by calling [Statistics.generate] on [statistics], and periodically - * pushes the latest collected statistics to the [StatsTransport]s that have been added to it. - * - * @author Hristo Terezov - * @author Lyubomir Marinov - */ -class StatsCollector( - /** - * The instance which can collect statistics via [Statistics.generate]. The [StatsCollector] invokes this - * periodically. - */ - val statistics: Statistics -) { - /** - * The [RecurringRunnableExecutor] which periodically invokes [statisticsRunnable]. - */ - private val statisticsExecutor = RecurringRunnableExecutor( - StatsCollector::class.java.simpleName + "-statisticsExecutor" - ) - - /** - * The [RecurringRunnableExecutor] which periodically invokes [transportRunnables]. - */ - private val transportExecutor = RecurringRunnableExecutor( - StatsCollector::class.java.simpleName + "-transportExecutor" - ) - - /** - * The periodic runnable which collects statistics by invoking `statistics.generate()`. - */ - private val statisticsRunnable: StatisticsPeriodicRunnable - - /** - * The runnables which periodically push statistics to the [StatsTransport]s that have been added. - */ - private val transportRunnables: MutableList = CopyOnWriteArrayList() - - private val running = AtomicBoolean() - - init { - val period = config.interval.toMillis() - require(period >= 1) { "period $period" } - statisticsRunnable = StatisticsPeriodicRunnable(statistics, period) - } - - /** - * Adds a specific StatsTransport through which this [StatsCollector] is to periodically send statistics. - * - * @param transport the [StatsTransport] to add - * @param updatePeriodMs the period in milliseconds at which this [StatsCollector] is to repeatedly send statistics - * to the specified [transport]. - */ - fun addTransport(transport: StatsTransport, updatePeriodMs: Long) { - require(updatePeriodMs >= 1) { "period $updatePeriodMs" } - - TransportPeriodicRunnable(transport, updatePeriodMs).also { - transportRunnables.add(it) - if (running.get()) { - transportExecutor.registerRecurringRunnable(it) - } - } - } - - fun removeTransport(transport: StatsTransport) { - val runnable = transportRunnables.find { it.o == transport } - runnable?.let { - transportExecutor.deRegisterRecurringRunnable(it) - transportRunnables.remove(it) - } - } - - /** - * {@inheritDoc} - * - * Starts the [StatsTransport]s added to this [StatsCollector]. Commences the collection of statistics. - */ - fun start() { - if (running.compareAndSet(false, true)) { - // Register statistics and transports with their respective RecurringRunnableExecutor in order to have them - // periodically executed. - statisticsExecutor.registerRecurringRunnable(statisticsRunnable) - transportRunnables.forEach { transportExecutor.registerRecurringRunnable(it) } - } - } - - /** - * {@inheritDoc} - * - * Stops the [StatsTransport]s added to this [StatsCollector] and the [StatisticsPeriodicRunnable]. - */ - fun stop() { - if (running.compareAndSet(true, false)) { - // De-register statistics and transports from their respective - // RecurringRunnableExecutor in order to have them no longer - // periodically executed. - statisticsExecutor.deRegisterRecurringRunnable(statisticsRunnable) - // Stop the StatTransports added to this StatsManager - transportRunnables.forEach { transportExecutor.deRegisterRecurringRunnable(it) } - transportRunnables.clear() - } - } - - /** - * Implements a [RecurringRunnable] which periodically collects statistics from a specific [Statistics]. - */ - private class StatisticsPeriodicRunnable(statistics: Statistics, period: Long) : - PeriodicRunnableWithObject(statistics, period) { - - override fun doRun() { - o.generate() - } - } - - /** - * Implements a [RecurringRunnable] which periodically publishes statistics through a specific [StatsTransport]. - */ - private inner class TransportPeriodicRunnable(transport: StatsTransport, period: Long) : - PeriodicRunnableWithObject(transport, period) { - - override fun doRun() { - // FIXME measurementInterval was meant to be the actual interval of time that the information of the - // Statistics covers. However, it became difficult after a refactoring to calculate measurementInterval. - val measurementInterval = period - o.publishStatistics(statisticsRunnable.o, measurementInterval) - } - } -} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/VideobridgeStatisticsShim.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/VideobridgeStatisticsShim.kt new file mode 100644 index 0000000000..6b481bcfa1 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/VideobridgeStatisticsShim.kt @@ -0,0 +1,236 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.stats + +import org.jitsi.nlj.rtcp.RembHandler +import org.jitsi.videobridge.EndpointConnectionStatusMonitor +import org.jitsi.videobridge.VersionConfig +import org.jitsi.videobridge.health.JvbHealthChecker +import org.jitsi.videobridge.load_management.JvbLoadManager +import org.jitsi.videobridge.metrics.Metrics +import org.jitsi.videobridge.metrics.ThreadsMetric +import org.jitsi.videobridge.metrics.VideobridgeMetrics +import org.jitsi.videobridge.metrics.VideobridgePeriodicMetrics +import org.jitsi.videobridge.relay.RelayConfig +import org.jitsi.videobridge.transport.ice.IceTransport +import org.jitsi.videobridge.version.JvbVersionService +import org.jitsi.videobridge.xmpp.XmppConnection +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.BITRATE_DOWNLOAD +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.BITRATE_UPLOAD +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.CONFERENCES +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.DRAIN +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.ENDPOINTS_SENDING_AUDIO +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.ENDPOINTS_SENDING_VIDEO +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.INACTIVE_CONFERENCES +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.INACTIVE_ENDPOINTS +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.LARGEST_CONFERENCE +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.OCTO_CONFERENCES +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.OCTO_ENDPOINTS +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.OCTO_RECEIVE_BITRATE +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.OCTO_RECEIVE_PACKET_RATE +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.OCTO_SEND_BITRATE +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.OCTO_SEND_PACKET_RATE +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.P2P_CONFERENCES +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.PACKET_RATE_DOWNLOAD +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.PACKET_RATE_UPLOAD +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.PARTICIPANTS +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.RECEIVE_ONLY_ENDPOINTS +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.REGION +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.RELAY_ID +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.RELEASE +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.RTT_AGGREGATE +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.SHUTDOWN_IN_PROGRESS +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.Stat +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.THREADS +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TIMESTAMP +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_BYTES_RECEIVED +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_BYTES_RECEIVED_OCTO +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_BYTES_SENT +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_BYTES_SENT_OCTO +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_COLIBRI_WEB_SOCKET_MESSAGES_RECEIVED +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_COLIBRI_WEB_SOCKET_MESSAGES_SENT +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_CONFERENCES_COMPLETED +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_CONFERENCES_CREATED +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_CONFERENCE_SECONDS +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_DATA_CHANNEL_MESSAGES_RECEIVED +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_DATA_CHANNEL_MESSAGES_SENT +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_DOMINANT_SPEAKER_CHANGES +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_ICE_FAILED +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_ICE_SUCCEEDED +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_ICE_SUCCEEDED_TCP +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_PACKETS_RECEIVED +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_PACKETS_RECEIVED_OCTO +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_PACKETS_SENT +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_PACKETS_SENT_OCTO +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_PARTICIPANTS +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.VERSION +import org.json.simple.JSONObject +import java.text.SimpleDateFormat +import java.util.Date +import java.util.TimeZone + +/** + * A shim layer which translates the Prometheus metrics (from [VideobridgeMetricsContainer]) to the legacy + * [ColibriStatsExtension] suitable to be added to XMPP presence or converted to JSON for the response to the legacy + * /colibri/stats + */ +object VideobridgeStatisticsShim { + fun getStatsJson() = JSONObject().apply { + getStats().forEach { (k, v) -> + this[k] = v + } + } + + /** + * Formats statistics in ColibriStatsExtension object + * @param statistics the statistics instance + * @return the ColibriStatsExtension instance. + */ + fun getColibriStatsExtension() = ColibriStatsExtension().apply { + getStats().forEach { (key, value) -> + addStat(Stat(key, value)) + } + } + + /** + * Formats statistics in ColibriStatsExtension object + * @param statistics the statistics instance + * @param whitelist which of the statistics to use + * @return the ColibriStatsExtension instance. + */ + fun getColibriStatsExtension(whitelist: List) = ColibriStatsExtension().apply { + val allStats = getStats() + whitelist.forEach { whitelistedKey -> + val value = allStats[whitelistedKey] + if (value != null) { + addStat(Stat(whitelistedKey, value)) + } + } + } + + private val timestampFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + private fun getStats(): Map = synchronized(Metrics.lock) { + return buildMap { + put("incoming_loss", VideobridgePeriodicMetrics.incomingLoss.get()) + put("outgoing_loss", VideobridgePeriodicMetrics.outgoingLoss.get()) + put("overall_loss", VideobridgePeriodicMetrics.loss.get()) + put("endpoints_with_high_outgoing_loss", VideobridgePeriodicMetrics.endpointsWithHighOutgoingLoss.get()) + put("local_active_endpoints", VideobridgePeriodicMetrics.activeEndpoints.get()) + put(BITRATE_DOWNLOAD, VideobridgePeriodicMetrics.incomingBitrate.get() / 1000) + put(BITRATE_UPLOAD, VideobridgePeriodicMetrics.outgoingBitrate.get() / 1000) + put(PACKET_RATE_DOWNLOAD, VideobridgePeriodicMetrics.incomingPacketRate.get()) + put(PACKET_RATE_UPLOAD, VideobridgePeriodicMetrics.outgoingPacketRate.get()) + put(RTT_AGGREGATE, VideobridgePeriodicMetrics.averageRtt.get()) + put("num_eps_oversending", VideobridgePeriodicMetrics.endpointsOversending.get()) + put(OCTO_CONFERENCES, VideobridgePeriodicMetrics.conferencesWithRelay.get()) + put(INACTIVE_CONFERENCES, VideobridgePeriodicMetrics.conferencesInactive.get()) + put(P2P_CONFERENCES, VideobridgePeriodicMetrics.conferencesP2p.get()) + put("endpoints", VideobridgePeriodicMetrics.endpoints.get()) + put(PARTICIPANTS, VideobridgePeriodicMetrics.endpoints.get()) + put(RECEIVE_ONLY_ENDPOINTS, VideobridgePeriodicMetrics.endpointsReceiveOnly.get()) + put(INACTIVE_ENDPOINTS, VideobridgePeriodicMetrics.endpointsInactive.get()) + put(OCTO_ENDPOINTS, VideobridgePeriodicMetrics.endpointsRelayed.get()) + put(ENDPOINTS_SENDING_AUDIO, VideobridgePeriodicMetrics.endpointsSendingAudio.get()) + put(ENDPOINTS_SENDING_VIDEO, VideobridgePeriodicMetrics.endpointsSendingVideo.get()) + put(LARGEST_CONFERENCE, VideobridgePeriodicMetrics.largestConference.get()) + put(OCTO_RECEIVE_BITRATE, VideobridgePeriodicMetrics.relayIncomingBitrate.get()) + put(OCTO_RECEIVE_PACKET_RATE, VideobridgePeriodicMetrics.relayIncomingPacketRate.get()) + put(OCTO_SEND_BITRATE, VideobridgePeriodicMetrics.relayOutgoingBitrate.get()) + put(OCTO_SEND_PACKET_RATE, VideobridgePeriodicMetrics.relayOutgoingPacketRate.get()) + put("endpoints_with_suspended_sources", VideobridgePeriodicMetrics.endpointsWithSuspendedSources.get()) + + put(TOTAL_CONFERENCES_CREATED, VideobridgeMetrics.conferencesCreated.get()) + put(TOTAL_CONFERENCES_COMPLETED, VideobridgeMetrics.conferencesCompleted.get()) + put(TOTAL_CONFERENCE_SECONDS, VideobridgeMetrics.totalConferenceSeconds.get()) + put(TOTAL_PARTICIPANTS, VideobridgeMetrics.totalEndpoints.get()) + put("total_visitors", VideobridgeMetrics.totalVisitors.get()) + put( + "num_eps_no_msg_transport_after_delay", + VideobridgeMetrics.numEndpointsNoMessageTransportAfterDelay.get() + ) + put("total_relays", VideobridgeMetrics.totalRelays.get()) + put( + "num_relays_no_msg_transport_after_delay", + VideobridgeMetrics.numRelaysNoMessageTransportAfterDelay.get() + ) + put("total_keyframes_received", VideobridgeMetrics.keyframesReceived.get()) + put("total_layering_changes_received", VideobridgeMetrics.layeringChangesReceived.get()) + put( + "total_video_stream_milliseconds_received", + VideobridgeMetrics.totalVideoStreamMillisecondsReceived.get() + ) + put("stress_level", VideobridgeMetrics.stressLevel.get()) + put(CONFERENCES, VideobridgeMetrics.currentConferences.get()) + put("visitors", VideobridgeMetrics.currentVisitors.get()) + put("local_endpoints", VideobridgeMetrics.currentLocalEndpoints.get()) + put(TOTAL_DATA_CHANNEL_MESSAGES_RECEIVED, VideobridgeMetrics.dataChannelMessagesReceived.get()) + put(TOTAL_DATA_CHANNEL_MESSAGES_SENT, VideobridgeMetrics.dataChannelMessagesSent.get()) + put(TOTAL_COLIBRI_WEB_SOCKET_MESSAGES_RECEIVED, VideobridgeMetrics.colibriWebSocketMessagesReceived.get()) + put(TOTAL_COLIBRI_WEB_SOCKET_MESSAGES_SENT, VideobridgeMetrics.colibriWebSocketMessagesSent.get()) + put(TOTAL_BYTES_RECEIVED, VideobridgeMetrics.totalBytesReceived.get()) + put("dtls_failed_endpoints", VideobridgeMetrics.endpointsDtlsFailed.get()) + put(TOTAL_BYTES_SENT, VideobridgeMetrics.totalBytesSent.get()) + put(TOTAL_PACKETS_RECEIVED, VideobridgeMetrics.packetsReceived.get()) + put(TOTAL_PACKETS_SENT, VideobridgeMetrics.packetsSent.get()) + put(TOTAL_BYTES_RECEIVED_OCTO, VideobridgeMetrics.totalRelayBytesReceived.get()) + put(TOTAL_BYTES_SENT_OCTO, VideobridgeMetrics.totalRelayBytesSent.get()) + put(TOTAL_PACKETS_RECEIVED_OCTO, VideobridgeMetrics.relayPacketsReceived.get()) + put(TOTAL_PACKETS_SENT_OCTO, VideobridgeMetrics.relayPacketsSent.get()) + put(TOTAL_DOMINANT_SPEAKER_CHANGES, VideobridgeMetrics.dominantSpeakerChanges.get()) + put("preemptive_kfr_sent", VideobridgeMetrics.preemptiveKeyframeRequestsSent.get()) + put("preemptive_kfr_suppressed", VideobridgeMetrics.preemptiveKeyframeRequestsSuppressed.get()) + + put(TOTAL_ICE_FAILED, IceTransport.iceFailed.get()) + put(TOTAL_ICE_SUCCEEDED, IceTransport.iceSucceeded.get()) + put(TOTAL_ICE_SUCCEEDED_TCP, IceTransport.iceSucceededTcp.get()) + put("total_ice_succeeded_relayed", IceTransport.iceSucceededRelayed.get()) + + put("average_participant_stress", JvbLoadManager.averageParticipantStress) + + put(THREADS, ThreadsMetric.threadCount.get()) + + put(SHUTDOWN_IN_PROGRESS, VideobridgeMetrics.gracefulShutdown.get()) + put("shutting_down", VideobridgeMetrics.shuttingDown.get()) + put(DRAIN, VideobridgeMetrics.drainMode.get()) + + put(TIMESTAMP, timestampFormat.format(Date())) + if (RelayConfig.config.enabled) { + put(RELAY_ID, RelayConfig.config.relayId) + } + put("muc_clients_configured", XmppConnection.mucClientsConfigured.get()) + put("muc_clients_connected", XmppConnection.mucClientsConnected.get()) + put("mucs_configured", XmppConnection.mucsConfigured.get()) + put("mucs_joined", XmppConnection.mucsJoined.get()) + + put("endpoints_with_spurious_remb", RembHandler.endpointsWithSpuriousRemb()) + put("healthy", JvbHealthChecker.healthyMetric.get()) + put("endpoints_disconnected", EndpointConnectionStatusMonitor.endpointsDisconnected.get()) + put("endpoints_reconnected", EndpointConnectionStatusMonitor.endpointsReconnected.get()) + + put(VERSION, JvbVersionService.instance.currentVersion.toString()) + VersionConfig.config.release?.let { + put(RELEASE, it) + } + VideobridgeMetrics.regionInfo?.let { + put(REGION, it.get()) + } + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/config/StatsManagerConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/config/StatsManagerConfig.kt deleted file mode 100644 index 9c7a790b58..0000000000 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/config/StatsManagerConfig.kt +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright @ 2018 - present 8x8, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.jitsi.videobridge.stats.config - -import com.typesafe.config.Config -import com.typesafe.config.ConfigList -import com.typesafe.config.ConfigObject -import org.jitsi.config.JitsiConfig -import org.jitsi.metaconfig.ConfigException -import org.jitsi.metaconfig.config -import org.jitsi.videobridge.xmpp.XmppConnection -import java.time.Duration - -class StatsManagerConfig private constructor() { - /** The interval at which the stats are collected. */ - val interval: Duration by config { - "org.jitsi.videobridge.STATISTICS_INTERVAL".from(JitsiConfig.legacyConfig).convertFrom(Duration::ofMillis) - "videobridge.stats.interval".from(JitsiConfig.newConfig) - } - - /** - * The enabled stat transports - * - * Note that if 'org.jitsi.videobridge.STATISTICS_TRANSPORT' is present at all - * in the legacy config, we won't search the new config (we don't support merging - * stats transport configs from old and new config together). - * - * These are now obsolete and only maintained for backward compatibility. Transports should be configured in the - * modules that define them. See e.g. the implementation and [XmppConnection]. - */ - val transportConfigs: List by config { - "org.jitsi.videobridge." - .from(JitsiConfig.legacyConfig) - .convertFrom> { - if ("org.jitsi.videobridge.STATISTICS_TRANSPORT" in it) { - it.toStatsTransportConfig() - } else { - throw ConfigException.UnableToRetrieve.NotFound("not found in legacy config") - } - } - "videobridge.stats" - .from(JitsiConfig.newConfig) - .convertFrom { cfg -> - val transports = cfg["transports"] - ?: throw ConfigException.UnableToRetrieve.NotFound("Could not find transports within stats") - transports as ConfigList - transports.map { it as ConfigObject } - .map { it.toConfig() } - .mapNotNull { it.toStatsTransportConfig() } - } - "default" { emptyList() } - } - - /** - * From a map of properties pulled from the legacy config file, create a list of [StatsTransportConfig] - */ - private fun Map.toStatsTransportConfig(): List { - val transportTypes = - this["org.jitsi.videobridge.STATISTICS_TRANSPORT"]?.split(",") ?: return listOf() - return transportTypes.mapNotNull { transportType -> - val interval = this["org.jitsi.videobridge.STATISTICS_INTERVAL.$transportType"]?.let { - Duration.ofMillis(it.toLong()) - } ?: this@StatsManagerConfig.interval - when (transportType) { - "muc" -> StatsTransportConfig.MucStatsTransportConfig(interval) - else -> null - } - } - } - - private fun Config.toStatsTransportConfig(): StatsTransportConfig? { - val interval = if (hasPath("interval")) { - getDuration("interval") - } else { - this@StatsManagerConfig.interval - } - return when (getString("type")) { - "muc" -> StatsTransportConfig.MucStatsTransportConfig(interval) - else -> null - } - } - - companion object { - @JvmField - val config = StatsManagerConfig() - } -} - -/** - * Helper classes which model the config parameters for each stats transport - */ -sealed class StatsTransportConfig( - val interval: Duration -) { - class MucStatsTransportConfig(interval: Duration) : StatsTransportConfig(interval) -} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/version/JvbVersionService.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/version/JvbVersionService.kt index 7d1ee13e28..9c1cf96701 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/version/JvbVersionService.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/version/JvbVersionService.kt @@ -51,6 +51,7 @@ class JvbVersionService : VersionService { private const val DEFAULT_MAJOR_VERSION = 2 private const val DEFAULT_MINOR_VERSION = 1 private val defaultBuildId: String? = null + val instance = JvbVersionService() } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt index 7aa792c40a..2fbf52ca07 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt @@ -20,6 +20,7 @@ import org.jitsi.nlj.stats.DelayStats import org.jitsi.utils.OrderedJsonObject import org.jitsi.utils.logging2.cdebug import org.jitsi.utils.logging2.createLogger +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer import org.jitsi.videobridge.xmpp.config.XmppClientConnectionConfig.Companion.config import org.jitsi.xmpp.extensions.colibri.ForcefulShutdownIQ import org.jitsi.xmpp.extensions.colibri.GracefulShutdownIQ @@ -66,6 +67,7 @@ class XmppConnection : IQListener { } config.clientConfigs.forEach { cfg -> mucClientManager.addMucClient(cfg) } + org.jitsi.videobridge.metrics.Metrics.metricsUpdater.addUpdateTask { updateMetrics() } } else { logger.info("Already started") } @@ -77,6 +79,13 @@ class XmppConnection : IQListener { } } + fun updateMetrics() { + mucClientsConfigured.set(mucClientManager.clientCount) + mucClientsConnected.set(mucClientManager.clientConnectedCount) + mucsConfigured.set(mucClientManager.mucCount) + mucsJoined.set(mucClientManager.mucJoinedCount) + } + /** * Adds an [ExtensionElement] to our presence, and removes any other * extensions with the same element name and namespace, if any exists. @@ -289,6 +298,23 @@ class XmppConnection : IQListener { private val healthDelayStats = DelayStats(delayThresholds) private val versionDelayStats = DelayStats(delayThresholds) + val mucClientsConfigured = VideobridgeMetricsContainer.instance.registerLongGauge( + "muc_clients_configured", + "Number of MUC clients that are configured." + ) + val mucClientsConnected = VideobridgeMetricsContainer.instance.registerLongGauge( + "muc_clients_connected", + "Number of MUC clients that are connected." + ) + val mucsConfigured = VideobridgeMetricsContainer.instance.registerLongGauge( + "mucs_connected", + "Number of MUCs that are configured." + ) + val mucsJoined = VideobridgeMetricsContainer.instance.registerLongGauge( + "mucs_joined", + "Number of MUCs that are joined." + ) + @JvmStatic fun getStatsJson(): OrderedJsonObject = OrderedJsonObject().apply { put("colibri", colibriDelayStats.toJson()) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/config/XmppClientConnectionConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/config/XmppClientConnectionConfig.kt index 6f3dd5446a..883023373b 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/config/XmppClientConnectionConfig.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/config/XmppClientConnectionConfig.kt @@ -21,8 +21,6 @@ import com.typesafe.config.ConfigValue import org.jitsi.config.JitsiConfig import org.jitsi.metaconfig.ConfigException import org.jitsi.metaconfig.config -import org.jitsi.videobridge.stats.config.StatsManagerConfig -import org.jitsi.videobridge.stats.config.StatsTransportConfig import org.jitsi.xmpp.mucclient.MucClientConfiguration import java.time.Duration @@ -44,20 +42,10 @@ class XmppClientConnectionConfig private constructor() { } } - private val presenceIntervalProperty: Duration by config { + val presenceInterval: Duration by config { "videobridge.apis.xmpp-client.presence-interval".from(JitsiConfig.newConfig) } - /** - * The interval at which presence updates (with updates stats/status) are published. Allow to be overridden by - * legacy-style "stats-transports" config. - */ - val presenceInterval: Duration = StatsManagerConfig.config.transportConfigs - .filterIsInstance() - .map(StatsTransportConfig::interval) - .firstOrNull() - ?: presenceIntervalProperty - /** * Whether to filter the statistics. */ diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/ConferenceTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/ConferenceTest.kt index b0eb96f306..a43d5f5eca 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/ConferenceTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/ConferenceTest.kt @@ -16,7 +16,6 @@ package org.jitsi.videobridge import io.kotest.matchers.shouldBe -import io.mockk.every import io.mockk.mockk import org.jitsi.ConfigTest import org.json.simple.JSONObject @@ -27,9 +26,7 @@ import org.jxmpp.jid.impl.JidCreate * This is a high-level test for [Conference] and related functionality. */ class ConferenceTest : ConfigTest() { - private val videobridge = mockk(relaxed = true) { - every { statistics } returns Videobridge.Statistics() - } + private val videobridge = mockk(relaxed = true) init { val name = JidCreate.entityBareFrom("roomName@somedomain.com") diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/VideobridgeTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/VideobridgeTest.kt index 960a2084a2..7d359c9a92 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/VideobridgeTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/VideobridgeTest.kt @@ -26,6 +26,7 @@ import org.jitsi.config.withNewConfig import org.jitsi.shutdown.ShutdownServiceImpl import org.jitsi.utils.OrderedJsonObject import org.jitsi.utils.concurrent.FakeScheduledExecutorService +import org.jitsi.videobridge.metrics.VideobridgeMetrics import org.jitsi.videobridge.shutdown.ShutdownConfig import org.jitsi.videobridge.shutdown.ShutdownState import org.jitsi.videobridge.util.TaskPools @@ -45,7 +46,7 @@ class VideobridgeTest : ShouldSpec() { override suspend fun afterAny(testCase: TestCase, result: TestResult) = super.afterAny(testCase, result).also { TaskPools.resetScheduledPool() // we need this to test shutdown behavior since metrics are preserved across tests - videobridge.statistics.currentLocalEndpoints.set(0) + VideobridgeMetrics.currentLocalEndpoints.set(0) } init { diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/rest/root/colibri/stats/StatsTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/rest/root/colibri/stats/StatsTest.kt index 7f4c4ea31d..60a2d88201 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/rest/root/colibri/stats/StatsTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/rest/root/colibri/stats/StatsTest.kt @@ -18,34 +18,24 @@ package org.jitsi.videobridge.rest.root.colibri.stats import io.kotest.matchers.shouldBe import io.kotest.matchers.types.shouldBeInstanceOf -import io.mockk.every -import io.mockk.mockk import jakarta.ws.rs.core.Application import jakarta.ws.rs.core.MediaType -import jakarta.ws.rs.core.Response import org.eclipse.jetty.http.HttpStatus import org.glassfish.jersey.server.ResourceConfig import org.glassfish.jersey.test.JerseyTest import org.glassfish.jersey.test.TestProperties -import org.jitsi.videobridge.rest.MockBinder -import org.jitsi.videobridge.stats.StatsCollector -import org.jitsi.videobridge.stats.VideobridgeStatistics import org.json.simple.JSONObject import org.json.simple.parser.JSONParser import org.junit.Test class StatsTest : JerseyTest() { - private lateinit var statsCollector: StatsCollector private val baseUrl = "/colibri/stats" override fun configure(): Application { - statsCollector = mockk() - enable(TestProperties.LOG_TRAFFIC) enable(TestProperties.DUMP_ENTITY) return object : ResourceConfig() { init { - register(MockBinder(statsCollector, StatsCollector::class.java)) register(Stats::class.java) } } @@ -53,20 +43,10 @@ class StatsTest : JerseyTest() { @Test fun testGetStats() { - val fakeStats = mutableMapOf("stat1" to "value1", "stat2" to "value2") - val videobridgeStatistics = mockk() - every { videobridgeStatistics.stats } returns fakeStats - every { statsCollector.statistics } returns videobridgeStatistics - val resp = target(baseUrl).request().get() resp.status shouldBe HttpStatus.OK_200 resp.mediaType shouldBe MediaType.APPLICATION_JSON_TYPE - resp.getResultAsJson() shouldBe mapOf("stat1" to "value1", "stat2" to "value2") - } - - private fun Response.getResultAsJson(): JSONObject { - val obj = JSONParser().parse(readEntity(String::class.java)) - obj.shouldBeInstanceOf() - return obj + val parsed = JSONParser().parse(resp.readEntity(String::class.java)) + parsed.shouldBeInstanceOf() } } diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/stats/config/StatsManagerConfigTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/stats/config/StatsManagerConfigTest.kt deleted file mode 100644 index 58295e9fd6..0000000000 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/stats/config/StatsManagerConfigTest.kt +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright @ 2018 - present 8x8, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.jitsi.videobridge.stats.config - -import io.kotest.inspectors.forOne -import io.kotest.matchers.collections.shouldHaveSize -import io.kotest.matchers.shouldBe -import org.jitsi.ConfigTest -import org.jitsi.config.withLegacyConfig -import org.jitsi.config.withNewConfig -import java.time.Duration - -internal class StatsManagerConfigTest : ConfigTest() { - - init { - context("When only new config contains stats transport config") { - context("a stats transport config") { - context("with multiple, valid stats transports configured") { - withNewConfig(newConfigAllStatsTransports()) { - StatsManagerConfig.config.transportConfigs shouldHaveSize 1 - StatsManagerConfig.config.transportConfigs.forOne { - it as StatsTransportConfig.MucStatsTransportConfig - it.interval shouldBe Duration.ofSeconds(5) - } - } - } - context("with an invalid stats transport configured") { - withNewConfig(newConfigInvalidStatsTransports()) { - should("ignore the invalid config and parse the valid transport correctly") { - StatsManagerConfig.config.transportConfigs shouldHaveSize 1 - StatsManagerConfig.config.transportConfigs.forOne { - it as StatsTransportConfig.MucStatsTransportConfig - } - } - } - } - context("which has a custom interval") { - withNewConfig(newConfigOneStatsTransportCustomInterval()) { - should("reflect the custom interval") { - StatsManagerConfig.config.transportConfigs.forOne { - it as StatsTransportConfig.MucStatsTransportConfig - it.interval shouldBe Duration.ofSeconds(10) - } - } - } - } - } - } - context("When old and new config contain stats transport configs") { - withLegacyConfig(legacyConfigAllStatsTransports()) { - withNewConfig(newConfigOneStatsTransport()) { - should("use the values from the old config") { - StatsManagerConfig.config.transportConfigs shouldHaveSize 1 - StatsManagerConfig.config.transportConfigs.forOne { - it as StatsTransportConfig.MucStatsTransportConfig - } - } - } - } - } - } -} - -private fun newConfigAllStatsTransports(enabled: Boolean = true) = """ - videobridge { - stats { - interval=5 seconds - enabled=$enabled - transports = [ - { - type="muc" - } - ] - } - } -""".trimIndent() - -private fun newConfigOneStatsTransport(enabled: Boolean = true) = """ - videobridge { - stats { - enabled=$enabled - interval=5 seconds - transports = [ - { - type="muc" - } - ] - } - } -""".trimIndent() - -private fun newConfigOneStatsTransportCustomInterval(enabled: Boolean = true) = """ - videobridge { - stats { - enabled=$enabled - interval=5 seconds - transports = [ - { - type="muc" - interval=10 seconds - } - ] - } - } -""".trimIndent() - -private fun newConfigInvalidStatsTransports(enabled: Boolean = true) = """ - videobridge { - stats { - interval=5 seconds - enabled=$enabled - transports = [ - { - type="invalid" - }, - { - type="muc" - }, - ] - } - } -""".trimIndent() - -private fun legacyConfigAllStatsTransports(enabled: Boolean = true) = """ - org.jitsi.videobridge.ENABLE_STATISTICS=$enabled - org.jitsi.videobridge.STATISTICS_TRANSPORT=muc -""".trimIndent() diff --git a/pom.xml b/pom.xml index b6152eb6a0..d2235b3213 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 5.7.2 5.10.0 1.0-127-g6c65524 - 1.1-132-g906f995 + 1.1-139-gc42fce4 1.13.8 3.0.0 3.5.1 From 2958c4dd50f14b807ed459a7851d2c7ff393150b Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 2 Apr 2024 21:56:08 -0400 Subject: [PATCH 097/189] Bump srtp version (get correct path for ppc64le library). (#2115) --- jitsi-media-transform/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jitsi-media-transform/pom.xml b/jitsi-media-transform/pom.xml index 25a4beee56..de6d1404e8 100644 --- a/jitsi-media-transform/pom.xml +++ b/jitsi-media-transform/pom.xml @@ -22,7 +22,7 @@ ${project.groupId} jitsi-srtp - 1.1-15-ga19c05a + 1.1-16-g7fbe7e3 ${project.groupId} From 21b42fb2f2b5f3e92d0fdbbe9dce507d0fc2731d Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 2 Apr 2024 21:56:15 -0400 Subject: [PATCH 098/189] Bump jitsi-sctp version: assert if soref() is called on a freed socket. (#2116) --- jvb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/pom.xml b/jvb/pom.xml index 7cf98e8893..7a7078d1ba 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -108,7 +108,7 @@ ${project.groupId} jitsi-sctp - 1.0-21-gfe0d028 + 1.0-22-gda328b9 From 84c7cba0d68b4bf0862a6070be9e9b0ca04b165b Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Thu, 4 Apr 2024 12:12:40 -0400 Subject: [PATCH 099/189] Revert "Bump jitsi-sctp version: assert if soref() is called on a freed socket. (#2116)" (#2118) This reverts commit 21b42fb2f2b5f3e92d0fdbbe9dce507d0fc2731d. --- jvb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/pom.xml b/jvb/pom.xml index 7a7078d1ba..7cf98e8893 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -108,7 +108,7 @@ ${project.groupId} jitsi-sctp - 1.0-22-gda328b9 + 1.0-21-gfe0d028 From ac3de3d965ad47065492f135e2bbf5874004d495 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 9 Apr 2024 15:31:40 -0400 Subject: [PATCH 100/189] Update Jetty (including transitive dependency through jicoco). (#2119) --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index d2235b3213..21dd83acb2 100644 --- a/pom.xml +++ b/pom.xml @@ -23,12 +23,12 @@ pom - 11.0.14 + 11.0.20 1.9.10 5.7.2 5.10.0 1.0-127-g6c65524 - 1.1-139-gc42fce4 + 1.1-140-g8f45a9f 1.13.8 3.0.0 3.5.1 From b08f133c24cdb3f5d4a213a3535f144a26d1a901 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Wed, 10 Apr 2024 06:39:48 -0700 Subject: [PATCH 101/189] fix: Catch RTP parsing exceptions. (#2117) --- .../org/jitsi/nlj/transform/node/RtpParser.kt | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/RtpParser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/RtpParser.kt index a5ae97a8a9..c13cf86567 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/RtpParser.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/RtpParser.kt @@ -43,12 +43,25 @@ class RtpParser( } val rtpPacket = when (payloadType.mediaType) { - MediaType.AUDIO -> when (payloadType.encoding) { - RED -> packet.toOtherType(::RedAudioRtpPacket) - else -> packet.toOtherType(::AudioRtpPacket) + MediaType.AUDIO -> try { + when (payloadType.encoding) { + RED -> packet.toOtherType(::RedAudioRtpPacket) + else -> packet.toOtherType(::AudioRtpPacket) + } + } catch (e: Exception) { + logger.info("Dropping audio packet due to parse failure: ${e.message}") + return null + } + MediaType.VIDEO -> try { + packet.toOtherType(::VideoRtpPacket) + } catch (e: Exception) { + logger.info("Dropping video packet due to parse failure: ${e.message}") + return null + } + else -> { + logger.info("Dropping packet with unrecognized media type: '${payloadType.mediaType}'") + return null } - MediaType.VIDEO -> packet.toOtherType(::VideoRtpPacket) - else -> throw Exception("Unrecognized media type: '${payloadType.mediaType}'") } packetInfo.packet = rtpPacket if (rtpPacket.extensionsProfileType == 0xC0DE || rtpPacket.extensionsProfileType == 0xC2DE) { From 37674374c7b2fd20b701d54afa18d94ec42c5fee Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Mon, 15 Apr 2024 18:02:31 -0400 Subject: [PATCH 102/189] Update sctp again; add config param to set sctp debug flags. (#2122) --- jvb/pom.xml | 2 +- jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java | 2 +- jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpConfig.kt | 1 + jvb/src/main/resources/reference.conf | 2 ++ 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/jvb/pom.xml b/jvb/pom.xml index 7cf98e8893..c5479f7d5e 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -108,7 +108,7 @@ ${project.groupId} jitsi-sctp - 1.0-21-gfe0d028 + 1.0-23-ge04a9c9 diff --git a/jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java b/jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java index 72e0cacce8..af7f71a1b8 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java +++ b/jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java @@ -62,7 +62,7 @@ public class SctpManager classLogger.info("Initializing Sctp4j"); // "If UDP encapsulation is not necessary, the UDP port has to be set to 0" // All our SCTP is encapsulated in DTLS, we don't use direct UDP encapsulation. - Sctp4j.init(0); + Sctp4j.init(0, config.getDebugMask()); } else { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpConfig.kt index ffed0aad65..4293cfdc62 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpConfig.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpConfig.kt @@ -21,6 +21,7 @@ import org.jitsi.metaconfig.config class SctpConfig private constructor() { val enabled: Boolean by config { "videobridge.sctp.enabled".from(JitsiConfig.newConfig) } + val debugMask: Int by config { "videobridge.sctp.debug-mask".from(JitsiConfig.newConfig) } fun enabled() = enabled diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index 94eaeddb0e..377f6b5231 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -219,6 +219,8 @@ videobridge { sctp { // Whether SCTP data channels are enabled. enabled = true + // Debug flags to enable in usrsctp. -1 for all, otherwise see usrsctp source + debug-flags = 0 } stats { // The interval at which stats are gathered. From 2e0cb5eb45b15a7e876147a57a3c63b7384c0c3f Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Mon, 15 Apr 2024 18:31:50 -0400 Subject: [PATCH 103/189] Fix new config value name in reference.conf. --- jvb/src/main/resources/reference.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index 377f6b5231..43d9b37afa 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -219,8 +219,8 @@ videobridge { sctp { // Whether SCTP data channels are enabled. enabled = true - // Debug flags to enable in usrsctp. -1 for all, otherwise see usrsctp source - debug-flags = 0 + // Debug mask of categories to enable in usrsctp. 0 for none, -1 for all, otherwise see usrsctp source + debug-mask = 0 } stats { // The interval at which stats are gathered. From d7324cb397123a65d64da2cb460d1bf5ab44469d Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 16 Apr 2024 14:46:18 -0400 Subject: [PATCH 104/189] Fix or suppress some ktlint-1.2.1 warnings. (#2120) * Fix or suppress some ktlint-1.2.1 warnings. * Update ktlint-maven-plugin version to match. --- .../main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt | 2 +- .../main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt | 2 +- .../org/jitsi/nlj/stats/EndpointConnectionStatsTest.kt | 6 +++--- pom.xml | 2 +- rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpPacket.kt | 1 + rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpHeaderTest.kt | 4 ++-- 6 files changed, 9 insertions(+), 8 deletions(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt index 2798125d7e..33d98323ff 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt @@ -79,7 +79,7 @@ class Vp8Packet private constructor( val hasTL0PICIDX = DePacketizer.VP8PayloadDescriptor.hasTL0PICIDX(buffer, payloadOffset, payloadLength) - @field:Suppress("ktlint:standard:property-naming") + @field:Suppress("ktlint:standard:property-naming", "ktlint:standard:backing-property-naming") private var _TL0PICIDX = TL0PICIDX ?: DePacketizer.VP8PayloadDescriptor.getTL0PICIDX(buffer, payloadOffset, payloadLength) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt index ef26c01f89..7baf2379d5 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt @@ -99,7 +99,7 @@ class Vp9Packet private constructor( val isInterPicturePredicted = DePacketizer.VP9PayloadDescriptor.isInterPicturePredicted(buffer, payloadOffset, payloadLength) - @field:Suppress("ktlint:standard:property-naming") + @field:Suppress("ktlint:standard:property-naming", "ktlint:standard:backing-property-naming") private var _TL0PICIDX = TL0PICIDX ?: DePacketizer.VP9PayloadDescriptor.getTL0PICIDX(buffer, payloadOffset, payloadLength) diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/stats/EndpointConnectionStatsTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/stats/EndpointConnectionStatsTest.kt index 5c48de7d0d..cf498c233d 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/stats/EndpointConnectionStatsTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/stats/EndpointConnectionStatsTest.kt @@ -61,7 +61,7 @@ class EndpointConnectionStatsTest : ShouldSpec() { stats.rtcpPacketReceived(rrPacket, clock.instant()) context("the rtt") { should("be updated correctly") { - mostRecentPublishedRtt shouldBe(10.0 plusOrMinus .1) + mostRecentPublishedRtt shouldBe (10.0 plusOrMinus .1) } } } @@ -78,7 +78,7 @@ class EndpointConnectionStatsTest : ShouldSpec() { stats.rtcpPacketReceived(rrPacket, clock.instant()) context("the rtt") { should("be updated correctly") { - mostRecentPublishedRtt shouldBe(10.0.plusOrMinus(.1)) + mostRecentPublishedRtt shouldBe (10.0.plusOrMinus(.1)) } } } @@ -124,7 +124,7 @@ class EndpointConnectionStatsTest : ShouldSpec() { context("the rtt") { should("be updated correctly") { - mostRecentPublishedRtt shouldBe(10.0.plusOrMinus(.1)) + mostRecentPublishedRtt shouldBe (10.0.plusOrMinus(.1)) } } } diff --git a/pom.xml b/pom.xml index 21dd83acb2..0e846bc1ec 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ 1.0-127-g6c65524 1.1-140-g8f45a9f 1.13.8 - 3.0.0 + 3.2.0 3.5.1 4.6.0 3.0.10 diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpPacket.kt index f0246eca04..50d19bd070 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpPacket.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpPacket.kt @@ -173,6 +173,7 @@ open class RtpPacket( private val headerExtensionParser get() = HeaderExtensionHelpers.getHeaderExtensionParser(extensionsProfileType) + @field:Suppress("ktlint:standard:backing-property-naming") private val _encodedHeaderExtensions: EncodedHeaderExtensions = EncodedHeaderExtensions() private val encodedHeaderExtensions: EncodedHeaderExtensions get() { diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpHeaderTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpHeaderTest.kt index 8a4beb0aaa..f204812ee8 100644 --- a/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpHeaderTest.kt +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpHeaderTest.kt @@ -184,13 +184,13 @@ class RtpHeaderTest : ShouldSpec() { context("csrcs") { context("get") { should("work correctly") { - RtpHeader.getCsrcs(headerData, 0) shouldContainInOrder(listOf(123456, 45678)) + RtpHeader.getCsrcs(headerData, 0) shouldContainInOrder (listOf(123456, 45678)) } } context("set") { should("work correctly") { RtpHeader.setCsrcs(headerData, 0, listOf(2468, 1357)) - RtpHeader.getCsrcs(headerData, 0) shouldContainInOrder(listOf(2468, 1357)) + RtpHeader.getCsrcs(headerData, 0) shouldContainInOrder (listOf(2468, 1357)) } } } From 0f4454018d7c608741cf9ca709cf03204c05bdb4 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 16 Apr 2024 17:15:09 -0400 Subject: [PATCH 105/189] Add an sctp-pcap-dump feature. (#2124) * Make PcapWriter not be a Node It's never used as one. * Distinguish inbound and outbound packets in pcaps. * Expose a buf/off/len API to PcapWriter. * Add an sctp-pcap-dump feature. --- .../kotlin/org/jitsi/nlj/RtpReceiverImpl.kt | 4 +- .../kotlin/org/jitsi/nlj/RtpSenderImpl.kt | 4 +- .../jitsi/nlj/transform/node/PcapWriter.kt | 46 ++++++++++++++----- .../transform/node/ToggleablePcapWriter.kt | 10 ++-- .../kotlin/org/jitsi/nlj/dtls/DtlsTest.kt | 4 +- .../root/debug/EndpointDebugFeatures.java | 4 +- .../kotlin/org/jitsi/videobridge/Endpoint.kt | 21 ++++++++- .../org/jitsi/videobridge/relay/Relay.kt | 21 ++++++++- 8 files changed, 89 insertions(+), 25 deletions(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt index 8f749fef38..3e2d66bfaa 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt @@ -224,7 +224,7 @@ class RtpReceiverImpl @JvmOverloads constructor( node(remoteBandwidthEstimator) // This reads audio levels from packets that use cryptex. TODO: should it go in the Audio path? node(audioLevelReader.postDecryptNode) - node(toggleablePcapWriter.newObserverNode()) + node(toggleablePcapWriter.newObserverNode(outbound = false)) node(statsTracker) node(PaddingTermination(logger)) demux("Media Type") { @@ -259,7 +259,7 @@ class RtpReceiverImpl @JvmOverloads constructor( predicate = PacketPredicate(Packet::looksLikeRtcp) path = pipeline { node(srtcpDecryptWrapper) - node(toggleablePcapWriter.newObserverNode()) + node(toggleablePcapWriter.newObserverNode(outbound = false)) node(CompoundRtcpParser(logger)) node(rtcpTermination) node(packetHandlerWrapper) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt index b568116e5f..e8f4f89be5 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt @@ -147,7 +147,7 @@ class RtpSenderImpl( node(statsTracker) node(TccSeqNumTagger(transportCcEngine, streamInformationStore)) node(HeaderExtEncoder(streamInformationStore, logger)) - node(toggleablePcapWriter.newObserverNode()) + node(toggleablePcapWriter.newObserverNode(outbound = true)) node(srtpEncryptWrapper) node(packetStreamStats.createNewNode()) node(PacketLossNode(packetLossConfig), condition = { packetLossConfig.enabled }) @@ -181,7 +181,7 @@ class RtpSenderImpl( packetInfo } node(rtcpSrUpdater) - node(toggleablePcapWriter.newObserverNode()) + node(toggleablePcapWriter.newObserverNode(outbound = true)) node(srtcpEncryptWrapper) node(packetStreamStats.createNewNode()) node(PacketLossNode(packetLossConfig), condition = { packetLossConfig.enabled }) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PcapWriter.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PcapWriter.kt index 47507b693c..133f821d0a 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PcapWriter.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PcapWriter.kt @@ -42,7 +42,7 @@ import kotlin.io.path.Path class PcapWriter( parentLogger: Logger, filePath: Path = Path(directory, "${Random().nextLong()}.pcap") -) : ObserverNode("PCAP writer") { +) { constructor(parentLogger: Logger, filePath: String) : this(parentLogger, Path(filePath)) private val logger = createChildLogger(parentLogger) @@ -60,28 +60,52 @@ class PcapWriter( companion object { private val localhost = Inet4Address.getByName("127.0.0.1") as Inet4Address + private val remotehost = Inet4Address.getByName("192.0.2.0") as Inet4Address + + private val localport = UdpPort(123, "blah") + private val remoteport = UdpPort(456, "blah") + val directory: String by config("jmt.debug.pcap.directory".from(JitsiConfig.newConfig)) } - override fun observe(packetInfo: PacketInfo) { + fun observe(packetInfo: PacketInfo, outbound: Boolean) = + observe(packetInfo.packet.buffer, packetInfo.packet.offset, packetInfo.packet.length, outbound) + + fun observe(buffer: ByteArray, offset: Int, length: Int, outbound: Boolean) { val udpPayload = UnknownPacket.Builder() // We can't pass offset/limit values to udpPayload.rawData, so we need to create an array that contains // only exactly what we want to write - val subBuf = ByteArray(packetInfo.packet.length) - System.arraycopy(packetInfo.packet.buffer, packetInfo.packet.offset, subBuf, 0, packetInfo.packet.length) + val subBuf = ByteArray(length) + System.arraycopy(buffer, offset, subBuf, 0, length) udpPayload.rawData(subBuf) + val srchost: Inet4Address + val dsthost: Inet4Address + val srcport: UdpPort + val dstport: UdpPort + if (outbound) { + srchost = localhost + srcport = localport + dsthost = remotehost + dstport = remoteport + } else { + srchost = remotehost + srcport = remoteport + dsthost = localhost + dstport = localport + } + val udp = UdpPacket.Builder() - .srcPort(UdpPort(123, "blah")) - .dstPort(UdpPort(456, "blah")) - .srcAddr(localhost) - .dstAddr(localhost) + .srcPort(srcport) + .dstPort(dstport) + .srcAddr(srchost) + .dstAddr(dsthost) .correctChecksumAtBuild(true) .correctLengthAtBuild(true) .payloadBuilder(udpPayload) val ipPacket = IpV4Packet.Builder() - .srcAddr(localhost) - .dstAddr(localhost) + .srcAddr(srchost) + .dstAddr(dsthost) .protocol(IpNumber.UDP) .version(IpVersion.IPV4) .tos(IpV4Rfc1349Tos.newInstance(0)) @@ -99,8 +123,6 @@ class PcapWriter( writer.dump(eth) } - override fun trace(f: () -> Unit) = f.invoke() - fun close() { if (lazyWriter.isInitialized() && writer.isOpen) { writer.close() diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/ToggleablePcapWriter.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/ToggleablePcapWriter.kt index 46403f6365..5715f8ae1e 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/ToggleablePcapWriter.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/ToggleablePcapWriter.kt @@ -51,11 +51,15 @@ class ToggleablePcapWriter( fun isEnabled(): Boolean = pcapWriter != null - fun newObserverNode(): Node = PcapWriterNode("Toggleable pcap writer: $prefix") + fun newObserverNode(outbound: Boolean) = PcapWriterNode("Toggleable pcap writer: $prefix", outbound) - private inner class PcapWriterNode(name: String) : ObserverNode(name) { + inner class PcapWriterNode(name: String, val outbound: Boolean) : ObserverNode(name) { override fun observe(packetInfo: PacketInfo) { - pcapWriter?.processPacket(packetInfo) + pcapWriter?.observe(packetInfo, outbound) + } + + fun observe(buffer: ByteArray, offset: Int, length: Int) { + pcapWriter?.observe(buffer, offset, length, outbound) } override fun trace(f: () -> Unit) = f.invoke() diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsTest.kt index 5ca9c9190f..2b0d9db85e 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsTest.kt @@ -57,7 +57,7 @@ class DtlsTest : ShouldSpec() { // The DTLS server's send is wired directly to the DTLS client's receive dtlsServer.outgoingDataHandler = object : DtlsStack.OutgoingDataHandler { override fun sendData(data: ByteArray, off: Int, len: Int) { - pcapWriter?.processPacket(PacketInfo(UnparsedPacket(data, off, len))) + pcapWriter?.observe(PacketInfo(UnparsedPacket(data, off, len)), outbound = false) dtlsClient.processIncomingProtocolData(data, off, len) } } @@ -79,7 +79,7 @@ class DtlsTest : ShouldSpec() { // The DTLS client's send is wired directly to the DTLS server's receive dtlsClient.outgoingDataHandler = object : DtlsStack.OutgoingDataHandler { override fun sendData(data: ByteArray, off: Int, len: Int) { - pcapWriter?.processPacket(PacketInfo(UnparsedPacket(data, off, len))) + pcapWriter?.observe(PacketInfo(UnparsedPacket(data, off, len)), outbound = true) dtlsServer.processIncomingProtocolData(data, off, len) } } diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/EndpointDebugFeatures.java b/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/EndpointDebugFeatures.java index d3ee037f3f..917a984ce4 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/EndpointDebugFeatures.java +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/EndpointDebugFeatures.java @@ -18,8 +18,8 @@ public enum EndpointDebugFeatures { - PCAP_DUMP("pcap-dump"); - + PCAP_DUMP("pcap-dump"), + SCTP_PCAP_DUMP("sctp-pcap-dump"); private final String value; diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index c1c3694a54..5ef171f061 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -33,6 +33,8 @@ import org.jitsi.nlj.rtp.VideoRtpPacket import org.jitsi.nlj.srtp.TlsRole import org.jitsi.nlj.stats.EndpointConnectionStats import org.jitsi.nlj.transform.node.ConsumerNode +import org.jitsi.nlj.transform.node.ToggleablePcapWriter +import org.jitsi.nlj.transform.pipeline import org.jitsi.nlj.util.Bandwidth import org.jitsi.nlj.util.LocalSsrcAssociation import org.jitsi.nlj.util.NEVER @@ -125,6 +127,15 @@ class Endpoint @JvmOverloads constructor( private val sctpHandler = SctpHandler() private val dataChannelHandler = DataChannelHandler() + private val toggleablePcapWriter = ToggleablePcapWriter(logger, "$id-sctp") + private val sctpRecvPcap = toggleablePcapWriter.newObserverNode(outbound = false) + private val sctpSendPcap = toggleablePcapWriter.newObserverNode(outbound = true) + + private val sctpPipeline = pipeline { + node(sctpRecvPcap) + node(sctpHandler) + } + /* TODO: do we ever want to support useUniquePort for an Endpoint? */ private val iceTransport = IceTransport(id, iceControlling, false, supportsPrivateAddresses, logger) private val dtlsTransport = DtlsTransport(logger).also { it.cryptex = CryptexConfig.endpoint } @@ -460,12 +471,19 @@ class Endpoint @JvmOverloads constructor( fun setFeature(feature: EndpointDebugFeatures, enabled: Boolean) { when (feature) { EndpointDebugFeatures.PCAP_DUMP -> transceiver.setFeature(Features.TRANSCEIVER_PCAP_DUMP, enabled) + EndpointDebugFeatures.SCTP_PCAP_DUMP -> + if (enabled) { + toggleablePcapWriter.enable() + } else { + toggleablePcapWriter.disable() + } } } fun isFeatureEnabled(feature: EndpointDebugFeatures): Boolean { return when (feature) { EndpointDebugFeatures.PCAP_DUMP -> transceiver.isFeatureEnabled(Features.TRANSCEIVER_PCAP_DUMP) + EndpointDebugFeatures.SCTP_PCAP_DUMP -> toggleablePcapWriter.isEnabled() } } @@ -522,7 +540,7 @@ class Endpoint @JvmOverloads constructor( */ // TODO(brian): change sctp handler to take buf, off, len fun dtlsAppPacketReceived(data: ByteArray, off: Int, len: Int) = - sctpHandler.processPacket(PacketInfo(UnparsedPacket(data, off, len))) + sctpPipeline.processPacket(PacketInfo(UnparsedPacket(data, off, len))) private fun effectiveVideoConstraintsChanged( oldEffectiveConstraints: EffectiveConstraintsMap, @@ -577,6 +595,7 @@ class Endpoint @JvmOverloads constructor( // Create the SctpManager and provide it a method for sending SCTP data sctpManager = SctpManager( { data, offset, length -> + sctpSendPcap.observe(data, offset, length) dtlsTransport.sendDtlsData(data, offset, length) 0 }, diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt index 1b69be4e31..72ec37e5ae 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt @@ -35,6 +35,8 @@ import org.jitsi.nlj.srtp.SrtpUtil import org.jitsi.nlj.srtp.TlsRole import org.jitsi.nlj.stats.EndpointConnectionStats import org.jitsi.nlj.transform.node.ConsumerNode +import org.jitsi.nlj.transform.node.ToggleablePcapWriter +import org.jitsi.nlj.transform.pipeline import org.jitsi.nlj.util.Bandwidth import org.jitsi.nlj.util.BufferPool import org.jitsi.nlj.util.LocalSsrcAssociation @@ -168,6 +170,15 @@ class Relay @JvmOverloads constructor( private val sctpHandler = SctpHandler() private val dataChannelHandler = DataChannelHandler() + private val toggleablePcapWriter = ToggleablePcapWriter(logger, "$id-sctp") + private val sctpRecvPcap = toggleablePcapWriter.newObserverNode(outbound = false) + private val sctpSendPcap = toggleablePcapWriter.newObserverNode(outbound = true) + + private val sctpPipeline = pipeline { + node(sctpRecvPcap) + node(sctpHandler) + } + private val iceTransport = IceTransport( id = id, controlling = iceControlling, @@ -447,6 +458,7 @@ class Relay @JvmOverloads constructor( // Create the SctpManager and provide it a method for sending SCTP data val sctpManager = SctpManager( { data, offset, length -> + sctpSendPcap.observe(data, offset, length) dtlsTransport.sendDtlsData(data, offset, length) 0 }, @@ -626,12 +638,19 @@ class Relay @JvmOverloads constructor( } senders.values.forEach { s -> s.setFeature(Features.TRANSCEIVER_PCAP_DUMP, enabled) } } + EndpointDebugFeatures.SCTP_PCAP_DUMP -> + if (enabled) { + toggleablePcapWriter.enable() + } else { + toggleablePcapWriter.disable() + } } } fun isFeatureEnabled(feature: EndpointDebugFeatures): Boolean { return when (feature) { EndpointDebugFeatures.PCAP_DUMP -> transceiver.isFeatureEnabled(Features.TRANSCEIVER_PCAP_DUMP) + EndpointDebugFeatures.SCTP_PCAP_DUMP -> toggleablePcapWriter.isEnabled() } } @@ -738,7 +757,7 @@ class Relay @JvmOverloads constructor( */ // TODO(brian): change sctp handler to take buf, off, len fun dtlsAppPacketReceived(data: ByteArray, off: Int, len: Int) = - sctpHandler.processPacket(PacketInfo(UnparsedPacket(data, off, len))) + sctpPipeline.processPacket(PacketInfo(UnparsedPacket(data, off, len))) /** * Return the newly created endpoint, or null if an endpoint with that ID already existed. Note that the new From e155b81eb4671208983390b48790d1bf063dd4b1 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Mon, 22 Apr 2024 10:31:20 -0400 Subject: [PATCH 106/189] Revert "Update sctp again; add config param to set sctp debug flags. (#2122)" (#2126) This reverts commit 37674374c7b2fd20b701d54afa18d94ec42c5fee. --- jvb/pom.xml | 2 +- jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java | 2 +- jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpConfig.kt | 1 - jvb/src/main/resources/reference.conf | 2 -- 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/jvb/pom.xml b/jvb/pom.xml index c5479f7d5e..7cf98e8893 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -108,7 +108,7 @@ ${project.groupId} jitsi-sctp - 1.0-23-ge04a9c9 + 1.0-21-gfe0d028 diff --git a/jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java b/jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java index af7f71a1b8..72e0cacce8 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java +++ b/jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java @@ -62,7 +62,7 @@ public class SctpManager classLogger.info("Initializing Sctp4j"); // "If UDP encapsulation is not necessary, the UDP port has to be set to 0" // All our SCTP is encapsulated in DTLS, we don't use direct UDP encapsulation. - Sctp4j.init(0, config.getDebugMask()); + Sctp4j.init(0); } else { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpConfig.kt index 4293cfdc62..ffed0aad65 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpConfig.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpConfig.kt @@ -21,7 +21,6 @@ import org.jitsi.metaconfig.config class SctpConfig private constructor() { val enabled: Boolean by config { "videobridge.sctp.enabled".from(JitsiConfig.newConfig) } - val debugMask: Int by config { "videobridge.sctp.debug-mask".from(JitsiConfig.newConfig) } fun enabled() = enabled diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index 43d9b37afa..94eaeddb0e 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -219,8 +219,6 @@ videobridge { sctp { // Whether SCTP data channels are enabled. enabled = true - // Debug mask of categories to enable in usrsctp. 0 for none, -1 for all, otherwise see usrsctp source - debug-mask = 0 } stats { // The interval at which stats are gathered. From 462b7f6295a8bb4e98e73002e326c3780cd944ed Mon Sep 17 00:00:00 2001 From: bgrozev Date: Tue, 23 Apr 2024 08:20:56 -0700 Subject: [PATCH 107/189] fix: Concurrent modification of receiverVideoConstraints. (#2125) --- jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt index cc78dcb859..b573eb561f 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt @@ -28,6 +28,7 @@ import org.jitsi.videobridge.cc.allocation.ReceiverConstraintsMap import org.jitsi.videobridge.cc.allocation.VideoConstraints import org.json.simple.JSONObject import java.time.Instant +import java.util.concurrent.ConcurrentHashMap /** * Represents an endpoint in a conference (i.e. the entity associated with @@ -60,7 +61,7 @@ abstract class AbstractEndpoint protected constructor( /** * The map of source name -> ReceiverConstraintsMap. */ - private val receiverVideoConstraints = mutableMapOf() + private val receiverVideoConstraints = ConcurrentHashMap() /** * The statistics id of this Endpoint. From 748a626bfa546762bdd7a1d91f5db9436fefcf23 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Mon, 29 Apr 2024 11:06:13 -0700 Subject: [PATCH 108/189] feat: Add metrics for colibri-ws closing and receiving an error. (#2128) --- .../videobridge/EndpointMessageTransport.java | 12 +++++++++++- .../videobridge/metrics/VideobridgeMetrics.kt | 18 ++++++++++++++++++ .../videobridge/relay/RelayMessageTransport.kt | 11 ++++++++++- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java b/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java index 4ab0df86be..3e1d0e366e 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java +++ b/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java @@ -353,9 +353,18 @@ public void webSocketClosed(ColibriWebSocket ws, int statusCode, String reason) webSocket = null; webSocketLastActive = false; getLogger().info(() -> "Websocket closed, statusCode " + statusCode + " ( " + reason + ")."); + // 1000 is normal, 1001 is e.g. a tab closing. 1005 is "No Status Rcvd" and we see the majority of + // sockets close this way. + if (statusCode == 1000 || statusCode == 1001 || statusCode == 1005) + { + VideobridgeMetrics.colibriWebSocketCloseNormal.inc(); + } + else + { + VideobridgeMetrics.colibriWebSocketCloseAbnormal.inc(); + } } } - } /** @@ -365,6 +374,7 @@ public void webSocketClosed(ColibriWebSocket ws, int statusCode, String reason) public void webSocketError(ColibriWebSocket ws, Throwable cause) { getLogger().error("Colibri websocket error: " + cause.getMessage()); + VideobridgeMetrics.colibriWebSocketErrors.inc(); } /** diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetrics.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetrics.kt index e16ace9c2f..d5c1a931d1 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetrics.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetrics.kt @@ -72,6 +72,24 @@ object VideobridgeMetrics { "Number of messages sent via the data channels of the endpoints of this conference." ) + @JvmField + val colibriWebSocketCloseNormal = metricsContainer.registerCounter( + "colibri_web_socket_close_normal", + "Number of times a colibri web socket was closed normally." + ) + + @JvmField + val colibriWebSocketCloseAbnormal = metricsContainer.registerCounter( + "colibri_web_socket_close_abnormal", + "Number of times a colibri web socket was closed abnormally." + ) + + @JvmField + val colibriWebSocketErrors = metricsContainer.registerCounter( + "colibri_web_socket_error", + "Number of times a colibri web socket reported an error." + ) + @JvmField val packetsReceived = metricsContainer.registerCounter( "packets_received", diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt index 24540b14c7..a4cdd3548f 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt @@ -340,6 +340,13 @@ class RelayMessageTransport( webSocket = null webSocketLastActive = false logger.debug { "Web socket closed, statusCode $statusCode ( $reason)." } + // 1000 is normal, 1001 is e.g. a tab closing. 1005 is "No Status Rcvd" and we see the majority of + // sockets close this way. + if (statusCode == 1000 || statusCode == 1001 || statusCode == 1005) { + VideobridgeMetrics.colibriWebSocketCloseNormal.inc() + } else { + VideobridgeMetrics.colibriWebSocketCloseAbnormal.inc() + } } } @@ -350,8 +357,10 @@ class RelayMessageTransport( } } - override fun webSocketError(ws: ColibriWebSocket, cause: Throwable) = + override fun webSocketError(ws: ColibriWebSocket, cause: Throwable) { logger.error("Colibri websocket error: ${cause.message}") + VideobridgeMetrics.colibriWebSocketErrors.inc() + } /** * {@inheritDoc} From b623d6af714a17922750aa9d1c8060406ecb6e0a Mon Sep 17 00:00:00 2001 From: bgrozev Date: Mon, 29 Apr 2024 11:37:25 -0700 Subject: [PATCH 109/189] Use JitsiXmppStringprep, update smack (#2127) * ref: Do not register xmpp extensions for RAW UDP, we only support ICE. * ref: Remove unused code. * feat: Use JitsiXmppStringprep * Initialize Smack using the default configuration from jitsi-xmpp-extensions. * Move registering extension providers out of Videobridge.java * doc: Fix javadocs. * ref: Move queue stats out of Videobridge.java. * ref: Move ulimit check out of Videobridge.java. * ref: Move videobridgeExpireThread out of Videobridge.java. * squash: Remove obsolete TODO. * chore: Update jitsi-xmpp-extensions (JitsiXmppStringprep). --- .../org/jitsi/videobridge/Conference.java | 2 +- .../org/jitsi/videobridge/Videobridge.java | 167 +----------------- .../videobridge/rest/root/debug/Debug.java | 2 +- .../main/kotlin/org/jitsi/videobridge/Main.kt | 10 +- .../org/jitsi/videobridge/stats/QueueStats.kt | 76 ++++++++ .../org/jitsi/videobridge/xmpp/Smack.kt | 77 ++++++++ .../org/jitsi/videobridge/VideobridgeTest.kt | 2 +- pom.xml | 2 +- 8 files changed, 166 insertions(+), 172 deletions(-) create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/stats/QueueStats.kt create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/Smack.kt diff --git a/jvb/src/main/java/org/jitsi/videobridge/Conference.java b/jvb/src/main/java/org/jitsi/videobridge/Conference.java index 4e8c24d5ff..a146e3cc04 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Conference.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Conference.java @@ -565,7 +565,7 @@ private double getMaxReceiverRtt(String excludedEndpointId) * respective Channels. Releases the resources acquired by this * instance throughout its life time and prepares it to be garbage * collected. - * + *

* NOTE: this should _only_ be called by the Conference "manager" (in this * case, Videobridge). If you need to expire a Conference from elsewhere, use * {@link Videobridge#expireConference(Conference)} diff --git a/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java b/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java index 9dbaa6da77..70476e6280 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java @@ -19,27 +19,20 @@ import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.*; import org.jitsi.health.Result; -import org.jitsi.nlj.*; import org.jitsi.shutdown.*; import org.jitsi.utils.*; import org.jitsi.utils.logging2.*; -import org.jitsi.utils.queue.*; import org.jitsi.utils.version.*; import org.jitsi.videobridge.health.*; import org.jitsi.videobridge.load_management.*; import org.jitsi.videobridge.metrics.*; -import org.jitsi.videobridge.relay.*; import org.jitsi.videobridge.shutdown.*; import org.jitsi.videobridge.stats.*; import org.jitsi.videobridge.util.*; import org.jitsi.videobridge.xmpp.*; -import org.jitsi.xmpp.extensions.*; -import org.jitsi.xmpp.extensions.colibri.*; import org.jitsi.xmpp.extensions.colibri2.*; import org.jitsi.xmpp.extensions.health.*; -import org.jitsi.xmpp.extensions.jingle.*; import org.jivesoftware.smack.packet.*; -import org.jivesoftware.smack.provider.*; import org.json.simple.*; import org.jxmpp.jid.*; import org.jxmpp.jid.impl.*; @@ -78,7 +71,7 @@ public class Videobridge /** * The Conferences of this Videobridge mapped by their local IDs. - * + *

* TODO: The only remaining uses of this ID are for the HTTP debug interface and the colibri WebSocket conference * identifier. This should be replaced with meetingId (while making sure jvb-rtcstats-push doesn't break). */ @@ -94,18 +87,12 @@ public class Videobridge /** * The clock to use, pluggable for testing purposes. - * + *

* Note that currently most code uses the system clock directly. */ @NotNull private final Clock clock; - /** - * Thread that checks expiration for conferences, contents, channels and - * execute expire procedure for any of them. - */ - private final VideobridgeExpireThread videobridgeExpireThread; - /** * The {@link JvbLoadManager} instance used for this bridge. */ @@ -113,7 +100,6 @@ public class Videobridge private final JvbLoadManager jvbLoadManager; @NotNull private final Version version; - @Nullable private final String releaseId; @NotNull private final ShutdownManager shutdownManager; @@ -138,18 +124,15 @@ public Videobridge( @Nullable XmppConnection xmppConnection, @NotNull ShutdownServiceImpl shutdownService, @NotNull Version version, - @Nullable String releaseId, @NotNull Clock clock) { this.clock = clock; - videobridgeExpireThread = new VideobridgeExpireThread(this); jvbLoadManager = JvbLoadManager.create(this); if (xmppConnection != null) { xmppConnection.setEventHandler(new XmppConnectionEventHandler()); } this.version = version; - this.releaseId = releaseId; this.shutdownManager = new ShutdownManager(shutdownService, logger); jvbHealthChecker.start(); } @@ -495,75 +478,11 @@ public ShutdownState getShutdownState() return shutdownManager.getState(); } - /** - * Starts this {@link Videobridge}. - * - * NOTE: we have to make this public so Jicofo can call it from its tests. - */ - public void start() - { - UlimitCheck.printUlimits(); - - videobridgeExpireThread.start(); - - // - ForcefulShutdownIqProvider.registerIQProvider(); - - // - GracefulShutdownIqProvider.registerIQProvider(); - - // - new ColibriStatsIqProvider(); // registers itself with Smack - - // ICE-UDP - ProviderManager.addExtensionProvider( - IceUdpTransportPacketExtension.ELEMENT, - IceUdpTransportPacketExtension.NAMESPACE, - new DefaultPacketExtensionProvider<>(IceUdpTransportPacketExtension.class)); - - // RAW-UDP - DefaultPacketExtensionProvider udpCandidatePacketExtensionProvider - = new DefaultPacketExtensionProvider<>(UdpCandidatePacketExtension.class); - ProviderManager.addExtensionProvider( - UdpCandidatePacketExtension.ELEMENT, - UdpCandidatePacketExtension.NAMESPACE, - udpCandidatePacketExtensionProvider); - - // ICE-UDP - DefaultPacketExtensionProvider iceCandidatePacketExtensionProvider - = new DefaultPacketExtensionProvider<>(IceCandidatePacketExtension.class); - ProviderManager.addExtensionProvider( - IceCandidatePacketExtension.ELEMENT, - IceCandidatePacketExtension.NAMESPACE, - iceCandidatePacketExtensionProvider); - - // ICE - ProviderManager.addExtensionProvider( - IceRtcpmuxPacketExtension.ELEMENT, - IceRtcpmuxPacketExtension.NAMESPACE, - new DefaultPacketExtensionProvider<>(IceRtcpmuxPacketExtension.class)); - - // DTLS-SRTP - ProviderManager.addExtensionProvider( - DtlsFingerprintPacketExtension.ELEMENT, - DtlsFingerprintPacketExtension.NAMESPACE, - new DefaultPacketExtensionProvider<>(DtlsFingerprintPacketExtension.class)); - - // Health-check - HealthCheckIQProvider.registerIQProvider(); - - // Colibri2 - IqProviderUtils.registerProviders(); - } - /** * Stops this {@link Videobridge}. - * - * NOTE: we have to make this public so Jicofo can call it from its tests. */ - public void stop() + void stop() { - videobridgeExpireThread.stop(); jvbLoadManager.stop(); } @@ -622,83 +541,12 @@ public OrderedJsonObject getDebugState(String conferenceId, String endpointId, b return debugState; } - /** - * Gets statistics for the different {@code PacketQueue}s that this bridge - * uses. - * TODO: is there a better place for this? - */ - @SuppressWarnings("unchecked") - public JSONObject getQueueStats() - { - JSONObject queueStats = new JSONObject(); - - queueStats.put( - "srtp_send_queue", - getJsonFromQueueStatisticsAndErrorHandler(Endpoint.queueErrorCounter, - "Endpoint-outgoing-packet-queue")); - queueStats.put( - "relay_srtp_send_queue", - getJsonFromQueueStatisticsAndErrorHandler(Relay.queueErrorCounter, - "Relay-outgoing-packet-queue")); - queueStats.put( - "relay_endpoint_sender_srtp_send_queue", - getJsonFromQueueStatisticsAndErrorHandler(RelayEndpointSender.queueErrorCounter, - "RelayEndpointSender-outgoing-packet-queue")); - queueStats.put( - "rtp_receiver_queue", - getJsonFromQueueStatisticsAndErrorHandler(RtpReceiverImpl.Companion.getQueueErrorCounter(), - "rtp-receiver-incoming-packet-queue")); - queueStats.put( - "rtp_sender_queue", - getJsonFromQueueStatisticsAndErrorHandler(RtpSenderImpl.Companion.getQueueErrorCounter(), - "rtp-sender-incoming-packet-queue")); - queueStats.put( - "colibri_queue", - QueueStatistics.Companion.getStatistics().get("colibri-queue") - ); - - queueStats.put( - AbstractEndpointMessageTransport.INCOMING_MESSAGE_QUEUE_ID, - getJsonFromQueueStatisticsAndErrorHandler( - null, - AbstractEndpointMessageTransport.INCOMING_MESSAGE_QUEUE_ID)); - - return queueStats; - } - - private OrderedJsonObject getJsonFromQueueStatisticsAndErrorHandler( - CountingErrorHandler countingErrorHandler, - String queueName) - { - OrderedJsonObject json = (OrderedJsonObject)QueueStatistics.Companion.getStatistics().get(queueName); - if (countingErrorHandler != null) - { - if (json == null) - { - json = new OrderedJsonObject(); - json.put("dropped_packets", countingErrorHandler.getNumPacketsDropped()); - } - json.put("exceptions", countingErrorHandler.getNumExceptions()); - } - - return json; - } - @NotNull public Version getVersion() { return version; } - /** - * Get the release ID of this videobridge. - * @return The release ID. Returns null if not in use. - */ - public @Nullable String getReleaseId() - { - return releaseId; - } - private class XmppConnectionEventHandler implements XmppConnection.EventHandler { @Override @@ -741,15 +589,6 @@ public IQ healthCheckIqReceived(@NotNull HealthCheckIQ iq) } } - /** - * Basic statistics/metrics about the videobridge like cumulative/total - * number of channels created, cumulative/total number of channels failed, - * etc. - */ - public static class Statistics - { - } - private static class ConferenceNotFoundException extends Exception {} private static class ConferenceAlreadyExistsException extends Exception {} private static class InGracefulShutdownException extends Exception {} diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/Debug.java b/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/Debug.java index 7a82d812ac..94b9993c54 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/Debug.java +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/Debug.java @@ -370,7 +370,7 @@ public String getJvbFeatureStats(@PathParam("feature") DebugFeatures feature) return ByteBufferPool.getStatsJson().toJSONString(); } case QUEUE_STATS: { - return videobridge.getQueueStats().toJSONString(); + return QueueStats.getQueueStats().toJSONString(); } case TRANSIT_STATS: { return PacketTransitStats.getStatsJson().toJSONString(); diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt index 7df9dd6cff..920d87caeb 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt @@ -38,11 +38,11 @@ import org.jitsi.videobridge.metrics.VideobridgePeriodicMetrics import org.jitsi.videobridge.rest.root.Application import org.jitsi.videobridge.stats.MucPublisher import org.jitsi.videobridge.util.TaskPools +import org.jitsi.videobridge.util.UlimitCheck import org.jitsi.videobridge.version.JvbVersionService import org.jitsi.videobridge.websocket.ColibriWebSocketService import org.jitsi.videobridge.xmpp.XmppConnection import org.jitsi.videobridge.xmpp.config.XmppClientConnectionConfig -import org.jxmpp.stringprep.XmppStringPrepUtil import java.time.Clock import kotlin.concurrent.thread import kotlin.system.exitProcess @@ -75,12 +75,13 @@ fun main() { logger.info("Starting jitsi-videobridge version ${JvbVersionService.instance.currentVersion}") + UlimitCheck.printUlimits() startIce4j() // Initialize, binding on the main ICE port. Harvesters.init() - XmppStringPrepUtil.setMaxCacheSizes(XmppClientConnectionConfig.config.jidCacheSize) + org.jitsi.videobridge.xmpp.Smack.initialize() PacketQueue.setEnableStatisticsDefault(true) // Trigger an exception early in case the DTLS cipher suites are misconfigured @@ -99,9 +100,9 @@ fun main() { xmppConnection, shutdownService, JvbVersionService.instance.currentVersion, - VersionConfig.config.release, Clock.systemUTC() - ).apply { start() } + ) + val videobridgeExpireThread = VideobridgeExpireThread(videobridge) Metrics.metricsUpdater.addUpdateTask { VideobridgePeriodicMetrics.update(videobridge) } @@ -169,6 +170,7 @@ fun main() { } catch (t: Throwable) { logger.error("Error shutting down http servers", t) } + videobridgeExpireThread.stop() videobridge.stop() stopIce4j() Metrics.stop() diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/QueueStats.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/QueueStats.kt new file mode 100644 index 0000000000..9adf21a3a8 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/QueueStats.kt @@ -0,0 +1,76 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.stats + +import org.jitsi.nlj.RtpReceiverImpl +import org.jitsi.nlj.RtpSenderImpl +import org.jitsi.utils.OrderedJsonObject +import org.jitsi.utils.queue.CountingErrorHandler +import org.jitsi.utils.queue.QueueStatistics.Companion.getStatistics +import org.jitsi.videobridge.AbstractEndpointMessageTransport +import org.jitsi.videobridge.Endpoint +import org.jitsi.videobridge.relay.Relay +import org.jitsi.videobridge.relay.RelayEndpointSender +import org.json.simple.JSONObject + +object QueueStats { + /** Gets statistics for the different `PacketQueue`s that this bridge uses. */ + @JvmStatic + fun getQueueStats() = JSONObject().apply { + this["srtp_send_queue"] = getJsonFromQueueStatisticsAndErrorHandler( + Endpoint.queueErrorCounter, + "Endpoint-outgoing-packet-queue" + ) + this["relay_srtp_send_queue"] = getJsonFromQueueStatisticsAndErrorHandler( + Relay.queueErrorCounter, + "Relay-outgoing-packet-queue" + ) + this["relay_endpoint_sender_srtp_send_queue"] = getJsonFromQueueStatisticsAndErrorHandler( + RelayEndpointSender.queueErrorCounter, + "RelayEndpointSender-outgoing-packet-queue" + ) + this["rtp_receiver_queue"] = getJsonFromQueueStatisticsAndErrorHandler( + RtpReceiverImpl.queueErrorCounter, + "rtp-receiver-incoming-packet-queue" + ) + this["rtp_sender_queue"] = getJsonFromQueueStatisticsAndErrorHandler( + RtpSenderImpl.queueErrorCounter, + "rtp-sender-incoming-packet-queue" + ) + this["colibri_queue"] = getStatistics()["colibri-queue"] + this[AbstractEndpointMessageTransport.INCOMING_MESSAGE_QUEUE_ID] = + getJsonFromQueueStatisticsAndErrorHandler( + null, + AbstractEndpointMessageTransport.INCOMING_MESSAGE_QUEUE_ID + ) + } + + private fun getJsonFromQueueStatisticsAndErrorHandler( + countingErrorHandler: CountingErrorHandler?, + queueName: String + ): OrderedJsonObject? { + var json = getStatistics()[queueName] as OrderedJsonObject? + if (countingErrorHandler != null) { + if (json == null) { + json = OrderedJsonObject() + json["dropped_packets"] = countingErrorHandler.numPacketsDropped + } + json["exceptions"] = countingErrorHandler.numExceptions + } + + return json + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/Smack.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/Smack.kt new file mode 100644 index 0000000000..436cc9654a --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/Smack.kt @@ -0,0 +1,77 @@ +/* + * Copyright @ 2024 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.xmpp + +import org.jitsi.videobridge.xmpp.config.XmppClientConnectionConfig +import org.jitsi.xmpp.extensions.DefaultPacketExtensionProvider +import org.jitsi.xmpp.extensions.colibri.ColibriStatsIqProvider +import org.jitsi.xmpp.extensions.colibri.ForcefulShutdownIqProvider +import org.jitsi.xmpp.extensions.colibri.GracefulShutdownIqProvider +import org.jitsi.xmpp.extensions.colibri2.IqProviderUtils +import org.jitsi.xmpp.extensions.health.HealthCheckIQProvider +import org.jitsi.xmpp.extensions.jingle.DtlsFingerprintPacketExtension +import org.jitsi.xmpp.extensions.jingle.IceCandidatePacketExtension +import org.jitsi.xmpp.extensions.jingle.IceRtcpmuxPacketExtension +import org.jitsi.xmpp.extensions.jingle.IceUdpTransportPacketExtension +import org.jivesoftware.smack.provider.ProviderManager +import org.jxmpp.stringprep.XmppStringPrepUtil + +object Smack { + fun initialize() { + org.jitsi.xmpp.Smack.initialize() + + XmppStringPrepUtil.setMaxCacheSizes(XmppClientConnectionConfig.config.jidCacheSize) + + registerProviders() + } + + private fun registerProviders() { + // + ForcefulShutdownIqProvider.registerIQProvider() + // + GracefulShutdownIqProvider.registerIQProvider() + // + ColibriStatsIqProvider() // registers itself with Smack + // ICE-UDP + ProviderManager.addExtensionProvider( + IceUdpTransportPacketExtension.ELEMENT, + IceUdpTransportPacketExtension.NAMESPACE, + DefaultPacketExtensionProvider(IceUdpTransportPacketExtension::class.java) + ) + // ICE-UDP + ProviderManager.addExtensionProvider( + IceCandidatePacketExtension.ELEMENT, + IceCandidatePacketExtension.NAMESPACE, + DefaultPacketExtensionProvider(IceCandidatePacketExtension::class.java) + ) + // ICE + ProviderManager.addExtensionProvider( + IceRtcpmuxPacketExtension.ELEMENT, + IceRtcpmuxPacketExtension.NAMESPACE, + DefaultPacketExtensionProvider(IceRtcpmuxPacketExtension::class.java) + ) + // DTLS-SRTP + ProviderManager.addExtensionProvider( + DtlsFingerprintPacketExtension.ELEMENT, + DtlsFingerprintPacketExtension.NAMESPACE, + DefaultPacketExtensionProvider(DtlsFingerprintPacketExtension::class.java) + ) + // Health-check + HealthCheckIQProvider.registerIQProvider() + // Colibri2 + IqProviderUtils.registerProviders() + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/VideobridgeTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/VideobridgeTest.kt index 7d359c9a92..2144142166 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/VideobridgeTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/VideobridgeTest.kt @@ -37,7 +37,7 @@ class VideobridgeTest : ShouldSpec() { private val shutdownService: ShutdownServiceImpl = mockk(relaxed = true) private val fakeExecutor = FakeScheduledExecutorService() - private val videobridge = Videobridge(null, shutdownService, mockk(), null, fakeExecutor.clock) + private val videobridge = Videobridge(null, shutdownService, mockk(), fakeExecutor.clock) override suspend fun beforeAny(testCase: TestCase) = super.beforeAny(testCase).also { TaskPools.SCHEDULED_POOL = fakeExecutor diff --git a/pom.xml b/pom.xml index 0e846bc1ec..94f1b20ce6 100644 --- a/pom.xml +++ b/pom.xml @@ -111,7 +111,7 @@ ${project.groupId} jitsi-xmpp-extensions - 1.0-78-g62d03d4 + 1.0-80-g0ce9883 From acf024b797cd1899e29a4888a32ab8f84f70aa28 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 2 May 2024 13:25:08 -0700 Subject: [PATCH 110/189] feat: Export more metrics to prometheus. (#2129) * feat: Export JVM metrics to prometheus. * feat: Add a startup_time metric for detection of restarts. * feat: Add metrics for queue drops/exceptions. --- .../kotlin/org/jitsi/nlj/RtpReceiverImpl.kt | 2 +- .../kotlin/org/jitsi/nlj/RtpSenderImpl.kt | 2 +- .../org/jitsi/videobridge/Conference.java | 15 +-- .../AbstractEndpointMessageTransport.kt | 25 ++++- .../org/jitsi/videobridge/ColibriQueue.kt | 68 +++++++++++ .../kotlin/org/jitsi/videobridge/Endpoint.kt | 27 ++++- .../jitsi/videobridge/metrics/JvmMetrics.kt | 106 ++++++++++++++++++ .../org/jitsi/videobridge/metrics/Metrics.kt | 7 +- .../jitsi/videobridge/metrics/QueueMetrics.kt | 73 ++++++++++++ .../videobridge/metrics/ThreadsMetric.kt | 29 ----- .../videobridge/metrics/VideobridgeMetrics.kt | 7 ++ .../org/jitsi/videobridge/relay/Relay.kt | 28 ++++- .../videobridge/relay/RelayEndpointSender.kt | 28 ++++- .../stats/VideobridgeStatisticsShim.kt | 6 +- jvb/src/main/resources/reference.conf | 4 + 15 files changed, 369 insertions(+), 58 deletions(-) create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/ColibriQueue.kt create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/metrics/JvmMetrics.kt create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/metrics/QueueMetrics.kt delete mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/metrics/ThreadsMetric.kt diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt index 3e2d66bfaa..152a1587e6 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt @@ -152,7 +152,7 @@ class RtpReceiverImpl @JvmOverloads constructor( } companion object { - val queueErrorCounter = CountingErrorHandler() + var queueErrorCounter = CountingErrorHandler() private const val PACKET_QUEUE_ENTRY_EVENT = "Entered RTP receiver incoming queue" private const val PACKET_QUEUE_EXIT_EVENT = "Exited RTP receiver incoming queue" diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt index e8f4f89be5..3c8272d11f 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt @@ -324,7 +324,7 @@ class RtpSenderImpl( } companion object { - val queueErrorCounter = CountingErrorHandler() + var queueErrorCounter = CountingErrorHandler() private const val PACKET_QUEUE_ENTRY_EVENT = "Entered RTP sender incoming queue" private const val PACKET_QUEUE_EXIT_EVENT = "Exited RTP sender incoming queue" diff --git a/jvb/src/main/java/org/jitsi/videobridge/Conference.java b/jvb/src/main/java/org/jitsi/videobridge/Conference.java index a146e3cc04..95bb08a5ea 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Conference.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Conference.java @@ -219,10 +219,7 @@ public Conference(Videobridge videobridge, this.id = Objects.requireNonNull(id, "id"); this.conferenceName = conferenceName; this.colibri2Handler = new Colibri2ConferenceHandler(this, logger); - colibriQueue = new PacketQueue<>( - Integer.MAX_VALUE, - true, - "colibri-queue", + colibriQueue = new ColibriQueue( request -> { try @@ -256,12 +253,10 @@ public Conference(Videobridge videobridge, e.getMessage())); } return true; - }, - TaskPools.IO_POOL, - Clock.systemUTC(), // TODO: using the Videobridge clock breaks tests somehow - /* Allow running tasks to complete (so we can close the queue from within the task. */ - false - ); + } + ) + { + }; speechActivity = new ConferenceSpeechActivity(new SpeechActivityListener()); updateLastNEndpointsFuture = TaskPools.SCHEDULED_POOL.scheduleAtFixedRate(() -> { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpointMessageTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpointMessageTransport.kt index 9e605d798b..3e39fafa01 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpointMessageTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpointMessageTransport.kt @@ -16,10 +16,13 @@ package org.jitsi.videobridge import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.queue.CountingErrorHandler import org.jitsi.utils.queue.PacketQueue import org.jitsi.videobridge.message.BridgeChannelMessage import org.jitsi.videobridge.message.BridgeChannelMessage.Companion.parse import org.jitsi.videobridge.message.MessageHandler +import org.jitsi.videobridge.metrics.QueueMetrics +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer import org.jitsi.videobridge.util.TaskPools import org.json.simple.JSONObject import java.io.IOException @@ -31,7 +34,7 @@ abstract class AbstractEndpointMessageTransport(parentLogger: Logger) : MessageH abstract val isConnected: Boolean - private val incomingMessageQueue: PacketQueue = PacketQueue( + private val incomingMessageQueue: PacketQueue = PacketQueue( 50, true, INCOMING_MESSAGE_QUEUE_ID, @@ -47,7 +50,7 @@ abstract class AbstractEndpointMessageTransport(parentLogger: Logger) : MessageH }, TaskPools.IO_POOL, Clock.systemUTC() - ) + ).apply { setErrorHandler(queueErrorCounter) } /** * Fires the message transport ready event for the associated endpoint. @@ -97,5 +100,23 @@ abstract class AbstractEndpointMessageTransport(parentLogger: Logger) : MessageH companion object { const val INCOMING_MESSAGE_QUEUE_ID = "bridge-channel-message-incoming-queue" + private val droppedPacketsMetric = VideobridgeMetricsContainer.instance.registerCounter( + "endpoint_receive_message_queue_dropped_packets", + "Number of packets dropped out of the Endpoint receive message queue." + ) + private val exceptionsMetric = VideobridgeMetricsContainer.instance.registerCounter( + "endpoint_receive_message_queue_exceptions", + "Number of exceptions from the Endpoint receive message queue." + ) + val queueErrorCounter = object : CountingErrorHandler() { + override fun packetDropped() = super.packetDropped().also { + droppedPacketsMetric.inc() + QueueMetrics.droppedPackets.inc() + } + override fun packetHandlingFailed(t: Throwable?) = super.packetHandlingFailed(t).also { + exceptionsMetric.inc() + QueueMetrics.exceptions.inc() + } + } } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/ColibriQueue.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/ColibriQueue.kt new file mode 100644 index 0000000000..a491566cb6 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/ColibriQueue.kt @@ -0,0 +1,68 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge + +import org.jitsi.utils.queue.CountingErrorHandler +import org.jitsi.utils.queue.PacketQueue +import org.jitsi.videobridge.metrics.QueueMetrics +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer +import org.jitsi.videobridge.util.TaskPools +import org.jitsi.videobridge.xmpp.XmppConnection +import java.time.Clock +import kotlin.Int.Companion.MAX_VALUE + +abstract class ColibriQueue(packetHandler: PacketHandler) : + PacketQueue( + MAX_VALUE, + true, + QUEUE_NAME, + packetHandler, + TaskPools.IO_POOL, + // TODO: using the Videobridge clock breaks tests somehow + Clock.systemUTC(), + // Allow running tasks to complete (so we can close the queue from within the task). + false, + ) { + init { + setErrorHandler(queueErrorCounter) + } + + companion object { + val QUEUE_NAME = "colibri-queue" + + val droppedPacketsMetric = VideobridgeMetricsContainer.instance.registerCounter( + "colibri_queue_dropped_packets", + "Number of packets dropped out of the Colibri queue." + ) + + val exceptionsMetric = VideobridgeMetricsContainer.instance.registerCounter( + "colibri_queue_exceptions", + "Number of exceptions from the Colibri queue." + ) + + /** Count the number of dropped packets and exceptions. */ + val queueErrorCounter = object : CountingErrorHandler() { + override fun packetDropped() = super.packetDropped().also { + droppedPacketsMetric.inc() + QueueMetrics.droppedPackets.inc() + } + override fun packetHandlingFailed(t: Throwable?) = super.packetHandlingFailed(t).also { + exceptionsMetric.inc() + QueueMetrics.exceptions.inc() + } + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index 5ef171f061..3a2a56de2f 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -66,7 +66,9 @@ import org.jitsi.videobridge.message.BridgeChannelMessage import org.jitsi.videobridge.message.ForwardedSourcesMessage import org.jitsi.videobridge.message.ReceiverVideoConstraintsMessage import org.jitsi.videobridge.message.SenderSourceConstraintsMessage +import org.jitsi.videobridge.metrics.QueueMetrics import org.jitsi.videobridge.metrics.VideobridgeMetrics +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer import org.jitsi.videobridge.relay.AudioSourceDesc import org.jitsi.videobridge.relay.RelayedEndpoint import org.jitsi.videobridge.rest.root.debug.EndpointDebugFeatures @@ -1109,11 +1111,28 @@ class Endpoint @JvmOverloads constructor( */ private const val OPEN_DATA_CHANNEL_LOCALLY = false - /** - * Count the number of dropped packets and exceptions. - */ + private val droppedPacketsMetric = VideobridgeMetricsContainer.instance.registerCounter( + "srtp_send_queue_dropped_packets", + "Number of packets dropped out of the Endpoint SRTP send queue." + ) + + private val exceptionsMetric = VideobridgeMetricsContainer.instance.registerCounter( + "srtp_send_queue_exceptions", + "Number of exceptions from the Endpoint SRTP send queue." + ) + + /** Count the number of dropped packets and exceptions. */ @JvmField - val queueErrorCounter = CountingErrorHandler() + val queueErrorCounter = object : CountingErrorHandler() { + override fun packetDropped() = super.packetDropped().also { + droppedPacketsMetric.inc() + QueueMetrics.droppedPackets.inc() + } + override fun packetHandlingFailed(t: Throwable?) = super.packetHandlingFailed(t).also { + exceptionsMetric.inc() + QueueMetrics.exceptions.inc() + } + } /** * The executor which runs bandwidth probing. diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/JvmMetrics.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/JvmMetrics.kt new file mode 100644 index 0000000000..b9cca88af1 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/JvmMetrics.kt @@ -0,0 +1,106 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.metrics + +import com.sun.management.UnixOperatingSystemMXBean +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.utils.logging2.createLogger +import java.lang.management.ManagementFactory +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer.Companion.instance as metricsContainer + +class JvmMetrics private constructor() { + val logger = createLogger() + + private val gcType = ManagementFactory.getGarbageCollectorMXBeans().firstOrNull()?.name.let { + when { + it?.contains("shenandoah", ignoreCase = true) == true -> GcType.Shenandoah + it?.contains("zgc", ignoreCase = true) == true -> GcType.Zgc + it?.contains("g1", ignoreCase = true) == true -> GcType.G1 + else -> GcType.Other + } + }.also { + logger.info("Detected GC type $it") + } + + fun update() { + threadCount.set(ManagementFactory.getThreadMXBean().threadCount.toLong()) + gcCount.set( + ManagementFactory.getGarbageCollectorMXBeans().sumOf { it.collectionCount } + ) + gcTime.set( + ManagementFactory.getGarbageCollectorMXBeans().sumOf { it.collectionTime } + ) + (ManagementFactory.getOperatingSystemMXBean() as? UnixOperatingSystemMXBean)?.let { + openFdCount.set(it.openFileDescriptorCount) + } + if (gcType != GcType.Other) { + ManagementFactory.getMemoryPoolMXBeans().find { it.name == gcType.memoryPoolName }?.let { + heapUsed.set(it.usage.used) + heapCommitted.set(it.usage.committed) + } + } + } + + val threadCount = metricsContainer.registerLongGauge( + "thread_count", + "Current number of JVM threads." + ) + + private val gcCount = metricsContainer.registerLongGauge( + "jvm_gc_count", + "Garbage collection count." + ) + + private val gcTime = metricsContainer.registerLongGauge( + "jvm_gc_time", + "Garbage collection time." + ) + + private val heapCommitted = metricsContainer.registerLongGauge( + "jvm_heap_committed", + "Capacity of the main memory pool for the heap (GC type specific)." + ) + + private val heapUsed = metricsContainer.registerLongGauge( + "jvm_heap_used", + "Usage of the main memory pool for the heap (GC type specific)." + ) + + private val openFdCount = metricsContainer.registerLongGauge( + "jvm_open_fd_count", + "Number of open file descriptors." + ) + + private enum class GcType( + /** The name of the memory pool we're interested with this type of GC */ + val memoryPoolName: String? + ) { + G1("G1 Old Gen"), + Zgc("ZHeap"), + Shenandoah("Shenandoah"), + Other(null) + } + + companion object { + val enable: Boolean by config { + "videobridge.stats.jvm.enabled".from(JitsiConfig.newConfig) + } + + val INSTANCE = if (enable) JvmMetrics() else null + fun update() = INSTANCE?.update() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/Metrics.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/Metrics.kt index 61944800b3..e7eb2cff9b 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/Metrics.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/Metrics.kt @@ -40,7 +40,12 @@ object Metrics { val lock: Any get() = metricsUpdater - fun start() = metricsUpdater.addUpdateTask { ThreadsMetric.update() } + fun start() { + if (JvmMetrics.enable) { + metricsUpdater.addUpdateTask { JvmMetrics.update() } + } + QueueMetrics.init() + } fun stop() { metricsUpdater.stop() executor.shutdown() diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/QueueMetrics.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/QueueMetrics.kt new file mode 100644 index 0000000000..ceab84561a --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/QueueMetrics.kt @@ -0,0 +1,73 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.metrics + +import org.jitsi.nlj.RtpReceiverImpl +import org.jitsi.nlj.RtpSenderImpl +import org.jitsi.utils.queue.CountingErrorHandler +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer.Companion.instance as metricsContainer + +object QueueMetrics { + private val rtpReceiverDroppedPackets = metricsContainer.registerCounter( + "rtp_receiver_dropped_packets", + "Number of packets dropped out of the RTP receiver queue." + ) + private val rtpReceiverExceptions = metricsContainer.registerCounter( + "rtp_receiver_exceptions", + "Number of exceptions from the RTP receiver queue." + ) + private val rtpSenderDroppedPackets = metricsContainer.registerCounter( + "rtp_sender_dropped_packets", + "Number of packets dropped out of the RTP sender queue." + ) + private val rtpSenderExceptions = metricsContainer.registerCounter( + "rtp_sender_exceptions", + "Number of exceptions from the RTP sender queue." + ) + + val droppedPackets = metricsContainer.registerCounter( + "queue_dropped_packets", + "Number of packets dropped from any of the queues." + ) + val exceptions = metricsContainer.registerCounter( + "queue_exceptions", + "Number of exceptions from any of the queues." + ) + + fun init() { + RtpReceiverImpl.queueErrorCounter = object : CountingErrorHandler() { + override fun packetDropped() = super.packetDropped().also { + rtpReceiverDroppedPackets.inc() + droppedPackets.inc() + } + override fun packetHandlingFailed(t: Throwable?) = super.packetHandlingFailed(t).also { + rtpReceiverExceptions.inc() + exceptions.inc() + } + } + + RtpSenderImpl.queueErrorCounter = object : CountingErrorHandler() { + override fun packetDropped() = super.packetDropped().also { + rtpSenderDroppedPackets.inc() + droppedPackets.inc() + } + override fun packetHandlingFailed(t: Throwable?) = super.packetHandlingFailed(t).also { + rtpSenderExceptions.inc() + exceptions.inc() + } + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/ThreadsMetric.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/ThreadsMetric.kt deleted file mode 100644 index 978d290bbd..0000000000 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/ThreadsMetric.kt +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright @ 2024 - present 8x8, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.jitsi.videobridge.metrics - -import java.lang.management.ManagementFactory - -object ThreadsMetric { - fun update() { - threadCount.set(ManagementFactory.getThreadMXBean().threadCount.toLong()) - } - - val threadCount = VideobridgeMetricsContainer.instance.registerLongGauge( - "thread_count", - "Current number of JVM threads." - ) -} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetrics.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetrics.kt index d5c1a931d1..6012b1873f 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetrics.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetrics.kt @@ -268,4 +268,11 @@ object VideobridgeMetrics { } else { null } + + /** Just set once to allow detection of restarts */ + private val startupTime = metricsContainer.registerLongGauge( + "startup_time", + "The startup time of the service.", + System.currentTimeMillis() + ) } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt index 72ec37e5ae..7caca61aae 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt @@ -78,7 +78,9 @@ import org.jitsi.videobridge.datachannel.protocol.DataChannelPacket import org.jitsi.videobridge.datachannel.protocol.DataChannelProtocolConstants import org.jitsi.videobridge.message.BridgeChannelMessage import org.jitsi.videobridge.message.SourceVideoTypeMessage +import org.jitsi.videobridge.metrics.QueueMetrics import org.jitsi.videobridge.metrics.VideobridgeMetrics +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer import org.jitsi.videobridge.rest.root.debug.EndpointDebugFeatures import org.jitsi.videobridge.sctp.DataChannelHandler import org.jitsi.videobridge.sctp.SctpHandler @@ -1135,11 +1137,29 @@ class Relay @JvmOverloads constructor( } companion object { - /** - * Count the number of dropped packets and exceptions. - */ + private val droppedPacketsMetric = VideobridgeMetricsContainer.instance.registerCounter( + "relay_srtp_send_queue_dropped_packets", + "Number of packets dropped out of the Relay SRTP send queue." + ) + + private val exceptionsMetric = VideobridgeMetricsContainer.instance.registerCounter( + "relay_srtp_send_queue_exceptions", + "Number of exceptions from the Relay SRTP send queue." + ) + + /** Count the number of dropped packets and exceptions. */ @JvmField - val queueErrorCounter = CountingErrorHandler() + val queueErrorCounter = object : CountingErrorHandler() { + override fun packetDropped() = super.packetDropped().also { + droppedPacketsMetric.inc() + QueueMetrics.droppedPackets.inc() + } + + override fun packetHandlingFailed(t: Throwable?) = super.packetHandlingFailed(t).also { + exceptionsMetric.inc() + QueueMetrics.exceptions.inc() + } + } private const val SRTP_QUEUE_ENTRY_EVENT = "Entered Relay SRTP sender outgoing queue" private const val SRTP_QUEUE_EXIT_EVENT = "Exited Relay SRTP sender outgoing queue" diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayEndpointSender.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayEndpointSender.kt index 1c56d27424..1bee9c3ebc 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayEndpointSender.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayEndpointSender.kt @@ -37,6 +37,8 @@ import org.jitsi.utils.logging2.cdebug import org.jitsi.utils.logging2.createChildLogger import org.jitsi.utils.queue.CountingErrorHandler import org.jitsi.videobridge.TransportConfig +import org.jitsi.videobridge.metrics.QueueMetrics +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer import org.jitsi.videobridge.transport.ice.IceTransport import org.jitsi.videobridge.util.TaskPools import org.json.simple.JSONObject @@ -163,11 +165,29 @@ class RelayEndpointSender( } companion object { - /** - * Count the number of dropped packets and exceptions. - */ + private val droppedPacketsMetric = VideobridgeMetricsContainer.instance.registerCounter( + "relay_endpoint_srtp_send_queue_dropped_packets", + "Number of packets dropped out of the Relay SRTP send queue." + ) + + private val exceptionsMetric = VideobridgeMetricsContainer.instance.registerCounter( + "relay_endpoint_srtp_send_queue_exceptions", + "Number of exceptions from the Relay SRTP send queue." + ) + + /** Count the number of dropped packets and exceptions. */ @JvmField - val queueErrorCounter = CountingErrorHandler() + val queueErrorCounter = object : CountingErrorHandler() { + override fun packetDropped() = super.packetDropped().also { + droppedPacketsMetric.inc() + QueueMetrics.droppedPackets.inc() + } + + override fun packetHandlingFailed(t: Throwable?) = super.packetHandlingFailed(t).also { + exceptionsMetric.inc() + QueueMetrics.exceptions.inc() + } + } private const val SRTP_QUEUE_ENTRY_EVENT = "Entered RelayEndpointSender SRTP sender outgoing queue" } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/VideobridgeStatisticsShim.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/VideobridgeStatisticsShim.kt index 6b481bcfa1..7007fd7c3b 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/VideobridgeStatisticsShim.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/VideobridgeStatisticsShim.kt @@ -20,8 +20,8 @@ import org.jitsi.videobridge.EndpointConnectionStatusMonitor import org.jitsi.videobridge.VersionConfig import org.jitsi.videobridge.health.JvbHealthChecker import org.jitsi.videobridge.load_management.JvbLoadManager +import org.jitsi.videobridge.metrics.JvmMetrics import org.jitsi.videobridge.metrics.Metrics -import org.jitsi.videobridge.metrics.ThreadsMetric import org.jitsi.videobridge.metrics.VideobridgeMetrics import org.jitsi.videobridge.metrics.VideobridgePeriodicMetrics import org.jitsi.videobridge.relay.RelayConfig @@ -204,7 +204,9 @@ object VideobridgeStatisticsShim { put("average_participant_stress", JvbLoadManager.averageParticipantStress) - put(THREADS, ThreadsMetric.threadCount.get()) + JvmMetrics.INSTANCE?.threadCount?.let { + put(THREADS, it.get()) + } put(SHUTDOWN_IN_PROGRESS, VideobridgeMetrics.gracefulShutdown.get()) put("shutting_down", VideobridgeMetrics.shuttingDown.get()) diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index 94eaeddb0e..7418d9c000 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -236,6 +236,10 @@ videobridge { // /debug/jvb/stats/transit-time enable-jitter = false } + jvm { + // Whether to enable collection of JVM metrics (thread count, garbage collection) + enabled = true + } } websockets { enabled = false From 0bce9043871c933da7f0f9c042d7eb70e851d14b Mon Sep 17 00:00:00 2001 From: bgrozev Date: Mon, 20 May 2024 14:01:42 -0700 Subject: [PATCH 111/189] fix: Remove mappings when an endpoint expires. (#2132) * ref: Use [] syntax to access map, cleanup. * fix: Remove mappings when an endpoint expires. With SSRC rewriting source mappings are not removed when an endpoint expires. This results in duplicate entries in the other endpoints' mappings when an endpoint does an ICE restart (i.e. is recreated with the same ID). Subsequently, if an endpoint's WS reconnects, we signal all mapping including duplicates which may cause a failure to playback/render some sources. --- .../org/jitsi/videobridge/Conference.java | 2 +- .../org/jitsi/videobridge/AbstractEndpoint.kt | 5 ++ .../kotlin/org/jitsi/videobridge/Endpoint.kt | 9 ++++ .../kotlin/org/jitsi/videobridge/SsrcCache.kt | 50 +++++++++---------- 4 files changed, 39 insertions(+), 27 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/Conference.java b/jvb/src/main/java/org/jitsi/videobridge/Conference.java index 95bb08a5ea..f133689b9a 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Conference.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Conference.java @@ -922,7 +922,7 @@ public void endpointExpired(AbstractEndpoint endpoint) { // The removed endpoint was a local Endpoint as opposed to a RelayedEndpoint. updateEndpointsCache(); - endpointsById.forEach((i, senderEndpoint) -> senderEndpoint.removeReceiver(id)); + endpointsById.values().forEach(e -> e.otherEndpointExpired(removedEndpoint)); videobridge.localEndpointExpired(removedEndpoint.getVisitor()); } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt index b573eb561f..c801c19608 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt @@ -296,6 +296,11 @@ abstract class AbstractEndpoint protected constructor( } } + /** Notify this endpoint that another endpoint expired */ + open fun otherEndpointExpired(expired: AbstractEndpoint) { + removeReceiver(expired.id) + } + /** * Notifies this instance that the specified receiver no longer wants or * needs to receive anything from the endpoint attached to this diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index 3a2a56de2f..3ee8c6b4cf 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -940,6 +940,15 @@ class Endpoint @JvmOverloads constructor( return false } + override fun otherEndpointExpired(expired: AbstractEndpoint) = super.otherEndpointExpired(expired).also { + if (doSsrcRewriting) { + // Remove the expired endpoint's sources. We don't want duplicate entries in case the endpoint is + // re-created (e.g. after an ICE restart). + audioSsrcs.removeByOwner(expired.id) + videoSsrcs.removeByOwner(expired.id) + } + } + fun setLastN(lastN: Int) { bitrateController.lastN = lastN } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt index 2e691ea36e..fc6dd1aeff 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt @@ -295,17 +295,6 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: */ private var remapCount = 0 - companion object { - /** - * Print packet fields relevant to rewriting mode. - */ - private fun debugInfo(packet: RtpPacket): String { - val codecState = packet.getCodecState() - val codecInfo = codecState?.toString() ?: "" - return "ssrc=${packet.ssrc} seq=${packet.sequenceNumber} ts=${packet.timestamp}" + codecInfo - } - } - /** * Find the properties of the source indicated by the given SSRC. Returns null if not found. */ @@ -319,7 +308,6 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: /** * Assign a group of send SSRCs to use for the specified source. * If remapping the send SSRCs from another source, transfer RTP state from the old source. - * Collect any remapped sources into the provided list. * Returns null if no current mapping exists and allowCreate is false. * Otherwise, returns the send source information to use. */ @@ -327,10 +315,11 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: ssrc: Long, props: SourceDesc, allowCreate: Boolean, - remappings: MutableList + /* Collect remapped sources into this list, if provided. */ + remappings: MutableList? = null ): SendSource? { /* Moves to end of LRU when found. */ - var sendSource = sendSources.get(ssrc) + var sendSource = sendSources[ssrc] if (sendSource == null) { if (!allowCreate) { @@ -341,9 +330,9 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: sendSource = SendSource(props, eldest.value.send1, eldest.value.send2) logger.debug { "Remapping SSRC: ${props.ssrc1}->$sendSource. ${eldest.key}->inactive" } /* Request new deltas on next sent packet */ - receivedSsrcs.get(props.ssrc1)?.hasDeltas = false + receivedSsrcs[props.ssrc1]?.hasDeltas = false if (props.ssrc2 != -1L) { - receivedSsrcs.get(props.ssrc2)?.hasDeltas = false + receivedSsrcs[props.ssrc2]?.hasDeltas = false } ++remapCount } else { @@ -352,13 +341,25 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: sendSource = SendSource(props, ssrc1, ssrc2) logger.debug { "Added send SSRC: ${props.ssrc1}->$sendSource" } } - sendSources.put(ssrc, sendSource) - remappings.add(sendSource) + sendSources[ssrc] = sendSource + remappings?.add(sendSource) } return sendSource } + /** + * Remove all send sources owned by [owner]. Does not signal anything to the client, as only additions need to be + * signaled. + */ + fun removeByOwner(owner: String) { + synchronized(sendSources) { + sendSources.values.removeIf { sendSource -> + sendSource.props.owner == owner + } + } + } + /** * Assign send SSRCs to the given sources. Any remapped SSRCs will be notified to the client. */ @@ -370,7 +371,7 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: touches the already active sources, to prevent them from being bumped out of the LRU by earlier elements of the list. */ sources.filter { source -> - sendSources.get(source.primarySSRC) == null + sendSources[source.primarySSRC] == null }.forEach { source -> getSendSource(source.primarySSRC, SourceDesc(source), allowCreate = true, remappings) } @@ -411,11 +412,11 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: var send = false synchronized(sendSources) { - var rs = receivedSsrcs.get(packet.ssrc) + var rs = receivedSsrcs[packet.ssrc] if (rs == null) { val props = findSourceProps(packet.ssrc) ?: return false rs = ReceiveSsrc(props) - receivedSsrcs.put(packet.ssrc, rs) + receivedSsrcs[packet.ssrc] = rs logger.debug { "Added receive SSRC: ${packet.ssrc}" } } @@ -437,13 +438,10 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: * For packets in the same direction as media flow; feedback messages handled separately. */ fun rewriteRtcp(packet: RtcpPacket): Boolean { - val remappings = mutableListOf() // unused - val senderSsrc = packet.senderSsrc - synchronized(sendSources) { /* Don't activate a source on RTCP. */ - val rs = receivedSsrcs.get(packet.senderSsrc) ?: return false - val ss = getSendSource(rs.props.ssrc1, rs.props, allowCreate = false, remappings) ?: return false + val rs = receivedSsrcs[packet.senderSsrc] ?: return false + val ss = getSendSource(rs.props.ssrc1, rs.props, allowCreate = false) ?: return false ss.rewriteRtcp(packet) return true } From 5524b852480c4d294418380903b99a7b2f6958af Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Wed, 22 May 2024 14:42:25 -0400 Subject: [PATCH 112/189] Maintain a VideoCodecParser per source, not per endpoint. (#2133) --- .../kotlin/org/jitsi/nlj/MediaSourceDesc.kt | 7 ++ .../jitsi/nlj/rtp/codec/VideoCodecParser.kt | 19 +-- .../jitsi/nlj/rtp/codec/av1/Av1DDParser.kt | 12 +- .../org/jitsi/nlj/rtp/codec/vp8/Vp8Parser.kt | 4 +- .../org/jitsi/nlj/rtp/codec/vp9/Vp9Parser.kt | 12 +- .../transform/node/incoming/VideoParser.kt | 111 +++++++++++------- 6 files changed, 94 insertions(+), 71 deletions(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt index 7256df7659..2a6fb30188 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt @@ -19,6 +19,7 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import org.jitsi.nlj.rtp.VideoRtpPacket import org.jitsi.nlj.util.Bandwidth import org.jitsi.nlj.util.bps +import org.jitsi.rtp.rtp.RtpPacket import org.jitsi.utils.ArrayUtils import java.util.Collections import java.util.NavigableMap @@ -190,6 +191,12 @@ fun Array.findRtpLayerDescs(packet: VideoRtpPacket): Collection return this.flatMap { it.findRtpLayerDescs(packet) } } +fun Array.findRtpSource(ssrc: Long): MediaSourceDesc? { + return this.find { it.matches(ssrc) } +} + +fun Array.findRtpSource(packet: RtpPacket): MediaSourceDesc? = findRtpSource(packet.ssrc) + fun Array.findRtpEncodingId(packet: VideoRtpPacket): Int? { for (source in this) { source.findRtpEncodingDesc(packet.ssrc)?.let { diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/VideoCodecParser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/VideoCodecParser.kt index 28fad3f251..623bb516d4 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/VideoCodecParser.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/VideoCodecParser.kt @@ -29,27 +29,16 @@ import org.jitsi.nlj.rtp.VideoRtpPacket * and verify stream consistency. */ abstract class VideoCodecParser( - var sources: Array + var source: MediaSourceDesc ) { abstract fun parse(packetInfo: PacketInfo) protected fun findRtpEncodingDesc(packet: VideoRtpPacket): RtpEncodingDesc? { - for (source in sources) { - source.findRtpEncodingDesc(packet.ssrc)?.let { - return it - } + source.findRtpEncodingDesc(packet.ssrc)?.let { + return it } return null } - protected fun findSourceDescAndRtpEncodingDesc(packet: VideoRtpPacket): Pair? { - for (source in sources) { - source.findRtpEncodingDesc(packet.ssrc)?.let { - return Pair(source, it) - } - } - return null - } - - protected fun findRtpLayerDescs(packet: VideoRtpPacket) = sources.findRtpLayerDescs(packet) + protected fun findRtpLayerDescs(packet: VideoRtpPacket) = source.findRtpLayerDescs(packet) } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDParser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDParser.kt index db169f1a76..0a786636f9 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDParser.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDParser.kt @@ -36,10 +36,10 @@ import org.jitsi.utils.logging2.createChildLogger * won't be able to route. */ class Av1DDParser( - sources: Array, + source: MediaSourceDesc, parentLogger: Logger, private val diagnosticContext: DiagnosticContext -) : VideoCodecParser(sources) { +) : VideoCodecParser(source) { private val logger = createChildLogger(parentLogger) /** History of AV1 templates. */ @@ -121,13 +121,13 @@ class Av1DDParser( "now 0x${Integer.toHexString(activeDecodeTargets)}. Updating layering." } - findSourceDescAndRtpEncodingDesc(av1Packet)?.let { (src, enc) -> + findRtpEncodingDesc(av1Packet)?.let { enc -> av1Packet.getScalabilityStructure(eid = enc.eid)?.let { - src.setEncodingLayers(it.layers, av1Packet.ssrc) + source.setEncodingLayers(it.layers, av1Packet.ssrc) } - for (otherEnc in src.rtpEncodings) { + for (otherEnc in source.rtpEncodings) { if (!ddStateHistory.keys.contains(otherEnc.primarySSRC)) { - src.setEncodingLayers(emptyArray(), otherEnc.primarySSRC) + source.setEncodingLayers(emptyArray(), otherEnc.primarySSRC) } } } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Parser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Parser.kt index 024f45d683..86f92e97d4 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Parser.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Parser.kt @@ -30,9 +30,9 @@ import org.jitsi.utils.logging2.createChildLogger * from frames, and also diagnoses packet format variants that the Jitsi videobridge won't be able to route. */ class Vp8Parser( - sources: Array, + source: MediaSourceDesc, parentLogger: Logger -) : VideoCodecParser(sources) { +) : VideoCodecParser(source) { private val logger = createChildLogger(parentLogger) // Consistency diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Parser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Parser.kt index 04518ddbf6..d1fa5a9493 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Parser.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Parser.kt @@ -31,9 +31,9 @@ import org.jitsi.utils.logging2.createChildLogger * from frames, and also diagnoses packet format variants that the Jitsi videobridge won't be able to route. */ class Vp9Parser( - sources: Array, + source: MediaSourceDesc, parentLogger: Logger -) : VideoCodecParser(sources) { +) : VideoCodecParser(source) { private val logger = createChildLogger(parentLogger) private val pictureIdState = StateChangeLogger("missing picture id", logger) @@ -58,13 +58,13 @@ class Vp9Parser( } numSpatialLayers = packetSpatialLayers } - findSourceDescAndRtpEncodingDesc(vp9Packet)?.let { (src, enc) -> + findRtpEncodingDesc(vp9Packet)?.let { enc -> vp9Packet.getScalabilityStructure(eid = enc.eid)?.let { - src.setEncodingLayers(it.layers, vp9Packet.ssrc) + source.setEncodingLayers(it.layers, vp9Packet.ssrc) } - for (otherEnc in src.rtpEncodings) { + for (otherEnc in source.rtpEncodings) { if (!ssrcsSeen.contains(otherEnc.primarySSRC)) { - src.setEncodingLayers(emptyArray(), otherEnc.primarySSRC) + source.setEncodingLayers(emptyArray(), otherEnc.primarySSRC) } } } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoParser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoParser.kt index 685c70639a..850e39fa9d 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoParser.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoParser.kt @@ -19,6 +19,7 @@ import org.jitsi.nlj.Event import org.jitsi.nlj.MediaSourceDesc import org.jitsi.nlj.PacketInfo import org.jitsi.nlj.SetMediaSourcesEvent +import org.jitsi.nlj.findRtpSource import org.jitsi.nlj.format.Vp8PayloadType import org.jitsi.nlj.format.Vp9PayloadType import org.jitsi.nlj.rtp.RtpExtensionType @@ -55,7 +56,7 @@ class VideoParser( private var av1DDExtId: Int? = null - private var videoCodecParser: VideoCodecParser? = null + private val videoCodecParsers = mutableMapOf() init { streamInformationStore.onRtpExtensionMapping(RtpExtensionType.AV1_DEPENDENCY_DESCRIPTOR) { @@ -71,6 +72,7 @@ class VideoParser( stats.numPacketsDroppedUnknownPt++ return null } + val videoCodecParser: VideoCodecParser? val parsedPacket = try { when { payloadType is Vp8PayloadType -> { @@ -78,14 +80,10 @@ class VideoParser( packetInfo.packet = vp8Packet packetInfo.resetPayloadVerification() - if (videoCodecParser !is Vp8Parser) { - logger.cdebug { - "Creating new VP8Parser, current videoCodecParser is ${videoCodecParser?.javaClass}" - } - resetSources() - packetInfo.layeringChanged = true - videoCodecParser = Vp8Parser(sources, logger) + videoCodecParser = checkParserType(packetInfo) { source -> + Vp8Parser(source, logger) } + vp8Packet } payloadType is Vp9PayloadType -> { @@ -93,39 +91,37 @@ class VideoParser( packetInfo.packet = vp9Packet packetInfo.resetPayloadVerification() - if (videoCodecParser !is Vp9Parser) { - logger.cdebug { - "Creating new VP9Parser, current videoCodecParser is ${videoCodecParser?.javaClass}" - } - resetSources() - packetInfo.layeringChanged = true - videoCodecParser = Vp9Parser(sources, logger) + videoCodecParser = checkParserType(packetInfo) { source -> + Vp9Parser(source, logger) } + vp9Packet } av1DDExtId != null && packet.getHeaderExtension(av1DDExtId) != null -> { - if (videoCodecParser !is Av1DDParser) { - logger.cdebug { - "Creating new Av1DDParser, current videoCodecParser is ${videoCodecParser?.javaClass}" - } - resetSources() - packetInfo.layeringChanged = true - videoCodecParser = Av1DDParser(sources, logger, diagnosticContext) + videoCodecParser = checkParserType(packetInfo) { source -> + Av1DDParser(source, logger, diagnosticContext) } - val av1DDPacket = (videoCodecParser as Av1DDParser).createFrom(packet, av1DDExtId) - packetInfo.packet = av1DDPacket - packetInfo.resetPayloadVerification() + val av1DDPacket = videoCodecParser?.createFrom(packet, av1DDExtId)?.also { + packetInfo.packet = it + packetInfo.resetPayloadVerification() + } av1DDPacket } else -> { - if (videoCodecParser != null) { + val curParser = videoCodecParsers[packet.ssrc] + if (curParser != null) { logger.cdebug { "Removing videoCodecParser on ${payloadType.javaClass} packet, " + - "current videoCodecParser is ${videoCodecParser?.javaClass}" + "current videoCodecParser is ${curParser.javaClass}" + } + sources.findRtpSource(packet)?.let { source -> + resetSource(source) + source.rtpEncodings.forEach { + videoCodecParsers.remove(it.primarySSRC) + } } - resetSources() packetInfo.layeringChanged = true videoCodecParser = null } @@ -146,7 +142,7 @@ class VideoParser( /* Some codecs mark keyframes in every packet of the keyframe - only count the start of the frame, * so the count is correct. */ /* Alternately we could keep track of keyframes we've already seen, by timestamp, but that seems unnecessary. */ - if (parsedPacket.isKeyframe && parsedPacket.isStartOfFrame) { + if (parsedPacket != null && parsedPacket.isKeyframe && parsedPacket.isStartOfFrame) { logger.cdebug { "Received a keyframe for ssrc ${packet.ssrc} ${packet.sequenceNumber}" } stats.numKeyframes++ } @@ -158,29 +154,60 @@ class VideoParser( return packetInfo } + private inline fun checkParserType( + packetInfo: PacketInfo, + constructor: (MediaSourceDesc) -> T + ): T? { + val packet = packetInfo.packetAs() + val parser = videoCodecParsers[packet.ssrc] + if (parser is T) { + return parser + } + + val source = sources.findRtpSource(packet) + ?: // VideoQualityLayerLookup will drop this packet later, so no need to warn about it now + return null + logger.cdebug { + "Creating new ${T::class.java} for source ${source.sourceName}, " + + "current videoCodecParser is ${parser?.javaClass}" + } + resetSource(source) + packetInfo.layeringChanged = true + val newParser = constructor(source) + source.rtpEncodings.forEach { + videoCodecParsers[it.primarySSRC] = newParser + } + + return newParser + } + override fun handleEvent(event: Event) { when (event) { is SetMediaSourcesEvent -> { sources = event.mediaSourceDescs signaledSources = event.signaledMediaSourceDescs - videoCodecParser?.sources = sources + val ssrcsSeen = mutableSetOf() + sources.forEach { source -> + source.rtpEncodings.forEach { + videoCodecParsers[it.primarySSRC]?.source = source + ssrcsSeen.add(it.primarySSRC) + } + } + videoCodecParsers.keys.removeIf { !ssrcsSeen.contains(it) } } } super.handleEvent(event) } - private fun resetSources() { - logger.cdebug { "Resetting sources to signaled sources: ${signaledSources.joinToString(separator = "\n")}" } - for (signaledSource in signaledSources) { - for (source in sources) { - if (source.primarySSRC != signaledSource.primarySSRC) { - continue - } - for (signaledEncoding in signaledSource.rtpEncodings) { - source.setEncodingLayers(signaledEncoding.layers, signaledEncoding.primarySSRC) - } - break - } + private fun resetSource(source: MediaSourceDesc) { + val signaledSource = signaledSources.findRtpSource(source.primarySSRC) + if (signaledSource == null) { + logger.warn("Unable to find signaled source corresponding to ${source.primarySSRC}") + return + } + logger.cdebug { "Resetting source ${source.sourceName} to signaled source: $signaledSource" } + for (signaledEncoding in signaledSource.rtpEncodings) { + source.setEncodingLayers(signaledEncoding.layers, signaledEncoding.primarySSRC) } } From 2875cc4720a1f6adfe206eb9215e57d5beba9fd0 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Wed, 22 May 2024 15:14:27 -0400 Subject: [PATCH 113/189] Support using dcsctp4j for SCTP. (#2131) --------- Co-authored-by: bgrozev --- jvb/pom.xml | 5 + .../protocol/DataChannelPacket.java | 8 + .../jitsi/videobridge/sctp/SctpManager.java | 3 +- .../kotlin/org/jitsi/videobridge/Endpoint.kt | 184 ++++++++++++----- .../colibri2/Colibri2ConferenceHandler.kt | 10 +- .../jitsi/videobridge/dcsctp/DcSctpHandler.kt | 69 +++++++ .../videobridge/dcsctp/DcSctpTransport.kt | 163 +++++++++++++++ .../org/jitsi/videobridge/relay/Relay.kt | 189 +++++++++++++++--- .../org/jitsi/videobridge/sctp/SctpConfig.kt | 8 +- jvb/src/main/resources/reference.conf | 3 + pom.xml | 2 +- 11 files changed, 564 insertions(+), 80 deletions(-) create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpHandler.kt create mode 100644 jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpTransport.kt diff --git a/jvb/pom.xml b/jvb/pom.xml index 7cf98e8893..484f50dc35 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -110,6 +110,11 @@ jitsi-sctp 1.0-21-gfe0d028 + + ${project.groupId} + jitsi-dcsctp + 1.0-1-ga91e110 + diff --git a/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelPacket.java b/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelPacket.java index 96507b8d95..10f51aa897 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelPacket.java +++ b/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelPacket.java @@ -17,6 +17,7 @@ package org.jitsi.videobridge.datachannel.protocol; import org.jetbrains.annotations.*; +import org.jitsi.dcsctp4j.DcSctpMessage; import org.jitsi.rtp.*; /** @@ -38,6 +39,13 @@ public DataChannelPacket( this.ppid = ppid; } + public DataChannelPacket(DcSctpMessage message) + { + super(message.getPayload(), 0, message.getPayload().length); + this.sid = message.getStreamID(); + this.ppid = message.getPpid(); + } + /** * {@inheritDoc} */ diff --git a/jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java b/jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java index 72e0cacce8..25fdd353d0 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java +++ b/jvb/src/main/java/org/jitsi/videobridge/sctp/SctpManager.java @@ -19,6 +19,7 @@ import org.jetbrains.annotations.*; import org.jitsi.nlj.*; import org.jitsi.utils.logging2.*; +import org.jitsi.videobridge.dcsctp.*; import org.jitsi.videobridge.util.*; import org.jitsi_modified.sctp4j.*; @@ -54,7 +55,7 @@ public class SctpManager /** * We always use symmetric ports with SCTP (local port = remote port). */ - public static int DEFAULT_SCTP_PORT = 5000; + public static int DEFAULT_SCTP_PORT = DcSctpTransport.DEFAULT_SCTP_PORT; static { if (config.enabled()) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index 3ee8c6b4cf..ae893252ed 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -17,6 +17,10 @@ package org.jitsi.videobridge import org.jitsi.config.JitsiConfig +import org.jitsi.dcsctp4j.DcSctpMessage +import org.jitsi.dcsctp4j.ErrorKind +import org.jitsi.dcsctp4j.SendPacketStatus +import org.jitsi.dcsctp4j.SendStatus import org.jitsi.metaconfig.config import org.jitsi.nlj.Features import org.jitsi.nlj.MediaSourceDesc @@ -61,7 +65,9 @@ import org.jitsi.videobridge.cc.allocation.EffectiveConstraintsMap import org.jitsi.videobridge.cc.allocation.VideoConstraints import org.jitsi.videobridge.datachannel.DataChannelStack import org.jitsi.videobridge.datachannel.protocol.DataChannelPacket -import org.jitsi.videobridge.datachannel.protocol.DataChannelProtocolConstants +import org.jitsi.videobridge.dcsctp.DcSctpBaseCallbacks +import org.jitsi.videobridge.dcsctp.DcSctpHandler +import org.jitsi.videobridge.dcsctp.DcSctpTransport import org.jitsi.videobridge.message.BridgeChannelMessage import org.jitsi.videobridge.message.ForwardedSourcesMessage import org.jitsi.videobridge.message.ReceiverVideoConstraintsMessage @@ -97,6 +103,9 @@ import java.time.Instant import java.util.Optional import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicLong +import kotlin.collections.ArrayList +import kotlin.collections.HashSet +import org.jitsi.videobridge.sctp.SctpConfig.Companion.config as sctpConfig /** * Models a local endpoint (participant) in a [Conference] @@ -121,13 +130,25 @@ class Endpoint @JvmOverloads constructor( PotentialPacketHandler, EncodingsManager.EncodingsUpdateListener, SsrcRewriter { + + /** The time at which this endpoint was created */ + val creationTime = clock.instant() + /** - * The time at which this endpoint was created + * The two SCTP implementations (using usrsctp and dcsctp) are implemented side by side here. The intention is + * for the new dcsctp one to replace the old one, which will eventually be removed. */ - val creationTime = clock.instant() + private val sctpHandler = if (sctpConfig.enabled && !sctpConfig.useUsrSctp) DcSctpHandler() else null + private val usrSctpHandler = if (sctpConfig.enabled && sctpConfig.useUsrSctp) SctpHandler() else null + + /** The [DcSctpTransport] instance we'll use to manage the SCTP connection */ + private var sctpTransport: DcSctpTransport? = null - private val sctpHandler = SctpHandler() + // usrsctp private val dataChannelHandler = DataChannelHandler() + private var sctpManager: SctpManager? = null + private var sctpSocket: Optional = Optional.empty() + private var dataChannelStack: DataChannelStack? = null private val toggleablePcapWriter = ToggleablePcapWriter(logger, "$id-sctp") private val sctpRecvPcap = toggleablePcapWriter.newObserverNode(outbound = false) @@ -135,7 +156,12 @@ class Endpoint @JvmOverloads constructor( private val sctpPipeline = pipeline { node(sctpRecvPcap) - node(sctpHandler) + sctpHandler?.let { + node(it) + } + usrSctpHandler?.let { + node(it) + } } /* TODO: do we ever want to support useUniquePort for an Endpoint? */ @@ -150,19 +176,6 @@ class Endpoint @JvmOverloads constructor( private val timelineLogger = logger.createChildLogger("timeline.${this.javaClass.name}") - /** - * The [SctpManager] instance we'll use to manage the SCTP connection - */ - private var sctpManager: SctpManager? = null - - private var dataChannelStack: DataChannelStack? = null - - /** - * The [SctpSocket] for this endpoint, if an SCTP connection was - * negotiated. - */ - private var sctpSocket: Optional = Optional.empty() - /** * Whether this endpoint should accept audio packets. We set this according * to whether the endpoint has an audio Colibri channel whose direction @@ -428,10 +441,8 @@ class Endpoint @JvmOverloads constructor( ) { logger.info("DTLS handshake complete") transceiver.setSrtpInformation(chosenSrtpProtectionProfile, tlsRole, keyingMaterial, cryptex) - // TODO(brian): the old code would work even if the sctp connection was created after - // the handshake had completed, but this won't (since this is a one-time event). do - // we need to worry about that case? - sctpSocket.ifPresent(::acceptSctpConnection) + /* If we were the client of the SCTP connection, we would start it here. */ + sctpSocket.ifPresent(::acceptUsrSctpConnection) scheduleEndpointMessageTransportTimeout() } } @@ -588,11 +599,31 @@ class Endpoint @JvmOverloads constructor( } /** - * Create an SCTP connection for this Endpoint. If [OPEN_DATA_CHANNEL_LOCALLY] is true, + * Create an SCTP connection for this Endpoint. If [OPEN_DATA_CHANNEL_LOCALLY] is true, * we will create the data channel locally, otherwise we will wait for the remote side * to open it. */ fun createSctpConnection() { + if (sctpConfig.enabled) { + if (sctpConfig.useUsrSctp) { + createUsrSctpConnection() + } else { + createDcSctpConnection() + } + } else { + logger.error("Not creating SCTP connection, SCTP is disabled in configuration.") + } + } + + private fun createDcSctpConnection() { + logger.cdebug { "Creating SCTP transport" } + sctpTransport = DcSctpTransport(id, logger).also { + it.start(SctpCallbacks(it)) + sctpHandler?.setSctpTransport(it) + } + } + + private fun createUsrSctpConnection() { logger.cdebug { "Creating SCTP manager" } // Create the SctpManager and provide it a method for sending SCTP data sctpManager = SctpManager( @@ -603,7 +634,7 @@ class Endpoint @JvmOverloads constructor( }, logger ) - sctpHandler.setSctpManager(sctpManager!!) + usrSctpHandler!!.setSctpManager(sctpManager!!) // NOTE(brian): as far as I know we always act as the 'server' for sctp // connections, but if not we can make which type we use dynamic val socket = sctpManager!!.createServerSocket(logger) @@ -620,21 +651,8 @@ class Endpoint @JvmOverloads constructor( messageTransport.setDataChannel(dataChannel) } dataChannelHandler.setDataChannelStack(dataChannelStack!!) - if (OPEN_DATA_CHANNEL_LOCALLY) { - // This logic is for opening the data channel locally - logger.info("Will open the data channel.") - val dataChannel = dataChannelStack!!.createDataChannel( - DataChannelProtocolConstants.RELIABLE, - 0, - 0, - 0, - "default" - ) - messageTransport.setDataChannel(dataChannel) - dataChannel.open() - } else { - logger.info("Will wait for the remote side to open the data channel.") - } + + logger.info("Will wait for the remote side to open the data channel.") } override fun onDisconnected() { @@ -654,7 +672,7 @@ class Endpoint @JvmOverloads constructor( sctpSocket = Optional.of(socket) } - fun acceptSctpConnection(sctpServerSocket: SctpServerSocket) { + fun acceptUsrSctpConnection(sctpServerSocket: SctpServerSocket) { TaskPools.IO_POOL.execute { // We don't want to block the thread calling // onDtlsHandshakeComplete so run the socket acceptance in an IO @@ -1066,6 +1084,9 @@ class Endpoint @JvmOverloads constructor( put("audioSsrcs", audioSsrcs.getDebugState()) put("videoSsrcs", videoSsrcs.getDebugState()) } + sctpTransport?.let { + put("sctp", it.getDebugState()) + } } override fun expire() { @@ -1087,8 +1108,10 @@ class Endpoint @JvmOverloads constructor( transceiver.teardown() messageTransport.close() - sctpHandler.stop() + sctpHandler?.stop() + usrSctpHandler?.stop() sctpManager?.closeConnection() + sctpTransport?.socket?.close() } catch (t: Throwable) { logger.error("Exception while expiring: ", t) } @@ -1114,12 +1137,6 @@ class Endpoint @JvmOverloads constructor( } companion object { - /** - * Whether or not the bridge should be the peer which opens the data channel - * (as opposed to letting the far peer/client open it). - */ - private const val OPEN_DATA_CHANNEL_LOCALLY = false - private val droppedPacketsMetric = VideobridgeMetricsContainer.instance.registerCounter( "srtp_send_queue_dropped_packets", "Number of packets dropped out of the Endpoint SRTP send queue." @@ -1181,6 +1198,79 @@ class Endpoint @JvmOverloads constructor( private val random = SecureRandom() } + private inner class SctpCallbacks(transport: DcSctpTransport) : DcSctpBaseCallbacks(transport) { + override fun sendPacketWithStatus(packet: ByteArray): SendPacketStatus { + try { + val newBuf = ByteBufferPool.getBuffer(packet.size) + System.arraycopy(packet, 0, newBuf, 0, packet.size) + + sctpSendPcap.observe(newBuf, 0, packet.size) + dtlsTransport.sendDtlsData(newBuf, 0, packet.size) + + return SendPacketStatus.kSuccess + } catch (e: Throwable) { + logger.warn("Exception sending SCTP packet", e) + return SendPacketStatus.kError + } + } + + override fun OnMessageReceived(message: DcSctpMessage) { + try { + // We assume all data coming over SCTP will be datachannel data + val dataChannelPacket = DataChannelPacket(message) + // Post the rest of the task here because the current context is + // holding a lock inside the SctpSocket which can cause a deadlock + // if two endpoints are trying to send datachannel messages to one + // another (with stats broadcasting it can happen often) + incomingDataChannelMessagesQueue.add(PacketInfo(dataChannelPacket)) + } catch (e: Throwable) { + logger.warn("Exception processing SCTP message", e) + } + } + + override fun OnError(error: ErrorKind, message: String) { + logger.warn("SCTP error $error: $message") + } + + override fun OnAborted(error: ErrorKind, message: String) { + logger.warn("SCTP aborted with error $error: $message") + } + + override fun OnConnected() { + try { + logger.info("SCTP connection is ready, creating the Data channel stack") + val dataChannelStack = DataChannelStack( + { data, sid, ppid -> + val message = DcSctpMessage(sid.toShort(), ppid, data.array()) + val status = sctpTransport?.socket?.send(message, DcSctpTransport.DEFAULT_SEND_OPTIONS) + return@DataChannelStack if (status == SendStatus.kSuccess) { + 0 + } else { + logger.error("Error sending to SCTP: $status") + -1 + } + }, + logger + ) + this@Endpoint.dataChannelStack = dataChannelStack + // This handles if the remote side will be opening the data channel + dataChannelStack.onDataChannelStackEvents { dataChannel -> + logger.info("Remote side opened a data channel.") + messageTransport.setDataChannel(dataChannel) + } + dataChannelHandler.setDataChannelStack(dataChannelStack) + logger.info("Will wait for the remote side to open the data channel.") + } catch (e: Throwable) { + logger.warn("Exception processing SCTP connected event", e) + } + } + + override fun OnClosed() { + // I don't think this should happen, except during shutdown. + logger.info("SCTP connection closed") + } + } + private inner class TransceiverEventHandlerImpl : TransceiverEventHandler { /** * Forward audio level events from the Transceiver to the conference. We use the same thread, because this fires diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt index 0e33906f70..c0f466e09d 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt @@ -26,11 +26,11 @@ import org.jitsi.utils.logging2.Logger import org.jitsi.utils.logging2.createChildLogger import org.jitsi.videobridge.AbstractEndpoint import org.jitsi.videobridge.Conference +import org.jitsi.videobridge.dcsctp.DcSctpTransport import org.jitsi.videobridge.relay.AudioSourceDesc import org.jitsi.videobridge.relay.Relay import org.jitsi.videobridge.relay.RelayConfig import org.jitsi.videobridge.sctp.SctpConfig -import org.jitsi.videobridge.sctp.SctpManager import org.jitsi.videobridge.util.PayloadTypeUtil.Companion.create import org.jitsi.videobridge.websocket.config.WebsocketServiceConfig import org.jitsi.videobridge.xmpp.MediaSourceFactory @@ -203,7 +203,7 @@ class Colibri2ConferenceHandler( "Unsupported SCTP role: ${sctp.role}" ) } - if (sctp.port != null && sctp.port != SctpManager.DEFAULT_SCTP_PORT) { + if (sctp.port != null && sctp.port != DcSctpTransport.DEFAULT_SCTP_PORT) { throw IqProcessingException( Condition.bad_request, "Specific SCTP port requested, not supported." @@ -260,7 +260,7 @@ class Colibri2ConferenceHandler( if (c2endpoint.transport?.sctp != null) { transBuilder.setSctp( Sctp.Builder() - .setPort(SctpManager.DEFAULT_SCTP_PORT) + .setPort(DcSctpTransport.DEFAULT_SCTP_PORT) .setRole(Sctp.Role.SERVER) .build() ) @@ -392,7 +392,7 @@ class Colibri2ConferenceHandler( "SCTP support is not configured" ) } - if (sctp.port != null && sctp.port != SctpManager.DEFAULT_SCTP_PORT) { + if (sctp.port != null && sctp.port != DcSctpTransport.DEFAULT_SCTP_PORT) { throw IqProcessingException( Condition.bad_request, "Specific SCTP port requested, not supported." @@ -414,7 +414,7 @@ class Colibri2ConferenceHandler( } transBuilder.setSctp( Sctp.Builder() - .setPort(SctpManager.DEFAULT_SCTP_PORT) + .setPort(DcSctpTransport.DEFAULT_SCTP_PORT) .setRole(role) .build() ) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpHandler.kt new file mode 100644 index 0000000000..f2cf2d72b9 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpHandler.kt @@ -0,0 +1,69 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.dcsctp + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.ConsumerNode +import org.jitsi.videobridge.sctp.SctpConfig +import org.jitsi.videobridge.util.TaskPools +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.atomic.AtomicLong + +/** + * A node which can be placed in the pipeline to cache SCTP packets until + * the DcSctpTransport is ready to handle them. + */ +class DcSctpHandler : ConsumerNode("SCTP handler") { + private val sctpTransportLock = Any() + private var sctpTransport: DcSctpTransport? = null + private val numCachedSctpPackets = AtomicLong(0) + private val cachedSctpPackets = LinkedBlockingQueue(100) + + override fun consume(packetInfo: PacketInfo) { + synchronized(sctpTransportLock) { + if (SctpConfig.config.enabled) { + sctpTransport?.handleIncomingSctp(packetInfo) ?: run { + numCachedSctpPackets.incrementAndGet() + cachedSctpPackets.add(packetInfo) + } + } + } + } + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + addNumber("num_cached_packets", numCachedSctpPackets.get()) + } + + fun setSctpTransport(sctpTransport: DcSctpTransport) { + // Submit this to the pool since we wait on the lock and process any + // cached packets here as well + TaskPools.IO_POOL.execute { + // We grab the lock here so that we can set the SCTP transport and + // process any previously-cached packets as an atomic operation. + // It also prevents another thread from coming in via + // #doProcessPackets and processing packets at the same time in + // another thread, which would be a problem. + synchronized(sctpTransportLock) { + this.sctpTransport = sctpTransport + cachedSctpPackets.forEach { sctpTransport.handleIncomingSctp(it) } + cachedSctpPackets.clear() + } + } + } + + override fun trace(f: () -> Unit) = f.invoke() +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpTransport.kt new file mode 100644 index 0000000000..a048c465a3 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpTransport.kt @@ -0,0 +1,163 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.dcsctp + +import org.jitsi.dcsctp4j.DcSctpOptions +import org.jitsi.dcsctp4j.DcSctpSocketCallbacks +import org.jitsi.dcsctp4j.DcSctpSocketFactory +import org.jitsi.dcsctp4j.DcSctpSocketInterface +import org.jitsi.dcsctp4j.SendOptions +import org.jitsi.dcsctp4j.Timeout +import org.jitsi.nlj.PacketInfo +import org.jitsi.utils.OrderedJsonObject +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.videobridge.sctp.SctpConfig +import org.jitsi.videobridge.util.TaskPools +import java.time.Clock +import java.time.Instant +import java.util.concurrent.Future +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.ThreadLocalRandom +import java.util.concurrent.TimeUnit + +class DcSctpTransport( + val name: String, + parentLogger: Logger +) { + val logger = createChildLogger(parentLogger) + lateinit var socket: DcSctpSocketInterface + + fun start(callbacks: DcSctpSocketCallbacks, options: DcSctpOptions = DEFAULT_SOCKET_OPTIONS) { + socket = factory.create(name, callbacks, null, options) + } + + fun handleIncomingSctp(packetInfo: PacketInfo) { + val packet = packetInfo.packet + socket.receivePacket(packet.getBuffer(), packet.getOffset(), packet.getLength()) + } + + fun getDebugState(): OrderedJsonObject { + val metrics = socket.metrics + return OrderedJsonObject().apply { + if (metrics != null) { + put("tx_packets_count", metrics.txPacketsCount) + put("tx_messages_count", metrics.txMessagesCount) + put("rtx_packets_count", metrics.rtxPacketsCount) + put("rtx_bytes_count", metrics.rtxBytesCount) + put("cwnd_bytes", metrics.cwndBytes) + put("srtt_ms", metrics.srttMs) + put("unack_data_count", metrics.unackDataCount) + put("rx_packets_count", metrics.rxPacketsCount) + put("rx_messages_count", metrics.rxMessagesCount) + put("peer_rwnd_bytes", metrics.peerRwndBytes) + put("peer_implementation", metrics.peerImplementation.name) + put("uses_message_interleaving", metrics.usesMessageInterleaving()) + put("uses_zero_checksum", metrics.usesZeroChecksum()) + put("negotiated_maximum_incoming_streams", metrics.negotiatedMaximumIncomingStreams) + put("negotiated_maximum_outgoing_streams", metrics.negotiatedMaximumOutgoingStreams) + } + } + } + + companion object { + private val factory by lazy { + check(SctpConfig.config.enabled()) { "SCTP is disabled in configuration" } + DcSctpSocketFactory() + } + + /* Copying value set by Chrome's dcsctp_transport. */ + const val DEFAULT_MAX_TIMER_DURATION = 3000L + + val DEFAULT_SOCKET_OPTIONS by lazy { + check(SctpConfig.config.enabled()) { "SCTP is disabled in configuration" } + DcSctpOptions().apply { + maxTimerBackoffDuration = DEFAULT_MAX_TIMER_DURATION + } + } + + val DEFAULT_SEND_OPTIONS by lazy { + check(SctpConfig.config.enabled()) { "SCTP is disabled in configuration" } + SendOptions() + } + + const val DEFAULT_SCTP_PORT: Int = 5000 + } +} + +abstract class DcSctpBaseCallbacks( + val transport: DcSctpTransport, + val clock: Clock = Clock.systemUTC() +) : DcSctpSocketCallbacks { + /* Methods we can usefully implement for every JVB socket */ + override fun createTimeout(p0: DcSctpSocketCallbacks.DelayPrecision): Timeout { + return ATimeout() + } + + override fun Now(): Instant { + return clock.instant() + } + + override fun getRandomInt(low: Long, high: Long): Long { + return ThreadLocalRandom.current().nextLong(low, high) + } + + /* Methods we wouldn't normally expect to be called for a JVB SCTP socket. */ + override fun OnConnectionRestarted() { + transport.logger.info("Surprising SCTP callback: connection restarted") + } + + override fun OnStreamsResetFailed(outgoingStreams: ShortArray, reason: String) { + transport.logger.info( + "Surprising SCTP callback: streams ${outgoingStreams.joinToString()} reset failed: $reason" + ) + } + + override fun OnStreamsResetPerformed(outgoingStreams: ShortArray) { + transport.logger.info("Surprising SCTP callback: outgoing streams ${outgoingStreams.joinToString()} reset") + } + + override fun OnIncomingStreamsReset(incomingStreams: ShortArray) { + /* Does Chrome ever reset streams? */ + transport.logger.info("Surprising SCTP callback: incoming streams ${incomingStreams.joinToString()} reset") + } + + private inner class ATimeout : Timeout { + private var timeoutId: Long = 0 + private var scheduledFuture: ScheduledFuture<*>? = null + private var future: Future<*>? = null + override fun start(duration: Long, timeoutId: Long) { + try { + this.timeoutId = timeoutId + scheduledFuture = TaskPools.SCHEDULED_POOL.schedule({ + /* Execute it on the IO_POOL, because a timer may trigger sending new SCTP packets. */ + future = TaskPools.IO_POOL.submit { + transport.socket.handleTimeout(timeoutId) + } + }, duration, TimeUnit.MILLISECONDS) + } catch (e: Throwable) { + transport.logger.warn("Exception scheduling DCSCTP timeout", e) + } + } + + override fun stop() { + scheduledFuture?.cancel(false) + future?.cancel(false) + scheduledFuture = null + future = null + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt index 7caca61aae..4badda98d7 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt @@ -15,6 +15,10 @@ */ package org.jitsi.videobridge.relay +import org.jitsi.dcsctp4j.DcSctpMessage +import org.jitsi.dcsctp4j.ErrorKind +import org.jitsi.dcsctp4j.SendPacketStatus +import org.jitsi.dcsctp4j.SendStatus import org.jitsi.nlj.Features import org.jitsi.nlj.MediaSourceDesc import org.jitsi.nlj.PacketHandler @@ -76,6 +80,9 @@ import org.jitsi.videobridge.TransportConfig import org.jitsi.videobridge.datachannel.DataChannelStack import org.jitsi.videobridge.datachannel.protocol.DataChannelPacket import org.jitsi.videobridge.datachannel.protocol.DataChannelProtocolConstants +import org.jitsi.videobridge.dcsctp.DcSctpBaseCallbacks +import org.jitsi.videobridge.dcsctp.DcSctpHandler +import org.jitsi.videobridge.dcsctp.DcSctpTransport import org.jitsi.videobridge.message.BridgeChannelMessage import org.jitsi.videobridge.message.SourceVideoTypeMessage import org.jitsi.videobridge.metrics.QueueMetrics @@ -112,6 +119,7 @@ import kotlin.collections.ArrayList import kotlin.collections.HashMap import kotlin.collections.HashSet import kotlin.collections.sumOf +import org.jitsi.videobridge.sctp.SctpConfig.Companion.config as sctpConfig /** * Models a relay (remote videobridge) in a [Conference]. @@ -169,8 +177,24 @@ class Relay @JvmOverloads constructor( */ private var expired = false - private val sctpHandler = SctpHandler() + /** + * The two SCTP implementations (using usrsctp and dcsctp) are implemented side by side here. The intention is + * for the new dcsctp one to replace the old one, which will eventually be removed. + */ + private val sctpHandler = if (sctpConfig.enabled && !sctpConfig.useUsrSctp) DcSctpHandler() else null + private val usrSctpHandler = if (sctpConfig.enabled && sctpConfig.useUsrSctp) SctpHandler() else null + + /** The [DcSctpTransport] instance we'll use to manage the SCTP connection */ + private var sctpTransport: DcSctpTransport? = null + + /** The role we'll play in the SCTP handshake, if negotiated */ + private var sctpRole: Sctp.Role? = null + + // usrsctp + private var sctpManager: SctpManager? = null + private var sctpSocket: SctpSocket? = null private val dataChannelHandler = DataChannelHandler() + private var dataChannelStack: DataChannelStack? = null private val toggleablePcapWriter = ToggleablePcapWriter(logger, "$id-sctp") private val sctpRecvPcap = toggleablePcapWriter.newObserverNode(outbound = false) @@ -178,7 +202,12 @@ class Relay @JvmOverloads constructor( private val sctpPipeline = pipeline { node(sctpRecvPcap) - node(sctpHandler) + sctpHandler?.let { + node(it) + } + usrSctpHandler?.let { + node(it) + } } private val iceTransport = IceTransport( @@ -201,19 +230,6 @@ class Relay @JvmOverloads constructor( private val timelineLogger = logger.createChildLogger("timeline.${this.javaClass.name}") - /** - * The [SctpManager] instance we'll use to manage the SCTP connection - */ - private var sctpManager: SctpManager? = null - - private var dataChannelStack: DataChannelStack? = null - - /** - * The [SctpSocket] for this endpoint, if an SCTP connection was - * negotiated. - */ - private var sctpSocket: SctpSocket? = null - private val relayedEndpoints = HashMap() private val endpointsBySsrc = HashMap() private val endpointsLock = Any() @@ -342,6 +358,9 @@ class Relay @JvmOverloads constructor( endpointSenders[s.id] = s.getDebugState() } put("senders", endpointSenders) + sctpTransport?.let { + put("sctp", it.getDebugState()) + } } private fun setupIceTransport() { @@ -411,12 +430,18 @@ class Relay @JvmOverloads constructor( ) { logger.info("DTLS handshake complete") setSrtpInformation(chosenSrtpProtectionProfile, tlsRole, keyingMaterial) - when (val socket = sctpSocket) { - is SctpClientSocket -> connectSctpConnection(socket) - is SctpServerSocket -> acceptSctpConnection(socket) - else -> Unit + if (sctpConfig.enabled && sctpConfig.useUsrSctp) { + when (val socket = sctpSocket) { + is SctpClientSocket -> connectUsrSctpConnection(socket) + is SctpServerSocket -> acceptUsrSctpConnection(socket) + else -> Unit + } + scheduleRelayMessageTransportTimeout() + } else if (sctpConfig.enabled) { + if (sctpRole == Sctp.Role.CLIENT) { + sctpTransport!!.socket.connect() + } } - scheduleRelayMessageTransportTimeout() } } } @@ -454,6 +479,31 @@ class Relay @JvmOverloads constructor( * to open it. */ fun createSctpConnection(sctpDesc: Sctp) { + if (sctpConfig.enabled) { + if (sctpConfig.useUsrSctp) { + createUsrSctpConnection(sctpDesc) + } else { + createDcSctpConnection(sctpDesc) + } + } else { + logger.error("Not creating SCTP connection, SCTP is disabled in configuration.") + } + } + + private fun createDcSctpConnection(sctpDesc: Sctp) { + sctpRole = sctpDesc.role + + logger.cdebug { "Creating SCTP transport" } + sctpTransport = DcSctpTransport(id, logger).also { + it.start(SctpCallbacks(it)) + sctpHandler!!.setSctpTransport(it) + if (dtlsTransport.isConnected && sctpDesc.role == Sctp.Role.CLIENT) { + it.socket.connect() + } + } + } + + private fun createUsrSctpConnection(sctpDesc: Sctp) { val openDataChannelLocally = sctpDesc.role == Sctp.Role.CLIENT logger.cdebug { "Creating SCTP manager" } @@ -467,7 +517,7 @@ class Relay @JvmOverloads constructor( logger ) this.sctpManager = sctpManager - sctpHandler.setSctpManager(sctpManager) + usrSctpHandler!!.setSctpManager(sctpManager) val socket = if (sctpDesc.role == Sctp.Role.CLIENT) { sctpManager.createClientSocket(logger) } else { @@ -523,7 +573,7 @@ class Relay @JvmOverloads constructor( sctpSocket = socket } - fun connectSctpConnection(sctpClientSocket: SctpClientSocket) { + fun connectUsrSctpConnection(sctpClientSocket: SctpClientSocket) { TaskPools.IO_POOL.execute { // We don't want to block the thread calling // onDtlsHandshakeComplete so run the socket acceptance in an IO @@ -536,7 +586,7 @@ class Relay @JvmOverloads constructor( } } - fun acceptSctpConnection(sctpServerSocket: SctpServerSocket) { + fun acceptUsrSctpConnection(sctpServerSocket: SctpServerSocket) { TaskPools.IO_POOL.execute { // We don't want to block the thread calling // onDtlsHandshakeComplete so run the socket acceptance in an IO @@ -597,7 +647,7 @@ class Relay @JvmOverloads constructor( iceTransport.describe(iceUdpTransportPacketExtension) dtlsTransport.describe(iceUdpTransportPacketExtension) - if (sctpSocket == null) { + if (sctpTransport == null && sctpSocket == null) { /* TODO: this should be dependent on videobridge.websockets.enabled, if we support that being * disabled for relay. */ @@ -1121,8 +1171,10 @@ class Relay @JvmOverloads constructor( transceiver.teardown() messageTransport.close() - sctpHandler.stop() + sctpHandler?.stop() + usrSctpHandler?.stop() sctpManager?.closeConnection() + sctpTransport?.socket?.close() } catch (t: Throwable) { logger.error("Exception while expiring: ", t) } @@ -1181,6 +1233,93 @@ class Relay @JvmOverloads constructor( } } + private inner class SctpCallbacks(transport: DcSctpTransport) : DcSctpBaseCallbacks(transport) { + override fun sendPacketWithStatus(packet: ByteArray): SendPacketStatus { + try { + val newBuf = ByteBufferPool.getBuffer(packet.size) + System.arraycopy(packet, 0, newBuf, 0, packet.size) + + sctpSendPcap.observe(newBuf, 0, packet.size) + dtlsTransport.sendDtlsData(newBuf, 0, packet.size) + + return SendPacketStatus.kSuccess + } catch (e: Throwable) { + logger.warn("Exception sending SCTP packet", e) + return SendPacketStatus.kError + } + } + + override fun OnMessageReceived(message: DcSctpMessage) { + try { + // We assume all data coming over SCTP will be datachannel data + val dataChannelPacket = DataChannelPacket(message) + // Post the rest of the task here because the current context is + // holding a lock inside the SctpSocket which can cause a deadlock + // if two endpoints are trying to send datachannel messages to one + // another (with stats broadcasting it can happen often) + incomingDataChannelMessagesQueue.add(PacketInfo(dataChannelPacket)) + } catch (e: Throwable) { + logger.warn("Exception processing SCTP message", e) + } + } + + override fun OnError(error: ErrorKind, message: String) { + logger.warn("SCTP error $error: $message") + } + + override fun OnAborted(error: ErrorKind, message: String) { + logger.warn("SCTP aborted with error $error: $message") + } + + override fun OnConnected() { + try { + logger.info("SCTP connection is ready, creating the Data channel stack") + val dataChannelStack = DataChannelStack( + { data, sid, ppid -> + val message = DcSctpMessage(sid.toShort(), ppid, data.array()) + val status = sctpTransport?.socket?.send(message, DcSctpTransport.DEFAULT_SEND_OPTIONS) + return@DataChannelStack if (status == SendStatus.kSuccess) { + 0 + } else { + logger.error("Error sending to SCTP: $status") + -1 + } + }, + logger + ) + this@Relay.dataChannelStack = dataChannelStack + // This handles if the remote side will be opening the data channel + dataChannelStack.onDataChannelStackEvents { dataChannel -> + logger.info("Remote side opened a data channel.") + messageTransport.setDataChannel(dataChannel) + } + dataChannelHandler.setDataChannelStack(dataChannelStack) + if (sctpRole == Sctp.Role.CLIENT) { + // This logic is for opening the data channel locally + logger.info("Will open the data channel.") + val dataChannel = dataChannelStack.createDataChannel( + DataChannelProtocolConstants.RELIABLE, + 0, + 0, + 0, + "default" + ) + messageTransport.setDataChannel(dataChannel) + dataChannel.open() + } else { + logger.info("Will wait for the remote side to open the data channel.") + } + } catch (e: Throwable) { + logger.warn("Exception processing SCTP connected event", e) + } + } + + override fun OnClosed() { + // I don't think this should happen, except during shutdown. + logger.info("SCTP connection closed") + } + } + private inner class TransceiverEventHandlerImpl : TransceiverEventHandler { /** * Forward audio level events from the Transceiver to the conference. We use the same thread, because this fires diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpConfig.kt index ffed0aad65..9994fcfac3 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpConfig.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpConfig.kt @@ -20,10 +20,16 @@ import org.jitsi.config.JitsiConfig import org.jitsi.metaconfig.config class SctpConfig private constructor() { + /** Whether SCTP should be signaled or used when signaled to us */ val enabled: Boolean by config { "videobridge.sctp.enabled".from(JitsiConfig.newConfig) } - fun enabled() = enabled + /** + * If [enabled], whether to use the usrsctp based implementation. Otherwise, the new dcsctp implementation will be + * used. + */ + val useUsrSctp: Boolean by config { "videobridge.sctp.use-usrsctp".from(JitsiConfig.newConfig) } + companion object { @JvmField val config = SctpConfig() diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index 7418d9c000..e0991d042a 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -219,6 +219,9 @@ videobridge { sctp { // Whether SCTP data channels are enabled. enabled = true + + // Whether to use the usrsctp-based implementation instead of the new dcsctp-based one. + use-usrsctp = true } stats { // The interval at which stats are gathered. diff --git a/pom.xml b/pom.xml index 94f1b20ce6..ccae9b9f54 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ 1.9.10 5.7.2 5.10.0 - 1.0-127-g6c65524 + 1.0-131-ge0b9606 1.1-140-g8f45a9f 1.13.8 3.2.0 From 249e10a26e8148183f29a6be358d4901395510cc Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 23 May 2024 13:04:10 -0700 Subject: [PATCH 114/189] chore(ice4j): Configureable addresses and interfaces. (#2135) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ccae9b9f54..88742ad0c0 100644 --- a/pom.xml +++ b/pom.xml @@ -106,7 +106,7 @@ ${project.groupId} ice4j - 3.0-68-gd289f12 + 3.0-69-ga53b402 ${project.groupId} From 47ac3f04b7013457997845bc5d1a86848a86e431 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Thu, 23 May 2024 16:54:32 -0400 Subject: [PATCH 115/189] feat(VideoParser): In some cases, process VP8 or VP9 packets as AV1. (#2134) If the packets don't have the necessary fields to be routed as their payload type, but they do have an AV1 DD, route them based on the AV1 DD instead. --- .../org/jitsi/nlj/rtp/ParsedVideoPacket.kt | 6 ++ .../jitsi/nlj/rtp/codec/VideoCodecParser.kt | 1 - .../jitsi/nlj/rtp/codec/av1/Av1DDPacket.kt | 3 + .../org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt | 4 ++ .../org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt | 6 ++ .../transform/node/incoming/VideoParser.kt | 62 ++++++++++++++----- 6 files changed, 64 insertions(+), 18 deletions(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ParsedVideoPacket.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ParsedVideoPacket.kt index d4928295b8..5614e6f445 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ParsedVideoPacket.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ParsedVideoPacket.kt @@ -31,4 +31,10 @@ abstract class ParsedVideoPacket( abstract val isKeyframe: Boolean abstract val isStartOfFrame: Boolean abstract val isEndOfFrame: Boolean + + /** Whether the packet meets the needs of the routing infrastructure. + * If a packet could be parsed more than one way (e.g. it is VP8 or VP9 but also has an AV1 DD) + * this will let us choose which parse to prefer. + */ + abstract fun meetsRoutingNeeds(): Boolean } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/VideoCodecParser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/VideoCodecParser.kt index 623bb516d4..0f83e7e8ab 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/VideoCodecParser.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/VideoCodecParser.kt @@ -19,7 +19,6 @@ package org.jitsi.nlj.rtp.codec import org.jitsi.nlj.MediaSourceDesc import org.jitsi.nlj.PacketInfo import org.jitsi.nlj.RtpEncodingDesc -import org.jitsi.nlj.findRtpLayerDescs import org.jitsi.nlj.rtp.VideoRtpPacket /** diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacket.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacket.kt index 476beca682..dbb83f0d58 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacket.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacket.kt @@ -98,6 +98,9 @@ class Av1DDPacket : ParsedVideoPacket { override val isEndOfFrame: Boolean get() = statelessDescriptor.endOfFrame + override fun meetsRoutingNeeds(): Boolean = + true // If it didn't parse as AV1 we would have failed in the constructor + override val layerIds: Collection get() = frameInfo?.dtisPresent ?: run { super.layerIds } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt index 33d98323ff..38c2457dad 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt @@ -69,6 +69,10 @@ class Vp8Packet private constructor( /** This uses [get] rather than initialization because [isMarked] is a var. */ get() = isMarked + override fun meetsRoutingNeeds(): Boolean { + return hasPictureId && hasTemporalLayerIndex + } + val hasTemporalLayerIndex = DePacketizer.VP8PayloadDescriptor.hasTemporalLayerIndex(buffer, payloadOffset, payloadLength) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt index 7baf2379d5..ab05bf5d82 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt @@ -68,6 +68,12 @@ class Vp9Packet private constructor( override val isEndOfFrame: Boolean = isEndOfFrame ?: DePacketizer.VP9PayloadDescriptor.isEndOfFrame(buffer, payloadOffset, payloadLength) + override fun meetsRoutingNeeds(): Boolean { + // Question: should we include hasLayerIndices here? I.e. if we get a VP9 packet with an AV1 DD and + // a VP9 picture ID, but no VP9 layer indices, are we better off parsing it as VP9 or AV1? + return hasPictureId + } + override val layerIds: Collection get() = if (hasLayerIndices) { listOf(RtpLayerDesc.getIndex(0, spatialLayerIndex, temporalLayerIndex)) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoParser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoParser.kt index 850e39fa9d..d38b3ffeb7 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoParser.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoParser.kt @@ -22,6 +22,7 @@ import org.jitsi.nlj.SetMediaSourcesEvent import org.jitsi.nlj.findRtpSource import org.jitsi.nlj.format.Vp8PayloadType import org.jitsi.nlj.format.Vp9PayloadType +import org.jitsi.nlj.rtp.ParsedVideoPacket import org.jitsi.nlj.rtp.RtpExtensionType import org.jitsi.nlj.rtp.codec.VideoCodecParser import org.jitsi.nlj.rtp.codec.av1.Av1DDParser @@ -76,25 +77,17 @@ class VideoParser( val parsedPacket = try { when { payloadType is Vp8PayloadType -> { - val vp8Packet = packetInfo.packet.toOtherType(::Vp8Packet) - packetInfo.packet = vp8Packet - packetInfo.resetPayloadVerification() - - videoCodecParser = checkParserType(packetInfo) { source -> + val (vp8Packet, parser) = parseNormalPayload(packetInfo, ::Vp8Packet) { source -> Vp8Parser(source, logger) } - + videoCodecParser = parser vp8Packet } payloadType is Vp9PayloadType -> { - val vp9Packet = packetInfo.packet.toOtherType(::Vp9Packet) - packetInfo.packet = vp9Packet - packetInfo.resetPayloadVerification() - - videoCodecParser = checkParserType(packetInfo) { source -> + val (vp9Packet, parser) = parseNormalPayload(packetInfo, ::Vp9Packet) { source -> Vp9Parser(source, logger) } - + videoCodecParser = parser vp9Packet } av1DDExtId != null && packet.getHeaderExtension(av1DDExtId) != null -> { @@ -123,7 +116,6 @@ class VideoParser( } } packetInfo.layeringChanged = true - videoCodecParser = null } return packetInfo } @@ -143,17 +135,53 @@ class VideoParser( * so the count is correct. */ /* Alternately we could keep track of keyframes we've already seen, by timestamp, but that seems unnecessary. */ if (parsedPacket != null && parsedPacket.isKeyframe && parsedPacket.isStartOfFrame) { - logger.cdebug { "Received a keyframe for ssrc ${packet.ssrc} ${packet.sequenceNumber}" } + logger.cdebug { "Received a keyframe for ssrc ${packet.ssrc} at seq ${packet.sequenceNumber}" } stats.numKeyframes++ } if (packetInfo.layeringChanged) { - logger.cdebug { "Layering structure changed for ssrc ${packet.ssrc} ${packet.sequenceNumber}" } + logger.cdebug { "Layering structure changed for ssrc ${packet.ssrc} at seq ${packet.sequenceNumber}" } stats.numLayeringChanges++ } return packetInfo } + /** A normal payload is one where we choose the subclass of the ParsedVideoPacket and VideoCodecParser + * based on the payload type, as opposed to the header extension (like AV1). If the packet doesn't + * satisfy [ParsedVideoPacket.meetsRoutingNeeds] but it has an AV1 DD header extension, we will parse + * this packet as AV1 rather than as its normal type. + * */ + private inline fun parseNormalPayload( + packetInfo: PacketInfo, + otherTypeCreator: (ByteArray, Int, Int) -> ParsedVideoPacket, + parserConstructor: (MediaSourceDesc) -> T + ): Pair { + val parsedPacket = packetInfo.packet.toOtherType(otherTypeCreator) + if (!parsedPacket.meetsRoutingNeeds()) { + // See if we can parse this packet as AV1 + val packet = packetInfo.packetAs() + val av1DDExtId = this.av1DDExtId // So null checks work + if (av1DDExtId != null && packet.getHeaderExtension(av1DDExtId) != null) { + val parser = checkParserType(packetInfo) { source -> + Av1DDParser(source, logger, diagnosticContext) + } + + val av1DDPacket = parser?.createFrom(packet, av1DDExtId)?.also { + packetInfo.packet = it + packetInfo.resetPayloadVerification() + } + + return Pair(av1DDPacket, parser) + } + } + packetInfo.packet = parsedPacket + packetInfo.resetPayloadVerification() + + val parser = checkParserType(packetInfo, parserConstructor) + + return Pair(parsedPacket, parser) + } + private inline fun checkParserType( packetInfo: PacketInfo, constructor: (MediaSourceDesc) -> T @@ -168,8 +196,8 @@ class VideoParser( ?: // VideoQualityLayerLookup will drop this packet later, so no need to warn about it now return null logger.cdebug { - "Creating new ${T::class.java} for source ${source.sourceName}, " + - "current videoCodecParser is ${parser?.javaClass}" + "Creating new ${T::class.java.simpleName} for source ${source.sourceName}, " + + "current videoCodecParser is ${parser?.javaClass?.simpleName}" } resetSource(source) packetInfo.layeringChanged = true From 5ccdaa2b6b637fbd511b5c14344cbf45bd1efcee Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Thu, 23 May 2024 17:29:28 -0400 Subject: [PATCH 116/189] fix(AV1 DD): Separate AV1 DD template's decodeTargetLayers and decodeTargetProtectedBy fields. (#2136) The former can be present when the latter is not. --- .../jitsi/nlj/rtp/codec/av1/Av1DDPacket.kt | 2 +- .../Av1DDAdaptiveSourceProjectionContext.kt | 6 +- .../videobridge/cc/av1/Av1DDQualityFilter.kt | 3 +- .../Av1DependencyDescriptorHeaderExtension.kt | 45 +++++++------- ...DependencyDescriptorHeaderExtensionTest.kt | 61 +++++++++++++++++++ 5 files changed, 89 insertions(+), 28 deletions(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacket.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacket.kt index dbb83f0d58..27ea15920f 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacket.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacket.kt @@ -193,7 +193,7 @@ fun Av1DependencyDescriptorHeaderExtension.getScalabilityStructure( val layers = ArrayList() - structure.decodeTargetInfo.forEachIndexed { i, dt -> + structure.decodeTargetLayers.forEachIndexed { i, dt -> if (!activeDecodeTargetsBitmask.containsDecodeTarget(i)) { return@forEachIndexed } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt index 6388dc3f04..af464bef34 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt @@ -182,7 +182,7 @@ class Av1DDAdaptiveSourceProjectionContext( } else { frame.frameInfo?.dtisPresent ?: emptySet() } - val chainsToCheck = dtsToCheck.map { structure.decodeTargetInfo[it].protectedBy }.toSet() + val chainsToCheck = dtsToCheck.mapNotNull { structure.decodeTargetProtectedBy.getOrNull(it) }.toSet() return map.nextFrameWith(frame) { if (it.isAccepted) return@nextFrameWith false if (it.frameInfo == null) { @@ -197,8 +197,8 @@ class Av1DDAdaptiveSourceProjectionContext( private fun Av1DDFrame.partOfActiveChain(chainIdx: Int): Boolean { val structure = structure ?: return false val frameInfo = frameInfo ?: return false - for (i in structure.decodeTargetInfo.indices) { - if (structure.decodeTargetInfo[i].protectedBy != chainIdx) { + for (i in structure.decodeTargetProtectedBy.indices) { + if (structure.decodeTargetProtectedBy[i] != chainIdx) { continue } if (frameInfo.dti[i] == DTI.NOT_PRESENT || frameInfo.dti[i] == DTI.DISCARDABLE) { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt index 01808bd5a2..dc83977718 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt @@ -19,7 +19,6 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import org.jitsi.nlj.RtpLayerDesc.Companion.SUSPENDED_ENCODING_ID import org.jitsi.nlj.RtpLayerDesc.Companion.SUSPENDED_INDEX import org.jitsi.nlj.RtpLayerDesc.Companion.getEidFromIndex -import org.jitsi.nlj.RtpLayerDesc.Companion.indexString import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc.Companion.SUSPENDED_DT import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc.Companion.getDtFromIndex @@ -108,7 +107,7 @@ internal class Av1DDQualityFilter( val accept = doAcceptFrame(frame, incomingEncoding, externalTargetIndex, receivedTime) val currentDt = getDtFromIndex(currentIndex) val mark = currentDt != SUSPENDED_DT && - (frame.frameInfo?.spatialId == frame.structure?.decodeTargetInfo?.get(currentDt)?.spatialId) + (frame.frameInfo?.spatialId == frame.structure?.decodeTargetLayers?.get(currentDt)?.spatialId) val isResumption = (prevIndex == SUSPENDED_INDEX && currentIndex != SUSPENDED_INDEX) if (isResumption) { check(accept) { diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtension.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtension.kt index 679c6e3e83..c18061c8f7 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtension.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtension.kt @@ -284,7 +284,8 @@ fun Int.bitsForFdiff() = when { class Av1TemplateDependencyStructure( var templateIdOffset: Int, val templateInfo: List, - val decodeTargetInfo: List, + val decodeTargetProtectedBy: List, + val decodeTargetLayers: List, val maxRenderResolutions: List, val maxSpatialId: Int, val maxTemporalId: Int @@ -293,7 +294,7 @@ class Av1TemplateDependencyStructure( get() = templateInfo.size val decodeTargetCount - get() = decodeTargetInfo.size + get() = decodeTargetLayers.size val chainCount: Int = templateInfo.first().chains.size @@ -327,8 +328,8 @@ class Av1TemplateDependencyStructure( // TemplateChains length += nsBits(decodeTargetCount + 1, chainCount) if (chainCount > 0) { - decodeTargetInfo.forEach { - length += nsBits(chainCount, it.protectedBy) + decodeTargetProtectedBy.forEach { + length += nsBits(chainCount, it) } length += templateCount * chainCount * 4 } @@ -343,7 +344,8 @@ class Av1TemplateDependencyStructure( templateIdOffset, // These objects are not mutable so it's safe to copy them by reference templateInfo, - decodeTargetInfo, + decodeTargetProtectedBy, + decodeTargetLayers, maxRenderResolutions, maxSpatialId, maxTemporalId @@ -412,8 +414,8 @@ class Av1TemplateDependencyStructure( private fun writeTemplateChains(writer: BitWriter) { writer.writeNs(decodeTargetCount + 1, chainCount) - decodeTargetInfo.forEach { - writer.writeNs(chainCount, it.protectedBy) + decodeTargetProtectedBy.forEach { + writer.writeNs(chainCount, it) } templateInfo.forEach { t -> t.chains.forEach { chain -> @@ -461,7 +463,8 @@ class Av1TemplateDependencyStructure( return OrderedJsonObject().apply { put("templateIdOffset", templateIdOffset) put("templateInfo", templateInfo.toIndexedMap()) - put("decodeTargetInfo", decodeTargetInfo.toIndexedMap()) + put("decodeTargetProtectedBy", decodeTargetProtectedBy.toIndexedMap()) + put("decodeTargetLayers", decodeTargetLayers.toIndexedMap()) if (maxRenderResolutions.isNotEmpty()) { put("maxRenderResolutions", maxRenderResolutions.toIndexedMap()) } @@ -612,7 +615,8 @@ class Av1DependencyDescriptorReader( /* Data for template dependency structure */ private var templateIdOffset: Int = 0 private val templateInfo = mutableListOf() - private val decodeTargetInfo = mutableListOf() + private val decodeTargetProtectedBy = mutableListOf() + private val decodeTargetLayers = mutableListOf() private val maxRenderResolutions = mutableListOf() private var dtCnt = 0 @@ -623,7 +627,8 @@ class Av1DependencyDescriptorReader( */ templateCnt = 0 templateInfo.clear() - decodeTargetInfo.clear() + decodeTargetProtectedBy.clear() + decodeTargetLayers.clear() maxRenderResolutions.clear() } @@ -650,7 +655,8 @@ class Av1DependencyDescriptorReader( return Av1TemplateDependencyStructure( templateIdOffset, templateInfo.toList(), - decodeTargetInfo.toList(), + decodeTargetProtectedBy.toList(), + decodeTargetLayers.toList(), maxRenderResolutions.toList(), maxSpatialId, maxTemporalId @@ -733,7 +739,7 @@ class Av1DependencyDescriptorReader( val chainCount = reader.ns(dtCnt + 1) if (chainCount != 0) { for (dtIndex in 0 until dtCnt) { - decodeTargetInfo.add(dtIndex, DecodeTargetInfo(reader.ns(chainCount))) + decodeTargetProtectedBy.add(dtIndex, reader.ns(chainCount)) } for (templateIndex in 0 until templateCnt) { for (chainIndex in 0 until chainCount) { @@ -764,10 +770,9 @@ class Av1DependencyDescriptorReader( } } } - decodeTargetInfo[dtIndex].spatialId = spatialId - decodeTargetInfo[dtIndex].temporalId = temporalId + decodeTargetLayers.add(dtIndex, DecodeTargetLayer(spatialId, temporalId)) } - check(decodeTargetInfo.size == dtCnt) + check(decodeTargetLayers.size == dtCnt) } private fun readRenderResolutions() { @@ -843,16 +848,12 @@ class TemplateFrameInfo( override val chains: MutableList = mutableListOf() ) : FrameInfo(spatialId, temporalId, dti, fdiff, chains) -class DecodeTargetInfo( - val protectedBy: Int +class DecodeTargetLayer( + val spatialId: Int, + val temporalId: Int ) : JSONAware { - /** Todo: only want to be able to set these from the constructor */ - var spatialId: Int = -1 - var temporalId: Int = -1 - override fun toJSONString(): String { return OrderedJsonObject().apply { - put("protectedBy", protectedBy) put("spatialId", spatialId) put("temporalId", temporalId) }.toJSONString() diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtensionTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtensionTest.kt index 4a69bb9c93..6be28bd3de 100644 --- a/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtensionTest.kt +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtensionTest.kt @@ -68,6 +68,11 @@ class Av1DependencyDescriptorHeaderExtensionTest : ShouldSpec() { "8700ed80e3061eaa82804028280514d14134518010a091889a09409fc059c13fc0b3c0" ) + /* The header Chrome 126 generates for VP8 keyframes when AV1 DD is enabled for VP8. It has no chains. */ + val descNoChains = parseHexBinary( + "8000138002044eaaaf2860414d34538a0940413fc0b3c0" + ) + init { context("AV1 Dependency Descriptors") { context("a descriptor with a single-layer dependency structure") { @@ -544,6 +549,62 @@ class Av1DependencyDescriptorHeaderExtensionTest : ShouldSpec() { buf shouldBe descS3T3 } } + context("A descriptor with no chains") { + val ldsr = Av1DependencyDescriptorReader(descNoChains, 0, descNoChains.size) + val lds = ldsr.parse(null) + should("be parsed properly") { + lds.startOfFrame shouldBe true + lds.endOfFrame shouldBe false + lds.frameNumber shouldBe 0x0013 + lds.activeDecodeTargetsBitmask shouldBe 0x7 + + val structure = lds.newTemplateDependencyStructure + structure shouldNotBe null + structure!!.decodeTargetCount shouldBe 3 + structure.maxTemporalId shouldBe 2 + structure.maxSpatialId shouldBe 0 + } + should("calculate correct frame info") { + val ldsi = lds.frameInfo + ldsi.spatialId shouldBe 0 + ldsi.temporalId shouldBe 0 + } + should("calculate correctly whether layer switching needs keyframes") { + val structure = lds.newTemplateDependencyStructure!! + val fromS = 0 + for (fromT in 0..2) { + val fromDT = 3 * fromS + fromT + val toS = 0 + for (toT in 0..2) { + val toDT = 3 * toS + toT + /* With this structure you can switch down spatial layers, or to other temporal + * layers within the same spatial layer, without a keyframe; but switching up + * spatial layers needs a keyframe. + */ + withClue("from DT $fromDT to DT $toDT") { + structure.canSwitchWithoutKeyframe( + fromDt = fromDT, + toDt = toDT + ) shouldBe true + } + } + } + } + should("calculate DTI bitmasks corresponding to a given DT") { + val structure = lds.newTemplateDependencyStructure!! + structure.getDtBitmaskForDt(0) shouldBe 0b001 + structure.getDtBitmaskForDt(1) shouldBe 0b011 + structure.getDtBitmaskForDt(2) shouldBe 0b111 + } + should("Calculate its own length properly") { + lds.encodedLength shouldBe descNoChains.size + } + should("Be re-encoded to the same bytes") { + val buf = ByteArray(lds.encodedLength) + lds.write(buf, 0, buf.size) + buf shouldBe descNoChains + } + } } } } From 674f841f1044f0ea3ff3e82b8ceb2364369af676 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 28 May 2024 15:27:50 -0400 Subject: [PATCH 117/189] fix: Update dcsctp, fixing DcSctpOptions initialization. (#2137) --- jvb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/pom.xml b/jvb/pom.xml index 484f50dc35..5962b4fc9e 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -113,7 +113,7 @@ ${project.groupId} jitsi-dcsctp - 1.0-1-ga91e110 + 1.0-2-g2d8eee4 From 8de0373f470daf27e143d71316a01bcd273fc8a7 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Wed, 29 May 2024 10:42:41 -0700 Subject: [PATCH 118/189] chore: Add a dependabot.yml file. (#2138) * Add a dependabot.yml file. --- .github/dependabot.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..e0c85f963c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "maven" + directories: + - "/" + - "/rtp" + - "/jitsi-media-transform" + - "/jvb" + schedule: + interval: "weekly" From 63d7bf8b06ba9a2e36f3a7b8792588bfe661492e Mon Sep 17 00:00:00 2001 From: dkirov-dev <147876663+dkirov-dev@users.noreply.github.com> Date: Wed, 29 May 2024 11:38:55 -0700 Subject: [PATCH 119/189] Add telephone-event payload type (#2071) --- .../org/jitsi/nlj/format/PayloadType.kt | 20 +++++++++++++++++-- .../jitsi/videobridge/util/PayloadTypeUtil.kt | 3 +++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/format/PayloadType.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/format/PayloadType.kt index 6f0c5bba25..386731c9ab 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/format/PayloadType.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/format/PayloadType.kt @@ -77,15 +77,21 @@ enum class PayloadTypeEncoding { H264, RED, RTX, - OPUS; + OPUS, + TELEPHONE_EVENT; companion object { + private const val TEL_EVENT_TEXT = "telephone-event" + /** * [valueOf] does not allow for case-insensitivity and can't be overridden, so this * method should be used when creating an instance of this enum from a string */ fun createFrom(value: String): PayloadTypeEncoding { return try { + if (value.lowercase() == TEL_EVENT_TEXT) { + return TELEPHONE_EVENT + } valueOf(value.uppercase()) } catch (e: IllegalArgumentException) { return OTHER @@ -94,7 +100,11 @@ enum class PayloadTypeEncoding { } override fun toString(): String = with(StringBuffer()) { - append(super.toString()) + if (super.equals(TELEPHONE_EVENT)) { + append(TEL_EVENT_TEXT) + } else { + append(super.toString()) + } toString() } } @@ -145,6 +155,12 @@ class OpusPayloadType( parameters: PayloadTypeParams = ConcurrentHashMap() ) : AudioPayloadType(pt, PayloadTypeEncoding.OPUS, parameters = parameters) +class TelephoneEventPayloadType( + pt: Byte, + clockRate: Int, + parameters: PayloadTypeParams = ConcurrentHashMap() +) : AudioPayloadType(pt, PayloadTypeEncoding.TELEPHONE_EVENT, clockRate, parameters) + class AudioRedPayloadType( pt: Byte, clockRate: Int = 48000, diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/util/PayloadTypeUtil.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/util/PayloadTypeUtil.kt index c0459a5b5a..144d21718a 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/util/PayloadTypeUtil.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/util/PayloadTypeUtil.kt @@ -27,9 +27,11 @@ import org.jitsi.nlj.format.PayloadTypeEncoding.OPUS import org.jitsi.nlj.format.PayloadTypeEncoding.OTHER import org.jitsi.nlj.format.PayloadTypeEncoding.RED import org.jitsi.nlj.format.PayloadTypeEncoding.RTX +import org.jitsi.nlj.format.PayloadTypeEncoding.TELEPHONE_EVENT import org.jitsi.nlj.format.PayloadTypeEncoding.VP8 import org.jitsi.nlj.format.PayloadTypeEncoding.VP9 import org.jitsi.nlj.format.RtxPayloadType +import org.jitsi.nlj.format.TelephoneEventPayloadType import org.jitsi.nlj.format.VideoRedPayloadType import org.jitsi.nlj.format.Vp8PayloadType import org.jitsi.nlj.format.Vp9PayloadType @@ -95,6 +97,7 @@ class PayloadTypeUtil { H264 -> H264PayloadType(id, parameters, rtcpFeedbackSet) RTX -> RtxPayloadType(id, parameters) OPUS -> OpusPayloadType(id, parameters) + TELEPHONE_EVENT -> TelephoneEventPayloadType(id, clockRate, parameters) RED -> when (mediaType) { AUDIO -> AudioRedPayloadType(id, clockRate, parameters) VIDEO -> VideoRedPayloadType(id, clockRate, parameters, rtcpFeedbackSet) From 5216cddcb70773654de18346217309deac181081 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 May 2024 12:08:50 -0700 Subject: [PATCH 120/189] chore(deps): Bump jetty.version from 11.0.20 to 11.0.21 (#2140) Bumps `jetty.version` from 11.0.20 to 11.0.21. Updates `org.eclipse.jetty:jetty-rewrite` from 11.0.20 to 11.0.21 Updates `org.eclipse.jetty:jetty-servlets` from 11.0.20 to 11.0.21 Updates `org.eclipse.jetty.websocket:websocket-jetty-server` from 11.0.20 to 11.0.21 Updates `org.eclipse.jetty.websocket:websocket-jetty-client` from 11.0.20 to 11.0.21 Updates `org.eclipse.jetty:jetty-maven-plugin` from 11.0.20 to 11.0.21 --- updated-dependencies: - dependency-name: org.eclipse.jetty:jetty-rewrite dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.jetty:jetty-servlets dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.jetty.websocket:websocket-jetty-server dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.jetty.websocket:websocket-jetty-client dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.eclipse.jetty:jetty-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 88742ad0c0..fb3d366a5d 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ pom - 11.0.20 + 11.0.21 1.9.10 5.7.2 5.10.0 From 274e392058f385f56794a436e87694faacff742f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 May 2024 12:12:42 -0700 Subject: [PATCH 121/189] chore(deps): Bump kotest.version from 5.7.2 to 5.9.0 (#2141) Bumps `kotest.version` from 5.7.2 to 5.9.0. Updates `io.kotest:kotest-runner-junit5-jvm` from 5.7.2 to 5.9.0 - [Release notes](https://github.com/kotest/kotest/releases) - [Commits](https://github.com/kotest/kotest/compare/v5.7.2...v5.9.0) Updates `io.kotest:kotest-assertions-core-jvm` from 5.7.2 to 5.9.0 - [Release notes](https://github.com/kotest/kotest/releases) - [Commits](https://github.com/kotest/kotest/compare/v5.7.2...v5.9.0) Updates `io.kotest:kotest-property-jvm` from 5.7.2 to 5.9.0 - [Release notes](https://github.com/kotest/kotest/releases) - [Commits](https://github.com/kotest/kotest/compare/v5.7.2...v5.9.0) --- updated-dependencies: - dependency-name: io.kotest:kotest-runner-junit5-jvm dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: io.kotest:kotest-assertions-core-jvm dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: io.kotest:kotest-property-jvm dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fb3d366a5d..994ddf396f 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ 11.0.21 1.9.10 - 5.7.2 + 5.9.0 5.10.0 1.0-131-ge0b9606 1.1-140-g8f45a9f From aebbda947a222594d4e8dc852cdac6f08d254959 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 May 2024 12:44:10 -0700 Subject: [PATCH 122/189] chore(deps): Bump slf4j.version from 1.7.32 to 2.0.13 (#2139) Bumps `slf4j.version` from 1.7.32 to 2.0.13. Updates `org.slf4j:slf4j-api` from 1.7.32 to 2.0.13 Updates `org.slf4j:slf4j-jdk14` from 1.7.32 to 2.0.13 --- updated-dependencies: - dependency-name: org.slf4j:slf4j-api dependency-type: direct:production update-type: version-update:semver-major - dependency-name: org.slf4j:slf4j-jdk14 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- jvb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/pom.xml b/jvb/pom.xml index 5962b4fc9e..efc0441c26 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -17,7 +17,7 @@ false org.jitsi.videobridge.MainKt - 1.7.32 + 2.0.13 From 2e0434d847cae2eb945e2b1d6af072f71e67ee75 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Wed, 29 May 2024 16:02:29 -0400 Subject: [PATCH 123/189] fix: Run lastNEndpointsChanged asynchronously in Conference#endpointSourcesChanged. (#1611) This is called in the Colibri message processing critical path. --- jvb/src/main/java/org/jitsi/videobridge/Conference.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/Conference.java b/jvb/src/main/java/org/jitsi/videobridge/Conference.java index f133689b9a..4e751391c7 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Conference.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Conference.java @@ -730,7 +730,8 @@ private void endpointsChanged(boolean includesNonVisitors) private void endpointSourcesChanged(@NotNull Endpoint endpoint) { // Force an update to be propagated to each endpoint's bitrate controller. - lastNEndpointsChanged(); + // We run this async because this method is called in the Colibri critical path. + lastNEndpointsChangedAsync(); } /** From b3113c529f5e5c0ddeb0830b54a1b728742b7d7f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 May 2024 13:39:44 -0700 Subject: [PATCH 124/189] chore(deps): Bump kotlin.version from 1.9.10 to 2.0.0 (#2143) Bumps `kotlin.version` from 1.9.10 to 2.0.0. Updates `org.jetbrains.kotlin:kotlin-stdlib-jdk8` from 1.9.10 to 2.0.0 - [Release notes](https://github.com/JetBrains/kotlin/releases) - [Changelog](https://github.com/JetBrains/kotlin/blob/master/ChangeLog.md) - [Commits](https://github.com/JetBrains/kotlin/compare/v1.9.10...v2.0.0) Updates `org.jetbrains.kotlin:kotlin-reflect` from 1.9.10 to 2.0.0 - [Release notes](https://github.com/JetBrains/kotlin/releases) - [Changelog](https://github.com/JetBrains/kotlin/blob/master/ChangeLog.md) - [Commits](https://github.com/JetBrains/kotlin/compare/v1.9.10...v2.0.0) Updates `org.jetbrains.kotlin:kotlin-maven-plugin` from 1.9.10 to 2.0.0 --- updated-dependencies: - dependency-name: org.jetbrains.kotlin:kotlin-stdlib-jdk8 dependency-type: direct:production update-type: version-update:semver-major - dependency-name: org.jetbrains.kotlin:kotlin-reflect dependency-type: direct:production update-type: version-update:semver-major - dependency-name: org.jetbrains.kotlin:kotlin-maven-plugin dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 994ddf396f..0410146955 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ 11.0.21 - 1.9.10 + 2.0.0 5.9.0 5.10.0 1.0-131-ge0b9606 From 6051205ada0c2269c94ee593689ab856028632f1 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Wed, 29 May 2024 14:03:44 -0700 Subject: [PATCH 125/189] chore(dependabot): Bump open prs limit. (#2144) --- .github/dependabot.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e0c85f963c..f3fcd4bae1 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,3 +8,4 @@ updates: - "/jvb" schedule: interval: "weekly" + open-pull-requests-limit: 10 From 3fe96745d7e7996b0c327810cc2f91c6d1ddabe4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 May 2024 14:49:04 -0700 Subject: [PATCH 126/189] chore(deps): Bump org.jitsi:jitsi-utils from 1.0-131-ge0b9606 to 1.0-132-g83984af (#2153) chore(deps): Bump org.jitsi:jitsi-utils Bumps [org.jitsi:jitsi-utils](https://github.com/jitsi/jitsi-utils) from 1.0-131-ge0b9606 to 1.0-132-g83984af. - [Commits](https://github.com/jitsi/jitsi-utils/commits) --- updated-dependencies: - dependency-name: org.jitsi:jitsi-utils dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0410146955..5f89c44b97 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ 2.0.0 5.9.0 5.10.0 - 1.0-131-ge0b9606 + 1.0-132-g83984af 1.1-140-g8f45a9f 1.13.8 3.2.0 From a3a500d0669a1d606a5ca45cb8063b586ced471b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 May 2024 15:37:57 -0700 Subject: [PATCH 127/189] chore(deps): Bump com.puppycrawl.tools:checkstyle from 9.1 to 10.17.0 (#2151) Bumps [com.puppycrawl.tools:checkstyle](https://github.com/checkstyle/checkstyle) from 9.1 to 10.17.0. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-9.1...checkstyle-10.17.0) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- jvb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/pom.xml b/jvb/pom.xml index efc0441c26..67a76e9473 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -534,7 +534,7 @@ com.puppycrawl.tools checkstyle - 9.1 + 10.17.0 From 1f0a090a093cf8c5dcc74db1183b000119da9522 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 May 2024 15:38:11 -0700 Subject: [PATCH 128/189] chore(deps): Bump com.google.guava:guava from 32.0.0-jre to 33.2.0-jre (#2150) Bumps [com.google.guava:guava](https://github.com/google/guava) from 32.0.0-jre to 33.2.0-jre. - [Release notes](https://github.com/google/guava/releases) - [Commits](https://github.com/google/guava/commits) --- updated-dependencies: - dependency-name: com.google.guava:guava dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- jvb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/pom.xml b/jvb/pom.xml index 67a76e9473..33f3e9283a 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -61,7 +61,7 @@ com.google.guava guava - 32.0.0-jre + 33.2.0-jre org.eclipse.jetty From 5d7024ecce40465fffea07c3fee373df75be0caf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 May 2024 08:37:30 -0700 Subject: [PATCH 129/189] chore(deps): Bump io.sentry:sentry from 5.3.0 to 7.9.0 (#2149) Bumps [io.sentry:sentry](https://github.com/getsentry/sentry-java) from 5.3.0 to 7.9.0. - [Release notes](https://github.com/getsentry/sentry-java/releases) - [Changelog](https://github.com/getsentry/sentry-java/blob/main/CHANGELOG.md) - [Commits](https://github.com/getsentry/sentry-java/compare/5.3.0...7.9.0) --- updated-dependencies: - dependency-name: io.sentry:sentry dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- jvb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/pom.xml b/jvb/pom.xml index 33f3e9283a..56983f8769 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -208,7 +208,7 @@ io.sentry sentry - 5.3.0 + 7.9.0 runtime From 39882bee178b961940a4ed699e3d63742fb51430 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 May 2024 08:38:56 -0700 Subject: [PATCH 130/189] chore(deps): Bump io.pkts:pkts-core from 3.0.3 to 3.0.10 (#2148) Bumps [io.pkts:pkts-core](https://github.com/aboutsip/pkts) from 3.0.3 to 3.0.10. - [Changelog](https://github.com/aboutsip/pkts/blob/master/CHANGELOG.md) - [Commits](https://github.com/aboutsip/pkts/compare/release-pkts-3.0.3...release-pkts-3.0.10) --- updated-dependencies: - dependency-name: io.pkts:pkts-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- jitsi-media-transform/pom.xml | 2 +- rtp/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jitsi-media-transform/pom.xml b/jitsi-media-transform/pom.xml index de6d1404e8..5c8b01f495 100644 --- a/jitsi-media-transform/pom.xml +++ b/jitsi-media-transform/pom.xml @@ -94,7 +94,7 @@ io.pkts pkts-core - 3.0.3 + 3.0.10 test diff --git a/rtp/pom.xml b/rtp/pom.xml index 001ead4e8c..28bfbdbe73 100644 --- a/rtp/pom.xml +++ b/rtp/pom.xml @@ -52,7 +52,7 @@ io.pkts pkts-core - 3.0.3 + 3.0.10 test From af5e4689a42c34746be505e457bca18205976773 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 May 2024 08:40:18 -0700 Subject: [PATCH 131/189] chore(deps-dev): Bump jakarta.xml.bind:jakarta.xml.bind-api from 4.0.0 to 4.0.2 (#2147) chore(deps-dev): Bump jakarta.xml.bind:jakarta.xml.bind-api Bumps [jakarta.xml.bind:jakarta.xml.bind-api](https://github.com/jakartaee/jaxb-api) from 4.0.0 to 4.0.2. - [Release notes](https://github.com/jakartaee/jaxb-api/releases) - [Commits](https://github.com/jakartaee/jaxb-api/compare/4.0.0...4.0.2) --- updated-dependencies: - dependency-name: jakarta.xml.bind:jakarta.xml.bind-api dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- rtp/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rtp/pom.xml b/rtp/pom.xml index 28bfbdbe73..987590ffe1 100644 --- a/rtp/pom.xml +++ b/rtp/pom.xml @@ -58,7 +58,7 @@ jakarta.xml.bind jakarta.xml.bind-api - 4.0.0 + 4.0.2 test From a14d4919dc0d17e183e5ed99d0b540474c2fb7c2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 May 2024 08:40:56 -0700 Subject: [PATCH 132/189] chore(deps): Bump jersey.version from 3.0.10 to 3.1.7 (#2145) Bumps `jersey.version` from 3.0.10 to 3.1.7. Updates `org.glassfish.jersey.containers:jersey-container-jetty-http` from 3.0.10 to 3.1.7 Updates `org.glassfish.jersey.containers:jersey-container-servlet` from 3.0.10 to 3.1.7 Updates `org.glassfish.jersey.inject:jersey-hk2` from 3.0.10 to 3.1.7 Updates `org.glassfish.jersey.media:jersey-media-json-jackson` from 3.0.10 to 3.1.7 Updates `org.glassfish.jersey.test-framework:jersey-test-framework-core` from 3.0.10 to 3.1.7 Updates `org.glassfish.jersey.test-framework.providers:jersey-test-framework-provider-jetty` from 3.0.10 to 3.1.7 Updates `org.glassfish.jersey.test-framework.providers:jersey-test-framework-provider-grizzly2` from 3.0.10 to 3.1.7 --- updated-dependencies: - dependency-name: org.glassfish.jersey.containers:jersey-container-jetty-http dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: org.glassfish.jersey.containers:jersey-container-servlet dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: org.glassfish.jersey.inject:jersey-hk2 dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: org.glassfish.jersey.media:jersey-media-json-jackson dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: org.glassfish.jersey.test-framework:jersey-test-framework-core dependency-type: direct:development update-type: version-update:semver-minor - dependency-name: org.glassfish.jersey.test-framework.providers:jersey-test-framework-provider-jetty dependency-type: direct:development update-type: version-update:semver-minor - dependency-name: org.glassfish.jersey.test-framework.providers:jersey-test-framework-provider-grizzly2 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5f89c44b97..f1199c9662 100644 --- a/pom.xml +++ b/pom.xml @@ -33,7 +33,7 @@ 3.2.0 3.5.1 4.6.0 - 3.0.10 + 3.1.7 2.12.4 1.77 0.16.0 From 24dfb8e95a3739b27a4ee39853dd7aab438c616d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 May 2024 08:41:27 -0700 Subject: [PATCH 133/189] chore(deps): Bump org.apache.maven.plugins:maven-surefire-plugin from 2.22.0 to 3.2.5 (#2146) chore(deps): Bump org.apache.maven.plugins:maven-surefire-plugin Bumps [org.apache.maven.plugins:maven-surefire-plugin](https://github.com/apache/maven-surefire) from 2.22.0 to 3.2.5. - [Release notes](https://github.com/apache/maven-surefire/releases) - [Commits](https://github.com/apache/maven-surefire/compare/surefire-2.22.0...surefire-3.2.5) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-surefire-plugin dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- jitsi-media-transform/pom.xml | 2 +- jvb/pom.xml | 2 +- rtp/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jitsi-media-transform/pom.xml b/jitsi-media-transform/pom.xml index 5c8b01f495..0bab23ca28 100644 --- a/jitsi-media-transform/pom.xml +++ b/jitsi-media-transform/pom.xml @@ -238,7 +238,7 @@ maven-surefire-plugin - 2.22.0 + 3.2.5 diff --git a/jvb/pom.xml b/jvb/pom.xml index 56983f8769..6a6286f058 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -464,7 +464,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.22.2 + 3.2.5 ${project.basedir}/lib/logging.properties diff --git a/rtp/pom.xml b/rtp/pom.xml index 987590ffe1..fae009cca3 100644 --- a/rtp/pom.xml +++ b/rtp/pom.xml @@ -172,7 +172,7 @@ maven-surefire-plugin - 2.22.0 + 3.2.5 com.github.gantsign.maven From 304ae3871cf8a631eb8d4d6d5a5a5d7c254db70d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 10:34:26 -0700 Subject: [PATCH 134/189] chore(deps): Bump junit.version from 5.10.0 to 5.10.2 (#2160) Bumps `junit.version` from 5.10.0 to 5.10.2. Updates `org.junit.jupiter:junit-jupiter-api` from 5.10.0 to 5.10.2 - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.10.0...r5.10.2) Updates `org.junit.vintage:junit-vintage-engine` from 5.10.0 to 5.10.2 - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.10.0...r5.10.2) --- updated-dependencies: - dependency-name: org.junit.jupiter:junit-jupiter-api dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.junit.vintage:junit-vintage-engine dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f1199c9662..20a79581cb 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ 11.0.21 2.0.0 5.9.0 - 5.10.0 + 5.10.2 1.0-132-g83984af 1.1-140-g8f45a9f 1.13.8 From bc61ea15472876557728bd8fea6b389ac0eaf51d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 10:35:06 -0700 Subject: [PATCH 135/189] chore(deps): Bump org.codehaus.mojo:build-helper-maven-plugin from 3.0.0 to 3.6.0 (#2159) chore(deps): Bump org.codehaus.mojo:build-helper-maven-plugin Bumps [org.codehaus.mojo:build-helper-maven-plugin](https://github.com/mojohaus/build-helper-maven-plugin) from 3.0.0 to 3.6.0. - [Release notes](https://github.com/mojohaus/build-helper-maven-plugin/releases) - [Commits](https://github.com/mojohaus/build-helper-maven-plugin/compare/build-helper-maven-plugin-3.0.0...3.6.0) --- updated-dependencies: - dependency-name: org.codehaus.mojo:build-helper-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- jitsi-media-transform/pom.xml | 2 +- rtp/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jitsi-media-transform/pom.xml b/jitsi-media-transform/pom.xml index 0bab23ca28..d2a1c77e4b 100644 --- a/jitsi-media-transform/pom.xml +++ b/jitsi-media-transform/pom.xml @@ -174,7 +174,7 @@ org.codehaus.mojo build-helper-maven-plugin - 3.0.0 + 3.6.0 generate-sources diff --git a/rtp/pom.xml b/rtp/pom.xml index fae009cca3..b24a4920b8 100644 --- a/rtp/pom.xml +++ b/rtp/pom.xml @@ -97,7 +97,7 @@ org.codehaus.mojo build-helper-maven-plugin - 3.0.0 + 3.6.0 generate-sources From 740576a6a6082b7df219c0edb50efb20476ae4fa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 10:35:20 -0700 Subject: [PATCH 136/189] chore(deps): Bump org.apache.maven.plugins:maven-enforcer-plugin from 3.0.0-M3 to 3.5.0 (#2158) chore(deps): Bump org.apache.maven.plugins:maven-enforcer-plugin Bumps [org.apache.maven.plugins:maven-enforcer-plugin](https://github.com/apache/maven-enforcer) from 3.0.0-M3 to 3.5.0. - [Release notes](https://github.com/apache/maven-enforcer/releases) - [Commits](https://github.com/apache/maven-enforcer/compare/enforcer-3.0.0-M3...enforcer-3.5.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-enforcer-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 20a79581cb..a57f3e3bf4 100644 --- a/pom.xml +++ b/pom.xml @@ -45,7 +45,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.0.0-M3 + 3.5.0 enforce-maven From ad8da4dcb065129eae5cef237d758155793494fa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 10:42:54 -0700 Subject: [PATCH 137/189] chore(deps): Bump org.apache.maven.plugins:maven-jar-plugin from 3.2.0 to 3.4.1 (#2155) chore(deps): Bump org.apache.maven.plugins:maven-jar-plugin Bumps [org.apache.maven.plugins:maven-jar-plugin](https://github.com/apache/maven-jar-plugin) from 3.2.0 to 3.4.1. - [Release notes](https://github.com/apache/maven-jar-plugin/releases) - [Commits](https://github.com/apache/maven-jar-plugin/compare/maven-jar-plugin-3.2.0...maven-jar-plugin-3.4.1) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-jar-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- jvb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/pom.xml b/jvb/pom.xml index 6a6286f058..2949b41ea3 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -514,7 +514,7 @@ --> org.apache.maven.plugins maven-jar-plugin - 3.2.0 + 3.4.1 From e51f48757ec5b292cce8b05cc1325ddb5bb2c952 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 10:43:11 -0700 Subject: [PATCH 138/189] chore(deps): Bump io.mockk:mockk-jvm from 1.13.8 to 1.13.11 (#2154) Bumps [io.mockk:mockk-jvm](https://github.com/mockk/mockk) from 1.13.8 to 1.13.11. - [Release notes](https://github.com/mockk/mockk/releases) - [Commits](https://github.com/mockk/mockk/compare/1.13.8...1.13.11) --- updated-dependencies: - dependency-name: io.mockk:mockk-jvm dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a57f3e3bf4..f07f5f140d 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ 5.10.2 1.0-132-g83984af 1.1-140-g8f45a9f - 1.13.8 + 1.13.11 3.2.0 3.5.1 4.6.0 From df722ac9c37795982b24bc86face0e8d3c817ebb Mon Sep 17 00:00:00 2001 From: bgrozev Date: Tue, 4 Jun 2024 13:48:57 -0700 Subject: [PATCH 139/189] ref(test): Disable kotest classpath autoscan (#2164) This change reduces test execution time by about 1 min (30%) on my machine. Since the update to kotest 5.9 the following warning gets printed during tests: Warning: Kotest autoscan is enabled. This means Kotest will scan the classpath for extensions that are annotated with @AutoScan. To avoid this startup cost, disable autoscan by setting the system property 'kotest.framework.classpath.scanning.autoscan.disable=true'. In 6.0 this value will default to true. For further details see https://kotest.io/docs/next/framework/project-config.html#runtime-detection --- jitsi-media-transform/pom.xml | 3 +++ jvb/pom.xml | 1 + rtp/pom.xml | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/jitsi-media-transform/pom.xml b/jitsi-media-transform/pom.xml index d2a1c77e4b..b77219c8d7 100644 --- a/jitsi-media-transform/pom.xml +++ b/jitsi-media-transform/pom.xml @@ -240,6 +240,9 @@ maven-surefire-plugin 3.2.5 + + true + DtlsStackTest.* diff --git a/jvb/pom.xml b/jvb/pom.xml index 2949b41ea3..4cd358354f 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -468,6 +468,7 @@ ${project.basedir}/lib/logging.properties + true diff --git a/rtp/pom.xml b/rtp/pom.xml index b24a4920b8..d9dd9cf63a 100644 --- a/rtp/pom.xml +++ b/rtp/pom.xml @@ -173,6 +173,11 @@ maven-surefire-plugin 3.2.5 + + + true + + com.github.gantsign.maven From b76c7b507d1cbd73d17ec44254114b43e7e686cd Mon Sep 17 00:00:00 2001 From: bgrozev Date: Mon, 10 Jun 2024 07:40:46 -0700 Subject: [PATCH 140/189] feat(config): Count iceTransport.send as part of transit time, confugurable timeline. (#2163) * feat(stats): Count iceTransport.send as part of transit time. * feat(config): Make the packet timeline configurable. * ref: Make PacketInfo.timeline nullable. * fix: Make the packet timeline thread safe. --- .../main/kotlin/org/jitsi/nlj/PacketInfo.kt | 65 ++++++++++++------- .../org/jitsi/nlj/transform/node/Node.kt | 6 +- .../node/debug/PayloadVerificationPlugin.kt | 2 +- .../src/main/resources/reference.conf | 14 +++- .../kotlin/org/jitsi/nlj/PacketInfoTest.kt | 45 +++++++++++++ .../kotlin/org/jitsi/videobridge/Endpoint.kt | 12 ++-- .../cc/allocation/PacketHandler.kt | 4 +- .../org/jitsi/videobridge/relay/Relay.kt | 8 +-- 8 files changed, 116 insertions(+), 40 deletions(-) create mode 100644 jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/PacketInfoTest.kt diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketInfo.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketInfo.kt index f23e064045..c943f9325c 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketInfo.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketInfo.kt @@ -16,7 +16,10 @@ package org.jitsi.nlj import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config import org.jitsi.rtp.Packet +import org.jitsi.utils.logging2.createLogger import java.time.Clock import java.time.Duration import java.time.Instant @@ -24,9 +27,18 @@ import java.util.Collections @SuppressFBWarnings("CN_IMPLEMENTS_CLONE_BUT_NOT_CLONEABLE") class EventTimeline( - private val timeline: MutableList> = mutableListOf(), + /** + * We want this thread safe, because while PacketInfo objects are only handled by a single thread at a time, + * [StatsKeepingNode] may add an "exit" event after the packets has been added to another queue potentially handled + * by a different thread. This is not critical as it only affects the timeline and the result is just some "exit" + * events missing from the trace logs. + */ + timelineArg: MutableList> = mutableListOf(), private val clock: Clock = Clock.systemUTC() ) : Iterable> { + + private val timeline = Collections.synchronizedList(timelineArg) + /** * The [referenceTime] refers to the first timestamp we have * in the timeline. In the timeline this is used as time "0" and @@ -34,6 +46,9 @@ class EventTimeline( */ var referenceTime: Instant? = null + val size: Int + get() = timeline.size + fun addEvent(desc: String) { val now = clock.instant() if (referenceTime == null) { @@ -43,7 +58,7 @@ class EventTimeline( } fun clone(): EventTimeline { - val clone = EventTimeline(timeline.toMutableList()) + val clone = EventTimeline(timeline.toMutableList(), clock) clone.referenceTime = referenceTime return clone } @@ -64,7 +79,9 @@ class EventTimeline( return with(StringBuffer()) { referenceTime?.let { append("Reference time: $referenceTime; ") - append(timeline.joinToString(separator = "; ")) + synchronized(timeline) { + append(timeline.joinToString(separator = "; ")) + } } ?: run { append("[No timeline]") } @@ -84,7 +101,7 @@ open class PacketInfo @JvmOverloads constructor( var packet: Packet, /** The original length of the packet, i.e. before decryption. Stays unchanged even if the packet is updated. */ val originalLength: Int = packet.length, - val timeline: EventTimeline = EventTimeline() + val timeline: EventTimeline? = if (enableTimeline) EventTimeline() else null ) { /** * An explicit tag for when this packet was originally received (assuming it @@ -93,7 +110,7 @@ open class PacketInfo @JvmOverloads constructor( var receivedTime: Instant? = null set(value) { field = value - if (ENABLE_TIMELINE && timeline.referenceTime == null) { + if (timeline != null && timeline.referenceTime == null) { timeline.referenceTime = value } } @@ -121,7 +138,7 @@ open class PacketInfo @JvmOverloads constructor( * The payload verification string for the packet, or 'null' if payload verification is disabled. Calculating the * it is expensive, thus we only do it when the flag is enabled. */ - var payloadVerification = if (ENABLE_PAYLOAD_VERIFICATION) packet.payloadVerification else null + var payloadVerification = if (enablePayloadVerification) packet.payloadVerification else null /** * Re-calculates the expected payload verification string. This should be called any time that the code @@ -129,7 +146,7 @@ open class PacketInfo @JvmOverloads constructor( * it with a new type (parsing), or intentionally modifies the payload (SRTP)). */ fun resetPayloadVerification() { - payloadVerification = if (ENABLE_PAYLOAD_VERIFICATION) packet.payloadVerification else null + payloadVerification = if (enablePayloadVerification) packet.payloadVerification else null } /** @@ -145,13 +162,7 @@ open class PacketInfo @JvmOverloads constructor( * will be copied for the cloned PacketInfo). */ fun clone(): PacketInfo { - val clone = if (ENABLE_TIMELINE) { - PacketInfo(packet.clone(), originalLength, timeline.clone()) - } else { - // If the timeline isn't enabled, we can just share the same one. - // (This would change if we allowed enabling the timeline at runtime) - PacketInfo(packet.clone(), originalLength, timeline) - } + val clone = PacketInfo(packet.clone(), originalLength, timeline?.clone()) clone.receivedTime = receivedTime clone.originalHadCryptex = originalHadCryptex clone.shouldDiscard = shouldDiscard @@ -163,11 +174,7 @@ open class PacketInfo @JvmOverloads constructor( return clone } - fun addEvent(desc: String) { - if (ENABLE_TIMELINE) { - timeline.addEvent(desc) - } - } + fun addEvent(desc: String) = timeline?.addEvent(desc) /** * The list of pending actions, or [null] if none. @@ -209,14 +216,28 @@ open class PacketInfo @JvmOverloads constructor( } companion object { - // TODO: we could make this a public var to allow changing this at runtime - private const val ENABLE_TIMELINE = false + private val enableTimeline: Boolean by config { + "jmt.debug.packet-timeline.enabled".from(JitsiConfig.newConfig) + } + + private val enablePayloadVerificationDefault: Boolean by config { + "jmt.debug.payload-verification.enabled".from(JitsiConfig.newConfig) + } /** * If this is enabled all [Node]s will verify that the payload didn't unexpectedly change. This is expensive. */ @field:Suppress("ktlint:standard:property-naming") - var ENABLE_PAYLOAD_VERIFICATION = false + var enablePayloadVerification = enablePayloadVerificationDefault + + init { + if (enableTimeline) { + createLogger().info("Packet timeline is enabled.") + } + if (enablePayloadVerification) { + createLogger().info("Payload verification is enabled.") + } + } } } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/Node.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/Node.kt index a2ece3eacd..5a9ea245d5 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/Node.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/Node.kt @@ -144,15 +144,15 @@ sealed class Node( if (enable) { PLUGINS_ENABLED = true plugins.add(PayloadVerificationPlugin) - PacketInfo.ENABLE_PAYLOAD_VERIFICATION = true + PacketInfo.enablePayloadVerification = true } else { plugins.remove(PayloadVerificationPlugin) PLUGINS_ENABLED = plugins.isNotEmpty() - PacketInfo.ENABLE_PAYLOAD_VERIFICATION = false + PacketInfo.enablePayloadVerification = false } } - fun isPayloadVerificationEnabled(): Boolean = PacketInfo.ENABLE_PAYLOAD_VERIFICATION + fun isPayloadVerificationEnabled(): Boolean = PacketInfo.enablePayloadVerification fun enableNodeTracing(enable: Boolean) { TRACE_ENABLED = enable diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/debug/PayloadVerificationPlugin.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/debug/PayloadVerificationPlugin.kt index 952ffb5093..2a37a862fb 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/debug/PayloadVerificationPlugin.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/debug/PayloadVerificationPlugin.kt @@ -38,7 +38,7 @@ class PayloadVerificationPlugin { fun getStatsJson() = JSONObject().apply { this["num_payload_verification_failures"] = numFailures.get() } override fun observe(after: Node, packetInfo: PacketInfo) { - if (PacketInfo.ENABLE_PAYLOAD_VERIFICATION && + if (PacketInfo.enablePayloadVerification && packetInfo.payloadVerification != null ) { val expected = packetInfo.payloadVerification diff --git a/jitsi-media-transform/src/main/resources/reference.conf b/jitsi-media-transform/src/main/resources/reference.conf index 3c3e1438cc..52ee7c5e21 100644 --- a/jitsi-media-transform/src/main/resources/reference.conf +++ b/jitsi-media-transform/src/main/resources/reference.conf @@ -137,9 +137,9 @@ jmt { pcap { // Whether to permit the API to dynamically enable the capture of // unencrypted PCAP files of media traffic. - enabled=false + enabled = false // The directory in which to place captured PCAP files. - directory="/tmp" + directory = "/tmp" } packet-loss { // Artificial loss to introduce in the receive pipeline. @@ -161,5 +161,15 @@ jmt { burst-interval = 0 } } + packet-timeline { + // Whether to enable the packet timeline. This is an expensive option used for debugging. + enabled = false + // Log a packet timeline for every one out of [log-fraction] packets. + log-fraction = 10000 + } + payload-verification { + // Whether to enable payload verification on startup. This is a very expensive option only used for debugging. + enabled = false + } } } diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/PacketInfoTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/PacketInfoTest.kt new file mode 100644 index 0000000000..99f4959130 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/PacketInfoTest.kt @@ -0,0 +1,45 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe + +class PacketInfoTest : ShouldSpec() { + override fun isolationMode(): IsolationMode = IsolationMode.InstancePerLeaf + + init { + context("EventTimeline test") { + val timeline = EventTimeline().apply { + addEvent("A") + addEvent("B") + } + val clone = timeline.clone() + timeline.size shouldBe 2 + clone.size shouldBe 2 + + timeline.addEvent("original") + timeline.size shouldBe 3 + clone.size shouldBe 2 + + clone.addEvent("clone") + clone.addEvent("clone2") + timeline.size shouldBe 3 + clone.size shouldBe 4 + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index ae893252ed..21103e37da 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -512,14 +512,14 @@ class Endpoint @JvmOverloads constructor( private fun doSendSrtp(packetInfo: PacketInfo): Boolean { packetInfo.addEvent(SRTP_QUEUE_EXIT_EVENT) - PacketTransitStats.packetSent(packetInfo) + iceTransport.send(packetInfo.packet.buffer, packetInfo.packet.offset, packetInfo.packet.length) + PacketTransitStats.packetSent(packetInfo) + ByteBufferPool.returnBuffer(packetInfo.packet.buffer) packetInfo.sent() if (timelineLogger.isTraceEnabled && logTimeline()) { timelineLogger.trace { packetInfo.timeline.toString() } } - iceTransport.send(packetInfo.packet.buffer, packetInfo.packet.offset, packetInfo.packet.length) - ByteBufferPool.returnBuffer(packetInfo.packet.buffer) return true } @@ -1176,9 +1176,11 @@ class Endpoint @JvmOverloads constructor( private val epTimeout = 2.mins private val timelineCounter = AtomicLong() - private val TIMELINE_FRACTION = 10000L + private val timelineFraction: Long by config { + "jmt.debug.packet-timeline.log-fraction".from(JitsiConfig.newConfig) + } - fun logTimeline() = timelineCounter.getAndIncrement() % TIMELINE_FRACTION == 0L + fun logTimeline() = timelineCounter.getAndIncrement() % timelineFraction == 0L private const val SRTP_QUEUE_ENTRY_EVENT = "Entered Endpoint SRTP sender outgoing queue" private const val SRTP_QUEUE_EXIT_EVENT = "Exited Endpoint SRTP sender outgoing queue" diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/PacketHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/PacketHandler.kt index 3cff98225c..6fd407dd43 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/PacketHandler.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/PacketHandler.kt @@ -17,7 +17,7 @@ package org.jitsi.videobridge.cc.allocation import org.jitsi.nlj.MediaSourceDesc import org.jitsi.nlj.PacketInfo -import org.jitsi.nlj.PacketInfo.Companion.ENABLE_PAYLOAD_VERIFICATION +import org.jitsi.nlj.PacketInfo.Companion.enablePayloadVerification import org.jitsi.nlj.RtpLayerDesc import org.jitsi.nlj.rtp.VideoRtpPacket import org.jitsi.rtp.rtcp.RtcpSrPacket @@ -83,7 +83,7 @@ internal class PacketHandler( adaptiveSourceProjection.rewriteRtp(packetInfo) // The rewriteRtp operation must not modify the VP8 payload. - if (ENABLE_PAYLOAD_VERIFICATION) { + if (enablePayloadVerification) { val expected = packetInfo.payloadVerification val actual = videoPacket.payloadVerification if ("" != expected && expected != actual) { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt index 4badda98d7..2199ac57f6 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt @@ -766,16 +766,14 @@ class Relay @JvmOverloads constructor( fun doSendSrtp(packetInfo: PacketInfo): Boolean { packetInfo.addEvent(SRTP_QUEUE_EXIT_EVENT) - PacketTransitStats.packetSent(packetInfo) + iceTransport.send(packetInfo.packet.buffer, packetInfo.packet.offset, packetInfo.packet.length) + PacketTransitStats.packetSent(packetInfo) + ByteBufferPool.returnBuffer(packetInfo.packet.buffer) packetInfo.sent() - if (timelineLogger.isTraceEnabled && Endpoint.logTimeline()) { timelineLogger.trace { packetInfo.timeline.toString() } } - - iceTransport.send(packetInfo.packet.buffer, packetInfo.packet.offset, packetInfo.packet.length) - ByteBufferPool.returnBuffer(packetInfo.packet.buffer) return true } From 8ed7ab8db463a19a2c83b29175f76973f4532cc1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 08:05:55 -0700 Subject: [PATCH 141/189] chore(deps): Bump bouncycastle.version from 1.77 to 1.78.1 (#2161) * chore(deps): Bump bouncycastle.version from 1.77 to 1.78.1 Bumps `bouncycastle.version` from 1.77 to 1.78.1. Updates `org.bouncycastle:bctls-jdk18on` from 1.77 to 1.78.1 - [Changelog](https://github.com/bcgit/bc-java/blob/main/docs/releasenotes.html) - [Commits](https://github.com/bcgit/bc-java/commits) Updates `org.bouncycastle:bcprov-jdk18on` from 1.77 to 1.78.1 - [Changelog](https://github.com/bcgit/bc-java/blob/main/docs/releasenotes.html) - [Commits](https://github.com/bcgit/bc-java/commits) Updates `org.bouncycastle:bcpkix-jdk18on` from 1.77 to 1.78.1 - [Changelog](https://github.com/bcgit/bc-java/blob/main/docs/releasenotes.html) - [Commits](https://github.com/bcgit/bc-java/commits) --- updated-dependencies: - dependency-name: org.bouncycastle:bctls-jdk18on dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: org.bouncycastle:bcprov-jdk18on dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: org.bouncycastle:bcpkix-jdk18on dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * chore: Update jitsi-srtp to 1.1-18-g98e4c5d. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Boris Grozev --- jitsi-media-transform/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jitsi-media-transform/pom.xml b/jitsi-media-transform/pom.xml index b77219c8d7..1687031bf5 100644 --- a/jitsi-media-transform/pom.xml +++ b/jitsi-media-transform/pom.xml @@ -22,7 +22,7 @@ ${project.groupId} jitsi-srtp - 1.1-16-g7fbe7e3 + 1.1-18-g98e4c5d ${project.groupId} diff --git a/pom.xml b/pom.xml index f07f5f140d..e90a528284 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ 4.6.0 3.1.7 2.12.4 - 1.77 + 1.78.1 0.16.0 UTF-8 From ab7c0a5237e37359b9a0c71b2ec5bd2924b4cff3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 08:41:44 -0700 Subject: [PATCH 142/189] chore(deps): Bump com.fasterxml.jackson.module:jackson-module-kotlin from 2.12.4 to 2.17.1 (#2166) chore(deps): Bump com.fasterxml.jackson.module:jackson-module-kotlin Bumps [com.fasterxml.jackson.module:jackson-module-kotlin](https://github.com/FasterXML/jackson-module-kotlin) from 2.12.4 to 2.17.1. - [Commits](https://github.com/FasterXML/jackson-module-kotlin/compare/jackson-module-kotlin-2.12.4...jackson-module-kotlin-2.17.1) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.module:jackson-module-kotlin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e90a528284..f6e562b307 100644 --- a/pom.xml +++ b/pom.xml @@ -34,7 +34,7 @@ 3.5.1 4.6.0 3.1.7 - 2.12.4 + 2.17.1 1.78.1 0.16.0 UTF-8 From 33c67b6defafa56d8febff99550510253157e91b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 08:42:15 -0700 Subject: [PATCH 143/189] chore(deps): Bump kotest.version from 5.9.0 to 5.9.1 (#2165) Bumps `kotest.version` from 5.9.0 to 5.9.1. Updates `io.kotest:kotest-runner-junit5-jvm` from 5.9.0 to 5.9.1 - [Release notes](https://github.com/kotest/kotest/releases) - [Commits](https://github.com/kotest/kotest/compare/v5.9.0...v5.9.1) Updates `io.kotest:kotest-assertions-core-jvm` from 5.9.0 to 5.9.1 - [Release notes](https://github.com/kotest/kotest/releases) - [Commits](https://github.com/kotest/kotest/compare/v5.9.0...v5.9.1) Updates `io.kotest:kotest-property-jvm` from 5.9.0 to 5.9.1 - [Release notes](https://github.com/kotest/kotest/releases) - [Commits](https://github.com/kotest/kotest/compare/v5.9.0...v5.9.1) --- updated-dependencies: - dependency-name: io.kotest:kotest-runner-junit5-jvm dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: io.kotest:kotest-assertions-core-jvm dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: io.kotest:kotest-property-jvm dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f6e562b307..dee73ea88b 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ 11.0.21 2.0.0 - 5.9.0 + 5.9.1 5.10.2 1.0-132-g83984af 1.1-140-g8f45a9f From b1066f38dc2b0c552a1d62c67a1ce948b458a725 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 08:42:28 -0700 Subject: [PATCH 144/189] chore(deps): Bump org.jetbrains:annotations from 23.0.0 to 24.1.0 (#2167) Bumps [org.jetbrains:annotations](https://github.com/JetBrains/java-annotations) from 23.0.0 to 24.1.0. - [Release notes](https://github.com/JetBrains/java-annotations/releases) - [Changelog](https://github.com/JetBrains/java-annotations/blob/master/CHANGELOG.md) - [Commits](https://github.com/JetBrains/java-annotations/compare/23.0.0...24.1.0) --- updated-dependencies: - dependency-name: org.jetbrains:annotations dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- jvb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/pom.xml b/jvb/pom.xml index 4cd358354f..46da1f21a6 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -86,7 +86,7 @@ org.jetbrains annotations - 23.0.0 + 24.1.0 From 5273f2fd9e7b12d5b4bd84cac1c7035705f16b22 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 08:42:49 -0700 Subject: [PATCH 145/189] chore(deps): Bump com.github.spotbugs:spotbugs-maven-plugin from 4.7.0.0 to 4.8.5.0 (#2169) chore(deps): Bump com.github.spotbugs:spotbugs-maven-plugin Bumps [com.github.spotbugs:spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.7.0.0 to 4.8.5.0. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.7.0.0...spotbugs-maven-plugin-4.8.5.0) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- jitsi-media-transform/pom.xml | 2 +- jvb/pom.xml | 2 +- rtp/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jitsi-media-transform/pom.xml b/jitsi-media-transform/pom.xml index 1687031bf5..6ce881ca0b 100644 --- a/jitsi-media-transform/pom.xml +++ b/jitsi-media-transform/pom.xml @@ -288,7 +288,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.7.0.0 + 4.8.5.0 ${project.basedir}/spotbugs-exclude.xml true diff --git a/jvb/pom.xml b/jvb/pom.xml index 46da1f21a6..f50e07fb3c 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -475,7 +475,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.7.0.0 + 4.8.5.0 ${project.basedir}/spotbugs-exclude.xml true diff --git a/rtp/pom.xml b/rtp/pom.xml index d9dd9cf63a..d28d38664d 100644 --- a/rtp/pom.xml +++ b/rtp/pom.xml @@ -203,7 +203,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.7.0.0 + 4.8.5.0 ${project.basedir}/spotbugs-exclude.xml true From 0083ed0660563baec71b01b2a76e42c114e460e4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 08:43:04 -0700 Subject: [PATCH 146/189] chore(deps): Bump org.apache.maven.plugins:maven-shade-plugin from 3.5.1 to 3.6.0 (#2170) chore(deps): Bump org.apache.maven.plugins:maven-shade-plugin Bumps [org.apache.maven.plugins:maven-shade-plugin](https://github.com/apache/maven-shade-plugin) from 3.5.1 to 3.6.0. - [Release notes](https://github.com/apache/maven-shade-plugin/releases) - [Commits](https://github.com/apache/maven-shade-plugin/compare/maven-shade-plugin-3.5.1...maven-shade-plugin-3.6.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-shade-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dee73ea88b..700e5b63d7 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ 1.1-140-g8f45a9f 1.13.11 3.2.0 - 3.5.1 + 3.6.0 4.6.0 3.1.7 2.17.1 From 059cd3e75f6e5683b3ae0c7e487d7ae0fd05ca54 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Tue, 11 Jun 2024 13:04:58 -0700 Subject: [PATCH 147/189] feat: Add a timeline event when the packet was sent over ICE. (#2171) * feat: Add a timeline event when the packet was sent over ICE. * feat: Add another analize-timeline script. --- .../kotlin/org/jitsi/videobridge/Endpoint.kt | 2 + resources/analyze-timeline2.pl | 244 ++++++++++++++++++ 2 files changed, 246 insertions(+) create mode 100755 resources/analyze-timeline2.pl diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index 21103e37da..9ce14e42f4 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -514,6 +514,7 @@ class Endpoint @JvmOverloads constructor( packetInfo.addEvent(SRTP_QUEUE_EXIT_EVENT) iceTransport.send(packetInfo.packet.buffer, packetInfo.packet.offset, packetInfo.packet.length) + packetInfo.addEvent(SENT_OVER_ICE_TRANSPORT_EVENT) PacketTransitStats.packetSent(packetInfo) ByteBufferPool.returnBuffer(packetInfo.packet.buffer) packetInfo.sent() @@ -1184,6 +1185,7 @@ class Endpoint @JvmOverloads constructor( private const val SRTP_QUEUE_ENTRY_EVENT = "Entered Endpoint SRTP sender outgoing queue" private const val SRTP_QUEUE_EXIT_EVENT = "Exited Endpoint SRTP sender outgoing queue" + private const val SENT_OVER_ICE_TRANSPORT_EVENT = "Sent over the ICE transport" private val statsFilterThreshold: Int by config { "videobridge.stats-filter-threshold".from(JitsiConfig.newConfig) diff --git a/resources/analyze-timeline2.pl b/resources/analyze-timeline2.pl new file mode 100755 index 0000000000..a4b1df63e6 --- /dev/null +++ b/resources/analyze-timeline2.pl @@ -0,0 +1,244 @@ +#!/usr/bin/perl -w + +use Statistics::Descriptive; +use strict; + +my %stats; +my @stat_names; + +/* +This is extremely ugly, I'm driving perl in first gear because I don't know how to shift :) + +It parses the PacketInfo EventTimeline logs (see example below) and produces stats for +the following periods from the life cycle of a packet: + +1.Receiver Queue: From the moment the packet is read from the ICE transport (in an IO thread) and added the the Receiver Queue +to the moment it is removed from the Receiver Queue (in a CPU thread) and the pipeline processing starts. + +2.Receiver Pipeline: From the start of the Receiver Pipeline to the end, excluding the last Termination node. This is all +performed in the same CPU thread, and includes things like SRTP decryption/authentication, RTP parsing, updating bandwidth +estimation, video codec parsing, etc. + +3.Termination Node: This is technically the last Node in the Receiver Pipeline and executes in the same thread, but +it is conceptually different and worth looking at separately. It consists of conference-level handling of the packet: it loops +through all local endpoints and relays, checks whether they "want" the packet, and if they do then clones the packet and puts +the clone on the sender queue (see Conference#sendOut). This workload scales with the number of local endpoits and relays, +and in a large conference is the most computationally expensive. + +Note that currently Endpoint#send(PacketInfo) contains code for SSRC rewriting and a transformation from the BitrateController. +This can be offloaded to the sender queue and we plan to do so soon. + +4.Sender Queue: From the time the Receiver Pipeline thread places the packet on the Sender Queue, to the time another CPU thread +removes it from the queue and starts executing the Sender Pipeline. + +5.Sender Pipeline: From the start of the Sender Pipeline to the end. This executes in a CPU thread and includes stripping unknown (for +this specific receiving endpoint) RTP header extensions, saving a copy of the packet in the NACK cache, adding/updating the transport-cc +header extension, and SRTP encryption. + +This script looks only at RTP packets (ignoring RTCP and retransmissions) for simplicity. + +6.SRTP Queue: From the time a SRTP packet is placed on the queue by the Sender Pipeline thread to the time an IO thread removes it +from the queue and starts sending it over the ICE transport. + +7.Send over ICE transport: The time it took to execute IceTransport.send, which sends the packet through the ice4j socket representation. + +8.Total: From the start of 1. to the end of 7. + + + +Example log line: +JVB 2024-06-11 17:57:44.989 FINER: [1531] [confId=6149c0c339432f97 conf_name=loadtest0@conference.xxx meeting_id=ec7261c3 epId=c88aca35 stats_id=Otto-emV] Endpoint.doSendSrtp#544: Reference time: 2024-06-11T17:57:44.988517466Z; (Entered RTP receiver incoming queue, PT0.00000192S); (Exited RTP receiver incoming queue, PT0.000324242S); (Entered node PacketStreamStats, PT0.000324602S); (Exited node PacketStreamStats, PT0.000325042S); (Entered node SRTP/SRTCP demuxer, PT0.000325162S); (Exited node SRTP/SRTCP demuxer, PT0.000325722S); (Entered node RTP Parser, PT0.000325962S); (Exited node RTP Parser, PT0.000326762S); (Entered node Audio level reader (pre-srtp), PT0.000326922S); (Exited node Audio level reader (pre-srtp), PT0.000327242S); (Entered node Video mute node, PT0.000327362S); (Exited node Video mute node, PT0.000327642S); (Entered node SRTP Decrypt Node, PT0.000327762S); (Exited node SRTP Decrypt Node, PT0.000332442S); (Entered node TCC generator, PT0.000332562S); (Exited node TCC generator, PT0.000333562S); (Entered node Remote Bandwidth Estimator, PT0.000344962S); (Exited node Remote Bandwidth Estimator, PT0.000345202S); (Entered node Audio level reader (post-srtp), PT0.000345322S); (Exited node Audio level reader (post-srtp), PT0.000345442S); (Entered node Toggleable pcap writer: 21913831-rx, PT0.000345562S); (Exited node Toggleable pcap writer: 21913831-rx, PT0.000345762S); (Entered node Incoming statistics tracker, PT0.000345882S); (Exited node Incoming statistics tracker, PT0.000351242S); (Entered node Padding termination, PT0.000351362S); (Exited node Padding termination, PT0.000351522S); (Entered node Media Type demuxer, PT0.000351642S); (Exited node Media Type demuxer, PT0.000351962S); (Entered node RTX handler, PT0.000352082S); (Exited node RTX handler, PT0.000352322S); (Entered node Duplicate termination, PT0.000352722S); (Exited node Duplicate termination, PT0.000355442S); (Entered node Retransmission requester, PT0.000355882S); (Exited node Retransmission requester, PT0.000357922S); (Entered node Padding-only discarder, PT0.000358642S); (Exited node Padding-only discarder, PT0.000359842S); (Entered node Video parser, PT0.000360242S); (Exited node Video parser, PT0.000363082S); (Entered node Video quality layer lookup, PT0.000363362S); (Exited node Video quality layer lookup, PT0.000364362S); (Entered node Video bitrate calculator, PT0.000364682S); (Exited node Video bitrate calculator, PT0.000369122S); (Entered node Input pipeline termination node, PT0.000369362S); (Entered node receiver chain handler, PT0.000370122S); (Entered RTP sender incoming queue, PT0.000488083S); (Exited RTP sender incoming queue, PT0.000551283S); (Entered node Pre-processor, PT0.000551723S); (Exited node Pre-processor, PT0.000555843S); (Entered node RedHandler, PT0.000556083S); (Exited node RedHandler, PT0.000556563S); (Entered node Strip header extensions, PT0.000556843S); (Exited node Strip header extensions, PT0.000557523S); (Entered node Packet cache, PT0.000557923S); (Exited node Packet cache, PT0.000561083S); (Entered node Absolute send time, PT0.000561243S); (Exited node Absolute send time, PT0.000561643S); (Entered node Outgoing statistics tracker, PT0.000561923S); (Exited node Outgoing statistics tracker, PT0.000562683S); (Entered node TCC sequence number tagger, PT0.000563003S); (Exited node TCC sequence number tagger, PT0.000563843S); (Entered node Header extension encoder, PT0.000564083S); (Exited node Header extension encoder, PT0.000565163S); (Entered node Toggleable pcap writer: c88aca35-tx, PT0.000565443S); (Exited node Toggleable pcap writer: c88aca35-tx, PT0.000565763S); (Entered node SRTP Encrypt Node, PT0.000566083S); (Exited node SRTP Encrypt Node, PT0.000572883S); (Entered node PacketStreamStats, PT0.000573243S); (Exited node PacketStreamStats, PT0.000573883S); (Entered node Output pipeline termination node, PT0.000574123S); (Entered Endpoint SRTP sender outgoing queue, PT0.000574443S); (Exited node Output pipeline termination node, PT0.000582923S); (Exited Endpoint SRTP sender outgoing queue, PT0.000668003S); (Sent over the ICE transport, PT0.000693284S) + +*/ + +my $receiverQStartName = "Entered RTP receiver incoming queue"; +my $receiverQEndName = "Exited RTP receiver incoming queue"; +my $receiverPStartName = "Entered node SRTP/SRTCP demuxer"; +my $receiverPEndName = "Entered node Input pipeline termination node"; +my $terminationNodeStartName = "Entered node Input pipeline termination node"; +my $terminationNodeEndName = "Exited node Input pipeline termination node"; +my $senderQStartName = "Entered RTP sender incoming queue"; +my $senderQEndName = "Exited RTP sender incoming queue"; +my $senderPStartName = "Entered node RedHandler"; +my $senderPEndName = "Entered Endpoint SRTP sender outgoing queue"; +my $srtpQStartName = "Entered Endpoint SRTP sender outgoing queue"; +my $srtpQEndName = "Exited Endpoint SRTP sender outgoing queue"; +my $iceStartName = "Exited Endpoint SRTP sender outgoing queue"; +my $iceEndName = "Sent over the ICE transport"; +my $totalStartName = "Entered RTP receiver incoming queue"; +my $totalEndName = "Sent over the ICE transport"; + +while (<>) { + if (/Reference time: /) { + my $prev_time; + my $receiverQStart = -1; + my $receiverQEnd = -1; + my $receiverPStart = -1; + my $receiverPEnd = -1 ; + my $terminationNodeStart = -1 ; + my $terminationNodeEnd = -1; + my $senderQStart = -1; + my $senderQEnd = -1; + my $senderPStart = -1; + my $senderPEnd = -1 ; + my $srtpQStart = -1; + my $srtpQEnd = -1; + my $iceStart = -1; + my $iceEnd = -1; + my $totalStart = -1; + my $totalEnd = -1; + + while (/; \(([^)]*), PT([0-9.]*)/g) { + my $name = $1; + my $time = $2; + + if ($name eq $receiverQStartName) { + $receiverQStart = $time; + } + if ($name eq $receiverQEndName) { + $receiverQEnd = $time; + } + if ($name eq $receiverPStartName) { + $receiverPStart = $time; + } + if ($name eq $receiverPEndName) { + $receiverPEnd = $time; + } + if ($name eq $terminationNodeStartName) { + $terminationNodeStart = $time; + } + if ($name eq $terminationNodeEndName) { + $terminationNodeEnd = $time; + } + if ($name eq $senderQStartName) { + $senderQStart = $time; + } + if ($name eq $senderQEndName) { + $senderQEnd = $time; + } + if ($name eq $senderPStartName) { + $senderPStart = $time; + } + if ($name eq $senderPEndName) { + $senderPEnd = $time; + } + if ($name eq $srtpQStartName) { + $srtpQStart = $time; + } + if ($name eq $srtpQEndName) { + $srtpQEnd = $time; + } + if ($name eq $iceStartName) { + $iceStart = $time; + } + if ($name eq $iceEndName) { + $iceEnd = $time; + } + if ($name eq $totalStartName) { + $totalStart = $time; + } + if ($name eq $totalEndName) { + $totalEnd = $time; + } + + #my $delta_time; + #if (defined $prev_time) { + # $delta_time = $time - $prev_time; + #} + #else { + # $delta_time = $time; + #} + #$prev_time = $time; + + #if ($name =~ /Toggleable pcap writer:/) { + # $name =~ s/writer: .*/writer/; + #} + } + if ($receiverQStart > -1 && $receiverQEnd > -1) { + my $name = "1.Receiver Queue"; + my $time = $receiverQEnd - $receiverQStart; + + if (!defined ($stats{$name})) { + $stats{$name} = Statistics::Descriptive::Full->new(); + push(@stat_names, $name); + } + $stats{$name}->add_data($time * 1e3); + } + if ($receiverPStart > -1 && $receiverPEnd > -1) { + my $name = "2.Receiver Pipeline"; + my $time = $receiverPEnd - $receiverPStart; + + if (!defined ($stats{$name})) { + $stats{$name} = Statistics::Descriptive::Full->new(); + push(@stat_names, $name); + } + $stats{$name}->add_data($time * 1e3); + } + if ($terminationNodeStart > -1 && $terminationNodeEnd > -1) { + my $name = "3.Termination Node"; + my $time = $terminationNodeEnd - $terminationNodeStart; + + if (!defined ($stats{$name})) { + $stats{$name} = Statistics::Descriptive::Full->new(); + push(@stat_names, $name); + } + $stats{$name}->add_data($time * 1e3); + } + if ($senderQStart > -1 && $senderQEnd > -1) { + my $name = "4.Sender Queue"; + my $time = $senderQEnd - $senderQStart; + + if (!defined ($stats{$name})) { + $stats{$name} = Statistics::Descriptive::Full->new(); + push(@stat_names, $name); + } + $stats{$name}->add_data($time * 1e3); + } + if ($senderPStart > -1 && $senderPEnd > -1) { + my $name = "5.Sender Pipeline"; + my $time = $senderPEnd - $senderPStart; + + if (!defined ($stats{$name})) { + $stats{$name} = Statistics::Descriptive::Full->new(); + push(@stat_names, $name); + } + $stats{$name}->add_data($time * 1e3); + } + if ($srtpQStart > -1 && $srtpQEnd > -1) { + my $name = "6.SRTP Queue"; + my $time = $srtpQEnd - $srtpQStart; + + if (!defined ($stats{$name})) { + $stats{$name} = Statistics::Descriptive::Full->new(); + push(@stat_names, $name); + } + $stats{$name}->add_data($time * 1e3); + } + if ($iceStart > -1 && $iceEnd > -1) { + my $name = "7.Send over ICE transport"; + my $time = $iceEnd - $iceStart; + + if (!defined ($stats{$name})) { + $stats{$name} = Statistics::Descriptive::Full->new(); + push(@stat_names, $name); + } + $stats{$name}->add_data($time * 1e3); + } + if ($totalStart > -1 && $totalEnd > -1) { + my $name = "8.Total"; + my $time = $totalEnd - $totalStart; + + if (!defined ($stats{$name})) { + $stats{$name} = Statistics::Descriptive::Full->new(); + push(@stat_names, $name); + } + $stats{$name}->add_data($time * 1e3); + } + } +} + +for my $name (@stat_names) { + my $s = $stats{$name}; + printf("%s: min %.3f ms, mean %.3f ms, median %.3f ms, 90%% %.3f ms, 99%% %.3f ms, max %.3f ms\n", + $name, $s->min(), $s->mean(), $s->median(), scalar($s->percentile(90)), scalar($s->percentile(99)), $s->max()); +} From 93c41cc1b1004dd6ad6e0dcfe35d62de9ef63f61 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Wed, 12 Jun 2024 07:31:40 -0700 Subject: [PATCH 148/189] feat: Move some processing to the sender pipeline (#2172) Moves the bitrate controller transform (applying source projection logic) and SSRC rewriting to the sender pipeline. Previously an RtpReceiver's pipeline was running this code for every target endpoint. This reduces the packet processing time for large conferences since the work is split to multiple threads. --- .../kotlin/org/jitsi/nlj/RtpReceiverImpl.kt | 2 +- .../main/kotlin/org/jitsi/nlj/RtpSender.kt | 5 ++ .../kotlin/org/jitsi/nlj/RtpSenderImpl.kt | 7 ++ .../main/kotlin/org/jitsi/nlj/Transceiver.kt | 2 +- .../kotlin/org/jitsi/videobridge/Endpoint.kt | 71 ++++++++++--------- resources/analyze-timeline2.pl | 3 - 6 files changed, 51 insertions(+), 39 deletions(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt index 152a1587e6..079001f481 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt @@ -78,7 +78,7 @@ import java.util.concurrent.ScheduledExecutorService class RtpReceiverImpl @JvmOverloads constructor( val id: String, /** - * A function to be used when these receiver wants to send RTCP packets to the + * A function to be used when the receiver wants to send RTCP packets to the * participant it's receiving data from (NACK packets, for example) */ private val rtcpSender: (RtcpPacket) -> Unit = {}, diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSender.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSender.kt index 639a4373ab..241fbde39e 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSender.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSender.kt @@ -48,5 +48,10 @@ abstract class RtpSender : abstract fun isFeatureEnabled(feature: Features): Boolean abstract fun tearDown() + /** + * An optional function to be executed for each RTP packet, as the first step of the send pipeline. + */ + var preProcesor: ((PacketInfo) -> PacketInfo?)? = null + abstract val bandwidthEstimator: BandwidthEstimator } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt index 3c8272d11f..db2ae9d031 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt @@ -41,6 +41,7 @@ import org.jitsi.nlj.transform.node.PacketStreamStatsNode import org.jitsi.nlj.transform.node.SrtcpEncryptNode import org.jitsi.nlj.transform.node.SrtpEncryptNode import org.jitsi.nlj.transform.node.ToggleablePcapWriter +import org.jitsi.nlj.transform.node.TransformerNode import org.jitsi.nlj.transform.node.outgoing.AbsSendTime import org.jitsi.nlj.transform.node.outgoing.HeaderExtEncoder import org.jitsi.nlj.transform.node.outgoing.HeaderExtStripper @@ -140,6 +141,12 @@ class RtpSenderImpl( incomingPacketQueue.setErrorHandler(queueErrorCounter) outgoingRtpRoot = pipeline { + node(object : TransformerNode("Pre-processor") { + override fun transform(packetInfo: PacketInfo): PacketInfo? { + return preProcesor?.invoke(packetInfo) ?: packetInfo + } + override fun trace(f: () -> Unit) {} + }) node(AudioRedHandler(streamInformationStore, logger)) node(HeaderExtStripper(streamInformationStore)) node(outgoingPacketCache) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Transceiver.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Transceiver.kt index d2302cf63f..8c4b468490 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Transceiver.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Transceiver.kt @@ -108,7 +108,7 @@ class Transceiver( */ fun isReceivingVideo(): Boolean = rtpReceiver.isReceivingVideo() - private val rtpSender: RtpSender = RtpSenderImpl( + val rtpSender: RtpSender = RtpSenderImpl( id, rtcpEventNotifier, senderExecutor, diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index 9ce14e42f4..1b6a3c7bab 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -297,6 +297,42 @@ class Endpoint @JvmOverloads constructor( addEndpointConnectionStatsListener(rttListener) setLocalSsrc(MediaType.AUDIO, conference.localAudioSsrc) setLocalSsrc(MediaType.VIDEO, conference.localVideoSsrc) + rtpSender.preProcesor = { packetInfo -> preProcess(packetInfo) } + } + + /** + * Perform processing of the packet before it goes through the rest of the [transceiver] send pipeline: + * 1. Update the bitrate controller state and apply the source projection logic + * 2. Perform SSRC re-writing if [doSsrcRewriting] is set. + */ + private fun preProcess(packetInfo: PacketInfo): PacketInfo? { + when (val packet = packetInfo.packet) { + is VideoRtpPacket -> { + if (!bitrateController.transformRtp(packetInfo)) { + logger.warn("Dropping a packet which was supposed to be accepted:$packet") + return null + } + // The original packet was transformed in place. + if (doSsrcRewriting) { + val start = packet !is ParsedVideoPacket || (packet.isKeyframe && packet.isStartOfFrame) + if (!videoSsrcs.rewriteRtp(packet, start)) { + return null + } + } + } + is AudioRtpPacket -> if (doSsrcRewriting) audioSsrcs.rewriteRtp(packet) + is RtcpSrPacket -> { + // Allow the BC to update the timestamp (in place). + bitrateController.transformRtcp(packet) + if (doSsrcRewriting) { + // Just check both tables instead of looking up the type first. + if (!videoSsrcs.rewriteRtcp(packet) && !audioSsrcs.rewriteRtcp(packet)) { + return null + } + } + } + } + return packetInfo } private val bandwidthProbing = BandwidthProbing( @@ -890,40 +926,7 @@ class Endpoint @JvmOverloads constructor( } } - override fun send(packetInfo: PacketInfo) { - when (val packet = packetInfo.packet) { - is VideoRtpPacket -> { - if (bitrateController.transformRtp(packetInfo)) { - // The original packet was transformed in place. - if (doSsrcRewriting) { - val start = packet !is ParsedVideoPacket || (packet.isKeyframe && packet.isStartOfFrame) - if (!videoSsrcs.rewriteRtp(packet, start)) { - return - } - } - transceiver.sendPacket(packetInfo) - } else { - logger.warn("Dropping a packet which was supposed to be accepted:$packet") - } - return - } - is AudioRtpPacket -> if (doSsrcRewriting) audioSsrcs.rewriteRtp(packet) - is RtcpSrPacket -> { - // Allow the BC to update the timestamp (in place). - bitrateController.transformRtcp(packet) - if (doSsrcRewriting) { - // Just check both tables instead of looking up the type first. - if (!videoSsrcs.rewriteRtcp(packet) && !audioSsrcs.rewriteRtcp(packet)) { - return - } - } - logger.trace { - "relaying an sr from ssrc=${packet.senderSsrc}, timestamp=${packet.senderInfo.rtpTimestamp}" - } - } - } - transceiver.sendPacket(packetInfo) - } + override fun send(packetInfo: PacketInfo) = transceiver.sendPacket(packetInfo) /** * To find out whether the endpoint should be expired, we check the activity timestamps from the transceiver. diff --git a/resources/analyze-timeline2.pl b/resources/analyze-timeline2.pl index a4b1df63e6..9b4987b306 100755 --- a/resources/analyze-timeline2.pl +++ b/resources/analyze-timeline2.pl @@ -25,9 +25,6 @@ the clone on the sender queue (see Conference#sendOut). This workload scales with the number of local endpoits and relays, and in a large conference is the most computationally expensive. -Note that currently Endpoint#send(PacketInfo) contains code for SSRC rewriting and a transformation from the BitrateController. -This can be offloaded to the sender queue and we plan to do so soon. - 4.Sender Queue: From the time the Receiver Pipeline thread places the packet on the Sender Queue, to the time another CPU thread removes it from the queue and starts executing the Sender Pipeline. From af2ff245f7cca5d923350a49882e71cf09d52445 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Mon, 17 Jun 2024 16:01:26 -0400 Subject: [PATCH 149/189] Fix(dcsctp): Allow unlimited sctp retransmissions with dcsctp. (#2180) Don't consider streams reset a "Surprising" SCTP callback. --- .../org/jitsi/videobridge/dcsctp/DcSctpTransport.kt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpTransport.kt index a048c465a3..8269c5c76a 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpTransport.kt @@ -86,6 +86,11 @@ class DcSctpTransport( check(SctpConfig.config.enabled()) { "SCTP is disabled in configuration" } DcSctpOptions().apply { maxTimerBackoffDuration = DEFAULT_MAX_TIMER_DURATION + // Because we're making retransmits faster, we need to allow unlimited retransmits + // or SCTP can time out (which we don't handle). Peer connection timeouts are handled at + // a higher layer. + maxRetransmissions = null + maxInitRetransmits = null } } @@ -127,7 +132,9 @@ abstract class DcSctpBaseCallbacks( } override fun OnStreamsResetPerformed(outgoingStreams: ShortArray) { - transport.logger.info("Surprising SCTP callback: outgoing streams ${outgoingStreams.joinToString()} reset") + // This is normal following a call to close(), which is a hard-close (as opposed to shutdown() which is + // soft-close) + transport.logger.info("Outgoing streams ${outgoingStreams.joinToString()} reset") } override fun OnIncomingStreamsReset(incomingStreams: ShortArray) { From 793df5a91a51b253c75dd0b2ab9045fd4d05e230 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Fri, 21 Jun 2024 14:40:09 -0400 Subject: [PATCH 150/189] Fix(dcsctp): Fix memory leak. (#2182) There were reference cycles through JNI, which the garbage collector can't detect. --- jvb/pom.xml | 2 +- .../org/jitsi/videobridge/dcsctp/DcSctpTransport.kt | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/jvb/pom.xml b/jvb/pom.xml index f50e07fb3c..1aeab10a92 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -113,7 +113,7 @@ ${project.groupId} jitsi-dcsctp - 1.0-2-g2d8eee4 + 1.0-3-gaf9d564 diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpTransport.kt index 8269c5c76a..07d64f9343 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpTransport.kt @@ -27,6 +27,7 @@ import org.jitsi.utils.logging2.Logger import org.jitsi.utils.logging2.createChildLogger import org.jitsi.videobridge.sctp.SctpConfig import org.jitsi.videobridge.util.TaskPools +import java.lang.ref.WeakReference import java.time.Clock import java.time.Instant import java.util.concurrent.Future @@ -109,7 +110,7 @@ abstract class DcSctpBaseCallbacks( ) : DcSctpSocketCallbacks { /* Methods we can usefully implement for every JVB socket */ override fun createTimeout(p0: DcSctpSocketCallbacks.DelayPrecision): Timeout { - return ATimeout() + return ATimeout(transport) } override fun Now(): Instant { @@ -142,7 +143,11 @@ abstract class DcSctpBaseCallbacks( transport.logger.info("Surprising SCTP callback: incoming streams ${incomingStreams.joinToString()} reset") } - private inner class ATimeout : Timeout { + private class ATimeout(transport: DcSctpTransport) : Timeout { + // This holds a weak reference to the transport, to break JNI reference cycles + private val weakTransport = WeakReference(transport) + private val transport: DcSctpTransport? + get() = weakTransport.get() private var timeoutId: Long = 0 private var scheduledFuture: ScheduledFuture<*>? = null private var future: Future<*>? = null @@ -152,11 +157,11 @@ abstract class DcSctpBaseCallbacks( scheduledFuture = TaskPools.SCHEDULED_POOL.schedule({ /* Execute it on the IO_POOL, because a timer may trigger sending new SCTP packets. */ future = TaskPools.IO_POOL.submit { - transport.socket.handleTimeout(timeoutId) + transport?.socket?.handleTimeout(timeoutId) } }, duration, TimeUnit.MILLISECONDS) } catch (e: Throwable) { - transport.logger.warn("Exception scheduling DCSCTP timeout", e) + transport?.logger?.warn("Exception scheduling DCSCTP timeout", e) } } From b67f0c0de54db2c1c21ddfabc79ef4baafe11b7c Mon Sep 17 00:00:00 2001 From: bgrozev Date: Tue, 25 Jun 2024 15:36:24 -0500 Subject: [PATCH 151/189] feat(log): Always log when a MucClient is added/removed. (#2183) --- .../kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt index 2fbf52ca07..bfe91f869f 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt @@ -145,6 +145,7 @@ class XmppConnection : IQListener { } } + logger.info("Adding MucClient for ${config.id}") mucClientManager.addMucClient(config) return true } @@ -179,10 +180,13 @@ class XmppConnection : IQListener { * returns {@code false}. */ fun removeMucClient(jsonObject: JSONObject): Boolean { - if (jsonObject["id"] !is String) { + val id = jsonObject["id"] + if (id !is String) { + logger.info("Invalid ID: $id") return false } - return mucClientManager.removeMucClient(jsonObject["id"] as String) + logger.info("Removing muc client $id") + return mucClientManager.removeMucClient(id) } /** From 448da465d5b81febf964d7fad534d9f83605defa Mon Sep 17 00:00:00 2001 From: bgrozev Date: Tue, 9 Jul 2024 12:04:26 +0300 Subject: [PATCH 152/189] Fix jetty versions (#2186) * chore: Update jicoco. * fix(#2184): Revert "chore(deps): Bump jersey.version from 3.0.10 to 3.1.7 (#2145)" Fixes issues with mismatched jetty versions reported in #2184 This reverts commit a14d4919dc0d17e183e5ed99d0b540474c2fb7c2. --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 700e5b63d7..855cd6c104 100644 --- a/pom.xml +++ b/pom.xml @@ -28,14 +28,14 @@ 5.9.1 5.10.2 1.0-132-g83984af - 1.1-140-g8f45a9f + 1.1-141-g30ec741 1.13.11 3.2.0 3.6.0 4.6.0 - 3.1.7 2.17.1 1.78.1 + 3.0.10 0.16.0 UTF-8 From d98c8b56fc18536039d91c0d4c6fe4c5bc74cfb3 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 9 Jul 2024 10:51:40 -0400 Subject: [PATCH 153/189] Use dcsctp by default for sctp. (#2181) --- jvb/src/main/resources/reference.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index e0991d042a..cfff64409c 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -221,7 +221,7 @@ videobridge { enabled = true // Whether to use the usrsctp-based implementation instead of the new dcsctp-based one. - use-usrsctp = true + use-usrsctp = false } stats { // The interval at which stats are gathered. From 877b6a2f0fdb98f440e094678116b3b090f59533 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Wed, 10 Jul 2024 14:12:16 -0400 Subject: [PATCH 154/189] Feat: Redact remote endpoint IP addresses in log messages. (#2188) --- .../java/org/jitsi/videobridge/Conference.java | 16 ++++++++++++++-- .../org/jitsi/videobridge/VideobridgeConfig.kt | 5 +++++ jvb/src/main/resources/application.conf | 1 + jvb/src/main/resources/reference.conf | 3 +++ pom.xml | 4 ++-- 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/Conference.java b/jvb/src/main/java/org/jitsi/videobridge/Conference.java index 4e751391c7..cb5ff73882 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Conference.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Conference.java @@ -224,7 +224,14 @@ public Conference(Videobridge videobridge, { try { - logger.info("RECV colibri2 request: " + XmlStringBuilderUtil.toStringOpt(request.getRequest())); + logger.info( () -> { + String reqStr = XmlStringBuilderUtil.toStringOpt(request.getRequest()); + if (VideobridgeConfig.getRedactRemoteAddresses()) + { + reqStr = RedactColibriIp.Companion.redact(reqStr); + } + return "RECV colibri2 request: " + reqStr; + }); long start = System.currentTimeMillis(); Pair p = colibri2Handler.handleConferenceModifyIQ(request.getRequest()); IQ response = p.getFirst(); @@ -236,8 +243,13 @@ public Conference(Videobridge videobridge, request.getTotalDelayStats().addDelay(totalDelay); if (processingDelay > 100) { + String reqStr = XmlStringBuilderUtil.toStringOpt(request.getRequest()); + if (VideobridgeConfig.getRedactRemoteAddresses()) + { + reqStr = RedactColibriIp.Companion.redact(reqStr); + } logger.warn("Took " + processingDelay + " ms to process an IQ (total delay " - + totalDelay + " ms): " + XmlStringBuilderUtil.toStringOpt(request.getRequest())); + + totalDelay + " ms): " + reqStr); } logger.info("SENT colibri2 response: " + XmlStringBuilderUtil.toStringOpt(response)); request.getCallback().invoke(response); diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/VideobridgeConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/VideobridgeConfig.kt index 228c041744..212a6f170e 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/VideobridgeConfig.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/VideobridgeConfig.kt @@ -24,5 +24,10 @@ class VideobridgeConfig private constructor() { val initialDrainMode: Boolean by config { "videobridge.initial-drain-mode".from(JitsiConfig.newConfig) } + + @JvmStatic + val redactRemoteAddresses: Boolean by config { + "videobridge.redact-remote-addresses".from(JitsiConfig.newConfig) + } } } diff --git a/jvb/src/main/resources/application.conf b/jvb/src/main/resources/application.conf index 9afb3c9bf8..09d391d298 100644 --- a/jvb/src/main/resources/application.conf +++ b/jvb/src/main/resources/application.conf @@ -19,4 +19,5 @@ ice4j { use-dynamic-ports = false } } + redact-remote-addresses = ${videobridge.redact-remote-addresses} } diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index cfff64409c..9c0cc72fea 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -388,6 +388,9 @@ videobridge { # size for which the filter is enabled. stats-filter-threshold = 20 + # Whether to redact remote endpoint IP addresses from logs + redact-remote-addresses = true + ssrc-limit { # maximum number of SSRCs to send to an endpoint for video video = 50 diff --git a/pom.xml b/pom.xml index 855cd6c104..3a00efafcb 100644 --- a/pom.xml +++ b/pom.xml @@ -106,12 +106,12 @@ ${project.groupId} ice4j - 3.0-69-ga53b402 + 3.0-72-g824cd4b ${project.groupId} jitsi-xmpp-extensions - 1.0-80-g0ce9883 + 1.0-81-g3816e5a From e2b1c473e153e0a09a9d1ed2710b0771f8b201c3 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Thu, 18 Jul 2024 09:07:44 -0400 Subject: [PATCH 155/189] Fix: Catch exceptions processing packets. (#2190) Otherwise the IceTransport reader job exits. --- .../jitsi/videobridge/transport/ice/IceTransport.kt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt index 317c35b8aa..38f89d257b 100755 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt @@ -228,9 +228,13 @@ class IceTransport @JvmOverloads constructor( break } packetStats.numPacketsReceived.increment() - incomingDataHandler?.dataReceived(receiveBuf, packet.offset, packet.length, receivedTime) ?: run { - logger.cdebug { "Data handler is null, dropping data" } - packetStats.numIncomingPacketsDroppedNoHandler.increment() + try { + incomingDataHandler?.dataReceived(receiveBuf, packet.offset, packet.length, receivedTime) ?: run { + logger.cdebug { "Data handler is null, dropping data" } + packetStats.numIncomingPacketsDroppedNoHandler.increment() + } + } catch (e: Throwable) { + logger.error("Uncaught exception processing packet", e) } } logger.info("No longer running, stopped reading packets") From 97e03bb2eb74d853405a06226e2bf173d2f09fd0 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Thu, 18 Jul 2024 09:08:04 -0400 Subject: [PATCH 156/189] Fix(AV1): Prevent some exceptions in AV1 processing. (#2191) --- .../kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt | 2 +- .../Av1DependencyDescriptorHeaderExtension.kt | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt index dc83977718..0ecd90324a 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt @@ -107,7 +107,7 @@ internal class Av1DDQualityFilter( val accept = doAcceptFrame(frame, incomingEncoding, externalTargetIndex, receivedTime) val currentDt = getDtFromIndex(currentIndex) val mark = currentDt != SUSPENDED_DT && - (frame.frameInfo?.spatialId == frame.structure?.decodeTargetLayers?.get(currentDt)?.spatialId) + (frame.frameInfo?.spatialId == frame.structure?.decodeTargetLayers?.getOrNull(currentDt)?.spatialId) val isResumption = (prevIndex == SUSPENDED_INDEX && currentIndex != SUSPENDED_INDEX) if (isResumption) { check(accept) { diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtension.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtension.kt index c18061c8f7..58aa07729c 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtension.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtension.kt @@ -441,7 +441,8 @@ class Av1TemplateDependencyStructure( * Note this makes certain assumptions about the encoding structure. */ fun canSwitchWithoutKeyframe(fromDt: Int, toDt: Int): Boolean = templateInfo.any { - it.hasInterPictureDependency() && it.dti[fromDt] != DTI.NOT_PRESENT && it.dti[toDt] == DTI.SWITCH + it.hasInterPictureDependency() && it.dti.size > fromDt && it.dti.size > toDt && + it.dti[fromDt] != DTI.NOT_PRESENT && it.dti[toDt] == DTI.SWITCH } /** Given that we are sending packets for a given DT, return a decodeTargetBitmask corresponding to all DTs From 1eb3054e192471df327be3709f1edbc95b8accb2 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Thu, 18 Jul 2024 09:08:21 -0400 Subject: [PATCH 157/189] Fix(SSRC rewriting): Catch exceptions rewriting SSRCs in the SSRC cache. (#2192) --- .../kotlin/org/jitsi/videobridge/SsrcCache.kt | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt index fc6dd1aeff..ca178a5852 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt @@ -411,23 +411,28 @@ abstract class SsrcCache(val size: Int, val ep: SsrcRewriter, val parentLogger: val remappings = mutableListOf() var send = false - synchronized(sendSources) { - var rs = receivedSsrcs[packet.ssrc] - if (rs == null) { - val props = findSourceProps(packet.ssrc) ?: return false - rs = ReceiveSsrc(props) - receivedSsrcs[packet.ssrc] = rs - logger.debug { "Added receive SSRC: ${packet.ssrc}" } - } + try { + synchronized(sendSources) { + var rs = receivedSsrcs[packet.ssrc] + if (rs == null) { + val props = findSourceProps(packet.ssrc) ?: return false + rs = ReceiveSsrc(props) + receivedSsrcs[packet.ssrc] = rs + logger.debug { "Added receive SSRC: ${packet.ssrc}" } + } - val ss = getSendSource(rs.props.ssrc1, rs.props, allowCreateOnPacket, remappings) - if (ss != null) { - send = ss.rewriteRtp(packet, start, rs) + val ss = getSendSource(rs.props.ssrc1, rs.props, allowCreateOnPacket, remappings) + if (ss != null) { + send = ss.rewriteRtp(packet, start, rs) + } } - } - if (remappings.isNotEmpty()) { - notifyMappings(remappings) + if (remappings.isNotEmpty()) { + notifyMappings(remappings) + } + } catch (e: Exception) { + logger.error("Error rewriting SSRC", e) + send = false } return send From c6183464af571018348930e0e0b34dda198d553e Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 18 Jul 2024 17:19:05 +0300 Subject: [PATCH 158/189] Remove use-component-socket, cleanup (#2193) * ref: Move BufferPool setup to Main. * ref: Remove the use-component-socket option. The option was required, IceTransport unconditionally reads from the component socket. * log: Remove endpoint ID from IceMediaStream name. --- .../java/org/jitsi/videobridge/Videobridge.java | 16 ---------------- .../main/kotlin/org/jitsi/videobridge/Main.kt | 11 +++++++++++ .../org/jitsi/videobridge/ice/IceConfig.kt | 8 -------- .../videobridge/transport/ice/IceTransport.kt | 12 ++---------- jvb/src/main/resources/reference.conf | 3 --- 5 files changed, 13 insertions(+), 37 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java b/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java index 70476e6280..3f078369e1 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java +++ b/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java @@ -15,7 +15,6 @@ */ package org.jitsi.videobridge; -import kotlin.*; import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.*; import org.jitsi.health.Result; @@ -28,7 +27,6 @@ import org.jitsi.videobridge.metrics.*; import org.jitsi.videobridge.shutdown.*; import org.jitsi.videobridge.stats.*; -import org.jitsi.videobridge.util.*; import org.jitsi.videobridge.xmpp.*; import org.jitsi.xmpp.extensions.colibri2.*; import org.jitsi.xmpp.extensions.health.*; @@ -103,20 +101,6 @@ public class Videobridge @NotNull private final ShutdownManager shutdownManager; - static - { - org.jitsi.rtp.util.BufferPool.Companion.setGetArray(ByteBufferPool::getBuffer); - org.jitsi.rtp.util.BufferPool.Companion.setReturnArray(buffer -> { - ByteBufferPool.returnBuffer(buffer); - return Unit.INSTANCE; - }); - org.jitsi.nlj.util.BufferPool.Companion.setGetBuffer(ByteBufferPool::getBuffer); - org.jitsi.nlj.util.BufferPool.Companion.setReturnBuffer(buffer -> { - ByteBufferPool.returnBuffer(buffer); - return Unit.INSTANCE; - }); - } - /** * Initializes a new Videobridge instance. */ diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt index 920d87caeb..ec623884ae 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt @@ -37,6 +37,7 @@ import org.jitsi.videobridge.metrics.Metrics import org.jitsi.videobridge.metrics.VideobridgePeriodicMetrics import org.jitsi.videobridge.rest.root.Application import org.jitsi.videobridge.stats.MucPublisher +import org.jitsi.videobridge.util.ByteBufferPool import org.jitsi.videobridge.util.TaskPools import org.jitsi.videobridge.util.UlimitCheck import org.jitsi.videobridge.version.JvbVersionService @@ -78,6 +79,8 @@ fun main() { UlimitCheck.printUlimits() startIce4j() + setupBufferPools() + // Initialize, binding on the main ICE port. Harvesters.init() @@ -229,3 +232,11 @@ private fun stopIce4j() { // Shut down harvesters. Harvesters.close() } + +/** Configure our libraries to use JVB's global [ByteBufferPool] */ +private fun setupBufferPools() { + org.jitsi.rtp.util.BufferPool.getArray = { ByteBufferPool.getBuffer(it) } + org.jitsi.rtp.util.BufferPool.returnArray = { ByteBufferPool.returnBuffer(it) } + org.jitsi.nlj.util.BufferPool.getBuffer = { ByteBufferPool.getBuffer(it) } + org.jitsi.nlj.util.BufferPool.returnBuffer = { ByteBufferPool.returnBuffer(it) } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/ice/IceConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/ice/IceConfig.kt index 6090b444b1..8fe001081b 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/ice/IceConfig.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/ice/IceConfig.kt @@ -83,14 +83,6 @@ class IceConfig private constructor() { .convertFrom { KeepAliveStrategy.fromString(it) } } - /** - * Whether the ice4j "component socket" mode is used. - */ - val useComponentSocket: Boolean by config { - "org.jitsi.videobridge.USE_COMPONENT_SOCKET".from(JitsiConfig.legacyConfig) - "videobridge.ice.use-component-socket".from(JitsiConfig.newConfig) - } - val resolveRemoteCandidates: Boolean by config( "videobridge.ice.resolve-remote-candidates".from(JitsiConfig.newConfig) ) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt index 38f89d257b..2a4f0edbd2 100755 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt @@ -131,19 +131,12 @@ class IceTransport @JvmOverloads constructor( logger.addContext("local_ufrag", it.localUfrag) } - // TODO: Do we still need the id here now that we have logContext? - private val iceStream = iceAgent.createMediaStream("stream-$id").apply { + private val iceStream = iceAgent.createMediaStream("stream").apply { addPairChangeListener(iceStreamPairChangedListener) } - private val iceComponent = iceAgent.createComponent( - iceStream, - IceConfig.config.keepAliveStrategy, - IceConfig.config.useComponentSocket - ) - + private val iceComponent = iceAgent.createComponent(iceStream, IceConfig.config.keepAliveStrategy, true) private val packetStats = PacketStats() - val icePassword: String get() = iceAgent.localPassword @@ -267,7 +260,6 @@ class IceTransport @JvmOverloads constructor( } fun getDebugState(): OrderedJsonObject = OrderedJsonObject().apply { - put("useComponentSocket", IceConfig.config.useComponentSocket) put("keepAliveStrategy", IceConfig.config.keepAliveStrategy.toString()) put("nominationStrategy", IceConfig.config.nominationStrategy.toString()) put("advertisePrivateCandidates", IceConfig.config.advertisePrivateCandidates) diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index 9c0cc72fea..fcee8c4394 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -289,9 +289,6 @@ videobridge { # "selected_and_tcp", "selected_only", or "all_succeeded". keep-alive-strategy = "selected_and_tcp" - # Whether to use the "component socket" feature of ice4j. - use-component-socket = true - # Whether to attempt DNS resolution for remote candidates that contain a non-literal address. When set to 'false' # such candidates will be ignored. resolve-remote-candidates = false From 89dde6ea7114d792e1931545eb7028db24dd82f0 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 18 Jul 2024 17:19:26 +0300 Subject: [PATCH 159/189] feat: Remove support for ICE/TCP. (#2194) * feat: Remove support for ICE/TCP. * TURN/TLS provides a better solution for the same problem. * The implementation has been known to have issues with threads getting blocked. * ICE/TCP gets in the way of improvements we're currently working on. * We (jitsi team) haven't used it in any of our deployments since at least 2018. * See also: https://github.com/jitsi/docker-jitsi-meet/commit/7a939785a7f018601afe2d49d8f66078812a3dd0 --- CONFIG.md | 4 - doc/statistics.md | 1 - doc/tcp.md | 75 ------------------- .../org/jitsi/videobridge/ice/Harvesters.kt | 27 +------ .../org/jitsi/videobridge/ice/IceConfig.kt | 34 --------- .../stats/VideobridgeStatisticsShim.kt | 2 - .../videobridge/transport/ice/IceTransport.kt | 28 +------ jvb/src/main/resources/reference.conf | 17 +---- 8 files changed, 5 insertions(+), 183 deletions(-) delete mode 100644 doc/tcp.md diff --git a/CONFIG.md b/CONFIG.md index 2ebe205976..88c6ce8e52 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -57,10 +57,6 @@ This list will be updated as properties are migrated: | org.jitsi.videobridge.rest.COLIBRI_WS_DOMAIN | videobridge.websockets.domain | | | org.jitsi.videobridge.rest.COLIBRI_WS_TLS | videobridge.websockets.tls | | | org.jitsi.videobridge.rest.COLIBRI_WS_SERVER_ID | videobridge.websockets.server-id | | -| org.jitsi.videobridge.DISABLE_TCP_HARVESTER | videobridge.ice.tcp.enabled | The semantics of this property have been inverted (disable -> enable) | -| org.jitsi.videobridge.TCP_HARVESTER_SSLTCP | videobridge.ice.tcp.ssltcp | | -| org.jitsi.videobridge.TCP_HARVESTER_PORT | videobridge.ice.tcp.port | | -| org.jitsi.videobridge.TCP_HARVESTER_MAPPED_PORT | videobridge.ice.tcp.mapped-port | | | org.jitsi.videobridge.SINGLE_PORT_HARVESTER_PORT | videobridge.ice.udp.port | | | org.jitsi.videobridge.ICE_UFRAG_PREFIX | videobridge.ice.ufrag-prefix | | | org.jitsi.videobridge.KEEP_ALIVE_STRATEGY | videobridge.ice.keep-alive-strategy | | diff --git a/doc/statistics.md b/doc/statistics.md index e095d19ed8..b83eedac2a 100644 --- a/doc/statistics.md +++ b/doc/statistics.md @@ -121,7 +121,6 @@ connection. * `total_ice_failed` - total number of endpoints which failed to establish an ICE connection. * `total_ice_succeeded` - total number of endpoints which successfully established an ICE connection. * `total_ice_succeeded_relayed` - total number of endpoints which connected through a TURN relay (currently broken). -* `total_ice_succeeded_tcp` - total number of endpoints which connected through via ICE/TCP (currently broken). * `total_packets_dropped_octo` - total number of packets dropped on the `octo` channel. * `total_packets_received` - total number of RTP packets received. * `total_packets_received_octo` - total number packets received on the `octo` channel. diff --git a/doc/tcp.md b/doc/tcp.md deleted file mode 100644 index 0de6b1f093..0000000000 --- a/doc/tcp.md +++ /dev/null @@ -1,75 +0,0 @@ -# General -Jitsi Videobridge can accept and route RTP traffic over ICE/TCP. -The feature is off by default. When turned on, the bridge will listen -on a TCP port and advertise ICE candidates of type TCP via COLIBRI. - -# Warning -ICE/TCP is not the recommended way to deal with clients connecting -from networks where UDP traffic is restricted. The recommended way -is to use jitsi-videobridge in conjunction with a TURN server. The -main reason is that using TURN/TLS uses a real TLS handshake, while -ICE/TCP uses a hard-coded handshake which is known to be recognized -by some firewalls. - -# Configuration -ICE/TCP is configured in the `videobridge.ice.tcp` section in `jvb.conf`. - -By default TCP support is disabled. When enabled, the default is to -use port 443 with fallback to port 4443. A fallback would occur in -case something else, like a web server, is already listening on -port 443. Note, however, that the very point of using TCP is to -simulate HTTP traffic in a number of environments where it is the -only allowed form of communication, so you may want to make sure that -port 443 will be free on the machine where you are running the -bridge. - -In order to avoid binding to port 443 directly: -* Redirect 443 to 4443 by external means -* Use `port=4443` -* Use `mapped-port=443` -See below for details. - -```hocon -videobridge { - ice { - tcp { - enabled = false - - // Configures the port number to be used by the TCP harvester. If this property is unset (and the TCP harvester - // is enabled), jitsi-videobridge will first try to bind on port 443, and if this fails, it will try port 4443 - // instead. If the property is set, it will only try to bind to the specified port, with no fallback. - port = 443 - - // If this property is set, Jitsi Videobridge will use the given port in the candidates that it advertises, but - // the actual port it listens on will not change. - #mapped-port = 8443 // Defaults unset - - // Configures the use of "ssltcp" candidates. - // - // When enabled, Jitsi Videobridge will generate candidates with protocol "ssltcp", and the TCP harvester will - // expect connecting clients to send a special pseudo-SSL ClientHello message right after they connect, before any - // STUN messages. Chrome sends this message if a candidate in its SDP offer has the "ssltcp" protocol. - // - // When disabled, Jitsi Videobridge will generate candidates with protocol "tcp" candidates and will expect to - // receive STUN messages right away. - ssltcp = true - } - } -} -``` - -## Configuration of ice4j - -Some of the networking-related behavior of jitsi-videobridge can be configured -through properties for the ICE library -- [ice4j](https://github.com/jitsi/ice4j). -These properties can also be set in the jitsi-videobridge properties file. See -[documentation](https://github.com/jitsi/ice4j/blob/master/doc/configuration.md) and -[reference.conf](https://github.com/jitsi/ice4j/blob/master/src/main/resources/reference.conf#L37) in ice4j for -details. - -# Examples -## None of the TCP specific properties set, successful bind on port 443 -Jitsi Videobridge will bind to port 443 and announce port 443. - -## None of the TCP specific properties set, failure to bind on port 443 (lack of privileges, or web-server already bound on 443) -Jitsi Videobridge will bind to port 4443 and announce port 4443. diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/ice/Harvesters.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/ice/Harvesters.kt index 7ce6de52c0..eb9253bddb 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/ice/Harvesters.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/ice/Harvesters.kt @@ -16,21 +16,15 @@ package org.jitsi.videobridge.ice import org.ice4j.ice.harvest.SinglePortUdpHarvester -import org.ice4j.ice.harvest.TcpHarvester import org.jitsi.utils.logging2.createLogger -import java.io.IOException -class Harvesters private constructor( - val tcpHarvester: TcpHarvester?, - val singlePortHarvesters: List -) { +class Harvesters private constructor(val singlePortHarvesters: List) { /* We're unhealthy if there are no single port harvesters. */ val healthy: Boolean get() = singlePortHarvesters.isNotEmpty() private fun close() { singlePortHarvesters.forEach { it.close() } - tcpHarvester?.close() } companion object { @@ -48,25 +42,8 @@ class Harvesters private constructor( if (singlePortHarvesters.isEmpty()) { logger.warn("No single-port harvesters created.") } - val tcpHarvester: TcpHarvester? = if (IceConfig.config.tcpEnabled) { - val port = IceConfig.config.tcpPort - try { - TcpHarvester(port, IceConfig.config.iceSslTcp).apply { - logger.info("Initialized TCP harvester on port $port, ssltcp=${IceConfig.config.iceSslTcp}") - IceConfig.config.tcpMappedPort?.let { mappedPort -> - logger.info("Adding mapped port $mappedPort") - addMappedPort(mappedPort) - } - } - } catch (ioe: IOException) { - logger.warn("Failed to initialize TCP harvester on port $port") - null - } - } else { - null - } - Harvesters(tcpHarvester, singlePortHarvesters) + Harvesters(singlePortHarvesters) } } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/ice/IceConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/ice/IceConfig.kt index 8fe001081b..fc68992ff5 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/ice/IceConfig.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/ice/IceConfig.kt @@ -24,40 +24,6 @@ import org.jitsi.metaconfig.from import org.jitsi.metaconfig.optionalconfig class IceConfig private constructor() { - /** - * Is ICE/TCP enabled. - */ - val tcpEnabled: Boolean by config { - // The old property is named 'disable', while the new one - // is 'enable', so invert the old value - "org.jitsi.videobridge.DISABLE_TCP_HARVESTER".from(JitsiConfig.legacyConfig).transformedBy { !it } - "videobridge.ice.tcp.enabled".from(JitsiConfig.newConfig) - } - - /** - * The ICE/TCP port. - */ - val tcpPort: Int by config { - "org.jitsi.videobridge.TCP_HARVESTER_PORT".from(JitsiConfig.legacyConfig) - "videobridge.ice.tcp.port".from(JitsiConfig.newConfig) - } - - /** - * The additional port to advertise, or null if none is configured. - */ - val tcpMappedPort: Int? by optionalconfig { - "org.jitsi.videobridge.TCP_HARVESTER_MAPPED_PORT".from(JitsiConfig.legacyConfig) - "videobridge.ice.tcp.mapped-port".from(JitsiConfig.newConfig) - } - - /** - * Whether ICE/TCP should use "ssltcp" or not. - */ - val iceSslTcp: Boolean by config { - "org.jitsi.videobridge.TCP_HARVESTER_SSLTCP".from(JitsiConfig.legacyConfig) - "videobridge.ice.tcp.ssltcp".from(JitsiConfig.newConfig) - } - /** * The ICE UDP port. */ diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/VideobridgeStatisticsShim.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/VideobridgeStatisticsShim.kt index 7007fd7c3b..8957b725a1 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/VideobridgeStatisticsShim.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/VideobridgeStatisticsShim.kt @@ -71,7 +71,6 @@ import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_DATA_CHANNE import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_DOMINANT_SPEAKER_CHANGES import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_ICE_FAILED import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_ICE_SUCCEEDED -import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_ICE_SUCCEEDED_TCP import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_PACKETS_RECEIVED import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_PACKETS_RECEIVED_OCTO import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_PACKETS_SENT @@ -199,7 +198,6 @@ object VideobridgeStatisticsShim { put(TOTAL_ICE_FAILED, IceTransport.iceFailed.get()) put(TOTAL_ICE_SUCCEEDED, IceTransport.iceSucceeded.get()) - put(TOTAL_ICE_SUCCEEDED_TCP, IceTransport.iceSucceededTcp.get()) put("total_ice_succeeded_relayed", IceTransport.iceSucceededRelayed.get()) put("average_participant_stress", JvbLoadManager.averageParticipantStress) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt index 2a4f0edbd2..9f7ae36245 100755 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt @@ -344,9 +344,6 @@ class IceTransport @JvmOverloads constructor( transition.completed() -> { if (iceConnected.compareAndSet(false, true)) { eventHandler?.connected() - if (iceComponent.selectedPair.remoteCandidate.transport.isTcpType()) { - iceSucceededTcp.inc() - } if (iceComponent.selectedPair.remoteCandidate.type == CandidateType.RELAYED_CANDIDATE || iceComponent.selectedPair.localCandidate.type == CandidateType.RELAYED_CANDIDATE ) { @@ -389,9 +386,6 @@ class IceTransport @JvmOverloads constructor( companion object { fun appendHarvesters(iceAgent: Agent) { - Harvesters.INSTANCE.tcpHarvester?.let { - iceAgent.addCandidateHarvester(it) - } Harvesters.INSTANCE.singlePortHarvesters.forEach(iceAgent::addCandidateHarvester) } @@ -412,15 +406,6 @@ class IceTransport @JvmOverloads constructor( "Number of times an ICE Agent succeeded." ) - /** - * The total number of times an ICE Agent succeeded and the selected - * candidate was a TCP candidate. - */ - val iceSucceededTcp = VideobridgeMetricsContainer.instance.registerCounter( - "ice_succeeded_tcp", - "Number of times an ICE Agent succeeded and the selected candidate was a TCP candidate." - ) - /** * The total number of times an ICE Agent succeeded and the selected * candidate pair included a relayed candidate. @@ -498,8 +483,6 @@ private fun TransportAddress.isPrivateAddress(): Boolean = address.isSiteLocalAd /* 0xfc00::/7 */ ((address is Inet6Address) && ((addressBytes[0].toInt() and 0xfe) == 0xfc)) -private fun Transport.isTcpType(): Boolean = this == Transport.TCP || this == Transport.SSLTCP - private fun generateCandidateId(candidate: LocalCandidate): String = buildString { append(java.lang.Long.toHexString(hashCode().toLong())) append(java.lang.Long.toHexString(candidate.parentComponent.parentStream.parentAgent.hashCode().toLong())) @@ -522,16 +505,7 @@ private fun LocalCandidate.toCandidatePacketExtension(advertisePrivateAddresses: cpe.network = 0 cpe.setPriority(priority) - // Advertise 'tcp' candidates for which SSL is enabled as 'ssltcp' - // (although internally their transport protocol remains "tcp") - cpe.protocol = if (transport == Transport.TCP && isSSL) { - Transport.SSLTCP.toString() - } else { - transport.toString() - } - if (transport.isTcpType()) { - cpe.tcpType = tcpType.toString() - } + cpe.protocol = transport.toString() cpe.type = org.jitsi.xmpp.extensions.jingle.CandidateType.valueOf(type.toString()) cpe.ip = transportAddress.hostAddress cpe.port = transportAddress.port diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index fcee8c4394..60974d8e68 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -264,19 +264,6 @@ videobridge { relay-domains = [] } ice { - tcp { - # Whether ICE/TCP is enabled. - enabled = false - - # The port to bind to for ICE/TCP. - port = 443 - - # An optional additional port to advertise. - # mapped-port = 8443 - # Whether to use "ssltcp" or plain "tcp". - ssltcp = true - } - udp { # The port for ICE/UDP. port = 10000 @@ -286,8 +273,8 @@ videobridge { #ufrag-prefix = "jvb-123:" # Which candidate pairs to keep alive. The accepted values are defined in ice4j's KeepAliveStrategy: - # "selected_and_tcp", "selected_only", or "all_succeeded". - keep-alive-strategy = "selected_and_tcp" + # "selected_only", or "all_succeeded". + keep-alive-strategy = "selected_only" # Whether to attempt DNS resolution for remote candidates that contain a non-literal address. When set to 'false' # such candidates will be ignored. From 6f3f568471fd55701a8aa3a235eac57dda226e41 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Thu, 18 Jul 2024 14:23:40 -0400 Subject: [PATCH 160/189] Fix: Push packets received from ICE onto a queue. (#2196) Rather than processing them synchronously in the I/O thread. --- .../main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt index 37beba0136..e2734be1ce 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt @@ -188,7 +188,7 @@ class RelayedEndpoint( rtpReceiver.setSrtpTransformers(srtpTransformers) } - override fun handleIncomingPacket(packetInfo: RelayedPacketInfo) = rtpReceiver.processPacket(packetInfo) + override fun handleIncomingPacket(packetInfo: RelayedPacketInfo) = rtpReceiver.enqueuePacket(packetInfo) fun setFeature(feature: Features, enabled: Boolean) { rtpReceiver.setFeature(feature, enabled) From 97a1f15bc5f396c417b3550ced3cbedf54bf4c3b Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Mon, 29 Jul 2024 13:34:59 -0700 Subject: [PATCH 161/189] Fix: Support VP9 flexible mode. (#2199) VP9 flexible mode doesn't announce temporal layers in advance, so add them to the encoding desc as they are encountered. --- .../kotlin/org/jitsi/nlj/MediaSourceDesc.kt | 8 ++ .../main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt | 2 +- .../nlj/rtp/codec/av1/Av1DDRtpLayerDesc.kt | 7 +- .../org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt | 3 + .../org/jitsi/nlj/rtp/codec/vp9/Vp9Parser.kt | 80 +++++++++++++++++-- .../nlj/rtp/codec/vpx/VpxRtpLayerDesc.kt | 10 ++- .../node/incoming/BitrateCalculator.kt | 8 +- .../cc/allocation/BitrateControllerTest.kt | 2 +- 8 files changed, 104 insertions(+), 16 deletions(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt index 2a6fb30188..b529791bcc 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt @@ -148,6 +148,14 @@ class MediaSourceDesc @Synchronized fun findRtpEncodingDesc(ssrc: Long): RtpEncodingDesc? = rtpEncodings.find { it.matches(ssrc) } + @Synchronized + fun getEncodingLayers(ssrc: Long): Array { + val enc = findRtpEncodingDesc(ssrc) ?: return emptyArray() + return Array(enc.layers.size) { i -> + enc.layers[i].copy() + } + } + @Synchronized fun setEncodingLayers(layers: Array, ssrc: Long) { val enc = findRtpEncodingDesc(ssrc) ?: return diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt index 9cd30d9383..49f0474c45 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt @@ -56,7 +56,7 @@ constructor( */ val frameRate: Double, ) { - abstract fun copy(height: Int = this.height): RtpLayerDesc + abstract fun copy(height: Int = this.height, tid: Int = this.tid, inherit: Boolean = true): RtpLayerDesc /** * The [BitrateTracker] instance used to calculate the receiving bitrate of this RTP layer. diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDRtpLayerDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDRtpLayerDesc.kt index 38952a2415..2d1eccbf0b 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDRtpLayerDesc.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDRtpLayerDesc.kt @@ -52,7 +52,12 @@ class Av1DDRtpLayerDesc( */ frameRate: Double, ) : RtpLayerDesc(eid, tid, sid, height, frameRate) { - override fun copy(height: Int): RtpLayerDesc = Av1DDRtpLayerDesc(eid, dt, tid, sid, height, frameRate) + override fun copy(height: Int, tid: Int, inherit: Boolean): RtpLayerDesc = + Av1DDRtpLayerDesc(eid, dt, tid, sid, height, frameRate).also { + if (inherit) { + it.inheritFrom(this) + } + } override val layerId = dt override val index = getIndex(eid, dt) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt index ab05bf5d82..3b1a31e99b 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt @@ -96,6 +96,9 @@ class Vp9Packet private constructor( val hasExtendedPictureId = DePacketizer.VP9PayloadDescriptor.hasExtendedPictureId(buffer, payloadOffset, payloadLength) + val isFlexibleMode = + DePacketizer.VP9PayloadDescriptor.isFlexibleMode(buffer, payloadOffset, payloadLength) + val hasScalabilityStructure = DePacketizer.VP9PayloadDescriptor.hasScalabilityStructure(buffer, payloadOffset, payloadLength) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Parser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Parser.kt index d1fa5a9493..05300fff8b 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Parser.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Parser.kt @@ -18,12 +18,14 @@ package org.jitsi.nlj.rtp.codec.vp9 import org.jitsi.nlj.MediaSourceDesc import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.RtpLayerDesc import org.jitsi.nlj.rtp.codec.VideoCodecParser import org.jitsi.nlj.rtp.codec.vpx.VpxRtpLayerDesc import org.jitsi.nlj.util.StateChangeLogger import org.jitsi.rtp.extensions.toHex import org.jitsi.utils.logging2.Logger import org.jitsi.utils.logging2.createChildLogger +import kotlin.math.max /** * Some [Vp9Packet] fields are not able to be determined by looking at a single VP9 packet (for example the scalability @@ -40,13 +42,23 @@ class Vp9Parser( private val extendedPictureIdState = StateChangeLogger("missing extended picture ID", logger) private var numSpatialLayers = -1 - /** Encodings we've actually seen. Used to clear out inferred-from-signaling encoding information. */ - private val ssrcsSeen = HashSet() + /** Encodings we've actually seen, and the layers seen for each one. + * Used to clear out inferred-from-signaling encoding information, and to synthesize temporal layers + * for flexible-mode encodings. */ + private val ssrcsInfo = HashMap>() override fun parse(packetInfo: PacketInfo) { val vp9Packet = packetInfo.packetAs() - ssrcsSeen.add(vp9Packet.ssrc) + val layerMap = ssrcsInfo.getOrPut(vp9Packet.ssrc) { + HashMap() + } + + layerMap[vp9Packet.spatialLayerIndex]?.let { + layerMap[vp9Packet.spatialLayerIndex] = max(it, vp9Packet.temporalLayerIndex) + } ?: run { + layerMap[vp9Packet.spatialLayerIndex] = vp9Packet.temporalLayerIndex + } if (vp9Packet.hasScalabilityStructure) { // TODO: handle case where new SS is from a packet older than the @@ -58,12 +70,31 @@ class Vp9Parser( } numSpatialLayers = packetSpatialLayers } - findRtpEncodingDesc(vp9Packet)?.let { enc -> - vp9Packet.getScalabilityStructure(eid = enc.eid)?.let { - source.setEncodingLayers(it.layers, vp9Packet.ssrc) - } + val ss = findRtpEncodingDesc(vp9Packet)?.let { enc -> + vp9Packet.getScalabilityStructure(eid = enc.eid) + } + + if (ss != null) { + val layers = + if (vp9Packet.isFlexibleMode) { + /* In flexible mode, the number of temporal layers isn't announced in the keyframe. + * Thus, add temporal layer information to the source's encoding layers based on the temporal + * layers we've seen previously. + */ + val layersList = ss.layers.toMutableList() + + for ((sid, maxTid) in layerMap) { + addTemporalLayers(layersList, sid, maxTid) + } + layersList.toTypedArray() + } else { + ss.layers + } + + source.setEncodingLayers(layers, vp9Packet.ssrc) + for (otherEnc in source.rtpEncodings) { - if (!ssrcsSeen.contains(otherEnc.primarySSRC)) { + if (!ssrcsInfo.contains(otherEnc.primarySSRC)) { source.setEncodingLayers(emptyArray(), otherEnc.primarySSRC) } } @@ -82,6 +113,19 @@ class Vp9Parser( } } + if (vp9Packet.isFlexibleMode && findRtpLayerDescs(vp9Packet).isEmpty()) { + val layers = source.getEncodingLayers(vp9Packet.ssrc).toMutableList() + /* In flexible mode, the number of temporal layers isn't announced in the keyframe. + * Thus, add temporal layer information to the source's encoding layers as we see packets with + * temporal layers. + */ + val changed = addTemporalLayers(layers, vp9Packet.spatialLayerIndex, vp9Packet.temporalLayerIndex) + if (changed) { + source.setEncodingLayers(layers.toTypedArray(), vp9Packet.ssrc) + packetInfo.layeringChanged = true + } + } + pictureIdState.setState(vp9Packet.hasPictureId, vp9Packet) { "Packet Data: ${vp9Packet.toHex(80)}" } @@ -89,4 +133,24 @@ class Vp9Parser( "Packet Data: ${vp9Packet.toHex(80)}" } } + + /** Add temporal layers to the list of layers. Needed if VP9 is encoded in flexible mode, because + * in flexible mode the scalability structure doesn't describe the temporal layers. + */ + private fun addTemporalLayers(layers: MutableList, sid: Int, maxTid: Int): Boolean { + var changed = false + + for (tid in 1..maxTid) { + val layer = layers.find { it.sid == sid && it.tid == tid } + if (layer == null) { + val prevLayer = layers.find { it.sid == sid && it.tid == tid - 1 } + if (prevLayer != null) { + val newLayer = prevLayer.copy(tid = tid, inherit = false) + layers.add(newLayer) + changed = true + } + } + } + return changed + } } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vpx/VpxRtpLayerDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vpx/VpxRtpLayerDesc.kt index 9c42b14e04..0244e0bd1a 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vpx/VpxRtpLayerDesc.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vpx/VpxRtpLayerDesc.kt @@ -70,19 +70,21 @@ constructor( } /** - * Clone an existing layer desc, inheriting its statistics, + * Clone an existing layer desc, inheriting its statistics if [inherit], * modifying only specific values. */ - override fun copy(height: Int) = VpxRtpLayerDesc( + override fun copy(height: Int, tid: Int, inherit: Boolean) = VpxRtpLayerDesc( eid = this.eid, - tid = this.tid, + tid = tid, sid = this.sid, height = height, frameRate = this.frameRate, dependencyLayers = this.dependencyLayers, softDependencyLayers = this.softDependencyLayers ).also { - it.inheritFrom(this) + if (inherit) { + it.inheritFrom(this) + } } /** diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/BitrateCalculator.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/BitrateCalculator.kt index 2daf4cc1da..e20eb85f7e 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/BitrateCalculator.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/BitrateCalculator.kt @@ -56,7 +56,13 @@ class VideoBitrateCalculator( val videoRtpPacket: VideoRtpPacket = packetInfo.packet as VideoRtpPacket val now = clock.millis() - mediaSourceDescs.findRtpLayerDescs(videoRtpPacket).forEach { + val layerDescs = mediaSourceDescs.findRtpLayerDescs(videoRtpPacket) + + if (layerDescs.isEmpty()) { + logger.warn("No layer found for packet $videoRtpPacket") + } + + layerDescs.forEach { if (it.updateBitrate(videoRtpPacket.length.bytes, now)) { /* When a layer is started when it was previously inactive, * we want to recalculate bandwidth allocation. diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt index 4649e80586..a3f0f13156 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt @@ -1528,7 +1528,7 @@ class MockRtpLayerDesc( var bitrate: Bandwidth, sid: Int = -1 ) : RtpLayerDesc(eid, tid, sid, height, frameRate) { - override fun copy(height: Int): RtpLayerDesc { + override fun copy(height: Int, tid: Int, inherit: Boolean): RtpLayerDesc { TODO("Not yet implemented") } From 6e441e62bd4a47bce8a75ded2dc8d886459eaaf9 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Mon, 26 Aug 2024 15:20:22 -0400 Subject: [PATCH 162/189] Modify Debian postinst to use usrsctp on ppc64el. (#2214) --- debian/postinst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/postinst b/debian/postinst index 67aeac999c..3543109f01 100644 --- a/debian/postinst +++ b/debian/postinst @@ -124,6 +124,12 @@ case "$1" in sed -i 's/.*--apis.*//' $CONFIG fi + # dcsctp4j isn't built for ppc64el, so use usrsctp there if no explicit option has been set. + DEBARCH=$(dpkg --print-architecture) + if [ "$DEBARCH" = "ppc64el" ] && ! hocon -f $HOCON_CONFIG get "videobridge.sctp.use-usrsctp" > /dev/null 2>&1 ;then + hocon -f $HOCON_CONFIG set "videobridge.sctp.use-usrsctp" "true" + fi + # we don't want to start the daemon as root if ! getent group jitsi > /dev/null ; then groupadd jitsi From f625842088b518a0be419f72a7f928970d817b02 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 5 Sep 2024 15:09:36 -0500 Subject: [PATCH 163/189] Add hackerone link to SECURITY.md. (#2218) --- SECURITY.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index d8441922de..cf2773d177 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,9 +1,9 @@ # Security -## Reporting security issuess +## Reporting security issues We take security very seriously and develop all Jitsi projects to be secure and safe. -If you find (or simply suspect) a security issue in any of the Jitsi projects, please send us an email to security@jitsi.org. +If you find (or simply suspect) a security issue in any of the Jitsi projects, please report it to us via [HackerOne](https://hackerone.com/8x8-bounty) or send us an email to security@jitsi.org. **We encourage responsible disclosure for the sake of our users, so please reach out before posting in a public space.** From d3f5db4c886ed613ec56e7b42750aa70f0dbc729 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 17 Sep 2024 13:30:56 -0400 Subject: [PATCH 164/189] Convert TCC packets to use Instant and Duration (#2219) Rather than integer microseconds. --- .../org/jitsi/nlj/rtp/TransportCcEngine.kt | 3 +- .../node/incoming/TccGeneratorNode.kt | 10 ++- .../kotlin/org/jitsi/nlj/util/ClockUtils.kt | 41 ------------ .../jitsi/nlj/rtp/TransportCcEngineTest.kt | 27 ++++---- pom.xml | 2 +- .../transport_layer_fb/tcc/RtcpFbTccPacket.kt | 64 +++++++++++-------- .../tcc/RtcpFbTccPacketTest.kt | 14 ++-- 7 files changed, 65 insertions(+), 96 deletions(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/TransportCcEngine.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/TransportCcEngine.kt index 8e43a057e7..8853ae0fda 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/TransportCcEngine.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/TransportCcEngine.kt @@ -22,7 +22,6 @@ import org.jitsi.nlj.util.DataSize import org.jitsi.nlj.util.NEVER import org.jitsi.nlj.util.Rfc3711IndexTracker import org.jitsi.nlj.util.formatMilli -import org.jitsi.nlj.util.instantOfEpochMicro import org.jitsi.rtp.rtcp.RtcpPacket import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.PacketReport import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.ReceivedPacketReport @@ -126,7 +125,7 @@ class TransportCcEngine( private fun tccReceived(tccPacket: RtcpFbTccPacket) { val now = clock.instant() - var currArrivalTimestamp = instantOfEpochMicro(tccPacket.GetBaseTimeUs()) + var currArrivalTimestamp = tccPacket.BaseTime() if (remoteReferenceTime == NEVER) { remoteReferenceTime = currArrivalTimestamp localReferenceTime = now diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNode.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNode.kt index e22fb088da..c12996ce82 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNode.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNode.kt @@ -25,7 +25,6 @@ import org.jitsi.nlj.util.NEVER import org.jitsi.nlj.util.ReadOnlyStreamInformationStore import org.jitsi.nlj.util.Rfc3711IndexTracker import org.jitsi.nlj.util.bytes -import org.jitsi.nlj.util.toEpochMicro import org.jitsi.rtp.rtcp.RtcpPacket import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacketBuilder @@ -182,20 +181,19 @@ class TccGeneratorNode( mediaSourceSsrc = mediaSsrc, feedbackPacketSeqNum = currTccSeqNum++ ) - currentTccPacket.SetBase(windowStartSeq, firstEntry.value.toEpochMicro()) + currentTccPacket.SetBase(windowStartSeq, firstEntry.value) var nextSequenceNumber = windowStartSeq val feedbackBlockPackets = packetArrivalTimes.tailMap(windowStartSeq) feedbackBlockPackets.forEach { (seq, timestamp) -> - val timestampUs = timestamp.toEpochMicro() - if (!currentTccPacket.AddReceivedPacket(seq, timestampUs)) { + if (!currentTccPacket.AddReceivedPacket(seq, timestamp)) { tccPackets.add(currentTccPacket.build()) currentTccPacket = RtcpFbTccPacketBuilder( mediaSourceSsrc = mediaSsrc, feedbackPacketSeqNum = currTccSeqNum++ ).apply { - SetBase(seq, timestampUs) - AddReceivedPacket(seq, timestampUs) + SetBase(seq, timestamp) + AddReceivedPacket(seq, timestamp) } } nextSequenceNumber = seq + 1 diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ClockUtils.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ClockUtils.kt index d5f7a2573d..823ef31e90 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ClockUtils.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ClockUtils.kt @@ -29,47 +29,6 @@ fun Instant.formatMilli(): String = TimeUtils.formatTimeAsFullMillis(this.epochS fun Duration.formatMilli(): String = TimeUtils.formatTimeAsFullMillis(this.seconds, this.nano) -/** - * Converts this instant to the number of microseconds from the epoch - * of 1970-01-01T00:00:00Z. - * - * If this instant represents a point on the time-line too far in the future - * or past to fit in a [Long] microseconds, then an exception is thrown. - * - * If this instant has greater than microsecond precision, then the conversion - * will drop any excess precision information as though the amount in nanoseconds - * was subject to integer division by one thousand. - * - * @return the number of microseconds since the epoch of 1970-01-01T00:00:00Z - * @throws ArithmeticException if numeric overflow occurs - */ -fun Instant.toEpochMicro(): Long { - return if (this.epochSecond < 0 && this.nano > 0) { - val micros = Math.multiplyExact(this.epochSecond + 1, 1000_000L) - val adjustment: Long = (this.nano / 1000 - 1000_000).toLong() - Math.addExact(micros, adjustment) - } else { - val micros = Math.multiplyExact(this.epochSecond, 1000_000L) - Math.addExact(micros, (this.nano / 1000).toLong()) - } -} - -/** - * Obtains an instance of [Instant] using microseconds from the - * epoch of 1970-01-01T00:00:00Z. - *

- * The seconds and nanoseconds are extracted from the specified milliseconds. - * - * @param epochMicro the number of microseconds from 1970-01-01T00:00:00Z - * @return an instant, not null - * @throws DateTimeException if the instant exceeds the maximum or minimum instant - */ -fun instantOfEpochMicro(epochMicro: Long): Instant { - val secs = Math.floorDiv(epochMicro, 1000_000L) - val micros = Math.floorMod(epochMicro, 1000_000L) - return Instant.ofEpochSecond(secs, micros * 1000L) -} - fun Iterable.sumOf(selector: (T) -> Duration): Duration { var sum: Duration = Duration.ZERO for (element in this) { diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/TransportCcEngineTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/TransportCcEngineTest.kt index 2bde7dd289..b0b480f01b 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/TransportCcEngineTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/TransportCcEngineTest.kt @@ -23,6 +23,7 @@ import org.jitsi.nlj.resources.logging.StdoutLogger import org.jitsi.nlj.rtp.bandwidthestimation.BandwidthEstimator import org.jitsi.nlj.util.bytes import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacketBuilder +import org.jitsi.utils.instantOfEpochMicro import org.jitsi.utils.time.FakeClock import java.util.logging.Level @@ -58,11 +59,11 @@ class TransportCcEngineTest : FunSpec() { transportCcEngine.mediaPacketSent(4, 1300.bytes) val tccPacket = with(RtcpFbTccPacketBuilder(mediaSourceSsrc = 123, feedbackPacketSeqNum = 0)) { - SetBase(1, 100) - AddReceivedPacket(1, 100) - AddReceivedPacket(2, 110) - AddReceivedPacket(3, 120) - AddReceivedPacket(4, 130) + SetBase(1, instantOfEpochMicro(100)) + AddReceivedPacket(1, instantOfEpochMicro(100)) + AddReceivedPacket(2, instantOfEpochMicro(110)) + AddReceivedPacket(3, instantOfEpochMicro(120)) + AddReceivedPacket(4, instantOfEpochMicro(130)) build() } @@ -81,15 +82,15 @@ class TransportCcEngineTest : FunSpec() { transportCcEngine.mediaPacketSent(4, 1300.bytes) val tccPacket = with(RtcpFbTccPacketBuilder(mediaSourceSsrc = 123, feedbackPacketSeqNum = 1)) { - SetBase(4, 130) - AddReceivedPacket(4, 130) + SetBase(4, instantOfEpochMicro(130)) + AddReceivedPacket(4, instantOfEpochMicro(130)) build() } transportCcEngine.rtcpPacketReceived(tccPacket, clock.instant()) val tccPacket2 = with(RtcpFbTccPacketBuilder(mediaSourceSsrc = 123, feedbackPacketSeqNum = 2)) { - SetBase(4, 130) - AddReceivedPacket(4, 130) + SetBase(4, instantOfEpochMicro(130)) + AddReceivedPacket(4, instantOfEpochMicro(130)) build() } transportCcEngine.rtcpPacketReceived(tccPacket2, clock.instant()) @@ -108,8 +109,8 @@ class TransportCcEngineTest : FunSpec() { transportCcEngine.mediaPacketSent(5, 1300.bytes) val tccPacket = with(RtcpFbTccPacketBuilder(mediaSourceSsrc = 123, feedbackPacketSeqNum = 1)) { - SetBase(4, 130) - AddReceivedPacket(5, 130) + SetBase(4, instantOfEpochMicro(130)) + AddReceivedPacket(5, instantOfEpochMicro(130)) build() } transportCcEngine.rtcpPacketReceived(tccPacket, clock.instant()) @@ -118,8 +119,8 @@ class TransportCcEngineTest : FunSpec() { lossListener.numLost shouldBe 1 val tccPacket2 = with(RtcpFbTccPacketBuilder(mediaSourceSsrc = 123, feedbackPacketSeqNum = 2)) { - SetBase(4, 130) - AddReceivedPacket(4, 130) + SetBase(4, instantOfEpochMicro(130)) + AddReceivedPacket(4, instantOfEpochMicro(130)) build() } diff --git a/pom.xml b/pom.xml index 3a00efafcb..631ee0f0d6 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ 2.0.0 5.9.1 5.10.2 - 1.0-132-g83984af + 1.0-133-g6af1020 1.1-141-g30ec741 1.13.11 3.2.0 diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacket.kt index 8566f34421..65a08be936 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacket.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacket.kt @@ -30,7 +30,7 @@ import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket.Companio import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket.Companion.kDeltaScaleFactor import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket.Companion.kMaxReportedPackets import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket.Companion.kMaxSizeBytes -import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket.Companion.kTimeWrapPeriodUs +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket.Companion.kTimeWrapPeriod import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket.Companion.kTransportFeedbackHeaderSizeBytes import org.jitsi.rtp.rtp.RtpSequenceNumber import org.jitsi.rtp.rtp.toRtpSequenceNumber @@ -39,8 +39,12 @@ import org.jitsi.rtp.util.RtpUtils import org.jitsi.rtp.util.get3BytesAsInt import org.jitsi.rtp.util.getByteAsInt import org.jitsi.rtp.util.getShortAsInt +import org.jitsi.utils.micros +import org.jitsi.utils.times +import org.jitsi.utils.toEpochMicro +import org.jitsi.utils.toMicros import java.time.Duration -import java.time.temporal.ChronoUnit +import java.time.Instant sealed class PacketReport(val seqNum: Int) @@ -52,7 +56,7 @@ typealias DeltaSize = Int class ReceivedPacketReport(seqNum: Int, val deltaTicks: Short) : PacketReport(seqNum) { val deltaDuration: Duration - get() = Duration.of(deltaTicks * 250L, ChronoUnit.MICROS) + get() = deltaTicks.toInt() * kDeltaScaleFactor } /** @@ -95,23 +99,25 @@ class RtcpFbTccPacketBuilder( // The size of the entire packet, in bytes private var size_bytes_ = kTransportFeedbackHeaderSizeBytes - private var last_timestamp_us_: Long = 0 + private var last_timestamp_: Instant = Instant.EPOCH private val packets_ = mutableListOf() - fun SetBase(base_sequence: Int, ref_timestamp_us: Long) { + fun SetBase(base_sequence: Int, ref_timestamp: Instant) { base_seq_no_ = base_sequence.toRtpSequenceNumber() - base_time_ticks_ = (ref_timestamp_us % kTimeWrapPeriodUs) / kBaseScaleFactor - last_timestamp_us_ = GetBaseTimeUs() + base_time_ticks_ = (ref_timestamp.toEpochMicro() % kTimeWrapPeriod.toMicros()) / kBaseScaleFactor.toMicros() + last_timestamp_ = BaseTime() } - fun AddReceivedPacket(seqNum: Int, timestamp_us: Long): Boolean { + fun AddReceivedPacket(seqNum: Int, timestamp: Instant): Boolean { val sequence_number = seqNum.toRtpSequenceNumber() - var delta_full = (timestamp_us - last_timestamp_us_) % kTimeWrapPeriodUs - if (delta_full > kTimeWrapPeriodUs / 2) { - delta_full -= kTimeWrapPeriodUs + var delta_full = Duration.between(last_timestamp_, timestamp).toMicros() % kTimeWrapPeriod.toMicros() + if (delta_full > kTimeWrapPeriod.toMicros() / 2) { + delta_full -= kTimeWrapPeriod.toMicros() + delta_full -= kDeltaScaleFactor.toMicros() / 2 + } else { + delta_full += kDeltaScaleFactor.toMicros() / 2 } - delta_full += if (delta_full < 0) -(kDeltaScaleFactor / 2) else kDeltaScaleFactor / 2 - delta_full /= kDeltaScaleFactor + delta_full /= kDeltaScaleFactor.toMicros() val delta = delta_full.toShort() // If larger than 16bit signed, we can't represent it - need new fb packet. @@ -137,13 +143,15 @@ class RtcpFbTccPacketBuilder( } packets_.add(ReceivedPacketReport(sequence_number.value, delta)) - last_timestamp_us_ += delta * kDeltaScaleFactor + last_timestamp_ += delta.toInt() * kDeltaScaleFactor size_bytes_ += delta_size return true } - fun GetBaseTimeUs(): Long = base_time_ticks_ * kBaseScaleFactor + fun BaseTime(): Instant { + return Instant.EPOCH + base_time_ticks_ * kBaseScaleFactor + } private fun AddDeltaSize(deltaSize: DeltaSize): Boolean { if (num_seq_no_ == kMaxReportedPackets) { @@ -295,7 +303,7 @@ class RtcpFbTccPacket( val encoded_chunks_: MutableList, var last_chunk_: LastChunk, var num_seq_no_: Int, - var last_timestamp_us_: Long, + var last_timestamp_: Instant, val packets_: MutableList ) @@ -305,7 +313,7 @@ class RtcpFbTccPacket( val encoded_chunks_ = mutableListOf() val last_chunk_ = LastChunk() val num_seq_no_: Int - var last_timestamp_us_: Long = 0 + var last_timestamp_: Instant = Instant.EPOCH val packets_ = mutableListOf() val base_time_ticks_ = getReferenceTimeTicks(buffer, offset) @@ -343,13 +351,13 @@ class RtcpFbTccPacket( 1 -> { val delta = buffer[index] packets_.add(ReceivedPacketReport(seq_no.value, delta.toPositiveShort())) - last_timestamp_us_ += delta * kDeltaScaleFactor + last_timestamp_ += delta.toInt() * kDeltaScaleFactor index += delta_size } 2 -> { val delta = buffer.getShortAsInt(index) packets_.add(ReceivedPacketReport(seq_no.value, delta.toShort())) - last_timestamp_us_ += delta * kDeltaScaleFactor + last_timestamp_ += delta * kDeltaScaleFactor index += delta_size } 3 -> { @@ -376,7 +384,7 @@ class RtcpFbTccPacket( encoded_chunks_, last_chunk_, num_seq_no_, - last_timestamp_us_, + last_timestamp_, packets_ ) } @@ -401,10 +409,10 @@ class RtcpFbTccPacket( } private val packets_: MutableList get() = data.packets_ - private var last_timestamp_us_: Long - get() = data.last_timestamp_us_ + private var last_timestamp_: Instant + get() = data.last_timestamp_ set(value) { - data.last_timestamp_us_ = value + data.last_timestamp_ = value } // The reference time, in ticks. @@ -416,7 +424,9 @@ class RtcpFbTccPacket( val feedbackSeqNum: Int = getFeedbackPacketCount(buffer, offset) - fun GetBaseTimeUs(): Long = base_time_ticks_ * kBaseScaleFactor + fun BaseTime(): Instant { + return Instant.EPOCH + base_time_ticks_ * kBaseScaleFactor + } override fun iterator(): Iterator = packets_.iterator() @@ -426,7 +436,7 @@ class RtcpFbTccPacket( const val FMT = 15 // Convert to multiples of 0.25ms - const val kDeltaScaleFactor = 250 + val kDeltaScaleFactor = 250.micros // Maximum number of packets_ (including missing) TransportFeedback can report. const val kMaxReportedPackets = 0xFFFF @@ -442,11 +452,11 @@ class RtcpFbTccPacket( const val kTransportFeedbackHeaderSizeBytes = 4 + 8 + 8 // Used to convert from microseconds to multiples of 64ms - const val kBaseScaleFactor = kDeltaScaleFactor * (1 shl 8) + val kBaseScaleFactor = kDeltaScaleFactor * (1 shl 8) // The reference time field is 24 bits and are represented as multiples of 64ms // When the reference time field would need to wrap around - const val kTimeWrapPeriodUs: Long = (1 shl 24).toLong() * kBaseScaleFactor + val kTimeWrapPeriod = (1 shl 24).toLong() * kBaseScaleFactor const val BASE_SEQ_NUM_OFFSET = RtcpFbPacket.HEADER_SIZE const val PACKET_STATUS_COUNT_OFFSET = RtcpFbPacket.HEADER_SIZE + 2 diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacketTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacketTest.kt index 6144aa2230..12146eeff4 100644 --- a/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacketTest.kt +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacketTest.kt @@ -24,6 +24,8 @@ import io.kotest.matchers.shouldBe import io.kotest.matchers.types.beInstanceOf import org.jitsi.rtp.rtcp.RtcpHeaderBuilder import org.jitsi.rtp.util.byteBufferOf +import org.jitsi.utils.instantOfEpochMicro +import org.jitsi.utils.micros import java.time.Duration class RtcpFbTccPacketTest : ShouldSpec() { @@ -177,13 +179,13 @@ class RtcpFbTccPacketTest : ShouldSpec() { mediaSourceSsrc = 2397376430, feedbackPacketSeqNum = 162 ) - rtcpFbTccPacketBuilder.SetBase(6227, 107784064) - rtcpFbTccPacketBuilder.AddReceivedPacket(6228, 107784064) shouldBe true + rtcpFbTccPacketBuilder.SetBase(6227, instantOfEpochMicro(107784064)) + rtcpFbTccPacketBuilder.AddReceivedPacket(6228, instantOfEpochMicro(107784064)) shouldBe true } context("Creating and parsing an RtcpFbTccPacket") { context("with missing packets") { val kBaseSeqNo = 1000 - val kBaseTimestampUs = 10000L + val kBaseTimestamp = instantOfEpochMicro(10000L) val rtcpFbTccPacketBuilder = RtcpFbTccPacketBuilder( rtcpHeader = RtcpHeaderBuilder( senderSsrc = 839852602 @@ -191,9 +193,9 @@ class RtcpFbTccPacketTest : ShouldSpec() { mediaSourceSsrc = 2397376430, feedbackPacketSeqNum = 163 ) - rtcpFbTccPacketBuilder.SetBase(kBaseSeqNo, kBaseTimestampUs) - rtcpFbTccPacketBuilder.AddReceivedPacket(kBaseSeqNo + 0, kBaseTimestampUs) - rtcpFbTccPacketBuilder.AddReceivedPacket(kBaseSeqNo + 3, kBaseTimestampUs + 2000) + rtcpFbTccPacketBuilder.SetBase(kBaseSeqNo, kBaseTimestamp) + rtcpFbTccPacketBuilder.AddReceivedPacket(kBaseSeqNo + 0, kBaseTimestamp) + rtcpFbTccPacketBuilder.AddReceivedPacket(kBaseSeqNo + 3, kBaseTimestamp + 2000.micros) val coded = rtcpFbTccPacketBuilder.build() From d23d24405f84da4cdb7f632340639e52d68c6989 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Sep 2024 08:13:15 -0500 Subject: [PATCH 165/189] chore(deps): Bump io.sentry:sentry from 7.9.0 to 7.14.0 in /jvb (#2209) Bumps [io.sentry:sentry](https://github.com/getsentry/sentry-java) from 7.9.0 to 7.14.0. - [Release notes](https://github.com/getsentry/sentry-java/releases) - [Changelog](https://github.com/getsentry/sentry-java/blob/main/CHANGELOG.md) - [Commits](https://github.com/getsentry/sentry-java/compare/7.9.0...7.14.0) --- updated-dependencies: - dependency-name: io.sentry:sentry dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- jvb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvb/pom.xml b/jvb/pom.xml index 1aeab10a92..2aaa72ad20 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -208,7 +208,7 @@ io.sentry sentry - 7.9.0 + 7.14.0 runtime From b7dba8242cfaa84364bf6988cc097611346505c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Sep 2024 08:14:52 -0500 Subject: [PATCH 166/189] chore(deps): Bump io.sentry:sentry from 7.9.0 to 7.14.0 (#2205) Bumps [io.sentry:sentry](https://github.com/getsentry/sentry-java) from 7.9.0 to 7.14.0. - [Release notes](https://github.com/getsentry/sentry-java/releases) - [Changelog](https://github.com/getsentry/sentry-java/blob/main/CHANGELOG.md) - [Commits](https://github.com/getsentry/sentry-java/compare/7.9.0...7.14.0) --- updated-dependencies: - dependency-name: io.sentry:sentry dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> From 93b2699d4f7a2e804031a25325b7b5f9c0711b98 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Wed, 25 Sep 2024 16:38:40 -0400 Subject: [PATCH 167/189] Don't feed SCTP packets to the SCTP stack after it's been closed. (#2221) * Don't feed SCTP packets to the SCTP stack after it's been closed. (It results in harmless-but-scary warnings in the logs about verification tags being wrong.) * Change an sctp log to info. --- .../kotlin/org/jitsi/videobridge/Endpoint.kt | 6 +-- .../videobridge/dcsctp/DcSctpTransport.kt | 44 ++++++++++++++++--- .../org/jitsi/videobridge/relay/Relay.kt | 10 ++--- 3 files changed, 47 insertions(+), 13 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index 1b6a3c7bab..40a7df8278 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -1115,7 +1115,7 @@ class Endpoint @JvmOverloads constructor( sctpHandler?.stop() usrSctpHandler?.stop() sctpManager?.closeConnection() - sctpTransport?.socket?.close() + sctpTransport?.stop() } catch (t: Throwable) { logger.error("Exception while expiring: ", t) } @@ -1240,7 +1240,7 @@ class Endpoint @JvmOverloads constructor( } override fun OnAborted(error: ErrorKind, message: String) { - logger.warn("SCTP aborted with error $error: $message") + logger.info("SCTP aborted with error $error: $message") } override fun OnConnected() { @@ -1249,7 +1249,7 @@ class Endpoint @JvmOverloads constructor( val dataChannelStack = DataChannelStack( { data, sid, ppid -> val message = DcSctpMessage(sid.toShort(), ppid, data.array()) - val status = sctpTransport?.socket?.send(message, DcSctpTransport.DEFAULT_SEND_OPTIONS) + val status = sctpTransport?.send(message, DcSctpTransport.DEFAULT_SEND_OPTIONS) return@DataChannelStack if (status == SendStatus.kSuccess) { 0 } else { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpTransport.kt index 07d64f9343..0435f03a05 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpTransport.kt @@ -15,11 +15,13 @@ */ package org.jitsi.videobridge.dcsctp +import org.jitsi.dcsctp4j.DcSctpMessage import org.jitsi.dcsctp4j.DcSctpOptions import org.jitsi.dcsctp4j.DcSctpSocketCallbacks import org.jitsi.dcsctp4j.DcSctpSocketFactory import org.jitsi.dcsctp4j.DcSctpSocketInterface import org.jitsi.dcsctp4j.SendOptions +import org.jitsi.dcsctp4j.SendStatus import org.jitsi.dcsctp4j.Timeout import org.jitsi.nlj.PacketInfo import org.jitsi.utils.OrderedJsonObject @@ -40,19 +42,51 @@ class DcSctpTransport( parentLogger: Logger ) { val logger = createChildLogger(parentLogger) - lateinit var socket: DcSctpSocketInterface + private val lock = Any() + private var socket: DcSctpSocketInterface? = null fun start(callbacks: DcSctpSocketCallbacks, options: DcSctpOptions = DEFAULT_SOCKET_OPTIONS) { - socket = factory.create(name, callbacks, null, options) + synchronized(lock) { + socket = factory.create(name, callbacks, null, options) + } } fun handleIncomingSctp(packetInfo: PacketInfo) { val packet = packetInfo.packet - socket.receivePacket(packet.getBuffer(), packet.getOffset(), packet.getLength()) + synchronized(lock) { + socket?.receivePacket(packet.getBuffer(), packet.getOffset(), packet.getLength()) + } + } + + fun stop() { + synchronized(lock) { + socket?.close() + socket = null + } + } + + fun connect() { + synchronized(lock) { + socket?.connect() + } + } + + fun send(message: DcSctpMessage, options: SendOptions): SendStatus { + synchronized(lock) { + return socket?.send(message, options) ?: SendStatus.kErrorShuttingDown + } + } + + fun handleTimeout(timeoutId: Long) { + synchronized(lock) { + socket?.handleTimeout(timeoutId) + } } fun getDebugState(): OrderedJsonObject { - val metrics = socket.metrics + val metrics = synchronized(lock) { + socket?.metrics + } return OrderedJsonObject().apply { if (metrics != null) { put("tx_packets_count", metrics.txPacketsCount) @@ -157,7 +191,7 @@ abstract class DcSctpBaseCallbacks( scheduledFuture = TaskPools.SCHEDULED_POOL.schedule({ /* Execute it on the IO_POOL, because a timer may trigger sending new SCTP packets. */ future = TaskPools.IO_POOL.submit { - transport?.socket?.handleTimeout(timeoutId) + transport?.handleTimeout(timeoutId) } }, duration, TimeUnit.MILLISECONDS) } catch (e: Throwable) { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt index 2199ac57f6..3f1052bee1 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt @@ -439,7 +439,7 @@ class Relay @JvmOverloads constructor( scheduleRelayMessageTransportTimeout() } else if (sctpConfig.enabled) { if (sctpRole == Sctp.Role.CLIENT) { - sctpTransport!!.socket.connect() + sctpTransport!!.connect() } } } @@ -498,7 +498,7 @@ class Relay @JvmOverloads constructor( it.start(SctpCallbacks(it)) sctpHandler!!.setSctpTransport(it) if (dtlsTransport.isConnected && sctpDesc.role == Sctp.Role.CLIENT) { - it.socket.connect() + it.connect() } } } @@ -1172,7 +1172,7 @@ class Relay @JvmOverloads constructor( sctpHandler?.stop() usrSctpHandler?.stop() sctpManager?.closeConnection() - sctpTransport?.socket?.close() + sctpTransport?.stop() } catch (t: Throwable) { logger.error("Exception while expiring: ", t) } @@ -1266,7 +1266,7 @@ class Relay @JvmOverloads constructor( } override fun OnAborted(error: ErrorKind, message: String) { - logger.warn("SCTP aborted with error $error: $message") + logger.info("SCTP aborted with error $error: $message") } override fun OnConnected() { @@ -1275,7 +1275,7 @@ class Relay @JvmOverloads constructor( val dataChannelStack = DataChannelStack( { data, sid, ppid -> val message = DcSctpMessage(sid.toShort(), ppid, data.array()) - val status = sctpTransport?.socket?.send(message, DcSctpTransport.DEFAULT_SEND_OPTIONS) + val status = sctpTransport?.send(message, DcSctpTransport.DEFAULT_SEND_OPTIONS) return@DataChannelStack if (status == SendStatus.kSuccess) { 0 } else { From 0b2e8d6aadfad06e5ef25929ec15f255425b3400 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Wed, 2 Oct 2024 14:25:33 -0500 Subject: [PATCH 168/189] feat: Use the ice4j push API. (#2195) * feat: Use the ice4j push API. * feat: Put incoming DTLS packets on a queue. * Start writing to the ICE transport as soon as we have any validated pair. * chore: Bump to ice4j 3.2 --------- Co-authored-by: Jonathan Lennox --- .../kotlin/org/jitsi/videobridge/Endpoint.kt | 32 +++----- .../main/kotlin/org/jitsi/videobridge/Main.kt | 14 ++++ .../org/jitsi/videobridge/relay/Relay.kt | 28 +++---- .../transport/dtls/DtlsTransport.kt | 31 ++++++- .../videobridge/transport/ice/IceTransport.kt | 80 ++++++++++++++----- pom.xml | 2 +- 6 files changed, 129 insertions(+), 58 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index 40a7df8278..1b5ce37861 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -16,6 +16,7 @@ package org.jitsi.videobridge +import org.ice4j.util.Buffer import org.jitsi.config.JitsiConfig import org.jitsi.dcsctp4j.DcSctpMessage import org.jitsi.dcsctp4j.ErrorKind @@ -45,13 +46,11 @@ import org.jitsi.nlj.util.NEVER import org.jitsi.nlj.util.PacketInfoQueue import org.jitsi.nlj.util.RemoteSsrcAssociation import org.jitsi.nlj.util.sumOf -import org.jitsi.rtp.Packet import org.jitsi.rtp.UnparsedPacket import org.jitsi.rtp.rtcp.RtcpSrPacket import org.jitsi.rtp.rtcp.rtcpfb.RtcpFbPacket import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbFirPacket import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbPliPacket -import org.jitsi.rtp.rtp.RtpPacket import org.jitsi.utils.MediaType import org.jitsi.utils.concurrent.RecurringRunnableExecutor import org.jitsi.utils.logging2.Logger @@ -166,7 +165,7 @@ class Endpoint @JvmOverloads constructor( /* TODO: do we ever want to support useUniquePort for an Endpoint? */ private val iceTransport = IceTransport(id, iceControlling, false, supportsPrivateAddresses, logger) - private val dtlsTransport = DtlsTransport(logger).also { it.cryptex = CryptexConfig.endpoint } + private val dtlsTransport = DtlsTransport(logger, id).also { it.cryptex = CryptexConfig.endpoint } private var cryptex: Boolean = CryptexConfig.endpoint @@ -407,30 +406,21 @@ class Endpoint @JvmOverloads constructor( private fun setupIceTransport() { iceTransport.incomingDataHandler = object : IceTransport.IncomingDataHandler { - override fun dataReceived(data: ByteArray, offset: Int, length: Int, receivedTime: Instant) { - // DTLS data will be handled by the DtlsTransport, but SRTP data can go - // straight to the transceiver - if (looksLikeDtls(data, offset, length)) { - // DTLS transport is responsible for making its own copy, because it will manage its own - // buffers - dtlsTransport.dtlsDataReceived(data, offset, length) + override fun dataReceived(buffer: Buffer) { + if (looksLikeDtls(buffer.buffer, buffer.offset, buffer.length)) { + // DTLS transport is responsible for making its own copy, because it will manage its own buffers + dtlsTransport.enqueueBuffer(buffer) } else { - val copy = ByteBufferPool.getBuffer( - length + - RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET + - Packet.BYTES_TO_LEAVE_AT_END_OF_PACKET - ) - System.arraycopy(data, offset, copy, RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET, length) val pktInfo = - PacketInfo(UnparsedPacket(copy, RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET, length)).apply { - this.receivedTime = receivedTime + PacketInfo(UnparsedPacket(buffer.buffer, buffer.offset, buffer.length)).apply { + this.receivedTime = buffer.receivedTime } transceiver.handleIncomingPacket(pktInfo) } } } iceTransport.eventHandler = object : IceTransport.EventHandler { - override fun connected() { + override fun writeable() { logger.info("ICE connected") transceiver.setOutgoingPacketHandler(object : PacketHandler { override fun processPacket(packetInfo: PacketInfo) { @@ -438,10 +428,12 @@ class Endpoint @JvmOverloads constructor( outgoingSrtpPacketQueue.add(packetInfo) } }) - TaskPools.IO_POOL.execute(iceTransport::startReadingData) TaskPools.IO_POOL.execute(dtlsTransport::startDtlsHandshake) } + override fun connected() { + } + override fun failed() { } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt index ec623884ae..07b4e4d668 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt @@ -18,7 +18,9 @@ package org.jitsi.videobridge import org.eclipse.jetty.servlet.ServletHolder import org.glassfish.jersey.servlet.ServletContainer +import org.ice4j.ice.harvest.AbstractUdpListener import org.ice4j.ice.harvest.MappingCandidateHarvesters +import org.ice4j.util.Buffer import org.jitsi.config.JitsiConfig import org.jitsi.metaconfig.ConfigException import org.jitsi.metaconfig.MetaconfigLogger @@ -29,6 +31,8 @@ import org.jitsi.rest.createServer import org.jitsi.rest.enableCors import org.jitsi.rest.isEnabled import org.jitsi.rest.servletContextHandler +import org.jitsi.rtp.Packet +import org.jitsi.rtp.rtp.RtpPacket import org.jitsi.shutdown.ShutdownServiceImpl import org.jitsi.utils.logging2.LoggerImpl import org.jitsi.utils.queue.PacketQueue @@ -220,6 +224,7 @@ private fun getSystemPropertyDefaults(): Map { } private fun startIce4j() { + AbstractUdpListener.USE_PUSH_API = true // Start the initialization of the mapping candidate harvesters. // Asynchronous, because the AWS and STUN harvester may take a long // time to initialize. @@ -239,4 +244,13 @@ private fun setupBufferPools() { org.jitsi.rtp.util.BufferPool.returnArray = { ByteBufferPool.returnBuffer(it) } org.jitsi.nlj.util.BufferPool.getBuffer = { ByteBufferPool.getBuffer(it) } org.jitsi.nlj.util.BufferPool.returnBuffer = { ByteBufferPool.returnBuffer(it) } + org.ice4j.util.BufferPool.getBuffer = { len -> + val b = ByteBufferPool.getBuffer(len) + Buffer(b, 0, b.size) + } + org.ice4j.util.BufferPool.returnBuffer = { ByteBufferPool.returnBuffer(it.buffer) } + org.ice4j.ice.harvest.AbstractUdpListener.BYTES_TO_LEAVE_AT_START_OF_PACKET = + RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET + org.ice4j.ice.harvest.AbstractUdpListener.BYTES_TO_LEAVE_AT_END_OF_PACKET = + Packet.BYTES_TO_LEAVE_AT_END_OF_PACKET } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt index 3f1052bee1..cbb6e70e1c 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt @@ -15,6 +15,7 @@ */ package org.jitsi.videobridge.relay +import org.ice4j.util.Buffer import org.jitsi.dcsctp4j.DcSctpMessage import org.jitsi.dcsctp4j.ErrorKind import org.jitsi.dcsctp4j.SendPacketStatus @@ -47,7 +48,6 @@ import org.jitsi.nlj.util.LocalSsrcAssociation import org.jitsi.nlj.util.PacketInfoQueue import org.jitsi.nlj.util.RemoteSsrcAssociation import org.jitsi.nlj.util.sumOf -import org.jitsi.rtp.Packet import org.jitsi.rtp.UnparsedPacket import org.jitsi.rtp.extensions.looksLikeRtcp import org.jitsi.rtp.extensions.looksLikeRtp @@ -220,7 +220,7 @@ class Relay @JvmOverloads constructor( clock = clock ) - private val dtlsTransport = DtlsTransport(logger).also { it.cryptex = CryptexConfig.relay } + private val dtlsTransport = DtlsTransport(logger, id).also { it.cryptex = CryptexConfig.relay } private var cryptex = CryptexConfig.relay @@ -365,33 +365,26 @@ class Relay @JvmOverloads constructor( private fun setupIceTransport() { iceTransport.incomingDataHandler = object : IceTransport.IncomingDataHandler { - override fun dataReceived(data: ByteArray, offset: Int, length: Int, receivedTime: Instant) { + override fun dataReceived(buffer: Buffer) { // DTLS data will be handled by the DtlsTransport, but SRTP data can go // straight to the transceiver - if (looksLikeDtls(data, offset, length)) { - // DTLS transport is responsible for making its own copy, because it will manage its own - // buffers - dtlsTransport.dtlsDataReceived(data, offset, length) + if (looksLikeDtls(buffer.buffer, buffer.offset, buffer.length)) { + // DTLS transport is responsible for making its own copy, because it will manage its own buffers + dtlsTransport.enqueueBuffer(buffer) } else { - val copy = ByteBufferPool.getBuffer( - length + - RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET + - Packet.BYTES_TO_LEAVE_AT_END_OF_PACKET - ) - System.arraycopy(data, offset, copy, RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET, length) val pktInfo = RelayedPacketInfo( - UnparsedPacket(copy, RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET, length), + UnparsedPacket(buffer.buffer, buffer.offset, buffer.length), meshId ).apply { - this.receivedTime = receivedTime + this.receivedTime = buffer.receivedTime } handleMediaPacket(pktInfo) } } } iceTransport.eventHandler = object : IceTransport.EventHandler { - override fun connected() { + override fun writeable() { logger.info("ICE connected") transceiver.setOutgoingPacketHandler(object : PacketHandler { override fun processPacket(packetInfo: PacketInfo) { @@ -399,10 +392,11 @@ class Relay @JvmOverloads constructor( outgoingSrtpPacketQueue.add(packetInfo) } }) - TaskPools.IO_POOL.execute(iceTransport::startReadingData) TaskPools.IO_POOL.execute(dtlsTransport::startDtlsHandshake) } + override fun connected() {} + override fun failed() {} override fun consentUpdated(time: Instant) { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/dtls/DtlsTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/dtls/DtlsTransport.kt index ed9c2276a0..e6db0a8293 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/dtls/DtlsTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/dtls/DtlsTransport.kt @@ -16,13 +16,17 @@ package org.jitsi.videobridge.transport.dtls +import org.ice4j.util.Buffer import org.jitsi.nlj.dtls.DtlsClient import org.jitsi.nlj.dtls.DtlsServer import org.jitsi.nlj.dtls.DtlsStack import org.jitsi.nlj.srtp.TlsRole +import org.jitsi.nlj.util.BufferPool import org.jitsi.utils.OrderedJsonObject import org.jitsi.utils.logging2.Logger import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.utils.queue.PacketQueue +import org.jitsi.videobridge.util.TaskPools import org.jitsi.xmpp.extensions.jingle.DtlsFingerprintPacketExtension import org.jitsi.xmpp.extensions.jingle.IceUdpTransportPacketExtension import java.util.concurrent.atomic.AtomicBoolean @@ -39,7 +43,7 @@ import java.util.concurrent.atomic.AtomicBoolean * be passed to the [outgoingDataHandler], which should be set by an * interested party. */ -class DtlsTransport(parentLogger: Logger) { +class DtlsTransport(parentLogger: Logger, id: String) { private val logger = createChildLogger(parentLogger) private val running = AtomicBoolean(true) @@ -62,6 +66,26 @@ class DtlsTransport(parentLogger: Logger) { /** Whether to advertise cryptex to peers. */ var cryptex = false + val dtlsQueue = object : PacketQueue( + 128, + null, + "dtls-queue-$id", + { buffer: Buffer -> + try { + dtlsDataReceived(buffer.buffer, buffer.offset, buffer.length) + true + } catch (e: Exception) { + logger.warn("Failed to handle DTLS data", e) + false + } + }, + TaskPools.IO_POOL, + ) { + override fun releasePacket(buffer: Buffer) { + BufferPool.returnBuffer(buffer.buffer) + } + } + /** * The DTLS stack instance */ @@ -170,10 +194,13 @@ class DtlsTransport(parentLogger: Logger) { } } + fun enqueueBuffer(buffer: Buffer) = dtlsQueue.add(buffer) + /** * Notify this layer that DTLS data has been received from the network */ - fun dtlsDataReceived(data: ByteArray, off: Int, len: Int) = dtlsStack.processIncomingProtocolData(data, off, len) + private fun dtlsDataReceived(data: ByteArray, off: Int, len: Int) = + dtlsStack.processIncomingProtocolData(data, off, len) /** * Send out DTLS data diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt index 9f7ae36245..b33681dce2 100755 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt @@ -27,7 +27,10 @@ import org.ice4j.ice.IceProcessingState import org.ice4j.ice.LocalCandidate import org.ice4j.ice.RemoteCandidate import org.ice4j.ice.harvest.MappingCandidateHarvesters -import org.ice4j.socket.SocketClosedException +import org.ice4j.util.Buffer +import org.ice4j.util.BufferHandler +import org.jitsi.rtp.Packet +import org.jitsi.rtp.rtp.RtpPacket import org.jitsi.utils.OrderedJsonObject import org.jitsi.utils.logging2.Logger import org.jitsi.utils.logging2.cdebug @@ -36,6 +39,8 @@ import org.jitsi.videobridge.ice.Harvesters import org.jitsi.videobridge.ice.IceConfig import org.jitsi.videobridge.ice.TransportUtils import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer +import org.jitsi.videobridge.util.ByteBufferPool +import org.jitsi.videobridge.util.TaskPools import org.jitsi.xmpp.extensions.jingle.CandidatePacketExtension import org.jitsi.xmpp.extensions.jingle.IceCandidatePacketExtension import org.jitsi.xmpp.extensions.jingle.IceRtcpmuxPacketExtension @@ -62,7 +67,7 @@ class IceTransport @JvmOverloads constructor( * Whether the ICE agent created by this transport should use * unique local ports, rather than the configured port. */ - useUniquePort: Boolean, + val useUniquePort: Boolean, /** * Use private addresses for this [IceTransport] even if [IceConfig.advertisePrivateCandidates] is false. */ @@ -73,14 +78,9 @@ class IceTransport @JvmOverloads constructor( private val logger = createChildLogger(parentLogger) /** - * The handler which will be invoked when data is received. The handler - * does *not* own the buffer passed to it, so a copy must be made if it wants - * to use the data after the handler call finishes. This field should be - * set by some other entity which wishes to handle the incoming data + * The handler which will be invoked when data is received. + * This field should be set by some other entity which wishes to handle the incoming data * received over the ICE connection. - * NOTE: we don't create a packet in [IceTransport] because - * RTP packets want space before and after and [IceTransport] - * has no notion of what kind of data is contained within the buffer. */ @JvmField var incomingDataHandler: IncomingDataHandler? = null @@ -94,6 +94,13 @@ class IceTransport @JvmOverloads constructor( @JvmField var eventHandler: EventHandler? = null + /** + * Whether or not it is possible to write to this [IceTransport]. + * + * This happens as soon as any candidate pair is validated, and happens (usually) before iceConnected. + */ + private val iceWriteable = AtomicBoolean(false) + /** * Whether or not this [IceTransport] has connected. */ @@ -106,6 +113,8 @@ class IceTransport @JvmOverloads constructor( fun hasFailed(): Boolean = iceFailed.get() + fun isWriteable(): Boolean = iceWriteable.get() + fun isConnected(): Boolean = iceConnected.get() /** @@ -135,7 +144,16 @@ class IceTransport @JvmOverloads constructor( addPairChangeListener(iceStreamPairChangedListener) } - private val iceComponent = iceAgent.createComponent(iceStream, IceConfig.config.keepAliveStrategy, true) + private val iceComponent = iceAgent.createComponent(iceStream, IceConfig.config.keepAliveStrategy, false).apply { + setBufferCallback(object : BufferHandler { + override fun handleBuffer(buffer: Buffer) { + incomingDataHandler?.dataReceived(buffer) ?: run { + packetStats.numIncomingPacketsDroppedNoHandler.increment() + ByteBufferPool.returnBuffer(buffer.buffer) + } + } + }) + } private val packetStats = PacketStats() val icePassword: String get() = iceAgent.localPassword @@ -204,7 +222,7 @@ class IceTransport @JvmOverloads constructor( fun startReadingData() { logger.cdebug { "Starting to read incoming data" } - val socket = iceComponent.socket + val socket = iceComponent.selectedPair.iceSocketWrapper val receiveBuf = ByteArray(1500) val packet = DatagramPacket(receiveBuf, 0, receiveBuf.size) var receivedTime: Instant @@ -213,16 +231,25 @@ class IceTransport @JvmOverloads constructor( try { socket.receive(packet) receivedTime = clock.instant() - } catch (e: SocketClosedException) { - logger.info("Socket closed, stopping reader") - break } catch (e: IOException) { logger.warn("Stopping reader", e) break } packetStats.numPacketsReceived.increment() try { - incomingDataHandler?.dataReceived(receiveBuf, packet.offset, packet.length, receivedTime) ?: run { + val b = ByteBufferPool.getBuffer( + RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET + packet.length + Packet.BYTES_TO_LEAVE_AT_END_OF_PACKET + ) + System.arraycopy( + packet.data, + packet.offset, + b, + RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET, + packet.length + ) + val buffer = Buffer(b, RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET, packet.length, receivedTime) + + incomingDataHandler?.dataReceived(buffer) ?: run { logger.cdebug { "Data handler is null, dropping data" } packetStats.numIncomingPacketsDroppedNoHandler.increment() } @@ -239,7 +266,7 @@ class IceTransport @JvmOverloads constructor( fun send(data: ByteArray, off: Int, length: Int) { if (running.get()) { try { - iceComponent.socket.send(DatagramPacket(data, off, length)) + iceComponent.send(data, off, length) packetStats.numPacketsSent.increment() } catch (e: IOException) { logger.error("Error sending packet", e) @@ -264,6 +291,7 @@ class IceTransport @JvmOverloads constructor( put("nominationStrategy", IceConfig.config.nominationStrategy.toString()) put("advertisePrivateCandidates", IceConfig.config.advertisePrivateCandidates) put("closed", !running.get()) + put("iceWriteable", iceWriteable.get()) put("iceConnected", iceConnected.get()) put("iceFailed", iceFailed.get()) putAll(packetStats.toJson()) @@ -344,6 +372,13 @@ class IceTransport @JvmOverloads constructor( transition.completed() -> { if (iceConnected.compareAndSet(false, true)) { eventHandler?.connected() + if (useUniquePort) { + // ice4j's push API only works with the single port harvester. With unique ports we still need + // to read from the socket. + TaskPools.IO_POOL.submit { + startReadingData() + } + } if (iceComponent.selectedPair.remoteCandidate.type == CandidateType.RELAYED_CANDIDATE || iceComponent.selectedPair.localCandidate.type == CandidateType.RELAYED_CANDIDATE ) { @@ -375,7 +410,11 @@ class IceTransport @JvmOverloads constructor( } private fun iceStreamPairChanged(ev: PropertyChangeEvent) { - if (IceMediaStream.PROPERTY_PAIR_CONSENT_FRESHNESS_CHANGED == ev.propertyName) { + if (IceMediaStream.PROPERTY_PAIR_VALIDATED == ev.propertyName) { + if (iceWriteable.compareAndSet(false, true)) { + eventHandler?.writeable() + } + } else if (IceMediaStream.PROPERTY_PAIR_CONSENT_FRESHNESS_CHANGED == ev.propertyName) { /* TODO: Currently ice4j only triggers this event for the selected * pair, but should we double-check the pair anyway? */ @@ -435,10 +474,15 @@ class IceTransport @JvmOverloads constructor( * Notify the handler that data was received (contained * within [data] at [offset] with [length]) at [receivedTime] */ - fun dataReceived(data: ByteArray, offset: Int, length: Int, receivedTime: Instant) + fun dataReceived(buffer: Buffer) } interface EventHandler { + /** + * Notify the event handler that it is possible to write to the ICE stack + */ + fun writeable() + /** * Notify the event handler that ICE connected successfully */ diff --git a/pom.xml b/pom.xml index 631ee0f0d6..517a7c9199 100644 --- a/pom.xml +++ b/pom.xml @@ -106,7 +106,7 @@ ${project.groupId} ice4j - 3.0-72-g824cd4b + 3.2-0-g63cddcd ${project.groupId} From 28674f78e7470b53bb47478302afaa626a84c625 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Mon, 7 Oct 2024 12:52:27 -0400 Subject: [PATCH 169/189] Update ice4j. (#2232) * Agent should fail on SocketNotFoundException * Don't start multiple PaceMakers for a single CheckList. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 517a7c9199..e2f1265b1f 100644 --- a/pom.xml +++ b/pom.xml @@ -106,7 +106,7 @@ ${project.groupId} ice4j - 3.2-0-g63cddcd + 3.2-2-gb4c8cd1 ${project.groupId} From 4e95a314801a79bbdba11fbf72e8a62e1f9b11a8 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Mon, 21 Oct 2024 15:17:55 -0500 Subject: [PATCH 170/189] chore: Update jicoco to 1.1-142-gfed0320. (#2233) --- jvb/pom.xml | 4 ++++ pom.xml | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/jvb/pom.xml b/jvb/pom.xml index 2aaa72ad20..ec9e0f3d45 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -143,6 +143,10 @@ ${project.groupId} jicoco-metrics + + ${project.groupId} + jicoco-metrics-jetty + ${project.groupId} jitsi-xmpp-extensions diff --git a/pom.xml b/pom.xml index e2f1265b1f..818ea628ab 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 5.9.1 5.10.2 1.0-133-g6af1020 - 1.1-141-g30ec741 + 1.1-142-gfed0320 1.13.11 3.2.0 3.6.0 @@ -93,6 +93,11 @@ jicoco-metrics ${jicoco.version} + + ${project.groupId} + jicoco-metrics-jetty + ${jicoco.version} + ${project.groupId} jitsi-utils From 9925c61120674b56cad6fd5142e13d336b9e42ca Mon Sep 17 00:00:00 2001 From: bgrozev Date: Tue, 22 Oct 2024 11:31:28 -0500 Subject: [PATCH 171/189] ref: Move Prometheus from jicoco. (#2235) * ref: Move Prometheus from jicoco. * chore: Update jicoco to 1.1-143. --- jvb/pom.xml | 4 -- .../rest/prometheus/Prometheus.java | 66 +++++++++++++++++++ .../videobridge/rest/root/Application.java | 4 +- pom.xml | 7 +- 4 files changed, 69 insertions(+), 12 deletions(-) create mode 100644 jvb/src/main/java/org/jitsi/videobridge/rest/prometheus/Prometheus.java diff --git a/jvb/pom.xml b/jvb/pom.xml index ec9e0f3d45..2aaa72ad20 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -143,10 +143,6 @@ ${project.groupId} jicoco-metrics - - ${project.groupId} - jicoco-metrics-jetty - ${project.groupId} jitsi-xmpp-extensions diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/prometheus/Prometheus.java b/jvb/src/main/java/org/jitsi/videobridge/rest/prometheus/Prometheus.java new file mode 100644 index 0000000000..0e67d409f9 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/prometheus/Prometheus.java @@ -0,0 +1,66 @@ +/* + * Copyright @ 2022 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.prometheus; + +import io.prometheus.client.exporter.common.*; +import jakarta.ws.rs.*; +import jakarta.ws.rs.core.*; +import org.jetbrains.annotations.NotNull; +import org.jitsi.metrics.*; + +/** + * A REST endpoint exposing JVB stats for Prometheus. + * Any scraper supporting Prometheus' text-based formats ({@code text/plain; version=0.0.4} or OpenMetrics) + * is compatible with this {@code /metrics} endpoint.
+ * JSON is provided when the client performs a request with the 'Accept' header set to {@code application/json}.
+ * The response defaults to {@code text/plain; version=0.0.4} formatted output. + * + * @see + * Prometheus' exposition formats + */ +@Path("/metrics") +public class Prometheus +{ + @NotNull + private final MetricsContainer metricsContainer; + + public Prometheus(@NotNull MetricsContainer metricsContainer) + { + this.metricsContainer = metricsContainer; + } + + @GET + @Produces(TextFormat.CONTENT_TYPE_004) + public String getPrometheusPlainText() + { + return metricsContainer.getPrometheusMetrics(TextFormat.CONTENT_TYPE_004); + } + + @GET + @Produces(TextFormat.CONTENT_TYPE_OPENMETRICS_100) + public String getPrometheusOpenMetrics() + { + return metricsContainer.getPrometheusMetrics(TextFormat.CONTENT_TYPE_OPENMETRICS_100); + } + + @GET + @Produces(MediaType.APPLICATION_JSON) + public String getJsonString() + { + return metricsContainer.getJsonString(); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/root/Application.java b/jvb/src/main/java/org/jitsi/videobridge/rest/root/Application.java index 0dfa4e2462..8e85f63011 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/rest/root/Application.java +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/root/Application.java @@ -25,7 +25,7 @@ import org.jitsi.videobridge.rest.*; import org.jitsi.videobridge.rest.binders.*; import org.jitsi.videobridge.rest.filters.*; -import org.jitsi.videobridge.stats.*; +import org.jitsi.videobridge.rest.prometheus.*; import org.jitsi.videobridge.xmpp.*; import static org.jitsi.videobridge.rest.RestConfig.config; @@ -61,7 +61,7 @@ public Application( } if (config.isEnabled(RestApis.PROMETHEUS)) { - register(new org.jitsi.rest.prometheus.Prometheus(VideobridgeMetricsContainer.getInstance())); + register(new Prometheus(VideobridgeMetricsContainer.getInstance())); } } } diff --git a/pom.xml b/pom.xml index 818ea628ab..9f3e11d7b6 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 5.9.1 5.10.2 1.0-133-g6af1020 - 1.1-142-gfed0320 + 1.1-143-g175c44b 1.13.11 3.2.0 3.6.0 @@ -93,11 +93,6 @@ jicoco-metrics ${jicoco.version} - - ${project.groupId} - jicoco-metrics-jetty - ${jicoco.version} - ${project.groupId} jitsi-utils From 8c4bcf9e24134c2cc686a40ac71bd61122957359 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Wed, 23 Oct 2024 11:35:27 -0400 Subject: [PATCH 172/189] Bump ice4j to use SocketPool. Remove usePlainDatagramSocketImpl flag. (#2236) Bump ice4j to use SocketPool. Remove usePlainDatagramSocketImpl flag (no longer needed). --- jvb/resources/jvb.sh | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jvb/resources/jvb.sh b/jvb/resources/jvb.sh index 97821b128b..fb460a67c3 100755 --- a/jvb/resources/jvb.sh +++ b/jvb/resources/jvb.sh @@ -19,4 +19,4 @@ fi if [ -z "$VIDEOBRIDGE_MAX_MEMORY" ]; then VIDEOBRIDGE_MAX_MEMORY=3072m; fi if [ -z "$VIDEOBRIDGE_GC_TYPE" ]; then VIDEOBRIDGE_GC_TYPE=G1GC; fi -exec java -Xmx$VIDEOBRIDGE_MAX_MEMORY $VIDEOBRIDGE_DEBUG_OPTIONS -XX:+Use$VIDEOBRIDGE_GC_TYPE -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp -Djdk.tls.ephemeralDHKeySize=2048 -Djdk.net.usePlainDatagramSocketImpl=true $LOGGING_CONFIG_PARAM $JAVA_SYS_PROPS -cp $cp $mainClass $@ +exec java -Xmx$VIDEOBRIDGE_MAX_MEMORY $VIDEOBRIDGE_DEBUG_OPTIONS -XX:+Use$VIDEOBRIDGE_GC_TYPE -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp -Djdk.tls.ephemeralDHKeySize=2048 $LOGGING_CONFIG_PARAM $JAVA_SYS_PROPS -cp $cp $mainClass $@ diff --git a/pom.xml b/pom.xml index 9f3e11d7b6..96df738c07 100644 --- a/pom.xml +++ b/pom.xml @@ -106,7 +106,7 @@ ${project.groupId} ice4j - 3.2-2-gb4c8cd1 + 3.2-4-g1373788 ${project.groupId} From 3ad96973a877165a1adf2a53022650b6eb4c91b9 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Wed, 23 Oct 2024 11:36:26 -0400 Subject: [PATCH 173/189] Add the source name to the source projection log context. (#2223) --- .../org/jitsi/videobridge/cc/AdaptiveSourceProjection.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjection.java b/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjection.java index 3cf093ea26..ae5ba2261f 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjection.java +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjection.java @@ -105,7 +105,9 @@ public AdaptiveSourceProjection( this.diagnosticContext = diagnosticContext; this.logger = parentLogger.createChildLogger(AdaptiveSourceProjection.class.getName(), Map.of("targetSsrc", Long.toString(targetSsrc), - "srcEpId", Objects.toString(source.getOwner(), ""))); + "srcEpId", Objects.toString(source.getOwner(), ""), + "srcName", source.getSourceName() + )); this.keyframeRequester = keyframeRequester; } From 7245c6a76ff0c40921ee7a1b0d5fad9bece779a5 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Wed, 23 Oct 2024 19:04:45 -0500 Subject: [PATCH 174/189] Fix RTT when SSRC rewriting is enabled. (#2237) * fix: Drop the packet if a preProcessor exists and rejects it. * fix: Also pre-process RTCP packets. * fix: Move the notifyRtcpSent call to the sender pipeline. Notably, this calls the notifier after the RTCP packet has passed the preProcessor where SSRC rewriting may change the packet. The callback is also moved from the receiver pipeline thread (calling into Conference.sendOut -> Endpoint.send) to the sender pipeline thread. The only code which uses this callback is EndpointConnectionStats, which saves sent SRs to calculate RTT. This commit fixes the RTT calculation when SSRC rewriting is used. --- .../kotlin/org/jitsi/nlj/RtpSenderImpl.kt | 25 +++++++++++-------- .../org/jitsi/nlj/transform/node/Node.kt | 13 ++++++++++ 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt index db2ae9d031..6a89f12b99 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt @@ -34,14 +34,15 @@ import org.jitsi.nlj.transform.NodeTeardownVisitor import org.jitsi.nlj.transform.node.AudioRedHandler import org.jitsi.nlj.transform.node.ConsumerNode import org.jitsi.nlj.transform.node.Node +import org.jitsi.nlj.transform.node.ObserverNode import org.jitsi.nlj.transform.node.PacketCacher import org.jitsi.nlj.transform.node.PacketLossConfig import org.jitsi.nlj.transform.node.PacketLossNode import org.jitsi.nlj.transform.node.PacketStreamStatsNode +import org.jitsi.nlj.transform.node.PluggableTransformerNode import org.jitsi.nlj.transform.node.SrtcpEncryptNode import org.jitsi.nlj.transform.node.SrtpEncryptNode import org.jitsi.nlj.transform.node.ToggleablePcapWriter -import org.jitsi.nlj.transform.node.TransformerNode import org.jitsi.nlj.transform.node.outgoing.AbsSendTime import org.jitsi.nlj.transform.node.outgoing.HeaderExtEncoder import org.jitsi.nlj.transform.node.outgoing.HeaderExtStripper @@ -141,12 +142,7 @@ class RtpSenderImpl( incomingPacketQueue.setErrorHandler(queueErrorCounter) outgoingRtpRoot = pipeline { - node(object : TransformerNode("Pre-processor") { - override fun transform(packetInfo: PacketInfo): PacketInfo? { - return preProcesor?.invoke(packetInfo) ?: packetInfo - } - override fun trace(f: () -> Unit) {} - }) + node(PluggableTransformerNode("RTP pre-processor") { preProcesor }) node(AudioRedHandler(streamInformationStore, logger)) node(HeaderExtStripper(streamInformationStore)) node(outgoingPacketCache) @@ -173,6 +169,7 @@ class RtpSenderImpl( // TODO: are we setting outgoing rtcp sequence numbers correctly? just add a simple node here to rewrite them outgoingRtcpRoot = pipeline { + node(PluggableTransformerNode("RTCP pre-processor") { preProcesor }) node(keyframeRequester) node(SentRtcpStats()) // TODO(brian): not sure this is a great idea. it works as a catch-call but can also be error-prone @@ -189,6 +186,16 @@ class RtpSenderImpl( } node(rtcpSrUpdater) node(toggleablePcapWriter.newObserverNode(outbound = true)) + node(object : ObserverNode("RTCP sent notifier") { + override fun observe(packetInfo: PacketInfo) { + val packet = packetInfo.packet + if (packet is RtcpPacket) { + rtcpEventNotifier.notifyRtcpSent(packet) + } + } + + override fun trace(f: () -> Unit) {} + }) node(srtcpEncryptWrapper) node(packetStreamStats.createNewNode()) node(PacketLossNode(packetLossConfig), condition = { packetLossConfig.enabled }) @@ -216,10 +223,6 @@ class RtpSenderImpl( */ override fun doProcessPacket(packetInfo: PacketInfo) { if (running) { - val packet = packetInfo.packet - if (packet is RtcpPacket) { - rtcpEventNotifier.notifyRtcpSent(packet) - } packetInfo.addEvent(PACKET_QUEUE_ENTRY_EVENT) incomingPacketQueue.add(packetInfo) } else { diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/Node.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/Node.kt index 5a9ea245d5..2b5fcdd5db 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/Node.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/Node.kt @@ -393,6 +393,19 @@ abstract class TransformerNode(name: String) : StatsKeepingNode(name) { } } +/** A [TransformerNode] which gets its transformation function dynamically. */ +class PluggableTransformerNode( + name: String, + val transform: () -> ((PacketInfo) -> PacketInfo?)? +) : TransformerNode(name) { + override fun transform(packetInfo: PacketInfo): PacketInfo? { + transform()?.let { return it.invoke(packetInfo) } + return packetInfo + } + override fun trace(f: () -> Unit) {} + override val aggregationKey = this.name +} + /** * Unlike a [TransformerNode], [ModifierNode] modifies a packet in-place and never * outright 'fails', meaning the original [PacketInfo] will *always* be forwarded. From d011ddf71bc8569e9faa3327fb354f81fd566828 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Tue, 5 Nov 2024 13:47:25 -0600 Subject: [PATCH 175/189] fix(#2234): Metrics content type. (#2247) We used to have non-deterministic behavior when no Accept header was included, or it was set to */* (presumably depending on the order in which the jakarta annotations were loaded). We now always default to openmetrics. --- .../rest/prometheus/Prometheus.java | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/prometheus/Prometheus.java b/jvb/src/main/java/org/jitsi/videobridge/rest/prometheus/Prometheus.java index 0e67d409f9..454b06f774 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/rest/prometheus/Prometheus.java +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/prometheus/Prometheus.java @@ -16,12 +16,15 @@ package org.jitsi.videobridge.rest.prometheus; -import io.prometheus.client.exporter.common.*; import jakarta.ws.rs.*; import jakarta.ws.rs.core.*; +import kotlin.*; import org.jetbrains.annotations.NotNull; import org.jitsi.metrics.*; +import java.util.*; +import java.util.stream.*; + /** * A REST endpoint exposing JVB stats for Prometheus. * Any scraper supporting Prometheus' text-based formats ({@code text/plain; version=0.0.4} or OpenMetrics) @@ -35,6 +38,13 @@ @Path("/metrics") public class Prometheus { + static private final Comparator comparator = Comparator.comparing(Prometheus::getQValue).reversed(); + + private static double getQValue(MediaType m) + { + return m.getParameters().get("q") == null ? 1.0 : Double.parseDouble(m.getParameters().get("q")); + } + @NotNull private final MetricsContainer metricsContainer; @@ -44,23 +54,16 @@ public Prometheus(@NotNull MetricsContainer metricsContainer) } @GET - @Produces(TextFormat.CONTENT_TYPE_004) - public String getPrometheusPlainText() - { - return metricsContainer.getPrometheusMetrics(TextFormat.CONTENT_TYPE_004); - } - - @GET - @Produces(TextFormat.CONTENT_TYPE_OPENMETRICS_100) - public String getPrometheusOpenMetrics() + public Response x(@HeaderParam("Accept") String accept) { - return metricsContainer.getPrometheusMetrics(TextFormat.CONTENT_TYPE_OPENMETRICS_100); - } + List acceptMediaTypes + = Arrays.stream(accept.split(",")) + .map(MediaType::valueOf) + .sorted(comparator) + .map(m -> m.getType() + "/" + m.getSubtype()) + .collect(Collectors.toList()); + Pair m = metricsContainer.getMetrics(acceptMediaTypes); - @GET - @Produces(MediaType.APPLICATION_JSON) - public String getJsonString() - { - return metricsContainer.getJsonString(); + return Response.ok(m.getFirst(), m.getSecond()).build(); } } From cc337a2abe033a4a2785ee6ef6f0303c1ca2fc9a Mon Sep 17 00:00:00 2001 From: bgrozev Date: Mon, 11 Nov 2024 09:42:03 -0600 Subject: [PATCH 176/189] feat: Use the cpu-usage measurement for stress. (#2248) --- jvb/src/main/resources/reference.conf | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index 60974d8e68..8f947845a2 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -182,8 +182,11 @@ videobridge { recovery-threshold = 0.72 } - # Which of the available measurements to use, either "packet-rate" or "cpu-usage". - load-measurement = "packet-rate" + # Which of the available measurements to use, either "packet-rate" or "cpu-usage". This determines how the stress + # metric is calculated, which is just the measured load divided by the configured load-threshold. + # The packet-rate metric load-threshold needs to be configured based on the machine that the bridge is running on, + # while the cpu-usage metric should behave well with the default configuration on any machine. + load-measurement = "cpu-usage" } load-reducers { last-n { From fe5577d41d2da9d4a78696bee9e0f46e055fd061 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Wed, 13 Nov 2024 12:08:27 -0600 Subject: [PATCH 177/189] Do not signal mappings for disabled sources. (#2250) This fixes an issue when a receiver joins a conference and one of the senders has a disabled source. The source was signaled with type NONE, which is not handled correctly by receivers. * fix: Do not signal mappings for disabled sources. * feat: Read initial VideoType from colibri. * feat: Log a warning when video type changes between CAMERA and DESKTOP. --- .../videobridge/xmpp/MediaSourceFactory.java | 32 +++++++++++++------ .../org/jitsi/videobridge/AbstractEndpoint.kt | 6 ++++ .../kotlin/org/jitsi/videobridge/Endpoint.kt | 4 ++- .../cc/allocation/BandwidthAllocation.kt | 4 +-- 4 files changed, 34 insertions(+), 12 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java b/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java index 3d9c777131..8f879ccc2b 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java +++ b/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java @@ -15,6 +15,7 @@ */ package org.jitsi.videobridge.xmpp; +import org.jetbrains.annotations.*; import org.jitsi.nlj.*; import org.jitsi.nlj.rtp.*; import org.jitsi.nlj.rtp.codec.vpx.*; @@ -536,12 +537,10 @@ public static MediaSourceDesc[] createMediaSources( { final Collection finalSourceGroups = sourceGroups == null ? new ArrayList<>() : sourceGroups; - if (sources == null) - { - sources = new ArrayList<>(); - } + final Collection finalSources + = sources == null ? new ArrayList<>() : sources; - List sourceSsrcsList = getSourceSsrcs(sources, finalSourceGroups); + List sourceSsrcsList = getSourceSsrcs(finalSources, finalSourceGroups); List mediaSources = new ArrayList<>(); sourceSsrcsList.forEach(sourceSsrcs -> { @@ -567,7 +566,8 @@ public static MediaSourceDesc[] createMediaSources( numTemporalLayersPerStream, secondarySsrcs, sourceSsrcs.owner, - sourceSsrcs.name + sourceSsrcs.name, + getVideoType(finalSources) ); mediaSources.add(mediaSource); }); @@ -624,7 +624,8 @@ public static MediaSourceDesc createMediaSource( numTemporalLayersPerStream, secondarySsrcs, owner, - name + name, + getVideoType(sources) ); } else @@ -779,7 +780,8 @@ private static MediaSourceDesc createSource( int numTemporalLayersPerStream, Map allSecondarySsrcs, String owner, - String name + String name, + VideoType videoType ) { RtpEncodingDesc[] encodings = @@ -817,6 +819,18 @@ private static MediaSourceDesc createSource( throw new IllegalArgumentException("The 'owner' is missing in the source description"); } - return new MediaSourceDesc(encodings, owner, name); + return new MediaSourceDesc(encodings, owner, name, videoType); + } + + private static VideoType getVideoType(@NotNull Collection sources) + { + for (SourcePacketExtension source : sources) + { + if (source.getVideoType() != null) + { + return VideoType.valueOf(source.getVideoType().toUpperCase()); + } + } + return VideoType.CAMERA; } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt index c801c19608..35bd4a54c0 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt @@ -106,6 +106,12 @@ abstract class AbstractEndpoint protected constructor( val mediaSourceDesc = findMediaSourceDesc(sourceName) if (mediaSourceDesc != null) { if (mediaSourceDesc.videoType !== videoType) { + if (mediaSourceDesc.videoType.isEnabled() && videoType.isEnabled()) { + logger.warn( + "Changing video type from ${mediaSourceDesc.videoType} to $videoType for $sourceName. " + + "This will not trigger re-signaling the mapping." + ) + } mediaSourceDesc.videoType = videoType conference.speechActivity.endpointVideoAvailabilityChanged() } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index 1b5ce37861..f904078f9b 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -603,7 +603,9 @@ class Endpoint @JvmOverloads constructor( if (doSsrcRewriting) { val newActiveSources = - newEffectiveConstraints.entries.filter { !it.value.isDisabled() }.map { it.key }.toList() + newEffectiveConstraints.entries.filter { + !it.value.isDisabled() && it.key.videoType.isEnabled() + }.map { it.key }.toList() val newActiveSourceNames = newActiveSources.map { it.sourceName }.toSet() /* safe unlocked access of activeSources. BitrateController will not overlap calls to this method. */ if (activeSources != newActiveSourceNames) { diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt index db791bc68e..9644bdb38c 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt @@ -45,8 +45,8 @@ class BandwidthAllocation @JvmOverloads constructor( allocations.all { allocation -> other.allocations.any { otherAllocation -> allocation.endpointId == otherAllocation.endpointId && - allocation.mediaSource?.primarySSRC == - otherAllocation.mediaSource?.primarySSRC && + allocation.mediaSource?.primarySSRC == otherAllocation.mediaSource?.primarySSRC && + allocation.mediaSource?.videoType == otherAllocation.mediaSource?.videoType && allocation.targetLayer?.index == otherAllocation.targetLayer?.index } } From 94fb80515e1a388e12521057fb3ce6d4f84ebb23 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Wed, 13 Nov 2024 15:05:29 -0600 Subject: [PATCH 178/189] Minor cleanup, removing unused code (#2251) * fix: Improve logging. * ref: Remove unused code. * ref: Remove unnecessary functions. * ref: Remove unused config option. * ref: Remove unused code. * ref: Single line comments. --- .../videobridge/xmpp/MediaSourceFactory.java | 12 +------ .../cc/allocation/AllocationSettings.kt | 2 +- .../cc/allocation/BandwidthAllocation.kt | 5 ++- .../cc/allocation/BandwidthAllocator.kt | 32 ++++++------------ .../cc/allocation/BitrateController.kt | 2 +- .../cc/allocation/SingleSourceAllocation.kt | 6 ++-- .../cc/config/BitrateControllerConfig.kt | 33 +++++-------------- jvb/src/main/resources/reference.conf | 1 - .../cc/allocation/BitrateControllerTest.kt | 2 +- .../cc/allocation/EffectiveConstraintsTest.kt | 2 +- 10 files changed, 28 insertions(+), 69 deletions(-) diff --git a/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java b/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java index 8f879ccc2b..3c99d27bc2 100644 --- a/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java +++ b/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java @@ -24,7 +24,6 @@ import org.jitsi.xmpp.extensions.jingle.*; import org.jitsi.xmpp.extensions.jitsimeet.*; import org.jitsi.xmpp.util.*; -import org.jivesoftware.smack.packet.*; import org.jxmpp.jid.parts.*; import org.jxmpp.util.*; @@ -43,16 +42,7 @@ public class MediaSourceFactory * The {@link Logger} used by the {@link MediaSourceDesc} class and its * instances for logging output. */ - private static final Logger logger - = new LoggerImpl(MediaSourceFactory.class.getName()); - - /** - * The default number of temporal layers to use for VP8 simulcast. - * - * FIXME: hardcoded ugh.. this should be either signaled or somehow included - * in the RTP stream. - */ - private static final int VP8_SIMULCAST_TEMPORAL_LAYERS = 3; + private static final Logger logger = new LoggerImpl(MediaSourceFactory.class.getName()); /** * The resolution of the base stream when activating simulcast for VP8. diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt index c9a1762e65..749ff8d213 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt @@ -80,7 +80,7 @@ internal class AllocationSettingsWrapper( private var videoConstraints: Map = emptyMap() - private var defaultConstraints: VideoConstraints = VideoConstraints(config.thumbnailMaxHeightPx()) + private var defaultConstraints: VideoConstraints = VideoConstraints(config.thumbnailMaxHeightPx) private var assumedBandwidthBps: Long = -1 diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt index 9644bdb38c..aa362ccc8f 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt @@ -31,8 +31,6 @@ class BandwidthAllocation @JvmOverloads constructor( private val suspendedSources: List = emptyList() ) { val hasSuspendedSources: Boolean = suspendedSources.isNotEmpty() - val forwardedEndpoints: Set = - allocations.filter { it.isForwarded() }.map { it.endpointId }.toSet() val forwardedSources: Set = allocations.filter { it.isForwarded() }.mapNotNull { it.mediaSource?.sourceName }.toSet() @@ -98,7 +96,8 @@ data class SingleAllocation( get() = targetLayer?.index ?: -1 fun isForwarded(): Boolean = targetIndex > -1 - override fun toString(): String = "[id=$endpointId target=${targetLayer?.height}/${targetLayer?.frameRate} " + + override fun toString(): String = "[epId=$endpointId sourceName=${mediaSource?.sourceName} " + + "target=${targetLayer?.height}/${targetLayer?.frameRate} " + "(${targetLayer?.indexString()}) " + "ideal=${idealLayer?.height}/${idealLayer?.frameRate} (${idealLayer?.indexString()})]" diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt index b26e141600..6412c50d89 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt @@ -52,14 +52,10 @@ internal class BandwidthAllocator( ) { private val logger = createChildLogger(parentLogger) - /** - * The estimated available bandwidth in bits per second. - */ + /** The estimated available bandwidth in bits per second. */ private var bweBps: Long = -1 - /** - * Whether this bandwidth estimator has been expired. Once expired we stop periodic re-allocation. - */ + /** Whether this bandwidth estimator has been expired. Once expired we stop periodic re-allocation. */ private var expired = false /** @@ -86,7 +82,7 @@ internal class BandwidthAllocator( * The allocations settings signalled by the receiver. */ private var allocationSettings = - AllocationSettings(defaultConstraints = VideoConstraints(BitrateControllerConfig.config.thumbnailMaxHeightPx())) + AllocationSettings(defaultConstraints = VideoConstraints(BitrateControllerConfig.config.thumbnailMaxHeightPx)) /** * The last time [BandwidthAllocator.update] was called. @@ -95,24 +91,18 @@ internal class BandwidthAllocator( */ private var lastUpdateTime: Instant = clock.instant() - /** - * The result of the bitrate control algorithm, the last time it ran. - */ + /** The result of the bitrate control algorithm, the last time it ran. */ var allocation = BandwidthAllocation(emptySet()) private set - /** - * The task scheduled to call [.update]. - */ + /** The task scheduled to call [.update]. */ private var updateTask: ScheduledFuture<*>? = null init { rescheduleUpdate() } - /** - * Gets a JSON representation of the parts of this object's state that are deemed useful for debugging. - */ + /** Gets a JSON representation of the parts of this object's state that are deemed useful for debugging. */ @get:SuppressFBWarnings( value = ["IS2_INCONSISTENT_SYNC"], justification = "We intentionally avoid synchronizing while reading fields only used in debug output." @@ -128,9 +118,7 @@ internal class BandwidthAllocator( return debugState } - /** - * Get the available bandwidth, taking into account the `trustBwe` option. - */ + /** Get the available bandwidth, taking into account the `trustBwe` option. */ private val availableBandwidth: Long get() = if (trustBwe.get()) bweBps else Long.MAX_VALUE @@ -186,7 +174,7 @@ internal class BandwidthAllocator( logger.trace { "Allocating: sortedSources=${sortedSources.map { it.sourceName }}, " + - " effectiveConstraints=$newEffectiveConstraints" + " effectiveConstraints=${newEffectiveConstraints.map { "${it.key.sourceName}=${it.value}" }}" } // Compute the bandwidth allocation. @@ -336,7 +324,7 @@ internal class BandwidthAllocator( return } val timeSinceLastUpdate = Duration.between(lastUpdateTime, clock.instant()) - val period = BitrateControllerConfig.config.maxTimeBetweenCalculations() + val period = BitrateControllerConfig.config.maxTimeBetweenCalculations val delayMs = if (timeSinceLastUpdate > period) { logger.debug("Running periodic re-allocation.") TaskPools.CPU_POOL.execute { this.update() } @@ -384,7 +372,7 @@ private fun bweChangeIsLargerThanThreshold(previousBwe: Long, currentBwe: Long): // In any case, there are other triggers for re-allocation, so any suppression we do here will only last up to // a few seconds. val deltaBwe = abs(currentBwe - previousBwe) - return deltaBwe > previousBwe * BitrateControllerConfig.config.bweChangeThreshold() + return deltaBwe > previousBwe * BitrateControllerConfig.config.bweChangeThreshold // If, on the other hand, the bwe has decreased, we require at least a 15% drop in order to update the bitrate // allocation. This is an ugly hack to prevent too many resolution/UI changes in case the bridge produces too diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt index 48b2ba21ac..bed8c99437 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt @@ -105,7 +105,7 @@ class BitrateController @JvmOverloads constructor( * TODO: Is this comment still accurate? */ private val trustBwe: Boolean - get() = config.trustBwe() && supportsRtx && packetHandler.timeSinceFirstMedia() >= 10.secs + get() = config.trustBwe && supportsRtx && packetHandler.timeSinceFirstMedia() >= 10.secs // Proxy to the allocator fun endpointOrderingChanged() = bandwidthAllocator.update() diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt index 82d139f860..d69a22f465 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt @@ -123,7 +123,7 @@ internal class SingleSourceAllocation( // `maxOversendBitrate`. if (allowOversending && layers.oversendIndex >= 0 && targetIdx < layers.oversendIndex) { for (i in layers.oversendIndex downTo targetIdx + 1) { - if (layers[i].bitrate <= maxBps + config.maxOversendBitrateBps()) { + if (layers[i].bitrate <= maxBps + config.maxOversendBitrate.bps) { targetIdx = i } } @@ -236,7 +236,7 @@ internal class SingleSourceAllocation( } } - val oversendIdx = if (onStage && config.allowOversendOnStage()) { + val oversendIdx = if (onStage && config.allowOversendOnStage) { val maxHeight = selectedLayers.maxOfOrNull { it.layer.height } ?: return Layers.noLayers // Of all layers with the highest resolution select the one with lowest bitrate. In case of VP9 the layers // are not necessarily ordered by bitrate. @@ -339,7 +339,7 @@ private fun List.lastIndexWhich(predicate: (T) -> Boolean): Int { */ private fun getPreferred(constraints: VideoConstraints): VideoConstraints { return if (constraints.maxHeight > 180 || !constraints.heightIsLimited()) { - VideoConstraints(config.onstagePreferredHeightPx(), config.onstagePreferredFramerate()) + VideoConstraints(config.onstagePreferredHeightPx, config.onstagePreferredFramerate) } else { VideoConstraints.UNLIMITED } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt index 6f25167015..10772e7316 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt @@ -30,7 +30,7 @@ class BitrateControllerConfig private constructor() { * In order to limit the resolution changes due to bandwidth changes we only react to bandwidth changes greater * than {@code bweChangeThreshold * last_bandwidth_estimation}. */ - private val bweChangeThreshold: Double by config { + val bweChangeThreshold: Double by config { "org.jitsi.videobridge.BWE_CHANGE_THRESHOLD_PCT".from(JitsiConfig.legacyConfig) .transformedBy { it / 100.0 } // This is an old version, include for backward compat. @@ -38,84 +38,67 @@ class BitrateControllerConfig private constructor() { .transformedBy { it / 100.0 } "videobridge.cc.bwe-change-threshold".from(JitsiConfig.newConfig) } - fun bweChangeThreshold() = bweChangeThreshold /** * The max resolution to allocate for the thumbnails. */ - private val thumbnailMaxHeightPx: Int by config { + val thumbnailMaxHeightPx: Int by config { "org.jitsi.videobridge.THUMBNAIL_MAX_HEIGHT".from(JitsiConfig.legacyConfig) "videobridge.cc.thumbnail-max-height-px".from(JitsiConfig.newConfig) } - fun thumbnailMaxHeightPx() = thumbnailMaxHeightPx /** * The default preferred resolution to allocate for the onstage participant, * before allocating bandwidth for the thumbnails. */ - private val onstagePreferredHeightPx: Int by config { + val onstagePreferredHeightPx: Int by config { "org.jitsi.videobridge.ONSTAGE_PREFERRED_HEIGHT".from(JitsiConfig.legacyConfig) "videobridge.cc.onstage-preferred-height-px".from(JitsiConfig.newConfig) } - fun onstagePreferredHeightPx() = onstagePreferredHeightPx /** * The preferred frame rate to allocate for the onstage participant. */ - private val onstagePreferredFramerate: Double by config { + val onstagePreferredFramerate: Double by config { "org.jitsi.videobridge.ONSTAGE_PREFERRED_FRAME_RATE".from(JitsiConfig.legacyConfig) "videobridge.cc.onstage-preferred-framerate".from(JitsiConfig.newConfig) } - fun onstagePreferredFramerate() = onstagePreferredFramerate /** * Whether or not we are allowed to oversend (exceed available bandwidth) for the video of the on-stage * participant. */ - private val allowOversendOnStage: Boolean by config { + val allowOversendOnStage: Boolean by config { "org.jitsi.videobridge.ENABLE_ONSTAGE_VIDEO_SUSPEND".from(JitsiConfig.legacyConfig).transformedBy { !it } "videobridge.cc.enable-onstage-video-suspend".from(JitsiConfig.newConfig).transformedBy { !it } "videobridge.cc.allow-oversend-onstage".from(JitsiConfig.newConfig) } - fun allowOversendOnStage(): Boolean = allowOversendOnStage /** * The maximum bitrate by which the bridge may exceed the estimated available bandwidth when oversending. */ - private val maxOversendBitrate: Bandwidth by config { + val maxOversendBitrate: Bandwidth by config { "videobridge.cc.max-oversend-bitrate".from(JitsiConfig.newConfig) .convertFrom { Bandwidth.fromString(it) } } - fun maxOversendBitrateBps(): Double = maxOversendBitrate.bps /** * Whether or not we should trust the bandwidth * estimations. If this is se to false, then we assume a bandwidth * estimation of Long.MAX_VALUE. */ - private val trustBwe: Boolean by config { + val trustBwe: Boolean by config { "org.jitsi.videobridge.TRUST_BWE".from(JitsiConfig.legacyConfig) "videobridge.cc.trust-bwe".from(JitsiConfig.newConfig) } - fun trustBwe(): Boolean = trustBwe - - /** - * The property for the max resolution to allocate for the onstage - * participant. - */ - private val onstageIdealHeightPx: Int by config( - "videobridge.cc.onstage-ideal-height-px".from(JitsiConfig.newConfig) - ) - fun onstageIdealHeightPx() = onstageIdealHeightPx /** * The maximum amount of time we'll run before recalculating which streams we'll * forward. */ - private val maxTimeBetweenCalculations: Duration by config( + val maxTimeBetweenCalculations: Duration by config( "videobridge.cc.max-time-between-calculations".from(JitsiConfig.newConfig) ) - fun maxTimeBetweenCalculations() = maxTimeBetweenCalculations val assumedBandwidthLimit: Bandwidth? by optionalconfig { "videobridge.cc.assumed-bandwidth-limit".from(JitsiConfig.newConfig) diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index 8f947845a2..1f80492af8 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -45,7 +45,6 @@ videobridge { cc { bwe-change-threshold = 0.15 thumbnail-max-height-px = 180 - onstage-ideal-height-px = 1080 onstage-preferred-height-px = 360 onstage-preferred-framerate = 30 // Whether the bridge is allowed to oversend (send the lowest layer regardless of BWE) for on-stage endpoints. If diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt index a3f0f13156..ff27df055c 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt @@ -96,7 +96,7 @@ class BitrateControllerTest : ShouldSpec() { val delayMs = TimeUnit.MILLISECONDS.convert(captureDelay.captured, captureDelayTimeunit.captured) delayMs.shouldBeWithinPercentageOf( - BitrateControllerConfig.config.maxTimeBetweenCalculations().toMillis(), + BitrateControllerConfig.config.maxTimeBetweenCalculations.toMillis(), 10.0 ) diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/EffectiveConstraintsTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/EffectiveConstraintsTest.kt index 0e572c3dfd..25a947ca5b 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/EffectiveConstraintsTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/EffectiveConstraintsTest.kt @@ -44,7 +44,7 @@ class EffectiveConstraintsTest : ShouldSpec() { val s5 = testSource("e1", "s5", videoType = VideoType.DISABLED) val s6 = testSource("e1", "s6", videoType = VideoType.DISABLED) - val defaultConstraints = VideoConstraints(BitrateControllerConfig.config.thumbnailMaxHeightPx()) + val defaultConstraints = VideoConstraints(BitrateControllerConfig.config.thumbnailMaxHeightPx) val sources = listOf(s1, s2, s3, s4, s5, s6) val zeroEffectiveConstraints = mutableMapOf( From 847cb3e14a5c2699708eb488f22aeb0204852df4 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 14 Nov 2024 10:09:08 -0600 Subject: [PATCH 179/189] feat: Split config for default constraints. (#2252) * feat: Split config for default constraints. * squash: Use the correct property. --- .../cc/allocation/AllocationSettings.kt | 2 +- .../cc/allocation/BandwidthAllocator.kt | 8 ++++---- .../cc/config/BitrateControllerConfig.kt | 15 ++++++++++----- jvb/src/main/resources/reference.conf | 6 +++++- .../cc/allocation/EffectiveConstraintsTest.kt | 3 +-- 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt index 749ff8d213..59940fba43 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt @@ -80,7 +80,7 @@ internal class AllocationSettingsWrapper( private var videoConstraints: Map = emptyMap() - private var defaultConstraints: VideoConstraints = VideoConstraints(config.thumbnailMaxHeightPx) + private var defaultConstraints: VideoConstraints = VideoConstraints(config.defaultMaxHeightPx) private var assumedBandwidthBps: Long = -1 diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt index 6412c50d89..d3732502f6 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt @@ -78,11 +78,11 @@ internal class BandwidthAllocator( addHandler(eventHandler) } - /** - * The allocations settings signalled by the receiver. - */ + /** The allocations settings signalled by the receiver. */ private var allocationSettings = - AllocationSettings(defaultConstraints = VideoConstraints(BitrateControllerConfig.config.thumbnailMaxHeightPx)) + AllocationSettings( + defaultConstraints = VideoConstraints(BitrateControllerConfig.config.initialMaxHeightPx) + ) /** * The last time [BandwidthAllocator.update] was called. diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt index 10772e7316..5c44519e43 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt @@ -39,12 +39,17 @@ class BitrateControllerConfig private constructor() { "videobridge.cc.bwe-change-threshold".from(JitsiConfig.newConfig) } - /** - * The max resolution to allocate for the thumbnails. - */ - val thumbnailMaxHeightPx: Int by config { - "org.jitsi.videobridge.THUMBNAIL_MAX_HEIGHT".from(JitsiConfig.legacyConfig) + val initialMaxHeightPx: Int by config { + // Support the old property name if the user has overridden it. + "videobridge.cc.thumbnail-max-height-px".from(JitsiConfig.newConfig) + .softDeprecated("use videobridge.cc.initial-max-height-px") + "videobridge.cc.initial-max-height-px".from(JitsiConfig.newConfig) + } + val defaultMaxHeightPx: Int by config { + // Support the old property name if the user has overridden it. "videobridge.cc.thumbnail-max-height-px".from(JitsiConfig.newConfig) + .softDeprecated("use videobridge.cc.default-max-height-px") + "videobridge.cc.default-max-height-px".from(JitsiConfig.newConfig) } /** diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index 1f80492af8..7eaf35696c 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -42,9 +42,13 @@ videobridge { # How often we check endpoint's connectivity status check-interval = 500 milliseconds } + # Congestion Control cc { bwe-change-threshold = 0.15 - thumbnail-max-height-px = 180 + # The default constraint to use if the receiver signals ReceiverConstraints with missing defaultConstraints. + default-max-height-px = 180 + # The constraint to use for the initial allocation, before the receiver has signaled ReceiverConstraints. + initial-max-height-px = 180 onstage-preferred-height-px = 360 onstage-preferred-framerate = 30 // Whether the bridge is allowed to oversend (send the lowest layer regardless of BWE) for on-stage endpoints. If diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/EffectiveConstraintsTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/EffectiveConstraintsTest.kt index 25a947ca5b..9601a205c3 100644 --- a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/EffectiveConstraintsTest.kt +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/EffectiveConstraintsTest.kt @@ -21,7 +21,6 @@ import io.kotest.core.spec.style.ShouldSpec import io.kotest.matchers.shouldBe import org.jitsi.nlj.MediaSourceDesc import org.jitsi.nlj.VideoType -import org.jitsi.videobridge.cc.config.BitrateControllerConfig fun testSource(endpointId: String, sourceName: String, videoType: VideoType = VideoType.CAMERA): MediaSourceDesc { return MediaSourceDesc( @@ -44,7 +43,7 @@ class EffectiveConstraintsTest : ShouldSpec() { val s5 = testSource("e1", "s5", videoType = VideoType.DISABLED) val s6 = testSource("e1", "s6", videoType = VideoType.DISABLED) - val defaultConstraints = VideoConstraints(BitrateControllerConfig.config.thumbnailMaxHeightPx) + val defaultConstraints = VideoConstraints(180) val sources = listOf(s1, s2, s3, s4, s5, s6) val zeroEffectiveConstraints = mutableMapOf( From a05e6818b3155d97e92bb6a04eb6bfc9a8d6c881 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 26 Nov 2024 11:13:35 -0500 Subject: [PATCH 180/189] Update fingerprint handling code for RFC 8122 conformance. (#2254) --- .../kotlin/org/jitsi/nlj/dtls/DtlsConfig.kt | 28 +++++++ .../kotlin/org/jitsi/nlj/dtls/DtlsStack.kt | 2 +- .../kotlin/org/jitsi/nlj/dtls/DtlsUtils.kt | 83 ++++++------------- .../src/main/resources/reference.conf | 5 ++ .../org/jitsi/nlj/dtls/DtlsConfigTest.kt | 54 ++++++++++++ .../kotlin/org/jitsi/nlj/dtls/DtlsTest.kt | 4 +- .../kotlin/org/jitsi/videobridge/Endpoint.kt | 5 +- .../org/jitsi/videobridge/relay/Relay.kt | 5 +- .../transport/dtls/DtlsTransport.kt | 2 +- 9 files changed, 122 insertions(+), 66 deletions(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsConfig.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsConfig.kt index 6764aa94f8..40f689da7e 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsConfig.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsConfig.kt @@ -15,6 +15,7 @@ */ package org.jitsi.nlj.dtls +import org.bouncycastle.operator.DefaultDigestAlgorithmIdentifierFinder import org.bouncycastle.tls.CipherSuite import org.jitsi.config.JitsiConfig import org.jitsi.metaconfig.ConfigException @@ -36,11 +37,38 @@ class DtlsConfig { } } + val localFingerprintHashFunction: String by config { + "jmt.dtls.local-fingerprint-hash-function".from(JitsiConfig.newConfig).transformedBy { + validateHashFunction(it) + } + } + + val acceptedFingerprintHashFunctions: List by config { + "jmt.dtls.accepted-fingerprint-hash-functions".from(JitsiConfig.newConfig).convertFrom> { list -> + if (list.isEmpty()) { + throw ConfigException.UnableToRetrieve.ConditionNotMet( + "accepted-fingerprint-hash-functions must not be empty" + ) + } + list.map { validateHashFunction(it) } + } + } + companion object { val config = DtlsConfig() } } +private fun validateHashFunction(func: String): String { + val ucFunc = func.uppercase() + DefaultDigestAlgorithmIdentifierFinder().find(ucFunc) + ?: throw ConfigException.UnableToRetrieve.WrongType("Unknown hash function $func") + if (ucFunc == "MD5" || ucFunc == "MD2") { + throw ConfigException.UnableToRetrieve.WrongType("Forbidden hash function $func") + } + return func.lowercase() +} + private fun String.toBcCipherSuite(): Int = try { CipherSuite::class.java.getDeclaredField(this).getInt(null) } catch (e: Exception) { diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsStack.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsStack.kt index 4d7c32ba5c..14f0b13a82 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsStack.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsStack.kt @@ -71,7 +71,7 @@ class DtlsStack( /** * The remote fingerprints sent to us over the signaling path. */ - var remoteFingerprints: Map = HashMap() + var remoteFingerprints: Map> = mapOf() /** * A handler which will be invoked when DTLS application data is received diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsUtils.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsUtils.kt index c1259c3ba1..6915fa3d04 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsUtils.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsUtils.kt @@ -65,7 +65,7 @@ class DtlsUtils { val cn = generateCN("TODO-APP-NAME", "TODO-APP-VERSION") val keyPair = generateEcKeyPair() val x509certificate = generateCertificate(cn, keyPair) - val localFingerprintHashFunction = x509certificate.getHashFunction() + val localFingerprintHashFunction = config.localFingerprintHashFunction val localFingerprint = x509certificate.getFingerprint(localFingerprintHashFunction) val certificate = org.bouncycastle.tls.Certificate( @@ -158,7 +158,7 @@ class DtlsUtils { */ fun verifyAndValidateCertificate( certificateInfo: org.bouncycastle.tls.Certificate, - remoteFingerprints: Map + remoteFingerprints: Map> ) { if (certificateInfo.certificateList.isEmpty()) { throw DtlsException("No remote fingerprints.") @@ -179,66 +179,33 @@ class DtlsUtils { * and validate against the fingerprints presented by the remote endpoint * via the signaling path. */ - private fun verifyAndValidateCertificate(certificate: Certificate, remoteFingerprints: Map) { - // RFC 4572 "Connection-Oriented Media Transport over the Transport - // Layer Security (TLS) Protocol in the Session Description Protocol - // (SDP)" defines that "[a] certificate fingerprint MUST be computed - // using the same one-way hash function as is used in the certificate's - // signature algorithm." - - val hashFunction = certificate.getHashFunction() - - // As RFC 5763 "Framework for Establishing a Secure Real-time Transport - // Protocol (SRTP) Security Context Using Datagram Transport Layer - // Security (DTLS)" states, "the certificate presented during the DTLS - // handshake MUST match the fingerprint exchanged via the signaling path - // in the SDP." - val remoteFingerprint = remoteFingerprints[hashFunction] ?: throw DtlsException( - "No fingerprint declared over the signaling path with hash function: $hashFunction" - ) - - // TODO(boris) check if the below is still true, and re-introduce the hack if it is. - // Unfortunately, Firefox does not comply with RFC 5763 at the time - // of this writing. Its certificate uses SHA-1 and it sends a - // fingerprint computed with SHA-256. We could, of course, wait for - // Mozilla to make Firefox compliant. However, we would like to - // support Firefox in the meantime. That is why we will allow the - // fingerprint to "upgrade" the hash function of the certificate - // much like SHA-256 is an "upgrade" of SHA-1. - /* - if (remoteFingerprint == null) - { - val hashFunctionUpgrade = findHashFunctionUpgrade(hashFunction, remoteFingerprints) - - if (hashFunctionUpgrade != null - && !hashFunctionUpgrade.equalsIgnoreCase(hashFunction)) { - fingerprint = fingerprints[hashFunctionUpgrade] - if (fingerprint != null) - hashFunction = hashFunctionUpgrade - } - } + private fun verifyAndValidateCertificate( + certificate: Certificate, + remoteFingerprints: Map> + ) { + /** RFC 8122: + * An endpoint MUST select the set of fingerprints that use its most + * preferred hash function (out of those offered by the peer) and verify + * that each certificate used matches one fingerprint out of that set. */ + config.acceptedFingerprintHashFunctions.forEach { hashFunction -> + val fingerprints = remoteFingerprints[hashFunction] ?: return@forEach - val certificateFingerprint = certificate.getFingerprint(hashFunction) + val certificateFingerprint = certificate.getFingerprint(hashFunction) - if (remoteFingerprint != certificateFingerprint) { - throw DtlsException( - "Fingerprint $remoteFingerprint does not match the $hashFunction-hashed " + - "certificate $certificateFingerprint" - ) + if (fingerprints.none { it == certificateFingerprint }) { + throw DtlsException( + "None of the fingerprints ${fingerprints.joinToString()} match the $hashFunction-hashed " + + "certificate $certificateFingerprint" + ) + } + return@verifyAndValidateCertificate } - } - - /** - * Determine and return the hash function (as a [String]) used by this certificate - */ - private fun Certificate.getHashFunction(): String { - val digAlgId = DefaultDigestAlgorithmIdentifierFinder().find(signatureAlgorithm) - - return BcDefaultDigestProvider.INSTANCE - .get(digAlgId) - .algorithmName - .lowercase() + /* If we got here none of our accepted fingerprint functions were listed. */ + throw DtlsException( + "No fingerprint declared over the signaling path with any of the accepted hash functions: " + + config.acceptedFingerprintHashFunctions.joinToString() + ) } /** diff --git a/jitsi-media-transform/src/main/resources/reference.conf b/jitsi-media-transform/src/main/resources/reference.conf index 52ee7c5e21..9fc672b7a0 100644 --- a/jitsi-media-transform/src/main/resources/reference.conf +++ b/jitsi-media-transform/src/main/resources/reference.conf @@ -43,6 +43,11 @@ jmt { // TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, // TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 ] + // The hash function to use to generate certificate fingerprints + local-fingerprint-hash-function = sha-256 + // The hash functions that are accepted for remote certificate fingerprints, in decreasing strength order + accepted-fingerprint-hash-functions = [ sha-512, sha-384, sha-256, sha-1 ] + } srtp { // The maximum number of packets that can be discarded early (without going through the SRTP stack for diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsConfigTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsConfigTest.kt index 0029ffd101..b0b9ad900b 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsConfigTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsConfigTest.kt @@ -17,6 +17,7 @@ package org.jitsi.nlj.dtls import io.kotest.assertions.throwables.shouldThrow import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.collections.shouldContainExactly import io.kotest.matchers.shouldBe import org.bouncycastle.tls.CipherSuite import org.jitsi.config.withNewConfig @@ -64,5 +65,58 @@ class DtlsConfigTest : ShouldSpec() { } } } + context("Valid fingerprint hash functions") { + withNewConfig( + """ + jmt.dtls.local-fingerprint-hash-function = sha-512 + jmt.dtls.accepted-fingerprint-hash-functions = [ sha-512, sha-384, sha-256 ] + """.trimIndent() + ) { + DtlsConfig.config.localFingerprintHashFunction shouldBe "sha-512" + DtlsConfig.config.acceptedFingerprintHashFunctions shouldContainExactly + setOf("sha-512", "sha-384", "sha-256") + } + context("With inconsistent capitalization") { + withNewConfig( + """ + jmt.dtls.local-fingerprint-hash-function = SHA-512 + jmt.dtls.accepted-fingerprint-hash-functions = [ Sha-512, sHa-384, shA-256 ] + """.trimIndent() + ) { + DtlsConfig.config.localFingerprintHashFunction shouldBe "sha-512" + DtlsConfig.config.acceptedFingerprintHashFunctions shouldContainExactly + setOf("sha-512", "sha-384", "sha-256") + } + } + } + context("Invalid local fingerprint hash function") { + context("Invalid name") { + withNewConfig("jmt.dtls.local-fingerprint-hash-function = sha-257") { + shouldThrow { DtlsConfig.config.localFingerprintHashFunction } + } + } + context("Forbidden function") { + withNewConfig("jmt.dtls.local-fingerprint-hash-function = md5") { + shouldThrow { DtlsConfig.config.localFingerprintHashFunction } + } + } + } + context("Invalid accepted accepted fingerprint hash functions") { + context("Invalid entry") { + withNewConfig("jmt.dtls.accepted-fingerprint-hash-functions = [ sha-256, sha-257 ]") { + shouldThrow { DtlsConfig.config.acceptedFingerprintHashFunctions } + } + } + context("Empty") { + withNewConfig("jmt.dtls.accepted-fingerprint-hash-functions = []") { + shouldThrow { DtlsConfig.config.acceptedFingerprintHashFunctions } + } + } + context("Forbidden function") { + withNewConfig("jmt.dtls.accepted-fingerprint-hash-functions = [ sha-1, md5 ]") { + shouldThrow { DtlsConfig.config.acceptedFingerprintHashFunctions } + } + } + } } } diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsTest.kt index 2b0d9db85e..82bc4601fa 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsTest.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsTest.kt @@ -48,10 +48,10 @@ class DtlsTest : ShouldSpec() { val pcapWriter = if (pcapEnabled) PcapWriter(logger, "/tmp/dtls-test.pcap") else null dtlsClient.remoteFingerprints = mapOf( - dtlsServer.localFingerprintHashFunction to dtlsServer.localFingerprint + dtlsServer.localFingerprintHashFunction to listOf(dtlsServer.localFingerprint) ) dtlsServer.remoteFingerprints = mapOf( - dtlsClient.localFingerprintHashFunction to dtlsClient.localFingerprint + dtlsClient.localFingerprintHashFunction to listOf(dtlsClient.localFingerprint) ) // The DTLS server's send is wired directly to the DTLS client's receive diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt index f904078f9b..99146041c5 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -788,11 +788,12 @@ class Endpoint @JvmOverloads constructor( * transport information. */ fun setTransportInfo(transportInfo: IceUdpTransportPacketExtension) { - val remoteFingerprints = mutableMapOf() + val remoteFingerprints = mutableMapOf>() val fingerprintExtensions = transportInfo.getChildExtensionsOfType(DtlsFingerprintPacketExtension::class.java) fingerprintExtensions.forEach { fingerprintExtension -> if (fingerprintExtension.hash != null && fingerprintExtension.fingerprint != null) { - remoteFingerprints[fingerprintExtension.hash] = fingerprintExtension.fingerprint + remoteFingerprints.getOrPut(fingerprintExtension.hash.lowercase()) { mutableListOf() } + .add(fingerprintExtension.fingerprint) } else { logger.info("Ignoring empty DtlsFingerprint extension: ${transportInfo.toStringOpt()}") } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt index cbb6e70e1c..6d0beff72e 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt @@ -612,11 +612,12 @@ class Relay @JvmOverloads constructor( * transport information. */ fun setTransportInfo(transportInfo: IceUdpTransportPacketExtension) { - val remoteFingerprints = mutableMapOf() + val remoteFingerprints = mutableMapOf>() val fingerprintExtensions = transportInfo.getChildExtensionsOfType(DtlsFingerprintPacketExtension::class.java) fingerprintExtensions.forEach { fingerprintExtension -> if (fingerprintExtension.hash != null && fingerprintExtension.fingerprint != null) { - remoteFingerprints[fingerprintExtension.hash] = fingerprintExtension.fingerprint + remoteFingerprints.getOrPut(fingerprintExtension.hash.lowercase()) { mutableListOf() } + .add(fingerprintExtension.fingerprint) } else { logger.info("Ignoring empty DtlsFingerprint extension: ${transportInfo.toStringOpt()}") } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/dtls/DtlsTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/dtls/DtlsTransport.kt index e6db0a8293..ae59765f08 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/dtls/DtlsTransport.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/dtls/DtlsTransport.kt @@ -163,7 +163,7 @@ class DtlsTransport(parentLogger: Logger, id: String) { } } - fun setRemoteFingerprints(remoteFingerprints: Map) { + fun setRemoteFingerprints(remoteFingerprints: Map>) { // Don't pass an empty list to the stack in order to avoid wiping // certificates that were contained in a previous request. if (remoteFingerprints.isEmpty()) { From e85304b564ae8495c817e93f5f7158bf5181f535 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Mon, 2 Dec 2024 17:15:54 -0500 Subject: [PATCH 181/189] Add Java 21 to GitHub testing matrix. (#2255) --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index c635edeb0c..d0c374dc5a 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -20,7 +20,7 @@ jobs: strategy: matrix: - java: [ 11, 17 ] + java: [ 11, 17, 21 ] name: Java ${{ matrix.java }} From 8b04b03e0ba832bd6f143b253964c9127b4354a9 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Tue, 3 Dec 2024 10:35:54 -0500 Subject: [PATCH 182/189] Fix block comment syntax in analyze-timeline2.pl. (#2256) --- resources/analyze-timeline2.pl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/resources/analyze-timeline2.pl b/resources/analyze-timeline2.pl index 9b4987b306..52d1fe1857 100755 --- a/resources/analyze-timeline2.pl +++ b/resources/analyze-timeline2.pl @@ -6,7 +6,8 @@ my %stats; my @stat_names; -/* +=begin comment + This is extremely ugly, I'm driving perl in first gear because I don't know how to shift :) It parses the PacketInfo EventTimeline logs (see example below) and produces stats for @@ -46,7 +47,9 @@ Example log line: JVB 2024-06-11 17:57:44.989 FINER: [1531] [confId=6149c0c339432f97 conf_name=loadtest0@conference.xxx meeting_id=ec7261c3 epId=c88aca35 stats_id=Otto-emV] Endpoint.doSendSrtp#544: Reference time: 2024-06-11T17:57:44.988517466Z; (Entered RTP receiver incoming queue, PT0.00000192S); (Exited RTP receiver incoming queue, PT0.000324242S); (Entered node PacketStreamStats, PT0.000324602S); (Exited node PacketStreamStats, PT0.000325042S); (Entered node SRTP/SRTCP demuxer, PT0.000325162S); (Exited node SRTP/SRTCP demuxer, PT0.000325722S); (Entered node RTP Parser, PT0.000325962S); (Exited node RTP Parser, PT0.000326762S); (Entered node Audio level reader (pre-srtp), PT0.000326922S); (Exited node Audio level reader (pre-srtp), PT0.000327242S); (Entered node Video mute node, PT0.000327362S); (Exited node Video mute node, PT0.000327642S); (Entered node SRTP Decrypt Node, PT0.000327762S); (Exited node SRTP Decrypt Node, PT0.000332442S); (Entered node TCC generator, PT0.000332562S); (Exited node TCC generator, PT0.000333562S); (Entered node Remote Bandwidth Estimator, PT0.000344962S); (Exited node Remote Bandwidth Estimator, PT0.000345202S); (Entered node Audio level reader (post-srtp), PT0.000345322S); (Exited node Audio level reader (post-srtp), PT0.000345442S); (Entered node Toggleable pcap writer: 21913831-rx, PT0.000345562S); (Exited node Toggleable pcap writer: 21913831-rx, PT0.000345762S); (Entered node Incoming statistics tracker, PT0.000345882S); (Exited node Incoming statistics tracker, PT0.000351242S); (Entered node Padding termination, PT0.000351362S); (Exited node Padding termination, PT0.000351522S); (Entered node Media Type demuxer, PT0.000351642S); (Exited node Media Type demuxer, PT0.000351962S); (Entered node RTX handler, PT0.000352082S); (Exited node RTX handler, PT0.000352322S); (Entered node Duplicate termination, PT0.000352722S); (Exited node Duplicate termination, PT0.000355442S); (Entered node Retransmission requester, PT0.000355882S); (Exited node Retransmission requester, PT0.000357922S); (Entered node Padding-only discarder, PT0.000358642S); (Exited node Padding-only discarder, PT0.000359842S); (Entered node Video parser, PT0.000360242S); (Exited node Video parser, PT0.000363082S); (Entered node Video quality layer lookup, PT0.000363362S); (Exited node Video quality layer lookup, PT0.000364362S); (Entered node Video bitrate calculator, PT0.000364682S); (Exited node Video bitrate calculator, PT0.000369122S); (Entered node Input pipeline termination node, PT0.000369362S); (Entered node receiver chain handler, PT0.000370122S); (Entered RTP sender incoming queue, PT0.000488083S); (Exited RTP sender incoming queue, PT0.000551283S); (Entered node Pre-processor, PT0.000551723S); (Exited node Pre-processor, PT0.000555843S); (Entered node RedHandler, PT0.000556083S); (Exited node RedHandler, PT0.000556563S); (Entered node Strip header extensions, PT0.000556843S); (Exited node Strip header extensions, PT0.000557523S); (Entered node Packet cache, PT0.000557923S); (Exited node Packet cache, PT0.000561083S); (Entered node Absolute send time, PT0.000561243S); (Exited node Absolute send time, PT0.000561643S); (Entered node Outgoing statistics tracker, PT0.000561923S); (Exited node Outgoing statistics tracker, PT0.000562683S); (Entered node TCC sequence number tagger, PT0.000563003S); (Exited node TCC sequence number tagger, PT0.000563843S); (Entered node Header extension encoder, PT0.000564083S); (Exited node Header extension encoder, PT0.000565163S); (Entered node Toggleable pcap writer: c88aca35-tx, PT0.000565443S); (Exited node Toggleable pcap writer: c88aca35-tx, PT0.000565763S); (Entered node SRTP Encrypt Node, PT0.000566083S); (Exited node SRTP Encrypt Node, PT0.000572883S); (Entered node PacketStreamStats, PT0.000573243S); (Exited node PacketStreamStats, PT0.000573883S); (Entered node Output pipeline termination node, PT0.000574123S); (Entered Endpoint SRTP sender outgoing queue, PT0.000574443S); (Exited node Output pipeline termination node, PT0.000582923S); (Exited Endpoint SRTP sender outgoing queue, PT0.000668003S); (Sent over the ICE transport, PT0.000693284S) -*/ +=end comment + +=cut my $receiverQStartName = "Entered RTP receiver incoming queue"; my $receiverQEndName = "Exited RTP receiver incoming queue"; From e2b6f4d033daa8e9cded28b0b72947b9c71e9af5 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 5 Dec 2024 08:02:17 -0600 Subject: [PATCH 183/189] Enable XMPP stream resumption (#2257) * feat: Add a metric for XMPP disconnects. * chore: Update jicoco to 1.1-148 (enable XMPP SM) feat: Add an endpointId field to StartEvent. feat: Move jwt utils from jibri into jicoco-jwt. chore: Update jitsi-metaconfig. Add Java 21 to GitHub testing matrix feat: Enables stream resumption. --- .../jitsi/videobridge/metrics/VideobridgeMetrics.kt | 5 +++++ .../org/jitsi/videobridge/xmpp/XmppConnection.kt | 10 ++++++++++ pom.xml | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetrics.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetrics.kt index 6012b1873f..582c163c9e 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetrics.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetrics.kt @@ -247,6 +247,11 @@ object VideobridgeMetrics { "Total duration of video received, in milliseconds (each SSRC counts separately)." ) + val xmppDisconnects = metricsContainer.registerCounter( + "xmpp_disconnects", + "The number of times one of the XMPP connections has disconnected." + ) + private val tossedPacketsEnergyBuckets = listOf(0, 7, 15, 23, 31, 39, 47, 55, 63, 71, 79, 87, 95, 103, 111, 119, 127).map { it.toDouble() } .toDoubleArray() diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt index bfe91f869f..37530e0f34 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt @@ -20,12 +20,14 @@ import org.jitsi.nlj.stats.DelayStats import org.jitsi.utils.OrderedJsonObject import org.jitsi.utils.logging2.cdebug import org.jitsi.utils.logging2.createLogger +import org.jitsi.videobridge.metrics.VideobridgeMetrics import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer import org.jitsi.videobridge.xmpp.config.XmppClientConnectionConfig.Companion.config import org.jitsi.xmpp.extensions.colibri.ForcefulShutdownIQ import org.jitsi.xmpp.extensions.colibri.GracefulShutdownIQ import org.jitsi.xmpp.extensions.colibri2.ConferenceModifyIQ import org.jitsi.xmpp.extensions.health.HealthCheckIQ +import org.jitsi.xmpp.mucclient.ConnectionStateListener import org.jitsi.xmpp.mucclient.IQListener import org.jitsi.xmpp.mucclient.MucClient import org.jitsi.xmpp.mucclient.MucClientConfiguration @@ -64,6 +66,14 @@ class XmppConnection : IQListener { registerIQ(GracefulShutdownIQ()) registerIQ(ConferenceModifyIQ.ELEMENT, ConferenceModifyIQ.NAMESPACE, false) setIQListener(this@XmppConnection) + addConnectionStateListener(object : ConnectionStateListener { + override fun connected(mucClient: MucClient) {} + override fun closed(mucClient: MucClient) = VideobridgeMetrics.xmppDisconnects.inc() + override fun closedOnError(mucClient: MucClient) = VideobridgeMetrics.xmppDisconnects.inc() + override fun reconnecting(mucClient: MucClient) {} + override fun reconnectionFailed(mucClient: MucClient) {} + override fun pingFailed(mucClient: MucClient) {} + }) } config.clientConfigs.forEach { cfg -> mucClientManager.addMucClient(cfg) } diff --git a/pom.xml b/pom.xml index 96df738c07..67676da56f 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 5.9.1 5.10.2 1.0-133-g6af1020 - 1.1-143-g175c44b + 1.1-148-g3afa2ac 1.13.11 3.2.0 3.6.0 From 87b4e72e8bd9ee0cf16c9a8276aaf7e831688773 Mon Sep 17 00:00:00 2001 From: damencho Date: Thu, 5 Dec 2024 17:11:40 -0600 Subject: [PATCH 184/189] chore: Update jicoco (smack 4.4.8) and a reconnect fix. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 67676da56f..0571db3d91 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 5.9.1 5.10.2 1.0-133-g6af1020 - 1.1-148-g3afa2ac + 1.1-149-g9a9091d 1.13.11 3.2.0 3.6.0 From 3a396ddd0e3623e00d7ae69a791964701eb69c68 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Wed, 11 Dec 2024 12:03:29 -0600 Subject: [PATCH 185/189] chore: Adapt to jicoco changes. (#2261) * chore: Adapt to jicoco changes. --- jitsi-media-transform/pom.xml | 5 ----- jvb/pom.xml | 14 +++++++++++--- pom.xml | 18 ++++++++++++++---- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/jitsi-media-transform/pom.xml b/jitsi-media-transform/pom.xml index 6ce881ca0b..aaa00cd380 100644 --- a/jitsi-media-transform/pom.xml +++ b/jitsi-media-transform/pom.xml @@ -34,11 +34,6 @@ jitsi-utils ${jitsi.utils.version} - - ${project.groupId} - jicoco - ${jicoco.version} - ${project.groupId} jicoco-config diff --git a/jvb/pom.xml b/jvb/pom.xml index 2aaa72ad20..ea1f582a02 100644 --- a/jvb/pom.xml +++ b/jvb/pom.xml @@ -132,17 +132,25 @@ ice4j - ${project.groupId} - jicoco + ${project.groupId} + jicoco-config ${project.groupId} - jicoco-config + jicoco-health-checker + + + ${project.groupId} + jicoco-jetty ${project.groupId} jicoco-metrics + + ${project.groupId} + jicoco-mucclient + ${project.groupId} jitsi-xmpp-extensions diff --git a/pom.xml b/pom.xml index 0571db3d91..ef43dac717 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 5.9.1 5.10.2 1.0-133-g6af1020 - 1.1-149-g9a9091d + 1.1-150-g57913c0 1.13.11 3.2.0 3.6.0 @@ -75,17 +75,17 @@ ${project.groupId} - jicoco + jicoco-config ${jicoco.version} ${project.groupId} - jicoco-test-kotlin + jicoco-health-checker ${jicoco.version} ${project.groupId} - jicoco-config + jicoco-jetty ${jicoco.version} @@ -93,6 +93,16 @@ jicoco-metrics ${jicoco.version} + + ${project.groupId} + jicoco-mucclient + ${jicoco.version} + + + ${project.groupId} + jicoco-test-kotlin + ${jicoco.version} + ${project.groupId} jitsi-utils From 80c49693f44de8cb0d2cc5a87d3c9b2d2888c4de Mon Sep 17 00:00:00 2001 From: damencho Date: Wed, 11 Dec 2024 18:57:02 -0600 Subject: [PATCH 186/189] chore: Updates jicoco. A reconnect fix on startup a race may lead to keep trying to reconnect filling up the logs with: WARNING: [25] [hostname=localhost id=shard] MucClient.lambda$getConnectAndLoginCallable$9#693: Error connecting: org.jivesoftware.smack.SmackException$AlreadyConnectedException: Client is already connected --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ef43dac717..9579d865e6 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 5.9.1 5.10.2 1.0-133-g6af1020 - 1.1-150-g57913c0 + 1.1-151-g63a0655 1.13.11 3.2.0 3.6.0 From f18bf2e4d4489ca6b5437c77f8af08e6ed796fd8 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 12 Dec 2024 10:29:00 -0600 Subject: [PATCH 187/189] chore: Update ice4j (#2258) * fix: Fail early if IMDS returns an error. (jitsi/ice4j#298) * Add IMDSv2 support to AwsCandidateHarvester and replace URLConnectionn with HttpRequest (jitsi/ice4j#297) * Add Java 21 to GitHub testing matrix (jitsi/ice4j#296) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9579d865e6..73c46196ac 100644 --- a/pom.xml +++ b/pom.xml @@ -116,7 +116,7 @@ ${project.groupId} ice4j - 3.2-4-g1373788 + 3.2-7-g4f13296 ${project.groupId} From c7ef8e66dce162ce5bccb6ac450fe779a0d67b76 Mon Sep 17 00:00:00 2001 From: bgrozev Date: Thu, 12 Dec 2024 15:42:54 -0600 Subject: [PATCH 188/189] Add support for the VLA RTP header extension (#2263) * feat: Add BitReader utilities and tests. * feat: Add a parser for the VLA RTP header extension. * ref: Remove unused function. * feat: Update layers with info found in VLA. * feat: Add an option to use targetBitrate instead of measured bitrate for allocation * feat: Warn if replacing width/frameRate. * test: Add tests for invalid VLAs. * ref: Simplify code, add a comment. * feat: Retain the VLA extension between relays. --- .../main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt | 15 +- .../kotlin/org/jitsi/nlj/RtpReceiverImpl.kt | 2 + .../main/kotlin/org/jitsi/nlj/RtpSender.kt | 2 + .../kotlin/org/jitsi/nlj/RtpSenderImpl.kt | 8 +- .../main/kotlin/org/jitsi/nlj/Transceiver.kt | 5 + .../kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt | 8 +- .../transform/node/incoming/VlaReaderNode.kt | 105 +++++ .../node/outgoing/HeaderExtStripper.kt | 18 +- .../cc/allocation/BitrateController.kt | 38 +- .../cc/allocation/SingleSourceAllocation.kt | 11 +- .../cc/config/BitrateControllerConfig.kt | 4 + .../org/jitsi/videobridge/relay/Relay.kt | 1 + jvb/src/main/resources/reference.conf | 4 + .../rtp/rtp/header_extensions/VlaExtension.kt | 129 ++++++ .../kotlin/org/jitsi/rtp/util/BitReader.kt | 30 ++ .../jitsi/rtp/extensions/VlaExtensionTest.kt | 372 ++++++++++++++++++ .../org/jitsi/rtp/util/BitReaderTest.kt | 132 +++++++ 17 files changed, 858 insertions(+), 26 deletions(-) create mode 100644 jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VlaReaderNode.kt create mode 100644 rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/VlaExtension.kt create mode 100644 rtp/src/test/kotlin/org/jitsi/rtp/extensions/VlaExtensionTest.kt create mode 100644 rtp/src/test/kotlin/org/jitsi/rtp/util/BitReaderTest.kt diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt index 49f0474c45..90e412f571 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt @@ -28,8 +28,7 @@ import org.jitsi.utils.OrderedJsonObject * * @author George Politis */ -abstract class RtpLayerDesc -constructor( +abstract class RtpLayerDesc( /** * The index of this instance's encoding in the source encoding array. */ @@ -54,7 +53,7 @@ constructor( * represents. The actual frame rate may be less due to bad network or * system load. [NO_FRAME_RATE] for unknown. */ - val frameRate: Double, + var frameRate: Double, ) { abstract fun copy(height: Int = this.height, tid: Int = this.tid, inherit: Boolean = true): RtpLayerDesc @@ -63,6 +62,8 @@ constructor( */ protected var bitrateTracker = BitrateCalculator.createBitrateTracker() + var targetBitrate: Bandwidth? = null + /** * @return the "id" of this layer within this encoding. This is a server-side id and should * not be confused with any encoding id defined in the client (such as the @@ -87,6 +88,7 @@ constructor( */ internal open fun inheritFrom(other: RtpLayerDesc) { inheritStatistics(other.bitrateTracker) + targetBitrate = other.targetBitrate } /** @@ -110,12 +112,6 @@ constructor( */ abstract fun getBitrate(nowMs: Long): Bandwidth - /** - * Expose [getBitrate] as a [Double] in order to make it accessible from java (since [Bandwidth] is an inline - * class). - */ - fun getBitrateBps(nowMs: Long): Double = getBitrate(nowMs).bps - /** * Recursively checks this layer and its dependencies to see if the bitrate is zero. * Note that unlike [calcBitrate] this does not avoid double-visiting layers; the overhead @@ -131,6 +127,7 @@ constructor( addNumber("height", height) addNumber("index", index) addNumber("bitrate_bps", getBitrate(System.currentTimeMillis()).bps) + addNumber("target_bitrate", targetBitrate?.bps ?: 0) } fun debugState(): OrderedJsonObject = getNodeStats().toJson().apply { put("indexString", indexString()) } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt index 079001f481..85b33a5e8a 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt @@ -56,6 +56,7 @@ import org.jitsi.nlj.transform.node.incoming.VideoBitrateCalculator import org.jitsi.nlj.transform.node.incoming.VideoMuteNode import org.jitsi.nlj.transform.node.incoming.VideoParser import org.jitsi.nlj.transform.node.incoming.VideoQualityLayerLookup +import org.jitsi.nlj.transform.node.incoming.VlaReaderNode import org.jitsi.nlj.transform.packetPath import org.jitsi.nlj.transform.pipeline import org.jitsi.nlj.util.Bandwidth @@ -248,6 +249,7 @@ class RtpReceiverImpl @JvmOverloads constructor( node(videoParser) node(VideoQualityLayerLookup(logger)) node(videoBitrateCalculator) + node(VlaReaderNode(streamInformationStore, logger)) node(packetHandlerWrapper) } } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSender.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSender.kt index 241fbde39e..c6d943a0d1 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSender.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSender.kt @@ -16,6 +16,7 @@ package org.jitsi.nlj import org.jitsi.nlj.rtp.LossListener +import org.jitsi.nlj.rtp.RtpExtensionType import org.jitsi.nlj.rtp.TransportCcEngine import org.jitsi.nlj.rtp.bandwidthestimation.BandwidthEstimator import org.jitsi.nlj.srtp.SrtpTransformers @@ -47,6 +48,7 @@ abstract class RtpSender : abstract fun setFeature(feature: Features, enabled: Boolean) abstract fun isFeatureEnabled(feature: Features): Boolean abstract fun tearDown() + abstract fun addRtpExtensionToRetain(extensionType: RtpExtensionType) /** * An optional function to be executed for each RTP packet, as the first step of the send pipeline. diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt index 6a89f12b99..15895389f2 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt @@ -23,6 +23,7 @@ import org.jitsi.nlj.rtcp.NackHandler import org.jitsi.nlj.rtcp.RtcpEventNotifier import org.jitsi.nlj.rtcp.RtcpSrUpdater import org.jitsi.nlj.rtp.LossListener +import org.jitsi.nlj.rtp.RtpExtensionType import org.jitsi.nlj.rtp.TransportCcEngine import org.jitsi.nlj.rtp.bandwidthestimation.BandwidthEstimator import org.jitsi.nlj.rtp.bandwidthestimation.GoogleCcEstimator @@ -111,6 +112,7 @@ class RtpSenderImpl( private val srtcpEncryptWrapper = SrtcpEncryptNode() private val toggleablePcapWriter = ToggleablePcapWriter(logger, "$id-tx") private val outgoingPacketCache = PacketCacher() + private val headerExtensionStripper = HeaderExtStripper(streamInformationStore) private val absSendTime = AbsSendTime(streamInformationStore) private val statsTracker = OutgoingStatisticsTracker() private val packetStreamStats = PacketStreamStatsNode() @@ -144,7 +146,7 @@ class RtpSenderImpl( outgoingRtpRoot = pipeline { node(PluggableTransformerNode("RTP pre-processor") { preProcesor }) node(AudioRedHandler(streamInformationStore, logger)) - node(HeaderExtStripper(streamInformationStore)) + node(headerExtensionStripper) node(outgoingPacketCache) node(absSendTime) node(statsTracker) @@ -333,6 +335,10 @@ class RtpSenderImpl( toggleablePcapWriter.disable() } + override fun addRtpExtensionToRetain(extensionType: RtpExtensionType) { + headerExtensionStripper.addRtpExtensionToRetain(extensionType) + } + companion object { var queueErrorCounter = CountingErrorHandler() diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Transceiver.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Transceiver.kt index 8c4b468490..b0c066e0d2 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Transceiver.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Transceiver.kt @@ -18,6 +18,7 @@ package org.jitsi.nlj import org.jitsi.nlj.format.PayloadType import org.jitsi.nlj.rtcp.RtcpEventNotifier import org.jitsi.nlj.rtp.RtpExtension +import org.jitsi.nlj.rtp.RtpExtensionType import org.jitsi.nlj.rtp.bandwidthestimation.BandwidthEstimator import org.jitsi.nlj.srtp.SrtpTransformers import org.jitsi.nlj.srtp.SrtpUtil @@ -211,6 +212,10 @@ class Transceiver( rtpReceiver.handleEvent(localSsrcSetEvent) } + fun addRtpExtensionToRetain(extensionType: RtpExtensionType) { + rtpSender.addRtpExtensionToRetain(extensionType) + } + fun receivesSsrc(ssrc: Long): Boolean = streamInformationStore.receiveSsrcs.contains(ssrc) val receiveSsrcs: Set diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt index 90799e11a8..8d86e8af7e 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt @@ -103,7 +103,13 @@ enum class RtpExtensionType(val uri: String) { */ AV1_DEPENDENCY_DESCRIPTOR( "https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension" - ); + ), + + /** + * Video Layers Allocation + * https://webrtc.googlesource.com/src/+/refs/heads/main/docs/native-code/rtp-hdrext/video-layers-allocation00 + */ + VLA("http://www.webrtc.org/experiments/rtp-hdrext/video-layers-allocation00"); companion object { private val uriMap = RtpExtensionType.values().associateBy(RtpExtensionType::uri) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VlaReaderNode.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VlaReaderNode.kt new file mode 100644 index 0000000000..b4375482de --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VlaReaderNode.kt @@ -0,0 +1,105 @@ +/* + * Copyright @ 2024-Present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.incoming + +import org.jitsi.nlj.Event +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.SetMediaSourcesEvent +import org.jitsi.nlj.findRtpSource +import org.jitsi.nlj.rtp.RtpExtensionType.VLA +import org.jitsi.nlj.transform.node.ObserverNode +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.nlj.util.kbps +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.VlaExtension +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.LoggerImpl +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createChildLogger + +/** + * A node which reads the Video Layers Allocation (VLA) RTP header extension and updates the media sources. + */ +class VlaReaderNode( + streamInformationStore: ReadOnlyStreamInformationStore, + parentLogger: Logger = LoggerImpl(VlaReaderNode::class.simpleName) +) : ObserverNode("Video Layers Allocation reader") { + private val logger = createChildLogger(parentLogger) + private var vlaExtId: Int? = null + private var mediaSourceDescs: Array = arrayOf() + + init { + streamInformationStore.onRtpExtensionMapping(VLA) { + vlaExtId = it + logger.debug("VLA extension ID set to $it") + } + } + + override fun handleEvent(event: Event) { + when (event) { + is SetMediaSourcesEvent -> { + mediaSourceDescs = event.mediaSourceDescs.copyOf() + logger.cdebug { "Media sources changed:\n${mediaSourceDescs.joinToString()}" } + } + } + } + + override fun observe(packetInfo: PacketInfo) { + val rtpPacket = packetInfo.packetAs() + vlaExtId?.let { + rtpPacket.getHeaderExtension(it)?.let { ext -> + val vla = try { + VlaExtension.parse(ext) + } catch (e: Exception) { + logger.warn("Failed to parse VLA extension", e) + return + } + + val sourceDesc = mediaSourceDescs.findRtpSource(rtpPacket) + + logger.debug("Found VLA=$vla for sourceDesc=$sourceDesc") + + vla.forEachIndexed { streamIdx, stream -> + val rtpEncoding = sourceDesc?.rtpEncodings?.get(streamIdx) + stream.spatialLayers.forEach { spatialLayer -> + spatialLayer.targetBitratesKbps.forEachIndexed { tlIdx, targetBitrateKbps -> + rtpEncoding?.layers?.find { + // With VP8 simulcast all layers have sid -1 + (it.sid == spatialLayer.id || it.sid == -1) && it.tid == tlIdx + }?.let { layer -> + logger.debug( + "Setting target bitrate for rtpEncoding=$rtpEncoding layer=$layer to " + + "${targetBitrateKbps.kbps} (res=${spatialLayer.res})" + ) + layer.targetBitrate = targetBitrateKbps.kbps + spatialLayer.res?.let { res -> + if (layer.height > 0 && layer.height != res.height) { + logger.warn("Updating layer height from ${layer.height} to ${res.height}") + } + layer.height = res.height + layer.frameRate = res.maxFramerate.toDouble() + } + } + } + } + } + } + } + } + + override fun trace(f: () -> Unit) {} +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/HeaderExtStripper.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/HeaderExtStripper.kt index b697d9a763..979f271946 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/HeaderExtStripper.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/HeaderExtStripper.kt @@ -23,34 +23,40 @@ import org.jitsi.nlj.util.ReadOnlyStreamInformationStore import org.jitsi.rtp.rtp.RtpPacket /** - * Strip all hop-by-hop header extensions. Currently this leaves ssrc-audio-level and video-orientation, + * Strip all hop-by-hop header extensions. By default, this leaves ssrc-audio-level and video-orientation, * plus the AV1 dependency descriptor if the packet is an Av1DDPacket. */ class HeaderExtStripper( - streamInformationStore: ReadOnlyStreamInformationStore + streamInformationStore: ReadOnlyStreamInformationStore, ) : ModifierNode("Strip header extensions") { private var retainedExts: Set = emptySet() private var retainedExtsWithAv1DD: Set = emptySet() + private var retainedExtTypes = defaultRetainedExtTypes init { retainedExtTypes.forEach { rtpExtensionType -> streamInformationStore.onRtpExtensionMapping(rtpExtensionType) { it?.let { - retainedExts = retainedExts.plus(it) - retainedExtsWithAv1DD = retainedExtsWithAv1DD.plus(it) + retainedExts += it + retainedExtsWithAv1DD += it } } } streamInformationStore.onRtpExtensionMapping(RtpExtensionType.AV1_DEPENDENCY_DESCRIPTOR) { - it?.let { retainedExtsWithAv1DD = retainedExtsWithAv1DD.plus(it) } + it?.let { retainedExtsWithAv1DD += it } } } + fun addRtpExtensionToRetain(extensionType: RtpExtensionType) { + retainedExtTypes += extensionType + } + override fun modify(packetInfo: PacketInfo): PacketInfo { val rtpPacket = packetInfo.packetAs() val retained = if (rtpPacket is Av1DDPacket) retainedExtsWithAv1DD else retainedExts + // TODO: we should also retain any extensions that were not signaled. rtpPacket.removeHeaderExtensionsExcept(retained) return packetInfo @@ -59,7 +65,7 @@ class HeaderExtStripper( override fun trace(f: () -> Unit) = f.invoke() companion object { - private val retainedExtTypes: Set = setOf( + val defaultRetainedExtTypes: Set = setOf( RtpExtensionType.SSRC_AUDIO_LEVEL, RtpExtensionType.VIDEO_ORIENTATION ) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt index bed8c99437..8aaa5d9f64 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt @@ -19,6 +19,7 @@ import org.jitsi.nlj.MediaSourceDesc import org.jitsi.nlj.PacketInfo import org.jitsi.nlj.format.PayloadType import org.jitsi.nlj.format.PayloadTypeEncoding +import org.jitsi.nlj.util.Bandwidth import org.jitsi.nlj.util.bps import org.jitsi.rtp.rtcp.RtcpSrPacket import org.jitsi.utils.event.SyncEventEmitter @@ -192,13 +193,26 @@ class BitrateController @JvmOverloads constructor( val nowMs = clock.instant().toEpochMilli() val allocation = bandwidthAllocator.allocation - allocation.allocations.forEach { - it.targetLayer?.getBitrate(nowMs)?.let { targetBitrate -> - totalTargetBitrate += targetBitrate - it.mediaSource?.primarySSRC?.let { primarySsrc -> activeSsrcs.add(primarySsrc) } + allocation.allocations.forEach { singleAllocation -> + val allocationTargetBitrate: Bandwidth? = if (config.useVlaTargetBitrate) { + singleAllocation.targetLayer?.targetBitrate ?: singleAllocation.targetLayer?.getBitrate(nowMs) + } else { + singleAllocation.targetLayer?.getBitrate(nowMs) + } + + allocationTargetBitrate?.let { + totalTargetBitrate += it + singleAllocation.mediaSource?.primarySSRC?.let { primarySsrc -> activeSsrcs.add(primarySsrc) } } - it.idealLayer?.getBitrate(nowMs)?.let { idealBitrate -> - totalIdealBitrate += idealBitrate + + val allocationIdealBitrate: Bandwidth? = if (config.useVlaTargetBitrate) { + singleAllocation.idealLayer?.targetBitrate ?: singleAllocation.idealLayer?.getBitrate(nowMs) + } else { + singleAllocation.idealLayer?.getBitrate(nowMs) + } + + allocationIdealBitrate?.let { + totalIdealBitrate += it } } @@ -220,18 +234,24 @@ class BitrateController @JvmOverloads constructor( var totalTargetBps = 0.0 var totalIdealBps = 0.0 + var totalTargetMeasuredBps = 0.0 + var totalIdealMeasuredBps = 0.0 allocation.allocations.forEach { it.targetLayer?.getBitrate(nowMs)?.let { bitrate -> totalTargetBps += bitrate.bps } it.idealLayer?.getBitrate(nowMs)?.let { bitrate -> totalIdealBps += bitrate.bps } + it.targetLayer?.targetBitrate?.let { bitrate -> totalTargetMeasuredBps += bitrate.bps } + it.idealLayer?.targetBitrate?.let { bitrate -> totalIdealMeasuredBps += bitrate.bps } trace( diagnosticContext .makeTimeSeriesPoint("allocation_for_source", nowMs) .addField("remote_endpoint_id", it.endpointId) .addField("target_idx", it.targetLayer?.index ?: -1) .addField("ideal_idx", it.idealLayer?.index ?: -1) - .addField("target_bps", it.targetLayer?.getBitrate(nowMs)?.bps ?: -1) - .addField("ideal_bps", it.idealLayer?.getBitrate(nowMs)?.bps ?: -1) + .addField("target_bps_measured", it.targetLayer?.getBitrate(nowMs)?.bps ?: -1) + .addField("target_bps", it.targetLayer?.targetBitrate?.bps ?: -1) + .addField("ideal_bps_measured", it.idealLayer?.getBitrate(nowMs)?.bps ?: -1) + .addField("ideal_bps", it.idealLayer?.targetBitrate?.bps ?: -1) ) } @@ -240,6 +260,8 @@ class BitrateController @JvmOverloads constructor( .makeTimeSeriesPoint("allocation", nowMs) .addField("total_target_bps", totalTargetBps) .addField("total_ideal_bps", totalIdealBps) + .addField("total_target_measured_bps", totalTargetMeasuredBps) + .addField("total_ideal_measured_bps", totalIdealMeasuredBps) ) } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt index d69a22f465..12f730edaf 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt @@ -269,7 +269,16 @@ internal class SingleSourceAllocation( if (constraints.maxHeight == 0 || !source.hasRtpLayers()) { return Layers.noLayers } - val layers = source.rtpLayers.map { LayerSnapshot(it, it.getBitrateBps(nowMs)) } + val layers = source.rtpLayers.map { + LayerSnapshot( + it, + if (config.useVlaTargetBitrate) { + it.targetBitrate?.bps ?: it.getBitrate(nowMs).bps + } else { + it.getBitrate(nowMs).bps + } + ) + } return when (source.videoType) { VideoType.CAMERA -> selectLayersForCamera(layers, constraints) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt index 5c44519e43..1f9f3ed82b 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt @@ -110,6 +110,10 @@ class BitrateControllerConfig private constructor() { .convertFrom { Bandwidth.fromString(it) } } + val useVlaTargetBitrate: Boolean by config { + "videobridge.cc.use-vla-target-bitrate".from(JitsiConfig.newConfig) + } + companion object { @JvmField val config = BitrateControllerConfig() diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt index 6d0beff72e..b5d2b35cd5 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt @@ -285,6 +285,7 @@ class Relay @JvmOverloads constructor( }, external = true ) + addRtpExtensionToRetain(RtpExtensionType.VLA) } /** diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf index 7eaf35696c..9b90f6cf0b 100644 --- a/jvb/src/main/resources/reference.conf +++ b/jvb/src/main/resources/reference.conf @@ -77,6 +77,10 @@ videobridge { # If set allows receivers to override bandwidth estimation (BWE) with a specific value signaled over the bridge # channel (limited to the configured value). If not set, receivers are not allowed to override BWE. // assumed-bandwidth-limit = 10 Mbps + + # Whether to use the target bitrate signaled in the VLA extension for allocation. When disabled we use the measured + # bitrate instead (preserving previous behavior). + use-vla-target-bitrate = false } # Whether to indicate support for cryptex header extension encryption (RFC 9335) cryptex { diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/VlaExtension.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/VlaExtension.kt new file mode 100644 index 0000000000..7efbee9ac1 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/VlaExtension.kt @@ -0,0 +1,129 @@ +/* + * Copyright @ 2024-Present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.rtp.header_extensions + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.VlaExtension.Stream +import org.jitsi.rtp.util.BitReader + +/** + * A parser for the Video Layers Allocation RTP header extension. + * https://webrtc.googlesource.com/src/+/refs/heads/main/docs/native-code/rtp-hdrext/video-layers-allocation00 + */ +@SuppressFBWarnings("SF_SWITCH_NO_DEFAULT", justification = "False positive") +class VlaExtension { + companion object { + fun parse(ext: RtpPacket.HeaderExtension): ParsedVla { + val empty = ext.dataLengthBytes == 1 && ext.buffer[ext.dataOffset] == 0.toByte() + if (empty) { + return emptyList() + } + + val reader = BitReader(ext.buffer, ext.dataOffset, ext.dataLengthBytes) + reader.skipBits(2) // RID + val ns = reader.bits(2) + 1 + val slBm = reader.bits(4) + val slBms = IntArray(4) { i -> if (i < ns) slBm else 0 } + if (slBm == 0) { + slBms[0] = reader.bits(4) + if (ns > 1) { + slBms[1] = reader.bits(4) + if (ns > 2) { + slBms[2] = reader.bits(4) + if (ns > 3) { + slBms[3] = reader.bits(4) + } + } + } + if (ns == 1 || ns == 3) { + reader.skipBits(4) + } + } + + val slCount = slBms.sumOf { it.countOneBits() } + val tlCountLenBytes = slCount / 4 + if (slCount % 4 != 0) 1 else 0 + + val tlCountReader = reader.clone(tlCountLenBytes) + reader.skipBits(tlCountLenBytes * 8) + + val streams = ArrayList(ns) + (0 until ns).forEach { streamIdx -> + val spatialLayers = ArrayList() + val stream = Stream(streamIdx, spatialLayers) + streams.add(stream) + + (0 until 4).forEach { slIdx -> + if ((slBms[streamIdx] and (1 shl slIdx)) != 0) { + val targetBitrates = buildList { + repeat(tlCountReader.bits(2) + 1) { + add(reader.leb128()) + } + } + spatialLayers.add( + SpatialLayer( + slIdx, + targetBitrates, + null + ) + ) + } + } + } + + (0 until ns).forEach outer@{ streamIdx -> + (0 until 4).forEach { slIdx -> + if ((slBms[streamIdx] and (1 shl slIdx)) != 0) { + if (reader.remainingBits() < 40) { + return@outer + } + val sl = streams[streamIdx].spatialLayers[slIdx] + sl.res = ResolutionAndFrameRate( + reader.bits(16) + 1, + reader.bits(16) + 1, + reader.bits(8) + ) + } + } + } + + return streams + } + } + + data class ResolutionAndFrameRate( + val width: Int, + val height: Int, + val maxFramerate: Int + ) + + data class SpatialLayer( + val id: Int, + // The target bitrates for each temporal layer in this spatial layer + val targetBitratesKbps: List, + var res: ResolutionAndFrameRate? + ) { + override fun toString(): String = "SpatialLayer(id=$id, targetBitratesKbps=$targetBitratesKbps, " + + "width=${res?.width}, height=${res?.height}, maxFramerate=${res?.maxFramerate})" + } + + data class Stream( + val id: Int, + val spatialLayers: List + ) +} + +typealias ParsedVla = List diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/util/BitReader.kt b/rtp/src/main/kotlin/org/jitsi/rtp/util/BitReader.kt index d141b7aed4..9dcbd9bc9f 100644 --- a/rtp/src/main/kotlin/org/jitsi/rtp/util/BitReader.kt +++ b/rtp/src/main/kotlin/org/jitsi/rtp/util/BitReader.kt @@ -26,6 +26,22 @@ class BitReader(val buf: ByteArray, private val byteOffset: Int = 0, private val private var offset = byteOffset * 8 private val byteBound = byteOffset + byteLength + init { + check(byteOffset >= 0) { "byteOffset must be >= 0" } + check(byteBound <= buf.size) { "byteOffset + byteLength must be <= buf.size" } + } + + /** Clone with the current state (offset) and a new length in bytes. */ + fun clone(newByteLength: Int): BitReader { + check(offset % 8 == 0) { "Cannot clone BitReader with unaligned offset" } + check(offset / 8 + newByteLength <= byteBound) { + "newByteLength $newByteLength exceeds buffer length $byteLength after offset $byteOffset" + } + return BitReader(buf, offset / 8, newByteLength) + } + + fun remainingBits(): Int = byteBound * 8 - offset + /** Read a single bit from the buffer, as a boolean, incrementing the offset. */ fun bitAsBoolean(): Boolean { val byteIdx = offset / 8 @@ -98,6 +114,20 @@ class BitReader(val buf: ByteArray, private val byteOffset: Int = 0, private val return (v shl 1) - m + extraBit } + /** + * Read a LEB128-encoded unsigned integer. + * https://aomediacodec.github.io/av1-spec/#leb128 + */ + fun leb128(): Long { + var value = 0L + (0..8).forEach { i -> + val hasNext = bitAsBoolean() + value = value or (bits(7).toLong() shl (i * 7)) + if (!hasNext) return value + } + return value + } + /** Reset the reader to the beginning of the buffer */ fun reset() { offset = byteOffset * 8 diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/extensions/VlaExtensionTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/extensions/VlaExtensionTest.kt new file mode 100644 index 0000000000..d712ee5e97 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/extensions/VlaExtensionTest.kt @@ -0,0 +1,372 @@ +/* + * Copyright @ 2024-Present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.extensions + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.rtp.rtp.RtpPacket.HeaderExtension +import org.jitsi.rtp.rtp.header_extensions.ParsedVla +import org.jitsi.rtp.rtp.header_extensions.VlaExtension +import org.jitsi.rtp.rtp.header_extensions.VlaExtension.ResolutionAndFrameRate +import org.jitsi.rtp.rtp.header_extensions.VlaExtension.SpatialLayer +import org.jitsi.rtp.rtp.header_extensions.VlaExtension.Stream + +@Suppress("ktlint:standard:no-multi-spaces") +class VlaExtensionTest : ShouldSpec() { + init { + context("Empty") { + parse(0x00) shouldBe emptyList() + } + context("VP8 single stream (with resolution)") { + parse( + 0b0000_0001, + 0b1000_0000.toByte(), + 0b0101_0000, + 0b0111_1000, + 0b1100_1000.toByte(), + 0b0000_0001, + 0b0000_0001, + 0b0011_1111, + 0b0000_0000, + 0b1011_0011.toByte(), + 0b0010_0001 + ) shouldBe listOf( + Stream( + 0, + listOf( + SpatialLayer( + 0, + listOf(80, 120, 200), + ResolutionAndFrameRate(320, 180, 33) + ) + ) + ) + ) + } + context("VP8 single stream (without resolution)") { + parse( + 0b0000_0001, + 0b1000_0000.toByte(), + 0b0101_0000, + 0b0111_1000, + 0b1100_1000.toByte(), + 0b0000_0001 + ) shouldBe listOf( + Stream( + 0, + listOf( + SpatialLayer( + 0, + listOf(80, 120, 200), + null + ) + ) + ) + ) + } + context("VP8 simulcast stream (with resolutions)") { + parse( + 0b0010_0001, + 0b1010_1000.toByte(), + 0b0011_1100, + 0b0101_1010, + 0b1001_0110.toByte(), + 0b0000_0001, + 0b1100_1000.toByte(), + 0b0000_0001, + 0b1010_1100.toByte(), + 0b0000_0010, + 0b1111_0100.toByte(), + 0b0000_0011, + 0b1110_1110.toByte(), + 0b0000_0011, + 0b1110_0110.toByte(), + 0b0000_0101, + 0b1101_0100.toByte(), + 0b0000_1001, + 0b0000_0001, + 0b0011_1111, + 0b0000_0000, + 0b1011_0011.toByte(), + 0b0001_1111, + 0b0000_0010, + 0b0111_1111, + 0b0000_0001, + 0b0110_0111, + 0b0001_1111, + 0b0000_0100, + 0b1111_1111.toByte(), + 0b0000_0010, + 0b1100_1111.toByte(), + 0b0001_1111 + ) shouldBe listOf( + Stream( + 0, + listOf( + SpatialLayer( + 0, + listOf(60, 90, 150), + ResolutionAndFrameRate(320, 180, 31) + ) + ) + ), + Stream( + 1, + listOf( + SpatialLayer( + 0, + listOf(200, 300, 500), + ResolutionAndFrameRate(640, 360, 31) + ) + ) + ), + Stream( + 2, + listOf( + SpatialLayer( + 0, + listOf(494, 742, 1236), + ResolutionAndFrameRate(1280, 720, 31) + ) + ) + ) + ) + } + context("VP8 simulcast stream (without resolutions)") { + parse( + 0b1010_0001.toByte(), + 0b1010_1000.toByte(), + 0b1001_0101.toByte(), + 0b0000_0010, + 0b1001_1111.toByte(), + 0b0000_0011, + 0b1011_0100.toByte(), + 0b0000_0101, + 0b1000_1101.toByte(), + 0b0000_1001, + 0b1101_0011.toByte(), + 0b0000_1101, + 0b1110_0000.toByte(), + 0b0001_0110, + 0b1101_0000.toByte(), + 0b0000_1111, + 0b1011_1000.toByte(), + 0b0001_0111, + 0b1000_1000.toByte(), + 0b0010_0111 + ) shouldBe listOf( + Stream( + 0, + listOf( + SpatialLayer( + 0, + listOf(277, 415, 692), + null + ) + ) + ), + Stream( + 1, + listOf( + SpatialLayer( + 0, + listOf(1165, 1747, 2912), + null + ) + ) + ), + Stream( + 2, + listOf( + SpatialLayer( + 0, + listOf(2000, 3000, 5000), + null + ) + ) + ) + ) + } + context("VP9 SVC (with resolutions)") { + parse( + 0b0000_0111, + 0b1010_1000.toByte(), + 0b0100_1101, + 0b0110_0100, + 0b1000_1110.toByte(), + 0b0000_0001, + 0b1101_0111.toByte(), + 0b0000_0001, + 0b1001_0111.toByte(), + 0b0000_0010, + 0b1000_1101.toByte(), + 0b0000_0011, + 0b1101_0110.toByte(), + 0b0000_0010, + 0b1011_1101.toByte(), + 0b0000_0011, + 0b1111_1001.toByte(), + 0b0000_0100, + 0b0000_0001, + 0b0011_1111, + 0b0000_0000, + 0b1011_0011.toByte(), + 0b0010_0000, + 0b0000_0010, + 0b0111_1111, + 0b0000_0001, + 0b0110_0111, + 0b0010_0000, + 0b0000_0100, + 0b1111_1111.toByte(), + 0b0000_0010, + 0b1100_1111.toByte(), + 0b0010_0000 + ) shouldBe listOf( + Stream( + 0, + listOf( + SpatialLayer( + 0, + listOf(77, 100, 142), + ResolutionAndFrameRate(320, 180, 32) + ), + SpatialLayer( + 1, + listOf(215, 279, 397), + ResolutionAndFrameRate(640, 360, 32) + ), + SpatialLayer( + 2, + listOf(342, 445, 633), + ResolutionAndFrameRate(1280, 720, 32) + ) + ) + ) + ) + } + context("VP9 SVC (without resolutions or TLs)") { + parse( + 0b0000_0111, + 0b0000_0000, + 0b1001_0110.toByte(), + 0b0000_0001, + 0b1111_0100.toByte(), + 0b0000_0011, + 0b1010_1010.toByte(), + 0b0000_1011 + ) shouldBe listOf( + Stream( + 0, + listOf( + SpatialLayer( + 0, + listOf(150), + null + ), + SpatialLayer( + 1, + listOf(500), + null + ), + SpatialLayer( + 2, + listOf(1450), + null + ) + ) + ) + ) + } + context("Invalid VLAs") { + context("Resolution incomplete") { + // Cuts short with not enough bytes to contain the resolution. We succeed with no resolution. + parse( + 0b0000_0001, + 0b1000_0000.toByte(), // #tls + 0b0101_0000, // targetBitrate 1 + 0b0111_1000, // targetBitrate 2 + 0b1100_1000.toByte(), // targetBitrate 3 + 0b0000_0001, // targetBitrate 3 + 0b0000_0001, // width + 0b0011_1111, // width + 0b0000_0000, // height + 0b1011_0011.toByte(), // height + // 0b0010_0001 // maxFramerate + ) shouldBe listOf( + Stream( + 0, + listOf( + SpatialLayer( + 0, + listOf(80, 120, 200), + null + ) + ) + ) + ) + } + context("Invalid leb128") { + // Cuts short in the middle of one of the leb128 encoding of one of the target bitrates. + shouldThrow { + parse( + 0b0000_0001, + 0b1000_0000.toByte(), // #tls + 0b0101_0000, // targetBitrate 1 + 0b0111_1000, // targetBitrate 2 + 0b1100_1000.toByte(), // targetBitrate 3 + // 0b0000_0001, // targetBitrate 3 + ) + } + } + context("Missing target bitrates") { + // Does not contain the expected number of target bitrates + shouldThrow { + parse( + 0b0000_0001, // 1 spatial layer + 0b1000_0000.toByte(), // 3 temporal layers + 0b0101_0000, // targetBitrate 1 + 0b1100_1000.toByte(), // targetBitrate 2 + 0b0000_0001, // targetBitrate 2 + // 0b0111_1000, // targetBitrate 3 + ) + } + } + context("Missing temporal layer counts") { + // Does not contain the expected temporal layer counts + shouldThrow { + parse( + 0b0001_0000, // spatial layer bitmask in the next byte, 2 streams + 0b1111_1111.toByte(), // 4 spatial layers for each stream + 0b0000_0000.toByte(), // 1 temporal layer for each spatial layer of stream 1, more must follow + ) + } + } + } + } +} + +private fun parse(vararg bytes: Byte): ParsedVla = VlaExtension.parse(RawHeaderExtension(bytes)) + +@SuppressFBWarnings("CN_IMPLEMENTS_CLONE_BUT_NOT_CLONEABLE") +class RawHeaderExtension(override val buffer: ByteArray) : HeaderExtension { + override val dataOffset: Int = 0 + override var id: Int = 0 + override val dataLengthBytes: Int = buffer.size + override val totalLengthBytes: Int = buffer.size +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/util/BitReaderTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/util/BitReaderTest.kt new file mode 100644 index 0000000000..2be8e5100d --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/util/BitReaderTest.kt @@ -0,0 +1,132 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.util + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe + +class BitReaderTest : ShouldSpec() { + override fun isolationMode(): IsolationMode = IsolationMode.InstancePerLeaf + + init { + val bitReader = BitReader( + byteArrayOf( + 0b0101_1010.toByte(), + 0b0000_1111.toByte(), + 0b1010_1010.toByte(), + 0b1010_1010.toByte(), + 0b0111_1111.toByte() + ) + ) + val bits = 40 + context("Reading single bits as boolean") { + bitReader.bitAsBoolean() shouldBe false + bitReader.bitAsBoolean() shouldBe true + bitReader.bitAsBoolean() shouldBe false + bitReader.bitAsBoolean() shouldBe true + bitReader.bitAsBoolean() shouldBe true + bitReader.remainingBits() shouldBe bits - 5 + } + + context("Reading single bits as Integer") { + bitReader.bit() shouldBe 0 + bitReader.bit() shouldBe 1 + bitReader.bit() shouldBe 0 + bitReader.bit() shouldBe 1 + bitReader.bit() shouldBe 1 + bitReader.remainingBits() shouldBe bits - 5 + } + + context("Reading multiple bits as Integer") { + bitReader.bits(4) shouldBe 0b0101 + bitReader.bits(4) shouldBe 0b1010 + bitReader.bits(6) shouldBe 0b000011 + bitReader.bits(2) shouldBe 0b11 + bitReader.remainingBits() shouldBe bits - 16 + } + + context("Reading multiple bits as Long") { + bitReader.bitsLong(4) shouldBe 0b0101L + bitReader.bitsLong(4) shouldBe 0b1010L + bitReader.bits(6) shouldBe 0b000011L + bitReader.bits(2) shouldBe 0b11L + bitReader.remainingBits() shouldBe bits - 16 + } + + context("skip bits correctly") { + bitReader.skipBits(4) + bitReader.remainingBits() shouldBe bits - 4 + bitReader.bit() shouldBe 1 + bitReader.remainingBits() shouldBe bits - 5 + } + + context("Reading LEB128-encoded unsigned integers") { + bitReader.leb128() shouldBe 0b0101_1010 + bitReader.remainingBits() shouldBe bits - 8 + bitReader.leb128() shouldBe 0b00001111 + bitReader.remainingBits() shouldBe bits - 16 + bitReader.leb128() shouldBe 0b111_1111__010_1010__010_1010 + bitReader.remainingBits() shouldBe 0 + } + + context("Cloning") { + bitReader.skipBits(8) + val clone = bitReader.clone(2) + bitReader.remainingBits() shouldBe bits - 8 + clone.remainingBits() shouldBe 16 + + bitReader.bits(8) shouldBe 0b0000_1111 + bitReader.remainingBits() shouldBe bits - 16 + clone.remainingBits() shouldBe 16 + + clone.bits(8) shouldBe 0b0000_1111 + bitReader.remainingBits() shouldBe bits - 16 + clone.remainingBits() shouldBe 8 + + clone.skipBits(8) + bitReader.remainingBits() shouldBe bits - 16 + clone.remainingBits() shouldBe 0 + + bitReader.bits(8) shouldBe 0b1010_1010 + shouldThrow { + clone.bits(8) + } + } + + context("Throwing exception when reading past the bounds") { + bitReader.skipBits(bits) + bitReader.remainingBits() shouldBe 0 + shouldThrow { + bitReader.bitAsBoolean() + } + shouldThrow { + bitReader.bit() + } + shouldThrow { + bitReader.bits(4) + } + shouldThrow { + bitReader.leb128() + } + shouldThrow { + BitReader(byteArrayOf(0b1111_0000.toByte())).leb128() + } + } + } +} From 8780d4353d5638c08b3bfdd24c052fb9c57f80c6 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Fri, 20 Dec 2024 17:15:45 -0500 Subject: [PATCH 189/189] Catch TERM, HUP, and INT signals, and do a clean shutdown. (#2265) --- .../main/kotlin/org/jitsi/videobridge/Main.kt | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt index 07b4e4d668..95a5d1cd02 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt @@ -48,6 +48,7 @@ import org.jitsi.videobridge.version.JvbVersionService import org.jitsi.videobridge.websocket.ColibriWebSocketService import org.jitsi.videobridge.xmpp.XmppConnection import org.jitsi.videobridge.xmpp.config.XmppClientConnectionConfig +import sun.misc.Signal import java.time.Clock import kotlin.concurrent.thread import kotlin.system.exitProcess @@ -163,6 +164,23 @@ fun main() { null } + var exitStatus = 0 + + /* Catch signals and cause them to trigger a clean shutdown. */ + listOf("TERM", "HUP", "INT").forEach { signalName -> + try { + Signal.handle(Signal(signalName)) { signal -> + exitStatus = signal.number + 128 // Matches java.lang.Terminator + logger.info("Caught signal $signal, shutting down.") + + shutdownService.beginShutdown() + } + } catch (e: IllegalArgumentException) { + /* Unknown signal on this platform, or not allowed to register this signal; that's fine. */ + logger.warn("Unable to register signal '$signalName'", e) + } + } + // Block here until the bridge shuts down shutdownService.waitForShutdown() @@ -186,7 +204,7 @@ fun main() { TaskPools.CPU_POOL.shutdownNow() TaskPools.IO_POOL.shutdownNow() - exitProcess(0) + exitProcess(exitStatus) } private fun setupMetaconfigLogger() {