Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for the VLA RTP header extension #2263

Merged
merged 11 commits into from
Dec 12, 2024
Prev Previous commit
Next Next commit
feat: Update layers with info found in VLA.
  • Loading branch information
bgrozev committed Dec 12, 2024
commit ccac12eb1f1f19519c14d864cc39c8e55df5e412
Original file line number Diff line number Diff line change
@@ -53,7 +53,7 @@ abstract class RtpLayerDesc(
* represents. The actual frame rate may be less due to bad network or
* system load. [NO_FRAME_RATE] for unknown.
*/
val frameRate: Double,
var frameRate: Double,
) {
abstract fun copy(height: Int = this.height, tid: Int = this.tid, inherit: Boolean = true): RtpLayerDesc

@@ -62,6 +62,8 @@ abstract class RtpLayerDesc(
*/
protected var bitrateTracker = BitrateCalculator.createBitrateTracker()

var targetBitrate: Bandwidth? = null

/**
* @return the "id" of this layer within this encoding. This is a server-side id and should
* not be confused with any encoding id defined in the client (such as the
@@ -86,6 +88,7 @@ abstract class RtpLayerDesc(
*/
internal open fun inheritFrom(other: RtpLayerDesc) {
inheritStatistics(other.bitrateTracker)
targetBitrate = other.targetBitrate
}

/**
@@ -124,6 +127,7 @@ abstract class RtpLayerDesc(
addNumber("height", height)
addNumber("index", index)
addNumber("bitrate_bps", getBitrate(System.currentTimeMillis()).bps)
addNumber("target_bitrate", targetBitrate?.bps ?: 0)
}

fun debugState(): OrderedJsonObject = getNodeStats().toJson().apply { put("indexString", indexString()) }
Original file line number Diff line number Diff line change
@@ -19,9 +19,11 @@ import org.jitsi.nlj.Event
import org.jitsi.nlj.MediaSourceDesc
import org.jitsi.nlj.PacketInfo
import org.jitsi.nlj.SetMediaSourcesEvent
import org.jitsi.nlj.findRtpSource
import org.jitsi.nlj.rtp.RtpExtensionType.VLA
import org.jitsi.nlj.transform.node.ObserverNode
import org.jitsi.nlj.util.ReadOnlyStreamInformationStore
import org.jitsi.nlj.util.kbps
import org.jitsi.rtp.rtp.RtpPacket
import org.jitsi.rtp.rtp.header_extensions.VlaExtension
import org.jitsi.utils.logging2.Logger
@@ -30,7 +32,7 @@ import org.jitsi.utils.logging2.cdebug
import org.jitsi.utils.logging2.createChildLogger

/**
* A node which reads the Video Layers Allocation (VLA) RTP header extension and updates the media sources (TODO).
* A node which reads the Video Layers Allocation (VLA) RTP header extension and updates the media sources.
*/
class VlaReaderNode(
streamInformationStore: ReadOnlyStreamInformationStore,
@@ -56,20 +58,41 @@ class VlaReaderNode(
}
}

@OptIn(ExperimentalStdlibApi::class)
override fun observe(packetInfo: PacketInfo) {
val rtpPacket = packetInfo.packetAs<RtpPacket>()
vlaExtId?.let {
rtpPacket.getHeaderExtension(it)?.let { ext ->
try {
val vla = VlaExtension.parse(ext)
// TODO update media sources
logger.info(
"VLA extension found: $vla " +
"raw=${ext.buffer.toHexString(ext.dataOffset, ext.dataOffset + ext.dataLengthBytes)}"
)
val vla = try {
VlaExtension.parse(ext)
} catch (e: Exception) {
logger.warn("Failed to parse VLA extension", e)
return
}

val sourceDesc = mediaSourceDescs.findRtpSource(rtpPacket)

logger.debug("Found VLA=$vla for sourceDesc=$sourceDesc")

vla.forEachIndexed { streamIdx, stream ->
val rtpEncoding = sourceDesc?.rtpEncodings?.get(streamIdx)
stream.spatialLayers.forEach { spatialLayer ->
spatialLayer.targetBitratesKbps.forEachIndexed { tlIdx, targetBitrateKbps ->
rtpEncoding?.layers?.find {
// With VP8 simulcast all layers have sid -1
(it.sid == spatialLayer.id || it.sid == -1) && it.tid == tlIdx
}?.let { layer ->
logger.debug(
"Setting target bitrate for rtpEncoding=$rtpEncoding layer=$layer to " +
"${targetBitrateKbps.kbps} (res=${spatialLayer.res})"
)
layer.targetBitrate = targetBitrateKbps.kbps
spatialLayer.res?.let { res ->
layer.height = res.height
layer.frameRate = res.maxFramerate.toDouble()
}
}
}
}
}
}
}