Optimise SendingFormat, Timeout on Wait and Fix bug in g722 (#1249)

* fix stream retrieval bug
RTC updates

* add translations for tray app and installer

* fix for ptz serialization
use wyze camera name from url

* fix bug in g722 codec

* Only get stream format once
Add timeout to ice gathering
Add prefer H264 flag for compatible formats

* revert

* Optimise SendVideo and SendAudio (only get SendingFormat once)
Add sanity check for ICE Gathering timeout

* Optimise SendVideo and SendAudio (only get SendingFormat once)
Add sanity check for ICE Gathering timeout
Fix bug in g722 codec

* remove turn folder

* Remove turn folder

* Move gather timeout to config

* Fix rounding bug in SendAudioFrame
Fix bug where duplicate durations were being added to local track timestamp in SendAudioFrame
Ignore H264 formats that use unsupported packetization modes
Clean up logic in AreMatch

* remove comment marker

src/app/Media/Codecs/G722Codec.cs

@@ -414,7 +414,15 @@ public int Encode(G722CodecState state, byte[] outputBuffer, short[] inputBuffer
                             state.QmfSignalHistory[i] = state.QmfSignalHistory[i + 2];
                         }
                         state.QmfSignalHistory[22] = inputBuffer[j++];
-                        state.QmfSignalHistory[23] = inputBuffer[j++];
+                        if (j < inputBufferCount)
+                        {
+                            state.QmfSignalHistory[23] = inputBuffer[j++];
+                        }
+                        else
+                        {
+                            //Duplicate the last sample - fix odd shorts issue
+                            state.QmfSignalHistory[23] = state.QmfSignalHistory[22];
+                        }
 
                         // Discard every other QMF output
                         sumeven = 0;
@@ -692,4 +700,4 @@ public enum G722Flags
         /// </summary>
         Packed = 0x0002
     }
-}
+}

src/net/RTP/AudioStream.cs

@@ -17,7 +17,6 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Net;
 using System.Net.Sockets;
 using System.Threading;
 using System.Threading.Tasks;
@@ -31,9 +30,11 @@ namespace SIPSorcery.net.RTP
     public class AudioStream : MediaStream
     {
         protected static ILogger logger = Log.Logger;
-
         protected Boolean rtpEventInProgress = false;
 
+        private SDPAudioVideoMediaFormat sendingFormat;
+        private bool sendingFormatFound = false;
+
         #region EVENTS
 
         /// <summary>
@@ -69,8 +70,12 @@ public bool HasAudio
         /// <param name="sample">The audio sample to set as the RTP packet payload.</param>
         public void SendAudio(uint durationRtpUnits, byte[] sample)
         {
-            var audioFormat = GetSendingFormat();
-            SendAudioFrame(durationRtpUnits, audioFormat.ID, sample);
+            if (!sendingFormatFound)
+            {
+                sendingFormat = GetSendingFormat();
+                sendingFormatFound = true;
+            }
+            SendAudioFrame(durationRtpUnits, sendingFormat.ID, sample);
         }
 
         /// <summary>
@@ -100,29 +105,43 @@ public void SendAudioFrame(uint duration, int payloadTypeID, byte[] buffer)
                     // paylaod.
                     // See https://github.com/sipsorcery/sipsorcery/issues/394.
 
-                    uint payloadDuration = 0;
+                    int maxPayload = RTPSession.RTP_MAX_PAYLOAD;
+                    int totalPackets = (buffer.Length + maxPayload - 1) / maxPayload;
+
+                    uint totalIncrement = 0;
+                    uint startTimestamp = LocalTrack.Timestamp; // Keep track of where we started.
 
-                    for (int index = 0; index * RTPSession.RTP_MAX_PAYLOAD < buffer.Length; index++)
+                    for (int index = 0; index < totalPackets; index++)
                     {
-                        int offset = (index == 0) ? 0 : (index * RTPSession.RTP_MAX_PAYLOAD);
-                        int payloadLength = (offset + RTPSession.RTP_MAX_PAYLOAD < buffer.Length) ? RTPSession.RTP_MAX_PAYLOAD : buffer.Length - offset;
-                        LocalTrack.Timestamp += payloadDuration;
-                        byte[] payload = new byte[payloadLength];
+                        int offset = index * maxPayload;
+                        int payloadLength = Math.Min(maxPayload, buffer.Length - offset);
+
+                        double fraction = (double)payloadLength / buffer.Length;
+                        uint packetDuration = (uint)Math.Round(fraction * duration);
 
+                        byte[] payload = new byte[payloadLength];
                         Buffer.BlockCopy(buffer, offset, payload, 0, payloadLength);
 
                         // RFC3551 specifies that for audio the marker bit should always be 0 except for when returning
                         // from silence suppression. For video the marker bit DOES get set to 1 for the last packet
                         // in a frame.
                         int markerBit = 0;
 
+                        // Send this packet at the current LocalTrack.Timestamp
                         SendRtpRaw(payload, LocalTrack.Timestamp, markerBit, payloadTypeID, true);
-                        //logger.LogDebug($"send audio { audioRtpChannel.RTPLocalEndPoint}->{AudioDestinationEndPoint}.");
 
-                        payloadDuration = (uint)(((decimal)payloadLength / buffer.Length) * duration); // Get the percentage duration of this payload.
+                        // After sending, increment the timestamp by this packet's portion.
+                        // This ensures the timestamp increments for the next packet, including the first one.
+                        LocalTrack.Timestamp += packetDuration;
+                        totalIncrement += packetDuration;
                     }
 
-                    LocalTrack.Timestamp += duration;
+                    // After all packets are sent, correct if we haven't incremented exactly by `duration`.
+                    if (totalIncrement != duration)
+                    {
+                        // Add or subtract the difference so total increment equals duration.
+                        LocalTrack.Timestamp += (duration - totalIncrement);
+                    }
                 }
                 catch (SocketException sockExcp)
                 {

src/net/RTP/VideoStream.cs

@@ -29,9 +29,11 @@ namespace SIPSorcery.net.RTP
     public class VideoStream : MediaStream
     {
         protected static ILogger logger = Log.Logger;
-
         protected RtpVideoFramer RtpVideoFramer;
 
+        private SDPAudioVideoMediaFormat sendingFormat;
+        private bool sendingFormatFound = false;
+
         #region EVENTS
 
         /// <summary>
@@ -244,12 +246,18 @@ public void SendVp8Frame(uint duration, int payloadTypeID, byte[] buffer)
         /// <param name="durationRtpUnits">The duration in RTP timestamp units of the video sample. This
         /// value is added to the previous RTP timestamp when building the RTP header.</param>
         /// <param name="sample">The video sample to set as the RTP packet payload.</param>
+        ///
         public void SendVideo(uint durationRtpUnits, byte[] sample)
         {
-            var videoSendingFormat = GetSendingFormat();
-            int payloadID = Convert.ToInt32(videoSendingFormat.ID);
+            if (!sendingFormatFound)
+            {
+                sendingFormat = GetSendingFormat();
+                sendingFormatFound = true;
+            }
+
+            int payloadID = Convert.ToInt32(sendingFormat.ID);
 
-            switch (videoSendingFormat.Name())
+            switch (sendingFormat.Name())
             {
                 case "VP8":
                     SendVp8Frame(durationRtpUnits, payloadID, sample);
@@ -258,7 +266,7 @@ public void SendVideo(uint durationRtpUnits, byte[] sample)
                     SendH264Frame(durationRtpUnits, payloadID, sample);
                     break;
                 default:
-                    throw new ApplicationException($"Unsupported video format selected {videoSendingFormat.Name()}.");
+                    throw new ApplicationException($"Unsupported video format selected {sendingFormat.Name()}.");
             }
         }
 

src/net/SDP/SDPAudioVideoMediaFormat.cs

@@ -18,6 +18,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using SIPSorceryMedia.Abstractions;
+using static Org.BouncyCastle.Bcpg.Attr.ImageAttrib;
 
 namespace SIPSorcery.Net
 {
@@ -142,6 +143,51 @@ public SDPAudioVideoMediaFormat(SDPWellKnownMediaFormatsEnum knownFormat)
             }
         }
 
+        public bool IsH264
+        {
+            get
+            {
+                return (Rtpmap ?? "").ToUpperInvariant().Trim().StartsWith("H264");
+            }
+        }
+
+        public bool CheckCompatible()
+        {
+            if (IsH264)
+            {
+                var parameters = ParseWebRtcParameters(Fmtp);
+                if (parameters.TryGetValue("packetization-mode", out string packetizationMode))
+                {
+                    if (packetizationMode != "1")
+                    {
+                        return false;
+                    }
+                }
+            }
+            return true;
+        }
+
+        private static Dictionary<string, string> ParseWebRtcParameters(string input)
+        {
+            var parameters = new Dictionary<string, string>();
+            if (string.IsNullOrEmpty(input))
+            {
+                return parameters;
+            }
+
+            foreach (var pair in input.Split(';'))
+            {
+                var keyValue = pair.Split('=');
+                if (keyValue.Length == 2)
+                {
+                    parameters[keyValue[0].Trim().ToLowerInvariant()] = keyValue[1].Trim();
+                }
+            }
+
+            return parameters;
+        }
+
+
         /// <summary>
         /// Creates a new SDP media format for a dynamic media type. Dynamic media types are those that use 
         /// ID's between 96 and 127 inclusive and require an rtpmap attribute and optionally an fmtp attribute.
@@ -324,23 +370,22 @@ public VideoFormat ToVideoFormat()
         public static bool AreMatch(SDPAudioVideoMediaFormat format1, SDPAudioVideoMediaFormat format2)
         {
             // rtpmap takes priority as well known format ID's can be overruled.
-            if (format1.Rtpmap != null
-                && format2.Rtpmap != null &&
-                string.Equals(format1.Rtpmap.Trim(), format2.Rtpmap.Trim(), StringComparison.OrdinalIgnoreCase))
+            if (format1.Rtpmap != null && format2.Rtpmap != null)
             {
-                return true;
+                if (string.Equals(format1.Rtpmap.Trim(), format2.Rtpmap.Trim(), StringComparison.OrdinalIgnoreCase))
+                {
+                    return true;
+                }
             }
-            else if (format1.ID < DYNAMIC_ID_MIN
+            if (format1.ID < DYNAMIC_ID_MIN
                 && format1.ID == format2.ID
                 && string.Equals(format1.Name(), format2.Name(), StringComparison.OrdinalIgnoreCase))
             {
                 // Well known format type.
                 return true;
             }
-            else
-            {
-                return false;
-            }
+            return false;
+
         }
 
         /// <summary>
@@ -368,9 +413,12 @@ public static List<SDPAudioVideoMediaFormat> GetCompatibleFormats(List<SDPAudioV
             {
                 foreach (var format in a)
                 {
-                    if (b.Any(x => SDPAudioVideoMediaFormat.AreMatch(format, x)))
+                    if (b.Any(x => AreMatch(format, x)))
                     {
-                        compatible.Add(format);
+                        if (format.CheckCompatible())
+                        {
+                            compatible.Add(format);
+                        }
                     }
                 }
             }

src/net/WebRTC/IRTCPeerConnection.cs

@@ -345,6 +345,11 @@ public class RTCConfiguration
         /// is to use ECDSA. Chrome has defaulted to ECDSA since 2016 (see https://developer.chrome.com/blog/webrtc-ecdsa).
         /// </summary>
         public bool X_UseRsaForDtlsCertificate;
+
+        /// <summary>
+        /// Timeout for gathering local IP addresses
+        /// </summary>
+        public int X_GatherTimeoutMs = 30000;
     }
 
     /// <summary>

src/net/WebRTC/RTCPeerConnection.cs

@@ -160,7 +160,7 @@ public class RTCPeerConnection : RTPSession, IRTCPeerConnection
         private const string NORMAL_CLOSE_REASON = "normal";
         private const ushort SCTP_DEFAULT_PORT = 5000;
         private const string UNKNOWN_DATACHANNEL_ERROR = "unknown";
-
+        
         /// <summary>
         /// The period to wait for the SCTP association to complete before giving up.
         /// In theory this should be very quick as the DTLS connection should already have been established
@@ -1204,7 +1204,18 @@ private SDP createBaseSdp(List<MediaStream> mediaStreamList, bool excludeIceCand
             // In theory it would be better to an async/await but that would result in a breaking
             // change to the API and for a one off (once per class instance not once per method call)
             // delay of a few hundred milliseconds it was decided not to break the API.
-            _iceGatheringTask.Wait();
+            using (var ct = new CancellationTokenSource(TimeSpan.FromMilliseconds(_configuration.X_GatherTimeoutMs)))
+            {
+                try
+                {
+                    _iceGatheringTask.Wait(ct.Token);
+                }
+                catch (OperationCanceledException)
+                {
+                    logger.LogWarning($"ICE gathering timed out after {_configuration.X_GatherTimeoutMs}Ms");
+
+                }
+            }
 
             SDP offerSdp = new SDP(IPAddress.Loopback);
             offerSdp.SessionId = LocalSdpSessionID;