From 9238f09b0b41c640324a1173043a4d592c2cc32e Mon Sep 17 00:00:00 2001 From: john Date: Wed, 25 Oct 2023 11:55:32 +0800 Subject: [PATCH] RTC: Fix FFmpeg opus audio noisy issue. v5.0.195 v6.0.95 (#3845) Follow the example in FFmpeg's doc, before calling the API `avcodec_send_frame`, always use `av_frame_alloc` to create a new frame. --------- Co-authored-by: Haibo Chen <495810242@qq.com> --- trunk/auto/options.sh | 2 +- trunk/conf/rtc.tcp.only.conf | 46 ++++++++++++++++++ trunk/conf/rtc.tcp.udp.conf | 47 +++++++++++++++++++ trunk/doc/CHANGELOG.md | 2 + trunk/ide/srs_clion/CMakeLists.txt | 1 - trunk/src/app/srs_app_rtc_codec.cpp | 36 +++++++------- trunk/src/app/srs_app_source.cpp | 2 +- trunk/src/core/srs_core_version5.hpp | 2 +- trunk/src/core/srs_core_version6.hpp | 2 +- .../protocol/srs_protocol_rtmp_handshake.cpp | 2 - 10 files changed, 119 insertions(+), 23 deletions(-) create mode 100644 trunk/conf/rtc.tcp.only.conf create mode 100644 trunk/conf/rtc.tcp.udp.conf diff --git a/trunk/auto/options.sh b/trunk/auto/options.sh index c90e97dce9..9d924d309c 100755 --- a/trunk/auto/options.sh +++ b/trunk/auto/options.sh @@ -37,7 +37,7 @@ SRS_FFMPEG_TOOL=NO # FFmpeg fit is the source code for RTC, to transcode audio or video in SRS. SRS_FFMPEG_FIT=RESERVED # Whether use FFmpeg native opus codec for RTC. If not, use libopus instead. -SRS_FFMPEG_OPUS=NO +SRS_FFMPEG_OPUS=YES # arguments SRS_PREFIX=/usr/local/srs SRS_DEFAULT_CONFIG=conf/srs.conf diff --git a/trunk/conf/rtc.tcp.only.conf b/trunk/conf/rtc.tcp.only.conf new file mode 100644 index 0000000000..a18474c8cc --- /dev/null +++ b/trunk/conf/rtc.tcp.only.conf @@ -0,0 +1,46 @@ +# WebRTC streaming config for SRS. +# @see full.conf for detail config. + +listen 1935; +max_connections 1000; +daemon off; +srs_log_tank console; + +http_server { + enabled on; + listen 8080; + dir ./objs/nginx/html; +} + +http_api { + enabled on; + listen 1985; +} +stats { + network 0; +} +rtc_server { + enabled on; + tcp { + enabled on; + listen 8000; + } + protocol tcp; + # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/webrtc#config-candidate + candidate $CANDIDATE; +} + +vhost __defaultVhost__ { + rtc { + enabled on; + # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/webrtc#rtmp-to-rtc + rtmp_to_rtc off; + # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/webrtc#rtc-to-rtmp + rtc_to_rtmp off; + } + http_remux { + enabled on; + mount [vhost]/[app]/[stream].flv; + } +} + diff --git a/trunk/conf/rtc.tcp.udp.conf b/trunk/conf/rtc.tcp.udp.conf new file mode 100644 index 0000000000..7fcd3a808a --- /dev/null +++ b/trunk/conf/rtc.tcp.udp.conf @@ -0,0 +1,47 @@ +# WebRTC streaming config for SRS. +# @see full.conf for detail config. + +listen 1935; +max_connections 1000; +daemon off; +srs_log_tank console; + +http_server { + enabled on; + listen 8080; + dir ./objs/nginx/html; +} + +http_api { + enabled on; + listen 1985; +} +stats { + network 0; +} +rtc_server { + enabled on; + listen 8000; # UDP port + tcp { + enabled on; + listen 8000; + } + protocol all; + # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/webrtc#config-candidate + candidate $CANDIDATE; +} + +vhost __defaultVhost__ { + rtc { + enabled on; + # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/webrtc#rtmp-to-rtc + rtmp_to_rtc off; + # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/webrtc#rtc-to-rtmp + rtc_to_rtmp off; + } + http_remux { + enabled on; + mount [vhost]/[app]/[stream].flv; + } +} + diff --git a/trunk/doc/CHANGELOG.md b/trunk/doc/CHANGELOG.md index b841a25b17..20e97cd0a6 100644 --- a/trunk/doc/CHANGELOG.md +++ b/trunk/doc/CHANGELOG.md @@ -7,6 +7,7 @@ The changelog for SRS. ## SRS 6.0 Changelog +* v6.0, 2023-10-25, Merge [#3845](https://github.com/ossrs/srs/pull/3845): RTC: Fix FFmpeg opus audio noisy issue. v6.0.95 (#3845) * v6.0, 2023-10-21, Merge [#3847](https://github.com/ossrs/srs/pull/3847): WebRTC: TCP transport should use read_fully instead of read. v6.0.94 (#3847) * v6.0, 2023-10-20, Merge [#3846](https://github.com/ossrs/srs/pull/3846): Added system library option for ffmpeg, srtp, srt libraries. v6.0.93 (#3846) * v6.0, 2023-10-17, Merge [#3840](https://github.com/ossrs/srs/pull/3840): Disable asan by default. v6.0.92 (#3840) @@ -106,6 +107,7 @@ The changelog for SRS. ## SRS 5.0 Changelog +* v5.0, 2023-10-25, Merge [#3845](https://github.com/ossrs/srs/pull/3845): RTC: Fix FFmpeg opus audio noisy issue. v5.0.195 (#3845) * v5.0, 2023-10-21, Merge [#3847](https://github.com/ossrs/srs/pull/3847): WebRTC: TCP transport should use read_fully instead of read. v5.0.194 (#3847) * v5.0, 2023-10-20, Merge [#3846](https://github.com/ossrs/srs/pull/3846): Added system library option for ffmpeg, srtp, srt libraries. v5.0.193 (#3846) * v5.0, 2023-10-17, Merge [#3840](https://github.com/ossrs/srs/pull/3840): Disable asan by default. v5.0.192 (#3840) diff --git a/trunk/ide/srs_clion/CMakeLists.txt b/trunk/ide/srs_clion/CMakeLists.txt index fbaa133c29..deb36a5df9 100755 --- a/trunk/ide/srs_clion/CMakeLists.txt +++ b/trunk/ide/srs_clion/CMakeLists.txt @@ -44,7 +44,6 @@ set(DEPS_LIBS ${SRS_DIR}/objs/st/libst.a ${SRS_DIR}/objs/srtp2/lib/libsrtp2.a ${SRS_DIR}/objs/ffmpeg/lib/libavcodec.a ${SRS_DIR}/objs/ffmpeg/lib/libavutil.a - ${SRS_DIR}/objs/opus/lib/libopus.a ${SRS_DIR}/objs/ffmpeg/lib/libswresample.a ${SRS_DIR}/objs/srt/lib/libsrt.a) foreach(DEPS_LIB ${DEPS_LIBS}) diff --git a/trunk/src/app/srs_app_rtc_codec.cpp b/trunk/src/app/srs_app_rtc_codec.cpp index 4ef7e821d7..338e065914 100644 --- a/trunk/src/app/srs_app_rtc_codec.cpp +++ b/trunk/src/app/srs_app_rtc_codec.cpp @@ -242,7 +242,7 @@ srs_error_t SrsAudioTranscoder::init_enc(SrsAudioCodecId dst_codec, int dst_chan enc_->channel_layout = av_get_default_channel_layout(dst_channels); enc_->bit_rate = dst_bit_rate; enc_->sample_fmt = codec->sample_fmts[0]; - enc_->time_base.num = 1; enc_->time_base.den = 1000; // {1, 1000} + enc_->time_base.num = 1; enc_->time_base.den = dst_samplerate; // {1, dst_samplerate} if (dst_codec == SrsAudioCodecIdOpus) { //TODO: for more level setting enc_->compression_level = 1; @@ -261,14 +261,6 @@ srs_error_t SrsAudioTranscoder::init_enc(SrsAudioCodecId dst_codec, int dst_chan return srs_error_new(ERROR_RTC_RTP_MUXER, "Could not allocate audio encode in frame"); } - enc_frame_->format = enc_->sample_fmt; - enc_frame_->nb_samples = enc_->frame_size; - enc_frame_->channel_layout = enc_->channel_layout; - - if (av_frame_get_buffer(enc_frame_, 0) < 0) { - return srs_error_new(ERROR_RTC_RTP_MUXER, "Could not get audio frame buffer"); - } - enc_packet_ = av_packet_alloc(); if (!enc_packet_) { return srs_error_new(ERROR_RTC_RTP_MUXER, "Could not allocate audio encode out packet"); @@ -380,25 +372,35 @@ srs_error_t SrsAudioTranscoder::encode(std::vector &pkts) if (next_out_pts_ == AV_NOPTS_VALUE) { next_out_pts_ = new_pkt_pts_; } else { - int64_t diff = llabs(new_pkt_pts_ - next_out_pts_); + int64_t diff = llabs(new_pkt_pts_ - av_rescale(next_out_pts_, 1000, enc_->time_base.den)); if (diff > 1000) { srs_trace("time diff to large=%lld, next out=%lld, new pkt=%lld, set to new pkt", diff, next_out_pts_, new_pkt_pts_); - next_out_pts_ = new_pkt_pts_; + next_out_pts_ = av_rescale(new_pkt_pts_, enc_->time_base.den, 1000); } } - int frame_cnt = 0; while (av_audio_fifo_size(fifo_) >= enc_->frame_size) { + enc_frame_->format = enc_->sample_fmt; + enc_frame_->nb_samples = enc_->frame_size; + enc_frame_->channel_layout = enc_->channel_layout; + + if (av_frame_get_buffer(enc_frame_, 0) < 0) { + av_frame_free(&enc_frame_); + return srs_error_new(ERROR_RTC_RTP_MUXER, "Could not get audio frame buffer"); + } + /* Read as many samples from the FIFO buffer as required to fill the frame. * The samples are stored in the frame temporarily. */ if (av_audio_fifo_read(fifo_, (void **)enc_frame_->data, enc_->frame_size) < enc_->frame_size) { + av_frame_free(&enc_frame_); return srs_error_new(ERROR_RTC_RTP_MUXER, "Could not read data from FIFO"); } /* send the frame for encoding */ - enc_frame_->pts = next_out_pts_ + av_rescale(enc_->frame_size * frame_cnt, 1000, enc_->sample_rate); - ++frame_cnt; + enc_frame_->pts = next_out_pts_; + next_out_pts_ += enc_->frame_size; int error = avcodec_send_frame(enc_, enc_frame_); + av_frame_unref(enc_frame_); if (error < 0) { return srs_error_new(ERROR_RTC_RTP_MUXER, "Error sending the frame to the encoder(%d,%s)", error, av_make_error_string(err_buf, AV_ERROR_MAX_STRING_SIZE, error)); @@ -419,6 +421,10 @@ srs_error_t SrsAudioTranscoder::encode(std::vector &pkts) av_make_error_string(err_buf, AV_ERROR_MAX_STRING_SIZE, error)); } + // rescale time base from sample_rate 1000. + enc_packet_->dts = av_rescale(enc_packet_->dts, 1000, enc_->time_base.den); + enc_packet_->pts = av_rescale(enc_packet_->pts, 1000, enc_->time_base.den); + SrsAudioFrame *out_frame = new SrsAudioFrame; char *buf = new char[enc_packet_->size]; memcpy(buf, enc_packet_->data, enc_packet_->size); @@ -429,8 +435,6 @@ srs_error_t SrsAudioTranscoder::encode(std::vector &pkts) } } - next_out_pts_ += av_rescale(enc_->frame_size * frame_cnt, 1000, enc_->sample_rate); - return srs_success; } diff --git a/trunk/src/app/srs_app_source.cpp b/trunk/src/app/srs_app_source.cpp index d78606896e..96c0cd82b4 100755 --- a/trunk/src/app/srs_app_source.cpp +++ b/trunk/src/app/srs_app_source.cpp @@ -2721,7 +2721,7 @@ srs_error_t SrsLiveSource::consumer_dumps(SrsLiveConsumer* consumer, bool ds, bo // print status. if (dg) { - srs_trace("create consumer, active=%d, queue_size=%.2f, jitter=%d", hub->active(), queue_size, jitter_algorithm); + srs_trace("create consumer, active=%d, queue_size=%dms, jitter=%d", hub->active(), srsu2msi(queue_size), jitter_algorithm); } else { srs_trace("create consumer, active=%d, ignore gop cache, jitter=%d", hub->active(), jitter_algorithm); } diff --git a/trunk/src/core/srs_core_version5.hpp b/trunk/src/core/srs_core_version5.hpp index 24c9e78ce6..0c4fbadf9c 100644 --- a/trunk/src/core/srs_core_version5.hpp +++ b/trunk/src/core/srs_core_version5.hpp @@ -9,6 +9,6 @@ #define VERSION_MAJOR 5 #define VERSION_MINOR 0 -#define VERSION_REVISION 194 +#define VERSION_REVISION 195 #endif diff --git a/trunk/src/core/srs_core_version6.hpp b/trunk/src/core/srs_core_version6.hpp index 4bcb50582a..1088a3217c 100644 --- a/trunk/src/core/srs_core_version6.hpp +++ b/trunk/src/core/srs_core_version6.hpp @@ -9,6 +9,6 @@ #define VERSION_MAJOR 6 #define VERSION_MINOR 0 -#define VERSION_REVISION 94 +#define VERSION_REVISION 95 #endif diff --git a/trunk/src/protocol/srs_protocol_rtmp_handshake.cpp b/trunk/src/protocol/srs_protocol_rtmp_handshake.cpp index 4e3677c6dd..751294de9f 100644 --- a/trunk/src/protocol/srs_protocol_rtmp_handshake.cpp +++ b/trunk/src/protocol/srs_protocol_rtmp_handshake.cpp @@ -298,8 +298,6 @@ namespace srs_internal { srs_error_t err = srs_success; - int32_t bits_count = 1024; - close(); //1. Create the DH