RTC2RTMP: Fix screen sharing stutter caused by packet loss. v5.0.216 v6.0.157 v7.0.18 (#4160)

## How to reproduce?

1. Refer this commit, which contains the web demo to capture screen as
video stream through RTC.
2. Copy the `trunk/research/players/whip.html` and
`trunk/research/players/js/srs.sdk.js` to replace the `develop` branch
source code.
3. `./configure && make`
4. `./objs/srs -c conf/rtc2rtmp.conf`
5. open `http://localhost:8080/players/whip.html?schema=http`
6. check `Screen` radio option.
7. click `publish`, then check the screen to share.
8. play the rtmp live stream: `rtmp://localhost/live/livestream`
9. check the video stuttering.

## Cause
When capture screen by the chrome web browser, which send RTP packet
with empty payload frequently, then all the cached RTP packets are
dropped before next key frame arrive in this case.

The OBS screen stream and camera stream do not have such problem.

## Add screen stream to WHIP demo

><img width="581" alt="Screenshot 2024-08-28 at 2 49 46 PM"
src="https://github.com/user-attachments/assets/9557dbd2-c799-4dfd-b336-5bbf2e4f8fb8">

---------

Co-authored-by: winlin <winlinvip@gmail.com>

trunk/doc/CHANGELOG.md

@@ -7,6 +7,7 @@ The changelog for SRS.
 <a name="v7-changes"></a>
 
 ## SRS 7.0 Changelog
+* v7.0, 2024-10-15, Merge [#4160](https://github.com/ossrs/srs/pull/4160): RTC2RTMP: Fix screen sharing stutter caused by packet loss. v7.0.18 (#4160)
 * v7.0, 2024-10-15, Merge [#3979](https://github.com/ossrs/srs/pull/3979): ST: Use clock_gettime to prevent time jumping backwards. v7.0.17 (#3979)
 * v7.0, 2024-09-09, Merge [#4158](https://github.com/ossrs/srs/pull/4158): Proxy: Support proxy server for SRS. v7.0.16 (#4158)
 * v7.0, 2024-09-09, Merge [#4171](https://github.com/ossrs/srs/pull/4171): Heartbeat: Report ports for proxy server. v7.0.15 (#4171)
@@ -29,6 +30,7 @@ The changelog for SRS.
 <a name="v6-changes"></a>
 
 ## SRS 6.0 Changelog
+* v6.0, 2024-10-15, Merge [#4160](https://github.com/ossrs/srs/pull/4160): RTC2RTMP: Fix screen sharing stutter caused by packet loss. v6.0.157 (#4160)
 * v6.0, 2024-09-09, Merge [#4171](https://github.com/ossrs/srs/pull/4171): Heartbeat: Report ports for proxy server. v6.0.156 (#4171)
 * v6.0, 2024-09-01, Merge [#4165](https://github.com/ossrs/srs/pull/4165): FLV: Refine source and http handler. v6.0.155 (#4165)
 * v6.0, 2024-09-01, Merge [#4166](https://github.com/ossrs/srs/pull/4166): Edge: Fix flv edge crash when http unmount. v6.0.154 (#4166)
@@ -189,6 +191,7 @@ The changelog for SRS.
 <a name="v5-changes"></a>
 
 ## SRS 5.0 Changelog
+* v5.0, 2024-10-15, Merge [#4160](https://github.com/ossrs/srs/pull/4160): RTC2RTMP: Fix screen sharing stutter caused by packet loss. v5.0.216 (#4160)
 * v5.0, 2024-09-09, Merge [#4171](https://github.com/ossrs/srs/pull/4171): Heartbeat: Report ports for proxy server. v5.0.215 (#4171)
 * v5.0, 2024-07-24, Merge [#4126](https://github.com/ossrs/srs/pull/4126): Edge: Improve stability for state and fd closing. v5.0.214 (#4126)
 * v5.0, 2024-06-03, Merge [#4057](https://github.com/ossrs/srs/pull/4057): RTC: Support dropping h.264 SEI from NALUs. v5.0.213 (#4057)

trunk/research/players/js/srs.sdk.js

@@ -527,36 +527,56 @@ function SrsRtcWhipWhepAsync() {
     // @url The WebRTC url to publish with, for example:
     //      http://localhost:1985/rtc/v1/whip/?app=live&stream=livestream
     // @options The options to control playing, supports:
-    //      videoOnly: boolean, whether only play video, default to false.
-    //      audioOnly: boolean, whether only play audio, default to false.
+    //      camera: boolean, whether capture video from camera, default to true.
+    //      screen: boolean, whether capture video from screen, default to false.
+    //      audio: boolean, whether play audio, default to true.
     self.publish = async function (url, options) {
         if (url.indexOf('/whip/') === -1) throw new Error(`invalid WHIP url ${url}`);
-        if (options?.videoOnly && options?.audioOnly) throw new Error(`The videoOnly and audioOnly in options can't be true at the same time`);
+        const hasAudio = options?.audio ?? true;
+        const useCamera = options?.camera ?? true;
+        const useScreen = options?.screen ?? false;
 
-        if (!options?.videoOnly) {
+        if (!hasAudio && !useCamera && !useScreen) throw new Error(`The camera, screen and audio can't be false at the same time`);
+
+        if (hasAudio) {
             self.pc.addTransceiver("audio", {direction: "sendonly"});
         } else {
             self.constraints.audio = false;
         }
 
-        if (!options?.audioOnly) {
+        if (useCamera || useScreen) {
             self.pc.addTransceiver("video", {direction: "sendonly"});
-        } else {
+        }
+
+        if (!useCamera) {
             self.constraints.video = false;
         }
 
         if (!navigator.mediaDevices && window.location.protocol === 'http:' && window.location.hostname !== 'localhost') {
             throw new SrsError('HttpsRequiredError', `Please use HTTPS or localhost to publish, read https://github.com/ossrs/srs/issues/2762#issuecomment-983147576`);
         }
-        var stream = await navigator.mediaDevices.getUserMedia(self.constraints);
 
+        if (useScreen) {
+            const displayStream = await navigator.mediaDevices.getDisplayMedia({
+                video: true
+            });
             // @see https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/addStream#Migrating_to_addTrack
-        stream.getTracks().forEach(function (track) {
+            displayStream.getTracks().forEach(function (track) {
                 self.pc.addTrack(track);
+				// Notify about local track when stream is ok.
+                self.ontrack && self.ontrack({track: track});
+            });
+        }
+
+       if (useCamera || hasAudio) {
+            const userStream = await navigator.mediaDevices.getUserMedia(self.constraints);
 
+            userStream.getTracks().forEach(function (track) {
+                self.pc.addTrack(track);
                 // Notify about local track when stream is ok.
                 self.ontrack && self.ontrack({track: track});
             });
+       }
 
         var offer = await self.pc.createOffer();
         await self.pc.setLocalDescription(offer);

trunk/research/players/whip.html

@@ -62,10 +62,16 @@
     <div class="form-inline">
         Controls:
         <label>
-            <input type="checkbox" id="ch_videoonly" style="margin-bottom: 8px"> Video Only
+            <input type="radio" id="ra_camera" name="video" value="Camera" checked style="margin-bottom: 8px"> Camera
         </label>
         <label>
-            <input type="checkbox" id="ch_audioonly" style="margin-bottom: 8px"> Audio Only
+            <input type="radio" id="ra_screen" name="video" value="Screen" style="margin-bottom: 8px"> Screen
+        </label>
+        <label>
+            <input type="radio" id="ra_none" name="video" value="None" style="margin-bottom: 8px"> No Video
+        </label>
+        <label>
+            <input type="checkbox" id="ch_audio" checked style="margin-bottom: 8px"> Audio
         </label>
     </div>
 
@@ -113,8 +119,9 @@ $(function(){
         // For example: webrtc://r.ossrs.net/live/livestream
         var url = $("#txt_url").val();
         sdk.publish(url, {
-            videoOnly: $('#ch_videoonly').prop('checked'),
-            audioOnly: $('#ch_audioonly').prop('checked'),
+            camera: $('#ra_camera').prop('checked'),
+            screen: $('#ra_screen').prop('checked'),
+            audio: $('#ch_audio').prop('checked')
         }).then(function(session){
             $('#sessionid').html(session.sessionid);
             $('#simulator-drop').attr('href', session.simulator + '?drop=1&username=' + session.sessionid);

trunk/src/app/srs_app_rtc_source.cpp

@@ -1775,6 +1775,23 @@ srs_error_t SrsRtcFrameBuilder::packet_video_rtmp(const uint16_t start, const ui
 
     if (0 == nb_payload) {
         srs_warn("empty nalu");
+
+        // The chrome web browser send RTP packet with empty payload frequently,
+        // reset header_sn_, lost_sn_ and continue to found next frame in this case,
+        // otherwise, all the cached RTP packets are dropped before next key frame arrive.
+        header_sn_ = end + 1;
+        uint16_t tail_sn = 0;
+        int sn = find_next_lost_sn(header_sn_, tail_sn);
+        if (-1 == sn) {
+            if (check_frame_complete(header_sn_, tail_sn)) {
+                err = packet_video_rtmp(header_sn_, tail_sn);
+            }
+        } else if (-2 == sn) {
+            return srs_error_new(ERROR_RTC_RTP_MUXER, "video cache is overflow");
+        } else {
+            lost_sn_ = sn;
+        }
+
         return err;
     }
 

trunk/src/core/srs_core_version5.hpp

@@ -9,6 +9,6 @@
 
 #define VERSION_MAJOR       5
 #define VERSION_MINOR       0
-#define VERSION_REVISION    215
+#define VERSION_REVISION    216
 
 #endif

trunk/src/core/srs_core_version6.hpp

@@ -9,6 +9,6 @@
 
 #define VERSION_MAJOR       6
 #define VERSION_MINOR       0
-#define VERSION_REVISION    156
+#define VERSION_REVISION    157
 
 #endif

trunk/src/core/srs_core_version7.hpp

@@ -9,6 +9,6 @@
 
 #define VERSION_MAJOR       7
 #define VERSION_MINOR       0
-#define VERSION_REVISION    17
+#define VERSION_REVISION    18
 
 #endif