H265: Support HEVC over HTTP-TS. v6.0.4 (#3275)

1. Update TS video codec to HEVC during streaming.
2. Return error when HEVC is disabled.
3. Parse HEVC NALU type by SrsHevcNaluTypeParse.
4. Show message when codec change for TS.

Co-authored-by: runner365 <shi.weibd@hotmail.com>

trunk/doc/CHANGELOG.md

@@ -8,6 +8,7 @@ The changelog for SRS.
 
 ## SRS 6.0 Changelog
 
+* v6.0, 2022-11-23, Merge [#3275](https://github.com/ossrs/srs/pull/3275): H265: Support HEVC over HTTP-TS. v6.0.4
 * v6.0, 2022-11-23, Merge [#3274](https://github.com/ossrs/srs/pull/3274): H265: Support parse multiple NALUs in a frame. v6.0.3
 * v6.0, 2022-11-22, Merge [#3272](https://github.com/ossrs/srs/pull/3272): H265: Support HEVC over RTMP or HTTP-FLV. v6.0.2
 * v6.0, 2022-11-22, Merge [#3268](https://github.com/ossrs/srs/pull/3268): H265: Update mpegts.js to play HEVC over HTTP-TS/FLV. v6.0.1

trunk/src/app/srs_app_hls.cpp

@@ -555,7 +555,7 @@ srs_error_t SrsHlsMuxer::flush_audio(SrsTsMessageCache* cache)
     return err;
 }
 
-srs_error_t SrsHlsMuxer::flush_video(SrsTsMessageCache* cache)
+srs_error_t SrsHlsMuxer::flush_video(SrsTsMessageCache* cache, SrsVideoFrame* frame)
 {
     srs_error_t err = srs_success;
     
@@ -573,6 +573,12 @@ srs_error_t SrsHlsMuxer::flush_video(SrsTsMessageCache* cache)
     
     // update the duration of segment.
     current->append(cache->video->dts / 90);
+
+    // The video codec might change during streaming. Note that the frame might be NULL, when reap segment.
+    if (frame && frame->vcodec()) {
+        SrsTsContextWriter* tscw = current->tscw;
+        tscw->update_video_codec(frame->vcodec()->id);
+    }
     
     if ((err = current->tscw->write_video(cache->video)) != srs_success) {
         return srs_error_wrap(err, "hls: write video");
@@ -1025,7 +1031,7 @@ srs_error_t SrsHlsController::write_video(SrsVideoFrame* frame, int64_t dts)
     }
     
     // flush video when got one
-    if ((err = muxer->flush_video(tsmc)) != srs_success) {
+    if ((err = muxer->flush_video(tsmc, frame)) != srs_success) {
         return srs_error_wrap(err, "hls: flush video");
     }
     
@@ -1057,7 +1063,7 @@ srs_error_t SrsHlsController::reap_segment()
     }
     
     // segment open, flush video first.
-    if ((err = muxer->flush_video(tsmc)) != srs_success) {
+    if ((err = muxer->flush_video(tsmc, NULL)) != srs_success) {
         return srs_error_wrap(err, "hls: flush video");
     }
     

trunk/src/app/srs_app_hls.hpp

@@ -193,7 +193,7 @@ public:
     // Whether current hls muxer is pure audio mode.
     virtual bool pure_audio();
     virtual srs_error_t flush_audio(SrsTsMessageCache* cache);
-    virtual srs_error_t flush_video(SrsTsMessageCache* cache);
+    virtual srs_error_t flush_video(SrsTsMessageCache* cache, SrsVideoFrame* frame);
     // Close segment(ts).
     virtual srs_error_t segment_close();
 private:

trunk/src/core/srs_core_version6.hpp

@@ -9,6 +9,6 @@
 
 #define VERSION_MAJOR       6
 #define VERSION_MINOR       0
-#define VERSION_REVISION    3
+#define VERSION_REVISION    4
 
 #endif

trunk/src/kernel/srs_kernel_codec.cpp

@@ -620,11 +620,11 @@ srs_error_t SrsVideoFrame::add_sample(char* bytes, int size)
     // For HEVC(H.265), try to parse the IDR from NALUs.
     if (c && c->id == SrsVideoCodecIdHEVC) {
 #ifdef SRS_H265
-        SrsHevcNaluType nalu_type = (SrsHevcNaluType)(uint8_t)((bytes[0] & 0x3f) >> 1);
+        SrsHevcNaluType nalu_type = SrsHevcNaluTypeParse(bytes[0]);
         has_idr = (SrsHevcNaluType_CODED_SLICE_BLA <= nalu_type) && (nalu_type <= SrsHevcNaluType_RESERVED_23);
         return err;
 #else
-        return srs_error_new(ERROR_HLS_DECODE_ERROR, "H.265 is disabled");
+        return srs_error_new(ERROR_HEVC_DISABLED, "H.265 is disabled");
 #endif
     }
 
@@ -854,7 +854,7 @@ srs_error_t SrsFormat::video_avc_demux(SrsBuffer* stream, int64_t timestamp)
         }
         return err;
 #else
-        return srs_error_new(ERROR_HLS_DECODE_ERROR, "H.265 is disabled");
+        return srs_error_new(ERROR_HEVC_DISABLED, "H.265 is disabled");
 #endif
     }
 
@@ -1373,7 +1373,7 @@ srs_error_t SrsFormat::video_nalu_demux(SrsBuffer* stream)
         // TODO: FIXME: Might need to guess format?
         return do_avc_demux_ibmf_format(stream);
 #else
-        return srs_error_new(ERROR_HLS_DECODE_ERROR, "H.265 is disabled");
+        return srs_error_new(ERROR_HEVC_DISABLED, "H.265 is disabled");
 #endif
     }
 
@@ -1500,6 +1500,7 @@ srs_error_t SrsFormat::do_avc_demux_ibmf_format(SrsBuffer* stream)
     // 5.3.4.2.1 Syntax, ISO_IEC_14496-15-AVC-format-2012.pdf, page 20
     for (int i = 0; i < PictureLength;) {
         // unsigned int((NAL_unit_length+1)*8) NALUnitLength;
+        // TODO: FIXME: Should ignore error? See https://github.com/ossrs/srs-gb28181/commit/a13b9b54938a14796abb9011e7a8ee779439a452
         if (!stream->require(vcodec->NAL_unit_length + 1)) {
             return srs_error_new(ERROR_HLS_DECODE_ERROR, "PictureLength:%d, i:%d, NaluLength:%d, left:%d",
                 PictureLength, i, vcodec->NAL_unit_length, stream->left());

trunk/src/kernel/srs_kernel_codec.hpp

@@ -468,6 +468,7 @@ enum SrsHevcNaluType {
     SrsHevcNaluType_UNSPECIFIED_63,
     SrsHevcNaluType_INVALID,
 };
+#define SrsHevcNaluTypeParse(code) (SrsHevcNaluType)((code & 0x7E) >> 1)
 
 struct SrsHevcNalData {
     uint16_t nal_unit_length;

trunk/src/kernel/srs_kernel_error.hpp

@@ -266,7 +266,8 @@
     XX(ERROR_INOTIFY_OPENFD                , 3094, "InotifyOpenFd", "Failed to open inotify fd for config listener") \
     XX(ERROR_INOTIFY_WATCH                 , 3095, "InotfyWatch", "Failed to watch inotify for config listener") \
     XX(ERROR_HTTP_URL_UNESCAPE             , 3096, "HttpUrlUnescape", "Failed to unescape URL for HTTP") \
-    XX(ERROR_HTTP_WITH_BODY                , 3097, "HttpWithBody", "Failed for HTTP body")
+    XX(ERROR_HTTP_WITH_BODY                , 3097, "HttpWithBody", "Failed for HTTP body") \
+    XX(ERROR_HEVC_DISABLED                 , 3098, "HevcDisabled", "HEVC is disabled")
 
 /**************************************************/
 /* HTTP/StreamConverter protocol error. */

trunk/src/kernel/srs_kernel_ts.cpp

@@ -43,6 +43,9 @@ string srs_ts_stream2string(SrsTsStream stream)
         case SrsTsStreamAudioAC3: return "AC3";
         case SrsTsStreamAudioDTS: return "AudioDTS";
         case SrsTsStreamVideoH264: return "H.264";
+#ifdef SRS_H265
+        case SrsTsStreamVideoHEVC: return "H.265";
+#endif
         case SrsTsStreamVideoMpeg4: return "MP4";
         case SrsTsStreamAudioMpeg4: return "MP4A";
         default: return "Other";
@@ -285,6 +288,14 @@ srs_error_t SrsTsContext::encode(ISrsStreamWriter* writer, SrsTsMessage* msg, Sr
             vs = SrsTsStreamVideoH264;
             video_pid = TS_VIDEO_AVC_PID;
             break;
+        case SrsVideoCodecIdHEVC:
+#ifdef SRS_H265
+            vs = SrsTsStreamVideoHEVC;
+            video_pid = TS_VIDEO_AVC_PID;
+            break;
+#else
+            return srs_error_new(ERROR_HEVC_DISABLED, "H.265 is disabled");
+#endif
         case SrsVideoCodecIdDisabled:
             vs = SrsTsStreamReserved;
             break;
@@ -296,7 +307,6 @@ srs_error_t SrsTsContext::encode(ISrsStreamWriter* writer, SrsTsMessage* msg, Sr
         case SrsVideoCodecIdOn2VP6:
         case SrsVideoCodecIdOn2VP6WithAlphaChannel:
         case SrsVideoCodecIdScreenVideoVersion2:
-        case SrsVideoCodecIdHEVC:
         case SrsVideoCodecIdAV1:
             vs = SrsTsStreamReserved;
             break;
@@ -335,10 +345,13 @@ srs_error_t SrsTsContext::encode(ISrsStreamWriter* writer, SrsTsMessage* msg, Sr
         return srs_error_new(ERROR_HLS_NO_STREAM, "ts: no a/v stream, vcodec=%d, acodec=%d", vc, ac);
     }
     
-    // when any codec changed, write PAT/PMT table.
+    // When any codec changed, write PAT/PMT table.
     if (vcodec != vc || acodec != ac) {
-        vcodec = vc;
-        acodec = ac;
+        if (vcodec != SrsVideoCodecIdReserved || acodec != SrsAudioCodecIdReserved1) {
+            srs_trace("TS: Refresh PMT when vcodec=%d=>%d, acodec=%d=>%d", vcodec, vc, acodec, ac);
+        }
+        vcodec = vc; acodec = ac;
+
         if ((err = encode_pat_pmt(writer, video_pid, vs, audio_pid, as)) != srs_success) {
             return srs_error_wrap(err, "ts: encode PAT/PMT");
         }
@@ -355,8 +368,12 @@ srs_error_t SrsTsContext::encode(ISrsStreamWriter* writer, SrsTsMessage* msg, Sr
 srs_error_t SrsTsContext::encode_pat_pmt(ISrsStreamWriter* writer, int16_t vpid, SrsTsStream vs, int16_t apid, SrsTsStream as)
 {
     srs_error_t err = srs_success;
-    
-    if (vs != SrsTsStreamVideoH264 && as != SrsTsStreamAudioAAC && as != SrsTsStreamAudioMp3) {
+
+    bool codec_ok = (vs == SrsTsStreamVideoH264 || as == SrsTsStreamAudioAAC || as == SrsTsStreamAudioMp3);
+#ifdef SRS_H265
+    codec_ok = codec_ok ? : (vs == SrsTsStreamVideoHEVC);
+#endif
+    if (!codec_ok) {
         return srs_error_new(ERROR_HLS_NO_STREAM, "ts: no PID, vs=%d, as=%d", vs, as);
     }
     
@@ -425,8 +442,12 @@ srs_error_t SrsTsContext::encode_pes(ISrsStreamWriter* writer, SrsTsMessage* msg
     if (msg->payload->length() == 0) {
         return err;
     }
-    
-    if (sid != SrsTsStreamVideoH264 && sid != SrsTsStreamAudioMp3 && sid != SrsTsStreamAudioAAC) {
+
+    bool codec_ok = (sid == SrsTsStreamVideoH264 || sid == SrsTsStreamAudioAAC || sid == SrsTsStreamAudioMp3);
+#ifdef SRS_H265
+    codec_ok = codec_ok ? : (sid == SrsTsStreamVideoHEVC);
+#endif
+    if (!codec_ok) {
         srs_info("ts: ignore the unknown stream, sid=%d", sid);
         return err;
     }
@@ -750,10 +771,14 @@ SrsTsPacket* SrsTsPacket::create_pmt(SrsTsContext* context,
     pmt->current_next_indicator = 1;
     pmt->section_number = 0;
     pmt->last_section_number = 0;
-    
-    // must got one valid codec.
-    srs_assert(vs == SrsTsStreamVideoH264 || as == SrsTsStreamAudioAAC || as == SrsTsStreamAudioMp3);
-    
+
+    // Here we must get the correct codec.
+    bool codec_ok = (vs == SrsTsStreamVideoH264 || as == SrsTsStreamAudioAAC || as == SrsTsStreamAudioMp3);
+#ifdef SRS_H265
+    codec_ok = codec_ok ? : (vs == SrsTsStreamVideoHEVC);
+#endif
+    srs_assert(codec_ok);
+
     // if mp3 or aac specified, use audio to carry pcr.
     if (as == SrsTsStreamAudioAAC || as == SrsTsStreamAudioMp3) {
         // use audio to carray pcr by default.
@@ -762,8 +787,12 @@ SrsTsPacket* SrsTsPacket::create_pmt(SrsTsContext* context,
         pmt->infos.push_back(new SrsTsPayloadPMTESInfo(as, apid));
     }
     
-    // if h.264 specified, use video to carry pcr.
-    if (vs == SrsTsStreamVideoH264) {
+    // If h.264/h.265 specified, use video to carry pcr.
+    codec_ok = (vs == SrsTsStreamVideoH264);
+#ifdef SRS_H265
+    codec_ok = codec_ok ? : (vs == SrsTsStreamVideoHEVC);
+#endif
+    if (codec_ok) {
         pmt->PCR_PID = vpid;
         pmt->infos.push_back(new SrsTsPayloadPMTESInfo(vs, vpid));
     }
@@ -2533,6 +2562,9 @@ srs_error_t SrsTsPayloadPMT::psi_decode(SrsBuffer* stream)
         // update the apply pid table
         switch (info->stream_type) {
             case SrsTsStreamVideoH264:
+#ifdef SRS_H265
+            case SrsTsStreamVideoHEVC:
+#endif
             case SrsTsStreamVideoMpeg4:
                 packet->context->set(info->elementary_PID, SrsTsPidApplyVideo, info->stream_type);
                 break;
@@ -2616,6 +2648,9 @@ srs_error_t SrsTsPayloadPMT::psi_encode(SrsBuffer* stream)
         // update the apply pid table
         switch (info->stream_type) {
             case SrsTsStreamVideoH264:
+#ifdef SRS_H265
+            case SrsTsStreamVideoHEVC:
+#endif
             case SrsTsStreamVideoMpeg4:
                 packet->context->set(info->elementary_PID, SrsTsPidApplyVideo, info->stream_type);
                 break;
@@ -2685,6 +2720,11 @@ SrsVideoCodecId SrsTsContextWriter::video_codec()
     return vcodec;
 }
 
+void SrsTsContextWriter::update_video_codec(SrsVideoCodecId v)
+{
+    vcodec = v;
+}
+
 SrsEncFileWriter::SrsEncFileWriter()
 {
     memset(iv,0,16);
@@ -2832,8 +2872,17 @@ srs_error_t SrsTsMessageCache::cache_video(SrsVideoFrame* frame, int64_t dts)
     video->dts = dts;
     video->pts = video->dts + frame->cts * 90;
     video->sid = SrsTsPESStreamIdVideoCommon;
-    
-    // write video to cache.
+
+    // Write H.265 video frame to cache.
+    if (frame && frame->vcodec()->id == SrsVideoCodecIdHEVC) {
+#ifdef SRS_H265
+        return do_cache_hevc(frame);
+#else
+        return srs_error_new(ERROR_HEVC_DISABLED, "H.265 is disabled");
+#endif
+    }
+
+    // Write H.264 video frame to cache.
     if ((err = do_cache_avc(frame)) != srs_success) {
         return srs_error_wrap(err, "ts: cache avc");
     }
@@ -2924,7 +2973,7 @@ srs_error_t SrsTsMessageCache::do_cache_aac(SrsAudioFrame* frame)
     return err;
 }
 
-void srs_avc_insert_aud(SrsSimpleStream* payload, bool& aud_inserted)
+void srs_avc_insert_aud(SrsSimpleStream* payload, bool aud_inserted)
 {
     // mux the samples in annexb format,
     // ISO_IEC_14496-10-AVC-2012.pdf, page 324.
@@ -3064,6 +3113,52 @@ srs_error_t SrsTsMessageCache::do_cache_avc(SrsVideoFrame* frame)
     return err;
 }
 
+#ifdef SRS_H265
+srs_error_t SrsTsMessageCache::do_cache_hevc(SrsVideoFrame* frame)
+{
+    srs_error_t err = srs_success;
+
+    // Whether aud inserted.
+    bool aud_inserted = false;
+
+    SrsVideoCodecConfig* codec = frame->vcodec();
+    srs_assert(codec);
+
+    bool is_sps_pps_appended = false;
+
+    // all sample use cont nalu header, except the sps-pps before IDR frame.
+    for (int i = 0; i < frame->nb_samples; i++) {
+        SrsSample* sample = &frame->samples[i];
+        int32_t size = sample->size;
+
+        if (!sample->bytes || size <= 0) {
+            return srs_error_new(ERROR_HLS_AVC_SAMPLE_SIZE, "ts: invalid avc sample length=%d", size);
+        }
+
+        // Insert aud before NALU for HEVC.
+        SrsHevcNaluType nalu_type = (SrsHevcNaluType)SrsHevcNaluTypeParse(sample->bytes[0]);
+        bool is_idr = (SrsHevcNaluType_CODED_SLICE_BLA <= nalu_type) && (nalu_type <= SrsHevcNaluType_RESERVED_23);
+        if (is_idr && !frame->has_sps_pps && !is_sps_pps_appended) {
+            for (size_t i = 0; i < codec->hevc_dec_conf_record_.nalu_vec.size(); i++) {
+                const SrsHevcHvccNalu& nalu = codec->hevc_dec_conf_record_.nalu_vec[i];
+                if (nalu.num_nalus <= 0 || nalu.nal_data_vec.empty()) continue;
+
+                srs_avc_insert_aud(video->payload, aud_inserted);
+                const SrsHevcNalData& data = nalu.nal_data_vec.at(0);
+                video->payload->append((char*)&data.nal_unit_data[0], (int)data.nal_unit_data.size());
+                is_sps_pps_appended = true;
+            }
+        }
+
+        // Insert the NALU to video in annexb.
+        srs_avc_insert_aud(video->payload, aud_inserted);
+        video->payload->append(sample->bytes, sample->size);
+    }
+
+    return err;
+}
+#endif
+
 SrsTsTransmuxer::SrsTsTransmuxer()
 {
     writer = NULL;
@@ -3158,10 +3253,17 @@ srs_error_t SrsTsTransmuxer::write_video(int64_t timestamp, char* data, int size
     if (format->video->frame_type == SrsVideoAvcFrameTypeVideoInfoFrame) {
         return err;
     }
-    
-    if (format->vcodec->id != SrsVideoCodecIdAVC) {
+
+    bool codec_ok = (format->vcodec->id != SrsVideoCodecIdAVC);
+#ifdef SRS_H265
+    codec_ok = codec_ok ? : (format->vcodec->id != SrsVideoCodecIdHEVC);
+#endif
+    if (!codec_ok) {
         return err;
     }
+
+    // The video codec might change during streaming.
+    tscw->update_video_codec(format->vcodec->id);
     
     // ignore sequence header
     if (format->video->frame_type == SrsVideoAvcFrameTypeKeyFrame && format->video->avc_packet_type == SrsVideoAvcFrameTraitSequenceHeader) {

trunk/src/kernel/srs_kernel_ts.hpp

@@ -131,6 +131,10 @@ enum SrsTsStream
     // ITU-T Rec. H.222.0 | ISO/IEC 13818-1 Reserved
     // 0x15-0x7F
     SrsTsStreamVideoH264 = 0x1b,
+#ifdef SRS_H265
+    // For HEVC(H.265).
+    SrsTsStreamVideoHEVC = 0x24,
+#endif
     // User Private
     // 0x80-0xFF
     SrsTsStreamAudioAC3 = 0x81,
@@ -1272,8 +1276,9 @@ public:
     // Write a video frame to ts,
     virtual srs_error_t write_video(SrsTsMessage* video);
 public:
-    // get the video codec of ts muxer.
+    // Get or update the video codec of ts muxer.
     virtual SrsVideoCodecId video_codec();
+    virtual void update_video_codec(SrsVideoCodecId v);
 };
 
 // Used for HLS Encryption
@@ -1315,6 +1320,9 @@ private:
     virtual srs_error_t do_cache_mp3(SrsAudioFrame* frame);
     virtual srs_error_t do_cache_aac(SrsAudioFrame* frame);
     virtual srs_error_t do_cache_avc(SrsVideoFrame* frame);
+#ifdef SRS_H265
+    virtual srs_error_t do_cache_hevc(SrsVideoFrame* frame);
+#endif
 };
 
 // Transmux the RTMP stream to HTTP-TS stream.

trunk/src/utest/srs_utest_kernel.cpp

@@ -4921,9 +4921,6 @@ VOID TEST(KernelTSTest, CoverContextEncode)
         
         srs_error_t err = ctx.encode(&f, &m, SrsVideoCodecIdDisabled, SrsAudioCodecIdDisabled);
         HELPER_EXPECT_FAILED(err);
-        
-        err = ctx.encode(&f, &m, SrsVideoCodecIdHEVC, SrsAudioCodecIdOpus);
-        HELPER_EXPECT_FAILED(err);
 
         err = ctx.encode(&f, &m, SrsVideoCodecIdAV1, SrsAudioCodecIdOpus);
         HELPER_EXPECT_FAILED(err);
@@ -4953,6 +4950,34 @@ VOID TEST(KernelTSTest, CoverContextEncode)
     }
 }
 
+VOID TEST(KernelTSTest, CoverContextEncodeHEVC)
+{
+    srs_error_t err;
+
+    SrsTsContext ctx;
+    MockTsHandler h;
+
+#ifndef SRS_H265
+    if (true) {
+        MockSrsFileWriter f;
+        SrsTsMessage m;
+
+        err = ctx.encode(&f, &m, SrsVideoCodecIdHEVC, SrsAudioCodecIdOpus);
+        HELPER_EXPECT_FAILED(err);
+    }
+#endif
+
+#ifdef SRS_H265
+    if (true) {
+        MockSrsFileWriter f;
+        SrsTsMessage m;
+
+        err = ctx.encode(&f, &m, SrsVideoCodecIdHEVC, SrsAudioCodecIdOpus);
+        HELPER_EXPECT_SUCCESS(err);
+    }
+#endif
+}
+
 VOID TEST(KernelTSTest, CoverContextDecode)
 {
 	srs_error_t err;