MP3: Fix bug for TS or HLS with mp3 codec. v4.0.269 (#296) (#3333)

* MP3: Fix bug for TS or HLS with mp3 codec. v4.0.269 (#296)

1. Refresh HLS audio codec if changed in stream.
2. Refresh TS audio codec if changed in stream.
3. Fix mp3 demux bug in SrsFormat::audio_mp3_demux.
4. Use 3(MPEG1) not 4(MPEG2) as PMT stream type, follow FFmpeg.
5. MP3: Update utest for mp3 sample parsing.
6. MP3: Ignore empty frame sample.
7. UTest: Fix utest failed, do not copy files.

trunk/Dockerfile

@@ -18,21 +18,6 @@ COPY . /srs
 WORKDIR /srs/trunk
 RUN ./configure --srt=on --jobs=${JOBS} && make -j${JOBS} && make install
 
-# All config files for SRS.
-RUN cp -R conf /usr/local/srs/conf && \
-    cp research/api-server/static-dir/index.html /usr/local/srs/objs/nginx/html/ && \
-    cp research/api-server/static-dir/favicon.ico /usr/local/srs/objs/nginx/html/ && \
-    cp research/players/crossdomain.xml /usr/local/srs/objs/nginx/html/ && \
-    cp -R research/console /usr/local/srs/objs/nginx/html/ && \
-    cp -R research/players /usr/local/srs/objs/nginx/html/ && \
-    cp -R 3rdparty/signaling/www/demos /usr/local/srs/objs/nginx/html/
-
-# Copy the shared libraries for FFmpeg.
-RUN mkdir -p /usr/local/shared && \
-    cp $(ldd /usr/local/bin/ffmpeg |grep libxml2 |awk '{print $3}') /usr/local/shared/ && \
-    cp $(ldd /usr/local/bin/ffmpeg |grep libicuuc |awk '{print $3}') /usr/local/shared/ && \
-    cp $(ldd /usr/local/bin/ffmpeg |grep libicudata |awk '{print $3}') /usr/local/shared/
-
 ############################################################
 # dist
 ############################################################
@@ -46,7 +31,6 @@ RUN echo "BUILDPLATFORM: $BUILDPLATFORM, TARGETPLATFORM: $TARGETPLATFORM"
 EXPOSE 1935 1985 8080 8000/udp 10080/udp
 
 # FFMPEG 4.1
-COPY --from=build /usr/local/shared/* /lib/
 COPY --from=build /usr/local/bin/ffmpeg /usr/local/srs/objs/ffmpeg/bin/ffmpeg
 # SRS binary, config files and srs-console.
 COPY --from=build /usr/local/srs /usr/local/srs

trunk/doc/CHANGELOG.md

@@ -8,6 +8,7 @@ The changelog for SRS.
 
 ## SRS 4.0 Changelog
 
+* v4.0, 2022-12-24, For [#296](https://github.com/ossrs/srs/issues/296): MP3: Fix bug for TS or HLS with mp3 codec. v4.0.269
 * v4.0, 2022-11-22, Pick [#3079](https://github.com/ossrs/srs/issues/3079): WebRTC: Fix no audio and video issue for Firefox. v4.0.268
 * v4.0, 2022-10-10, For [#2901](https://github.com/ossrs/srs/issues/2901): Edge: Fast disconnect and reconnect. v4.0.267
 * v4.0, 2022-09-27, For [#3167](https://github.com/ossrs/srs/issues/3167): WebRTC: Refine sequence jitter algorithm. v4.0.266

trunk/src/app/srs_app_hls.cpp

@@ -202,6 +202,7 @@ SrsHlsMuxer::SrsHlsMuxer()
     async = new SrsAsyncCallWorker();
     context = new SrsTsContext();
     segments = new SrsFragmentWindow();
+    latest_acodec_ = SrsAudioCodecIdForbidden;
     
     memset(key, 0, 16);
     memset(iv, 0, 16);
@@ -263,6 +264,24 @@ int SrsHlsMuxer::deviation()
     return deviation_ts;
 }
 
+SrsAudioCodecId SrsHlsMuxer::latest_acodec()
+{
+    // If current context writer exists, we query from it.
+    if (current && current->tscw) return current->tscw->acodec();
+
+    // Get the configured or updated config.
+    return latest_acodec_;
+}
+
+void SrsHlsMuxer::set_latest_acodec(SrsAudioCodecId v)
+{
+    // Refresh the codec in context writer for current segment.
+    if (current && current->tscw) current->tscw->set_acodec(v);
+
+    // Refresh the codec for future segments.
+    latest_acodec_ = v;
+}
+
 srs_error_t SrsHlsMuxer::initialize()
 {
     return srs_success;
@@ -371,6 +390,8 @@ srs_error_t SrsHlsMuxer::segment_open()
             srs_warn("hls: use aac for other codec=%s", default_acodec_str.c_str());
         }
     }
+    // Now that we know the latest audio codec in stream, use it.
+    if (latest_acodec_ != SrsAudioCodecIdForbidden) default_acodec = latest_acodec_;
     
     // load the default vcodec from config.
     SrsVideoCodecId default_vcodec = SrsVideoCodecIdAVC;
@@ -963,6 +984,13 @@ srs_error_t SrsHlsController::on_sequence_header()
 srs_error_t SrsHlsController::write_audio(SrsAudioFrame* frame, int64_t pts)
 {
     srs_error_t err = srs_success;
+
+    // Refresh the codec ASAP.
+    if (muxer->latest_acodec() != frame->acodec()->id) {
+        srs_trace("HLS: Switch audio codec %d(%s) to %d(%s)", muxer->latest_acodec(), srs_audio_codec_id2str(muxer->latest_acodec()).c_str(),
+            frame->acodec()->id, srs_audio_codec_id2str(frame->acodec()->id).c_str());
+        muxer->set_latest_acodec(frame->acodec()->id);
+    }
     
     // write audio to cache.
     if ((err = tsmc->cache_audio(frame, pts)) != srs_success) {

trunk/src/app/srs_app_hls.hpp

@@ -156,6 +156,9 @@ private:
     SrsHlsSegment* current;
     // The ts context, to keep cc continous between ts.
     SrsTsContext* context;
+private:
+    // Latest audio codec, parsed from stream.
+    SrsAudioCodecId latest_acodec_;
 public:
     SrsHlsMuxer();
     virtual ~SrsHlsMuxer();
@@ -166,6 +169,9 @@ public:
     virtual std::string ts_url();
     virtual srs_utime_t duration();
     virtual int deviation();
+public:
+    SrsAudioCodecId latest_acodec();
+    void set_latest_acodec(SrsAudioCodecId v);
 public:
     // Initialize the hls muxer.
     virtual srs_error_t initialize();

trunk/src/app/srs_app_http_stream.cpp

@@ -773,7 +773,9 @@ void SrsLiveStream::http_hooks_on_stop(ISrsHttpMessage* r)
 srs_error_t SrsLiveStream::streaming_send_messages(ISrsBufferEncoder* enc, SrsSharedPtrMessage** msgs, int nb_msgs)
 {
     srs_error_t err = srs_success;
-    
+
+    // TODO: In gop cache, we know both the audio and video codec, so we should notice the encoder, which might depends
+    // on setting the correct codec information, for example, HTTP-TS or HLS will write PMT.
     for (int i = 0; i < nb_msgs; i++) {
         SrsSharedPtrMessage* msg = msgs[i];
         

trunk/src/core/srs_core_version4.hpp

@@ -9,6 +9,6 @@
 
 #define VERSION_MAJOR       4
 #define VERSION_MINOR       0
-#define VERSION_REVISION    268
+#define VERSION_REVISION    269
 
 #endif

trunk/src/kernel/srs_kernel_codec.cpp

@@ -488,6 +488,9 @@ srs_error_t SrsFrame::initialize(SrsCodecConfig* c)
 srs_error_t SrsFrame::add_sample(char* bytes, int size)
 {
     srs_error_t err = srs_success;
+
+    // Ignore empty sample.
+    if (!bytes || size <= 0) return err;
     
     if (nb_samples >= SrsMaxNbSamples) {
         return srs_error_new(ERROR_HLS_DECODE_ERROR, "Frame samples overflow");
@@ -1407,20 +1410,13 @@ srs_error_t SrsFormat::audio_mp3_demux(SrsBuffer* stream, int64_t timestamp)
     // we always decode aac then mp3.
     srs_assert(acodec->id == SrsAudioCodecIdMP3);
     
-    // Update the RAW MP3 data.
+    // Update the RAW MP3 data. Note the start is 12 bits syncword 0xFFF, so we should not skip any bytes, for detail
+    // please see ISO_IEC_11172-3-MP3-1993.pdf page 20 and 26.
     raw = stream->data() + stream->pos();
     nb_raw = stream->size() - stream->pos();
     
-    stream->skip(1);
-    if (stream->empty()) {
-        return err;
-    }
-    
-    char* data = stream->data() + stream->pos();
-    int size = stream->size() - stream->pos();
-    
     // mp3 payload.
-    if ((err = audio->add_sample(data, size)) != srs_success) {
+    if ((err = audio->add_sample(raw, nb_raw)) != srs_success) {
         return srs_error_wrap(err, "add audio frame");
     }
     

trunk/src/kernel/srs_kernel_ts.cpp

@@ -2598,8 +2598,8 @@ SrsTsContextWriter::SrsTsContextWriter(ISrsStreamWriter* w, SrsTsContext* c, Srs
 {
     writer = w;
     context = c;
-    
-    acodec = ac;
+
+    acodec_ = ac;
     vcodec = vc;
 }
 
@@ -2614,7 +2614,7 @@ srs_error_t SrsTsContextWriter::write_audio(SrsTsMessage* audio)
     srs_info("hls: write audio pts=%" PRId64 ", dts=%" PRId64 ", size=%d",
         audio->pts, audio->dts, audio->PES_packet_length);
     
-    if ((err = context->encode(writer, audio, vcodec, acodec)) != srs_success) {
+    if ((err = context->encode(writer, audio, vcodec, acodec_)) != srs_success) {
         return srs_error_wrap(err, "ts: write audio");
     }
     srs_info("hls encode audio ok");
@@ -2629,7 +2629,7 @@ srs_error_t SrsTsContextWriter::write_video(SrsTsMessage* video)
     srs_info("hls: write video pts=%" PRId64 ", dts=%" PRId64 ", size=%d",
         video->pts, video->dts, video->PES_packet_length);
     
-    if ((err = context->encode(writer, video, vcodec, acodec)) != srs_success) {
+    if ((err = context->encode(writer, video, vcodec, acodec_)) != srs_success) {
         return srs_error_wrap(err, "ts: write video");
     }
     srs_info("hls encode video ok");
@@ -2642,6 +2642,16 @@ SrsVideoCodecId SrsTsContextWriter::video_codec()
     return vcodec;
 }
 
+SrsAudioCodecId SrsTsContextWriter::acodec()
+{
+    return acodec_;
+}
+
+void SrsTsContextWriter::set_acodec(SrsAudioCodecId v)
+{
+    acodec_ = v;
+}
+
 SrsEncFileWriter::SrsEncFileWriter()
 {
     memset(iv,0,16);
@@ -3079,6 +3089,13 @@ srs_error_t SrsTsTransmuxer::write_audio(int64_t timestamp, char* data, int size
     if (format->acodec->id == SrsAudioCodecIdAAC && format->audio->aac_packet_type == SrsAudioAacFrameTraitSequenceHeader) {
         return err;
     }
+
+    // Switch audio codec if not AAC.
+    if (tscw->acodec() != format->acodec->id) {
+        srs_trace("TS: Switch audio codec %d(%s) to %d(%s)", tscw->acodec(), srs_audio_codec_id2str(tscw->acodec()).c_str(),
+            format->acodec->id, srs_audio_codec_id2str(format->acodec->id).c_str());
+        tscw->set_acodec(format->acodec->id);
+    }
     
     // the dts calc from rtmp/flv header.
     // @remark for http ts stream, the timestamp is always monotonically increase,

trunk/src/kernel/srs_kernel_ts.hpp

@@ -97,7 +97,7 @@ enum SrsTsPidApply
     SrsTsPidApplyAudio, // vor audio
 };
 
-// Table 2-29 - Stream type assignments
+// Table 2-29 - Stream type assignments, hls-mpeg-ts-iso13818-1.pdf, page 66
 enum SrsTsStream
 {
     // ITU-T | ISO/IEC Reserved
@@ -106,8 +106,8 @@ enum SrsTsStream
     // ISO/IEC 11172 Video
     // ITU-T Rec. H.262 | ISO/IEC 13818-2 Video or ISO/IEC 11172-2 constrained parameter video stream
     // ISO/IEC 11172 Audio
+    SrsTsStreamAudioMp3 = 0x03,
     // ISO/IEC 13818-3 Audio
-    SrsTsStreamAudioMp3 = 0x04,
     // ITU-T Rec. H.222.0 | ISO/IEC 13818-1 private_sections
     // ITU-T Rec. H.222.0 | ISO/IEC 13818-1 PES packets containing private data
     // ISO/IEC 13522 MHEG
@@ -1243,7 +1243,7 @@ private:
     // User must config the codec in right way.
     // @see https://github.com/ossrs/srs/issues/301
     SrsVideoCodecId vcodec;
-    SrsAudioCodecId acodec;
+    SrsAudioCodecId acodec_;
 private:
     SrsTsContext* context;
     ISrsStreamWriter* writer;
@@ -1259,6 +1259,10 @@ public:
 public:
     // get the video codec of ts muxer.
     virtual SrsVideoCodecId video_codec();
+public:
+    // Get and set the audio codec.
+    SrsAudioCodecId acodec();
+    void set_acodec(SrsAudioCodecId v);
 };
 
 // Used for HLS Encryption

trunk/src/utest/srs_utest_kernel.cpp

@@ -3391,11 +3391,23 @@ VOID TEST(KernelCodecTest, AVFrame)
         EXPECT_TRUE(20 == f.samples[1].size);
         EXPECT_TRUE(2 == f.nb_samples);
 	}
+
+
+    if (true) {
+        SrsAudioFrame f;
+        EXPECT_TRUE(0 == f.nb_samples);
+
+        HELPER_EXPECT_SUCCESS(f.add_sample((char*)1, 0));
+        EXPECT_TRUE(0 == f.nb_samples);
+
+        HELPER_EXPECT_SUCCESS(f.add_sample(NULL, 1));
+        EXPECT_TRUE(0 == f.nb_samples);
+    }
     
     if (true) {
         SrsAudioFrame f;
         for (int i = 0; i < SrsMaxNbSamples; i++) {
-            HELPER_EXPECT_SUCCESS(f.add_sample((char*)(int64_t)i, i*10));
+            HELPER_EXPECT_SUCCESS(f.add_sample((char*)(int64_t)(i + 1), i*10 + 1));
         }
         
         srs_error_t err = f.add_sample((char*)1, 1);
@@ -3502,18 +3514,39 @@ VOID TEST(KernelCodecTest, AudioFormat)
         HELPER_EXPECT_SUCCESS(f.on_audio(0, (char*)"\x00", 0));
         HELPER_EXPECT_SUCCESS(f.on_audio(0, (char*)"\x00", 1));
     }
-    
+
+    // For MP3
     if (true) {
         SrsFormat f;
         HELPER_EXPECT_SUCCESS(f.initialize());
+        HELPER_EXPECT_SUCCESS(f.on_audio(0, (char*)"\x20", 1));
+        EXPECT_TRUE(0 == f.nb_raw);
+        EXPECT_TRUE(0 == f.audio->nb_samples);
+
         HELPER_EXPECT_SUCCESS(f.on_audio(0, (char*)"\x20\x00", 2));
         EXPECT_TRUE(1 == f.nb_raw);
-        EXPECT_TRUE(0 == f.audio->nb_samples);
+        EXPECT_TRUE(1 == f.audio->nb_samples);
         
         HELPER_EXPECT_SUCCESS(f.on_audio(0, (char*)"\x20\x00\x00", 3));
         EXPECT_TRUE(2 == f.nb_raw);
         EXPECT_TRUE(1 == f.audio->nb_samples);
     }
+
+    // For AAC
+    if (true) {
+        SrsFormat f;
+        HELPER_EXPECT_SUCCESS(f.initialize());
+        HELPER_EXPECT_FAILED(f.on_audio(0, (char*)"\xa0", 1));
+
+        HELPER_EXPECT_SUCCESS(f.on_audio(0, (char*)"\xaf\x00\x12\x10", 4));
+        HELPER_EXPECT_SUCCESS(f.on_audio(0, (char*)"\xa0\x01", 2));
+        EXPECT_TRUE(0 == f.nb_raw);
+        EXPECT_TRUE(0 == f.audio->nb_samples);
+
+        HELPER_EXPECT_SUCCESS(f.on_audio(0, (char*)"\xa0\x01\x00", 3));
+        EXPECT_TRUE(1 == f.nb_raw);
+        EXPECT_TRUE(1 == f.audio->nb_samples);
+    }
     
     if (true) {
         SrsFormat f;