lal/pkg/hls/muxer.go

// Copyright 2020, Chef.  All rights reserved.
// https://github.com/q191201771/lal
//
// Use of this source code is governed by a MIT-style license
// that can be found in the License file.
//
// Author: Chef (191201771@qq.com)

package hls

import (
	"bytes"
	"fmt"
	"os"

	"github.com/q191201771/naza/pkg/unique"

	"github.com/q191201771/lal/pkg/aac"

	"github.com/q191201771/lal/pkg/rtmp"
	"github.com/q191201771/naza/pkg/bele"
	"github.com/q191201771/naza/pkg/nazalog"
)

// 记录fragment的一些信息，注意，写m3u8文件时可能还需要用到历史fragment的信息
type fragmentInfo struct {
	id       int     // fragment的自增序号
	duration float64 // 当前fragment中数据的时长，单位秒
	discont  bool    // #EXT-X-DISCONTINUITY
}

type MuxerConfig struct {
	OutPath            string `json:"out_path"`
	FragmentDurationMS int    `json:"fragment_duration_ms"`
	FragmentNum        int    `json:"fragment_num"`
}

type Muxer struct {
	UniqueKey string

	streamName          string
	outPath             string
	playlistFilename    string
	playlistFilenameBak string

	config *MuxerConfig

	fragmentOP FragmentOP
	opened     bool
	adts       aac.ADTS
	spspps     []byte
	videoCC    uint8 //视频连续计数器
	audioCC    uint8 //音频连续计数器
	videoOut   []byte // 帧

	fragTS uint64 // 新建立fragment时的时间戳，毫秒 * 90

	nfrags int            //1~config.FragmentNum(winfrags)，增长到winfrags后就不变了，新分片就增长frag了
	frag   int            // 写入m3u8的EXT-X-MEDIA-SEQUENCE字段
	frags  []fragmentInfo // TS文件的环形队列，记录TS的信息，比如写M3U8文件时要用 2 * winfrags + 1

	aaframe   []byte
	aframePTS uint64 // 最新音频帧的时间戳
}

func NewMuxer(streamName string, config *MuxerConfig) *Muxer {
	uk := unique.GenUniqueKey("HLSMUXER")
	nazalog.Infof("lifecycle new hls muxer. [%s] streamName=%s", uk, streamName)

	op := getMuxerOutPath(config.OutPath, streamName)
	playlistFilename := getM3U8Filename(op, streamName)
	playlistFilenameBak := fmt.Sprintf("%s.bak", playlistFilename)
	videoOut := make([]byte, 1024*1024)
	videoOut = videoOut[0:0]
	frags := make([]fragmentInfo, 2*config.FragmentNum+1) // TODO chef: 为什么是 * 2 + 1
	return &Muxer{
		UniqueKey:           uk,
		streamName:          streamName,
		outPath:             op,
		playlistFilename:    playlistFilename,
		playlistFilenameBak: playlistFilenameBak,
		config:              config,
		videoOut:            videoOut,
		aaframe:             nil,
		frags:               frags,
	}
}

func (m *Muxer) Start() {
	nazalog.Infof("start hls muxer. [%s]", m.UniqueKey)
	m.ensureDir()
}

func (m *Muxer) Dispose() {
	nazalog.Infof("lifecycle dispose hls muxer. [%s]", m.UniqueKey)
	m.flushAudio()
	m.closeFragment()
}

func (m *Muxer) FeedRTMPMessage(msg rtmp.AVMsg) {
	switch msg.Header.MsgTypeID {
	case rtmp.TypeidAudio:
		m.feedAudio(msg)
	case rtmp.TypeidVideo:
		m.feedVideo(msg)
	}
}

// TODO chef: 可以考虑数据有问题时，返回给上层，直接主动关闭输入流的连接
func (m *Muxer) feedVideo(msg rtmp.AVMsg) {
	if len(msg.Payload) < 5 {
		nazalog.Errorf("invalid video message length. [%s] len=%d", m.UniqueKey, len(msg.Payload))
		return
	}
	/*
	@see: E.4.3 Video Tags, video_file_format_spec_v10_1.pdf, page 78
	Frame Type, Type of video frame.
	CodecID, Codec Identifier.
	set the rtmp header
	*/
	if msg.Payload[0]&0xF != 7 {
		// TODO chef: HLS视频现在只做了h264的支持
		return
	}

	/*
	Frame Type
	Type of video frame. The following values are defined:
	1 = key frame (for AVC, a seekable frame)
	2 = inter frame (for AVC, a non-seekable frame)
	3 = disposable inter frame (H.263 only)
	4 = generated key frame (reserved for server use only)
	5 = video info/command frame
	*/
	ftype := msg.Payload[0] & 0xF0 >> 4
	/*
	AVCPacketType
	0 = AVC sequence header
	1 = AVC NALU
	2 = AVC end of sequence (lower level NALU sequence ender is
	 */
	htype := msg.Payload[1]

	//当前msg是SPS_PPS缓存它们
	if ftype == 1 && htype == 0 {
		m.cacheSPSPPS(msg)
		return
	}
	//Composition time offset
	cts := bele.BEUint24(msg.Payload[2:])

	audSent := false
	spsppsSent := false
	// 优化这块buffer
	out := m.videoOut[0:0]
	/*
	一个msg里面可能会有多个NAL单元数据(一个ES + SEI/SPS_PPS)或多个ES帧数据(这种应该算是一种错误)
	正常情况一个msg只会包括一个ES帧I/P/B或一个NAL单元数据或一个SEI + 一个ES
	不会有msg=SPS_PPS+ES这种情况,因为它们专门msg传递
	注:
		如果要兼容多个ES帧在一个msg中这种情况,TS封装标准没有禁止多个帧封装到一个PES段中
		但是如果这么做会出现这几个帧就会使用相同的PTS/DTS,至于终端能否正常显示就看终端的兼容性了。
		或者考虑把这个msg中的多个帧分开封装，但是需要去调整PTS/DTS(要不然这几个帧还是相同的PTS/DTS)
	 */
	for i := 5; i != len(msg.Payload); {
		if i+4 > len(msg.Payload) {
			nazalog.Errorf("slice len not enough. [%s] i=%d, len=%d", m.UniqueKey, i, len(msg.Payload))
			return
		}
		/*
		NALUnitLength
		5.3.4.2.1 Syntax, ISO_IEC_14496-15-AVC-format-2012.pdf, page 16
		lengthSizeMinusOne, or NAL_unit_length, always use 4bytes size
		mux the avc NALU in "ISO Base Media File Format"
		from ISO_IEC_14496-15-AVC-format-2012.pdf, page 20
		*/
		nalBytes := int(bele.BEUint32(msg.Payload[i:]))
		i += 4
		if i+nalBytes > len(msg.Payload) {
			nazalog.Errorf("slice len not enough. [%s] i=%d, payload len=%d, nalBytes=%d", m.UniqueKey, i, len(msg.Payload), nalBytes)
			return
		}
		/*
		msg.Payload[9],这里开始是NAL单元数据(无pic_start_code)
		nal_unit_type，NAL单元语法见“H.264官方中文版.pdf”pg54
		 */
		srcNalType := msg.Payload[i]
		nalType := srcNalType & 0x1F
		//nazalog.Debugf("hls: h264 NAL type=%d, len=%d(%d) cts=%d.", nalType, nalBytes, len(msg.Payload), cts)

		/*
		如果NAL单元是sps_pps_aud则跳过处理
		因为前面m.cacheSPSPPS(msg)有专门的msg传递sps_pps且已经缓存了
		 */
		if nalType >= 7 && nalType <= 9 {
			//nazalog.Warn("should not reach here.")
			i += nalBytes
			continue
		}

		/*
		在P/IDR/SEI前面插入AUD数据，每个MSG中如果有多个NAL单元保证只插入一次AUD
		nal_unit_type
		0 = B SLICE types
		1 = P SLICE types
		5 = IDR SLICE types
		6 = SEI types
		7 = SPS types
		8 = PPS types
		*/
		if !audSent {
			switch nalType {
			case 1, 5, 6:
				out = append(out, audNal...)
				audSent = true
			case 9:
				//前面AUD数据已经跳过了
				audSent = true
			}
		}
		/*
		每个IDR帧前插入SPS_PPS，如果一个MSG中多个I帧则每个都需要插入
		*/
		switch nalType {
		case 1:
			spsppsSent = false //I后面有P，P后面还有I，继续插入
		case 5:
			if !spsppsSent {
				out = m.appendSPSPPS(out)
			}
			spsppsSent = true
		}

		//添加pic_start_code
		if len(out) == 0 {
			//其它未知NAL单元数据也保存，正常情况不会到这里
			out = append(out, nalStartCode...)
		} else {
			out = append(out, nalStartCode3...)
		}
		//保存NAL单元数据
		out = append(out, msg.Payload[i:i+nalBytes]...)
		//下一个NAL单元
		i += nalBytes
	}

	var frame mpegTSFrame
	frame.cc = m.videoCC
	//rmtp_timestamp单位是ms,TS的pts/dts单位是90k时钟,dts=timestamp*1000/90000
	frame.dts = uint64(msg.Header.TimestampAbs) * 90
	frame.pts = frame.dts + uint64(cts)*90
	frame.pid = PidVideo
	frame.sid = streamIDVideo
	frame.key = ftype == 1

	/*
	程序刚运行还未创建第1个分片文件时，等到I帧来时boundary=true让updateFragment中去创建第1个分片
	如果先来的是非I帧的其它数据则丢弃(因为updateFragment中不会创建分片)
	*/
	boundary := frame.key && (!m.opened || m.adts.IsNil() || m.aaframe != nil)
	//更新分片信息和playlist文件
	m.updateFragment(frame.dts, boundary, 1)

	if !m.opened {
		nazalog.Warnf("not opened. [%s]", m.UniqueKey)
		return
	}
	//一个ES或一个SEI+ES或多个ES,封装成TS,并写入当前分片文件
	m.fragmentOP.WriteFrame(&frame, out)
	//保存WriteFrame中修改的连续计数器给下次调用WriteFrame时使用
	m.videoCC = frame.cc
}

func (m *Muxer) feedAudio(msg rtmp.AVMsg) {
	if len(msg.Payload) < 3 {
		nazalog.Errorf("invalid audio message length. [%s] len=%d", m.UniqueKey, len(msg.Payload))
	}
	if msg.Payload[0]>>4 != 10 {
		return
	}

	if msg.Payload[1] == 0 {
		m.cacheAACSeqHeader(msg)
		return
	}

	if m.adts.IsNil() {
		nazalog.Warnf("feed audio message but aac seq header not exist. [%s]", m.UniqueKey)
		return
	}

	pts := uint64(msg.Header.TimestampAbs) * 90

	m.updateFragment(pts, m.spspps == nil, 2)

	if m.aaframe == nil {
		m.aframePTS = pts
	}

	adtsHeader, _ := m.adts.GetADTS(uint16(msg.Header.MsgLen))
	m.aaframe = append(m.aaframe, adtsHeader...)
	m.aaframe = append(m.aaframe, msg.Payload[2:]...)
}

func (m *Muxer) cacheAACSeqHeader(msg rtmp.AVMsg) {
	_ = m.adts.PutAACSequenceHeader(msg.Payload)
}

func (m *Muxer) cacheSPSPPS(msg rtmp.AVMsg) {
	m.spspps = msg.Payload
}

func (m *Muxer) appendSPSPPS(out []byte) []byte {
	if m.spspps == nil {
		nazalog.Warnf("append spspps by not exist. [%s]", m.UniqueKey)
		return out
	}

	index := 10
	nnals := m.spspps[index] & 0x1f
	index++
	for n := 0; ; n++ {
		for ; nnals != 0; nnals-- {
			length := int(bele.BEUint16(m.spspps[index:]))
			index += 2
			out = append(out, nalStartCode...)
			out = append(out, m.spspps[index:index+length]...)
			index += length
		}

		if n == 1 {
			break
		}
		nnals = m.spspps[index]
		index++
	}
	return out
}

func (m *Muxer) updateFragment(ts uint64, boundary bool, flushRate int) {
	force := false
	discont := true
	var f *fragmentInfo

	if m.opened {
		f = m.getFrag(m.nfrags)

		// 当前时间戳跳跃很大，或者是往回跳跃超过了阈值，强制开启新的fragment
		maxfraglen := uint64(m.config.FragmentDurationMS * 90 * 10)
		if (ts > m.fragTS && ts-m.fragTS > maxfraglen) || (m.fragTS > ts && m.fragTS-ts > negMaxfraglen) {
			nazalog.Warnf("force fragment split. [%s] fragTS=%d, ts=%d", m.UniqueKey, m.fragTS, ts)
			force = true
		} else {
			// TODO chef: 考虑ts比fragTS小的情况
			f.duration = float64(ts-m.fragTS) / 90000
			discont = false
		}
	}

	// 时长超过设置的ts文件切片阈值才行
	if f != nil && f.duration < float64(m.config.FragmentDurationMS)/1000 {
		boundary = false
	}

	// 开启新的fragment
	if boundary || force {
		m.closeFragment()
		m.openFragment(ts, discont)
	}

	// 音频已经缓存了一定时长的数据了，需要落盘了
	if m.opened && m.aaframe != nil && ((m.aframePTS + maxAudioDelay*90/uint64(flushRate)) < ts) {
		m.flushAudio()
	}
}
/*
创建一个新TS分片文件
ts: 当前帧的时间pts/dts
discont:
*/
func (m *Muxer) openFragment(ts uint64, discont bool) {
	if m.opened {
		return
	}
	//文件序号，一直增加唯一数值
	id := m.getFragmentID()

	filename := getTSFilename(m.outPath, m.streamName, id)
	_ = m.fragmentOP.OpenFile(filename)
	m.opened = true

	frag := m.getFrag(m.nfrags)
	frag.discont = discont
	frag.id = id
	//新分片的开始时间
	m.fragTS = ts
	//把好个分片还未写入文件的音频帧写入新分片中
	m.flushAudio()
}

func (m *Muxer) closeFragment() {
	if !m.opened {
		return
	}
	//关闭当前分片文件
	m.fragmentOP.CloseFile()
	m.opened = false
	m.clearTS()
	//更新序号，为下个分片准备好
	m.nextFrag()
	//一个分片完成后，把这个分片文件名写入playlist文件中
	m.writePlaylist()
}
//ljy added.删除没有在playlist中的TS文件
func (m *Muxer) clearTS() {
	//frag>0表示已经有超过config.FragmentNum个文件产生，可以删除了
	if m.frag > 0 {
		name := getTSFilename(m.outPath, m.streamName, m.frag - 1)
		nazalog.Println("delete ts file: ",name)
		if os.Remove(name) != nil {
			nazalog.Warnf("delete ts file %s error",name)
		}
	}
}
func (m *Muxer) writePlaylist() {
	fp, err := os.Create(m.playlistFilenameBak)
	nazalog.Assert(nil, err)

	// 找出时长最长的fragment
	maxFrag := float64(m.config.FragmentDurationMS) / 1000
	for i := 0; i < m.nfrags; i++ {
		frag := m.getFrag(i)
		if frag.duration > maxFrag {
			maxFrag = frag.duration + 0.5
		}
	}

	// TODO chef 优化这块buffer的构造
	var buf bytes.Buffer
	buf.WriteString("#EXTM3U\n")
	buf.WriteString("#EXT-X-VERSION:3\n")
	buf.WriteString("#EXT-X-ALLOW-CACHE:NO\n")
	buf.WriteString(fmt.Sprintf("#EXT-X-TARGETDURATION:%d\n", int(maxFrag)))
	buf.WriteString(fmt.Sprintf("#EXT-X-MEDIA-SEQUENCE:%d\n\n", m.frag))

	//写入当前config.FragmentNum(winfrags)内的所有分片时长和文件名信息
	for i := 0; i < m.nfrags; i++ {
		frag := m.getFrag(i)
		if frag.discont {
			buf.WriteString("#EXT-X-DISCONTINUITY\n")
		}
		buf.WriteString(fmt.Sprintf("#EXTINF:%.3f,\n%s\n", frag.duration, getTSFilenameWithoutPath(m.streamName, frag.id)))
	}

	_, err = fp.Write(buf.Bytes())
	nazalog.Assert(nil, err)
	_ = fp.Close()
	err = os.Rename(m.playlistFilenameBak, m.playlistFilename)
	nazalog.Assert(nil, err)
}

// 创建文件夹，如果文件夹已经存在，老的文件夹会被删除
func (m *Muxer) ensureDir() {
	err := os.RemoveAll(m.outPath)
	nazalog.Assert(nil, err)
	err = os.MkdirAll(m.outPath, 0777)
	nazalog.Assert(nil, err)
}

func (m *Muxer) getFragmentID() int {
	return m.frag + m.nfrags
}

func (m *Muxer) getFrag(n int) *fragmentInfo {
	return &m.frags[(m.frag+n)%(m.config.FragmentNum*2+1)]
}

// TODO chef: 这个函数重命名为incr更好些
func (m *Muxer) nextFrag() {
	if m.nfrags == m.config.FragmentNum {
		m.frag++
	} else {
		//当nfrags==m.config.FragmentNum时就不会执行这里了
		m.nfrags++
	}
}

// 将音频数据落盘的几种情况：
// 1. open fragment时，如果aframe中还有数据
// 2. update fragment时，判断音频的时间戳
// 3. 音频队列长度过长时
// 4. 流关闭时
func (m *Muxer) flushAudio() {
	if !m.opened {
		nazalog.Warnf("flushAudio by not opened. [%s]", m.UniqueKey)
		return
	}

	if m.aaframe == nil {
		return
	}

	frame := &mpegTSFrame{
		pts: m.aframePTS,
		dts: m.aframePTS,
		pid: PidAudio,
		sid: streamIDAudio,
		cc:  m.audioCC,
		key: false,
	}

	m.fragmentOP.WriteFrame(frame, m.aaframe)

	m.audioCC = frame.cc
	m.aaframe = nil
}