From 70c59da1bfd41d08c1b958e65d101f20a4160687 Mon Sep 17 00:00:00 2001 From: winlin Date: Fri, 3 Apr 2015 23:17:50 +0800 Subject: [PATCH] enhanced avc decode, parse the sps get width+height. 2.0.156. --- README.md | 3 +- trunk/src/core/srs_core.hpp | 2 +- trunk/src/kernel/srs_kernel_codec.cpp | 243 +++++++++++++++++++++++- trunk/src/kernel/srs_kernel_codec.hpp | 5 + trunk/src/kernel/srs_kernel_error.hpp | 2 +- trunk/src/kernel/srs_kernel_stream.cpp | 34 ++++ trunk/src/kernel/srs_kernel_stream.hpp | 18 ++ trunk/src/kernel/srs_kernel_utility.cpp | 47 +++++ trunk/src/kernel/srs_kernel_utility.hpp | 5 + 9 files changed, 351 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index a8af61624..dac38a66e 100755 --- a/README.md +++ b/README.md @@ -562,7 +562,8 @@ Supported operating systems and hardware: ### SRS 2.0 history -* v2.0, 2015-03-30, for [#372](https://github.com/winlinvip/simple-rtmp-server/issues/372), support transform vhost of edge 2.0.155. +* v2.0, 2015-04-03, enhanced avc decode, parse the sps get width+height. 2.0.156. +* v2.0, 2015-04-03, for [#372](https://github.com/winlinvip/simple-rtmp-server/issues/372), support transform vhost of edge 2.0.155. * v2.0, 2015-03-30, for [#366](https://github.com/winlinvip/simple-rtmp-server/issues/366), config hls to disable cleanup of ts. 2.0.154. * v2.0, 2015-03-31, support server cycle handler. 2.0.153. * v2.0, 2015-03-31, support on_hls for http hooks. 2.0.152. diff --git a/trunk/src/core/srs_core.hpp b/trunk/src/core/srs_core.hpp index 390700d10..fc6dc8140 100644 --- a/trunk/src/core/srs_core.hpp +++ b/trunk/src/core/srs_core.hpp @@ -31,7 +31,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // current release version #define VERSION_MAJOR 2 #define VERSION_MINOR 0 -#define VERSION_REVISION 155 +#define VERSION_REVISION 156 // server info. #define RTMP_SIG_SRS_KEY "SRS" diff --git a/trunk/src/kernel/srs_kernel_codec.cpp b/trunk/src/kernel/srs_kernel_codec.cpp index f695b4f49..78299692f 100644 --- a/trunk/src/kernel/srs_kernel_codec.cpp +++ b/trunk/src/kernel/srs_kernel_codec.cpp @@ -31,6 +31,7 @@ using namespace std; #include #include #include +#include string srs_codec_video2str(SrsCodecVideo codec) { @@ -713,7 +714,8 @@ int SrsAvcAacCodec::avc_demux_sps_pps(SrsStream* stream) return ret; } - // 1 sps + // 1 sps, 7.3.2.1 Sequence parameter set RBSP syntax + // H.264-AVC-ISO_IEC_14496-10.pdf, page 45. if (!stream->require(1)) { ret = ERROR_HLS_DECODE_ERROR; srs_error("avc decode sequenc header sps failed. ret=%d", ret); @@ -740,8 +742,7 @@ int SrsAvcAacCodec::avc_demux_sps_pps(SrsStream* stream) if (sequenceParameterSetLength > 0) { srs_freep(sequenceParameterSetNALUnit); sequenceParameterSetNALUnit = new char[sequenceParameterSetLength]; - memcpy(sequenceParameterSetNALUnit, stream->data() + stream->pos(), sequenceParameterSetLength); - stream->skip(sequenceParameterSetLength); + stream->read_bytes(sequenceParameterSetNALUnit, sequenceParameterSetLength); } // 1 pps if (!stream->require(1)) { @@ -770,10 +771,242 @@ int SrsAvcAacCodec::avc_demux_sps_pps(SrsStream* stream) if (pictureParameterSetLength > 0) { srs_freep(pictureParameterSetNALUnit); pictureParameterSetNALUnit = new char[pictureParameterSetLength]; - memcpy(pictureParameterSetNALUnit, stream->data() + stream->pos(), pictureParameterSetLength); - stream->skip(pictureParameterSetLength); + stream->read_bytes(pictureParameterSetNALUnit, pictureParameterSetLength); } + return avc_demux_sps(); +} + +int SrsAvcAacCodec::avc_demux_sps() +{ + int ret = ERROR_SUCCESS; + + if (!sequenceParameterSetLength) { + return ret; + } + + SrsStream stream; + if ((ret = stream.initialize(sequenceParameterSetNALUnit, sequenceParameterSetLength)) != ERROR_SUCCESS) { + return ret; + } + + // for NALU, 7.3.1 NAL unit syntax + // H.264-AVC-ISO_IEC_14496-10.pdf, page 44. + if (!stream.require(1)) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("avc decode sps failed. ret=%d", ret); + return ret; + } + int8_t nutv = stream.read_1bytes(); + + // forbidden_zero_bit shall be equal to 0. + int8_t forbidden_zero_bit = (nutv >> 7) & 0x01; + if (forbidden_zero_bit) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("forbidden_zero_bit shall be equal to 0. ret=%d", ret); + return ret; + } + + // nal_ref_idc not equal to 0 specifies that the content of the NAL unit contains a sequence parameter set or a picture + // parameter set or a slice of a reference picture or a slice data partition of a reference picture. + int8_t nal_ref_idc = (nutv >> 5) & 0x03; + if (!nal_ref_idc) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("for sps, nal_ref_idc shall be not be equal to 0. ret=%d", ret); + return ret; + } + + // nal_unit_type specifies the type of RBSP data structure contained in the NAL unit as specified in Table 7-1. + // VCL NAL units are specified as those NAL units having nal_unit_type equal to 1, 2, 3, 4, 5, or 12. + // All remaining NAL units are called non-VCL NAL units. + int8_t nal_unit_type = nutv & 0x1f; + if (nal_unit_type != 7) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("for sps, nal_unit_type shall be equal to 7. ret=%d", ret); + return ret; + } + + // decode the rbsp from sps. + // rbsp[ i ] a raw byte sequence payload is specified as an ordered sequence of bytes. + int8_t* rbsp = new int8_t[sequenceParameterSetLength]; + SrsAutoFree(int8_t, rbsp); + + int nb_rbsp = 0; + while (!stream.empty()) { + rbsp[nb_rbsp] = stream.read_1bytes(); + + // XX 00 00 03 XX, the 03 byte should be drop. + if (nb_rbsp > 2 && rbsp[nb_rbsp - 2] == 0 && rbsp[nb_rbsp - 1] == 0 && rbsp[nb_rbsp] == 3) { + continue; + } + + nb_rbsp++; + } + + return avc_demux_sps_rbsp((char*)rbsp, nb_rbsp); +} + + +int SrsAvcAacCodec::avc_demux_sps_rbsp(char* rbsp, int nb_rbsp) +{ + int ret = ERROR_SUCCESS; + + // reparse the rbsp. + SrsStream stream; + if ((ret = stream.initialize(rbsp, nb_rbsp)) != ERROR_SUCCESS) { + return ret; + } + + // for SPS, 7.3.2.1.1 Sequence parameter set data syntax + // H.264-AVC-ISO_IEC_14496-10-2012.pdf, page 62. + if (!stream.require(3)) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("sps shall atleast 3bytes. ret=%d", ret); + return ret; + } + u_int8_t profile_idc = stream.read_1bytes(); + if (!profile_idc) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("sps the profile_idc invalid. ret=%d", ret); + return ret; + } + + int8_t flags = stream.read_1bytes(); + if (flags & 0x03) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("sps the flags invalid. ret=%d", ret); + return ret; + } + + u_int8_t level_idc = stream.read_1bytes(); + if (!level_idc) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("sps the level_idc invalid. ret=%d", ret); + return ret; + } + + SrsBitStream bs; + if ((ret = bs.initialize(&stream)) != ERROR_SUCCESS) { + return ret; + } + + int64_t seq_parameter_set_id = -1; + if ((ret = srs_avc_nalu_read_uev(&bs, seq_parameter_set_id)) != ERROR_SUCCESS) { + return ret; + } + if (seq_parameter_set_id < 0) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("sps the seq_parameter_set_id invalid. ret=%d", ret); + return ret; + } + srs_info("sps parse profile=%d, level=%d, sps_id=%d", profile_idc, level_idc, seq_parameter_set_id); + + if (profile_idc == 100 || profile_idc == 110 || profile_idc == 122 || profile_idc == 244 + || profile_idc == 44 || profile_idc == 83 || profile_idc == 86 || profile_idc == 118 + || profile_idc == 128 + ) { + int64_t chroma_format_idc = -1; + if ((ret = srs_avc_nalu_read_uev(&bs, chroma_format_idc)) != ERROR_SUCCESS) { + return ret; + } + if (chroma_format_idc == 3) { + int8_t separate_colour_plane_flag = -1; + if ((ret = srs_avc_nalu_read_bit(&bs, separate_colour_plane_flag)) != ERROR_SUCCESS) { + return ret; + } + } + + int64_t bit_depth_luma_minus8 = -1; + if ((ret = srs_avc_nalu_read_uev(&bs, bit_depth_luma_minus8)) != ERROR_SUCCESS) { + return ret; + } + + int64_t bit_depth_chroma_minus8 = -1; + if ((ret = srs_avc_nalu_read_uev(&bs, bit_depth_chroma_minus8)) != ERROR_SUCCESS) { + return ret; + } + + int8_t qpprime_y_zero_transform_bypass_flag = -1; + if ((ret = srs_avc_nalu_read_bit(&bs, qpprime_y_zero_transform_bypass_flag)) != ERROR_SUCCESS) { + return ret; + } + + int8_t seq_scaling_matrix_present_flag = -1; + if ((ret = srs_avc_nalu_read_bit(&bs, seq_scaling_matrix_present_flag)) != ERROR_SUCCESS) { + return ret; + } + if (seq_scaling_matrix_present_flag) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("sps the seq_scaling_matrix_present_flag invalid. ret=%d", ret); + return ret; + } + } + + int64_t log2_max_frame_num_minus4 = -1; + if ((ret = srs_avc_nalu_read_uev(&bs, log2_max_frame_num_minus4)) != ERROR_SUCCESS) { + return ret; + } + + int64_t pic_order_cnt_type = -1; + if ((ret = srs_avc_nalu_read_uev(&bs, pic_order_cnt_type)) != ERROR_SUCCESS) { + return ret; + } + + if (pic_order_cnt_type == 0) { + int64_t log2_max_pic_order_cnt_lsb_minus4 = -1; + if ((ret = srs_avc_nalu_read_uev(&bs, log2_max_pic_order_cnt_lsb_minus4)) != ERROR_SUCCESS) { + return ret; + } + } else if (pic_order_cnt_type == 1) { + int8_t delta_pic_order_always_zero_flag = -1; + if ((ret = srs_avc_nalu_read_bit(&bs, delta_pic_order_always_zero_flag)) != ERROR_SUCCESS) { + return ret; + } + + int64_t offset_for_non_ref_pic = -1; + if ((ret = srs_avc_nalu_read_uev(&bs, offset_for_non_ref_pic)) != ERROR_SUCCESS) { + return ret; + } + + int64_t offset_for_top_to_bottom_field = -1; + if ((ret = srs_avc_nalu_read_uev(&bs, offset_for_top_to_bottom_field)) != ERROR_SUCCESS) { + return ret; + } + + int64_t num_ref_frames_in_pic_order_cnt_cycle = -1; + if ((ret = srs_avc_nalu_read_uev(&bs, num_ref_frames_in_pic_order_cnt_cycle)) != ERROR_SUCCESS) { + return ret; + } + if (num_ref_frames_in_pic_order_cnt_cycle) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("sps the num_ref_frames_in_pic_order_cnt_cycle invalid. ret=%d", ret); + return ret; + } + } + + int64_t max_num_ref_frames = -1; + if ((ret = srs_avc_nalu_read_uev(&bs, max_num_ref_frames)) != ERROR_SUCCESS) { + return ret; + } + + int8_t gaps_in_frame_num_value_allowed_flag = -1; + if ((ret = srs_avc_nalu_read_bit(&bs, gaps_in_frame_num_value_allowed_flag)) != ERROR_SUCCESS) { + return ret; + } + + int64_t pic_width_in_mbs_minus1 = -1; + if ((ret = srs_avc_nalu_read_uev(&bs, pic_width_in_mbs_minus1)) != ERROR_SUCCESS) { + return ret; + } + + int64_t pic_height_in_map_units_minus1 = -1; + if ((ret = srs_avc_nalu_read_uev(&bs, pic_height_in_map_units_minus1)) != ERROR_SUCCESS) { + return ret; + } + + width = (int)(pic_width_in_mbs_minus1 + 1) * 16; + height = (int)(pic_height_in_map_units_minus1 + 1) * 16; + return ret; } diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index b619bb9e1..ce174dc3a 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -580,6 +580,11 @@ private: * decode the sps and pps. */ virtual int avc_demux_sps_pps(SrsStream* stream); + /** + * decode the sps rbsp stream. + */ + virtual int avc_demux_sps(); + virtual int avc_demux_sps_rbsp(char* rbsp, int nb_rbsp); /** * demux the avc NALU in "AnnexB" * from H.264-AVC-ISO_IEC_14496-10.pdf, page 211. diff --git a/trunk/src/kernel/srs_kernel_error.hpp b/trunk/src/kernel/srs_kernel_error.hpp index 8d87eae1f..1b142dae7 100644 --- a/trunk/src/kernel/srs_kernel_error.hpp +++ b/trunk/src/kernel/srs_kernel_error.hpp @@ -215,7 +215,6 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #define ERROR_HTTP_DVR_CREATE_REQUEST 3053 #define ERROR_HTTP_DVR_NO_TAEGET 3054 #define ERROR_ADTS_ID_NOT_AAC 3055 - // HDS error code #define ERROR_HDS_OPEN_F4M_FAILED 3056 #define ERROR_HDS_WRITE_F4M_FAILED 3057 @@ -254,6 +253,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #define ERROR_STREAM_CASTER_FLV_TAG 4024 #define ERROR_HTTP_RESPONSE_EOF 4025 #define ERROR_HTTP_INVALID_CHUNK_HEADER 4026 +#define ERROR_AVC_NALU_UEV 4027 /////////////////////////////////////////////////////// // user-define error. diff --git a/trunk/src/kernel/srs_kernel_stream.cpp b/trunk/src/kernel/srs_kernel_stream.cpp index cacfafd11..eaf70a2f2 100644 --- a/trunk/src/kernel/srs_kernel_stream.cpp +++ b/trunk/src/kernel/srs_kernel_stream.cpp @@ -252,4 +252,38 @@ void SrsStream::write_bytes(char* data, int size) p += size; } +SrsBitStream::SrsBitStream() +{ + cb = 0; + cb_left = 0; + stream = NULL; +} + +SrsBitStream::~SrsBitStream() +{ +} + +int SrsBitStream::initialize(SrsStream* s) { + stream = s; + return ERROR_SUCCESS; +} + +bool SrsBitStream::empty() { + if (cb_left) { + return false; + } + return stream->empty(); +} + +int8_t SrsBitStream::read_bit() { + if (!cb_left) { + srs_assert(!stream->empty()); + cb = stream->read_1bytes(); + cb_left = 8; + } + + int8_t v = (cb >> (cb_left - 1)) & 0x01; + cb_left--; + return v; +} diff --git a/trunk/src/kernel/srs_kernel_stream.hpp b/trunk/src/kernel/srs_kernel_stream.hpp index 937e28f97..a708bb35e 100644 --- a/trunk/src/kernel/srs_kernel_stream.hpp +++ b/trunk/src/kernel/srs_kernel_stream.hpp @@ -154,4 +154,22 @@ public: virtual void write_bytes(char* data, int size); }; +/** + * the bit stream. + */ +class SrsBitStream +{ +private: + int8_t cb; + u_int8_t cb_left; + SrsStream* stream; +public: + SrsBitStream(); + virtual ~SrsBitStream(); +public: + virtual int initialize(SrsStream* s); + virtual bool empty(); + virtual int8_t read_bit(); +}; + #endif diff --git a/trunk/src/kernel/srs_kernel_utility.cpp b/trunk/src/kernel/srs_kernel_utility.cpp index 218ac401d..6e1a50b90 100644 --- a/trunk/src/kernel/srs_kernel_utility.cpp +++ b/trunk/src/kernel/srs_kernel_utility.cpp @@ -46,6 +46,53 @@ using namespace std; // @see SRS_SYS_TIME_RESOLUTION_MS_TIMES #define SYS_TIME_RESOLUTION_US 300*1000 +int srs_avc_nalu_read_uev(SrsBitStream* stream, int64_t& v) +{ + int ret = ERROR_SUCCESS; + + if (stream->empty()) { + return ERROR_AVC_NALU_UEV; + } + + // ue(v) in 9.1 Parsing process for Exp-Golomb codes + // H.264-AVC-ISO_IEC_14496-10-2012.pdf, page 227. + // Syntax elements coded as ue(v), me(v), or se(v) are Exp-Golomb-coded. + // leadingZeroBits = -1; + // for( b = 0; !b; leadingZeroBits++ ) + // b = read_bits( 1 ) + // The variable codeNum is then assigned as follows: + // codeNum = (2<empty(); leadingZeroBits++) { + b = stream->read_bit(); + } + + if (leadingZeroBits >= 64) { + return ERROR_AVC_NALU_UEV; + } + + v = (1 << leadingZeroBits) - 1; + for (int i = 0; i < leadingZeroBits; i++) { + int64_t b = stream->read_bit(); + v += b << (leadingZeroBits - 1); + } + + return ret; +} + +int srs_avc_nalu_read_bit(SrsBitStream* stream, int8_t& v) +{ + int ret = ERROR_SUCCESS; + + if (stream->empty()) { + return ERROR_AVC_NALU_UEV; + } + + v = stream->read_bit(); + + return ret; +} + static int64_t _srs_system_time_us_cache = 0; static int64_t _srs_system_time_startup_time = 0; diff --git a/trunk/src/kernel/srs_kernel_utility.hpp b/trunk/src/kernel/srs_kernel_utility.hpp index 7317759cc..a23474afa 100644 --- a/trunk/src/kernel/srs_kernel_utility.hpp +++ b/trunk/src/kernel/srs_kernel_utility.hpp @@ -33,11 +33,16 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #include class SrsStream; +class SrsBitStream; // compare #define srs_min(a, b) (((a) < (b))? (a) : (b)) #define srs_max(a, b) (((a) < (b))? (b) : (a)) +// read nalu uev. +extern int srs_avc_nalu_read_uev(SrsBitStream* stream, int64_t& v); +extern int srs_avc_nalu_read_bit(SrsBitStream* stream, int8_t& v); + // get current system time in ms, use cache to avoid performance problem extern int64_t srs_get_system_time_ms(); extern int64_t srs_get_system_startup_time_ms();