make webrtc audio work

pull/1658/head
七曦 5 years ago
parent 68ad006b73
commit a0a4337214

22
trunk/configure vendored

@ -150,6 +150,8 @@ LibSTRoot="${SRS_OBJS_DIR}/st"; LibSTfile="${LibSTRoot}/libst.a"
if [[ $SRS_SHARED_ST == YES ]]; then LibSTfile="-lst"; fi
# srtp
LibSrtpRoot="${SRS_OBJS_DIR}/srtp2/include"; LibSrtpFile="${SRS_OBJS_DIR}/srtp2/lib/libsrtp2.a"
# ffmpeg
LibFfmpegRoot="${SRS_OBJS_DIR}/ffmpeg/include"; LibFfmpegFile="${SRS_OBJS_DIR}/ffmpeg/lib/libavcodec.a ${SRS_OBJS_DIR}/ffmpeg/lib/libswresample.a ${SRS_OBJS_DIR}/ffmpeg/lib/libavutil.a ${SRS_OBJS_DIR}/ffmpeg/lib/libopus.a -lpthread"
# openssl-1.1.0e, for the RTMP complex handshake.
LibSSLRoot="";LibSSLfile=""
if [[ $SRS_SSL == YES && $SRS_USE_SYS_SSL == NO ]]; then
@ -235,7 +237,7 @@ fi
if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then
MODULE_ID="SERVICE"
MODULE_DEPENDS=("CORE" "KERNEL" "PROTOCOL")
ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${SRS_OBJS_DIR} ${LibSSLRoot})
ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${LibFfmpegRoot} ${SRS_OBJS_DIR} ${LibSSLRoot})
MODULE_FILES=("srs_service_log" "srs_service_st" "srs_service_http_client"
"srs_service_http_conn" "srs_service_rtmp_conn" "srs_service_utility"
"srs_service_conn")
@ -248,7 +250,7 @@ fi
if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then
MODULE_ID="APP"
MODULE_DEPENDS=("CORE" "KERNEL" "PROTOCOL" "SERVICE")
ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${SRS_OBJS_DIR} ${LibSSLRoot})
ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${LibFfmpegRoot} ${SRS_OBJS_DIR} ${LibSSLRoot})
MODULE_FILES=("srs_app_server" "srs_app_conn" "srs_app_rtmp_conn" "srs_app_source"
"srs_app_refer" "srs_app_hls" "srs_app_forward" "srs_app_encoder" "srs_app_http_stream"
"srs_app_thread" "srs_app_bandwidth" "srs_app_st" "srs_app_log" "srs_app_config"
@ -259,7 +261,7 @@ if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then
"srs_app_mpegts_udp" "srs_app_rtp" "srs_app_rtc_conn" "srs_app_dtls" "srs_app_rtsp" "srs_app_listener" "srs_app_async_call"
"srs_app_caster_flv" "srs_app_process" "srs_app_ng_exec"
"srs_app_hourglass" "srs_app_dash" "srs_app_fragment" "srs_app_dvr"
"srs_app_coworkers" "srs_app_hybrid")
"srs_app_coworkers" "srs_app_hybrid" "srs_app_audio_recode")
DEFINES=""
# add each modules for app
for SRS_MODULE in ${SRS_MODULES[*]}; do
@ -286,7 +288,7 @@ if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then
if [[ $SRS_SRT == YES ]]; then
MODULE_DEPENDS+=("SRT")
fi
ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${SRS_OBJS_DIR} ${LibGperfRoot} ${LibSSLRoot})
ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${LibFfmpegRoot} ${SRS_OBJS_DIR} ${LibGperfRoot} ${LibSSLRoot})
if [[ $SRS_SRT == YES ]]; then
ModuleLibIncs+=("${LibSRTRoot[*]}")
fi
@ -299,7 +301,7 @@ fi
if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then
MODULE_ID="MAIN"
MODULE_DEPENDS=("CORE" "KERNEL" "PROTOCOL" "SERVICE")
ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${SRS_OBJS_DIR} ${LibGperfRoot} ${LibSSLRoot})
ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${LibFfmpegRoot} ${SRS_OBJS_DIR} ${LibGperfRoot} ${LibSSLRoot})
MODULE_FILES=()
DEFINES=""
# add each modules for main
@ -326,13 +328,13 @@ if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then
done
#
# all depends libraries
ModuleLibFiles=(${LibSTfile} ${LibSrtpFile} ${LibSSLfile} ${LibGperfFile})
ModuleLibFiles=(${LibSTfile} ${LibSrtpFile} ${LibFfmpegFile} ${LibSSLfile} ${LibGperfFile})
if [[ $SRS_SRT == YES ]]; then
ModuleLibFiles+=("${LibSRTfile[*]}")
fi
# all depends objects
MODULE_OBJS="${CORE_OBJS[@]} ${KERNEL_OBJS[@]} ${PROTOCOL_OBJS[@]} ${SERVICE_OBJS[@]} ${APP_OBJS[@]} ${SERVER_OBJS[@]}"
ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${SRS_OBJS_DIR} ${LibGperfRoot} ${LibSSLRoot})
ModuleLibIncs=(${LibSTRoot} ${LibSrtpRoot} ${LibFfmpegRoot} ${SRS_OBJS_DIR} ${LibGperfRoot} ${LibSSLRoot})
if [[ $SRS_SRT == YES ]]; then
MODULE_OBJS="${MODULE_OBJS} ${SRT_OBJS[@]}"
fi
@ -343,7 +345,7 @@ if [ $SRS_EXPORT_LIBRTMP_PROJECT = NO ]; then
#
# For modules, without the app module.
MODULE_OBJS="${CORE_OBJS[@]} ${KERNEL_OBJS[@]} ${PROTOCOL_OBJS[@]} ${SERVICE_OBJS[@]} ${MAIN_OBJS[@]}"
ModuleLibFiles=(${LibSTfile} ${LibSrtpFile} ${LibSSLfile} ${LibGperfFile})
ModuleLibFiles=(${LibSTfile} ${LibSrtpFile} ${LibFfmpegFile} ${LibSSLfile} ${LibGperfFile})
#
for SRS_MODULE in ${SRS_MODULES[*]}; do
. $SRS_MODULE/config
@ -363,11 +365,11 @@ if [ $SRS_UTEST = YES ]; then
MODULE_FILES=("srs_utest" "srs_utest_amf0" "srs_utest_protocol" "srs_utest_kernel" "srs_utest_core"
"srs_utest_config" "srs_utest_rtmp" "srs_utest_http" "srs_utest_avc" "srs_utest_reload"
"srs_utest_mp4" "srs_utest_service" "srs_utest_app")
ModuleLibIncs=(${SRS_OBJS_DIR} ${LibSTRoot} ${LibSrtpRoot} ${LibSSLRoot})
ModuleLibIncs=(${SRS_OBJS_DIR} ${LibSTRoot} ${LibSrtpRoot} ${LibFfmpegRoot} ${LibSSLRoot})
if [[ $SRS_SRT == YES ]]; then
ModuleLibIncs+=("${LibSRTRoot[*]}")
fi
ModuleLibFiles=(${LibSTfile} ${LibSrtpFile} ${LibSSLfile})
ModuleLibFiles=(${LibSTfile} ${LibSrtpFile} ${LibFfmpegFile} ${LibSSLfile})
if [[ $SRS_SRT == YES ]]; then
ModuleLibFiles+=("${LibSRTfile[*]}")
fi

@ -0,0 +1,468 @@
/**
* The MIT License (MIT)
*
* Copyright (c) 2013-2020 Winlin
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
* the Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include <srs_kernel_codec.hpp>
#include <srs_kernel_error.hpp>
#include <srs_app_audio_recode.hpp>
static const int kOpusPacketMs = 20;
static const int kOpusMaxbytes = 8000;
static const int kFrameBufMax = 40960;
static const int kPacketBufMax = 8192;
static const int kPcmBufMax = 4096*4;
SrsAudioDecoder::SrsAudioDecoder(std::string codec)
: codec_name_(codec)
{
frame_ = NULL;
packet_ = NULL;
codec_ctx_ = NULL;
}
SrsAudioDecoder::~SrsAudioDecoder()
{
if (codec_ctx_) {
avcodec_free_context(&codec_ctx_);
codec_ctx_ = NULL;
}
if (frame_) {
av_frame_free(&frame_);
frame_ = NULL;
}
if (packet_) {
av_packet_free(&packet_);
packet_ = NULL;
}
}
srs_error_t SrsAudioDecoder::initialize()
{
srs_error_t err = srs_success;
if (codec_name_.compare("aac")) {
return srs_error_wrap(err, "Invalid codec name");
}
const AVCodec *codec = avcodec_find_decoder_by_name(codec_name_.c_str());
if (!codec) {
return srs_error_wrap(err, "Codec not found by name");
}
codec_ctx_ = avcodec_alloc_context3(codec);
if (!codec_ctx_) {
return srs_error_wrap(err, "Could not allocate audio codec context");
}
if (avcodec_open2(codec_ctx_, codec, NULL) < 0) {
return srs_error_wrap(err, "Could not open codec");
}
frame_ = av_frame_alloc();
if (!frame_) {
return srs_error_wrap(err, "Could not allocate audio frame");
}
packet_ = av_packet_alloc();
if (!packet_) {
return srs_error_wrap(err, "Could not allocate audio packet");
}
return err;
}
srs_error_t SrsAudioDecoder::decode(SrsSample *pkt, char *buf, int &size)
{
srs_error_t err = srs_success;
packet_->data = (uint8_t *)pkt->bytes;
packet_->size = pkt->size;
int ret = avcodec_send_packet(codec_ctx_, packet_);
if (ret < 0) {
return srs_error_wrap(err, "Error submitting the packet to the decoder");
}
int max = size;
size = 0;
while (ret >= 0) {
ret = avcodec_receive_frame(codec_ctx_, frame_);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
return err;
} else if (ret < 0) {
return srs_error_wrap(err, "Error during decoding");
}
int pcm_size = av_get_bytes_per_sample(codec_ctx_->sample_fmt);
if (pcm_size < 0) {
return srs_error_wrap(err, "Failed to calculate data size");
}
for (int i = 0; i < frame_->nb_samples; i++) {
if (size + pcm_size * codec_ctx_->channels <= max) {
memcpy(buf + size,frame_->data[0] + pcm_size*codec_ctx_->channels * i, pcm_size * codec_ctx_->channels);
size += pcm_size * codec_ctx_->channels;
}
}
}
return err;
}
AVCodecContext* SrsAudioDecoder::codec_ctx()
{
return codec_ctx_;
}
SrsAudioEncoder::SrsAudioEncoder(int samplerate, int channels, int fec, int complexity)
: inband_fec_(fec),
channels_(channels),
sampling_rate_(samplerate),
complexity_(complexity)
{
opus_ = NULL;
}
SrsAudioEncoder::~SrsAudioEncoder()
{
if (opus_) {
opus_encoder_destroy(opus_);
opus_ = NULL;
}
}
srs_error_t SrsAudioEncoder::initialize()
{
srs_error_t err = srs_success;
int error = 0;
opus_ = opus_encoder_create(sampling_rate_, channels_, OPUS_APPLICATION_VOIP, &error);
if (error != OPUS_OK) {
return srs_error_wrap(err, "Error create Opus encoder");
}
switch (sampling_rate_)
{
case 48000:
opus_encoder_ctl(opus_, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_FULLBAND));
break;
case 24000:
opus_encoder_ctl(opus_, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_SUPERWIDEBAND));
case 16000:
opus_encoder_ctl(opus_, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_WIDEBAND));
break;
case 12000:
opus_encoder_ctl(opus_, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_MEDIUMBAND));
break;
case 8000:
opus_encoder_ctl(opus_, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_NARROWBAND));
break;
default:
sampling_rate_ = 16000;
opus_encoder_ctl(opus_, OPUS_SET_MAX_BANDWIDTH(OPUS_BANDWIDTH_WIDEBAND));
break;
}
opus_encoder_ctl(opus_, OPUS_SET_INBAND_FEC(inband_fec_));
opus_encoder_ctl(opus_, OPUS_SET_COMPLEXITY(complexity_));
return err;
}
srs_error_t SrsAudioEncoder::encode(SrsSample *frame, char *buf, int &size)
{
srs_error_t err = srs_success;
int nb_samples = sampling_rate_ * kOpusPacketMs / 1000;
if (frame->size != nb_samples * 2 * channels_) {
return srs_error_wrap(err, "invalid frame size %d, should be %d", frame->size, nb_samples * 2 * channels_);
}
opus_int16 *data = (opus_int16 *)frame->bytes;
size = opus_encode(opus_, data, nb_samples, (unsigned char *)buf, kOpusMaxbytes);
return err;
}
SrsAudioResample::SrsAudioResample(int src_rate, int src_layout, enum AVSampleFormat src_fmt,
int src_nb, int dst_rate, int dst_layout, enum AVSampleFormat dst_fmt)
: src_rate_(src_rate),
src_ch_layout_(src_layout),
src_sample_fmt_(src_fmt),
src_nb_samples_(src_nb),
dst_rate_(dst_rate),
dst_ch_layout_(dst_layout),
dst_sample_fmt_(dst_fmt)
{
src_nb_channels_ = 0;
dst_nb_channels_ = 0;
src_linesize_ = 0;
dst_linesize_ = 0;
dst_nb_samples_ = 0;
src_data_ = NULL;
dst_data_ = 0;
max_dst_nb_samples_ = 0;
swr_ctx_ = NULL;
}
SrsAudioResample::~SrsAudioResample()
{
if (src_data_) {
av_freep(&src_data_[0]);
av_freep(&src_data_);
src_data_ = NULL;
}
if (dst_data_) {
av_freep(&dst_data_[0]);
av_freep(&dst_data_);
dst_data_ = NULL;
}
if (swr_ctx_) {
swr_free(&swr_ctx_);
swr_ctx_ = NULL;
}
}
srs_error_t SrsAudioResample::initialize()
{
srs_error_t err = srs_success;
swr_ctx_ = swr_alloc();
if (!swr_ctx_) {
return srs_error_wrap(err, "Could not allocate resampler context");
}
av_opt_set_int(swr_ctx_, "in_channel_layout", src_ch_layout_, 0);
av_opt_set_int(swr_ctx_, "in_sample_rate", src_rate_, 0);
av_opt_set_sample_fmt(swr_ctx_, "in_sample_fmt", src_sample_fmt_, 0);
av_opt_set_int(swr_ctx_, "out_channel_layout", dst_ch_layout_, 0);
av_opt_set_int(swr_ctx_, "out_sample_rate", dst_rate_, 0);
av_opt_set_sample_fmt(swr_ctx_, "out_sample_fmt", dst_sample_fmt_, 0);
int ret;
if ((ret = swr_init(swr_ctx_)) < 0) {
return srs_error_wrap(err, "Failed to initialize the resampling context");
}
src_nb_channels_ = av_get_channel_layout_nb_channels(src_ch_layout_);
ret = av_samples_alloc_array_and_samples(&src_data_, &src_linesize_, src_nb_channels_,
src_nb_samples_, src_sample_fmt_, 0);
if (ret < 0) {
return srs_error_wrap(err, "Could not allocate source samples");
}
max_dst_nb_samples_ = dst_nb_samples_ =
av_rescale_rnd(src_nb_samples_, dst_rate_, src_rate_, AV_ROUND_UP);
dst_nb_channels_ = av_get_channel_layout_nb_channels(dst_ch_layout_);
ret = av_samples_alloc_array_and_samples(&dst_data_, &dst_linesize_, dst_nb_channels_,
dst_nb_samples_, dst_sample_fmt_, 0);
if (ret < 0) {
return srs_error_wrap(err, "Could not allocate destination samples");
}
return err;
}
srs_error_t SrsAudioResample::resample(SrsSample *pcm, char *buf, int &size)
{
srs_error_t err = srs_success;
int ret, plane = 1;
if (src_sample_fmt_ == AV_SAMPLE_FMT_FLTP) {
plane = 2;
}
if (src_linesize_ * plane < pcm->size || pcm->size < 0) {
return srs_error_wrap(err, "size not ok");
}
memcpy(src_data_[0], pcm->bytes, pcm->size);
dst_nb_samples_ = av_rescale_rnd(swr_get_delay(swr_ctx_, src_rate_) +
src_nb_samples_, dst_rate_, src_rate_, AV_ROUND_UP);
if (dst_nb_samples_ > max_dst_nb_samples_) {
av_freep(&dst_data_[0]);
ret = av_samples_alloc(dst_data_, &dst_linesize_, dst_nb_channels_,
dst_nb_samples_, dst_sample_fmt_, 1);
if (ret < 0) {
return srs_error_wrap(err, "alloc error");
}
max_dst_nb_samples_ = dst_nb_samples_;
}
ret = swr_convert(swr_ctx_, dst_data_, dst_nb_samples_, (const uint8_t **)src_data_, src_nb_samples_);
if (ret < 0) {
return srs_error_wrap(err, "Error while converting");
}
int dst_bufsize = av_samples_get_buffer_size(&dst_linesize_, dst_nb_channels_,
ret, dst_sample_fmt_, 1);
if (dst_bufsize < 0) {
return srs_error_wrap(err, "Could not get sample buffer size");
}
int max = size;
size = 0;
if (max > dst_bufsize) {
memcpy(buf, dst_data_[0], dst_bufsize);
size = dst_bufsize;
}
return err;
}
SrsAudioRecode::SrsAudioRecode(int channels, int samplerate)
: dst_channels_(channels),
dst_samplerate_(samplerate)
{
size_ = 0;
data_ = new char[kPcmBufMax];
}
SrsAudioRecode::~SrsAudioRecode()
{
if (dec_) {
delete dec_;
dec_ = NULL;
}
if (enc_) {
delete enc_;
enc_ = NULL;
}
if (resample_) {
delete resample_;
resample_ = NULL;
}
delete[] data_;
}
srs_error_t SrsAudioRecode::initialize()
{
srs_error_t err = srs_success;
dec_ = new SrsAudioDecoder("aac");
if (!dec_) {
return srs_error_wrap(err, "SrsAudioDecoder failed");
}
dec_->initialize();
enc_ = new SrsAudioEncoder(dst_samplerate_, dst_channels_, 1, 1);
if (!enc_) {
return srs_error_wrap(err, "SrsAudioEncoder failed");
}
enc_->initialize();
resample_ = NULL;
return err;
}
srs_error_t SrsAudioRecode::recode(SrsSample *pkt, char **buf, int *buf_len, int &n)
{
srs_error_t err = srs_success;
static char decode_buffer[kPacketBufMax];
static char resample_buffer[kFrameBufMax];
static char encode_buffer[kPacketBufMax];
if (!dec_) {
return srs_error_wrap(err, "dec_ nullptr");
}
int decode_len = kPacketBufMax;
if ((err = dec_->decode(pkt, decode_buffer, decode_len)) != srs_success) {
return srs_error_wrap(err, "decode error");
}
if (!resample_) {
int channel_layout = av_get_default_channel_layout(dst_channels_);
AVCodecContext *codec_ctx = dec_->codec_ctx();
resample_ = new SrsAudioResample(codec_ctx->sample_rate, (int)codec_ctx->channel_layout, \
codec_ctx->sample_fmt, codec_ctx->frame_size, dst_samplerate_, channel_layout, \
AV_SAMPLE_FMT_S16);
if (!resample_) {
return srs_error_wrap(err, "SrsAudioResample failed");
}
resample_->initialize();
}
SrsSample pcm;
pcm.bytes = decode_buffer;
pcm.size = decode_len;
int resample_len = kFrameBufMax;
if ((err = resample_->resample(&pcm, resample_buffer, resample_len)) != srs_success) {
return srs_error_wrap(err, "decode error");
}
n = 0;
int data_left = resample_len;
int total;
total = (dst_samplerate_ * kOpusPacketMs / 1000) * 2 * dst_channels_;
if (size_ + data_left < total) {
memcpy(data_ + size_, resample_buffer, data_left);
size_ += data_left;
} else {
int index = 0;
while (1) {
data_left = data_left - (total - size_);
memcpy(data_ + size_, resample_buffer + index, total - size_);
index += total - size_;
size_ += total - size_;
if (!enc_) {
return srs_error_wrap(err, "enc_ nullptr");
}
int encode_len;
pcm.bytes = (char *)data_;
pcm.size = size_;
if ((err = enc_->encode(&pcm, encode_buffer, encode_len)) != srs_success) {
return srs_error_wrap(err, "decode error");
}
memcpy(buf[n], encode_buffer, encode_len);
buf_len[n] = encode_len;
n++;
size_ = 0;
if(!data_left)
break;
if(data_left < total) {
memcpy(data_ + size_, resample_buffer + index, data_left);
size_ += data_left;
break;
}
}
}
return err;
}

@ -0,0 +1,126 @@
/**
* The MIT License (MIT)
*
* Copyright (c) 2013-2020 Winlin
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
* the Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef SRS_APP_AUDIO_RECODE_HPP
#define SRS_APP_AUDIO_RECODE_HPP
#include <string>
#include <srs_core.hpp>
#ifdef __cplusplus
extern "C" {
#endif
#include <libavutil/frame.h>
#include <libavutil/mem.h>
#include <libavcodec/avcodec.h>
#include <libavutil/opt.h>
#include <libavutil/channel_layout.h>
#include <libavutil/samplefmt.h>
#include <libswresample/swresample.h>
#include <opus/opus.h>
#ifdef __cplusplus
}
#endif
class SrsSample;
class SrsAudioDecoder
{
private:
AVFrame* frame_;
AVPacket* packet_;
AVCodecContext* codec_ctx_;
std::string codec_name_;
public:
SrsAudioDecoder(std::string codec);
virtual ~SrsAudioDecoder();
srs_error_t initialize();
virtual srs_error_t decode(SrsSample *pkt, char *buf, int &size);
AVCodecContext* codec_ctx();
};
class SrsAudioEncoder
{
private:
int inband_fec_;
int channels_;
int sampling_rate_;
int complexity_;
OpusEncoder *opus_;
public:
SrsAudioEncoder(int samplerate, int channels, int fec, int complexity);
virtual ~SrsAudioEncoder();
srs_error_t initialize();
virtual srs_error_t encode(SrsSample *frame, char *buf, int &size);
};
class SrsAudioResample
{
private:
int src_rate_;
int src_ch_layout_;
int src_nb_channels_;
enum AVSampleFormat src_sample_fmt_;
int src_linesize_;
int src_nb_samples_;
uint8_t **src_data_;
int dst_rate_;
int dst_ch_layout_;
int dst_nb_channels_;
enum AVSampleFormat dst_sample_fmt_;
int dst_linesize_;
int dst_nb_samples_;
uint8_t **dst_data_;
int max_dst_nb_samples_;
struct SwrContext *swr_ctx_;
public:
SrsAudioResample(int src_rate, int src_layout, enum AVSampleFormat src_fmt,
int src_nb, int dst_rate, int dst_layout, enum AVSampleFormat dst_fmt);
virtual ~SrsAudioResample();
srs_error_t initialize();
virtual srs_error_t resample(SrsSample *pcm, char *buf, int &size);
};
class SrsAudioRecode
{
private:
SrsAudioDecoder *dec_;
SrsAudioEncoder *enc_;
SrsAudioResample *resample_;
int dst_channels_;
int dst_samplerate_;
int size_;
char *data_;
public:
SrsAudioRecode(int channels, int samplerate);
virtual ~SrsAudioRecode();
srs_error_t initialize();
virtual srs_error_t recode(SrsSample *pkt, char **buf, int *buf_len, int &n);
};
#endif /* SRS_APP_AUDIO_RECODE_HPP */

@ -50,6 +50,46 @@ using namespace std;
#include <srs_app_http_hooks.hpp>
#include <srs_protocol_format.hpp>
#include <openssl/rand.h>
#include <srs_app_audio_recode.hpp>
// TODO: Add this function into SrsRtpMux class.
srs_error_t aac_raw_append_adts_header(SrsSharedPtrMessage* shared_audio, SrsFormat* format, SrsBuffer** stream_ptr)
{
srs_error_t err = srs_success;
if (format->is_aac_sequence_header()) {
return err;
}
if (stream_ptr == NULL) {
return srs_error_new(ERROR_RTC_RTP_MUXER, "adts");
}
srs_verbose("audio samples=%d", format->audio->nb_samples);
if (format->audio->nb_samples != 1) {
return srs_error_new(ERROR_RTC_RTP_MUXER, "adts");
}
int nb_buf = format->audio->samples[0].size + 7;
char* buf = new char[nb_buf];
SrsBuffer* stream = new SrsBuffer(buf, nb_buf);
// TODO: Add comment.
stream->write_1bytes(0xFF);
stream->write_1bytes(0xF9);
stream->write_1bytes(((format->acodec->aac_object - 1) << 6) | ((format->acodec->aac_sample_rate & 0x0F) << 2) | ((format->acodec->aac_channels & 0x04) >> 2));
stream->write_1bytes(((format->acodec->aac_channels & 0x03) << 6) | ((nb_buf >> 11) & 0x03));
stream->write_1bytes((nb_buf >> 3) & 0xFF);
stream->write_1bytes(((nb_buf & 0x07) << 5) | 0x1F);
stream->write_1bytes(0xFC);
stream->write_bytes(format->audio->samples[0].bytes, format->audio->samples[0].size);
*stream_ptr = stream;
return err;
}
SrsRtpMuxer::SrsRtpMuxer()
{
@ -288,6 +328,104 @@ srs_error_t SrsRtpMuxer::packet_stap_a(const string &sps, const string& pps, Srs
return err;
}
SrsRtpOpusMuxer::SrsRtpOpusMuxer()
{
sequence = 0;
timestamp = 0;
recoder = NULL;
}
SrsRtpOpusMuxer::~SrsRtpOpusMuxer()
{
if (recoder) {
delete recoder;
recoder = NULL;
}
}
srs_error_t SrsRtpOpusMuxer::initialize()
{
srs_error_t err = srs_success;
recoder = new SrsAudioRecode(kChannel, kSamplerate);
if (!recoder) {
return srs_error_wrap(err, "SrsAacOpus init failed");
}
recoder->initialize();
return err;
}
srs_error_t SrsRtpOpusMuxer::frame_to_packet(SrsSharedPtrMessage* shared_audio, SrsFormat* format, SrsBuffer* stream)
{
srs_error_t err = srs_success;
vector<SrsRtpSharedPacket*> rtp_packet_vec;
char* data_ptr[kArrayLength];
static char data_array[kArrayLength][kArrayBuffer];
int elen[kArrayLength], number = 0;
data_ptr[0] = &data_array[0][0];
for (int i = 1; i < kArrayLength; i++) {
data_ptr[i] = data_array[i];
}
SrsSample pkt;
pkt.bytes = stream->data();
pkt.size = stream->pos();
if ((err = recoder->recode(&pkt, data_ptr, elen, number)) != srs_success) {
return srs_error_wrap(err, "recode error");
}
for (int i = 0; i < number; i++) {
SrsSample sample;
sample.size = elen[i];
sample.bytes = data_ptr[i];
packet_opus(shared_audio, &sample, rtp_packet_vec);
}
shared_audio->set_rtp_packets(rtp_packet_vec);
return err;
}
srs_error_t SrsRtpOpusMuxer::packet_opus(SrsSharedPtrMessage* shared_frame, SrsSample* sample, std::vector<SrsRtpSharedPacket*>& rtp_packet_vec)
{
srs_error_t err = srs_success;
char* buf = new char[kRtpPacketSize];
SrsBuffer* stream = new SrsBuffer(buf, kRtpPacketSize);
SrsAutoFree(SrsBuffer, stream);
// v=2,p=0,x=0,cc=0
stream->write_1bytes(0x80);
// marker payloadtype
stream->write_1bytes(kOpusPayloadType);
// sequenct
stream->write_2bytes(sequence);
// timestamp
stream->write_4bytes(int32_t(timestamp));
timestamp += 960;
// ssrc
stream->write_4bytes(int32_t(kAudioSSRC));
stream->write_bytes(sample->bytes, sample->size);
srs_verbose("sample=%s", srs_string_dumps_hex(sample->bytes, sample->size).c_str());
srs_verbose("opus, size=%u, seq=%u, timestamp=%lu, ssrc=%u, payloadtype=%u",
sample->size, sequence, timestamp, kAudioSSRC, kOpusPayloadType);
SrsRtpSharedPacket* rtp_shared_pkt = new SrsRtpSharedPacket();
rtp_shared_pkt->create(timestamp, sequence++, kAudioSSRC, kOpusPayloadType, stream->data(), stream->pos());
rtp_shared_pkt->set_marker(true);
rtp_packet_vec.push_back(rtp_shared_pkt);
return err;
}
SrsRtp::SrsRtp()
{
req = NULL;
@ -326,6 +464,11 @@ srs_error_t SrsRtp::initialize(SrsOriginHub* h, SrsRequest* r)
req = r;
rtp_h264_muxer = new SrsRtpMuxer();
rtp_opus_muxer = new SrsRtpOpusMuxer();
if (rtp_opus_muxer) {
rtp_opus_muxer->initialize();
}
return err;
}
@ -387,7 +530,16 @@ srs_error_t SrsRtp::on_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* forma
// ignore sequence header
srs_assert(format->audio);
// TODO: rtc no support aac
SrsBuffer* stream = NULL;
SrsAutoFree(SrsBuffer, stream);
if ((err = aac_raw_append_adts_header(shared_audio, format, &stream)) != srs_success) {
return srs_error_wrap(err, "aac append header");
}
if (stream) {
rtp_opus_muxer->frame_to_packet(shared_audio, format, stream);
}
return err;
}

@ -36,10 +36,13 @@ class SrsSharedPtrMessage;
class SrsRtpSharedPacket;
class SrsRequest;
class SrsOriginHub;
class SrsAudioRecode;
class SrsBuffer;
const int max_payload_size = 1200;
const int kRtpPacketSize = 1500;
const uint8_t kOpusPayloadType = 111;
const uint8_t kH264PayloadType = 102;
const uint8_t kNalTypeMask = 0x1F;
@ -50,7 +53,13 @@ const uint8_t kFuA = 28;
const uint8_t kStart = 0x80;
const uint8_t kEnd = 0x40;
const int kChannel = 2;
const int kSamplerate = 48000;
const int kArrayLength = 8;
const int kArrayBuffer = 4096;
// FIXME: ssrc can relate to source
const uint32_t kAudioSSRC = 3233846890;
const uint32_t kVideoSSRC = 3233846889;
class SrsRtpMuxer
@ -70,6 +79,22 @@ private:
srs_error_t packet_stap_a(const std::string &sps, const std::string& pps, SrsSharedPtrMessage* shared_frame, std::vector<SrsRtpSharedPacket*>& rtp_packet_vec);
};
class SrsRtpOpusMuxer
{
private:
uint32_t timestamp;
uint16_t sequence;
SrsAudioRecode* recoder;
public:
SrsRtpOpusMuxer();
virtual ~SrsRtpOpusMuxer();
virtual srs_error_t initialize();
public:
srs_error_t frame_to_packet(SrsSharedPtrMessage* shared_audio, SrsFormat* format, SrsBuffer* stream);
private:
srs_error_t packet_opus(SrsSharedPtrMessage* shared_frame, SrsSample* sample, std::vector<SrsRtpSharedPacket*>& rtp_packet_vec);
};
class SrsRtp
{
private:
@ -78,6 +103,7 @@ private:
bool disposable;
srs_utime_t last_update_time;
SrsRtpMuxer* rtp_h264_muxer;
SrsRtpOpusMuxer* rtp_opus_muxer;
SrsOriginHub* hub;
public:
SrsRtp();

@ -2261,6 +2261,11 @@ srs_error_t SrsSource::on_audio_imp(SrsSharedPtrMessage* msg)
}
}
// Copy to hub to all utilities.
if ((err = hub->on_audio(msg)) != srs_success) {
return srs_error_wrap(err, "consume audio");
}
// copy to all consumer
if (!drop_for_reduce) {
for (int i = 0; i < (int)consumers.size(); i++) {
@ -2271,11 +2276,6 @@ srs_error_t SrsSource::on_audio_imp(SrsSharedPtrMessage* msg)
}
}
// Copy to hub to all utilities.
if ((err = hub->on_audio(msg)) != srs_success) {
return srs_error_wrap(err, "consume audio");
}
// cache the sequence header of aac, or first packet of mp3.
// for example, the mp3 is used for hls to write the "right" audio codec.
// TODO: FIXME: to refine the stream info system.

Loading…
Cancel
Save