From ca73534d7ecf5bd2afad29e12f398e011c34c2ae Mon Sep 17 00:00:00 2001 From: winlin Date: Tue, 13 Oct 2015 11:27:39 +0800 Subject: [PATCH 01/46] refine comments for api key --- trunk/src/protocol/srs_kafka_stack.hpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/trunk/src/protocol/srs_kafka_stack.hpp b/trunk/src/protocol/srs_kafka_stack.hpp index e5df62b68..4be61c3eb 100644 --- a/trunk/src/protocol/srs_kafka_stack.hpp +++ b/trunk/src/protocol/srs_kafka_stack.hpp @@ -33,7 +33,10 @@ #ifdef SRS_AUTO_KAFKA -// https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ApiKeys +/** + * the api key used to identify the request type. + * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ApiKeys + */ enum SrsKafkaApiKey { SrsKafkaApiKeyProduceRequest = 0, @@ -178,7 +181,7 @@ public: */ virtual int header_size(); /** - * the size of message, the left bytes left after the header. + * the size of message, the bytes left after the header. * @remark total_size = 4 + header_size + message_size. */ virtual int message_size(); From d9f991ed2f95535d59bb2775d22f80bf2f9a430c Mon Sep 17 00:00:00 2001 From: winlin Date: Tue, 13 Oct 2015 16:06:37 +0800 Subject: [PATCH 02/46] use system utility for string finds --- trunk/src/app/srs_app_caster_flv.cpp | 15 ++-------- trunk/src/app/srs_app_dvr.cpp | 2 +- trunk/src/app/srs_app_http_conn.cpp | 33 +++++++++------------ trunk/src/app/srs_app_http_static.cpp | 2 +- trunk/src/app/srs_app_http_stream.cpp | 11 ++----- trunk/src/app/srs_app_ingest.cpp | 13 ++------ trunk/src/app/srs_app_mpegts_udp.cpp | 15 ++-------- trunk/src/app/srs_app_rtsp.cpp | 24 ++++----------- trunk/src/app/srs_app_st.hpp | 11 +++++++ trunk/src/kernel/srs_kernel_utility.cpp | 25 +++++++++++++++- trunk/src/kernel/srs_kernel_utility.hpp | 8 +++-- trunk/src/libs/srs_librtmp.cpp | 11 ++----- trunk/src/protocol/srs_http_stack.cpp | 6 +--- trunk/src/protocol/srs_protocol_utility.cpp | 16 ++++++++-- trunk/src/protocol/srs_protocol_utility.hpp | 26 +++++++++++++--- trunk/src/protocol/srs_rtsp_stack.cpp | 5 +--- 16 files changed, 111 insertions(+), 112 deletions(-) diff --git a/trunk/src/app/srs_app_caster_flv.cpp b/trunk/src/app/srs_app_caster_flv.cpp index 6e77ba07d..ac78c0dbc 100644 --- a/trunk/src/app/srs_app_caster_flv.cpp +++ b/trunk/src/app/srs_app_caster_flv.cpp @@ -268,19 +268,8 @@ int SrsDynamicHttpConn::connect() // parse uri if (!req) { req = new SrsRequest(); - - size_t pos = string::npos; - string uri = req->tcUrl = output; - - // tcUrl, stream - if ((pos = uri.rfind("/")) != string::npos) { - req->stream = uri.substr(pos + 1); - req->tcUrl = uri = uri.substr(0, pos); - } - - srs_discovery_tc_url(req->tcUrl, - req->schema, req->host, req->vhost, req->app, req->port, - req->param); + srs_parse_rtmp_url(output, req->tcUrl, req->stream); + srs_discovery_tc_url(req->tcUrl, req->schema, req->host, req->vhost, req->app, req->port, req->param); } // connect host. diff --git a/trunk/src/app/srs_app_dvr.cpp b/trunk/src/app/srs_app_dvr.cpp index 3b0971af8..a15974098 100644 --- a/trunk/src/app/srs_app_dvr.cpp +++ b/trunk/src/app/srs_app_dvr.cpp @@ -107,7 +107,7 @@ int SrsFlvSegment::open(bool use_tmp_file) bool fresh_flv_file = !srs_path_exists(path); // create dir first. - std::string dir = path.substr(0, path.rfind("/")); + std::string dir = srs_path_dirname(path); if ((ret = srs_create_dir_recursively(dir)) != ERROR_SUCCESS) { srs_error("create dir=%s failed. ret=%d", dir.c_str(), ret); return ret; diff --git a/trunk/src/app/srs_app_http_conn.cpp b/trunk/src/app/srs_app_http_conn.cpp index a64a953ab..1023ed0c1 100644 --- a/trunk/src/app/srs_app_http_conn.cpp +++ b/trunk/src/app/srs_app_http_conn.cpp @@ -572,12 +572,7 @@ int SrsHttpMessage::update(string url, bool allow_jsonp, http_parser* header, Sr } // parse ext. - _ext = _uri->get_path(); - if ((pos = _ext.rfind(".")) != string::npos) { - _ext = _ext.substr(pos); - } else { - _ext = ""; - } + _ext = srs_path_filext(_uri->get_path()); // parse jsonp request message. if (allow_jsonp) { @@ -816,23 +811,23 @@ SrsRequest* SrsHttpMessage::to_request(string vhost) { SrsRequest* req = new SrsRequest(); - req->app = _uri->get_path(); - size_t pos = string::npos; - if ((pos = req->app.rfind("/")) != string::npos) { - req->stream = req->app.substr(pos + 1); - req->app = req->app.substr(0, pos); - } - if ((pos = req->stream.rfind(".")) != string::npos) { - req->stream = req->stream.substr(0, pos); - } + // http path, for instance, /live/livestream.flv, parse to + // app: /live + // stream: livestream.flv + srs_parse_rtmp_url(_uri->get_path(), req->app, req->stream); + + // trim the start slash, for instance, /live to live + req->app = srs_string_trim_start(req->app, "/"); + + // remove the extension, for instance, livestream.flv to livestream + req->stream = srs_path_filename(req->stream); - req->tcUrl = "rtmp://" + vhost + req->app; + // generate others. + req->tcUrl = "rtmp://" + vhost + "/" + req->app; req->pageUrl = get_request_header("Referer"); req->objectEncoding = 0; - srs_discovery_tc_url(req->tcUrl, - req->schema, req->host, req->vhost, req->app, req->port, - req->param); + srs_discovery_tc_url(req->tcUrl, req->schema, req->host, req->vhost, req->app, req->port, req->param); req->strip(); return req; diff --git a/trunk/src/app/srs_app_http_static.cpp b/trunk/src/app/srs_app_http_static.cpp index 18a3869a8..6304d864e 100644 --- a/trunk/src/app/srs_app_http_static.cpp +++ b/trunk/src/app/srs_app_http_static.cpp @@ -242,7 +242,7 @@ int SrsHttpStaticServer::initialize() mount = srs_string_replace(mount, SRS_CONSTS_RTMP_DEFAULT_VHOST"/", "/"); // the dir mount must always ends with "/" - if (mount != "/" && mount.rfind("/") != mount.length() - 1) { + if (mount != "/" && !srs_string_ends_with(mount, "/")) { mount += "/"; } diff --git a/trunk/src/app/srs_app_http_stream.cpp b/trunk/src/app/srs_app_http_stream.cpp index 204e9eb62..3f3c4f7f1 100644 --- a/trunk/src/app/srs_app_http_stream.cpp +++ b/trunk/src/app/srs_app_http_stream.cpp @@ -611,11 +611,7 @@ SrsLiveEntry::SrsLiveEntry(std::string m, bool h) req = NULL; source = NULL; - std::string ext; - size_t pos = string::npos; - if ((pos = m.rfind(".")) != string::npos) { - ext = m.substr(pos); - } + std::string ext = srs_path_filext(m); _is_flv = (ext == ".flv"); _is_ts = (ext == ".ts"); _is_mp3 = (ext == ".mp3"); @@ -1319,10 +1315,7 @@ string SrsHttpStreamServer::hls_mount_generate(SrsRequest* r, string uri, string std::string mount = tmpl; // the ts is relative from the m3u8, the same start dir. - size_t pos = string::npos; - if ((pos = mount.rfind("/")) != string::npos) { - mount = mount.substr(0, pos); - } + mount = srs_path_dirname(mount); // replace the vhost variable mount = srs_string_replace(mount, "[vhost]", r->vhost); diff --git a/trunk/src/app/srs_app_ingest.cpp b/trunk/src/app/srs_app_ingest.cpp index fab6647cf..512aa7132 100644 --- a/trunk/src/app/srs_app_ingest.cpp +++ b/trunk/src/app/srs_app_ingest.cpp @@ -35,6 +35,7 @@ using namespace std; #include #include #include +#include // when error, ingester sleep for a while and retry. // ingest never sleep a long time, for we must start the stream ASAP. @@ -354,17 +355,9 @@ int SrsIngester::initialize_ffmpeg(SrsFFMPEG* ffmpeg, SrsConfDirective* vhost, S } // find the app and stream in rtmp url - std::string url = output; std::string app, stream; - size_t pos = std::string::npos; - if ((pos = url.rfind("/")) != std::string::npos) { - stream = url.substr(pos + 1); - url = url.substr(0, pos); - } - if ((pos = url.rfind("/")) != std::string::npos) { - app = url.substr(pos + 1); - url = url.substr(0, pos); - } + srs_parse_rtmp_url(output, app, stream); + size_t pos; if ((pos = app.rfind("?")) != std::string::npos) { app = app.substr(0, pos); } diff --git a/trunk/src/app/srs_app_mpegts_udp.cpp b/trunk/src/app/srs_app_mpegts_udp.cpp index 313279b11..efb4e5bcd 100644 --- a/trunk/src/app/srs_app_mpegts_udp.cpp +++ b/trunk/src/app/srs_app_mpegts_udp.cpp @@ -611,19 +611,8 @@ int SrsMpegtsOverUdp::connect() // parse uri if (!req) { req = new SrsRequest(); - - size_t pos = string::npos; - string uri = req->tcUrl = output; - - // tcUrl, stream - if ((pos = uri.rfind("/")) != string::npos) { - req->stream = uri.substr(pos + 1); - req->tcUrl = uri = uri.substr(0, pos); - } - - srs_discovery_tc_url(req->tcUrl, - req->schema, req->host, req->vhost, req->app, req->port, - req->param); + srs_parse_rtmp_url(output, req->tcUrl, req->stream); + srs_discovery_tc_url(req->tcUrl, req->schema, req->host, req->vhost, req->app, req->port, req->param); } // connect host. diff --git a/trunk/src/app/srs_app_rtsp.cpp b/trunk/src/app/srs_app_rtsp.cpp index c0c6b38e1..ca73a659a 100644 --- a/trunk/src/app/srs_app_rtsp.cpp +++ b/trunk/src/app/srs_app_rtsp.cpp @@ -270,10 +270,7 @@ int SrsRtspConn::do_cycle() if ((pos = rtsp_tcUrl.rfind(".sdp")) != string::npos) { rtsp_tcUrl = rtsp_tcUrl.substr(0, pos); } - if ((pos = rtsp_tcUrl.rfind("/")) != string::npos) { - rtsp_stream = rtsp_tcUrl.substr(pos + 1); - rtsp_tcUrl = rtsp_tcUrl.substr(0, pos); - } + srs_parse_rtmp_url(rtsp_tcUrl, rtsp_tcUrl, rtsp_stream); srs_assert(req->sdp); video_id = ::atoi(req->sdp->video_stream_id.c_str()); @@ -651,8 +648,6 @@ int SrsRtspConn::connect() // parse uri if (!req) { - req = new SrsRequest(); - std::string schema, host, vhost, app, port, param; srs_discovery_tc_url(rtsp_tcUrl, schema, host, vhost, app, port, param); @@ -660,19 +655,10 @@ int SrsRtspConn::connect() std::string output = output_template; output = srs_string_replace(output, "[app]", app); output = srs_string_replace(output, "[stream]", rtsp_stream); - - size_t pos = string::npos; - string uri = req->tcUrl = output; - - // tcUrl, stream - if ((pos = uri.rfind("/")) != string::npos) { - req->stream = uri.substr(pos + 1); - req->tcUrl = uri = uri.substr(0, pos); - } - - srs_discovery_tc_url(req->tcUrl, - req->schema, req->host, req->vhost, req->app, req->port, - req->param); + + req = new SrsRequest(); + srs_parse_rtmp_url(output, req->tcUrl, req->stream); + srs_discovery_tc_url(req->tcUrl, req->schema, req->host, req->vhost, req->app, req->port, req->param); } // connect host. diff --git a/trunk/src/app/srs_app_st.hpp b/trunk/src/app/srs_app_st.hpp index ff28c149f..31af7d41f 100644 --- a/trunk/src/app/srs_app_st.hpp +++ b/trunk/src/app/srs_app_st.hpp @@ -202,6 +202,17 @@ public: virtual int writev(const iovec *iov, int iov_size, ssize_t* nwrite); }; +/** + * the common tcp client, to connect to specified TCP server, + * reconnect and close the connection. + */ +class SrsTcpClient +{ +public: + SrsTcpClient(); + virtual ~SrsTcpClient(); +}; + // initialize st, requires epoll. extern int srs_st_init(); diff --git a/trunk/src/kernel/srs_kernel_utility.cpp b/trunk/src/kernel/srs_kernel_utility.cpp index 550c8d2cd..13f863979 100644 --- a/trunk/src/kernel/srs_kernel_utility.cpp +++ b/trunk/src/kernel/srs_kernel_utility.cpp @@ -464,6 +464,29 @@ string srs_path_basename(string path) return dirname; } + +string srs_path_filename(string path) +{ + std::string filename = path; + size_t pos = string::npos; + + if ((pos = filename.rfind(".")) != string::npos) { + return filename.substr(0, pos); + } + + return filename; +} + +string srs_path_filext(string path) +{ + size_t pos = string::npos; + + if ((pos = path.rfind(".")) != string::npos) { + return path.substr(pos); + } + + return ""; +} bool srs_avc_startswith_annexb(SrsBuffer* stream, int* pnb_start_code) { @@ -471,7 +494,7 @@ bool srs_avc_startswith_annexb(SrsBuffer* stream, int* pnb_start_code) char* p = bytes; for (;;) { - if (!stream->require(p - bytes + 3)) { + if (!stream->require((int)(p - bytes + 3))) { return false; } diff --git a/trunk/src/kernel/srs_kernel_utility.hpp b/trunk/src/kernel/srs_kernel_utility.hpp index d2bc344ec..4a5bc5cb4 100644 --- a/trunk/src/kernel/srs_kernel_utility.hpp +++ b/trunk/src/kernel/srs_kernel_utility.hpp @@ -96,10 +96,14 @@ extern int srs_create_dir_recursively(std::string dir); // whether path exists. extern bool srs_path_exists(std::string path); -// get the dirname of path +// get the dirname of path, for instance, filename("/live/livestream")="/live" extern std::string srs_path_dirname(std::string path); -// get the basename of path +// get the basename of path, for instance, filename("/live/livestream")="livestream" extern std::string srs_path_basename(std::string path); +// get the filename of path, for instance, filename("livestream.flv")="livestream" +extern std::string srs_path_filename(std::string path); +// get the file extension of path, for instance, filext("live.flv")=".flv" +extern std::string srs_path_filext(std::string path); /** * whether stream starts with the avc NALU in "AnnexB" diff --git a/trunk/src/libs/srs_librtmp.cpp b/trunk/src/libs/srs_librtmp.cpp index d8592d482..e2fa7b9dd 100644 --- a/trunk/src/libs/srs_librtmp.cpp +++ b/trunk/src/libs/srs_librtmp.cpp @@ -464,16 +464,9 @@ int srs_librtmp_context_parse_uri(Context* context) { int ret = ERROR_SUCCESS; - // parse uri - size_t pos = string::npos; - string uri = context->url; - // tcUrl, stream - if ((pos = uri.rfind("/")) != string::npos) { - context->stream = uri.substr(pos + 1); - context->tcUrl = uri = uri.substr(0, pos); - } - std::string schema; + + srs_parse_rtmp_url(context->url, context->tcUrl, context->stream); srs_discovery_tc_url(context->tcUrl, schema, context->host, context->vhost, context->app, context->port, context->param); diff --git a/trunk/src/protocol/srs_http_stack.cpp b/trunk/src/protocol/srs_http_stack.cpp index b19c8519b..3c88fc44c 100755 --- a/trunk/src/protocol/srs_http_stack.cpp +++ b/trunk/src/protocol/srs_http_stack.cpp @@ -377,11 +377,7 @@ int SrsHttpFileServer::serve_file(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, } if (true) { - size_t pos; - std::string ext = fullpath; - if ((pos = ext.rfind(".")) != string::npos) { - ext = ext.substr(pos); - } + std::string ext = srs_path_filext(fullpath); if (_mime.find(ext) == _mime.end()) { w->header()->set_content_type("application/octet-stream"); diff --git a/trunk/src/protocol/srs_protocol_utility.cpp b/trunk/src/protocol/srs_protocol_utility.cpp index 682f92602..09cfd6b5c 100644 --- a/trunk/src/protocol/srs_protocol_utility.cpp +++ b/trunk/src/protocol/srs_protocol_utility.cpp @@ -44,7 +44,7 @@ using namespace std; void srs_discovery_tc_url( string tcUrl, string& schema, string& host, string& vhost, - string& app, int& port, std::string& param + string& app, int& port, string& param ) { size_t pos = std::string::npos; std::string url = tcUrl; @@ -229,7 +229,7 @@ int srs_rtmp_create_msg(char type, u_int32_t timestamp, char* data, int size, in return ret; } -std::string srs_generate_stream_url(std::string vhost, std::string app, std::string stream) +string srs_generate_stream_url(string vhost, string app, string stream) { std::string url = ""; @@ -244,6 +244,18 @@ std::string srs_generate_stream_url(std::string vhost, std::string app, std::str return url; } +void srs_parse_rtmp_url(string url, string& tcUrl, string& stream) +{ + size_t pos; + + if ((pos = url.rfind("/")) != string::npos) { + stream = url.substr(pos + 1); + tcUrl = url.substr(0, pos); + } else { + tcUrl = url; + } +} + string srs_generate_rtmp_url(string server, int port, string vhost, string app, string stream) { std::stringstream ss; diff --git a/trunk/src/protocol/srs_protocol_utility.hpp b/trunk/src/protocol/srs_protocol_utility.hpp index f3650f0a9..d2ce19068 100644 --- a/trunk/src/protocol/srs_protocol_utility.hpp +++ b/trunk/src/protocol/srs_protocol_utility.hpp @@ -101,16 +101,34 @@ extern bool srs_bytes_equals(void* pa, void* pb, int size); * @param data the packet bytes. user should never free it. * @param ppmsg output the shared ptr message. user should free it. */ -extern int srs_rtmp_create_msg(char type, u_int32_t timestamp, char* data, int size, int stream_id, SrsSharedPtrMessage** ppmsg); +extern int srs_rtmp_create_msg( + char type, u_int32_t timestamp, char* data, int size, int stream_id, + SrsSharedPtrMessage** ppmsg +); // get the stream identify, vhost/app/stream. -extern std::string srs_generate_stream_url(std::string vhost, std::string app, std::string stream); +extern std::string srs_generate_stream_url( + std::string vhost, std::string app, std::string stream +); + +// parse the rtmp url to tcUrl/stream, +// for example, rtmp://v.ossrs.net/live/livestream to +// tcUrl: rtmp://v.ossrs.net/live +// stream: livestream +extern void srs_parse_rtmp_url( + std::string url, std::string& tcUrl, std::string& stream +); // genereate the rtmp url, for instance, rtmp://server:port/app...vhost...vhost/stream -extern std::string srs_generate_rtmp_url(std::string server, int port, std::string vhost, std::string app, std::string stream); +extern std::string srs_generate_rtmp_url( + std::string server, int port, std::string vhost, std::string app, std::string stream +); // write large numbers of iovs. -extern int srs_write_large_iovs(ISrsProtocolReaderWriter* skt, iovec* iovs, int size, ssize_t* pnwrite = NULL); +extern int srs_write_large_iovs( + ISrsProtocolReaderWriter* skt, iovec* iovs, int size, + ssize_t* pnwrite = NULL +); #endif diff --git a/trunk/src/protocol/srs_rtsp_stack.cpp b/trunk/src/protocol/srs_rtsp_stack.cpp index 772957fee..0987c7d9d 100644 --- a/trunk/src/protocol/srs_rtsp_stack.cpp +++ b/trunk/src/protocol/srs_rtsp_stack.cpp @@ -988,10 +988,7 @@ int SrsRtspStack::do_recv_message(SrsRtspRequest* req) // for setup, parse the stream id from uri. if (req->is_setup()) { size_t pos = string::npos; - std::string stream_id; - if ((pos = req->uri.rfind("/")) != string::npos) { - stream_id = req->uri.substr(pos + 1); - } + std::string stream_id = srs_path_basename(req->uri); if ((pos = stream_id.find("=")) != string::npos) { stream_id = stream_id.substr(pos + 1); } From 908365a61a0862d4ee4dfd13c2af5e0dbb8b6eb8 Mon Sep 17 00:00:00 2001 From: winlin Date: Tue, 13 Oct 2015 16:42:49 +0800 Subject: [PATCH 03/46] use string for const char* --- trunk/src/app/srs_app_http_conn.cpp | 20 ++++++++++---------- trunk/src/app/srs_app_http_conn.hpp | 10 +++++----- trunk/src/main/srs_main_ingest_hls.cpp | 10 +++++----- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/trunk/src/app/srs_app_http_conn.cpp b/trunk/src/app/srs_app_http_conn.cpp index 1023ed0c1..5476ae41a 100644 --- a/trunk/src/app/srs_app_http_conn.cpp +++ b/trunk/src/app/srs_app_http_conn.cpp @@ -1109,19 +1109,19 @@ int SrsHttpUri::initialize(string _url) return ret; } -const char* SrsHttpUri::get_url() +string SrsHttpUri::get_url() { - return url.data(); + return url; } -const char* SrsHttpUri::get_schema() +string SrsHttpUri::get_schema() { - return schema.data(); + return schema; } -const char* SrsHttpUri::get_host() +string SrsHttpUri::get_host() { - return host.data(); + return host; } int SrsHttpUri::get_port() @@ -1129,14 +1129,14 @@ int SrsHttpUri::get_port() return port; } -const char* SrsHttpUri::get_path() +string SrsHttpUri::get_path() { - return path.data(); + return path; } -const char* SrsHttpUri::get_query() +string SrsHttpUri::get_query() { - return query.data(); + return query; } string SrsHttpUri::get_uri_field(string uri, http_parser_url* hp_u, http_parser_url_fields field) diff --git a/trunk/src/app/srs_app_http_conn.hpp b/trunk/src/app/srs_app_http_conn.hpp index 64f826bc5..e54c3a1f5 100644 --- a/trunk/src/app/srs_app_http_conn.hpp +++ b/trunk/src/app/srs_app_http_conn.hpp @@ -369,12 +369,12 @@ public: */ virtual int initialize(std::string _url); public: - virtual const char* get_url(); - virtual const char* get_schema(); - virtual const char* get_host(); + virtual std::string get_url(); + virtual std::string get_schema(); + virtual std::string get_host(); virtual int get_port(); - virtual const char* get_path(); - virtual const char* get_query(); + virtual std::string get_path(); + virtual std::string get_query(); private: /** * get the parsed url field. diff --git a/trunk/src/main/srs_main_ingest_hls.cpp b/trunk/src/main/srs_main_ingest_hls.cpp index 8d84e0faa..fdf2a911a 100644 --- a/trunk/src/main/srs_main_ingest_hls.cpp +++ b/trunk/src/main/srs_main_ingest_hls.cpp @@ -378,7 +378,7 @@ int SrsIngestSrsInput::parseM3u8(SrsHttpUri* url, double& td, double& duration) int ret = ERROR_SUCCESS; SrsHttpClient client; - srs_trace("parse input hls %s", url->get_url()); + srs_trace("parse input hls %s", url->get_url().c_str()); if ((ret = client.initialize(url->get_host(), url->get_port())) != ERROR_SUCCESS) { srs_error("connect to server failed. ret=%d", ret); @@ -387,7 +387,7 @@ int SrsIngestSrsInput::parseM3u8(SrsHttpUri* url, double& td, double& duration) ISrsHttpMessage* msg = NULL; if ((ret = client.get(url->get_path(), "", &msg)) != ERROR_SUCCESS) { - srs_error("HTTP GET %s failed. ret=%d", url->get_url(), ret); + srs_error("HTTP GET %s failed. ret=%d", url->get_url().c_str(), ret); return ret; } @@ -609,7 +609,7 @@ int SrsIngestSrsInput::SrsTsPiece::fetch(string m3u8) ISrsHttpMessage* msg = NULL; if ((ret = client.get(uri.get_path(), "", &msg)) != ERROR_SUCCESS) { - srs_error("HTTP GET %s failed. ret=%d", uri.get_url(), ret); + srs_error("HTTP GET %s failed. ret=%d", uri.get_url().c_str(), ret); return ret; } @@ -1215,7 +1215,7 @@ int SrsIngestSrsOutput::connect() return ret; } - srs_trace("connect output=%s", out_rtmp->get_url()); + srs_trace("connect output=%s", out_rtmp->get_url().c_str()); // parse uri if (!req) { @@ -1322,7 +1322,7 @@ int SrsIngestSrsOutput::connect_app(string ep_server, int ep_port) void SrsIngestSrsOutput::close() { - srs_trace("close output=%s", out_rtmp->get_url()); + srs_trace("close output=%s", out_rtmp->get_url().c_str()); h264_sps_pps_sent = false; srs_freep(client); From 69cc01b696c64e6b99dda3918993f4d0b8bd6d56 Mon Sep 17 00:00:00 2001 From: winlin Date: Tue, 13 Oct 2015 16:54:51 +0800 Subject: [PATCH 04/46] use system utilities --- trunk/src/app/srs_app_dvr.cpp | 2 +- trunk/src/app/srs_app_ffmpeg.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/trunk/src/app/srs_app_dvr.cpp b/trunk/src/app/srs_app_dvr.cpp index a15974098..4f161208f 100644 --- a/trunk/src/app/srs_app_dvr.cpp +++ b/trunk/src/app/srs_app_dvr.cpp @@ -410,7 +410,7 @@ string SrsFlvSegment::generate_path() std::string path_config = _srs_config->get_dvr_path(req->vhost); // add [stream].[timestamp].flv as filename for dir - if (path_config.find(".flv") != path_config.length() - 4) { + if (!srs_string_ends_with(path_config, ".flv")) { path_config += "/[stream].[timestamp].flv"; } diff --git a/trunk/src/app/srs_app_ffmpeg.cpp b/trunk/src/app/srs_app_ffmpeg.cpp index cb28f0479..79f262916 100644 --- a/trunk/src/app/srs_app_ffmpeg.cpp +++ b/trunk/src/app/srs_app_ffmpeg.cpp @@ -219,7 +219,7 @@ int SrsFFMPEG::initialize_transcode(SrsConfDirective* engine) // for not rtmp input, donot append the iformat, // for example, "-f flv" before "-i udp://192.168.1.252:2222" // @see https://github.com/simple-rtmp-server/srs/issues/290 - if (input.find("rtmp://") != 0) { + if (!srs_string_starts_with(input, "rtmp://")) { iformat = ""; } From 20fcfb3eeed7940b77241b4c5667b3b28a704226 Mon Sep 17 00:00:00 2001 From: winlin Date: Tue, 13 Oct 2015 17:37:59 +0800 Subject: [PATCH 05/46] fix bug, use system utility --- trunk/src/app/srs_app_caster_flv.cpp | 6 +++--- trunk/src/app/srs_app_caster_flv.hpp | 2 +- trunk/src/app/srs_app_mpegts_udp.cpp | 6 +++--- trunk/src/app/srs_app_mpegts_udp.hpp | 2 +- trunk/src/app/srs_app_rtsp.cpp | 9 +++++---- trunk/src/app/srs_app_rtsp.hpp | 2 +- trunk/src/app/srs_app_st.cpp | 17 +++++++++++++++++ trunk/src/app/srs_app_st.hpp | 10 ++++++++++ 8 files changed, 41 insertions(+), 13 deletions(-) diff --git a/trunk/src/app/srs_app_caster_flv.cpp b/trunk/src/app/srs_app_caster_flv.cpp index ac78c0dbc..a0d38cda9 100644 --- a/trunk/src/app/srs_app_caster_flv.cpp +++ b/trunk/src/app/srs_app_caster_flv.cpp @@ -273,8 +273,8 @@ int SrsDynamicHttpConn::connect() } // connect host. - if ((ret = srs_socket_connect(req->host, ::atoi(req->port.c_str()), ST_UTIME_NO_TIMEOUT, &stfd)) != ERROR_SUCCESS) { - srs_error("mpegts: connect server %s:%s failed. ret=%d", req->host.c_str(), req->port.c_str(), ret); + if ((ret = srs_socket_connect(req->host, req->port, ST_UTIME_NO_TIMEOUT, &stfd)) != ERROR_SUCCESS) { + srs_error("mpegts: connect server %s:%d failed. ret=%d", req->host.c_str(), req->port, ret); return ret; } io = new SrsStSocket(stfd); @@ -308,7 +308,7 @@ int SrsDynamicHttpConn::connect() } // TODO: FIXME: refine the connect_app. -int SrsDynamicHttpConn::connect_app(string ep_server, string ep_port) +int SrsDynamicHttpConn::connect_app(string ep_server, int ep_port) { int ret = ERROR_SUCCESS; diff --git a/trunk/src/app/srs_app_caster_flv.hpp b/trunk/src/app/srs_app_caster_flv.hpp index cf0d11f86..f6a03838e 100644 --- a/trunk/src/app/srs_app_caster_flv.hpp +++ b/trunk/src/app/srs_app_caster_flv.hpp @@ -104,7 +104,7 @@ private: // connect to rtmp output url. // @remark ignore when not connected, reconnect when disconnected. virtual int connect(); - virtual int connect_app(std::string ep_server, std::string ep_port); + virtual int connect_app(std::string ep_server, int ep_port); // close the connected io and rtmp to ready to be re-connect. virtual void close(); }; diff --git a/trunk/src/app/srs_app_mpegts_udp.cpp b/trunk/src/app/srs_app_mpegts_udp.cpp index efb4e5bcd..71593f72c 100644 --- a/trunk/src/app/srs_app_mpegts_udp.cpp +++ b/trunk/src/app/srs_app_mpegts_udp.cpp @@ -616,8 +616,8 @@ int SrsMpegtsOverUdp::connect() } // connect host. - if ((ret = srs_socket_connect(req->host, ::atoi(req->port.c_str()), ST_UTIME_NO_TIMEOUT, &stfd)) != ERROR_SUCCESS) { - srs_error("mpegts: connect server %s:%s failed. ret=%d", req->host.c_str(), req->port.c_str(), ret); + if ((ret = srs_socket_connect(req->host, req->port, ST_UTIME_NO_TIMEOUT, &stfd)) != ERROR_SUCCESS) { + srs_error("mpegts: connect server %s:%d failed. ret=%d", req->host.c_str(), req->port, ret); return ret; } io = new SrsStSocket(stfd); @@ -651,7 +651,7 @@ int SrsMpegtsOverUdp::connect() } // TODO: FIXME: refine the connect_app. -int SrsMpegtsOverUdp::connect_app(string ep_server, string ep_port) +int SrsMpegtsOverUdp::connect_app(string ep_server, int ep_port) { int ret = ERROR_SUCCESS; diff --git a/trunk/src/app/srs_app_mpegts_udp.hpp b/trunk/src/app/srs_app_mpegts_udp.hpp index 400e8627e..98b4cd3c3 100644 --- a/trunk/src/app/srs_app_mpegts_udp.hpp +++ b/trunk/src/app/srs_app_mpegts_udp.hpp @@ -126,7 +126,7 @@ private: // connect to rtmp output url. // @remark ignore when not connected, reconnect when disconnected. virtual int connect(); - virtual int connect_app(std::string ep_server, std::string ep_port); + virtual int connect_app(std::string ep_server, int ep_port); // close the connected io and rtmp to ready to be re-connect. virtual void close(); }; diff --git a/trunk/src/app/srs_app_rtsp.cpp b/trunk/src/app/srs_app_rtsp.cpp index ca73a659a..b35eccb03 100644 --- a/trunk/src/app/srs_app_rtsp.cpp +++ b/trunk/src/app/srs_app_rtsp.cpp @@ -648,7 +648,8 @@ int SrsRtspConn::connect() // parse uri if (!req) { - std::string schema, host, vhost, app, port, param; + std::string schema, host, vhost, app, param; + int port; srs_discovery_tc_url(rtsp_tcUrl, schema, host, vhost, app, port, param); // generate output by template. @@ -662,8 +663,8 @@ int SrsRtspConn::connect() } // connect host. - if ((ret = srs_socket_connect(req->host, ::atoi(req->port.c_str()), ST_UTIME_NO_TIMEOUT, &stfd)) != ERROR_SUCCESS) { - srs_error("rtsp: connect server %s:%s failed. ret=%d", req->host.c_str(), req->port.c_str(), ret); + if ((ret = srs_socket_connect(req->host, req->port, ST_UTIME_NO_TIMEOUT, &stfd)) != ERROR_SUCCESS) { + srs_error("rtsp: connect server %s:%d failed. ret=%d", req->host.c_str(), req->port, ret); return ret; } io = new SrsStSocket(stfd); @@ -697,7 +698,7 @@ int SrsRtspConn::connect() } // TODO: FIXME: refine the connect_app. -int SrsRtspConn::connect_app(string ep_server, string ep_port) +int SrsRtspConn::connect_app(string ep_server, int ep_port) { int ret = ERROR_SUCCESS; diff --git a/trunk/src/app/srs_app_rtsp.hpp b/trunk/src/app/srs_app_rtsp.hpp index 40c9c599a..3fa245053 100644 --- a/trunk/src/app/srs_app_rtsp.hpp +++ b/trunk/src/app/srs_app_rtsp.hpp @@ -181,7 +181,7 @@ private: // connect to rtmp output url. // @remark ignore when not connected, reconnect when disconnected. virtual int connect(); - virtual int connect_app(std::string ep_server, std::string ep_port); + virtual int connect_app(std::string ep_server, int ep_port); }; /** diff --git a/trunk/src/app/srs_app_st.cpp b/trunk/src/app/srs_app_st.cpp index 810eccdb5..8d90e8084 100644 --- a/trunk/src/app/srs_app_st.cpp +++ b/trunk/src/app/srs_app_st.cpp @@ -23,6 +23,9 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #include +#include +using namespace std; + #include #include @@ -406,6 +409,20 @@ int SrsStSocket::writev(const iovec *iov, int iov_size, ssize_t* nwrite) return ret; } +SrsTcpClient::SrsTcpClient() +{ +} + +SrsTcpClient::~SrsTcpClient() +{ +} + +int SrsTcpClient::connect(string host, int port, int64_t timeout) +{ + int ret = ERROR_SUCCESS; + return ret; +} + #ifdef __linux__ #include diff --git a/trunk/src/app/srs_app_st.hpp b/trunk/src/app/srs_app_st.hpp index 31af7d41f..6125c50f0 100644 --- a/trunk/src/app/srs_app_st.hpp +++ b/trunk/src/app/srs_app_st.hpp @@ -30,6 +30,8 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #include +#include + #include #include @@ -211,6 +213,14 @@ class SrsTcpClient public: SrsTcpClient(); virtual ~SrsTcpClient(); +public: + /** + * connect to server over TCP. + * @param host the ip or hostname of server. + * @param port the port to connect to. + * @param timeout the timeout in us. + */ + virtual int connect(std::string host, int port, int64_t timeout); }; // initialize st, requires epoll. From a9bb6061c3bf282c646dd6071d20c6858496125b Mon Sep 17 00:00:00 2001 From: winlin Date: Tue, 13 Oct 2015 17:59:51 +0800 Subject: [PATCH 06/46] use tcp client for raw connect. --- trunk/src/app/srs_app_caster_flv.cpp | 16 +++-- trunk/src/app/srs_app_caster_flv.hpp | 4 +- trunk/src/app/srs_app_st.cpp | 89 ++++++++++++++++++++++++++++ trunk/src/app/srs_app_st.hpp | 29 ++++++++- 4 files changed, 126 insertions(+), 12 deletions(-) diff --git a/trunk/src/app/srs_app_caster_flv.cpp b/trunk/src/app/srs_app_caster_flv.cpp index a0d38cda9..20dc505f7 100644 --- a/trunk/src/app/srs_app_caster_flv.cpp +++ b/trunk/src/app/srs_app_caster_flv.cpp @@ -119,9 +119,8 @@ SrsDynamicHttpConn::SrsDynamicHttpConn(IConnectionManager* cm, st_netfd_t fd, Sr { req = NULL; - io = NULL; + transport = new SrsTcpClient(); client = NULL; - stfd = NULL; stream_id = 0; pprint = SrsPithyPrint::create_caster(); @@ -131,6 +130,7 @@ SrsDynamicHttpConn::~SrsDynamicHttpConn() { close(); + srs_freep(transport); srs_freep(pprint); } @@ -261,7 +261,7 @@ int SrsDynamicHttpConn::connect() // when ok, ignore. // TODO: FIXME: should reconnect when disconnected. - if (io || client) { + if (transport->connected()) { return ret; } @@ -273,12 +273,10 @@ int SrsDynamicHttpConn::connect() } // connect host. - if ((ret = srs_socket_connect(req->host, req->port, ST_UTIME_NO_TIMEOUT, &stfd)) != ERROR_SUCCESS) { - srs_error("mpegts: connect server %s:%d failed. ret=%d", req->host.c_str(), req->port, ret); + if ((ret = transport->connect(req->host, req->port, ST_UTIME_NO_TIMEOUT)) != ERROR_SUCCESS) { return ret; } - io = new SrsStSocket(stfd); - client = new SrsRtmpClient(io); + client = new SrsRtmpClient(transport); client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); @@ -360,10 +358,10 @@ int SrsDynamicHttpConn::connect_app(string ep_server, int ep_port) void SrsDynamicHttpConn::close() { + transport->close(); + srs_freep(client); - srs_freep(io); srs_freep(req); - srs_close_stfd(stfd); } SrsHttpFileReader::SrsHttpFileReader(ISrsHttpResponseReader* h) diff --git a/trunk/src/app/srs_app_caster_flv.hpp b/trunk/src/app/srs_app_caster_flv.hpp index f6a03838e..7020fde41 100644 --- a/trunk/src/app/srs_app_caster_flv.hpp +++ b/trunk/src/app/srs_app_caster_flv.hpp @@ -43,6 +43,7 @@ class SrsRequest; class SrsPithyPrint; class ISrsHttpResponseReader; class SrsFlvDecoder; +class SrsTcpClient; #include #include @@ -86,8 +87,7 @@ private: SrsPithyPrint* pprint; private: SrsRequest* req; - st_netfd_t stfd; - SrsStSocket* io; + SrsTcpClient* transport; SrsRtmpClient* client; int stream_id; public: diff --git a/trunk/src/app/srs_app_st.cpp b/trunk/src/app/srs_app_st.cpp index 8d90e8084..f240d6798 100644 --- a/trunk/src/app/srs_app_st.cpp +++ b/trunk/src/app/srs_app_st.cpp @@ -28,6 +28,7 @@ using namespace std; #include #include +#include namespace internal { @@ -411,18 +412,106 @@ int SrsStSocket::writev(const iovec *iov, int iov_size, ssize_t* nwrite) SrsTcpClient::SrsTcpClient() { + io = NULL; + stfd = NULL; } SrsTcpClient::~SrsTcpClient() { + close(); +} + +bool SrsTcpClient::connected() +{ + return io; } int SrsTcpClient::connect(string host, int port, int64_t timeout) { int ret = ERROR_SUCCESS; + + // when connected, ignore. + if (io) { + return ret; + } + + // connect host. + if ((ret = srs_socket_connect(host, port, timeout, &stfd)) != ERROR_SUCCESS) { + srs_error("mpegts: connect server %s:%d failed. ret=%d", host.c_str(), port, ret); + return ret; + } + + io = new SrsStSocket(stfd); + return ret; } +void SrsTcpClient::close() +{ + // when closed, ignore. + if (!io) { + return; + } + + srs_freep(io); + srs_close_stfd(stfd); +} + +bool SrsTcpClient::is_never_timeout(int64_t timeout_us) +{ + return io->is_never_timeout(timeout_us); +} + +void SrsTcpClient::set_recv_timeout(int64_t timeout_us) +{ + io->set_recv_timeout(timeout_us); +} + +int64_t SrsTcpClient::get_recv_timeout() +{ + return io->get_recv_timeout(); +} + +void SrsTcpClient::set_send_timeout(int64_t timeout_us) +{ + io->set_send_timeout(timeout_us); +} + +int64_t SrsTcpClient::get_send_timeout() +{ + return io->get_send_timeout(); +} + +int64_t SrsTcpClient::get_recv_bytes() +{ + return io->get_recv_bytes(); +} + +int64_t SrsTcpClient::get_send_bytes() +{ + return io->get_send_bytes(); +} + +int SrsTcpClient::read(void* buf, size_t size, ssize_t* nread) +{ + return io->read(buf, size, nread); +} + +int SrsTcpClient::read_fully(void* buf, size_t size, ssize_t* nread) +{ + return io->read_fully(buf, size, nread); +} + +int SrsTcpClient::write(void* buf, size_t size, ssize_t* nwrite) +{ + return io->write(buf, size, nwrite); +} + +int SrsTcpClient::writev(const iovec *iov, int iov_size, ssize_t* nwrite) +{ + return io->writev(iov, iov_size, nwrite); +} + #ifdef __linux__ #include diff --git a/trunk/src/app/srs_app_st.hpp b/trunk/src/app/srs_app_st.hpp index 6125c50f0..faa30f6d5 100644 --- a/trunk/src/app/srs_app_st.hpp +++ b/trunk/src/app/srs_app_st.hpp @@ -208,19 +208,46 @@ public: * the common tcp client, to connect to specified TCP server, * reconnect and close the connection. */ -class SrsTcpClient +class SrsTcpClient : public ISrsProtocolReaderWriter { +private: + st_netfd_t stfd; + SrsStSocket* io; public: SrsTcpClient(); virtual ~SrsTcpClient(); +public: + /** + * whether connected to server. + */ + virtual bool connected(); public: /** * connect to server over TCP. * @param host the ip or hostname of server. * @param port the port to connect to. * @param timeout the timeout in us. + * @remark ignore when connected. */ virtual int connect(std::string host, int port, int64_t timeout); + /** + * close the connection. + * @remark ignore when closed. + */ + virtual void close(); +// interface ISrsProtocolReaderWriter +public: + virtual bool is_never_timeout(int64_t timeout_us); + virtual void set_recv_timeout(int64_t timeout_us); + virtual int64_t get_recv_timeout(); + virtual void set_send_timeout(int64_t timeout_us); + virtual int64_t get_send_timeout(); + virtual int64_t get_recv_bytes(); + virtual int64_t get_send_bytes(); + virtual int read(void* buf, size_t size, ssize_t* nread); + virtual int read_fully(void* buf, size_t size, ssize_t* nread); + virtual int write(void* buf, size_t size, ssize_t* nwrite); + virtual int writev(const iovec *iov, int iov_size, ssize_t* nwrite); }; // initialize st, requires epoll. From 24b38999725b3516e48bef63acd0f3293a8c4636 Mon Sep 17 00:00:00 2001 From: winlin Date: Wed, 14 Oct 2015 10:48:08 +0800 Subject: [PATCH 07/46] use SrsTcpClient instead raw socket. --- trunk/src/app/srs_app_caster_flv.cpp | 4 ++- trunk/src/app/srs_app_edge.cpp | 50 ++++++++------------------ trunk/src/app/srs_app_edge.hpp | 9 ++--- trunk/src/app/srs_app_forward.cpp | 25 ++++--------- trunk/src/app/srs_app_forward.hpp | 4 +-- trunk/src/app/srs_app_http_client.cpp | 31 +++++++--------- trunk/src/app/srs_app_http_client.hpp | 4 +-- trunk/src/app/srs_app_http_conn.cpp | 14 ++++---- trunk/src/app/srs_app_http_conn.hpp | 10 +++--- trunk/src/app/srs_app_mpegts_udp.cpp | 23 ++++++------ trunk/src/app/srs_app_mpegts_udp.hpp | 3 +- trunk/src/app/srs_app_rtmp_conn.cpp | 25 +++++-------- trunk/src/app/srs_app_rtmp_conn.hpp | 2 +- trunk/src/app/srs_app_rtsp.cpp | 34 ++++++++++-------- trunk/src/app/srs_app_rtsp.hpp | 2 +- trunk/src/main/srs_main_ingest_hls.cpp | 20 +++++------ 16 files changed, 107 insertions(+), 153 deletions(-) diff --git a/trunk/src/app/srs_app_caster_flv.cpp b/trunk/src/app/srs_app_caster_flv.cpp index 20dc505f7..8edb04815 100644 --- a/trunk/src/app/srs_app_caster_flv.cpp +++ b/trunk/src/app/srs_app_caster_flv.cpp @@ -276,7 +276,9 @@ int SrsDynamicHttpConn::connect() if ((ret = transport->connect(req->host, req->port, ST_UTIME_NO_TIMEOUT)) != ERROR_SUCCESS) { return ret; } - client = new SrsRtmpClient(transport); + + srs_freep(client); + client = new SrsRtmpClient(transport); client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); diff --git a/trunk/src/app/srs_app_edge.cpp b/trunk/src/app/srs_app_edge.cpp index 2ac192bb5..3d88e6c63 100644 --- a/trunk/src/app/srs_app_edge.cpp +++ b/trunk/src/app/srs_app_edge.cpp @@ -63,13 +63,12 @@ using namespace std; SrsEdgeIngester::SrsEdgeIngester() { - io = NULL; + transport = new SrsTcpClient(); kbps = new SrsKbps(); client = NULL; _edge = NULL; _req = NULL; stream_id = 0; - stfd = NULL; lb = new SrsLbRoundRobin(); pthread = new SrsReusableThread2("edge-igs", this, SRS_EDGE_INGESTER_SLEEP_US); } @@ -78,6 +77,7 @@ SrsEdgeIngester::~SrsEdgeIngester() { stop(); + srs_freep(transport); srs_freep(lb); srs_freep(pthread); srs_freep(kbps); @@ -109,11 +109,9 @@ int SrsEdgeIngester::start() void SrsEdgeIngester::stop() { pthread->stop(); - - close_underlayer_socket(); + transport->close(); srs_freep(client); - srs_freep(io); kbps->set_io(NULL, NULL); // notice to unpublish. @@ -336,17 +334,12 @@ int SrsEdgeIngester::process_publish_message(SrsCommonMessage* msg) return ret; } -void SrsEdgeIngester::close_underlayer_socket() -{ - srs_close_stfd(stfd); -} - int SrsEdgeIngester::connect_server(string& ep_server, int& ep_port) { int ret = ERROR_SUCCESS; // reopen - close_underlayer_socket(); + transport->close(); SrsConfDirective* conf = _srs_config->get_vhost_edge_origin(_req->vhost); @@ -368,20 +361,16 @@ int SrsEdgeIngester::connect_server(string& ep_server, int& ep_port) // open socket. int64_t timeout = SRS_EDGE_INGESTER_TIMEOUT_US; - if ((ret = srs_socket_connect(ep_server, ep_port, timeout, &stfd)) != ERROR_SUCCESS) { + if ((ret = transport->connect(ep_server, ep_port, timeout)) != ERROR_SUCCESS) { srs_warn("edge pull failed, stream=%s, tcUrl=%s to server=%s, port=%d, timeout=%"PRId64", ret=%d", _req->stream.c_str(), _req->tcUrl.c_str(), ep_server.c_str(), ep_port, timeout, ret); return ret; } srs_freep(client); - srs_freep(io); - - srs_assert(stfd); - io = new SrsStSocket(stfd); - client = new SrsRtmpClient(io); + client = new SrsRtmpClient(transport); - kbps->set_io(io, io); + kbps->set_io(transport, transport); srs_trace("edge pull connected, url=%s/%s, server=%s:%d", _req->tcUrl.c_str(), _req->stream.c_str(), ep_server.c_str(), ep_port); @@ -391,14 +380,13 @@ int SrsEdgeIngester::connect_server(string& ep_server, int& ep_port) SrsEdgeForwarder::SrsEdgeForwarder() { - io = NULL; + transport = new SrsTcpClient(); kbps = new SrsKbps(); client = NULL; _edge = NULL; _req = NULL; lb = new SrsLbRoundRobin(); stream_id = 0; - stfd = NULL; pthread = new SrsReusableThread2("edge-fwr", this, SRS_EDGE_FORWARDER_SLEEP_US); queue = new SrsMessageQueue(); send_error_code = ERROR_SUCCESS; @@ -408,6 +396,7 @@ SrsEdgeForwarder::~SrsEdgeForwarder() { stop(); + srs_freep(transport); srs_freep(lb); srs_freep(pthread); srs_freep(queue); @@ -473,13 +462,11 @@ int SrsEdgeForwarder::start() void SrsEdgeForwarder::stop() { pthread->stop(); - - close_underlayer_socket(); + transport->close(); queue->clear(); srs_freep(client); - srs_freep(io); kbps->set_io(NULL, NULL); } @@ -586,17 +573,12 @@ int SrsEdgeForwarder::proxy(SrsCommonMessage* msg) return ret; } -void SrsEdgeForwarder::close_underlayer_socket() -{ - srs_close_stfd(stfd); -} - int SrsEdgeForwarder::connect_server(string& ep_server, int& ep_port) { int ret = ERROR_SUCCESS; // reopen - close_underlayer_socket(); + transport->close(); SrsConfDirective* conf = _srs_config->get_vhost_edge_origin(_req->vhost); srs_assert(conf); @@ -610,20 +592,16 @@ int SrsEdgeForwarder::connect_server(string& ep_server, int& ep_port) // open socket. int64_t timeout = SRS_EDGE_FORWARDER_TIMEOUT_US; - if ((ret = srs_socket_connect(ep_server, ep_port, timeout, &stfd)) != ERROR_SUCCESS) { + if ((ret = transport->connect(ep_server, ep_port, timeout)) != ERROR_SUCCESS) { srs_warn("edge push failed, stream=%s, tcUrl=%s to server=%s, port=%d, timeout=%"PRId64", ret=%d", _req->stream.c_str(), _req->tcUrl.c_str(), ep_server.c_str(), ep_port, timeout, ret); return ret; } srs_freep(client); - srs_freep(io); - - srs_assert(stfd); - io = new SrsStSocket(stfd); - client = new SrsRtmpClient(io); + client = new SrsRtmpClient(transport); - kbps->set_io(io, io); + kbps->set_io(transport, transport); // open socket. srs_trace("edge push connected, stream=%s, tcUrl=%s to server=%s, port=%d", diff --git a/trunk/src/app/srs_app_edge.hpp b/trunk/src/app/srs_app_edge.hpp index 3f05ce925..6fa22db2a 100644 --- a/trunk/src/app/srs_app_edge.hpp +++ b/trunk/src/app/srs_app_edge.hpp @@ -47,6 +47,7 @@ class SrsMessageQueue; class ISrsProtocolReaderWriter; class SrsKbps; class SrsLbRoundRobin; +class SrsTcpClient; /** * the state of edge, auto machine @@ -85,8 +86,7 @@ private: SrsPlayEdge* _edge; SrsRequest* _req; SrsReusableThread2* pthread; - st_netfd_t stfd; - ISrsProtocolReaderWriter* io; + SrsTcpClient* transport; SrsKbps* kbps; SrsRtmpClient* client; SrsLbRoundRobin* lb; @@ -103,7 +103,6 @@ public: virtual int cycle(); private: virtual int ingest(); - virtual void close_underlayer_socket(); virtual int connect_server(std::string& ep_server, int& ep_port); virtual int connect_app(std::string ep_server, int ep_port); virtual int process_publish_message(SrsCommonMessage* msg); @@ -121,8 +120,7 @@ private: SrsPublishEdge* _edge; SrsRequest* _req; SrsReusableThread2* pthread; - st_netfd_t stfd; - ISrsProtocolReaderWriter* io; + SrsTcpClient* transport; SrsKbps* kbps; SrsRtmpClient* client; SrsLbRoundRobin* lb; @@ -152,7 +150,6 @@ public: public: virtual int proxy(SrsCommonMessage* msg); private: - virtual void close_underlayer_socket(); virtual int connect_server(std::string& ep_server, int& ep_port); virtual int connect_app(std::string ep_server, int ep_port); }; diff --git a/trunk/src/app/srs_app_forward.cpp b/trunk/src/app/srs_app_forward.cpp index 21383b570..478cddcda 100644 --- a/trunk/src/app/srs_app_forward.cpp +++ b/trunk/src/app/srs_app_forward.cpp @@ -54,9 +54,8 @@ SrsForwarder::SrsForwarder(SrsSource* _source) source = _source; _req = NULL; - io = NULL; client = NULL; - stfd = NULL; + transport = new SrsTcpClient(); kbps = new SrsKbps(); stream_id = 0; @@ -71,6 +70,7 @@ SrsForwarder::~SrsForwarder() { on_unpublish(); + srs_freep(transport); srs_freep(pthread); srs_freep(queue); srs_freep(jitter); @@ -151,11 +151,9 @@ int SrsForwarder::on_publish() void SrsForwarder::on_unpublish() { pthread->stop(); - - close_underlayer_socket(); + transport->close(); srs_freep(client); - srs_freep(io); kbps->set_io(NULL, NULL); } @@ -271,11 +269,6 @@ int SrsForwarder::cycle() return ret; } -void SrsForwarder::close_underlayer_socket() -{ - srs_close_stfd(stfd); -} - void SrsForwarder::discovery_ep(string& server, int& port, string& tc_url) { SrsRequest* req = _req; @@ -292,7 +285,7 @@ int SrsForwarder::connect_server(string& ep_server, int& ep_port) int ret = ERROR_SUCCESS; // reopen - close_underlayer_socket(); + transport->close(); // discovery the server port and tcUrl from req and ep_forward. string tc_url; @@ -300,20 +293,16 @@ int SrsForwarder::connect_server(string& ep_server, int& ep_port) // open socket. int64_t timeout = SRS_FORWARDER_SLEEP_US; - if ((ret = srs_socket_connect(ep_server, ep_port, timeout, &stfd)) != ERROR_SUCCESS) { + if ((ret = transport->connect(ep_server, ep_port, timeout)) != ERROR_SUCCESS) { srs_warn("forward failed, stream=%s, tcUrl=%s to server=%s, port=%d, timeout=%"PRId64", ret=%d", _req->stream.c_str(), _req->tcUrl.c_str(), ep_server.c_str(), ep_port, timeout, ret); return ret; } srs_freep(client); - srs_freep(io); - - srs_assert(stfd); - io = new SrsStSocket(stfd); - client = new SrsRtmpClient(io); + client = new SrsRtmpClient(transport); - kbps->set_io(io, io); + kbps->set_io(transport, transport); srs_trace("forward connected, stream=%s, tcUrl=%s to server=%s, port=%d", _req->stream.c_str(), _req->tcUrl.c_str(), ep_server.c_str(), ep_port); diff --git a/trunk/src/app/srs_app_forward.hpp b/trunk/src/app/srs_app_forward.hpp index 5842f0ac6..6d108f899 100644 --- a/trunk/src/app/srs_app_forward.hpp +++ b/trunk/src/app/srs_app_forward.hpp @@ -56,11 +56,10 @@ private: SrsRequest* _req; int stream_id; private: - st_netfd_t stfd; SrsReusableThread2* pthread; private: SrsSource* source; - ISrsProtocolReaderWriter* io; + SrsTcpClient* transport; SrsKbps* kbps; SrsRtmpClient* client; SrsRtmpJitter* jitter; @@ -99,7 +98,6 @@ public: public: virtual int cycle(); private: - virtual void close_underlayer_socket(); virtual void discovery_ep(std::string& server, int& port, std::string& tc_url); virtual int connect_server(std::string& ep_server, int& ep_port); virtual int connect_app(std::string ep_server, int ep_port); diff --git a/trunk/src/app/srs_app_http_client.cpp b/trunk/src/app/srs_app_http_client.cpp index 08f4b2b0c..1bf9ff185 100644 --- a/trunk/src/app/srs_app_http_client.cpp +++ b/trunk/src/app/srs_app_http_client.cpp @@ -39,9 +39,7 @@ using namespace std; SrsHttpClient::SrsHttpClient() { - connected = false; - stfd = NULL; - skt = NULL; + transport = new SrsTcpClient(); parser = NULL; timeout_us = 0; port = 0; @@ -50,6 +48,8 @@ SrsHttpClient::SrsHttpClient() SrsHttpClient::~SrsHttpClient() { disconnect(); + + srs_freep(transport); srs_freep(parser); } @@ -102,7 +102,7 @@ int SrsHttpClient::post(string path, string req, ISrsHttpMessage** ppmsg) << req; std::string data = ss.str(); - if ((ret = skt->write((void*)data.c_str(), data.length(), NULL)) != ERROR_SUCCESS) { + if ((ret = transport->write((void*)data.c_str(), data.length(), NULL)) != ERROR_SUCCESS) { // disconnect when error. disconnect(); @@ -111,7 +111,7 @@ int SrsHttpClient::post(string path, string req, ISrsHttpMessage** ppmsg) } ISrsHttpMessage* msg = NULL; - if ((ret = parser->parse_message(skt, NULL, &msg)) != ERROR_SUCCESS) { + if ((ret = parser->parse_message(transport, NULL, &msg)) != ERROR_SUCCESS) { srs_error("parse http post response failed. ret=%d", ret); return ret; } @@ -148,7 +148,7 @@ int SrsHttpClient::get(string path, std::string req, ISrsHttpMessage** ppmsg) << req; std::string data = ss.str(); - if ((ret = skt->write((void*)data.c_str(), data.length(), NULL)) != ERROR_SUCCESS) { + if ((ret = transport->write((void*)data.c_str(), data.length(), NULL)) != ERROR_SUCCESS) { // disconnect when error. disconnect(); @@ -157,7 +157,7 @@ int SrsHttpClient::get(string path, std::string req, ISrsHttpMessage** ppmsg) } ISrsHttpMessage* msg = NULL; - if ((ret = parser->parse_message(skt, NULL, &msg)) != ERROR_SUCCESS) { + if ((ret = parser->parse_message(transport, NULL, &msg)) != ERROR_SUCCESS) { srs_error("parse http post response failed. ret=%d", ret); return ret; } @@ -171,37 +171,30 @@ int SrsHttpClient::get(string path, std::string req, ISrsHttpMessage** ppmsg) void SrsHttpClient::disconnect() { - connected = false; - - srs_close_stfd(stfd); - srs_freep(skt); + transport->close(); } int SrsHttpClient::connect() { int ret = ERROR_SUCCESS; - if (connected) { + if (transport->connected()) { return ret; } disconnect(); // open socket. - if ((ret = srs_socket_connect(host, port, timeout_us, &stfd)) != ERROR_SUCCESS) { + if ((ret = transport->connect(host, port, timeout_us)) != ERROR_SUCCESS) { srs_warn("http client failed, server=%s, port=%d, timeout=%"PRId64", ret=%d", host.c_str(), port, timeout_us, ret); return ret; } srs_info("connect to server success. server=%s, port=%d", host.c_str(), port); - srs_assert(!skt); - skt = new SrsStSocket(stfd); - connected = true; - // set the recv/send timeout in us. - skt->set_recv_timeout(timeout_us); - skt->set_send_timeout(timeout_us); + transport->set_recv_timeout(timeout_us); + transport->set_send_timeout(timeout_us); return ret; } diff --git a/trunk/src/app/srs_app_http_client.hpp b/trunk/src/app/srs_app_http_client.hpp index b05def7ca..c905277ee 100644 --- a/trunk/src/app/srs_app_http_client.hpp +++ b/trunk/src/app/srs_app_http_client.hpp @@ -49,9 +49,7 @@ class SrsStSocket; class SrsHttpClient { private: - bool connected; - st_netfd_t stfd; - SrsStSocket* skt; + SrsTcpClient* transport; SrsHttpParser* parser; private: int64_t timeout_us; diff --git a/trunk/src/app/srs_app_http_conn.cpp b/trunk/src/app/srs_app_http_conn.cpp index 5476ae41a..786e99dd0 100644 --- a/trunk/src/app/srs_app_http_conn.cpp +++ b/trunk/src/app/srs_app_http_conn.cpp @@ -303,7 +303,7 @@ int SrsHttpResponseWriter::send_header(char* data, int size) return skt->write((void*)buf.c_str(), buf.length(), NULL); } -SrsHttpResponseReader::SrsHttpResponseReader(SrsHttpMessage* msg, SrsStSocket* io) +SrsHttpResponseReader::SrsHttpResponseReader(SrsHttpMessage* msg, ISrsProtocolReaderWriter* io) { skt = io; owner = msg; @@ -494,7 +494,7 @@ int SrsHttpResponseReader::read_specified(char* data, int nb_data, int* nb_read) return ret; } -SrsHttpMessage::SrsHttpMessage(SrsStSocket* io, SrsConnection* c) : ISrsHttpMessage() +SrsHttpMessage::SrsHttpMessage(ISrsProtocolReaderWriter* io, SrsConnection* c) : ISrsHttpMessage() { conn = c; chunked = false; @@ -870,7 +870,7 @@ int SrsHttpParser::initialize(enum http_parser_type type, bool allow_jsonp) return ret; } -int SrsHttpParser::parse_message(SrsStSocket* skt, SrsConnection* conn, ISrsHttpMessage** ppmsg) +int SrsHttpParser::parse_message(ISrsProtocolReaderWriter* io, SrsConnection* conn, ISrsHttpMessage** ppmsg) { *ppmsg = NULL; @@ -887,7 +887,7 @@ int SrsHttpParser::parse_message(SrsStSocket* skt, SrsConnection* conn, ISrsHttp header_parsed = 0; // do parse - if ((ret = parse_message_imp(skt)) != ERROR_SUCCESS) { + if ((ret = parse_message_imp(io)) != ERROR_SUCCESS) { if (!srs_is_client_gracefully_close(ret)) { srs_error("parse http msg failed. ret=%d", ret); } @@ -895,7 +895,7 @@ int SrsHttpParser::parse_message(SrsStSocket* skt, SrsConnection* conn, ISrsHttp } // create msg - SrsHttpMessage* msg = new SrsHttpMessage(skt, conn); + SrsHttpMessage* msg = new SrsHttpMessage(io, conn); // initalize http msg, parse url. if ((ret = msg->update(url, jsonp, &header, buffer, headers)) != ERROR_SUCCESS) { @@ -910,7 +910,7 @@ int SrsHttpParser::parse_message(SrsStSocket* skt, SrsConnection* conn, ISrsHttp return ret; } -int SrsHttpParser::parse_message_imp(SrsStSocket* skt) +int SrsHttpParser::parse_message_imp(ISrsProtocolReaderWriter* io) { int ret = ERROR_SUCCESS; @@ -944,7 +944,7 @@ int SrsHttpParser::parse_message_imp(SrsStSocket* skt) // when nothing parsed, read more to parse. if (nparsed == 0) { // when requires more, only grow 1bytes, but the buffer will cache more. - if ((ret = buffer->grow(skt, buffer->size() + 1)) != ERROR_SUCCESS) { + if ((ret = buffer->grow(io, buffer->size() + 1)) != ERROR_SUCCESS) { if (!srs_is_client_gracefully_close(ret)) { srs_error("read body from server failed. ret=%d", ret); } diff --git a/trunk/src/app/srs_app_http_conn.hpp b/trunk/src/app/srs_app_http_conn.hpp index e54c3a1f5..2fab8765d 100644 --- a/trunk/src/app/srs_app_http_conn.hpp +++ b/trunk/src/app/srs_app_http_conn.hpp @@ -120,7 +120,7 @@ public: class SrsHttpResponseReader : virtual public ISrsHttpResponseReader { private: - SrsStSocket* skt; + ISrsProtocolReaderWriter* skt; SrsHttpMessage* owner; SrsFastStream* buffer; bool is_eof; @@ -131,7 +131,7 @@ private: // already read total bytes. int64_t nb_total_read; public: - SrsHttpResponseReader(SrsHttpMessage* msg, SrsStSocket* io); + SrsHttpResponseReader(SrsHttpMessage* msg, ISrsProtocolReaderWriter* io); virtual ~SrsHttpResponseReader(); public: /** @@ -208,7 +208,7 @@ private: // the method in QueryString will override the HTTP method. std::string jsonp_method; public: - SrsHttpMessage(SrsStSocket* io, SrsConnection* c); + SrsHttpMessage(ISrsProtocolReaderWriter* io, SrsConnection* c); virtual ~SrsHttpMessage(); public: /** @@ -332,12 +332,12 @@ public: * or error and *ppmsg must be NULL. * @remark, if success, *ppmsg always NOT-NULL, *ppmsg always is_complete(). */ - virtual int parse_message(SrsStSocket* skt, SrsConnection* conn, ISrsHttpMessage** ppmsg); + virtual int parse_message(ISrsProtocolReaderWriter* io, SrsConnection* conn, ISrsHttpMessage** ppmsg); private: /** * parse the HTTP message to member field: msg. */ - virtual int parse_message_imp(SrsStSocket* skt); + virtual int parse_message_imp(ISrsProtocolReaderWriter* io); private: static int on_message_begin(http_parser* parser); static int on_headers_complete(http_parser* parser); diff --git a/trunk/src/app/srs_app_mpegts_udp.cpp b/trunk/src/app/srs_app_mpegts_udp.cpp index 71593f72c..9bb92e46f 100644 --- a/trunk/src/app/srs_app_mpegts_udp.cpp +++ b/trunk/src/app/srs_app_mpegts_udp.cpp @@ -132,9 +132,8 @@ SrsMpegtsOverUdp::SrsMpegtsOverUdp(SrsConfDirective* c) output = _srs_config->get_stream_caster_output(c); req = NULL; - io = NULL; client = NULL; - stfd = NULL; + transport = new SrsTcpClient(); stream_id = 0; avc = new SrsRawH264Stream(); @@ -150,6 +149,7 @@ SrsMpegtsOverUdp::~SrsMpegtsOverUdp() { close(); + srs_freep(transport); srs_freep(buffer); srs_freep(stream); srs_freep(context); @@ -345,8 +345,8 @@ int SrsMpegtsOverUdp::on_ts_video(SrsTsMessage* msg, SrsBuffer* avs) } // ts tbn to flv tbn. - u_int32_t dts = msg->dts / 90; - u_int32_t pts = msg->dts / 90; + u_int32_t dts = (u_int32_t)(msg->dts / 90); + u_int32_t pts = (u_int32_t)(msg->dts / 90); // send each frame. while (!avs->empty()) { @@ -504,7 +504,7 @@ int SrsMpegtsOverUdp::on_ts_audio(SrsTsMessage* msg, SrsBuffer* avs) } // ts tbn to flv tbn. - u_int32_t dts = msg->dts / 90; + u_int32_t dts = (u_int32_t)(msg->dts / 90); // send each frame. while (!avs->empty()) { @@ -604,7 +604,7 @@ int SrsMpegtsOverUdp::connect() // when ok, ignore. // TODO: FIXME: should reconnect when disconnected. - if (io || client) { + if (transport->connected()) { return ret; } @@ -616,12 +616,13 @@ int SrsMpegtsOverUdp::connect() } // connect host. - if ((ret = srs_socket_connect(req->host, req->port, ST_UTIME_NO_TIMEOUT, &stfd)) != ERROR_SUCCESS) { + if ((ret = transport->connect(req->host, req->port, ST_UTIME_NO_TIMEOUT)) != ERROR_SUCCESS) { srs_error("mpegts: connect server %s:%d failed. ret=%d", req->host.c_str(), req->port, ret); return ret; } - io = new SrsStSocket(stfd); - client = new SrsRtmpClient(io); + + srs_freep(client); + client = new SrsRtmpClient(transport); client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); @@ -704,9 +705,9 @@ int SrsMpegtsOverUdp::connect_app(string ep_server, int ep_port) void SrsMpegtsOverUdp::close() { srs_freep(client); - srs_freep(io); srs_freep(req); - srs_close_stfd(stfd); + + transport->close(); } #endif diff --git a/trunk/src/app/srs_app_mpegts_udp.hpp b/trunk/src/app/srs_app_mpegts_udp.hpp index 98b4cd3c3..f2ba65727 100644 --- a/trunk/src/app/srs_app_mpegts_udp.hpp +++ b/trunk/src/app/srs_app_mpegts_udp.hpp @@ -86,8 +86,7 @@ private: std::string output; private: SrsRequest* req; - st_netfd_t stfd; - SrsStSocket* io; + SrsTcpClient* transport; SrsRtmpClient* client; int stream_id; private: diff --git a/trunk/src/app/srs_app_rtmp_conn.cpp b/trunk/src/app/srs_app_rtmp_conn.cpp index 5bc2d0dd2..c02dd8b75 100644 --- a/trunk/src/app/srs_app_rtmp_conn.cpp +++ b/trunk/src/app/srs_app_rtmp_conn.cpp @@ -1229,11 +1229,13 @@ int SrsRtmpConn::check_edge_token_traverse_auth() srs_assert(req); - st_netfd_t stsock = NULL; + SrsTcpClient* transport = new SrsTcpClient(); + SrsAutoFree(SrsTcpClient, transport); + vector args = _srs_config->get_vhost_edge_origin(req->vhost)->args; for (int i = 0; i < (int)args.size(); i++) { string hostport = args.at(i); - if ((ret = connect_server(hostport, &stsock)) == ERROR_SUCCESS) { + if ((ret = connect_server(hostport, transport)) == ERROR_SUCCESS) { break; } } @@ -1242,20 +1244,13 @@ int SrsRtmpConn::check_edge_token_traverse_auth() return ret; } - srs_assert(stsock); - SrsStSocket* io = new SrsStSocket(stsock); - SrsRtmpClient* client = new SrsRtmpClient(io); + SrsRtmpClient* client = new SrsRtmpClient(transport); + SrsAutoFree(SrsRtmpClient, client); - ret = do_token_traverse_auth(client); - - srs_freep(client); - srs_freep(io); - srs_close_stfd(stsock); - - return ret; + return do_token_traverse_auth(client); } -int SrsRtmpConn::connect_server(string hostport, st_netfd_t* pstsock) +int SrsRtmpConn::connect_server(string hostport, SrsTcpClient* transport) { int ret = ERROR_SUCCESS; @@ -1268,16 +1263,14 @@ int SrsRtmpConn::connect_server(string hostport, st_netfd_t* pstsock) srs_parse_hostport(hostport, server, port); // open socket. - st_netfd_t stsock = NULL; int64_t timeout = SRS_EDGE_TOKEN_TRAVERSE_TIMEOUT_US; - if ((ret = srs_socket_connect(server, port, timeout, &stsock)) != ERROR_SUCCESS) { + if ((ret = transport->connect(server, port, timeout)) != ERROR_SUCCESS) { srs_warn("edge token traverse failed, tcUrl=%s to server=%s, port=%d, timeout=%"PRId64", ret=%d", req->tcUrl.c_str(), server.c_str(), port, timeout, ret); return ret; } srs_info("edge token auth connected, url=%s/%s, server=%s:%d", req->tcUrl.c_str(), req->stream.c_str(), server.c_str(), port); - *pstsock = stsock; return ret; } diff --git a/trunk/src/app/srs_app_rtmp_conn.hpp b/trunk/src/app/srs_app_rtmp_conn.hpp index 005315aa4..5ae160537 100644 --- a/trunk/src/app/srs_app_rtmp_conn.hpp +++ b/trunk/src/app/srs_app_rtmp_conn.hpp @@ -134,7 +134,7 @@ private: virtual void set_sock_options(); private: virtual int check_edge_token_traverse_auth(); - virtual int connect_server(std::string hostport, st_netfd_t* pstsock); + virtual int connect_server(std::string hostport, SrsTcpClient* transport); virtual int do_token_traverse_auth(SrsRtmpClient* client); /** * when the connection disconnect, call this method. diff --git a/trunk/src/app/srs_app_rtsp.cpp b/trunk/src/app/srs_app_rtsp.cpp index b35eccb03..8ca22c1b1 100644 --- a/trunk/src/app/srs_app_rtsp.cpp +++ b/trunk/src/app/srs_app_rtsp.cpp @@ -195,8 +195,8 @@ SrsRtspConn::SrsRtspConn(SrsRtspCaster* c, st_netfd_t fd, std::string o) trd = new SrsOneCycleThread("rtsp", this); req = NULL; - io = NULL; client = NULL; + transport = new SrsTcpClient(); stream_id = 0; vjitter = new SrsRtspJitter(); ajitter = new SrsRtspJitter(); @@ -219,7 +219,7 @@ SrsRtspConn::~SrsRtspConn() srs_freep(rtsp); srs_freep(client); - srs_freep(io); + srs_freep(transport); srs_freep(req); srs_freep(vjitter); @@ -254,7 +254,7 @@ int SrsRtspConn::do_cycle() srs_info("rtsp: got rtsp request"); if (req->is_options()) { - SrsRtspOptionsResponse* res = new SrsRtspOptionsResponse(req->seq); + SrsRtspOptionsResponse* res = new SrsRtspOptionsResponse((int)req->seq); res->session = session; if ((ret = rtsp->send_message(res)) != ERROR_SUCCESS) { if (!srs_is_client_gracefully_close(ret)) { @@ -288,7 +288,7 @@ int SrsRtspConn::do_cycle() audio_sample_rate, audio_channel, rtsp_tcUrl.c_str(), rtsp_stream.c_str() ); - SrsRtspResponse* res = new SrsRtspResponse(req->seq); + SrsRtspResponse* res = new SrsRtspResponse((int)req->seq); res->session = session; if ((ret = rtsp->send_message(res)) != ERROR_SUCCESS) { if (!srs_is_client_gracefully_close(ret)) { @@ -328,7 +328,7 @@ int SrsRtspConn::do_cycle() session = "O9EaZ4bf"; // TODO: FIXME: generate session id. } - SrsRtspSetupResponse* res = new SrsRtspSetupResponse(req->seq); + SrsRtspSetupResponse* res = new SrsRtspSetupResponse((int)req->seq); res->client_port_min = req->transport->client_port_min; res->client_port_max = req->transport->client_port_max; res->local_port_min = lpm; @@ -341,7 +341,7 @@ int SrsRtspConn::do_cycle() return ret; } } else if (req->is_record()) { - SrsRtspResponse* res = new SrsRtspResponse(req->seq); + SrsRtspResponse* res = new SrsRtspResponse((int)req->seq); res->session = session; if ((ret = rtsp->send_message(res)) != ERROR_SUCCESS) { if (!srs_is_client_gracefully_close(ret)) { @@ -434,7 +434,11 @@ int SrsRtspConn::on_rtp_video(SrsRtpPacket* pkt, int64_t dts, int64_t pts) return ret; } - if ((ret = write_h264_ipb_frame(pkt->payload->bytes(), pkt->payload->length(), dts / 90, pts / 90)) != ERROR_SUCCESS) { + char* bytes = pkt->payload->bytes(); + int length = pkt->payload->length(); + u_int32_t fdts = (u_int32_t)(dts / 90); + u_int32_t fpts = (u_int32_t)(pts / 90); + if ((ret = write_h264_ipb_frame(bytes, length, fdts, fpts)) != ERROR_SUCCESS) { return ret; } @@ -476,7 +480,7 @@ int SrsRtspConn::kickoff_audio_cache(SrsRtpPacket* pkt, int64_t dts) int nb_frame = acache->audio_samples->sample_units[i].size; int64_t timestamp = (acache->dts + delta * i) / 90; acodec->aac_packet_type = 1; - if ((ret = write_audio_raw_frame(frame, nb_frame, acodec, timestamp)) != ERROR_SUCCESS) { + if ((ret = write_audio_raw_frame(frame, nb_frame, acodec, (u_int32_t)timestamp)) != ERROR_SUCCESS) { return ret; } } @@ -497,7 +501,7 @@ int SrsRtspConn::write_sequence_header() int64_t dts = vjitter->timestamp() / 90; // send video sps/pps - if ((ret = write_h264_sps_pps(dts, dts)) != ERROR_SUCCESS) { + if ((ret = write_h264_sps_pps((u_int32_t)dts, (u_int32_t)dts)) != ERROR_SUCCESS) { return ret; } @@ -535,7 +539,7 @@ int SrsRtspConn::write_sequence_header() break; }; - if ((ret = write_audio_raw_frame((char*)sh.data(), (int)sh.length(), acodec, dts)) != ERROR_SUCCESS) { + if ((ret = write_audio_raw_frame((char*)sh.data(), (int)sh.length(), acodec, (u_int32_t)dts)) != ERROR_SUCCESS) { return ret; } } @@ -642,7 +646,8 @@ int SrsRtspConn::connect() int ret = ERROR_SUCCESS; // when ok, ignore. - if (io || client) { + // TODO: FIXME: support reconnect. + if (transport->connected()) { return ret; } @@ -663,12 +668,13 @@ int SrsRtspConn::connect() } // connect host. - if ((ret = srs_socket_connect(req->host, req->port, ST_UTIME_NO_TIMEOUT, &stfd)) != ERROR_SUCCESS) { + if ((ret = transport->connect(req->host, req->port, ST_UTIME_NO_TIMEOUT)) != ERROR_SUCCESS) { srs_error("rtsp: connect server %s:%d failed. ret=%d", req->host.c_str(), req->port, ret); return ret; } - io = new SrsStSocket(stfd); - client = new SrsRtmpClient(io); + + srs_freep(client); + client = new SrsRtmpClient(transport); client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); diff --git a/trunk/src/app/srs_app_rtsp.hpp b/trunk/src/app/srs_app_rtsp.hpp index 3fa245053..135839b33 100644 --- a/trunk/src/app/srs_app_rtsp.hpp +++ b/trunk/src/app/srs_app_rtsp.hpp @@ -139,7 +139,7 @@ private: SrsOneCycleThread* trd; private: SrsRequest* req; - SrsStSocket* io; + SrsTcpClient* transport; SrsRtmpClient* client; SrsRtspJitter* vjitter; SrsRtspJitter* ajitter; diff --git a/trunk/src/main/srs_main_ingest_hls.cpp b/trunk/src/main/srs_main_ingest_hls.cpp index fdf2a911a..2fdde094f 100644 --- a/trunk/src/main/srs_main_ingest_hls.cpp +++ b/trunk/src/main/srs_main_ingest_hls.cpp @@ -637,8 +637,7 @@ private: int64_t raw_aac_dts; private: SrsRequest* req; - st_netfd_t stfd; - SrsStSocket* io; + SrsTcpClient* transport; SrsRtmpClient* client; int stream_id; private: @@ -658,9 +657,8 @@ public: raw_aac_dts = srs_update_system_time_ms(); req = NULL; - io = NULL; client = NULL; - stfd = NULL; + transport = new SrsTcpClient(); stream_id = 0; avc = new SrsRawH264Stream(); @@ -672,6 +670,7 @@ public: virtual ~SrsIngestSrsOutput() { close(); + srs_freep(transport); srs_freep(avc); srs_freep(aac); @@ -1211,7 +1210,7 @@ int SrsIngestSrsOutput::connect() // when ok, ignore. // TODO: FIXME: should reconnect when disconnected. - if (io || client) { + if (transport->connected()) { return ret; } @@ -1235,12 +1234,13 @@ int SrsIngestSrsOutput::connect() } // connect host. - if ((ret = srs_socket_connect(req->host, req->port, ST_UTIME_NO_TIMEOUT, &stfd)) != ERROR_SUCCESS) { + if ((ret = transport->connect(req->host, req->port, ST_UTIME_NO_TIMEOUT)) != ERROR_SUCCESS) { srs_error("mpegts: connect server %s:%d failed. ret=%d", req->host.c_str(), req->port, ret); return ret; } - io = new SrsStSocket(stfd); - client = new SrsRtmpClient(io); + + srs_freep(client); + client = new SrsRtmpClient(transport); client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); @@ -1326,9 +1326,9 @@ void SrsIngestSrsOutput::close() h264_sps_pps_sent = false; srs_freep(client); - srs_freep(io); srs_freep(req); - srs_close_stfd(stfd); + + transport->close(); } // the context for ingest hls stream. From adb8f243bfc021203c9267069a5ced82c4ad95b5 Mon Sep 17 00:00:00 2001 From: winlin Date: Wed, 14 Oct 2015 11:16:50 +0800 Subject: [PATCH 08/46] use simple rtmp client for raw connect app --- trunk/src/app/srs_app_caster_flv.cpp | 158 ++------------------------- trunk/src/app/srs_app_caster_flv.hpp | 15 +-- trunk/src/app/srs_app_rtmp_conn.cpp | 146 +++++++++++++++++++++++++ trunk/src/app/srs_app_rtmp_conn.hpp | 26 ++++- 4 files changed, 182 insertions(+), 163 deletions(-) diff --git a/trunk/src/app/srs_app_caster_flv.cpp b/trunk/src/app/srs_app_caster_flv.cpp index 8edb04815..6b9e88d3f 100644 --- a/trunk/src/app/srs_app_caster_flv.cpp +++ b/trunk/src/app/srs_app_caster_flv.cpp @@ -42,6 +42,7 @@ using namespace std; #include #include #include +#include #define SRS_HTTP_FLV_STREAM_BUFFER 4096 @@ -117,20 +118,13 @@ int SrsAppCasterFlv::serve_http(ISrsHttpResponseWriter* w, ISrsHttpMessage* r) SrsDynamicHttpConn::SrsDynamicHttpConn(IConnectionManager* cm, st_netfd_t fd, SrsHttpServeMux* m) : SrsHttpConn(cm, fd, m) { - - req = NULL; - transport = new SrsTcpClient(); - client = NULL; - stream_id = 0; - + sdk = new SrsSimpleRtmpClient(); pprint = SrsPithyPrint::create_caster(); } SrsDynamicHttpConn::~SrsDynamicHttpConn() { - close(); - - srs_freep(transport); + srs_freep(sdk); srs_freep(pprint); } @@ -176,7 +170,7 @@ int SrsDynamicHttpConn::proxy(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std } ret = do_proxy(rr, &dec); - close(); + sdk->close(); return ret; } @@ -189,7 +183,7 @@ int SrsDynamicHttpConn::do_proxy(ISrsHttpResponseReader* rr, SrsFlvDecoder* dec) while (!rr->eof()) { pprint->elapse(); - if ((ret = connect()) != ERROR_SUCCESS) { + if ((ret = sdk->connect(output)) != ERROR_SUCCESS) { return ret; } @@ -212,13 +206,17 @@ int SrsDynamicHttpConn::do_proxy(ISrsHttpResponseReader* rr, SrsFlvDecoder* dec) return ret; } - if ((ret = rtmp_write_packet(type, time, data, size)) != ERROR_SUCCESS) { + if ((ret = sdk->rtmp_write_packet(type, time, data, size)) != ERROR_SUCCESS) { if (!srs_is_client_gracefully_close(ret)) { srs_error("flv: proxy rtmp packet failed. ret=%d", ret); } return ret; } + if (pprint->can_print()) { + srs_trace("flv: send msg %d age=%d, dts=%d, size=%d", type, pprint->age(), time, size); + } + if ((ret = dec->read_previous_tag_size(pps)) != ERROR_SUCCESS) { if (!srs_is_client_gracefully_close(ret)) { srs_error("flv: proxy tag header pps failed. ret=%d", ret); @@ -230,142 +228,6 @@ int SrsDynamicHttpConn::do_proxy(ISrsHttpResponseReader* rr, SrsFlvDecoder* dec) return ret; } -int SrsDynamicHttpConn::rtmp_write_packet(char type, u_int32_t timestamp, char* data, int size) -{ - int ret = ERROR_SUCCESS; - - SrsSharedPtrMessage* msg = NULL; - - if ((ret = srs_rtmp_create_msg(type, timestamp, data, size, stream_id, &msg)) != ERROR_SUCCESS) { - srs_error("flv: create shared ptr msg failed. ret=%d", ret); - return ret; - } - srs_assert(msg); - - if (pprint->can_print()) { - srs_trace("flv: send msg %s age=%d, dts=%"PRId64", size=%d", - msg->is_audio()? "A":msg->is_video()? "V":"N", pprint->age(), msg->timestamp, msg->size); - } - - // send out encoded msg. - if ((ret = client->send_and_free_message(msg, stream_id)) != ERROR_SUCCESS) { - return ret; - } - - return ret; -} - -int SrsDynamicHttpConn::connect() -{ - int ret = ERROR_SUCCESS; - - // when ok, ignore. - // TODO: FIXME: should reconnect when disconnected. - if (transport->connected()) { - return ret; - } - - // parse uri - if (!req) { - req = new SrsRequest(); - srs_parse_rtmp_url(output, req->tcUrl, req->stream); - srs_discovery_tc_url(req->tcUrl, req->schema, req->host, req->vhost, req->app, req->port, req->param); - } - - // connect host. - if ((ret = transport->connect(req->host, req->port, ST_UTIME_NO_TIMEOUT)) != ERROR_SUCCESS) { - return ret; - } - - srs_freep(client); - client = new SrsRtmpClient(transport); - - client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); - - // connect to vhost/app - if ((ret = client->handshake()) != ERROR_SUCCESS) { - srs_error("mpegts: handshake with server failed. ret=%d", ret); - return ret; - } - if ((ret = connect_app(req->host, req->port)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed. ret=%d", ret); - return ret; - } - if ((ret = client->create_stream(stream_id)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed, stream_id=%d. ret=%d", stream_id, ret); - return ret; - } - - // publish. - if ((ret = client->publish(req->stream, stream_id)) != ERROR_SUCCESS) { - srs_error("mpegts: publish failed, stream=%s, stream_id=%d. ret=%d", - req->stream.c_str(), stream_id, ret); - return ret; - } - - return ret; -} - -// TODO: FIXME: refine the connect_app. -int SrsDynamicHttpConn::connect_app(string ep_server, int ep_port) -{ - int ret = ERROR_SUCCESS; - - // args of request takes the srs info. - if (req->args == NULL) { - req->args = SrsAmf0Any::object(); - } - - // notify server the edge identity, - // @see https://github.com/simple-rtmp-server/srs/issues/147 - SrsAmf0Object* data = req->args; - data->set("srs_sig", SrsAmf0Any::str(RTMP_SIG_SRS_KEY)); - data->set("srs_server", SrsAmf0Any::str(RTMP_SIG_SRS_KEY" "RTMP_SIG_SRS_VERSION" ("RTMP_SIG_SRS_URL_SHORT")")); - data->set("srs_license", SrsAmf0Any::str(RTMP_SIG_SRS_LICENSE)); - data->set("srs_role", SrsAmf0Any::str(RTMP_SIG_SRS_ROLE)); - data->set("srs_url", SrsAmf0Any::str(RTMP_SIG_SRS_URL)); - data->set("srs_version", SrsAmf0Any::str(RTMP_SIG_SRS_VERSION)); - data->set("srs_site", SrsAmf0Any::str(RTMP_SIG_SRS_WEB)); - data->set("srs_email", SrsAmf0Any::str(RTMP_SIG_SRS_EMAIL)); - data->set("srs_copyright", SrsAmf0Any::str(RTMP_SIG_SRS_COPYRIGHT)); - data->set("srs_primary", SrsAmf0Any::str(RTMP_SIG_SRS_PRIMARY)); - data->set("srs_authors", SrsAmf0Any::str(RTMP_SIG_SRS_AUTHROS)); - // for edge to directly get the id of client. - data->set("srs_pid", SrsAmf0Any::number(getpid())); - data->set("srs_id", SrsAmf0Any::number(_srs_context->get_id())); - - // local ip of edge - std::vector ips = srs_get_local_ipv4_ips(); - assert(_srs_config->get_stats_network() < (int)ips.size()); - std::string local_ip = ips[_srs_config->get_stats_network()]; - data->set("srs_server_ip", SrsAmf0Any::str(local_ip.c_str())); - - // generate the tcUrl - std::string param = ""; - std::string tc_url = srs_generate_tc_url(ep_server, req->vhost, req->app, ep_port, param); - - // upnode server identity will show in the connect_app of client. - // @see https://github.com/simple-rtmp-server/srs/issues/160 - // the debug_srs_upnode is config in vhost and default to true. - bool debug_srs_upnode = _srs_config->get_debug_srs_upnode(req->vhost); - if ((ret = client->connect_app(req->app, tc_url, req, debug_srs_upnode)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed, tcUrl=%s, dsu=%d. ret=%d", - tc_url.c_str(), debug_srs_upnode, ret); - return ret; - } - - return ret; -} - -void SrsDynamicHttpConn::close() -{ - transport->close(); - - srs_freep(client); - srs_freep(req); -} - SrsHttpFileReader::SrsHttpFileReader(ISrsHttpResponseReader* h) { http = h; diff --git a/trunk/src/app/srs_app_caster_flv.hpp b/trunk/src/app/srs_app_caster_flv.hpp index 7020fde41..9dd0f8550 100644 --- a/trunk/src/app/srs_app_caster_flv.hpp +++ b/trunk/src/app/srs_app_caster_flv.hpp @@ -44,6 +44,7 @@ class SrsPithyPrint; class ISrsHttpResponseReader; class SrsFlvDecoder; class SrsTcpClient; +class SrsSimpleRtmpClient; #include #include @@ -85,11 +86,7 @@ class SrsDynamicHttpConn : public SrsHttpConn private: std::string output; SrsPithyPrint* pprint; -private: - SrsRequest* req; - SrsTcpClient* transport; - SrsRtmpClient* client; - int stream_id; + SrsSimpleRtmpClient* sdk; public: SrsDynamicHttpConn(IConnectionManager* cm, st_netfd_t fd, SrsHttpServeMux* m); virtual ~SrsDynamicHttpConn(); @@ -99,14 +96,6 @@ public: virtual int proxy(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string o); private: virtual int do_proxy(ISrsHttpResponseReader* rr, SrsFlvDecoder* dec); - virtual int rtmp_write_packet(char type, u_int32_t timestamp, char* data, int size); -private: - // connect to rtmp output url. - // @remark ignore when not connected, reconnect when disconnected. - virtual int connect(); - virtual int connect_app(std::string ep_server, int ep_port); - // close the connected io and rtmp to ready to be re-connect. - virtual void close(); }; /** diff --git a/trunk/src/app/srs_app_rtmp_conn.cpp b/trunk/src/app/srs_app_rtmp_conn.cpp index c02dd8b75..d612af6e2 100644 --- a/trunk/src/app/srs_app_rtmp_conn.cpp +++ b/trunk/src/app/srs_app_rtmp_conn.cpp @@ -76,6 +76,152 @@ using namespace std; // when edge timeout, retry next. #define SRS_EDGE_TOKEN_TRAVERSE_TIMEOUT_US (int64_t)(3*1000*1000LL) +SrsSimpleRtmpClient::SrsSimpleRtmpClient() +{ + req = NULL; + client = NULL; + + transport = new SrsTcpClient(); + stream_id = 0; +} + +SrsSimpleRtmpClient::~SrsSimpleRtmpClient() +{ + close(); + + srs_freep(transport); +} + +int SrsSimpleRtmpClient::connect(string url) +{ + int ret = ERROR_SUCCESS; + + // when ok, ignore. + // TODO: FIXME: should reconnect when disconnected. + if (transport->connected()) { + return ret; + } + + // parse uri + if (!req) { + req = new SrsRequest(); + srs_parse_rtmp_url(url, req->tcUrl, req->stream); + srs_discovery_tc_url(req->tcUrl, req->schema, req->host, req->vhost, req->app, req->port, req->param); + } + + // connect host. + if ((ret = transport->connect(req->host, req->port, ST_UTIME_NO_TIMEOUT)) != ERROR_SUCCESS) { + return ret; + } + + srs_freep(client); + client = new SrsRtmpClient(transport); + + client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); + client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); + + // connect to vhost/app + if ((ret = client->handshake()) != ERROR_SUCCESS) { + srs_error("mpegts: handshake with server failed. ret=%d", ret); + return ret; + } + if ((ret = connect_app(req->host, req->port)) != ERROR_SUCCESS) { + srs_error("mpegts: connect with server failed. ret=%d", ret); + return ret; + } + if ((ret = client->create_stream(stream_id)) != ERROR_SUCCESS) { + srs_error("mpegts: connect with server failed, stream_id=%d. ret=%d", stream_id, ret); + return ret; + } + + // publish. + if ((ret = client->publish(req->stream, stream_id)) != ERROR_SUCCESS) { + srs_error("mpegts: publish failed, stream=%s, stream_id=%d. ret=%d", + req->stream.c_str(), stream_id, ret); + return ret; + } + + return ret; +} + +void SrsSimpleRtmpClient::close() +{ + transport->close(); + + srs_freep(client); + srs_freep(req); +} + +int SrsSimpleRtmpClient::rtmp_write_packet(char type, u_int32_t timestamp, char* data, int size) +{ + int ret = ERROR_SUCCESS; + + SrsSharedPtrMessage* msg = NULL; + + if ((ret = srs_rtmp_create_msg(type, timestamp, data, size, stream_id, &msg)) != ERROR_SUCCESS) { + srs_error("flv: create shared ptr msg failed. ret=%d", ret); + return ret; + } + srs_assert(msg); + + // send out encoded msg. + if ((ret = client->send_and_free_message(msg, stream_id)) != ERROR_SUCCESS) { + return ret; + } + + return ret; +} + +int SrsSimpleRtmpClient::connect_app(string ep_server, int ep_port) +{ + int ret = ERROR_SUCCESS; + + // args of request takes the srs info. + if (req->args == NULL) { + req->args = SrsAmf0Any::object(); + } + + // notify server the edge identity, + // @see https://github.com/simple-rtmp-server/srs/issues/147 + SrsAmf0Object* data = req->args; + data->set("srs_sig", SrsAmf0Any::str(RTMP_SIG_SRS_KEY)); + data->set("srs_server", SrsAmf0Any::str(RTMP_SIG_SRS_KEY" "RTMP_SIG_SRS_VERSION" ("RTMP_SIG_SRS_URL_SHORT")")); + data->set("srs_license", SrsAmf0Any::str(RTMP_SIG_SRS_LICENSE)); + data->set("srs_role", SrsAmf0Any::str(RTMP_SIG_SRS_ROLE)); + data->set("srs_url", SrsAmf0Any::str(RTMP_SIG_SRS_URL)); + data->set("srs_version", SrsAmf0Any::str(RTMP_SIG_SRS_VERSION)); + data->set("srs_site", SrsAmf0Any::str(RTMP_SIG_SRS_WEB)); + data->set("srs_email", SrsAmf0Any::str(RTMP_SIG_SRS_EMAIL)); + data->set("srs_copyright", SrsAmf0Any::str(RTMP_SIG_SRS_COPYRIGHT)); + data->set("srs_primary", SrsAmf0Any::str(RTMP_SIG_SRS_PRIMARY)); + data->set("srs_authors", SrsAmf0Any::str(RTMP_SIG_SRS_AUTHROS)); + // for edge to directly get the id of client. + data->set("srs_pid", SrsAmf0Any::number(getpid())); + data->set("srs_id", SrsAmf0Any::number(_srs_context->get_id())); + + // local ip of edge + std::vector ips = srs_get_local_ipv4_ips(); + assert(_srs_config->get_stats_network() < (int)ips.size()); + std::string local_ip = ips[_srs_config->get_stats_network()]; + data->set("srs_server_ip", SrsAmf0Any::str(local_ip.c_str())); + + // generate the tcUrl + std::string param = ""; + std::string tc_url = srs_generate_tc_url(ep_server, req->vhost, req->app, ep_port, param); + + // upnode server identity will show in the connect_app of client. + // @see https://github.com/simple-rtmp-server/srs/issues/160 + // the debug_srs_upnode is config in vhost and default to true. + bool debug_srs_upnode = _srs_config->get_debug_srs_upnode(req->vhost); + if ((ret = client->connect_app(req->app, tc_url, req, debug_srs_upnode)) != ERROR_SUCCESS) { + srs_error("mpegts: connect with server failed, tcUrl=%s, dsu=%d. ret=%d", + tc_url.c_str(), debug_srs_upnode, ret); + return ret; + } + + return ret; +} + SrsRtmpConn::SrsRtmpConn(SrsServer* svr, st_netfd_t c) : SrsConnection(svr, c) { diff --git a/trunk/src/app/srs_app_rtmp_conn.hpp b/trunk/src/app/srs_app_rtmp_conn.hpp index 5ae160537..be8192781 100644 --- a/trunk/src/app/srs_app_rtmp_conn.hpp +++ b/trunk/src/app/srs_app_rtmp_conn.hpp @@ -58,8 +58,30 @@ class SrsSecurity; class ISrsWakable; /** -* the client provides the main logic control for RTMP clients. -*/ + * the simple rtmp client stub, use SrsRtmpClient and provides high level APIs. + */ +class SrsSimpleRtmpClient +{ +private: + SrsRequest* req; + SrsTcpClient* transport; + SrsRtmpClient* client; + int stream_id; +public: + SrsSimpleRtmpClient(); + virtual ~SrsSimpleRtmpClient(); +public: + virtual int connect(std::string url); + virtual void close(); +public: + virtual int rtmp_write_packet(char type, u_int32_t timestamp, char* data, int size); +private: + virtual int connect_app(std::string ep_server, int ep_port); +}; + +/** + * the client provides the main logic control for RTMP clients. + */ class SrsRtmpConn : public virtual SrsConnection, public virtual ISrsReloadHandler { // for the thread to directly access any field of connection. From a9ad7b211bbd3bd78f0c70937faf2e0109d7ec0d Mon Sep 17 00:00:00 2001 From: winlin Date: Wed, 14 Oct 2015 11:22:12 +0800 Subject: [PATCH 09/46] refine simple rtmp client for post flv stream. --- trunk/src/app/srs_app_caster_flv.cpp | 3 ++- trunk/src/app/srs_app_rtmp_conn.cpp | 8 ++++---- trunk/src/app/srs_app_rtmp_conn.hpp | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/trunk/src/app/srs_app_caster_flv.cpp b/trunk/src/app/srs_app_caster_flv.cpp index 6b9e88d3f..dbee0cfd8 100644 --- a/trunk/src/app/srs_app_caster_flv.cpp +++ b/trunk/src/app/srs_app_caster_flv.cpp @@ -183,7 +183,7 @@ int SrsDynamicHttpConn::do_proxy(ISrsHttpResponseReader* rr, SrsFlvDecoder* dec) while (!rr->eof()) { pprint->elapse(); - if ((ret = sdk->connect(output)) != ERROR_SUCCESS) { + if ((ret = sdk->connect(output, SRS_CONSTS_RTMP_RECV_TIMEOUT_US)) != ERROR_SUCCESS) { return ret; } @@ -206,6 +206,7 @@ int SrsDynamicHttpConn::do_proxy(ISrsHttpResponseReader* rr, SrsFlvDecoder* dec) return ret; } + // TODO: FIXME: for post flv, reconnect when error. if ((ret = sdk->rtmp_write_packet(type, time, data, size)) != ERROR_SUCCESS) { if (!srs_is_client_gracefully_close(ret)) { srs_error("flv: proxy rtmp packet failed. ret=%d", ret); diff --git a/trunk/src/app/srs_app_rtmp_conn.cpp b/trunk/src/app/srs_app_rtmp_conn.cpp index d612af6e2..d4e91869f 100644 --- a/trunk/src/app/srs_app_rtmp_conn.cpp +++ b/trunk/src/app/srs_app_rtmp_conn.cpp @@ -92,7 +92,7 @@ SrsSimpleRtmpClient::~SrsSimpleRtmpClient() srs_freep(transport); } -int SrsSimpleRtmpClient::connect(string url) +int SrsSimpleRtmpClient::connect(string url, int64_t timeout) { int ret = ERROR_SUCCESS; @@ -110,15 +110,15 @@ int SrsSimpleRtmpClient::connect(string url) } // connect host. - if ((ret = transport->connect(req->host, req->port, ST_UTIME_NO_TIMEOUT)) != ERROR_SUCCESS) { + if ((ret = transport->connect(req->host, req->port, timeout)) != ERROR_SUCCESS) { return ret; } srs_freep(client); client = new SrsRtmpClient(transport); - client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); + client->set_recv_timeout(timeout); + client->set_send_timeout(timeout); // connect to vhost/app if ((ret = client->handshake()) != ERROR_SUCCESS) { diff --git a/trunk/src/app/srs_app_rtmp_conn.hpp b/trunk/src/app/srs_app_rtmp_conn.hpp index be8192781..c783cea5e 100644 --- a/trunk/src/app/srs_app_rtmp_conn.hpp +++ b/trunk/src/app/srs_app_rtmp_conn.hpp @@ -71,7 +71,7 @@ public: SrsSimpleRtmpClient(); virtual ~SrsSimpleRtmpClient(); public: - virtual int connect(std::string url); + virtual int connect(std::string url, int64_t timeout); virtual void close(); public: virtual int rtmp_write_packet(char type, u_int32_t timestamp, char* data, int size); From e4c852945feb32fecab88ecbe0ee75372cadbb9f Mon Sep 17 00:00:00 2001 From: winlin Date: Wed, 14 Oct 2015 11:47:35 +0800 Subject: [PATCH 10/46] refine code, support override vhost --- trunk/src/app/srs_app_caster_flv.cpp | 1 + trunk/src/app/srs_app_rtmp_conn.cpp | 87 ++++++++++++++++------------ trunk/src/app/srs_app_rtmp_conn.hpp | 6 +- 3 files changed, 55 insertions(+), 39 deletions(-) diff --git a/trunk/src/app/srs_app_caster_flv.cpp b/trunk/src/app/srs_app_caster_flv.cpp index dbee0cfd8..988628a7e 100644 --- a/trunk/src/app/srs_app_caster_flv.cpp +++ b/trunk/src/app/srs_app_caster_flv.cpp @@ -184,6 +184,7 @@ int SrsDynamicHttpConn::do_proxy(ISrsHttpResponseReader* rr, SrsFlvDecoder* dec) pprint->elapse(); if ((ret = sdk->connect(output, SRS_CONSTS_RTMP_RECV_TIMEOUT_US)) != ERROR_SUCCESS) { + srs_error("flv: connect %s failed. ret=%d", output.c_str(), ret); return ret; } diff --git a/trunk/src/app/srs_app_rtmp_conn.cpp b/trunk/src/app/srs_app_rtmp_conn.cpp index d4e91869f..d0e3011e8 100644 --- a/trunk/src/app/srs_app_rtmp_conn.cpp +++ b/trunk/src/app/srs_app_rtmp_conn.cpp @@ -93,6 +93,11 @@ SrsSimpleRtmpClient::~SrsSimpleRtmpClient() } int SrsSimpleRtmpClient::connect(string url, int64_t timeout) +{ + return connect(url, "", timeout); +} + +int SrsSimpleRtmpClient::connect(string url, string vhost, int64_t timeout) { int ret = ERROR_SUCCESS; @@ -122,21 +127,21 @@ int SrsSimpleRtmpClient::connect(string url, int64_t timeout) // connect to vhost/app if ((ret = client->handshake()) != ERROR_SUCCESS) { - srs_error("mpegts: handshake with server failed. ret=%d", ret); + srs_error("sdk: handshake with server failed. ret=%d", ret); return ret; } - if ((ret = connect_app(req->host, req->port)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed. ret=%d", ret); + if ((ret = connect_app(vhost)) != ERROR_SUCCESS) { + srs_error("sdk: connect with server failed. ret=%d", ret); return ret; } if ((ret = client->create_stream(stream_id)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed, stream_id=%d. ret=%d", stream_id, ret); + srs_error("sdk: connect with server failed, stream_id=%d. ret=%d", stream_id, ret); return ret; } // publish. if ((ret = client->publish(req->stream, stream_id)) != ERROR_SUCCESS) { - srs_error("mpegts: publish failed, stream=%s, stream_id=%d. ret=%d", + srs_error("sdk: publish failed, stream=%s, stream_id=%d. ret=%d", req->stream.c_str(), stream_id, ret); return ret; } @@ -144,35 +149,7 @@ int SrsSimpleRtmpClient::connect(string url, int64_t timeout) return ret; } -void SrsSimpleRtmpClient::close() -{ - transport->close(); - - srs_freep(client); - srs_freep(req); -} - -int SrsSimpleRtmpClient::rtmp_write_packet(char type, u_int32_t timestamp, char* data, int size) -{ - int ret = ERROR_SUCCESS; - - SrsSharedPtrMessage* msg = NULL; - - if ((ret = srs_rtmp_create_msg(type, timestamp, data, size, stream_id, &msg)) != ERROR_SUCCESS) { - srs_error("flv: create shared ptr msg failed. ret=%d", ret); - return ret; - } - srs_assert(msg); - - // send out encoded msg. - if ((ret = client->send_and_free_message(msg, stream_id)) != ERROR_SUCCESS) { - return ret; - } - - return ret; -} - -int SrsSimpleRtmpClient::connect_app(string ep_server, int ep_port) +int SrsSimpleRtmpClient::connect_app(string vhost) { int ret = ERROR_SUCCESS; @@ -185,7 +162,7 @@ int SrsSimpleRtmpClient::connect_app(string ep_server, int ep_port) // @see https://github.com/simple-rtmp-server/srs/issues/147 SrsAmf0Object* data = req->args; data->set("srs_sig", SrsAmf0Any::str(RTMP_SIG_SRS_KEY)); - data->set("srs_server", SrsAmf0Any::str(RTMP_SIG_SRS_KEY" "RTMP_SIG_SRS_VERSION" ("RTMP_SIG_SRS_URL_SHORT")")); + data->set("srs_server", SrsAmf0Any::str(RTMP_SIG_SRS_SERVER)); data->set("srs_license", SrsAmf0Any::str(RTMP_SIG_SRS_LICENSE)); data->set("srs_role", SrsAmf0Any::str(RTMP_SIG_SRS_ROLE)); data->set("srs_url", SrsAmf0Any::str(RTMP_SIG_SRS_URL)); @@ -207,14 +184,22 @@ int SrsSimpleRtmpClient::connect_app(string ep_server, int ep_port) // generate the tcUrl std::string param = ""; - std::string tc_url = srs_generate_tc_url(ep_server, req->vhost, req->app, ep_port, param); + std::string target_vhost = req->vhost; + if (vhost.empty()) { + target_vhost = vhost; + } + std::string tc_url = srs_generate_tc_url(req->host, target_vhost, req->app, req->port, param); + + // replace the tcUrl in request, + // which will replace the tc_url in client.connect_app(). + req->tcUrl = tc_url; // upnode server identity will show in the connect_app of client. // @see https://github.com/simple-rtmp-server/srs/issues/160 // the debug_srs_upnode is config in vhost and default to true. bool debug_srs_upnode = _srs_config->get_debug_srs_upnode(req->vhost); if ((ret = client->connect_app(req->app, tc_url, req, debug_srs_upnode)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed, tcUrl=%s, dsu=%d. ret=%d", + srs_error("sdk: connect with server failed, tcUrl=%s, dsu=%d. ret=%d", tc_url.c_str(), debug_srs_upnode, ret); return ret; } @@ -222,6 +207,34 @@ int SrsSimpleRtmpClient::connect_app(string ep_server, int ep_port) return ret; } +void SrsSimpleRtmpClient::close() +{ + transport->close(); + + srs_freep(client); + srs_freep(req); +} + +int SrsSimpleRtmpClient::rtmp_write_packet(char type, u_int32_t timestamp, char* data, int size) +{ + int ret = ERROR_SUCCESS; + + SrsSharedPtrMessage* msg = NULL; + + if ((ret = srs_rtmp_create_msg(type, timestamp, data, size, stream_id, &msg)) != ERROR_SUCCESS) { + srs_error("sdk: create shared ptr msg failed. ret=%d", ret); + return ret; + } + srs_assert(msg); + + // send out encoded msg. + if ((ret = client->send_and_free_message(msg, stream_id)) != ERROR_SUCCESS) { + return ret; + } + + return ret; +} + SrsRtmpConn::SrsRtmpConn(SrsServer* svr, st_netfd_t c) : SrsConnection(svr, c) { diff --git a/trunk/src/app/srs_app_rtmp_conn.hpp b/trunk/src/app/srs_app_rtmp_conn.hpp index c783cea5e..4c8741c4f 100644 --- a/trunk/src/app/srs_app_rtmp_conn.hpp +++ b/trunk/src/app/srs_app_rtmp_conn.hpp @@ -72,11 +72,13 @@ public: virtual ~SrsSimpleRtmpClient(); public: virtual int connect(std::string url, int64_t timeout); + virtual int connect(std::string url, std::string vhost, int64_t timeout); +private: + virtual int connect_app(std::string vhost); +public: virtual void close(); public: virtual int rtmp_write_packet(char type, u_int32_t timestamp, char* data, int size); -private: - virtual int connect_app(std::string ep_server, int ep_port); }; /** From bc274818864496a9241f501b46a646fb845ef6f1 Mon Sep 17 00:00:00 2001 From: winlin Date: Wed, 14 Oct 2015 14:37:24 +0800 Subject: [PATCH 11/46] refine code, use simple rtmp client. --- trunk/src/app/srs_app_caster_flv.cpp | 15 +- trunk/src/app/srs_app_edge.cpp | 213 ++++++------------------- trunk/src/app/srs_app_edge.hpp | 17 +- trunk/src/app/srs_app_rtmp_conn.cpp | 70 +++++++- trunk/src/app/srs_app_rtmp_conn.hpp | 11 ++ trunk/src/kernel/srs_kernel_consts.hpp | 3 + 6 files changed, 145 insertions(+), 184 deletions(-) diff --git a/trunk/src/app/srs_app_caster_flv.cpp b/trunk/src/app/srs_app_caster_flv.cpp index 988628a7e..400060188 100644 --- a/trunk/src/app/srs_app_caster_flv.cpp +++ b/trunk/src/app/srs_app_caster_flv.cpp @@ -179,15 +179,20 @@ int SrsDynamicHttpConn::do_proxy(ISrsHttpResponseReader* rr, SrsFlvDecoder* dec) { int ret = ERROR_SUCCESS; + if ((ret = sdk->connect(output, SRS_CONSTS_RTMP_RECV_TIMEOUT_US)) != ERROR_SUCCESS) { + srs_error("flv: connect %s failed. ret=%d", output.c_str(), ret); + return ret; + } + + if ((ret = sdk->publish()) != ERROR_SUCCESS) { + srs_error("flv: publish failed. ret=%d", ret); + return ret; + } + char pps[4]; while (!rr->eof()) { pprint->elapse(); - if ((ret = sdk->connect(output, SRS_CONSTS_RTMP_RECV_TIMEOUT_US)) != ERROR_SUCCESS) { - srs_error("flv: connect %s failed. ret=%d", output.c_str(), ret); - return ret; - } - char type; int32_t size; u_int32_t time; diff --git a/trunk/src/app/srs_app_edge.cpp b/trunk/src/app/srs_app_edge.cpp index 3d88e6c63..e942dca44 100644 --- a/trunk/src/app/srs_app_edge.cpp +++ b/trunk/src/app/srs_app_edge.cpp @@ -45,6 +45,7 @@ using namespace std; #include #include #include +#include // when error, edge ingester sleep for a while and retry. #define SRS_EDGE_INGESTER_SLEEP_US (int64_t)(1*1000*1000LL) @@ -63,12 +64,11 @@ using namespace std; SrsEdgeIngester::SrsEdgeIngester() { - transport = new SrsTcpClient(); - kbps = new SrsKbps(); - client = NULL; - _edge = NULL; - _req = NULL; - stream_id = 0; + source = NULL; + edge = NULL; + req = NULL; + + sdk = new SrsSimpleRtmpClient(); lb = new SrsLbRoundRobin(); pthread = new SrsReusableThread2("edge-igs", this, SRS_EDGE_INGESTER_SLEEP_US); } @@ -77,19 +77,18 @@ SrsEdgeIngester::~SrsEdgeIngester() { stop(); - srs_freep(transport); + srs_freep(sdk); srs_freep(lb); srs_freep(pthread); - srs_freep(kbps); } -int SrsEdgeIngester::initialize(SrsSource* source, SrsPlayEdge* edge, SrsRequest* req) +int SrsEdgeIngester::initialize(SrsSource* s, SrsPlayEdge* e, SrsRequest* r) { int ret = ERROR_SUCCESS; - _source = source; - _edge = edge; - _req = req; + source = s; + edge = e; + req = r; return ret; } @@ -98,7 +97,7 @@ int SrsEdgeIngester::start() { int ret = ERROR_SUCCESS; - if ((ret = _source->on_publish()) != ERROR_SUCCESS) { + if ((ret = source->on_publish()) != ERROR_SUCCESS) { srs_error("edge pull stream then publish to edge failed. ret=%d", ret); return ret; } @@ -109,13 +108,10 @@ int SrsEdgeIngester::start() void SrsEdgeIngester::stop() { pthread->stop(); - transport->close(); - - srs_freep(client); - kbps->set_io(NULL, NULL); + sdk->close(); // notice to unpublish. - _source->on_unpublish(); + source->on_unpublish(); } string SrsEdgeIngester::get_curr_origin() @@ -127,39 +123,47 @@ int SrsEdgeIngester::cycle() { int ret = ERROR_SUCCESS; - _source->on_source_id_changed(_srs_context->get_id()); + source->on_source_id_changed(_srs_context->get_id()); - std::string ep_server; - int ep_port; - if ((ret = connect_server(ep_server, ep_port)) != ERROR_SUCCESS) { - return ret; + std::string url, vhost; + if (true) { + SrsConfDirective* conf = _srs_config->get_vhost_edge_origin(req->vhost); + + // @see https://github.com/simple-rtmp-server/srs/issues/79 + // when origin is error, for instance, server is shutdown, + // then user remove the vhost then reload, the conf is empty. + if (!conf) { + ret = ERROR_EDGE_VHOST_REMOVED; + srs_warn("vhost %s removed. ret=%d", req->vhost.c_str(), ret); + return ret; + } + + // select the origin. + if (true) { + std::string server = lb->select(conf->args); + int port = SRS_CONSTS_RTMP_DEFAULT_PORT; + srs_parse_hostport(server, server, port); + + url = srs_generate_rtmp_url(server, port, req->vhost, req->app, req->stream); + } + + // support vhost tranform for edge, + // @see https://github.com/simple-rtmp-server/srs/issues/372 + vhost = _srs_config->get_vhost_edge_transform_vhost(req->vhost); + vhost = srs_string_replace(vhost, "[vhost]", req->vhost); } - srs_assert(client); - - client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); - - SrsRequest* req = _req; - if ((ret = client->handshake()) != ERROR_SUCCESS) { - srs_error("handshake with server failed. ret=%d", ret); - return ret; - } - if ((ret = connect_app(ep_server, ep_port)) != ERROR_SUCCESS) { - return ret; - } - if ((ret = client->create_stream(stream_id)) != ERROR_SUCCESS) { - srs_error("connect with server failed, stream_id=%d. ret=%d", stream_id, ret); + if ((ret = sdk->connect(url, vhost, SRS_CONSTS_RTMP_TIMEOUT_US)) != ERROR_SUCCESS) { + srs_error("edge pull %s failed. ret=%d", url.c_str(), ret); return ret; } - if ((ret = client->play(req->stream, stream_id)) != ERROR_SUCCESS) { - srs_error("connect with server failed, stream=%s, stream_id=%d. ret=%d", - req->stream.c_str(), stream_id, ret); + if ((ret = sdk->play()) != ERROR_SUCCESS) { + srs_error("edge pull %s stream failed. ret=%d", url.c_str(), ret); return ret; } - if ((ret = _edge->on_ingest_play()) != ERROR_SUCCESS) { + if ((ret = edge->on_ingest_play()) != ERROR_SUCCESS) { return ret; } @@ -176,27 +180,22 @@ int SrsEdgeIngester::ingest() { int ret = ERROR_SUCCESS; - client->set_recv_timeout(SRS_EDGE_INGESTER_TIMEOUT_US); + sdk->set_recv_timeout(SRS_EDGE_INGESTER_TIMEOUT_US); SrsPithyPrint* pprint = SrsPithyPrint::create_edge(); SrsAutoFree(SrsPithyPrint, pprint); - + while (!pthread->interrupted()) { pprint->elapse(); // pithy print if (pprint->can_print()) { - kbps->sample(); - srs_trace("<- "SRS_CONSTS_LOG_EDGE_PLAY - " time=%"PRId64", okbps=%d,%d,%d, ikbps=%d,%d,%d", - pprint->age(), - kbps->get_send_kbps(), kbps->get_send_kbps_30s(), kbps->get_send_kbps_5m(), - kbps->get_recv_kbps(), kbps->get_recv_kbps_30s(), kbps->get_recv_kbps_5m()); + sdk->kbps_sample(SRS_CONSTS_LOG_EDGE_PLAY, pprint->age()); } - + // read from client. SrsCommonMessage* msg = NULL; - if ((ret = client->recv_message(&msg)) != ERROR_SUCCESS) { + if ((ret = sdk->recv_message(&msg)) != ERROR_SUCCESS) { if (!srs_is_client_gracefully_close(ret)) { srs_error("pull origin server message failed. ret=%d", ret); } @@ -211,68 +210,6 @@ int SrsEdgeIngester::ingest() return ret; } } - - return ret; -} - -// TODO: FIXME: refine the connect_app. -int SrsEdgeIngester::connect_app(string ep_server, int ep_port) -{ - int ret = ERROR_SUCCESS; - - SrsRequest* req = _req; - - // args of request takes the srs info. - if (req->args == NULL) { - req->args = SrsAmf0Any::object(); - } - - // notify server the edge identity, - // @see https://github.com/simple-rtmp-server/srs/issues/147 - SrsAmf0Object* data = req->args; - data->set("srs_sig", SrsAmf0Any::str(RTMP_SIG_SRS_KEY)); - data->set("srs_server", SrsAmf0Any::str(RTMP_SIG_SRS_SERVER)); - data->set("srs_license", SrsAmf0Any::str(RTMP_SIG_SRS_LICENSE)); - data->set("srs_role", SrsAmf0Any::str(RTMP_SIG_SRS_ROLE)); - data->set("srs_url", SrsAmf0Any::str(RTMP_SIG_SRS_URL)); - data->set("srs_version", SrsAmf0Any::str(RTMP_SIG_SRS_VERSION)); - data->set("srs_site", SrsAmf0Any::str(RTMP_SIG_SRS_WEB)); - data->set("srs_email", SrsAmf0Any::str(RTMP_SIG_SRS_EMAIL)); - data->set("srs_copyright", SrsAmf0Any::str(RTMP_SIG_SRS_COPYRIGHT)); - data->set("srs_primary", SrsAmf0Any::str(RTMP_SIG_SRS_PRIMARY)); - data->set("srs_authors", SrsAmf0Any::str(RTMP_SIG_SRS_AUTHROS)); - // for edge to directly get the id of client. - data->set("srs_pid", SrsAmf0Any::number(getpid())); - data->set("srs_id", SrsAmf0Any::number(_srs_context->get_id())); - - // local ip of edge - std::vector ips = srs_get_local_ipv4_ips(); - assert(_srs_config->get_stats_network() < (int)ips.size()); - std::string local_ip = ips[_srs_config->get_stats_network()]; - data->set("srs_server_ip", SrsAmf0Any::str(local_ip.c_str())); - - // support vhost tranform for edge, - // @see https://github.com/simple-rtmp-server/srs/issues/372 - std::string vhost = _srs_config->get_vhost_edge_transform_vhost(req->vhost); - vhost = srs_string_replace(vhost, "[vhost]", req->vhost); - // generate the tcUrl - std::string param = ""; - std::string tc_url = srs_generate_tc_url(ep_server, vhost, req->app, ep_port, param); - srs_trace("edge ingest from %s:%d at %s", ep_server.c_str(), ep_port, tc_url.c_str()); - - // replace the tcUrl in request, - // which will replace the tc_url in client.connect_app(). - req->tcUrl = tc_url; - - // upnode server identity will show in the connect_app of client. - // @see https://github.com/simple-rtmp-server/srs/issues/160 - // the debug_srs_upnode is config in vhost and default to true. - bool debug_srs_upnode = _srs_config->get_debug_srs_upnode(req->vhost); - if ((ret = client->connect_app(req->app, tc_url, req, debug_srs_upnode)) != ERROR_SUCCESS) { - srs_error("connect with server failed, tcUrl=%s, dsu=%d. ret=%d", - tc_url.c_str(), debug_srs_upnode, ret); - return ret; - } return ret; } @@ -280,8 +217,6 @@ int SrsEdgeIngester::connect_app(string ep_server, int ep_port) int SrsEdgeIngester::process_publish_message(SrsCommonMessage* msg) { int ret = ERROR_SUCCESS; - - SrsSource* source = _source; // process audio packet if (msg->header.is_audio()) { @@ -311,7 +246,7 @@ int SrsEdgeIngester::process_publish_message(SrsCommonMessage* msg) // process onMetaData if (msg->header.is_amf0_data() || msg->header.is_amf3_data()) { SrsPacket* pkt = NULL; - if ((ret = client->decode_message(msg, &pkt)) != ERROR_SUCCESS) { + if ((ret = sdk->decode_message(msg, &pkt)) != ERROR_SUCCESS) { srs_error("decode onMetaData message failed. ret=%d", ret); return ret; } @@ -334,50 +269,6 @@ int SrsEdgeIngester::process_publish_message(SrsCommonMessage* msg) return ret; } -int SrsEdgeIngester::connect_server(string& ep_server, int& ep_port) -{ - int ret = ERROR_SUCCESS; - - // reopen - transport->close(); - - SrsConfDirective* conf = _srs_config->get_vhost_edge_origin(_req->vhost); - - // @see https://github.com/simple-rtmp-server/srs/issues/79 - // when origin is error, for instance, server is shutdown, - // then user remove the vhost then reload, the conf is empty. - if (!conf) { - ret = ERROR_EDGE_VHOST_REMOVED; - srs_warn("vhost %s removed. ret=%d", _req->vhost.c_str(), ret); - return ret; - } - - // select the origin. - if (true) { - std::string server = lb->select(conf->args); - ep_port = SRS_CONSTS_RTMP_DEFAULT_PORT; - srs_parse_hostport(server, ep_server, ep_port); - } - - // open socket. - int64_t timeout = SRS_EDGE_INGESTER_TIMEOUT_US; - if ((ret = transport->connect(ep_server, ep_port, timeout)) != ERROR_SUCCESS) { - srs_warn("edge pull failed, stream=%s, tcUrl=%s to server=%s, port=%d, timeout=%"PRId64", ret=%d", - _req->stream.c_str(), _req->tcUrl.c_str(), ep_server.c_str(), ep_port, timeout, ret); - return ret; - } - - srs_freep(client); - client = new SrsRtmpClient(transport); - - kbps->set_io(transport, transport); - - srs_trace("edge pull connected, url=%s/%s, server=%s:%d", - _req->tcUrl.c_str(), _req->stream.c_str(), ep_server.c_str(), ep_port); - - return ret; -} - SrsEdgeForwarder::SrsEdgeForwarder() { transport = new SrsTcpClient(); diff --git a/trunk/src/app/srs_app_edge.hpp b/trunk/src/app/srs_app_edge.hpp index 6fa22db2a..de3b4891f 100644 --- a/trunk/src/app/srs_app_edge.hpp +++ b/trunk/src/app/srs_app_edge.hpp @@ -48,6 +48,7 @@ class ISrsProtocolReaderWriter; class SrsKbps; class SrsLbRoundRobin; class SrsTcpClient; +class SrsSimpleRtmpClient; /** * the state of edge, auto machine @@ -80,21 +81,17 @@ enum SrsEdgeUserState class SrsEdgeIngester : public ISrsReusableThread2Handler { private: - int stream_id; -private: - SrsSource* _source; - SrsPlayEdge* _edge; - SrsRequest* _req; + SrsSource* source; + SrsPlayEdge* edge; + SrsRequest* req; SrsReusableThread2* pthread; - SrsTcpClient* transport; - SrsKbps* kbps; - SrsRtmpClient* client; + SrsSimpleRtmpClient* sdk; SrsLbRoundRobin* lb; public: SrsEdgeIngester(); virtual ~SrsEdgeIngester(); public: - virtual int initialize(SrsSource* source, SrsPlayEdge* edge, SrsRequest* req); + virtual int initialize(SrsSource* s, SrsPlayEdge* e, SrsRequest* r); virtual int start(); virtual void stop(); virtual std::string get_curr_origin(); @@ -103,8 +100,6 @@ public: virtual int cycle(); private: virtual int ingest(); - virtual int connect_server(std::string& ep_server, int& ep_port); - virtual int connect_app(std::string ep_server, int ep_port); virtual int process_publish_message(SrsCommonMessage* msg); }; diff --git a/trunk/src/app/srs_app_rtmp_conn.cpp b/trunk/src/app/srs_app_rtmp_conn.cpp index d0e3011e8..e45f2e8db 100644 --- a/trunk/src/app/srs_app_rtmp_conn.cpp +++ b/trunk/src/app/srs_app_rtmp_conn.cpp @@ -80,6 +80,7 @@ SrsSimpleRtmpClient::SrsSimpleRtmpClient() { req = NULL; client = NULL; + kbps = new SrsKbps(); transport = new SrsTcpClient(); stream_id = 0; @@ -89,7 +90,11 @@ SrsSimpleRtmpClient::~SrsSimpleRtmpClient() { close(); + srs_freep(kbps); srs_freep(transport); + + srs_freep(client); + kbps->set_io(NULL, NULL); } int SrsSimpleRtmpClient::connect(string url, int64_t timeout) @@ -122,6 +127,8 @@ int SrsSimpleRtmpClient::connect(string url, string vhost, int64_t timeout) srs_freep(client); client = new SrsRtmpClient(transport); + kbps->set_io(transport, transport); + client->set_recv_timeout(timeout); client->set_send_timeout(timeout); @@ -139,13 +146,6 @@ int SrsSimpleRtmpClient::connect(string url, string vhost, int64_t timeout) return ret; } - // publish. - if ((ret = client->publish(req->stream, stream_id)) != ERROR_SUCCESS) { - srs_error("sdk: publish failed, stream=%s, stream_id=%d. ret=%d", - req->stream.c_str(), stream_id, ret); - return ret; - } - return ret; } @@ -215,6 +215,47 @@ void SrsSimpleRtmpClient::close() srs_freep(req); } +int SrsSimpleRtmpClient::publish() +{ + int ret = ERROR_SUCCESS; + + // publish. + if ((ret = client->publish(req->stream, stream_id)) != ERROR_SUCCESS) { + srs_error("sdk: publish failed, stream=%s, stream_id=%d. ret=%d", + req->stream.c_str(), stream_id, ret); + return ret; + } + + return ret; +} + +int SrsSimpleRtmpClient::play() +{ + int ret = ERROR_SUCCESS; + + if ((ret = client->play(req->stream, stream_id)) != ERROR_SUCCESS) { + srs_error("connect with server failed, stream=%s, stream_id=%d. ret=%d", + req->stream.c_str(), stream_id, ret); + return ret; + } + + return ret; +} + +void SrsSimpleRtmpClient::kbps_sample(const char* label, int64_t age) +{ + kbps->sample(); + + int sr = kbps->get_send_kbps(); + int sr30s = kbps->get_send_kbps_30s(); + int sr5m = kbps->get_send_kbps_5m(); + int rr = kbps->get_recv_kbps(); + int rr30s = kbps->get_recv_kbps_30s(); + int rr5m = kbps->get_recv_kbps_5m(); + + srs_trace("<- %s time=%"PRId64", okbps=%d,%d,%d, ikbps=%d,%d,%d", age, sr, sr30s, sr5m, rr, rr30s, rr5m); +} + int SrsSimpleRtmpClient::rtmp_write_packet(char type, u_int32_t timestamp, char* data, int size) { int ret = ERROR_SUCCESS; @@ -235,6 +276,21 @@ int SrsSimpleRtmpClient::rtmp_write_packet(char type, u_int32_t timestamp, char* return ret; } +int SrsSimpleRtmpClient::recv_message(SrsCommonMessage** pmsg) +{ + return client->recv_message(pmsg); +} + +int SrsSimpleRtmpClient::decode_message(SrsCommonMessage* msg, SrsPacket** ppacket) +{ + return client->decode_message(msg, ppacket); +} + +void SrsSimpleRtmpClient::set_recv_timeout(int64_t timeout) +{ + transport->set_recv_timeout(timeout); +} + SrsRtmpConn::SrsRtmpConn(SrsServer* svr, st_netfd_t c) : SrsConnection(svr, c) { diff --git a/trunk/src/app/srs_app_rtmp_conn.hpp b/trunk/src/app/srs_app_rtmp_conn.hpp index 4c8741c4f..e274e471a 100644 --- a/trunk/src/app/srs_app_rtmp_conn.hpp +++ b/trunk/src/app/srs_app_rtmp_conn.hpp @@ -56,6 +56,8 @@ class SrsQueueRecvThread; class SrsPublishRecvThread; class SrsSecurity; class ISrsWakable; +class SrsCommonMessage; +class SrsPacket; /** * the simple rtmp client stub, use SrsRtmpClient and provides high level APIs. @@ -66,6 +68,7 @@ private: SrsRequest* req; SrsTcpClient* transport; SrsRtmpClient* client; + SrsKbps* kbps; int stream_id; public: SrsSimpleRtmpClient(); @@ -77,8 +80,16 @@ private: virtual int connect_app(std::string vhost); public: virtual void close(); +public: + virtual int publish(); + virtual int play(); + virtual void kbps_sample(const char* label, int64_t age); public: virtual int rtmp_write_packet(char type, u_int32_t timestamp, char* data, int size); + virtual int recv_message(SrsCommonMessage** pmsg); + virtual int decode_message(SrsCommonMessage* msg, SrsPacket** ppacket); +public: + virtual void set_recv_timeout(int64_t timeout); }; /** diff --git a/trunk/src/kernel/srs_kernel_consts.hpp b/trunk/src/kernel/srs_kernel_consts.hpp index bc05023b6..54788cecf 100644 --- a/trunk/src/kernel/srs_kernel_consts.hpp +++ b/trunk/src/kernel/srs_kernel_consts.hpp @@ -68,6 +68,9 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // the following is the timeout for rtmp protocol, // to avoid death connection. +// the common io timeout, for both recv and send. +#define SRS_CONSTS_RTMP_TIMEOUT_US (int64_t)(30*1000*1000LL) + // the timeout to send data to client, // if timeout, close the connection. #define SRS_CONSTS_RTMP_SEND_TIMEOUT_US (int64_t)(30*1000*1000LL) From 12e013142d2b67e8d5fa4989efcad4b3eb3d339b Mon Sep 17 00:00:00 2001 From: winlin Date: Wed, 14 Oct 2015 15:08:55 +0800 Subject: [PATCH 12/46] refine code, use simple rtmp client. --- trunk/src/app/srs_app_caster_flv.cpp | 6 +- trunk/src/app/srs_app_edge.cpp | 216 ++++++------------------- trunk/src/app/srs_app_edge.hpp | 17 +- trunk/src/app/srs_app_rtmp_conn.cpp | 55 ++++--- trunk/src/app/srs_app_rtmp_conn.hpp | 8 +- trunk/src/kernel/srs_kernel_consts.hpp | 2 +- 6 files changed, 102 insertions(+), 202 deletions(-) diff --git a/trunk/src/app/srs_app_caster_flv.cpp b/trunk/src/app/srs_app_caster_flv.cpp index 400060188..5da82e8fe 100644 --- a/trunk/src/app/srs_app_caster_flv.cpp +++ b/trunk/src/app/srs_app_caster_flv.cpp @@ -179,8 +179,10 @@ int SrsDynamicHttpConn::do_proxy(ISrsHttpResponseReader* rr, SrsFlvDecoder* dec) { int ret = ERROR_SUCCESS; - if ((ret = sdk->connect(output, SRS_CONSTS_RTMP_RECV_TIMEOUT_US)) != ERROR_SUCCESS) { - srs_error("flv: connect %s failed. ret=%d", output.c_str(), ret); + int64_t cto = SRS_CONSTS_RTMP_TIMEOUT_US; + int64_t sto = SRS_CONSTS_RTMP_PULSE_TIMEOUT_US; + if ((ret = sdk->connect(output, cto, sto)) != ERROR_SUCCESS) { + srs_error("flv: connect %s failed, cto=%"PRId64", sto=%"PRId64". ret=%d", output.c_str(), cto, sto, ret); return ret; } diff --git a/trunk/src/app/srs_app_edge.cpp b/trunk/src/app/srs_app_edge.cpp index e942dca44..d859e91dd 100644 --- a/trunk/src/app/srs_app_edge.cpp +++ b/trunk/src/app/srs_app_edge.cpp @@ -125,7 +125,7 @@ int SrsEdgeIngester::cycle() source->on_source_id_changed(_srs_context->get_id()); - std::string url, vhost; + std::string url; if (true) { SrsConfDirective* conf = _srs_config->get_vhost_edge_origin(req->vhost); @@ -139,22 +139,22 @@ int SrsEdgeIngester::cycle() } // select the origin. - if (true) { - std::string server = lb->select(conf->args); - int port = SRS_CONSTS_RTMP_DEFAULT_PORT; - srs_parse_hostport(server, server, port); - - url = srs_generate_rtmp_url(server, port, req->vhost, req->app, req->stream); - } + std::string server = lb->select(conf->args); + int port = SRS_CONSTS_RTMP_DEFAULT_PORT; + srs_parse_hostport(server, server, port); // support vhost tranform for edge, // @see https://github.com/simple-rtmp-server/srs/issues/372 - vhost = _srs_config->get_vhost_edge_transform_vhost(req->vhost); + std::string vhost = _srs_config->get_vhost_edge_transform_vhost(req->vhost); vhost = srs_string_replace(vhost, "[vhost]", req->vhost); + + url = srs_generate_rtmp_url(server, port, vhost, req->app, req->stream); } - if ((ret = sdk->connect(url, vhost, SRS_CONSTS_RTMP_TIMEOUT_US)) != ERROR_SUCCESS) { - srs_error("edge pull %s failed. ret=%d", url.c_str(), ret); + int64_t cto = SRS_EDGE_INGESTER_TIMEOUT_US; + int64_t sto = SRS_CONSTS_RTMP_PULSE_TIMEOUT_US; + if ((ret = sdk->connect(url, cto, sto)) != ERROR_SUCCESS) { + srs_error("edge pull %s failed, cto=%"PRId64", sto=%"PRId64". ret=%d", url.c_str(), cto, sto, ret); return ret; } @@ -180,8 +180,6 @@ int SrsEdgeIngester::ingest() { int ret = ERROR_SUCCESS; - sdk->set_recv_timeout(SRS_EDGE_INGESTER_TIMEOUT_US); - SrsPithyPrint* pprint = SrsPithyPrint::create_edge(); SrsAutoFree(SrsPithyPrint, pprint); @@ -271,27 +269,24 @@ int SrsEdgeIngester::process_publish_message(SrsCommonMessage* msg) SrsEdgeForwarder::SrsEdgeForwarder() { - transport = new SrsTcpClient(); - kbps = new SrsKbps(); - client = NULL; - _edge = NULL; - _req = NULL; + edge = NULL; + req = NULL; + send_error_code = ERROR_SUCCESS; + + sdk = new SrsSimpleRtmpClient(); lb = new SrsLbRoundRobin(); - stream_id = 0; pthread = new SrsReusableThread2("edge-fwr", this, SRS_EDGE_FORWARDER_SLEEP_US); queue = new SrsMessageQueue(); - send_error_code = ERROR_SUCCESS; } SrsEdgeForwarder::~SrsEdgeForwarder() { stop(); - srs_freep(transport); + srs_freep(sdk); srs_freep(lb); srs_freep(pthread); srs_freep(queue); - srs_freep(kbps); } void SrsEdgeForwarder::set_queue_size(double queue_size) @@ -299,13 +294,13 @@ void SrsEdgeForwarder::set_queue_size(double queue_size) return queue->set_queue_size(queue_size); } -int SrsEdgeForwarder::initialize(SrsSource* source, SrsPublishEdge* edge, SrsRequest* req) +int SrsEdgeForwarder::initialize(SrsSource* s, SrsPublishEdge* e, SrsRequest* r) { int ret = ERROR_SUCCESS; - _source = source; - _edge = edge; - _req = req; + source = s; + edge = e; + req = r; return ret; } @@ -314,36 +309,37 @@ int SrsEdgeForwarder::start() { int ret = ERROR_SUCCESS; + // reset the error code. send_error_code = ERROR_SUCCESS; - std::string ep_server; - int ep_port; - if ((ret = connect_server(ep_server, ep_port)) != ERROR_SUCCESS) { - return ret; + std::string url; + if (true) { + SrsConfDirective* conf = _srs_config->get_vhost_edge_origin(req->vhost); + srs_assert(conf); + + // select the origin. + std::string server = lb->select(conf->args); + int port = SRS_CONSTS_RTMP_DEFAULT_PORT; + srs_parse_hostport(server, server, port); + + // support vhost tranform for edge, + // @see https://github.com/simple-rtmp-server/srs/issues/372 + std::string vhost = _srs_config->get_vhost_edge_transform_vhost(req->vhost); + vhost = srs_string_replace(vhost, "[vhost]", req->vhost); + + url = srs_generate_rtmp_url(server, port, vhost, req->app, req->stream); } - srs_assert(client); - - client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); - - SrsRequest* req = _req; - if ((ret = client->handshake()) != ERROR_SUCCESS) { - srs_error("handshake with server failed. ret=%d", ret); - return ret; - } - if ((ret = connect_app(ep_server, ep_port)) != ERROR_SUCCESS) { - srs_error("connect with server failed. ret=%d", ret); - return ret; - } - if ((ret = client->create_stream(stream_id)) != ERROR_SUCCESS) { - srs_error("connect with server failed, stream_id=%d. ret=%d", stream_id, ret); + // open socket. + int64_t cto = SRS_EDGE_FORWARDER_TIMEOUT_US; + int64_t sto = SRS_CONSTS_RTMP_TIMEOUT_US; + if ((ret = sdk->connect(url, cto, sto)) != ERROR_SUCCESS) { + srs_warn("edge push %s failed, cto=%"PRId64", sto=%"PRId64". ret=%d", url.c_str(), cto, sto, ret); return ret; } - if ((ret = client->publish(req->stream, stream_id)) != ERROR_SUCCESS) { - srs_error("publish failed, stream=%s, stream_id=%d. ret=%d", - req->stream.c_str(), stream_id, ret); + if ((ret = sdk->publish()) != ERROR_SUCCESS) { + srs_error("edge push publish failed. ret=%d", ret); return ret; } @@ -353,12 +349,8 @@ int SrsEdgeForwarder::start() void SrsEdgeForwarder::stop() { pthread->stop(); - transport->close(); - + sdk->close(); queue->clear(); - - srs_freep(client); - kbps->set_io(NULL, NULL); } #define SYS_MAX_EDGE_SEND_MSGS 128 @@ -366,7 +358,7 @@ int SrsEdgeForwarder::cycle() { int ret = ERROR_SUCCESS; - client->set_recv_timeout(SRS_CONSTS_RTMP_PULSE_TIMEOUT_US); + sdk->set_recv_timeout(SRS_CONSTS_RTMP_PULSE_TIMEOUT_US); SrsPithyPrint* pprint = SrsPithyPrint::create_edge(); SrsAutoFree(SrsPithyPrint, pprint); @@ -382,7 +374,7 @@ int SrsEdgeForwarder::cycle() // read from client. if (true) { SrsCommonMessage* msg = NULL; - ret = client->recv_message(&msg); + ret = sdk->recv_message(&msg); srs_verbose("edge loop recv message. ret=%d", ret); if (ret != ERROR_SUCCESS && ret != ERROR_SOCKET_TIMEOUT) { @@ -406,22 +398,17 @@ int SrsEdgeForwarder::cycle() // pithy print if (pprint->can_print()) { - kbps->sample(); - srs_trace("-> "SRS_CONSTS_LOG_EDGE_PUBLISH - " time=%"PRId64", msgs=%d, okbps=%d,%d,%d, ikbps=%d,%d,%d", - pprint->age(), count, - kbps->get_send_kbps(), kbps->get_send_kbps_30s(), kbps->get_send_kbps_5m(), - kbps->get_recv_kbps(), kbps->get_recv_kbps_30s(), kbps->get_recv_kbps_5m()); + sdk->kbps_sample(SRS_CONSTS_LOG_EDGE_PUBLISH, pprint->age(), count); } // ignore when no messages. if (count <= 0) { - srs_verbose("no packets to push."); + srs_verbose("edge no packets to push."); continue; } // sendout messages, all messages are freed by send_and_free_messages(). - if ((ret = client->send_and_free_messages(msgs.msgs, count, stream_id)) != ERROR_SUCCESS) { + if ((ret = sdk->send_and_free_messages(msgs.msgs, count)) != ERROR_SUCCESS) { srs_error("edge publish push message to server failed. ret=%d", ret); return ret; } @@ -456,7 +443,7 @@ int SrsEdgeForwarder::proxy(SrsCommonMessage* msg) } srs_verbose("initialize shared ptr msg success."); - copy.stream_id = stream_id; + copy.stream_id = sdk->sid(); if ((ret = queue->enqueue(copy.copy())) != ERROR_SUCCESS) { srs_error("enqueue edge publish msg failed. ret=%d", ret); } @@ -464,105 +451,6 @@ int SrsEdgeForwarder::proxy(SrsCommonMessage* msg) return ret; } -int SrsEdgeForwarder::connect_server(string& ep_server, int& ep_port) -{ - int ret = ERROR_SUCCESS; - - // reopen - transport->close(); - - SrsConfDirective* conf = _srs_config->get_vhost_edge_origin(_req->vhost); - srs_assert(conf); - - // select the origin. - if (true) { - std::string server = lb->select(conf->args); - ep_port = SRS_CONSTS_RTMP_DEFAULT_PORT; - srs_parse_hostport(server, ep_server, ep_port); - } - - // open socket. - int64_t timeout = SRS_EDGE_FORWARDER_TIMEOUT_US; - if ((ret = transport->connect(ep_server, ep_port, timeout)) != ERROR_SUCCESS) { - srs_warn("edge push failed, stream=%s, tcUrl=%s to server=%s, port=%d, timeout=%"PRId64", ret=%d", - _req->stream.c_str(), _req->tcUrl.c_str(), ep_server.c_str(), ep_port, timeout, ret); - return ret; - } - - srs_freep(client); - client = new SrsRtmpClient(transport); - - kbps->set_io(transport, transport); - - // open socket. - srs_trace("edge push connected, stream=%s, tcUrl=%s to server=%s, port=%d", - _req->stream.c_str(), _req->tcUrl.c_str(), ep_server.c_str(), ep_port); - - return ret; -} - -// TODO: FIXME: refine the connect_app. -int SrsEdgeForwarder::connect_app(string ep_server, int ep_port) -{ - int ret = ERROR_SUCCESS; - - SrsRequest* req = _req; - - // args of request takes the srs info. - if (req->args == NULL) { - req->args = SrsAmf0Any::object(); - } - - // notify server the edge identity, - // @see https://github.com/simple-rtmp-server/srs/issues/147 - SrsAmf0Object* data = req->args; - data->set("srs_sig", SrsAmf0Any::str(RTMP_SIG_SRS_KEY)); - data->set("srs_server", SrsAmf0Any::str(RTMP_SIG_SRS_SERVER)); - data->set("srs_license", SrsAmf0Any::str(RTMP_SIG_SRS_LICENSE)); - data->set("srs_role", SrsAmf0Any::str(RTMP_SIG_SRS_ROLE)); - data->set("srs_url", SrsAmf0Any::str(RTMP_SIG_SRS_URL)); - data->set("srs_version", SrsAmf0Any::str(RTMP_SIG_SRS_VERSION)); - data->set("srs_site", SrsAmf0Any::str(RTMP_SIG_SRS_WEB)); - data->set("srs_email", SrsAmf0Any::str(RTMP_SIG_SRS_EMAIL)); - data->set("srs_copyright", SrsAmf0Any::str(RTMP_SIG_SRS_COPYRIGHT)); - data->set("srs_primary", SrsAmf0Any::str(RTMP_SIG_SRS_PRIMARY)); - data->set("srs_authors", SrsAmf0Any::str(RTMP_SIG_SRS_AUTHROS)); - // for edge to directly get the id of client. - data->set("srs_pid", SrsAmf0Any::number(getpid())); - data->set("srs_id", SrsAmf0Any::number(_srs_context->get_id())); - - // local ip of edge - std::vector ips = srs_get_local_ipv4_ips(); - assert(_srs_config->get_stats_network() < (int)ips.size()); - std::string local_ip = ips[_srs_config->get_stats_network()]; - data->set("srs_server_ip", SrsAmf0Any::str(local_ip.c_str())); - - // support vhost tranform for edge, - // @see https://github.com/simple-rtmp-server/srs/issues/372 - std::string vhost = _srs_config->get_vhost_edge_transform_vhost(req->vhost); - vhost = srs_string_replace(vhost, "[vhost]", req->vhost); - // generate the tcUrl - std::string param = ""; - std::string tc_url = srs_generate_tc_url(ep_server, vhost, req->app, ep_port, param); - srs_trace("edge forward to %s:%d at %s", ep_server.c_str(), ep_port, tc_url.c_str()); - - // replace the tcUrl in request, - // which will replace the tc_url in client.connect_app(). - req->tcUrl = tc_url; - - // upnode server identity will show in the connect_app of client. - // @see https://github.com/simple-rtmp-server/srs/issues/160 - // the debug_srs_upnode is config in vhost and default to true. - bool debug_srs_upnode = _srs_config->get_debug_srs_upnode(req->vhost); - if ((ret = client->connect_app(req->app, tc_url, req, debug_srs_upnode)) != ERROR_SUCCESS) { - srs_error("connect with server failed, tcUrl=%s, dsu=%d. ret=%d", - tc_url.c_str(), debug_srs_upnode, ret); - return ret; - } - - return ret; -} - SrsPlayEdge::SrsPlayEdge() { state = SrsEdgeStateInit; diff --git a/trunk/src/app/srs_app_edge.hpp b/trunk/src/app/srs_app_edge.hpp index de3b4891f..708a83f84 100644 --- a/trunk/src/app/srs_app_edge.hpp +++ b/trunk/src/app/srs_app_edge.hpp @@ -109,15 +109,11 @@ private: class SrsEdgeForwarder : public ISrsReusableThread2Handler { private: - int stream_id; -private: - SrsSource* _source; - SrsPublishEdge* _edge; - SrsRequest* _req; + SrsSource* source; + SrsPublishEdge* edge; + SrsRequest* req; SrsReusableThread2* pthread; - SrsTcpClient* transport; - SrsKbps* kbps; - SrsRtmpClient* client; + SrsSimpleRtmpClient* sdk; SrsLbRoundRobin* lb; /** * we must ensure one thread one fd principle, @@ -136,7 +132,7 @@ public: public: virtual void set_queue_size(double queue_size); public: - virtual int initialize(SrsSource* source, SrsPublishEdge* edge, SrsRequest* req); + virtual int initialize(SrsSource* s, SrsPublishEdge* e, SrsRequest* r); virtual int start(); virtual void stop(); // interface ISrsReusableThread2Handler @@ -144,9 +140,6 @@ public: virtual int cycle(); public: virtual int proxy(SrsCommonMessage* msg); -private: - virtual int connect_server(std::string& ep_server, int& ep_port); - virtual int connect_app(std::string ep_server, int ep_port); }; /** diff --git a/trunk/src/app/srs_app_rtmp_conn.cpp b/trunk/src/app/srs_app_rtmp_conn.cpp index e45f2e8db..52865a4a0 100644 --- a/trunk/src/app/srs_app_rtmp_conn.cpp +++ b/trunk/src/app/srs_app_rtmp_conn.cpp @@ -97,12 +97,7 @@ SrsSimpleRtmpClient::~SrsSimpleRtmpClient() kbps->set_io(NULL, NULL); } -int SrsSimpleRtmpClient::connect(string url, int64_t timeout) -{ - return connect(url, "", timeout); -} - -int SrsSimpleRtmpClient::connect(string url, string vhost, int64_t timeout) +int SrsSimpleRtmpClient::connect(string url, int64_t connect_timeout, int64_t stream_timeout) { int ret = ERROR_SUCCESS; @@ -113,14 +108,13 @@ int SrsSimpleRtmpClient::connect(string url, string vhost, int64_t timeout) } // parse uri - if (!req) { - req = new SrsRequest(); - srs_parse_rtmp_url(url, req->tcUrl, req->stream); - srs_discovery_tc_url(req->tcUrl, req->schema, req->host, req->vhost, req->app, req->port, req->param); - } + srs_freep(req); + req = new SrsRequest(); + srs_parse_rtmp_url(url, req->tcUrl, req->stream); + srs_discovery_tc_url(req->tcUrl, req->schema, req->host, req->vhost, req->app, req->port, req->param); // connect host. - if ((ret = transport->connect(req->host, req->port, timeout)) != ERROR_SUCCESS) { + if ((ret = transport->connect(req->host, req->port, connect_timeout)) != ERROR_SUCCESS) { return ret; } @@ -129,15 +123,15 @@ int SrsSimpleRtmpClient::connect(string url, string vhost, int64_t timeout) kbps->set_io(transport, transport); - client->set_recv_timeout(timeout); - client->set_send_timeout(timeout); + client->set_recv_timeout(stream_timeout); + client->set_send_timeout(stream_timeout); // connect to vhost/app if ((ret = client->handshake()) != ERROR_SUCCESS) { srs_error("sdk: handshake with server failed. ret=%d", ret); return ret; } - if ((ret = connect_app(vhost)) != ERROR_SUCCESS) { + if ((ret = connect_app()) != ERROR_SUCCESS) { srs_error("sdk: connect with server failed. ret=%d", ret); return ret; } @@ -149,7 +143,7 @@ int SrsSimpleRtmpClient::connect(string url, string vhost, int64_t timeout) return ret; } -int SrsSimpleRtmpClient::connect_app(string vhost) +int SrsSimpleRtmpClient::connect_app() { int ret = ERROR_SUCCESS; @@ -185,10 +179,7 @@ int SrsSimpleRtmpClient::connect_app(string vhost) // generate the tcUrl std::string param = ""; std::string target_vhost = req->vhost; - if (vhost.empty()) { - target_vhost = vhost; - } - std::string tc_url = srs_generate_tc_url(req->host, target_vhost, req->app, req->port, param); + std::string tc_url = srs_generate_tc_url(req->host, req->vhost, req->app, req->port, param); // replace the tcUrl in request, // which will replace the tc_url in client.connect_app(). @@ -256,6 +247,25 @@ void SrsSimpleRtmpClient::kbps_sample(const char* label, int64_t age) srs_trace("<- %s time=%"PRId64", okbps=%d,%d,%d, ikbps=%d,%d,%d", age, sr, sr30s, sr5m, rr, rr30s, rr5m); } +void SrsSimpleRtmpClient::kbps_sample(const char* label, int64_t age, int msgs) +{ + kbps->sample(); + + int sr = kbps->get_send_kbps(); + int sr30s = kbps->get_send_kbps_30s(); + int sr5m = kbps->get_send_kbps_5m(); + int rr = kbps->get_recv_kbps(); + int rr30s = kbps->get_recv_kbps_30s(); + int rr5m = kbps->get_recv_kbps_5m(); + + srs_trace("<- %s time=%"PRId64", msgs=%d, okbps=%d,%d,%d, ikbps=%d,%d,%d", age, msgs, sr, sr30s, sr5m, rr, rr30s, rr5m); +} + +int SrsSimpleRtmpClient::sid() +{ + return stream_id; +} + int SrsSimpleRtmpClient::rtmp_write_packet(char type, u_int32_t timestamp, char* data, int size) { int ret = ERROR_SUCCESS; @@ -286,6 +296,11 @@ int SrsSimpleRtmpClient::decode_message(SrsCommonMessage* msg, SrsPacket** ppack return client->decode_message(msg, ppacket); } +int SrsSimpleRtmpClient::send_and_free_messages(SrsSharedPtrMessage** msgs, int nb_msgs) +{ + return client->send_and_free_messages(msgs, nb_msgs, stream_id); +} + void SrsSimpleRtmpClient::set_recv_timeout(int64_t timeout) { transport->set_recv_timeout(timeout); diff --git a/trunk/src/app/srs_app_rtmp_conn.hpp b/trunk/src/app/srs_app_rtmp_conn.hpp index e274e471a..dab29ebfb 100644 --- a/trunk/src/app/srs_app_rtmp_conn.hpp +++ b/trunk/src/app/srs_app_rtmp_conn.hpp @@ -74,20 +74,22 @@ public: SrsSimpleRtmpClient(); virtual ~SrsSimpleRtmpClient(); public: - virtual int connect(std::string url, int64_t timeout); - virtual int connect(std::string url, std::string vhost, int64_t timeout); + virtual int connect(std::string url, int64_t connect_timeout, int64_t stream_timeout); private: - virtual int connect_app(std::string vhost); + virtual int connect_app(); public: virtual void close(); public: virtual int publish(); virtual int play(); virtual void kbps_sample(const char* label, int64_t age); + virtual void kbps_sample(const char* label, int64_t age, int msgs); + virtual int sid(); public: virtual int rtmp_write_packet(char type, u_int32_t timestamp, char* data, int size); virtual int recv_message(SrsCommonMessage** pmsg); virtual int decode_message(SrsCommonMessage* msg, SrsPacket** ppacket); + virtual int send_and_free_messages(SrsSharedPtrMessage** msgs, int nb_msgs); public: virtual void set_recv_timeout(int64_t timeout); }; diff --git a/trunk/src/kernel/srs_kernel_consts.hpp b/trunk/src/kernel/srs_kernel_consts.hpp index 54788cecf..7a7bb8d3d 100644 --- a/trunk/src/kernel/srs_kernel_consts.hpp +++ b/trunk/src/kernel/srs_kernel_consts.hpp @@ -71,10 +71,10 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // the common io timeout, for both recv and send. #define SRS_CONSTS_RTMP_TIMEOUT_US (int64_t)(30*1000*1000LL) +// TODO: FIXME: remove following two macros. // the timeout to send data to client, // if timeout, close the connection. #define SRS_CONSTS_RTMP_SEND_TIMEOUT_US (int64_t)(30*1000*1000LL) - // the timeout to wait client data, // if timeout, close the connection. #define SRS_CONSTS_RTMP_RECV_TIMEOUT_US (int64_t)(30*1000*1000LL) From 0f4cb8ee00df00ede5925087f0c7304173d077f0 Mon Sep 17 00:00:00 2001 From: winlin Date: Wed, 14 Oct 2015 15:16:59 +0800 Subject: [PATCH 13/46] refine code, remove the rtmp recv/send macro, use one macro. --- trunk/src/app/srs_app_forward.cpp | 4 ++-- trunk/src/app/srs_app_mpegts_udp.cpp | 4 ++-- trunk/src/app/srs_app_rtmp_conn.cpp | 12 ++++++------ trunk/src/app/srs_app_rtsp.cpp | 4 ++-- trunk/src/kernel/srs_kernel_consts.hpp | 8 -------- trunk/src/main/srs_main_ingest_hls.cpp | 4 ++-- 6 files changed, 14 insertions(+), 22 deletions(-) diff --git a/trunk/src/app/srs_app_forward.cpp b/trunk/src/app/srs_app_forward.cpp index 478cddcda..028b5d218 100644 --- a/trunk/src/app/srs_app_forward.cpp +++ b/trunk/src/app/srs_app_forward.cpp @@ -235,8 +235,8 @@ int SrsForwarder::cycle() } srs_assert(client); - client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); + client->set_recv_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); + client->set_send_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); if ((ret = client->handshake()) != ERROR_SUCCESS) { srs_error("handshake with server failed. ret=%d", ret); diff --git a/trunk/src/app/srs_app_mpegts_udp.cpp b/trunk/src/app/srs_app_mpegts_udp.cpp index 9bb92e46f..5f1497615 100644 --- a/trunk/src/app/srs_app_mpegts_udp.cpp +++ b/trunk/src/app/srs_app_mpegts_udp.cpp @@ -624,8 +624,8 @@ int SrsMpegtsOverUdp::connect() srs_freep(client); client = new SrsRtmpClient(transport); - client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); + client->set_recv_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); + client->set_send_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); // connect to vhost/app if ((ret = client->handshake()) != ERROR_SUCCESS) { diff --git a/trunk/src/app/srs_app_rtmp_conn.cpp b/trunk/src/app/srs_app_rtmp_conn.cpp index 52865a4a0..d3efb40b7 100644 --- a/trunk/src/app/srs_app_rtmp_conn.cpp +++ b/trunk/src/app/srs_app_rtmp_conn.cpp @@ -362,8 +362,8 @@ int SrsRtmpConn::do_cycle() srs_trace("RTMP client ip=%s", ip.c_str()); - rtmp->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - rtmp->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); + rtmp->set_recv_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); + rtmp->set_send_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); if ((ret = rtmp->handshake()) != ERROR_SUCCESS) { srs_error("rtmp handshake failed. ret=%d", ret); @@ -690,8 +690,8 @@ int SrsRtmpConn::stream_service_cycle() srs_info("security check ok"); // client is identified, set the timeout to service timeout. - rtmp->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - rtmp->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); + rtmp->set_recv_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); + rtmp->set_send_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); // find a source to serve. SrsSource* source = SrsSource::fetch(req); @@ -1510,8 +1510,8 @@ int SrsRtmpConn::do_token_traverse_auth(SrsRtmpClient* client) srs_assert(client); - client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); + client->set_recv_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); + client->set_send_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); if ((ret = client->handshake()) != ERROR_SUCCESS) { srs_error("handshake with server failed. ret=%d", ret); diff --git a/trunk/src/app/srs_app_rtsp.cpp b/trunk/src/app/srs_app_rtsp.cpp index 8ca22c1b1..0712f017d 100644 --- a/trunk/src/app/srs_app_rtsp.cpp +++ b/trunk/src/app/srs_app_rtsp.cpp @@ -676,8 +676,8 @@ int SrsRtspConn::connect() srs_freep(client); client = new SrsRtmpClient(transport); - client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); + client->set_recv_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); + client->set_send_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); // connect to vhost/app if ((ret = client->handshake()) != ERROR_SUCCESS) { diff --git a/trunk/src/kernel/srs_kernel_consts.hpp b/trunk/src/kernel/srs_kernel_consts.hpp index 7a7bb8d3d..d209c7762 100644 --- a/trunk/src/kernel/srs_kernel_consts.hpp +++ b/trunk/src/kernel/srs_kernel_consts.hpp @@ -71,14 +71,6 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // the common io timeout, for both recv and send. #define SRS_CONSTS_RTMP_TIMEOUT_US (int64_t)(30*1000*1000LL) -// TODO: FIXME: remove following two macros. -// the timeout to send data to client, -// if timeout, close the connection. -#define SRS_CONSTS_RTMP_SEND_TIMEOUT_US (int64_t)(30*1000*1000LL) -// the timeout to wait client data, -// if timeout, close the connection. -#define SRS_CONSTS_RTMP_RECV_TIMEOUT_US (int64_t)(30*1000*1000LL) - // the timeout to wait for client control message, // if timeout, we generally ignore and send the data to client, // generally, it's the pulse time for data seding. diff --git a/trunk/src/main/srs_main_ingest_hls.cpp b/trunk/src/main/srs_main_ingest_hls.cpp index 2fdde094f..8c13fb14b 100644 --- a/trunk/src/main/srs_main_ingest_hls.cpp +++ b/trunk/src/main/srs_main_ingest_hls.cpp @@ -1242,8 +1242,8 @@ int SrsIngestSrsOutput::connect() srs_freep(client); client = new SrsRtmpClient(transport); - client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); + client->set_recv_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); + client->set_send_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); // connect to vhost/app if ((ret = client->handshake()) != ERROR_SUCCESS) { From ad9b377d96deeebd8075179322f295e445f5cf5e Mon Sep 17 00:00:00 2001 From: winlin Date: Wed, 14 Oct 2015 15:34:18 +0800 Subject: [PATCH 14/46] refine forwarder code, use simple rtmp client. --- trunk/src/app/srs_app_forward.cpp | 195 +++++++--------------------- trunk/src/app/srs_app_forward.hpp | 15 +-- trunk/src/app/srs_app_rtmp_conn.cpp | 5 + trunk/src/app/srs_app_rtmp_conn.hpp | 1 + 4 files changed, 55 insertions(+), 161 deletions(-) diff --git a/trunk/src/app/srs_app_forward.cpp b/trunk/src/app/srs_app_forward.cpp index 028b5d218..c2882be55 100644 --- a/trunk/src/app/srs_app_forward.cpp +++ b/trunk/src/app/srs_app_forward.cpp @@ -45,51 +45,47 @@ using namespace std; #include #include #include +#include // when error, forwarder sleep for a while and retry. #define SRS_FORWARDER_SLEEP_US (int64_t)(3*1000*1000LL) -SrsForwarder::SrsForwarder(SrsSource* _source) +SrsForwarder::SrsForwarder(SrsSource* s) { - source = _source; + source = s; - _req = NULL; - client = NULL; - transport = new SrsTcpClient(); - kbps = new SrsKbps(); - stream_id = 0; + req = NULL; + sh_video = sh_audio = NULL; + sdk = new SrsSimpleRtmpClient(); pthread = new SrsReusableThread2("forward", this, SRS_FORWARDER_SLEEP_US); queue = new SrsMessageQueue(); jitter = new SrsRtmpJitter(); - - sh_video = sh_audio = NULL; } SrsForwarder::~SrsForwarder() { on_unpublish(); - srs_freep(transport); + srs_freep(sdk); srs_freep(pthread); srs_freep(queue); srs_freep(jitter); - srs_freep(kbps); srs_freep(sh_video); srs_freep(sh_audio); } -int SrsForwarder::initialize(SrsRequest* req, string ep_forward) +int SrsForwarder::initialize(SrsRequest* r, string ep) { int ret = ERROR_SUCCESS; // it's ok to use the request object, // SrsSource already copy it and never delete it. - _req = req; + req = r; // the ep(endpoint) to forward to - _ep_forward = ep_forward; + ep_forward = ep; return ret; } @@ -103,12 +99,17 @@ int SrsForwarder::on_publish() { int ret = ERROR_SUCCESS; - SrsRequest* req = _req; - // discovery the server port and tcUrl from req and ep_forward. - int port; - std::string server, tc_url; - discovery_ep(server, port, tc_url); + std::string server; + std::string tcUrl; + int port = SRS_CONSTS_RTMP_DEFAULT_PORT; + if (true) { + // parse host:port from hostport. + srs_parse_hostport(ep_forward, server, port); + + // generate tcUrl + tcUrl = srs_generate_tc_url(server, req->vhost, req->app, port, req->param); + } // dead loop check std::string source_ep = "rtmp://"; @@ -119,7 +120,7 @@ int SrsForwarder::on_publish() source_ep += req->vhost; std::string dest_ep = "rtmp://"; - if (_ep_forward == SRS_CONSTS_LOCALHOST) { + if (ep_forward == SRS_CONSTS_LOCALHOST) { dest_ep += req->host; } else { dest_ep += server; @@ -136,7 +137,7 @@ int SrsForwarder::on_publish() return ret; } srs_trace("start forward %s to %s, tcUrl=%s, stream=%s", - source_ep.c_str(), dest_ep.c_str(), tc_url.c_str(), + source_ep.c_str(), dest_ep.c_str(), tcUrl.c_str(), req->stream.c_str()); if ((ret = pthread->start()) != ERROR_SUCCESS) { @@ -151,10 +152,7 @@ int SrsForwarder::on_publish() void SrsForwarder::on_unpublish() { pthread->stop(); - transport->close(); - - srs_freep(client); - kbps->set_io(NULL, NULL); + sdk->close(); } int SrsForwarder::on_meta_data(SrsSharedPtrMessage* shared_metadata) @@ -228,32 +226,26 @@ int SrsForwarder::cycle() { int ret = ERROR_SUCCESS; - std::string ep_server; - int ep_port; - if ((ret = connect_server(ep_server, ep_port)) != ERROR_SUCCESS) { - return ret; + std::string url; + if (true) { + std::string server; + int port = SRS_CONSTS_RTMP_DEFAULT_PORT; + + // parse host:port from hostport. + srs_parse_hostport(ep_forward, server, port); + + // generate url + url = srs_generate_rtmp_url(server, port, req->vhost, req->app, req->stream); } - srs_assert(client); - - client->set_recv_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); - if ((ret = client->handshake()) != ERROR_SUCCESS) { - srs_error("handshake with server failed. ret=%d", ret); - return ret; - } - if ((ret = connect_app(ep_server, ep_port)) != ERROR_SUCCESS) { - srs_error("connect with server failed. ret=%d", ret); - return ret; - } - if ((ret = client->create_stream(stream_id)) != ERROR_SUCCESS) { - srs_error("connect with server failed, stream_id=%d. ret=%d", stream_id, ret); + int64_t cto = SRS_FORWARDER_SLEEP_US; + int64_t sto = SRS_CONSTS_RTMP_TIMEOUT_US; + if ((ret = sdk->connect(url, cto, sto)) != ERROR_SUCCESS) { + srs_warn("forward failed, url=%s, cto=%"PRId64", sto=%"PRId64". ret=%d", url.c_str(), cto, sto, ret); return ret; } - if ((ret = client->publish(_req->stream, stream_id)) != ERROR_SUCCESS) { - srs_error("connect with server failed, stream_name=%s, stream_id=%d. ret=%d", - _req->stream.c_str(), stream_id, ret); + if ((ret = sdk->publish()) != ERROR_SUCCESS) { return ret; } @@ -269,106 +261,12 @@ int SrsForwarder::cycle() return ret; } -void SrsForwarder::discovery_ep(string& server, int& port, string& tc_url) -{ - SrsRequest* req = _req; - - port = SRS_CONSTS_RTMP_DEFAULT_PORT; - srs_parse_hostport(_ep_forward, server, port); - - // generate tcUrl - tc_url = srs_generate_tc_url(server, req->vhost, req->app, port, req->param); -} - -int SrsForwarder::connect_server(string& ep_server, int& ep_port) -{ - int ret = ERROR_SUCCESS; - - // reopen - transport->close(); - - // discovery the server port and tcUrl from req and ep_forward. - string tc_url; - discovery_ep(ep_server, ep_port, tc_url); - - // open socket. - int64_t timeout = SRS_FORWARDER_SLEEP_US; - if ((ret = transport->connect(ep_server, ep_port, timeout)) != ERROR_SUCCESS) { - srs_warn("forward failed, stream=%s, tcUrl=%s to server=%s, port=%d, timeout=%"PRId64", ret=%d", - _req->stream.c_str(), _req->tcUrl.c_str(), ep_server.c_str(), ep_port, timeout, ret); - return ret; - } - - srs_freep(client); - client = new SrsRtmpClient(transport); - - kbps->set_io(transport, transport); - - srs_trace("forward connected, stream=%s, tcUrl=%s to server=%s, port=%d", - _req->stream.c_str(), _req->tcUrl.c_str(), ep_server.c_str(), ep_port); - - return ret; -} - -// TODO: FIXME: refine the connect_app. -int SrsForwarder::connect_app(string ep_server, int ep_port) -{ - int ret = ERROR_SUCCESS; - - SrsRequest* req = _req; - - // args of request takes the srs info. - if (req->args == NULL) { - req->args = SrsAmf0Any::object(); - } - - // notify server the edge identity, - // @see https://github.com/simple-rtmp-server/srs/issues/147 - SrsAmf0Object* data = req->args; - data->set("srs_sig", SrsAmf0Any::str(RTMP_SIG_SRS_KEY)); - data->set("srs_server", SrsAmf0Any::str(RTMP_SIG_SRS_SERVER)); - data->set("srs_license", SrsAmf0Any::str(RTMP_SIG_SRS_LICENSE)); - data->set("srs_role", SrsAmf0Any::str(RTMP_SIG_SRS_ROLE)); - data->set("srs_url", SrsAmf0Any::str(RTMP_SIG_SRS_URL)); - data->set("srs_version", SrsAmf0Any::str(RTMP_SIG_SRS_VERSION)); - data->set("srs_site", SrsAmf0Any::str(RTMP_SIG_SRS_WEB)); - data->set("srs_email", SrsAmf0Any::str(RTMP_SIG_SRS_EMAIL)); - data->set("srs_copyright", SrsAmf0Any::str(RTMP_SIG_SRS_COPYRIGHT)); - data->set("srs_primary", SrsAmf0Any::str(RTMP_SIG_SRS_PRIMARY)); - data->set("srs_authors", SrsAmf0Any::str(RTMP_SIG_SRS_AUTHROS)); - // for edge to directly get the id of client. - data->set("srs_pid", SrsAmf0Any::number(getpid())); - data->set("srs_id", SrsAmf0Any::number(_srs_context->get_id())); - - // local ip of edge - std::vector ips = srs_get_local_ipv4_ips(); - assert(_srs_config->get_stats_network() < (int)ips.size()); - std::string local_ip = ips[_srs_config->get_stats_network()]; - data->set("srs_server_ip", SrsAmf0Any::str(local_ip.c_str())); - - // generate the tcUrl - std::string param = ""; - std::string tc_url = srs_generate_tc_url(ep_server, req->vhost, req->app, ep_port, param); - - // upnode server identity will show in the connect_app of client. - // @see https://github.com/simple-rtmp-server/srs/issues/160 - // the debug_srs_upnode is config in vhost and default to true. - bool debug_srs_upnode = _srs_config->get_debug_srs_upnode(req->vhost); - if ((ret = client->connect_app(req->app, tc_url, req, debug_srs_upnode)) != ERROR_SUCCESS) { - srs_error("connect with server failed, tcUrl=%s, dsu=%d. ret=%d", - tc_url.c_str(), debug_srs_upnode, ret); - return ret; - } - - return ret; -} - #define SYS_MAX_FORWARD_SEND_MSGS 128 int SrsForwarder::forward() { int ret = ERROR_SUCCESS; - client->set_recv_timeout(SRS_CONSTS_RTMP_PULSE_TIMEOUT_US); + sdk->set_recv_timeout(SRS_CONSTS_RTMP_PULSE_TIMEOUT_US); SrsPithyPrint* pprint = SrsPithyPrint::create_forwarder(); SrsAutoFree(SrsPithyPrint, pprint); @@ -378,13 +276,13 @@ int SrsForwarder::forward() // update sequence header // TODO: FIXME: maybe need to zero the sequence header timestamp. if (sh_video) { - if ((ret = client->send_and_free_message(sh_video->copy(), stream_id)) != ERROR_SUCCESS) { + if ((ret = sdk->send_and_free_message(sh_video->copy())) != ERROR_SUCCESS) { srs_error("forwarder send sh_video to server failed. ret=%d", ret); return ret; } } if (sh_audio) { - if ((ret = client->send_and_free_message(sh_audio->copy(), stream_id)) != ERROR_SUCCESS) { + if ((ret = sdk->send_and_free_message(sh_audio->copy())) != ERROR_SUCCESS) { srs_error("forwarder send sh_audio to server failed. ret=%d", ret); return ret; } @@ -396,7 +294,7 @@ int SrsForwarder::forward() // read from client. if (true) { SrsCommonMessage* msg = NULL; - ret = client->recv_message(&msg); + ret = sdk->recv_message(&msg); srs_verbose("play loop recv message. ret=%d", ret); if (ret != ERROR_SUCCESS && ret != ERROR_SOCKET_TIMEOUT) { @@ -417,12 +315,7 @@ int SrsForwarder::forward() // pithy print if (pprint->can_print()) { - kbps->sample(); - srs_trace("-> "SRS_CONSTS_LOG_FOWARDER - " time=%"PRId64", msgs=%d, okbps=%d,%d,%d, ikbps=%d,%d,%d", - pprint->age(), count, - kbps->get_send_kbps(), kbps->get_send_kbps_30s(), kbps->get_send_kbps_5m(), - kbps->get_recv_kbps(), kbps->get_recv_kbps_30s(), kbps->get_recv_kbps_5m()); + sdk->kbps_sample(SRS_CONSTS_LOG_FOWARDER, pprint->age(), count); } // ignore when no messages. @@ -432,7 +325,7 @@ int SrsForwarder::forward() } // sendout messages, all messages are freed by send_and_free_messages(). - if ((ret = client->send_and_free_messages(msgs.msgs, count, stream_id)) != ERROR_SUCCESS) { + if ((ret = sdk->send_and_free_messages(msgs.msgs, count)) != ERROR_SUCCESS) { srs_error("forwarder messages to server failed. ret=%d", ret); return ret; } diff --git a/trunk/src/app/srs_app_forward.hpp b/trunk/src/app/srs_app_forward.hpp index 6d108f899..487cd2b83 100644 --- a/trunk/src/app/srs_app_forward.hpp +++ b/trunk/src/app/srs_app_forward.hpp @@ -43,6 +43,7 @@ class SrsRtmpClient; class SrsRequest; class SrsSource; class SrsKbps; +class SrsSimpleRtmpClient; /** * forward the stream to other servers. @@ -52,16 +53,13 @@ class SrsForwarder : public ISrsReusableThread2Handler { private: // the ep to forward, server[:port]. - std::string _ep_forward; - SrsRequest* _req; - int stream_id; + std::string ep_forward; + SrsRequest* req; private: SrsReusableThread2* pthread; private: SrsSource* source; - SrsTcpClient* transport; - SrsKbps* kbps; - SrsRtmpClient* client; + SrsSimpleRtmpClient* sdk; SrsRtmpJitter* jitter; SrsMessageQueue* queue; /** @@ -74,7 +72,7 @@ public: SrsForwarder(SrsSource* _source); virtual ~SrsForwarder(); public: - virtual int initialize(SrsRequest* req, std::string ep_forward); + virtual int initialize(SrsRequest* r, std::string ep); virtual void set_queue_size(double queue_size); public: virtual int on_publish(); @@ -98,9 +96,6 @@ public: public: virtual int cycle(); private: - virtual void discovery_ep(std::string& server, int& port, std::string& tc_url); - virtual int connect_server(std::string& ep_server, int& ep_port); - virtual int connect_app(std::string ep_server, int ep_port); virtual int forward(); }; diff --git a/trunk/src/app/srs_app_rtmp_conn.cpp b/trunk/src/app/srs_app_rtmp_conn.cpp index d3efb40b7..0301c4602 100644 --- a/trunk/src/app/srs_app_rtmp_conn.cpp +++ b/trunk/src/app/srs_app_rtmp_conn.cpp @@ -301,6 +301,11 @@ int SrsSimpleRtmpClient::send_and_free_messages(SrsSharedPtrMessage** msgs, int return client->send_and_free_messages(msgs, nb_msgs, stream_id); } +int SrsSimpleRtmpClient::send_and_free_message(SrsSharedPtrMessage* msg) +{ + return client->send_and_free_message(msg, stream_id); +} + void SrsSimpleRtmpClient::set_recv_timeout(int64_t timeout) { transport->set_recv_timeout(timeout); diff --git a/trunk/src/app/srs_app_rtmp_conn.hpp b/trunk/src/app/srs_app_rtmp_conn.hpp index dab29ebfb..be345660d 100644 --- a/trunk/src/app/srs_app_rtmp_conn.hpp +++ b/trunk/src/app/srs_app_rtmp_conn.hpp @@ -90,6 +90,7 @@ public: virtual int recv_message(SrsCommonMessage** pmsg); virtual int decode_message(SrsCommonMessage* msg, SrsPacket** ppacket); virtual int send_and_free_messages(SrsSharedPtrMessage** msgs, int nb_msgs); + virtual int send_and_free_message(SrsSharedPtrMessage* msg); public: virtual void set_recv_timeout(int64_t timeout); }; From a08d8f83d6eda658eb155da3f2a429ad9d9b6dc1 Mon Sep 17 00:00:00 2001 From: winlin Date: Wed, 14 Oct 2015 15:51:01 +0800 Subject: [PATCH 15/46] refine mpegts code, use simple rtmp client --- trunk/src/app/srs_app_caster_flv.cpp | 7 +- trunk/src/app/srs_app_mpegts_udp.cpp | 110 ++++----------------------- trunk/src/app/srs_app_mpegts_udp.hpp | 6 +- trunk/src/app/srs_app_rtmp_conn.cpp | 19 +++-- trunk/src/app/srs_app_rtmp_conn.hpp | 4 +- 5 files changed, 34 insertions(+), 112 deletions(-) diff --git a/trunk/src/app/srs_app_caster_flv.cpp b/trunk/src/app/srs_app_caster_flv.cpp index 5da82e8fe..f8258f896 100644 --- a/trunk/src/app/srs_app_caster_flv.cpp +++ b/trunk/src/app/srs_app_caster_flv.cpp @@ -214,8 +214,13 @@ int SrsDynamicHttpConn::do_proxy(ISrsHttpResponseReader* rr, SrsFlvDecoder* dec) return ret; } + SrsSharedPtrMessage* msg = NULL; + if ((ret = sdk->rtmp_create_msg(type, time, data, size, &msg)) != ERROR_SUCCESS) { + return ret; + } + // TODO: FIXME: for post flv, reconnect when error. - if ((ret = sdk->rtmp_write_packet(type, time, data, size)) != ERROR_SUCCESS) { + if ((ret = sdk->send_and_free_message(msg)) != ERROR_SUCCESS) { if (!srs_is_client_gracefully_close(ret)) { srs_error("flv: proxy rtmp packet failed. ret=%d", ret); } diff --git a/trunk/src/app/srs_app_mpegts_udp.cpp b/trunk/src/app/srs_app_mpegts_udp.cpp index 5f1497615..0d437166c 100644 --- a/trunk/src/app/srs_app_mpegts_udp.cpp +++ b/trunk/src/app/srs_app_mpegts_udp.cpp @@ -49,6 +49,7 @@ using namespace std; #include #include #include +#include SrsMpegtsQueue::SrsMpegtsQueue() { @@ -132,9 +133,7 @@ SrsMpegtsOverUdp::SrsMpegtsOverUdp(SrsConfDirective* c) output = _srs_config->get_stream_caster_output(c); req = NULL; - client = NULL; - transport = new SrsTcpClient(); - stream_id = 0; + sdk = new SrsSimpleRtmpClient(); avc = new SrsRawH264Stream(); aac = new SrsRawAacStream(); @@ -149,7 +148,7 @@ SrsMpegtsOverUdp::~SrsMpegtsOverUdp() { close(); - srs_freep(transport); + srs_freep(sdk); srs_freep(buffer); srs_freep(stream); srs_freep(context); @@ -565,8 +564,8 @@ int SrsMpegtsOverUdp::rtmp_write_packet(char type, u_int32_t timestamp, char* da int ret = ERROR_SUCCESS; SrsSharedPtrMessage* msg = NULL; - - if ((ret = srs_rtmp_create_msg(type, timestamp, data, size, stream_id, &msg)) != ERROR_SUCCESS) { + + if ((ret = sdk->rtmp_create_msg(type, timestamp, data, size, &msg)) != ERROR_SUCCESS) { srs_error("mpegts: create shared ptr msg failed. ret=%d", ret); return ret; } @@ -590,7 +589,7 @@ int SrsMpegtsOverUdp::rtmp_write_packet(char type, u_int32_t timestamp, char* da } // send out encoded msg. - if ((ret = client->send_and_free_message(msg, stream_id)) != ERROR_SUCCESS) { + if ((ret = sdk->send_and_free_message(msg)) != ERROR_SUCCESS) { return ret; } } @@ -604,110 +603,29 @@ int SrsMpegtsOverUdp::connect() // when ok, ignore. // TODO: FIXME: should reconnect when disconnected. - if (transport->connected()) { - return ret; - } - - // parse uri - if (!req) { - req = new SrsRequest(); - srs_parse_rtmp_url(output, req->tcUrl, req->stream); - srs_discovery_tc_url(req->tcUrl, req->schema, req->host, req->vhost, req->app, req->port, req->param); - } - - // connect host. - if ((ret = transport->connect(req->host, req->port, ST_UTIME_NO_TIMEOUT)) != ERROR_SUCCESS) { - srs_error("mpegts: connect server %s:%d failed. ret=%d", req->host.c_str(), req->port, ret); + if (sdk->connected()) { return ret; } - srs_freep(client); - client = new SrsRtmpClient(transport); - - client->set_recv_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); - - // connect to vhost/app - if ((ret = client->handshake()) != ERROR_SUCCESS) { - srs_error("mpegts: handshake with server failed. ret=%d", ret); - return ret; - } - if ((ret = connect_app(req->host, req->port)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed. ret=%d", ret); - return ret; - } - if ((ret = client->create_stream(stream_id)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed, stream_id=%d. ret=%d", stream_id, ret); + int64_t cto = SRS_CONSTS_RTMP_TIMEOUT_US; + int64_t sto = SRS_CONSTS_RTMP_PULSE_TIMEOUT_US; + if ((ret = sdk->connect(output, cto, sto)) != ERROR_SUCCESS) { + srs_error("mpegts: connect %s failed, cto=%"PRId64", sto=%"PRId64". ret=%d", output.c_str(), cto, sto, ret); return ret; } - // publish. - if ((ret = client->publish(req->stream, stream_id)) != ERROR_SUCCESS) { - srs_error("mpegts: publish failed, stream=%s, stream_id=%d. ret=%d", - req->stream.c_str(), stream_id, ret); + if ((ret = sdk->publish()) != ERROR_SUCCESS) { + srs_error("mpegts: publish failed. ret=%d", ret); return ret; } return ret; } -// TODO: FIXME: refine the connect_app. -int SrsMpegtsOverUdp::connect_app(string ep_server, int ep_port) -{ - int ret = ERROR_SUCCESS; - - // args of request takes the srs info. - if (req->args == NULL) { - req->args = SrsAmf0Any::object(); - } - - // notify server the edge identity, - // @see https://github.com/simple-rtmp-server/srs/issues/147 - SrsAmf0Object* data = req->args; - data->set("srs_sig", SrsAmf0Any::str(RTMP_SIG_SRS_KEY)); - data->set("srs_server", SrsAmf0Any::str(RTMP_SIG_SRS_KEY" "RTMP_SIG_SRS_VERSION" ("RTMP_SIG_SRS_URL_SHORT")")); - data->set("srs_license", SrsAmf0Any::str(RTMP_SIG_SRS_LICENSE)); - data->set("srs_role", SrsAmf0Any::str(RTMP_SIG_SRS_ROLE)); - data->set("srs_url", SrsAmf0Any::str(RTMP_SIG_SRS_URL)); - data->set("srs_version", SrsAmf0Any::str(RTMP_SIG_SRS_VERSION)); - data->set("srs_site", SrsAmf0Any::str(RTMP_SIG_SRS_WEB)); - data->set("srs_email", SrsAmf0Any::str(RTMP_SIG_SRS_EMAIL)); - data->set("srs_copyright", SrsAmf0Any::str(RTMP_SIG_SRS_COPYRIGHT)); - data->set("srs_primary", SrsAmf0Any::str(RTMP_SIG_SRS_PRIMARY)); - data->set("srs_authors", SrsAmf0Any::str(RTMP_SIG_SRS_AUTHROS)); - // for edge to directly get the id of client. - data->set("srs_pid", SrsAmf0Any::number(getpid())); - data->set("srs_id", SrsAmf0Any::number(_srs_context->get_id())); - - // local ip of edge - std::vector ips = srs_get_local_ipv4_ips(); - assert(_srs_config->get_stats_network() < (int)ips.size()); - std::string local_ip = ips[_srs_config->get_stats_network()]; - data->set("srs_server_ip", SrsAmf0Any::str(local_ip.c_str())); - - // generate the tcUrl - std::string param = ""; - std::string tc_url = srs_generate_tc_url(ep_server, req->vhost, req->app, ep_port, param); - - // upnode server identity will show in the connect_app of client. - // @see https://github.com/simple-rtmp-server/srs/issues/160 - // the debug_srs_upnode is config in vhost and default to true. - bool debug_srs_upnode = _srs_config->get_debug_srs_upnode(req->vhost); - if ((ret = client->connect_app(req->app, tc_url, req, debug_srs_upnode)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed, tcUrl=%s, dsu=%d. ret=%d", - tc_url.c_str(), debug_srs_upnode, ret); - return ret; - } - - return ret; -} - void SrsMpegtsOverUdp::close() { - srs_freep(client); srs_freep(req); - - transport->close(); + sdk->close(); } #endif diff --git a/trunk/src/app/srs_app_mpegts_udp.hpp b/trunk/src/app/srs_app_mpegts_udp.hpp index f2ba65727..e9de14c00 100644 --- a/trunk/src/app/srs_app_mpegts_udp.hpp +++ b/trunk/src/app/srs_app_mpegts_udp.hpp @@ -48,6 +48,7 @@ class SrsSharedPtrMessage; class SrsRawAacStream; struct SrsRawAacStreamCodec; class SrsPithyPrint; +class SrsSimpleRtmpClient; #include #include @@ -86,9 +87,7 @@ private: std::string output; private: SrsRequest* req; - SrsTcpClient* transport; - SrsRtmpClient* client; - int stream_id; + SrsSimpleRtmpClient* sdk; private: SrsRawH264Stream* avc; std::string h264_sps; @@ -125,7 +124,6 @@ private: // connect to rtmp output url. // @remark ignore when not connected, reconnect when disconnected. virtual int connect(); - virtual int connect_app(std::string ep_server, int ep_port); // close the connected io and rtmp to ready to be re-connect. virtual void close(); }; diff --git a/trunk/src/app/srs_app_rtmp_conn.cpp b/trunk/src/app/srs_app_rtmp_conn.cpp index 0301c4602..a9229611a 100644 --- a/trunk/src/app/srs_app_rtmp_conn.cpp +++ b/trunk/src/app/srs_app_rtmp_conn.cpp @@ -198,6 +198,11 @@ int SrsSimpleRtmpClient::connect_app() return ret; } +bool SrsSimpleRtmpClient::connected() +{ + return transport->connected(); +} + void SrsSimpleRtmpClient::close() { transport->close(); @@ -266,22 +271,16 @@ int SrsSimpleRtmpClient::sid() return stream_id; } -int SrsSimpleRtmpClient::rtmp_write_packet(char type, u_int32_t timestamp, char* data, int size) +int SrsSimpleRtmpClient::rtmp_create_msg(char type, u_int32_t timestamp, char* data, int size, SrsSharedPtrMessage** pmsg) { - int ret = ERROR_SUCCESS; + *pmsg = NULL; - SrsSharedPtrMessage* msg = NULL; + int ret = ERROR_SUCCESS; - if ((ret = srs_rtmp_create_msg(type, timestamp, data, size, stream_id, &msg)) != ERROR_SUCCESS) { + if ((ret = srs_rtmp_create_msg(type, timestamp, data, size, stream_id, pmsg)) != ERROR_SUCCESS) { srs_error("sdk: create shared ptr msg failed. ret=%d", ret); return ret; } - srs_assert(msg); - - // send out encoded msg. - if ((ret = client->send_and_free_message(msg, stream_id)) != ERROR_SUCCESS) { - return ret; - } return ret; } diff --git a/trunk/src/app/srs_app_rtmp_conn.hpp b/trunk/src/app/srs_app_rtmp_conn.hpp index be345660d..5cfc0624f 100644 --- a/trunk/src/app/srs_app_rtmp_conn.hpp +++ b/trunk/src/app/srs_app_rtmp_conn.hpp @@ -78,6 +78,7 @@ public: private: virtual int connect_app(); public: + virtual bool connected(); virtual void close(); public: virtual int publish(); @@ -86,7 +87,8 @@ public: virtual void kbps_sample(const char* label, int64_t age, int msgs); virtual int sid(); public: - virtual int rtmp_write_packet(char type, u_int32_t timestamp, char* data, int size); + virtual int rtmp_create_msg(char type, u_int32_t timestamp, char* data, int size, SrsSharedPtrMessage** pmsg); +public: virtual int recv_message(SrsCommonMessage** pmsg); virtual int decode_message(SrsCommonMessage* msg, SrsPacket** ppacket); virtual int send_and_free_messages(SrsSharedPtrMessage** msgs, int nb_msgs); From a9fdb630d958e05a515a3a6d20d9964f05e57681 Mon Sep 17 00:00:00 2001 From: winlin Date: Wed, 14 Oct 2015 16:06:45 +0800 Subject: [PATCH 16/46] refine code, replace all rtmp connect by simple rtmp client. --- trunk/src/app/srs_app_rtsp.cpp | 105 +++------------------ trunk/src/app/srs_app_rtsp.hpp | 6 +- trunk/src/main/srs_main_ingest_hls.cpp | 124 ++++--------------------- 3 files changed, 33 insertions(+), 202 deletions(-) diff --git a/trunk/src/app/srs_app_rtsp.cpp b/trunk/src/app/srs_app_rtsp.cpp index 0712f017d..e40539b8c 100644 --- a/trunk/src/app/srs_app_rtsp.cpp +++ b/trunk/src/app/srs_app_rtsp.cpp @@ -42,6 +42,7 @@ using namespace std; #include #include #include +#include #ifdef SRS_AUTO_STREAM_CASTER @@ -167,7 +168,7 @@ int SrsRtspJitter::correct(int64_t& ts) previous_timestamp = ts; } - delta = srs_max(0, ts - previous_timestamp); + delta = srs_max(0, (int)(ts - previous_timestamp)); if (delta > 90000) { delta = 0; } @@ -195,9 +196,7 @@ SrsRtspConn::SrsRtspConn(SrsRtspCaster* c, st_netfd_t fd, std::string o) trd = new SrsOneCycleThread("rtsp", this); req = NULL; - client = NULL; - transport = new SrsTcpClient(); - stream_id = 0; + sdk = new SrsSimpleRtmpClient(); vjitter = new SrsRtspJitter(); ajitter = new SrsRtspJitter(); @@ -218,8 +217,7 @@ SrsRtspConn::~SrsRtspConn() srs_freep(skt); srs_freep(rtsp); - srs_freep(client); - srs_freep(transport); + srs_freep(sdk); srs_freep(req); srs_freep(vjitter); @@ -626,14 +624,14 @@ int SrsRtspConn::rtmp_write_packet(char type, u_int32_t timestamp, char* data, i SrsSharedPtrMessage* msg = NULL; - if ((ret = srs_rtmp_create_msg(type, timestamp, data, size, stream_id, &msg)) != ERROR_SUCCESS) { + if ((ret = sdk->rtmp_create_msg(type, timestamp, data, size, &msg)) != ERROR_SUCCESS) { srs_error("rtsp: create shared ptr msg failed. ret=%d", ret); return ret; } srs_assert(msg); // send out encoded msg. - if ((ret = client->send_and_free_message(msg, stream_id)) != ERROR_SUCCESS) { + if ((ret = sdk->send_and_free_message(msg)) != ERROR_SUCCESS) { return ret; } @@ -647,11 +645,12 @@ int SrsRtspConn::connect() // when ok, ignore. // TODO: FIXME: support reconnect. - if (transport->connected()) { + if (sdk->connected()) { return ret; } - // parse uri + // generate rtmp url to connect to. + std::string url; if (!req) { std::string schema, host, vhost, app, param; int port; @@ -661,99 +660,25 @@ int SrsRtspConn::connect() std::string output = output_template; output = srs_string_replace(output, "[app]", app); output = srs_string_replace(output, "[stream]", rtsp_stream); - - req = new SrsRequest(); - srs_parse_rtmp_url(output, req->tcUrl, req->stream); - srs_discovery_tc_url(req->tcUrl, req->schema, req->host, req->vhost, req->app, req->port, req->param); } // connect host. - if ((ret = transport->connect(req->host, req->port, ST_UTIME_NO_TIMEOUT)) != ERROR_SUCCESS) { - srs_error("rtsp: connect server %s:%d failed. ret=%d", req->host.c_str(), req->port, ret); - return ret; - } - - srs_freep(client); - client = new SrsRtmpClient(transport); - - client->set_recv_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); - - // connect to vhost/app - if ((ret = client->handshake()) != ERROR_SUCCESS) { - srs_error("rtsp: handshake with server failed. ret=%d", ret); - return ret; - } - if ((ret = connect_app(req->host, req->port)) != ERROR_SUCCESS) { - srs_error("rtsp: connect with server failed. ret=%d", ret); - return ret; - } - if ((ret = client->create_stream(stream_id)) != ERROR_SUCCESS) { - srs_error("rtsp: connect with server failed, stream_id=%d. ret=%d", stream_id, ret); + int64_t cto = SRS_CONSTS_RTMP_TIMEOUT_US; + int64_t sto = SRS_CONSTS_RTMP_PULSE_TIMEOUT_US; + if ((ret = sdk->connect(url, cto, sto)) != ERROR_SUCCESS) { + srs_error("rtsp: connect %s failed, cto=%"PRId64", sto=%"PRId64". ret=%d", url.c_str(), cto, sto, ret); return ret; } // publish. - if ((ret = client->publish(req->stream, stream_id)) != ERROR_SUCCESS) { - srs_error("rtsp: publish failed, stream=%s, stream_id=%d. ret=%d", - req->stream.c_str(), stream_id, ret); + if ((ret = sdk->publish()) != ERROR_SUCCESS) { + srs_error("rtsp: publish %s failed. ret=%d", url.c_str(), ret); return ret; } return write_sequence_header(); } -// TODO: FIXME: refine the connect_app. -int SrsRtspConn::connect_app(string ep_server, int ep_port) -{ - int ret = ERROR_SUCCESS; - - // args of request takes the srs info. - if (req->args == NULL) { - req->args = SrsAmf0Any::object(); - } - - // notify server the edge identity, - // @see https://github.com/simple-rtmp-server/srs/issues/147 - SrsAmf0Object* data = req->args; - data->set("srs_sig", SrsAmf0Any::str(RTMP_SIG_SRS_KEY)); - data->set("srs_server", SrsAmf0Any::str(RTMP_SIG_SRS_KEY" "RTMP_SIG_SRS_VERSION" ("RTMP_SIG_SRS_URL_SHORT")")); - data->set("srs_license", SrsAmf0Any::str(RTMP_SIG_SRS_LICENSE)); - data->set("srs_role", SrsAmf0Any::str(RTMP_SIG_SRS_ROLE)); - data->set("srs_url", SrsAmf0Any::str(RTMP_SIG_SRS_URL)); - data->set("srs_version", SrsAmf0Any::str(RTMP_SIG_SRS_VERSION)); - data->set("srs_site", SrsAmf0Any::str(RTMP_SIG_SRS_WEB)); - data->set("srs_email", SrsAmf0Any::str(RTMP_SIG_SRS_EMAIL)); - data->set("srs_copyright", SrsAmf0Any::str(RTMP_SIG_SRS_COPYRIGHT)); - data->set("srs_primary", SrsAmf0Any::str(RTMP_SIG_SRS_PRIMARY)); - data->set("srs_authors", SrsAmf0Any::str(RTMP_SIG_SRS_AUTHROS)); - // for edge to directly get the id of client. - data->set("srs_pid", SrsAmf0Any::number(getpid())); - data->set("srs_id", SrsAmf0Any::number(_srs_context->get_id())); - - // local ip of edge - std::vector ips = srs_get_local_ipv4_ips(); - assert(_srs_config->get_stats_network() < (int)ips.size()); - std::string local_ip = ips[_srs_config->get_stats_network()]; - data->set("srs_server_ip", SrsAmf0Any::str(local_ip.c_str())); - - // generate the tcUrl - std::string param = ""; - std::string tc_url = srs_generate_tc_url(ep_server, req->vhost, req->app, ep_port, param); - - // upnode server identity will show in the connect_app of client. - // @see https://github.com/simple-rtmp-server/srs/issues/160 - // the debug_srs_upnode is config in vhost and default to true. - bool debug_srs_upnode = _srs_config->get_debug_srs_upnode(req->vhost); - if ((ret = client->connect_app(req->app, tc_url, req, debug_srs_upnode)) != ERROR_SUCCESS) { - srs_error("rtsp: connect with server failed, tcUrl=%s, dsu=%d. ret=%d", - tc_url.c_str(), debug_srs_upnode, ret); - return ret; - } - - return ret; -} - SrsRtspCaster::SrsRtspCaster(SrsConfDirective* c) { // TODO: FIXME: support reload. diff --git a/trunk/src/app/srs_app_rtsp.hpp b/trunk/src/app/srs_app_rtsp.hpp index 135839b33..353b1b351 100644 --- a/trunk/src/app/srs_app_rtsp.hpp +++ b/trunk/src/app/srs_app_rtsp.hpp @@ -56,6 +56,7 @@ class SrsSharedPtrMessage; class SrsCodecSample; class SrsSimpleStream; class SrsPithyPrint; +class SrsSimpleRtmpClient; /** * a rtp connection which transport a stream. @@ -139,11 +140,9 @@ private: SrsOneCycleThread* trd; private: SrsRequest* req; - SrsTcpClient* transport; - SrsRtmpClient* client; + SrsSimpleRtmpClient* sdk; SrsRtspJitter* vjitter; SrsRtspJitter* ajitter; - int stream_id; private: SrsRawH264Stream* avc; std::string h264_sps; @@ -181,7 +180,6 @@ private: // connect to rtmp output url. // @remark ignore when not connected, reconnect when disconnected. virtual int connect(); - virtual int connect_app(std::string ep_server, int ep_port); }; /** diff --git a/trunk/src/main/srs_main_ingest_hls.cpp b/trunk/src/main/srs_main_ingest_hls.cpp index 8c13fb14b..f32c20f2d 100644 --- a/trunk/src/main/srs_main_ingest_hls.cpp +++ b/trunk/src/main/srs_main_ingest_hls.cpp @@ -47,6 +47,7 @@ using namespace std; #include #include #include +#include // pre-declare int proxy_hls2rtmp(std::string hls, std::string rtmp); @@ -637,9 +638,7 @@ private: int64_t raw_aac_dts; private: SrsRequest* req; - SrsTcpClient* transport; - SrsRtmpClient* client; - int stream_id; + SrsSimpleRtmpClient* sdk; private: SrsRawH264Stream* avc; std::string h264_sps; @@ -657,9 +656,7 @@ public: raw_aac_dts = srs_update_system_time_ms(); req = NULL; - client = NULL; - transport = new SrsTcpClient(); - stream_id = 0; + sdk = new SrsSimpleRtmpClient(); avc = new SrsRawH264Stream(); aac = new SrsRawAacStream(); @@ -670,7 +667,7 @@ public: virtual ~SrsIngestSrsOutput() { close(); - srs_freep(transport); + srs_freep(sdk); srs_freep(avc); srs_freep(aac); @@ -707,7 +704,6 @@ public: */ virtual int flush_message_queue(); private: - virtual int connect_app(std::string ep_server, int ep_port); // close the connected io and rtmp to ready to be re-connect. virtual void close(); }; @@ -1187,7 +1183,7 @@ int SrsIngestSrsOutput::rtmp_write_packet(char type, u_int32_t timestamp, char* SrsSharedPtrMessage* msg = NULL; - if ((ret = srs_rtmp_create_msg(type, timestamp, data, size, stream_id, &msg)) != ERROR_SUCCESS) { + if ((ret = sdk->rtmp_create_msg(type, timestamp, data, size, &msg)) != ERROR_SUCCESS) { srs_error("mpegts: create shared ptr msg failed. ret=%d", ret); return ret; } @@ -1196,7 +1192,7 @@ int SrsIngestSrsOutput::rtmp_write_packet(char type, u_int32_t timestamp, char* srs_info("RTMP type=%d, dts=%d, size=%d", type, timestamp, size); // send out encoded msg. - if ((ret = client->send_and_free_message(msg, stream_id)) != ERROR_SUCCESS) { + if ((ret = sdk->send_and_free_message(msg)) != ERROR_SUCCESS) { srs_error("send RTMP type=%d, dts=%d, size=%d failed. ret=%d", type, timestamp, size, ret); return ret; } @@ -1210,110 +1206,24 @@ int SrsIngestSrsOutput::connect() // when ok, ignore. // TODO: FIXME: should reconnect when disconnected. - if (transport->connected()) { + if (sdk->connected()) { return ret; } - srs_trace("connect output=%s", out_rtmp->get_url().c_str()); - - // parse uri - if (!req) { - req = new SrsRequest(); - - string uri = req->tcUrl = out_rtmp->get_url(); - - // tcUrl, stream - if (srs_string_contains(uri, "/")) { - req->stream = srs_path_basename(uri); - req->tcUrl = uri = srs_path_dirname(uri); - } - - srs_discovery_tc_url(req->tcUrl, - req->schema, req->host, req->vhost, req->app, req->port, - req->param); - } + std::string url = out_rtmp->get_url(); + srs_trace("connect output=%s", url.c_str()); // connect host. - if ((ret = transport->connect(req->host, req->port, ST_UTIME_NO_TIMEOUT)) != ERROR_SUCCESS) { - srs_error("mpegts: connect server %s:%d failed. ret=%d", req->host.c_str(), req->port, ret); - return ret; - } - - srs_freep(client); - client = new SrsRtmpClient(transport); - - client->set_recv_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); - - // connect to vhost/app - if ((ret = client->handshake()) != ERROR_SUCCESS) { - srs_error("mpegts: handshake with server failed. ret=%d", ret); - return ret; - } - if ((ret = connect_app(req->host, req->port)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed. ret=%d", ret); - return ret; - } - if ((ret = client->create_stream(stream_id)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed, stream_id=%d. ret=%d", stream_id, ret); + int64_t cto = SRS_CONSTS_RTMP_TIMEOUT_US; + int64_t sto = SRS_CONSTS_RTMP_PULSE_TIMEOUT_US; + if ((ret = sdk->connect(url, cto, sto)) != ERROR_SUCCESS) { + srs_error("mpegts: connect %s failed, cto=%"PRId64", sto=%"PRId64". ret=%d", url.c_str(), cto, sto, ret); return ret; } // publish. - if ((ret = client->publish(req->stream, stream_id)) != ERROR_SUCCESS) { - srs_error("mpegts: publish failed, stream=%s, stream_id=%d. ret=%d", - req->stream.c_str(), stream_id, ret); - return ret; - } - - return ret; -} - -// TODO: FIXME: refine the connect_app. -int SrsIngestSrsOutput::connect_app(string ep_server, int ep_port) -{ - int ret = ERROR_SUCCESS; - - // args of request takes the srs info. - if (req->args == NULL) { - req->args = SrsAmf0Any::object(); - } - - // notify server the edge identity, - // @see https://github.com/simple-rtmp-server/srs/issues/147 - SrsAmf0Object* data = req->args; - data->set("srs_sig", SrsAmf0Any::str(RTMP_SIG_SRS_KEY)); - data->set("srs_server", SrsAmf0Any::str(RTMP_SIG_SRS_KEY" "RTMP_SIG_SRS_VERSION" ("RTMP_SIG_SRS_URL_SHORT")")); - data->set("srs_license", SrsAmf0Any::str(RTMP_SIG_SRS_LICENSE)); - data->set("srs_role", SrsAmf0Any::str(RTMP_SIG_SRS_ROLE)); - data->set("srs_url", SrsAmf0Any::str(RTMP_SIG_SRS_URL)); - data->set("srs_version", SrsAmf0Any::str(RTMP_SIG_SRS_VERSION)); - data->set("srs_site", SrsAmf0Any::str(RTMP_SIG_SRS_WEB)); - data->set("srs_email", SrsAmf0Any::str(RTMP_SIG_SRS_EMAIL)); - data->set("srs_copyright", SrsAmf0Any::str(RTMP_SIG_SRS_COPYRIGHT)); - data->set("srs_primary", SrsAmf0Any::str(RTMP_SIG_SRS_PRIMARY)); - data->set("srs_authors", SrsAmf0Any::str(RTMP_SIG_SRS_AUTHROS)); - // for edge to directly get the id of client. - data->set("srs_pid", SrsAmf0Any::number(getpid())); - data->set("srs_id", SrsAmf0Any::number(_srs_context->get_id())); - - // local ip of edge - std::vector ips = srs_get_local_ipv4_ips(); - assert(0 < (int)ips.size()); - std::string local_ip = ips[0]; - data->set("srs_server_ip", SrsAmf0Any::str(local_ip.c_str())); - - // generate the tcUrl - std::string param = ""; - std::string tc_url = srs_generate_tc_url(ep_server, req->vhost, req->app, ep_port, param); - - // upnode server identity will show in the connect_app of client. - // @see https://github.com/simple-rtmp-server/srs/issues/160 - // the debug_srs_upnode is config in vhost and default to true. - bool debug_srs_upnode = true; - if ((ret = client->connect_app(req->app, tc_url, req, debug_srs_upnode)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed, tcUrl=%s, dsu=%d. ret=%d", - tc_url.c_str(), debug_srs_upnode, ret); + if ((ret = sdk->publish()) != ERROR_SUCCESS) { + srs_error("mpegts: publish %s failed. ret=%d", url.c_str(), ret); return ret; } @@ -1325,10 +1235,8 @@ void SrsIngestSrsOutput::close() srs_trace("close output=%s", out_rtmp->get_url().c_str()); h264_sps_pps_sent = false; - srs_freep(client); srs_freep(req); - - transport->close(); + sdk->close(); } // the context for ingest hls stream. From 8974fe298b24e186df51aa31535c18b4a5389b8b Mon Sep 17 00:00:00 2001 From: winlin Date: Thu, 15 Oct 2015 17:45:58 +0800 Subject: [PATCH 17/46] connect to kafka and send metadata request. --- trunk/conf/full.conf | 5 ++- trunk/src/app/srs_app_config.cpp | 17 +++++++++ trunk/src/app/srs_app_config.hpp | 4 +++ trunk/src/app/srs_app_kafka.cpp | 34 +++++++++++++++--- trunk/src/app/srs_app_kafka.hpp | 4 +++ trunk/src/kernel/srs_kernel_consts.hpp | 8 +++++ trunk/src/protocol/srs_kafka_stack.cpp | 40 +++++++++++++++++++++ trunk/src/protocol/srs_kafka_stack.hpp | 50 ++++++++++++++++++++++---- 8 files changed, 151 insertions(+), 11 deletions(-) diff --git a/trunk/conf/full.conf b/trunk/conf/full.conf index 1ebc28790..dcffd69d9 100644 --- a/trunk/conf/full.conf +++ b/trunk/conf/full.conf @@ -242,8 +242,11 @@ kafka { enabled off; # the broker list, broker is # and use space to specify multple brokers. - # for exampl, 127.0.0.1:9092 127.0.0.1:9093 + # for example, 127.0.0.1:9092 127.0.0.1:9093 brokers 127.0.0.1:9092; + # the kafka topic to use. + # default: srs + topic srs; } ############################################################################################# diff --git a/trunk/src/app/srs_app_config.cpp b/trunk/src/app/srs_app_config.cpp index f0de5ac7a..12e6525ee 100644 --- a/trunk/src/app/srs_app_config.cpp +++ b/trunk/src/app/srs_app_config.cpp @@ -4298,6 +4298,23 @@ SrsConfDirective* SrsConfig::get_kafka_brokers() return conf; } +string SrsConfig::get_kafka_topic() +{ + static string DEFAULT = "srs"; + + SrsConfDirective* conf = root->get("kafka"); + if (!conf) { + return DEFAULT; + } + + conf = conf->get("topic"); + if (!conf || conf->arg0().empty()) { + return DEFAULT; + } + + return conf->arg0(); +} + SrsConfDirective* SrsConfig::get_vhost(string vhost, bool try_default_vhost) { srs_assert(root); diff --git a/trunk/src/app/srs_app_config.hpp b/trunk/src/app/srs_app_config.hpp index 48cbd59e3..662187b1d 100644 --- a/trunk/src/app/srs_app_config.hpp +++ b/trunk/src/app/srs_app_config.hpp @@ -638,6 +638,10 @@ public: * get the broker list, each is format in . */ virtual SrsConfDirective* get_kafka_brokers(); + /** + * get the kafka topic to use for srs. + */ + virtual std::string get_kafka_topic(); // vhost specified section public: /** diff --git a/trunk/src/app/srs_app_kafka.cpp b/trunk/src/app/srs_app_kafka.cpp index 9eb821941..2bcc0f2c7 100644 --- a/trunk/src/app/srs_app_kafka.cpp +++ b/trunk/src/app/srs_app_kafka.cpp @@ -33,6 +33,7 @@ using namespace std; #include #include #include +#include #ifdef SRS_AUTO_KAFKA @@ -40,12 +41,16 @@ SrsKafkaProducer::SrsKafkaProducer() { lb = new SrsLbRoundRobin(); worker = new SrsAsyncCallWorker(); + transport = new SrsTcpClient(); + kafka = new SrsKafkaClient(transport); } SrsKafkaProducer::~SrsKafkaProducer() { srs_freep(lb); srs_freep(worker); + srs_freep(kafka); + srs_freep(transport); } int SrsKafkaProducer::initialize() @@ -86,25 +91,46 @@ int SrsKafkaProducer::request_metadata() { int ret = ERROR_SUCCESS; + // ignore when disabled. bool enabled = _srs_config->get_kafka_enabled(); if (!enabled) { return ret; } + // select one broker to connect to. SrsConfDirective* brokers = _srs_config->get_kafka_brokers(); if (!brokers) { srs_warn("ignore for empty brokers."); return ret; } - srs_assert(!brokers->args.empty()); - std::string broker = lb->select(brokers->args); + std::string server; + int port = SRS_CONSTS_KAFKA_DEFAULT_PORT; + if (true) { + srs_assert(!brokers->args.empty()); + std::string broker = lb->select(brokers->args); + srs_parse_endpoint(broker, server, port); + } + + // connect to kafka server. + if ((ret = transport->connect(server, port, SRS_CONSTS_KAFKA_TIMEOUT_US)) != ERROR_SUCCESS) { + srs_error("kafka connect %s:%d failed. ret=%d", server.c_str(), port, ret); + return ret; + } + + // do fetch medata from broker. + std::string topic = _srs_config->get_kafka_topic(); + if ((ret = kafka->fetch_metadata(topic)) != ERROR_SUCCESS) { + srs_error("kafka fetch metadata failed. ret=%d", ret); + return ret; + } + // log when completed. if (true) { std::string senabled = srs_bool2switch(enabled); std::string sbrokers = srs_join_vector_string(brokers->args, ","); - srs_trace("kafka ok, enabled:%s, brokers:%s, current:[%d]%s", - senabled.c_str(), sbrokers.c_str(), lb->current(), broker.c_str()); + srs_trace("kafka ok, enabled:%s, brokers:%s, current:[%d]%s:%d, topic:%s", + senabled.c_str(), sbrokers.c_str(), lb->current(), server.c_str(), port, topic.c_str()); } return ret; diff --git a/trunk/src/app/srs_app_kafka.hpp b/trunk/src/app/srs_app_kafka.hpp index 02dc39724..81251317f 100644 --- a/trunk/src/app/srs_app_kafka.hpp +++ b/trunk/src/app/srs_app_kafka.hpp @@ -31,6 +31,8 @@ class SrsLbRoundRobin; class SrsAsyncCallWorker; +class SrsTcpClient; +class SrsKafkaClient; #ifdef SRS_AUTO_KAFKA @@ -42,6 +44,8 @@ class SrsKafkaProducer private: SrsLbRoundRobin* lb; SrsAsyncCallWorker* worker; + SrsTcpClient* transport; + SrsKafkaClient* kafka; public: SrsKafkaProducer(); virtual ~SrsKafkaProducer(); diff --git a/trunk/src/kernel/srs_kernel_consts.hpp b/trunk/src/kernel/srs_kernel_consts.hpp index d209c7762..34b4a0fa4 100644 --- a/trunk/src/kernel/srs_kernel_consts.hpp +++ b/trunk/src/kernel/srs_kernel_consts.hpp @@ -398,5 +398,13 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #define SRS_CONSTS_RTSP_RTSPVersionNotSupported_str "RTSP Version Not Supported" #define SRS_CONSTS_RTSP_OptionNotSupported_str "Option not support" +/////////////////////////////////////////////////////////// +// KAFKA consts values +/////////////////////////////////////////////////////////// +#define SRS_CONSTS_KAFKA_DEFAULT_PORT 9092 + +// the common io timeout, for both recv and send. +#define SRS_CONSTS_KAFKA_TIMEOUT_US (int64_t)(30*1000*1000LL) + #endif diff --git a/trunk/src/protocol/srs_kafka_stack.cpp b/trunk/src/protocol/srs_kafka_stack.cpp index f23d0d00a..deada64a1 100644 --- a/trunk/src/protocol/srs_kafka_stack.cpp +++ b/trunk/src/protocol/srs_kafka_stack.cpp @@ -23,8 +23,11 @@ #include +#include using namespace std; +#include + #ifdef SRS_AUTO_KAFKA SrsKafkaString::SrsKafkaString() @@ -196,5 +199,42 @@ SrsKafkaTopicMetadataRequest::~SrsKafkaTopicMetadataRequest() { } +SrsKafkaProtocol::SrsKafkaProtocol(ISrsProtocolReaderWriter* io) +{ + skt = io; +} + +SrsKafkaProtocol::~SrsKafkaProtocol() +{ +} + +int SrsKafkaProtocol::send_and_free_message(SrsKafkaMessage* msg) +{ + int ret = ERROR_SUCCESS; + + // TODO: FIXME: implements it. + + return ret; +} + +SrsKafkaClient::SrsKafkaClient(ISrsProtocolReaderWriter* io) +{ + protocol = new SrsKafkaProtocol(io); +} + +SrsKafkaClient::~SrsKafkaClient() +{ + srs_freep(protocol); +} + +int SrsKafkaClient::fetch_metadata(string topic) +{ + int ret = ERROR_SUCCESS; + + // TODO: FIXME: implements it. + + return ret; +} + #endif diff --git a/trunk/src/protocol/srs_kafka_stack.hpp b/trunk/src/protocol/srs_kafka_stack.hpp index 4be61c3eb..dc6a9c61d 100644 --- a/trunk/src/protocol/srs_kafka_stack.hpp +++ b/trunk/src/protocol/srs_kafka_stack.hpp @@ -30,6 +30,9 @@ #include #include +#include + +class ISrsProtocolReaderWriter; #ifdef SRS_AUTO_KAFKA @@ -94,7 +97,7 @@ public: * array of a structure foo as [foo]. * * Usage: - * SrsKafkaArray body; + * SrsKafkaArray body; * body.append(new SrsKafkaBytes()); * * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Requests @@ -104,8 +107,8 @@ class SrsKafkaArray { private: int length; - std::vector elems; - typedef typename std::vector::iterator SrsIterator; + std::vector elems; + typedef typename std::vector::iterator SrsIterator; public: SrsKafkaArray() { @@ -114,13 +117,13 @@ public: virtual ~SrsKafkaArray() { for (SrsIterator it = elems.begin(); it != elems.end(); ++it) { - T* elem = *it; + T elem = *it; srs_freep(elem); } elems.clear(); } public: - virtual void append(T* elem) + virtual void append(T elem) { length++; elems.push_back(elem); @@ -307,12 +310,47 @@ class SrsKafkaTopicMetadataRequest { private: SrsKafkaRequestHeader header; - SrsKafkaArray request; + SrsKafkaArray request; public: SrsKafkaTopicMetadataRequest(); virtual ~SrsKafkaTopicMetadataRequest(); }; +/** + * the kafka protocol stack, use to send and recv kakfa messages. + */ +class SrsKafkaProtocol +{ +private: + ISrsProtocolReaderWriter* skt; +public: + SrsKafkaProtocol(ISrsProtocolReaderWriter* io); + virtual ~SrsKafkaProtocol(); +public: + /** + * write the message to kafka server. + * @param msg the msg to send. user must not free it again. + */ + virtual int send_and_free_message(SrsKafkaMessage* msg); +}; + +/** + * the kafka client, for producer or consumer. + */ +class SrsKafkaClient +{ +private: + SrsKafkaProtocol* protocol; +public: + SrsKafkaClient(ISrsProtocolReaderWriter* io); + virtual ~SrsKafkaClient(); +public: + /** + * fetch the metadata from broker for topic. + */ + virtual int fetch_metadata(std::string topic); +}; + #endif #endif From 8a4ec49064fe6a5d7c1f39c60f97555852e535b2 Mon Sep 17 00:00:00 2001 From: winlin Date: Thu, 15 Oct 2015 18:08:17 +0800 Subject: [PATCH 18/46] add graph comments for size of request and response. --- trunk/src/protocol/srs_kafka_stack.cpp | 20 +++++++- trunk/src/protocol/srs_kafka_stack.hpp | 71 +++++++++++++++++++++----- 2 files changed, 77 insertions(+), 14 deletions(-) diff --git a/trunk/src/protocol/srs_kafka_stack.cpp b/trunk/src/protocol/srs_kafka_stack.cpp index deada64a1..1137f4add 100644 --- a/trunk/src/protocol/srs_kafka_stack.cpp +++ b/trunk/src/protocol/srs_kafka_stack.cpp @@ -150,13 +150,29 @@ void SrsKafkaRequestHeader::set_api_key(SrsKafkaApiKey key) api_key = (int16_t)key; } -SrsKafkaResponse::SrsKafkaResponse() +SrsKafkaResponseHeader::SrsKafkaResponseHeader() { + size = 0; correlation_id = 0; } -SrsKafkaResponse::~SrsKafkaResponse() +SrsKafkaResponseHeader::~SrsKafkaResponseHeader() +{ +} + +int SrsKafkaResponseHeader::header_size() +{ + return 4; +} + +int SrsKafkaResponseHeader::message_size() +{ + return size - header_size(); +} + +int SrsKafkaResponseHeader::total_size() { + return 4 + size; } SrsKafkaMessage::SrsKafkaMessage() diff --git a/trunk/src/protocol/srs_kafka_stack.hpp b/trunk/src/protocol/srs_kafka_stack.hpp index dc6a9c61d..bcb0b7ffa 100644 --- a/trunk/src/protocol/srs_kafka_stack.hpp +++ b/trunk/src/protocol/srs_kafka_stack.hpp @@ -179,18 +179,24 @@ public: virtual ~SrsKafkaRequestHeader(); public: /** - * the size of header, exclude the 4bytes size. - * @remark total_size = 4 + header_size + message_size. + * the layout of request: + * +-----------+----------------------------------+ + * | 4B size | [size] bytes | + * +-----------+------------+---------------------+ + * | 4B size | header | message | + * +-----------+------------+---------------------+ + * | total size = 4 + header + message | + * +----------------------------------------------+ + * where the header is specifies this request header without the start 4B size. + * @remark size = 4 + header + message. */ virtual int header_size(); /** * the size of message, the bytes left after the header. - * @remark total_size = 4 + header_size + message_size. */ virtual int message_size(); /** - * the total size of the request, 4bytes + size of header and message. - * @remark total_size = 4 + header_size + message_size. + * the total size of the request, includes the 4B size. */ virtual int total_size(); public: @@ -210,21 +216,53 @@ public: }; /** - * the common kafka response. - * The response will always match the paired request (e.g. we will + * the header of response, include the size of response. + * The response will always match the paired request (e.g. we will * send a MetadataResponse in return to a MetadataRequest). * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Responses */ -class SrsKafkaResponse +class SrsKafkaResponseHeader { -protected: +private: /** - * The server passes back whatever integer the client supplied as the correlation in the request. + * The MessageSize field gives the size of the subsequent request or response + * message in bytes. The client can read requests by first reading this 4 byte + * size as an integer N, and then reading and parsing the subsequent N bytes + * of the request. + */ + int32_t size; +private: + /** + * This is a user-supplied integer. It will be passed back in + * the response by the server, unmodified. It is useful for matching + * request and response between the client and server. */ int32_t correlation_id; public: - SrsKafkaResponse(); - virtual ~SrsKafkaResponse(); + SrsKafkaResponseHeader(); + virtual ~SrsKafkaResponseHeader(); +public: + /** + * the layout of response: + * +-----------+----------------------------------+ + * | 4B size | [size] bytes | + * +-----------+------------+---------------------+ + * | 4B size | 4B header | message | + * +-----------+------------+---------------------+ + * | total size = 4 + 4 + message | + * +----------------------------------------------+ + * where the header is specifies this request header without the start 4B size. + * @remark size = 4 + 4 + message. + */ + virtual int header_size(); + /** + * the size of message, the bytes left after the header. + */ + virtual int message_size(); + /** + * the total size of the request, includes the 4B size. + */ + virtual int total_size(); }; /** @@ -316,6 +354,15 @@ public: virtual ~SrsKafkaTopicMetadataRequest(); }; +class SrsKafkaTopicMetadataResponse +{ +private: + SrsKafkaRequestHeader header; +public: + SrsKafkaTopicMetadataResponse(); + virtual ~SrsKafkaTopicMetadataResponse(); +}; + /** * the kafka protocol stack, use to send and recv kakfa messages. */ From 45755fd1e71cb9c3f8a7b43019ebe9e4a79d4559 Mon Sep 17 00:00:00 2001 From: winlin Date: Fri, 16 Oct 2015 15:34:55 +0800 Subject: [PATCH 19/46] refine code for kakfa request/response, string/bytes. --- trunk/src/protocol/srs_kafka_stack.cpp | 63 ++++++++++++++++++++++++-- trunk/src/protocol/srs_kafka_stack.hpp | 50 ++++++++++++++++---- 2 files changed, 99 insertions(+), 14 deletions(-) diff --git a/trunk/src/protocol/srs_kafka_stack.cpp b/trunk/src/protocol/srs_kafka_stack.cpp index 1137f4add..efc8e89f3 100644 --- a/trunk/src/protocol/srs_kafka_stack.cpp +++ b/trunk/src/protocol/srs_kafka_stack.cpp @@ -27,6 +27,7 @@ using namespace std; #include +#include #ifdef SRS_AUTO_KAFKA @@ -36,6 +37,15 @@ SrsKafkaString::SrsKafkaString() data = NULL; } +SrsKafkaString::SrsKafkaString(string v) +{ + size = (int16_t)v.length(); + + srs_assert(size > 0); + data = new char[size]; + memcpy(data, v.data(), size); +} + SrsKafkaString::~SrsKafkaString() { srs_freep(data); @@ -62,6 +72,15 @@ SrsKafkaBytes::SrsKafkaBytes() data = NULL; } +SrsKafkaBytes::SrsKafkaBytes(const char* v, int nb_v) +{ + size = (int16_t)nb_v; + + srs_assert(size > 0); + data = new char[size]; + memcpy(data, v, size); +} + SrsKafkaBytes::~SrsKafkaBytes() { srs_freep(data); @@ -175,7 +194,7 @@ int SrsKafkaResponseHeader::total_size() return 4 + size; } -SrsKafkaMessage::SrsKafkaMessage() +SrsKafkaRawMessage::SrsKafkaRawMessage() { offset = 0; message_size = 0; @@ -186,7 +205,7 @@ SrsKafkaMessage::SrsKafkaMessage() value = new SrsKafkaBytes(); } -SrsKafkaMessage::~SrsKafkaMessage() +SrsKafkaRawMessage::~SrsKafkaRawMessage() { srs_freep(key); srs_freep(value); @@ -198,14 +217,30 @@ SrsKafkaMessageSet::SrsKafkaMessageSet() SrsKafkaMessageSet::~SrsKafkaMessageSet() { - vector::iterator it; + vector::iterator it; for (it = messages.begin(); it != messages.end(); ++it) { - SrsKafkaMessage* message = *it; + SrsKafkaRawMessage* message = *it; srs_freep(message); } messages.clear(); } +SrsKafkaRequest::SrsKafkaRequest() +{ +} + +SrsKafkaRequest::~SrsKafkaRequest() +{ +} + +SrsKafkaResponse::SrsKafkaResponse() +{ +} + +SrsKafkaResponse::~SrsKafkaResponse() +{ +} + SrsKafkaTopicMetadataRequest::SrsKafkaTopicMetadataRequest() { header.set_api_key(SrsKafkaApiKeyMetadataRequest); @@ -215,6 +250,19 @@ SrsKafkaTopicMetadataRequest::~SrsKafkaTopicMetadataRequest() { } +void SrsKafkaTopicMetadataRequest::add_topic(string topic) +{ + topics.append(new SrsKafkaString(topic)); +} + +SrsKafkaTopicMetadataResponse::SrsKafkaTopicMetadataResponse() +{ +} + +SrsKafkaTopicMetadataResponse::~SrsKafkaTopicMetadataResponse() +{ +} + SrsKafkaProtocol::SrsKafkaProtocol(ISrsProtocolReaderWriter* io) { skt = io; @@ -224,7 +272,7 @@ SrsKafkaProtocol::~SrsKafkaProtocol() { } -int SrsKafkaProtocol::send_and_free_message(SrsKafkaMessage* msg) +int SrsKafkaProtocol::send_and_free_message(SrsKafkaRequest* msg) { int ret = ERROR_SUCCESS; @@ -247,6 +295,11 @@ int SrsKafkaClient::fetch_metadata(string topic) { int ret = ERROR_SUCCESS; + SrsKafkaTopicMetadataRequest* req = new SrsKafkaTopicMetadataRequest(); + SrsAutoFree(SrsKafkaTopicMetadataRequest, req); + + req->add_topic(topic); + // TODO: FIXME: implements it. return ret; diff --git a/trunk/src/protocol/srs_kafka_stack.hpp b/trunk/src/protocol/srs_kafka_stack.hpp index bcb0b7ffa..92d30d50e 100644 --- a/trunk/src/protocol/srs_kafka_stack.hpp +++ b/trunk/src/protocol/srs_kafka_stack.hpp @@ -64,6 +64,7 @@ private: char* data; public: SrsKafkaString(); + SrsKafkaString(std::string v); virtual ~SrsKafkaString(); public: virtual bool null(); @@ -83,6 +84,7 @@ private: char* data; public: SrsKafkaBytes(); + SrsKafkaBytes(const char* v, int nb_v); virtual ~SrsKafkaBytes(); public: virtual bool null(); @@ -269,7 +271,7 @@ public: * the kafka message in message set. * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets */ -struct SrsKafkaMessage +struct SrsKafkaRawMessage { // metadata. public: @@ -313,8 +315,8 @@ public: */ SrsKafkaBytes* value; public: - SrsKafkaMessage(); - virtual ~SrsKafkaMessage(); + SrsKafkaRawMessage(); + virtual ~SrsKafkaRawMessage(); }; /** @@ -324,12 +326,32 @@ public: class SrsKafkaMessageSet { private: - std::vector messages; + std::vector messages; public: SrsKafkaMessageSet(); virtual ~SrsKafkaMessageSet(); }; +/** + * the kafka request message, for protocol to send. + */ +class SrsKafkaRequest +{ +public: + SrsKafkaRequest(); + virtual ~SrsKafkaRequest(); +}; + +/** + * the kafka response message, for protocol to recv. + */ +class SrsKafkaResponse +{ +public: + SrsKafkaResponse(); + virtual ~SrsKafkaResponse(); +}; + /** * request the metadata from broker. * This API answers the following questions: @@ -344,20 +366,30 @@ public: * * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-MetadataAPI */ -class SrsKafkaTopicMetadataRequest +class SrsKafkaTopicMetadataRequest : public SrsKafkaRequest { private: SrsKafkaRequestHeader header; - SrsKafkaArray request; + SrsKafkaArray topics; public: SrsKafkaTopicMetadataRequest(); virtual ~SrsKafkaTopicMetadataRequest(); +public: + virtual void add_topic(std::string topic); }; -class SrsKafkaTopicMetadataResponse +/** + * response for the metadata request from broker. + * The response contains metadata for each partition, + * with partitions grouped together by topic. This + * metadata refers to brokers by their broker id. + * The brokers each have a host and port. + * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-MetadataResponse + */ +class SrsKafkaTopicMetadataResponse : public SrsKafkaResponse { private: - SrsKafkaRequestHeader header; + SrsKafkaResponseHeader header; public: SrsKafkaTopicMetadataResponse(); virtual ~SrsKafkaTopicMetadataResponse(); @@ -378,7 +410,7 @@ public: * write the message to kafka server. * @param msg the msg to send. user must not free it again. */ - virtual int send_and_free_message(SrsKafkaMessage* msg); + virtual int send_and_free_message(SrsKafkaRequest* msg); }; /** From 9117e1e91859158e0a2b6771ff61468d394238af Mon Sep 17 00:00:00 2001 From: winlin Date: Fri, 16 Oct 2015 16:30:45 +0800 Subject: [PATCH 20/46] extract ISrsCodec for code and decode object. --- trunk/src/kernel/srs_kernel_buffer.cpp | 8 ++ trunk/src/kernel/srs_kernel_buffer.hpp | 60 +++++++++++- trunk/src/protocol/srs_kafka_stack.cpp | 129 +++++++++++++++++++++++-- trunk/src/protocol/srs_kafka_stack.hpp | 63 +++++++++--- trunk/src/protocol/srs_rtmp_stack.cpp | 6 +- 5 files changed, 236 insertions(+), 30 deletions(-) diff --git a/trunk/src/kernel/srs_kernel_buffer.cpp b/trunk/src/kernel/srs_kernel_buffer.cpp index 2c998b198..a4b58e197 100644 --- a/trunk/src/kernel/srs_kernel_buffer.cpp +++ b/trunk/src/kernel/srs_kernel_buffer.cpp @@ -29,6 +29,14 @@ using namespace std; #include #include +ISrsCodec::ISrsCodec() +{ +} + +ISrsCodec::~ISrsCodec() +{ +} + SrsBuffer::SrsBuffer() { p = bytes = NULL; diff --git a/trunk/src/kernel/srs_kernel_buffer.hpp b/trunk/src/kernel/srs_kernel_buffer.hpp index f20d57933..f79e08f33 100644 --- a/trunk/src/kernel/srs_kernel_buffer.hpp +++ b/trunk/src/kernel/srs_kernel_buffer.hpp @@ -33,11 +33,60 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #include #include +class SrsBuffer; + /** -* bytes utility, used to: -* convert basic types to bytes, -* build basic types from bytes. -*/ + * the srs codec, to code and decode object with bytes: + * code: to encode/serialize object to bytes in buffer, + * decode: to decode/deserialize object from bytes in buffer. + * we use SrsBuffer as bytes helper utility, + * for example, to code: + * ISrsCodec* obj = ... + * char* bytes = new char[obj->size()]; + * + * SrsBuffer* buf = new SrsBuffer(); + * buf->initialize(bytes, obj->size()) + * + * obj->encode(buf); + * for example, to decode: + * int nb_bytes = ... + * char* bytes = ... + * + * SrsBuffer* buf = new Srsbuffer(); + * buf->initialize(bytes, nb_bytes); + * + * ISrsCodec* obj = ... + * obj->decode(buf); + * @remark protocol or amf0 or json should implements this interface. + */ +// TODO: FIXME: protocol, amf0, json should implements it. +class ISrsCodec +{ +public: + ISrsCodec(); + virtual ~ISrsCodec(); +public: + /** + * get the size of object to encode object to bytes. + */ + virtual int size() = 0; + /** + * encode object to bytes in SrsBuffer. + */ + virtual int encode(SrsBuffer* buf) = 0; +public: + /** + * decode object from bytes in SrsBuffer. + */ + virtual int decode(SrsBuffer* buf) = 0; +}; + +/** + * bytes utility, used to: + * convert basic types to bytes, + * build basic types from bytes. + * @remark the buffer never mange the bytes, user must manage it. + */ class SrsBuffer { private: @@ -157,7 +206,8 @@ public: }; /** - * the bit stream. + * the bit stream, base on SrsBuffer, + * for exmaple, the h.264 avc stream is bit stream. */ class SrsBitBuffer { diff --git a/trunk/src/protocol/srs_kafka_stack.cpp b/trunk/src/protocol/srs_kafka_stack.cpp index efc8e89f3..67ee28907 100644 --- a/trunk/src/protocol/srs_kafka_stack.cpp +++ b/trunk/src/protocol/srs_kafka_stack.cpp @@ -28,6 +28,7 @@ using namespace std; #include #include +#include #ifdef SRS_AUTO_KAFKA @@ -103,7 +104,7 @@ int SrsKafkaBytes::total_size() SrsKafkaRequestHeader::SrsKafkaRequestHeader() { - size = 0; + _size = 0; api_key = api_version = 0; correlation_id = 0; client_id = new SrsKafkaString(); @@ -121,12 +122,12 @@ int SrsKafkaRequestHeader::header_size() int SrsKafkaRequestHeader::message_size() { - return size - header_size(); + return _size - header_size(); } int SrsKafkaRequestHeader::total_size() { - return 4 + size; + return 4 + _size; } bool SrsKafkaRequestHeader::is_producer_request() @@ -169,9 +170,28 @@ void SrsKafkaRequestHeader::set_api_key(SrsKafkaApiKey key) api_key = (int16_t)key; } +int SrsKafkaRequestHeader::size() +{ + return 4 + _size; +} + +int SrsKafkaRequestHeader::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + // TODO: FIXME: implements it. + return ret; +} + +int SrsKafkaRequestHeader::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + // TODO: FIXME: implements it. + return ret; +} + SrsKafkaResponseHeader::SrsKafkaResponseHeader() { - size = 0; + _size = 0; correlation_id = 0; } @@ -186,12 +206,31 @@ int SrsKafkaResponseHeader::header_size() int SrsKafkaResponseHeader::message_size() { - return size - header_size(); + return _size - header_size(); } int SrsKafkaResponseHeader::total_size() { - return 4 + size; + return 4 + _size; +} + +int SrsKafkaResponseHeader::size() +{ + return 4 + _size; +} + +int SrsKafkaResponseHeader::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + // TODO: FIXME: implements it. + return ret; +} + +int SrsKafkaResponseHeader::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + // TODO: FIXME: implements it. + return ret; } SrsKafkaRawMessage::SrsKafkaRawMessage() @@ -233,6 +272,21 @@ SrsKafkaRequest::~SrsKafkaRequest() { } +int SrsKafkaRequest::size() +{ + return header.size(); +} + +int SrsKafkaRequest::encode(SrsBuffer* buf) +{ + return header.encode(buf); +} + +int SrsKafkaRequest::decode(SrsBuffer* buf) +{ + return header.decode(buf); +} + SrsKafkaResponse::SrsKafkaResponse() { } @@ -241,6 +295,21 @@ SrsKafkaResponse::~SrsKafkaResponse() { } +int SrsKafkaResponse::size() +{ + return header.size(); +} + +int SrsKafkaResponse::encode(SrsBuffer* buf) +{ + return header.encode(buf); +} + +int SrsKafkaResponse::decode(SrsBuffer* buf) +{ + return header.decode(buf); +} + SrsKafkaTopicMetadataRequest::SrsKafkaTopicMetadataRequest() { header.set_api_key(SrsKafkaApiKeyMetadataRequest); @@ -255,6 +324,27 @@ void SrsKafkaTopicMetadataRequest::add_topic(string topic) topics.append(new SrsKafkaString(topic)); } +int SrsKafkaTopicMetadataRequest::size() +{ + int ret = ERROR_SUCCESS; + // TODO: FIXME: implements it. + return ret; +} + +int SrsKafkaTopicMetadataRequest::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + // TODO: FIXME: implements it. + return ret; +} + +int SrsKafkaTopicMetadataRequest::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + // TODO: FIXME: implements it. + return ret; +} + SrsKafkaTopicMetadataResponse::SrsKafkaTopicMetadataResponse() { } @@ -263,6 +353,27 @@ SrsKafkaTopicMetadataResponse::~SrsKafkaTopicMetadataResponse() { } +int SrsKafkaTopicMetadataResponse::size() +{ + int ret = ERROR_SUCCESS; + // TODO: FIXME: implements it. + return ret; +} + +int SrsKafkaTopicMetadataResponse::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + // TODO: FIXME: implements it. + return ret; +} + +int SrsKafkaTopicMetadataResponse::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + // TODO: FIXME: implements it. + return ret; +} + SrsKafkaProtocol::SrsKafkaProtocol(ISrsProtocolReaderWriter* io) { skt = io; @@ -296,10 +407,14 @@ int SrsKafkaClient::fetch_metadata(string topic) int ret = ERROR_SUCCESS; SrsKafkaTopicMetadataRequest* req = new SrsKafkaTopicMetadataRequest(); - SrsAutoFree(SrsKafkaTopicMetadataRequest, req); req->add_topic(topic); + if ((ret = protocol->send_and_free_message(req)) != ERROR_SUCCESS) { + srs_error("kafka send message failed. ret=%d", ret); + return ret; + } + // TODO: FIXME: implements it. return ret; diff --git a/trunk/src/protocol/srs_kafka_stack.hpp b/trunk/src/protocol/srs_kafka_stack.hpp index 92d30d50e..871403aab 100644 --- a/trunk/src/protocol/srs_kafka_stack.hpp +++ b/trunk/src/protocol/srs_kafka_stack.hpp @@ -32,6 +32,8 @@ #include #include +#include + class ISrsProtocolReaderWriter; #ifdef SRS_AUTO_KAFKA @@ -136,7 +138,7 @@ public: * the header of request, includes the size of request. * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Requests */ -class SrsKafkaRequestHeader +class SrsKafkaRequestHeader : public ISrsCodec { private: /** @@ -145,7 +147,7 @@ private: * size as an integer N, and then reading and parsing the subsequent N bytes * of the request. */ - int32_t size; + int32_t _size; private: /** * This is a numeric id for the API being invoked (i.e. is it @@ -179,13 +181,13 @@ private: public: SrsKafkaRequestHeader(); virtual ~SrsKafkaRequestHeader(); -public: +private: /** * the layout of request: * +-----------+----------------------------------+ - * | 4B size | [size] bytes | + * | 4B _size | [_size] bytes | * +-----------+------------+---------------------+ - * | 4B size | header | message | + * | 4B _size | header | message | * +-----------+------------+---------------------+ * | total size = 4 + header + message | * +----------------------------------------------+ @@ -215,6 +217,11 @@ public: virtual bool is_consumer_metadata_request(); // set the api key. virtual void set_api_key(SrsKafkaApiKey key); +// interface ISrsCodec +public: + virtual int size(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); }; /** @@ -223,7 +230,7 @@ public: * send a MetadataResponse in return to a MetadataRequest). * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Responses */ -class SrsKafkaResponseHeader +class SrsKafkaResponseHeader : public ISrsCodec { private: /** @@ -232,7 +239,7 @@ private: * size as an integer N, and then reading and parsing the subsequent N bytes * of the request. */ - int32_t size; + int32_t _size; private: /** * This is a user-supplied integer. It will be passed back in @@ -243,13 +250,13 @@ private: public: SrsKafkaResponseHeader(); virtual ~SrsKafkaResponseHeader(); -public: +private: /** * the layout of response: * +-----------+----------------------------------+ - * | 4B size | [size] bytes | + * | 4B _size | [_size] bytes | * +-----------+------------+---------------------+ - * | 4B size | 4B header | message | + * | 4B _size | 4B header | message | * +-----------+------------+---------------------+ * | total size = 4 + 4 + message | * +----------------------------------------------+ @@ -265,6 +272,11 @@ public: * the total size of the request, includes the 4B size. */ virtual int total_size(); +// interface ISrsCodec +public: + virtual int size(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); }; /** @@ -335,21 +347,35 @@ public: /** * the kafka request message, for protocol to send. */ -class SrsKafkaRequest +class SrsKafkaRequest : public ISrsCodec { +protected: + SrsKafkaRequestHeader header; public: SrsKafkaRequest(); virtual ~SrsKafkaRequest(); +// interface ISrsCodec +public: + virtual int size(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); }; /** * the kafka response message, for protocol to recv. */ -class SrsKafkaResponse +class SrsKafkaResponse : public ISrsCodec { +protected: + SrsKafkaResponseHeader header; public: SrsKafkaResponse(); virtual ~SrsKafkaResponse(); +// interface ISrsCodec +public: + virtual int size(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); }; /** @@ -369,13 +395,17 @@ public: class SrsKafkaTopicMetadataRequest : public SrsKafkaRequest { private: - SrsKafkaRequestHeader header; SrsKafkaArray topics; public: SrsKafkaTopicMetadataRequest(); virtual ~SrsKafkaTopicMetadataRequest(); public: virtual void add_topic(std::string topic); +// interface ISrsCodec +public: + virtual int size(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); }; /** @@ -388,11 +418,14 @@ public: */ class SrsKafkaTopicMetadataResponse : public SrsKafkaResponse { -private: - SrsKafkaResponseHeader header; public: SrsKafkaTopicMetadataResponse(); virtual ~SrsKafkaTopicMetadataResponse(); +// interface ISrsCodec +public: + virtual int size(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); }; /** diff --git a/trunk/src/protocol/srs_rtmp_stack.cpp b/trunk/src/protocol/srs_rtmp_stack.cpp index 0336f748e..8fc391d77 100644 --- a/trunk/src/protocol/srs_rtmp_stack.cpp +++ b/trunk/src/protocol/srs_rtmp_stack.cpp @@ -654,12 +654,12 @@ int SrsProtocol::do_simple_send(SrsMessageHeader* mh, char* payload, int size) int nbh = 0; if (p == payload) { nbh = srs_chunk_header_c0( - mh->perfer_cid, mh->timestamp, mh->payload_length, + mh->perfer_cid, (u_int32_t)mh->timestamp, mh->payload_length, mh->message_type, mh->stream_id, c0c3, sizeof(c0c3)); } else { nbh = srs_chunk_header_c3( - mh->perfer_cid, mh->timestamp, + mh->perfer_cid, (u_int32_t)mh->timestamp, c0c3, sizeof(c0c3)); } srs_assert(nbh > 0);; @@ -668,7 +668,7 @@ int SrsProtocol::do_simple_send(SrsMessageHeader* mh, char* payload, int size) iovs[0].iov_base = c0c3; iovs[0].iov_len = nbh; - int payload_size = srs_min(end - p, out_chunk_size); + int payload_size = srs_min((int)(end - p), out_chunk_size); iovs[1].iov_base = p; iovs[1].iov_len = payload_size; p += payload_size; From 3c64e4b957aafb48ee9fb1c75661f3c82947f2c9 Mon Sep 17 00:00:00 2001 From: winlin Date: Fri, 16 Oct 2015 17:18:16 +0800 Subject: [PATCH 21/46] kafka encode and send packet. --- trunk/src/kernel/srs_kernel_error.hpp | 6 +- trunk/src/protocol/srs_kafka_stack.cpp | 306 ++++++++++++++++++++++--- trunk/src/protocol/srs_kafka_stack.hpp | 62 ++++- 3 files changed, 335 insertions(+), 39 deletions(-) diff --git a/trunk/src/kernel/srs_kernel_error.hpp b/trunk/src/kernel/srs_kernel_error.hpp index 5e1fed433..bddb811a4 100755 --- a/trunk/src/kernel/srs_kernel_error.hpp +++ b/trunk/src/kernel/srs_kernel_error.hpp @@ -236,7 +236,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #define ERROR_REQUEST_DATA 3066 /////////////////////////////////////////////////////// -// HTTP/StreamCaster protocol error. +// HTTP/StreamCaster/KAFKA protocol error. /////////////////////////////////////////////////////// #define ERROR_HTTP_PATTERN_EMPTY 4000 #define ERROR_HTTP_PATTERN_DUPLICATED 4001 @@ -268,6 +268,10 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #define ERROR_AVC_NALU_UEV 4027 #define ERROR_AAC_BYTES_INVALID 4028 #define ERROR_HTTP_REQUEST_EOF 4029 +#define ERROR_KAFKA_CODEC_STRING 4030 +#define ERROR_KAFKA_CODEC_BYTES 4031 +#define ERROR_KAFKA_CODEC_REQUEST 4032 +#define ERROR_KAFKA_CODEC_RESPONSE 4033 /////////////////////////////////////////////////////// // HTTP API error. diff --git a/trunk/src/protocol/srs_kafka_stack.cpp b/trunk/src/protocol/srs_kafka_stack.cpp index 67ee28907..44e2faed7 100644 --- a/trunk/src/protocol/srs_kafka_stack.cpp +++ b/trunk/src/protocol/srs_kafka_stack.cpp @@ -29,22 +29,23 @@ using namespace std; #include #include #include +#include #ifdef SRS_AUTO_KAFKA SrsKafkaString::SrsKafkaString() { - size = -1; + _size = -1; data = NULL; } SrsKafkaString::SrsKafkaString(string v) { - size = (int16_t)v.length(); + _size = (int16_t)v.length(); - srs_assert(size > 0); - data = new char[size]; - memcpy(data, v.data(), size); + srs_assert(_size > 0); + data = new char[_size]; + memcpy(data, v.data(), _size); } SrsKafkaString::~SrsKafkaString() @@ -54,32 +55,92 @@ SrsKafkaString::~SrsKafkaString() bool SrsKafkaString::null() { - return size == -1; + return _size == -1; } bool SrsKafkaString::empty() { - return size <= 0; + return _size <= 0; } -int SrsKafkaString::total_size() +int SrsKafkaString::size() { - return 2 + (size == -1? 0 : size); + return _size == -1? 2 : 2 + _size; +} + +int SrsKafkaString::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(2)) { + ret = ERROR_KAFKA_CODEC_STRING; + srs_error("kafka encode string failed. ret=%d", ret); + return ret; + } + buf->write_2bytes(_size); + + if (_size <= 0) { + return ret; + } + + if (!buf->require(_size)) { + ret = ERROR_KAFKA_CODEC_STRING; + srs_error("kafka encode string data failed. ret=%d", ret); + return ret; + } + buf->write_bytes(data, _size); + + return ret; +} + +int SrsKafkaString::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(2)) { + ret = ERROR_KAFKA_CODEC_STRING; + srs_error("kafka decode string failed. ret=%d", ret); + return ret; + } + _size = buf->read_2bytes(); + + if (_size != -1 && _size < 0) { + ret = ERROR_KAFKA_CODEC_STRING; + srs_error("kafka string must be -1 or >=0, actual is %d. ret=%d", _size, ret); + return ret; + } + + if (_size <= 0) { + return ret; + } + + if (!buf->require(_size)) { + ret = ERROR_KAFKA_CODEC_STRING; + srs_error("kafka decode string data failed. ret=%d", ret); + return ret; + } + + srs_freep(data); + data = new char[_size]; + + buf->read_bytes(data, _size); + + return ret; } SrsKafkaBytes::SrsKafkaBytes() { - size = -1; + _size = -1; data = NULL; } SrsKafkaBytes::SrsKafkaBytes(const char* v, int nb_v) { - size = (int16_t)nb_v; + _size = (int16_t)nb_v; - srs_assert(size > 0); - data = new char[size]; - memcpy(data, v, size); + srs_assert(_size > 0); + data = new char[_size]; + memcpy(data, v, _size); } SrsKafkaBytes::~SrsKafkaBytes() @@ -89,17 +150,72 @@ SrsKafkaBytes::~SrsKafkaBytes() bool SrsKafkaBytes::null() { - return size == -1; + return _size == -1; } bool SrsKafkaBytes::empty() { - return size <= 0; + return _size <= 0; } -int SrsKafkaBytes::total_size() +int SrsKafkaBytes::size() { - return 4 + (size == -1? 0 : size); + return 4 + (_size == -1? 0 : _size); +} + +int SrsKafkaBytes::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_BYTES; + srs_error("kafka encode bytes failed. ret=%d", ret); + return ret; + } + buf->write_4bytes(_size); + + if (_size <= 0) { + return ret; + } + + if (!buf->require(_size)) { + ret = ERROR_KAFKA_CODEC_BYTES; + srs_error("kafka encode bytes data failed. ret=%d", ret); + return ret; + } + buf->write_bytes(data, _size); + + return ret; +} + +int SrsKafkaBytes::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_BYTES; + srs_error("kafka decode bytes failed. ret=%d", ret); + return ret; + } + _size = buf->read_4bytes(); + + if (_size != -1 && _size < 0) { + ret = ERROR_KAFKA_CODEC_BYTES; + srs_error("kafka bytes must be -1 or >=0, actual is %d. ret=%d", _size, ret); + return ret; + } + + if (!buf->require(_size)) { + ret = ERROR_KAFKA_CODEC_BYTES; + srs_error("kafka decode bytes data failed. ret=%d", ret); + return ret; + } + + srs_freep(data); + data = new char[_size]; + buf->read_bytes(data, _size); + + return ret; } SrsKafkaRequestHeader::SrsKafkaRequestHeader() @@ -117,7 +233,7 @@ SrsKafkaRequestHeader::~SrsKafkaRequestHeader() int SrsKafkaRequestHeader::header_size() { - return 2 + 2 + 4 + client_id->total_size(); + return 2 + 2 + 4 + client_id->size(); } int SrsKafkaRequestHeader::message_size() @@ -178,14 +294,56 @@ int SrsKafkaRequestHeader::size() int SrsKafkaRequestHeader::encode(SrsBuffer* buf) { int ret = ERROR_SUCCESS; - // TODO: FIXME: implements it. + + if (!buf->require(4 + _size)) { + ret = ERROR_KAFKA_CODEC_REQUEST; + srs_error("kafka encode request failed. ret=%d", ret); + return ret; + } + + buf->write_4bytes(_size); + buf->write_2bytes(api_key); + buf->write_2bytes(api_version); + buf->write_4bytes(correlation_id); + + if ((ret = client_id->encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode request client_id failed. ret=%d", ret); + return ret; + } + return ret; } int SrsKafkaRequestHeader::decode(SrsBuffer* buf) { int ret = ERROR_SUCCESS; - // TODO: FIXME: implements it. + + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_REQUEST; + srs_error("kafka decode request size failed. ret=%d", ret); + return ret; + } + _size = buf->read_4bytes(); + + if (_size <= 0) { + srs_warn("kafka got empty request"); + return ret; + } + + if (!buf->require(_size)) { + ret = ERROR_KAFKA_CODEC_REQUEST; + srs_error("kafka decode request message failed. ret=%d", ret); + return ret; + } + api_key = buf->read_2bytes(); + api_version = buf->read_2bytes(); + correlation_id = buf->read_4bytes(); + + if ((ret = client_id->decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode request client_id failed. ret=%d", ret); + return ret; + } + return ret; } @@ -222,14 +380,42 @@ int SrsKafkaResponseHeader::size() int SrsKafkaResponseHeader::encode(SrsBuffer* buf) { int ret = ERROR_SUCCESS; - // TODO: FIXME: implements it. + + if (!buf->require(4 + _size)) { + ret = ERROR_KAFKA_CODEC_RESPONSE; + srs_error("kafka encode response failed. ret=%d", ret); + return ret; + } + + buf->write_4bytes(_size); + buf->write_4bytes(correlation_id); + return ret; } int SrsKafkaResponseHeader::decode(SrsBuffer* buf) { int ret = ERROR_SUCCESS; - // TODO: FIXME: implements it. + + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_RESPONSE; + srs_error("kafka decode response size failed. ret=%d", ret); + return ret; + } + _size = buf->read_4bytes(); + + if (_size <= 0) { + srs_warn("kafka got empty response"); + return ret; + } + + if (!buf->require(_size)) { + ret = ERROR_KAFKA_CODEC_RESPONSE; + srs_error("kafka decode response message failed. ret=%d", ret); + return ret; + } + correlation_id = buf->read_4bytes(); + return ret; } @@ -326,22 +512,40 @@ void SrsKafkaTopicMetadataRequest::add_topic(string topic) int SrsKafkaTopicMetadataRequest::size() { - int ret = ERROR_SUCCESS; - // TODO: FIXME: implements it. - return ret; + return SrsKafkaRequest::size() + topics.size(); } int SrsKafkaTopicMetadataRequest::encode(SrsBuffer* buf) { int ret = ERROR_SUCCESS; - // TODO: FIXME: implements it. + + if ((ret = SrsKafkaRequest::encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode metadata request failed. ret=%d", ret); + return ret; + } + + if ((ret = topics.encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode metadata topics failed. ret=%d", ret); + return ret; + } + return ret; } int SrsKafkaTopicMetadataRequest::decode(SrsBuffer* buf) { int ret = ERROR_SUCCESS; - // TODO: FIXME: implements it. + + if ((ret = SrsKafkaRequest::decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode metadata request failed. ret=%d", ret); + return ret; + } + + if ((ret = topics.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode metadata topics failed. ret=%d", ret); + return ret; + } + return ret; } @@ -355,14 +559,19 @@ SrsKafkaTopicMetadataResponse::~SrsKafkaTopicMetadataResponse() int SrsKafkaTopicMetadataResponse::size() { - int ret = ERROR_SUCCESS; // TODO: FIXME: implements it. - return ret; + return SrsKafkaResponse::size(); } int SrsKafkaTopicMetadataResponse::encode(SrsBuffer* buf) { int ret = ERROR_SUCCESS; + + if ((ret = SrsKafkaResponse::encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode metadata response failed. ret=%d", ret); + return ret; + } + // TODO: FIXME: implements it. return ret; } @@ -370,6 +579,12 @@ int SrsKafkaTopicMetadataResponse::encode(SrsBuffer* buf) int SrsKafkaTopicMetadataResponse::decode(SrsBuffer* buf) { int ret = ERROR_SUCCESS; + + if ((ret = SrsKafkaResponse::decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode metadata response failed. ret=%d", ret); + return ret; + } + // TODO: FIXME: implements it. return ret; } @@ -387,7 +602,36 @@ int SrsKafkaProtocol::send_and_free_message(SrsKafkaRequest* msg) { int ret = ERROR_SUCCESS; - // TODO: FIXME: implements it. + // TODO: FIXME: refine for performance issue. + SrsAutoFree(SrsKafkaRequest, msg); + + int size = msg->size(); + if (size <= 0) { + return ret; + } + + // TODO: FIXME: refine for performance issue. + char* bytes = new char[size]; + SrsAutoFree(char, bytes); + + // TODO: FIXME: refine for performance issue. + SrsBuffer* buf = new SrsBuffer(); + SrsAutoFree(SrsBuffer, buf); + + if ((ret = buf->initialize(bytes, size)) != ERROR_SUCCESS) { + srs_error("kafka create buffer failed. ret=%d", ret); + return ret; + } + + if ((ret = msg->encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode message failed. ret=%d", ret); + return ret; + } + + if ((ret = skt->write(bytes, size, NULL)) != ERROR_SUCCESS) { + srs_error("kafka send message failed. ret=%d", ret); + return ret; + } return ret; } diff --git a/trunk/src/protocol/srs_kafka_stack.hpp b/trunk/src/protocol/srs_kafka_stack.hpp index 871403aab..933d2cd51 100644 --- a/trunk/src/protocol/srs_kafka_stack.hpp +++ b/trunk/src/protocol/srs_kafka_stack.hpp @@ -33,6 +33,7 @@ #include #include +#include class ISrsProtocolReaderWriter; @@ -59,10 +60,10 @@ enum SrsKafkaApiKey * A length of -1 indicates null. string uses an int16 for its size, and bytes uses an int32. * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ProtocolPrimitiveTypes */ -class SrsKafkaString +class SrsKafkaString : public ISrsCodec { private: - int16_t size; + int16_t _size; char* data; public: SrsKafkaString(); @@ -71,7 +72,11 @@ public: public: virtual bool null(); virtual bool empty(); - virtual int total_size(); +// interface ISrsCodec +public: + virtual int size(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); }; /** @@ -79,10 +84,10 @@ public: * A length of -1 indicates null. string uses an int16 for its size, and bytes uses an int32. * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ProtocolPrimitiveTypes */ -class SrsKafkaBytes +class SrsKafkaBytes : public ISrsCodec { private: - int32_t size; + int32_t _size; char* data; public: SrsKafkaBytes(); @@ -91,7 +96,11 @@ public: public: virtual bool null(); virtual bool empty(); - virtual int total_size(); +// interface ISrsCodec +public: + virtual int size(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); }; /** @@ -107,7 +116,7 @@ public: * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Requests */ template -class SrsKafkaArray +class SrsKafkaArray : public ISrsCodec { private: int length; @@ -132,6 +141,45 @@ public: length++; elems.push_back(elem); } +// interface ISrsCodec +public: + virtual int size() + { + int s = 0; + + for (SrsIterator it = elems.begin(); it != elems.end(); ++it) { + T elem = *it; + s += elem->size(); + } + + return s; + } + virtual int encode(SrsBuffer* buf) + { + int ret = ERROR_SUCCESS; + + for (SrsIterator it = elems.begin(); it != elems.end(); ++it) { + T elem = *it; + if ((ret = elem->encode(buf)) != ERROR_SUCCESS) { + return ret; + } + } + + return ret; + } + virtual int decode(SrsBuffer* buf) + { + int ret = ERROR_SUCCESS; + + for (SrsIterator it = elems.begin(); it != elems.end(); ++it) { + T elem = *it; + if ((ret = elem->decode(buf)) != ERROR_SUCCESS) { + return ret; + } + } + + return ret; + } }; /** From 2e67eb878644cf4283aa43d6f1adfeea0922c3df Mon Sep 17 00:00:00 2001 From: winlin Date: Fri, 16 Oct 2015 17:25:43 +0800 Subject: [PATCH 22/46] refine comments for kafka messages. --- trunk/src/protocol/srs_kafka_stack.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/trunk/src/protocol/srs_kafka_stack.hpp b/trunk/src/protocol/srs_kafka_stack.hpp index 933d2cd51..f1ced607b 100644 --- a/trunk/src/protocol/srs_kafka_stack.hpp +++ b/trunk/src/protocol/srs_kafka_stack.hpp @@ -112,6 +112,7 @@ public: * Usage: * SrsKafkaArray body; * body.append(new SrsKafkaBytes()); + * @remark the typename T must be a ISrsCodec* * * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Requests */ From 6e5ed11a008b89b06140d48d3287606cf873ddc5 Mon Sep 17 00:00:00 2001 From: winlin Date: Mon, 19 Oct 2015 11:34:18 +0800 Subject: [PATCH 23/46] for kafka, support correlation id cache. --- trunk/src/protocol/srs_kafka_stack.cpp | 137 +++++++++++++++++++++---- trunk/src/protocol/srs_kafka_stack.hpp | 83 ++++++++++++++- 2 files changed, 196 insertions(+), 24 deletions(-) diff --git a/trunk/src/protocol/srs_kafka_stack.cpp b/trunk/src/protocol/srs_kafka_stack.cpp index 44e2faed7..622507d51 100644 --- a/trunk/src/protocol/srs_kafka_stack.cpp +++ b/trunk/src/protocol/srs_kafka_stack.cpp @@ -221,8 +221,8 @@ int SrsKafkaBytes::decode(SrsBuffer* buf) SrsKafkaRequestHeader::SrsKafkaRequestHeader() { _size = 0; - api_key = api_version = 0; - correlation_id = 0; + _api_key = api_version = 0; + _correlation_id = 0; client_id = new SrsKafkaString(); } @@ -246,49 +246,69 @@ int SrsKafkaRequestHeader::total_size() return 4 + _size; } +void SrsKafkaRequestHeader::set_total_size(int s) +{ + _size = s - 4; +} + +int32_t SrsKafkaRequestHeader::correlation_id() +{ + return _correlation_id; +} + +void SrsKafkaRequestHeader::set_correlation_id(int32_t cid) +{ + _correlation_id = cid; +} + +SrsKafkaApiKey SrsKafkaRequestHeader::api_key() +{ + return (SrsKafkaApiKey)_api_key; +} + +void SrsKafkaRequestHeader::set_api_key(SrsKafkaApiKey key) +{ + _api_key = (int16_t)key; +} + bool SrsKafkaRequestHeader::is_producer_request() { - return api_key == SrsKafkaApiKeyProduceRequest; + return _api_key == SrsKafkaApiKeyProduceRequest; } bool SrsKafkaRequestHeader::is_fetch_request() { - return api_key == SrsKafkaApiKeyFetchRequest; + return _api_key == SrsKafkaApiKeyFetchRequest; } bool SrsKafkaRequestHeader::is_offset_request() { - return api_key == SrsKafkaApiKeyOffsetRequest; + return _api_key == SrsKafkaApiKeyOffsetRequest; } bool SrsKafkaRequestHeader::is_metadata_request() { - return api_key == SrsKafkaApiKeyMetadataRequest; + return _api_key == SrsKafkaApiKeyMetadataRequest; } bool SrsKafkaRequestHeader::is_offset_commit_request() { - return api_key == SrsKafkaApiKeyOffsetCommitRequest; + return _api_key == SrsKafkaApiKeyOffsetCommitRequest; } bool SrsKafkaRequestHeader::is_offset_fetch_request() { - return api_key == SrsKafkaApiKeyOffsetFetchRequest; + return _api_key == SrsKafkaApiKeyOffsetFetchRequest; } bool SrsKafkaRequestHeader::is_consumer_metadata_request() { - return api_key == SrsKafkaApiKeyConsumerMetadataRequest; -} - -void SrsKafkaRequestHeader::set_api_key(SrsKafkaApiKey key) -{ - api_key = (int16_t)key; + return _api_key == SrsKafkaApiKeyConsumerMetadataRequest; } int SrsKafkaRequestHeader::size() { - return 4 + _size; + return 4 + header_size(); } int SrsKafkaRequestHeader::encode(SrsBuffer* buf) @@ -302,9 +322,9 @@ int SrsKafkaRequestHeader::encode(SrsBuffer* buf) } buf->write_4bytes(_size); - buf->write_2bytes(api_key); + buf->write_2bytes(_api_key); buf->write_2bytes(api_version); - buf->write_4bytes(correlation_id); + buf->write_4bytes(_correlation_id); if ((ret = client_id->encode(buf)) != ERROR_SUCCESS) { srs_error("kafka encode request client_id failed. ret=%d", ret); @@ -335,9 +355,9 @@ int SrsKafkaRequestHeader::decode(SrsBuffer* buf) srs_error("kafka decode request message failed. ret=%d", ret); return ret; } - api_key = buf->read_2bytes(); + _api_key = buf->read_2bytes(); api_version = buf->read_2bytes(); - correlation_id = buf->read_4bytes(); + _correlation_id = buf->read_4bytes(); if ((ret = client_id->decode(buf)) != ERROR_SUCCESS) { srs_error("kafka decode request client_id failed. ret=%d", ret); @@ -372,9 +392,14 @@ int SrsKafkaResponseHeader::total_size() return 4 + _size; } +void SrsKafkaResponseHeader::set_total_size(int s) +{ + _size = s - 4; +} + int SrsKafkaResponseHeader::size() { - return 4 + _size; + return 4 + header_size(); } int SrsKafkaResponseHeader::encode(SrsBuffer* buf) @@ -452,12 +477,28 @@ SrsKafkaMessageSet::~SrsKafkaMessageSet() SrsKafkaRequest::SrsKafkaRequest() { + header.set_correlation_id(SrsKafkaCorrelationPool::instance()->generate_correlation_id()); } SrsKafkaRequest::~SrsKafkaRequest() { } +void SrsKafkaRequest::update_header(int s) +{ + header.set_total_size(s); +} + +int32_t SrsKafkaRequest::correlation_id() +{ + return header.correlation_id(); +} + +SrsKafkaApiKey SrsKafkaRequest::api_key() +{ + return header.api_key(); +} + int SrsKafkaRequest::size() { return header.size(); @@ -481,6 +522,11 @@ SrsKafkaResponse::~SrsKafkaResponse() { } +void SrsKafkaResponse::update_header(int s) +{ + header.set_total_size(s); +} + int SrsKafkaResponse::size() { return header.size(); @@ -589,6 +635,50 @@ int SrsKafkaTopicMetadataResponse::decode(SrsBuffer* buf) return ret; } +SrsKafkaCorrelationPool* SrsKafkaCorrelationPool::_instance = new SrsKafkaCorrelationPool(); + +SrsKafkaCorrelationPool* SrsKafkaCorrelationPool::instance() +{ + return _instance; +} + +SrsKafkaCorrelationPool::SrsKafkaCorrelationPool() +{ +} + +SrsKafkaCorrelationPool::~SrsKafkaCorrelationPool() +{ + correlation_ids.clear(); +} + +int32_t SrsKafkaCorrelationPool::generate_correlation_id() +{ + static int32_t cid = 1; + return cid++; +} + +void SrsKafkaCorrelationPool::set(int32_t correlation_id, SrsKafkaApiKey request) +{ + correlation_ids[correlation_id] = request; +} + +void SrsKafkaCorrelationPool::unset(int32_t correlation_id) +{ + std::map::iterator it = correlation_ids.find(correlation_id); + if (it != correlation_ids.end()) { + correlation_ids.erase(it); + } +} + +SrsKafkaApiKey SrsKafkaCorrelationPool::get(int32_t correlation_id) +{ + if (correlation_ids.find(correlation_id) == correlation_ids.end()) { + return SrsKafkaApiKeyUnknown; + } + + return correlation_ids[correlation_id]; +} + SrsKafkaProtocol::SrsKafkaProtocol(ISrsProtocolReaderWriter* io) { skt = io; @@ -610,6 +700,13 @@ int SrsKafkaProtocol::send_and_free_message(SrsKafkaRequest* msg) return ret; } + // update the header of message. + msg->update_header(size); + + // cache the request correlation id to discovery response message. + SrsKafkaCorrelationPool* pool = SrsKafkaCorrelationPool::instance(); + pool->set(msg->correlation_id(), msg->api_key()); + // TODO: FIXME: refine for performance issue. char* bytes = new char[size]; SrsAutoFree(char, bytes); diff --git a/trunk/src/protocol/srs_kafka_stack.hpp b/trunk/src/protocol/srs_kafka_stack.hpp index f1ced607b..520c9bdea 100644 --- a/trunk/src/protocol/srs_kafka_stack.hpp +++ b/trunk/src/protocol/srs_kafka_stack.hpp @@ -31,6 +31,7 @@ #include #include +#include #include #include @@ -45,6 +46,8 @@ class ISrsProtocolReaderWriter; */ enum SrsKafkaApiKey { + SrsKafkaApiKeyUnknown = -1, + SrsKafkaApiKeyProduceRequest = 0, SrsKafkaApiKeyFetchRequest = 1, SrsKafkaApiKeyOffsetRequest = 2, @@ -203,7 +206,7 @@ private: * a metadata request, a produce request, a fetch request, etc). * @remark MetadataRequest | ProduceRequest | FetchRequest | OffsetRequest | OffsetCommitRequest | OffsetFetchRequest */ - int16_t api_key; + int16_t _api_key; /** * This is a numeric version number for this api. We version each API and * this version number allows the server to properly interpret the request @@ -216,7 +219,7 @@ private: * the response by the server, unmodified. It is useful for matching * request and response between the client and server. */ - int32_t correlation_id; + int32_t _correlation_id; /** * This is a user supplied identifier for the client application. * The user can use any identifier they like and it will be used @@ -252,6 +255,28 @@ private: * the total size of the request, includes the 4B size. */ virtual int total_size(); +public: + /** + * when got the whole message size, update the header. + * @param s the whole message, including the 4 bytes size size. + */ + virtual void set_total_size(int s); + /** + * get the correlation id for message. + */ + virtual int32_t correlation_id(); + /** + * set the correlation id for message. + */ + virtual void set_correlation_id(int32_t cid); + /** + * get the api key of header for message. + */ + virtual SrsKafkaApiKey api_key(); + /** + * set the api key of header for message. + */ + virtual void set_api_key(SrsKafkaApiKey key); public: /** * the api key enumeration. @@ -264,8 +289,6 @@ public: virtual bool is_offset_commit_request(); virtual bool is_offset_fetch_request(); virtual bool is_consumer_metadata_request(); - // set the api key. - virtual void set_api_key(SrsKafkaApiKey key); // interface ISrsCodec public: virtual int size(); @@ -321,6 +344,12 @@ private: * the total size of the request, includes the 4B size. */ virtual int total_size(); +public: + /** + * when got the whole message size, update the header. + * @param s the whole message, including the 4 bytes size size. + */ + virtual void set_total_size(int s); // interface ISrsCodec public: virtual int size(); @@ -403,6 +432,20 @@ protected: public: SrsKafkaRequest(); virtual ~SrsKafkaRequest(); +public: + /** + * update the size in header. + * @param s an int value specifies the size of message in header. + */ + virtual void update_header(int s); + /** + * get the correlation id of header for message. + */ + virtual int32_t correlation_id(); + /** + * get the api key of request. + */ + virtual SrsKafkaApiKey api_key(); // interface ISrsCodec public: virtual int size(); @@ -420,6 +463,12 @@ protected: public: SrsKafkaResponse(); virtual ~SrsKafkaResponse(); +public: + /** + * update the size in header. + * @param s an int value specifies the size of message in header. + */ + virtual void update_header(int s); // interface ISrsCodec public: virtual int size(); @@ -477,6 +526,32 @@ public: virtual int decode(SrsBuffer* buf); }; +/** + * the poll to discovery reponse. + * @param CorrelationId This is a user-supplied integer. It will be passed back + * in the response by the server, unmodified. It is useful for matching + * request and response between the client and server. + * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Requests + */ +class SrsKafkaCorrelationPool +{ +private: + static SrsKafkaCorrelationPool* _instance; +public: + static SrsKafkaCorrelationPool* instance(); +private: + std::map correlation_ids; +private: + SrsKafkaCorrelationPool(); +public: + virtual ~SrsKafkaCorrelationPool(); +public: + virtual int32_t generate_correlation_id(); + virtual void set(int32_t correlation_id, SrsKafkaApiKey request); + virtual void unset(int32_t correlation_id); + virtual SrsKafkaApiKey get(int32_t correlation_id); +}; + /** * the kafka protocol stack, use to send and recv kakfa messages. */ From c4862878d1ccd13f2756858a6a3b9cdecd4db26d Mon Sep 17 00:00:00 2001 From: winlin Date: Mon, 19 Oct 2015 11:50:31 +0800 Subject: [PATCH 24/46] kafka rename message set to raw message set. --- trunk/src/protocol/srs_kafka_stack.cpp | 4 ++-- trunk/src/protocol/srs_kafka_stack.hpp | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/trunk/src/protocol/srs_kafka_stack.cpp b/trunk/src/protocol/srs_kafka_stack.cpp index 622507d51..e935b8c3f 100644 --- a/trunk/src/protocol/srs_kafka_stack.cpp +++ b/trunk/src/protocol/srs_kafka_stack.cpp @@ -461,11 +461,11 @@ SrsKafkaRawMessage::~SrsKafkaRawMessage() srs_freep(value); } -SrsKafkaMessageSet::SrsKafkaMessageSet() +SrsKafkaRawMessageSet::SrsKafkaRawMessageSet() { } -SrsKafkaMessageSet::~SrsKafkaMessageSet() +SrsKafkaRawMessageSet::~SrsKafkaRawMessageSet() { vector::iterator it; for (it = messages.begin(); it != messages.end(); ++it) { diff --git a/trunk/src/protocol/srs_kafka_stack.hpp b/trunk/src/protocol/srs_kafka_stack.hpp index 520c9bdea..1ea3ae8c1 100644 --- a/trunk/src/protocol/srs_kafka_stack.hpp +++ b/trunk/src/protocol/srs_kafka_stack.hpp @@ -413,13 +413,13 @@ public: * a set of kafka message. * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets */ -class SrsKafkaMessageSet +class SrsKafkaRawMessageSet { private: std::vector messages; public: - SrsKafkaMessageSet(); - virtual ~SrsKafkaMessageSet(); + SrsKafkaRawMessageSet(); + virtual ~SrsKafkaRawMessageSet(); }; /** From 493d2822559befdb9f63fc9a79b265fcdfd8ab44 Mon Sep 17 00:00:00 2001 From: winlin Date: Mon, 19 Oct 2015 13:55:53 +0800 Subject: [PATCH 25/46] kafka refine array, to decode and create object. --- trunk/src/kernel/srs_kernel_error.hpp | 1 + trunk/src/protocol/srs_kafka_stack.cpp | 40 +++++++++++++++++++ trunk/src/protocol/srs_kafka_stack.hpp | 53 +++++++++++++++++++------- 3 files changed, 81 insertions(+), 13 deletions(-) diff --git a/trunk/src/kernel/srs_kernel_error.hpp b/trunk/src/kernel/srs_kernel_error.hpp index bddb811a4..354dcd3cd 100755 --- a/trunk/src/kernel/srs_kernel_error.hpp +++ b/trunk/src/kernel/srs_kernel_error.hpp @@ -272,6 +272,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #define ERROR_KAFKA_CODEC_BYTES 4031 #define ERROR_KAFKA_CODEC_REQUEST 4032 #define ERROR_KAFKA_CODEC_RESPONSE 4033 +#define ERROR_KAFKA_CODEC_ARRAY 4034 /////////////////////////////////////////////////////// // HTTP API error. diff --git a/trunk/src/protocol/srs_kafka_stack.cpp b/trunk/src/protocol/srs_kafka_stack.cpp index e935b8c3f..144737913 100644 --- a/trunk/src/protocol/srs_kafka_stack.cpp +++ b/trunk/src/protocol/srs_kafka_stack.cpp @@ -30,6 +30,7 @@ using namespace std; #include #include #include +#include #ifdef SRS_AUTO_KAFKA @@ -682,10 +683,12 @@ SrsKafkaApiKey SrsKafkaCorrelationPool::get(int32_t correlation_id) SrsKafkaProtocol::SrsKafkaProtocol(ISrsProtocolReaderWriter* io) { skt = io; + reader = new SrsFastStream(); } SrsKafkaProtocol::~SrsKafkaProtocol() { + srs_freep(reader); } int SrsKafkaProtocol::send_and_free_message(SrsKafkaRequest* msg) @@ -733,6 +736,36 @@ int SrsKafkaProtocol::send_and_free_message(SrsKafkaRequest* msg) return ret; } +int SrsKafkaProtocol::recv_message(SrsKafkaResponse** pmsg) +{ + *pmsg = NULL; + + int ret = ERROR_SUCCESS; + + SrsKafkaResponseHeader header; + while (reader->size() < header.size()) { + if ((ret = reader->grow(skt, header.size())) != ERROR_SUCCESS) { + srs_error("kafka recv message failed. ret=%d", ret); + return ret; + } + } + + SrsBuffer buffer; + if ((ret = buffer.initialize(reader->bytes(), reader->size())) != ERROR_SUCCESS) { + return ret; + } + + SrsBuffer* buf = &buffer; + if ((ret = header.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode response header failed. ret=%d", ret); + return ret; + } + + // TODO: FIXME: decode message. + + return ret; +} + SrsKafkaClient::SrsKafkaClient(ISrsProtocolReaderWriter* io) { protocol = new SrsKafkaProtocol(io); @@ -756,6 +789,13 @@ int SrsKafkaClient::fetch_metadata(string topic) return ret; } + SrsKafkaResponse* res = NULL; + if ((ret = protocol->recv_message(&res)) != ERROR_SUCCESS) { + srs_error("kafka recv response failed. ret=%d", ret); + return ret; + } + SrsAutoFree(SrsKafkaResponse, res); + // TODO: FIXME: implements it. return ret; diff --git a/trunk/src/protocol/srs_kafka_stack.hpp b/trunk/src/protocol/srs_kafka_stack.hpp index 1ea3ae8c1..986f03b31 100644 --- a/trunk/src/protocol/srs_kafka_stack.hpp +++ b/trunk/src/protocol/srs_kafka_stack.hpp @@ -35,7 +35,9 @@ #include #include +#include +class SrsFastStream; class ISrsProtocolReaderWriter; #ifdef SRS_AUTO_KAFKA @@ -113,9 +115,9 @@ public: * array of a structure foo as [foo]. * * Usage: - * SrsKafkaArray body; + * SrsKafkaArray body; * body.append(new SrsKafkaBytes()); - * @remark the typename T must be a ISrsCodec* + * @remark array elem is the T*, which must be ISrsCodec* * * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Requests */ @@ -123,9 +125,9 @@ template class SrsKafkaArray : public ISrsCodec { private: - int length; - std::vector elems; - typedef typename std::vector::iterator SrsIterator; + int32_t length; + std::vector elems; + typedef typename std::vector::iterator SrsIterator; public: SrsKafkaArray() { @@ -134,13 +136,13 @@ public: virtual ~SrsKafkaArray() { for (SrsIterator it = elems.begin(); it != elems.end(); ++it) { - T elem = *it; + T* elem = *it; srs_freep(elem); } elems.clear(); } public: - virtual void append(T elem) + virtual void append(T* elem) { length++; elems.push_back(elem); @@ -149,10 +151,10 @@ public: public: virtual int size() { - int s = 0; + int s = 4; for (SrsIterator it = elems.begin(); it != elems.end(); ++it) { - T elem = *it; + T* elem = *it; s += elem->size(); } @@ -162,9 +164,17 @@ public: { int ret = ERROR_SUCCESS; + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_ARRAY; + srs_error("kafka encode array failed. ret=%d", ret); + return ret; + } + buf->write_4bytes(length); + for (SrsIterator it = elems.begin(); it != elems.end(); ++it) { - T elem = *it; + T* elem = *it; if ((ret = elem->encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode array elem failed. ret=%d", ret); return ret; } } @@ -175,11 +185,22 @@ public: { int ret = ERROR_SUCCESS; - for (SrsIterator it = elems.begin(); it != elems.end(); ++it) { - T elem = *it; + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_ARRAY; + srs_error("kafka decode array failed. ret=%d", ret); + return ret; + } + length = buf->read_2bytes(); + + for (int i = 0; i < length; i++) { + T* elem = new T(); if ((ret = elem->decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode array elem failed. ret=%d", ret); + srs_freep(elem); return ret; } + + elems.push_back(elem); } return ret; @@ -493,7 +514,7 @@ public: class SrsKafkaTopicMetadataRequest : public SrsKafkaRequest { private: - SrsKafkaArray topics; + SrsKafkaArray topics; public: SrsKafkaTopicMetadataRequest(); virtual ~SrsKafkaTopicMetadataRequest(); @@ -559,6 +580,7 @@ class SrsKafkaProtocol { private: ISrsProtocolReaderWriter* skt; + SrsFastStream* reader; public: SrsKafkaProtocol(ISrsProtocolReaderWriter* io); virtual ~SrsKafkaProtocol(); @@ -568,6 +590,11 @@ public: * @param msg the msg to send. user must not free it again. */ virtual int send_and_free_message(SrsKafkaRequest* msg); + /** + * read the message from kafka server. + * @param pmsg output the received message. user must free it. + */ + virtual int recv_message(SrsKafkaResponse** pmsg); }; /** From a108fa8d018e6b101b0ff9ca8c44fd85a59eb65d Mon Sep 17 00:00:00 2001 From: winlin Date: Mon, 19 Oct 2015 14:54:15 +0800 Subject: [PATCH 26/46] kafka recv and decode message. --- trunk/src/protocol/srs_kafka_stack.cpp | 104 ++++++++++++++++++++----- trunk/src/protocol/srs_kafka_stack.hpp | 27 ++++++- 2 files changed, 107 insertions(+), 24 deletions(-) diff --git a/trunk/src/protocol/srs_kafka_stack.cpp b/trunk/src/protocol/srs_kafka_stack.cpp index 144737913..73ddec2cd 100644 --- a/trunk/src/protocol/srs_kafka_stack.cpp +++ b/trunk/src/protocol/srs_kafka_stack.cpp @@ -371,7 +371,7 @@ int SrsKafkaRequestHeader::decode(SrsBuffer* buf) SrsKafkaResponseHeader::SrsKafkaResponseHeader() { _size = 0; - correlation_id = 0; + _correlation_id = 0; } SrsKafkaResponseHeader::~SrsKafkaResponseHeader() @@ -398,6 +398,11 @@ void SrsKafkaResponseHeader::set_total_size(int s) _size = s - 4; } +int32_t SrsKafkaResponseHeader::correlation_id() +{ + return _correlation_id; +} + int SrsKafkaResponseHeader::size() { return 4 + header_size(); @@ -414,7 +419,7 @@ int SrsKafkaResponseHeader::encode(SrsBuffer* buf) } buf->write_4bytes(_size); - buf->write_4bytes(correlation_id); + buf->write_4bytes(_correlation_id); return ret; } @@ -440,7 +445,7 @@ int SrsKafkaResponseHeader::decode(SrsBuffer* buf) srs_error("kafka decode response message failed. ret=%d", ret); return ret; } - correlation_id = buf->read_4bytes(); + _correlation_id = buf->read_4bytes(); return ret; } @@ -658,17 +663,31 @@ int32_t SrsKafkaCorrelationPool::generate_correlation_id() return cid++; } -void SrsKafkaCorrelationPool::set(int32_t correlation_id, SrsKafkaApiKey request) +SrsKafkaApiKey SrsKafkaCorrelationPool::set(int32_t correlation_id, SrsKafkaApiKey request) { + SrsKafkaApiKey previous = SrsKafkaApiKeyUnknown; + + std::map::iterator it = correlation_ids.find(correlation_id); + if (it != correlation_ids.end()) { + previous = it->second; + } + correlation_ids[correlation_id] = request; + + return previous; } -void SrsKafkaCorrelationPool::unset(int32_t correlation_id) +SrsKafkaApiKey SrsKafkaCorrelationPool::unset(int32_t correlation_id) { std::map::iterator it = correlation_ids.find(correlation_id); + if (it != correlation_ids.end()) { + SrsKafkaApiKey key = it->second; correlation_ids.erase(it); + return key; } + + return SrsKafkaApiKeyUnknown; } SrsKafkaApiKey SrsKafkaCorrelationPool::get(int32_t correlation_id) @@ -742,27 +761,72 @@ int SrsKafkaProtocol::recv_message(SrsKafkaResponse** pmsg) int ret = ERROR_SUCCESS; - SrsKafkaResponseHeader header; - while (reader->size() < header.size()) { + while (true) { + SrsKafkaResponseHeader header; + + // ensure enough bytes for response header. if ((ret = reader->grow(skt, header.size())) != ERROR_SUCCESS) { srs_error("kafka recv message failed. ret=%d", ret); return ret; } + + // decode response header. + SrsBuffer buffer; + if ((ret = buffer.initialize(reader->bytes(), reader->size())) != ERROR_SUCCESS) { + return ret; + } + + SrsBuffer* buf = &buffer; + if ((ret = header.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode response header failed. ret=%d", ret); + return ret; + } + + // skip the used buffer for header. + buf->skip(-1 * buf->pos()); + + // fetch cached api key. + SrsKafkaCorrelationPool* pool = SrsKafkaCorrelationPool::instance(); + SrsKafkaApiKey key = pool->unset(header.correlation_id()); + srs_trace("kafka got %d bytes response, key=%d", header.total_size(), header.correlation_id()); + + // create message by cached api key. + SrsKafkaResponse* res = NULL; + switch (key) { + case SrsKafkaApiKeyMetadataRequest: + srs_info("kafka got metadata response"); + res = new SrsKafkaTopicMetadataResponse(); + break; + case SrsKafkaApiKeyUnknown: + default: + break; + } + + // ensure enough bytes to decode message. + if ((ret = reader->grow(skt, header.total_size())) != ERROR_SUCCESS) { + srs_freep(res); + srs_error("kafka recv message body failed. ret=%d", ret); + return ret; + } + + // dropped message, fetch next. + if (!res) { + reader->skip(header.total_size()); + srs_warn("kafka ignore unknown message, size=%d.", header.total_size()); + continue; + } + + // parse the whole message. + if ((ret = res->decode(buf)) != ERROR_SUCCESS) { + srs_freep(res); + srs_error("kafka decode message failed. ret=%d", ret); + return ret; + } + + *pmsg = res; + break; } - SrsBuffer buffer; - if ((ret = buffer.initialize(reader->bytes(), reader->size())) != ERROR_SUCCESS) { - return ret; - } - - SrsBuffer* buf = &buffer; - if ((ret = header.decode(buf)) != ERROR_SUCCESS) { - srs_error("kafka decode response header failed. ret=%d", ret); - return ret; - } - - // TODO: FIXME: decode message. - return ret; } diff --git a/trunk/src/protocol/srs_kafka_stack.hpp b/trunk/src/protocol/srs_kafka_stack.hpp index 986f03b31..36c5e7fe7 100644 --- a/trunk/src/protocol/srs_kafka_stack.hpp +++ b/trunk/src/protocol/srs_kafka_stack.hpp @@ -339,7 +339,7 @@ private: * the response by the server, unmodified. It is useful for matching * request and response between the client and server. */ - int32_t correlation_id; + int32_t _correlation_id; public: SrsKafkaResponseHeader(); virtual ~SrsKafkaResponseHeader(); @@ -361,8 +361,9 @@ private: * the size of message, the bytes left after the header. */ virtual int message_size(); +public: /** - * the total size of the request, includes the 4B size. + * the total size of the request, includes the 4B size and message body. */ virtual int total_size(); public: @@ -371,6 +372,10 @@ public: * @param s the whole message, including the 4 bytes size size. */ virtual void set_total_size(int s); + /** + * get the correlation id of response message. + */ + virtual int32_t correlation_id(); // interface ISrsCodec public: virtual int size(); @@ -567,9 +572,23 @@ private: public: virtual ~SrsKafkaCorrelationPool(); public: + /** + * generate a global correlation id. + */ virtual int32_t generate_correlation_id(); - virtual void set(int32_t correlation_id, SrsKafkaApiKey request); - virtual void unset(int32_t correlation_id); + /** + * set the correlation id to specified request key. + */ + virtual SrsKafkaApiKey set(int32_t correlation_id, SrsKafkaApiKey request); + /** + * unset the correlation id. + * @return the previous api key; unknown if not set. + */ + virtual SrsKafkaApiKey unset(int32_t correlation_id); + /** + * get the key by specified correlation id. + * @return the specified api key; unknown if no correlation id. + */ virtual SrsKafkaApiKey get(int32_t correlation_id); }; From 7106934c3aaad5dc75a26969a204491e9a66d6c2 Mon Sep 17 00:00:00 2001 From: winlin Date: Mon, 19 Oct 2015 15:39:59 +0800 Subject: [PATCH 27/46] kafka producer use async interface to request metadata. --- trunk/src/app/srs_app_kafka.cpp | 90 +++++++++++++++++++++++++++++---- trunk/src/app/srs_app_kafka.hpp | 16 +++++- 2 files changed, 94 insertions(+), 12 deletions(-) diff --git a/trunk/src/app/srs_app_kafka.cpp b/trunk/src/app/srs_app_kafka.cpp index 2bcc0f2c7..89da50e5f 100644 --- a/trunk/src/app/srs_app_kafka.cpp +++ b/trunk/src/app/srs_app_kafka.cpp @@ -37,10 +37,18 @@ using namespace std; #ifdef SRS_AUTO_KAFKA +#define SRS_KAKFA_CYCLE_INTERVAL_MS 3000 + SrsKafkaProducer::SrsKafkaProducer() { - lb = new SrsLbRoundRobin(); + meatadata_ok = false; + metadata_expired = st_cond_new(); + + lock = st_mutex_new(); + pthread = new SrsReusableThread("kafka", this, SRS_KAKFA_CYCLE_INTERVAL_MS * 1000); worker = new SrsAsyncCallWorker(); + + lb = new SrsLbRoundRobin(); transport = new SrsTcpClient(); kafka = new SrsKafkaClient(transport); } @@ -48,21 +56,21 @@ SrsKafkaProducer::SrsKafkaProducer() SrsKafkaProducer::~SrsKafkaProducer() { srs_freep(lb); - srs_freep(worker); srs_freep(kafka); srs_freep(transport); + + srs_freep(worker); + srs_freep(pthread); + + st_mutex_destroy(lock); + st_cond_destroy(metadata_expired); } int SrsKafkaProducer::initialize() { int ret = ERROR_SUCCESS; - // when kafka enabled, request metadata when startup. - if ((ret = request_metadata()) != ERROR_SUCCESS) { - srs_error("request kafka metadata failed. ret=%d", ret); - return ret; - } - + meatadata_ok = false; srs_info("initialize kafka producer ok."); return ret; @@ -73,20 +81,78 @@ int SrsKafkaProducer::start() int ret = ERROR_SUCCESS; if ((ret = worker->start()) != ERROR_SUCCESS) { - srs_error("start kafka failed. ret=%d", ret); + srs_error("start kafka worker failed. ret=%d", ret); return ret; } - srs_info("kafka worker ok"); + if ((ret = pthread->start()) != ERROR_SUCCESS) { + srs_error("start kafka thread failed. ret=%d", ret); + } + + meatadata_ok = false; + st_cond_signal(metadata_expired); + srs_trace("kafka work in background"); return ret; } void SrsKafkaProducer::stop() { + pthread->stop(); worker->stop(); } +int SrsKafkaProducer::cycle() +{ + int ret = ERROR_SUCCESS; + + if ((ret = do_cycle()) != ERROR_SUCCESS) { + srs_warn("ignore kafka error. ret=%d", ret); + } + + return ret; +} + +int SrsKafkaProducer::on_before_cycle() +{ + // wait for the metadata expired. + // when metadata is ok, wait for it expired. + if (meatadata_ok) { + st_cond_wait(metadata_expired); + } + + // request to lock to acquire the socket. + st_mutex_lock(lock); + + return ERROR_SUCCESS; +} + +int SrsKafkaProducer::on_end_cycle() +{ + st_mutex_unlock(lock); + + return ERROR_SUCCESS; +} + +int SrsKafkaProducer::do_cycle() +{ + int ret = ERROR_SUCCESS; + + // ignore when disabled. + bool enabled = _srs_config->get_kafka_enabled(); + if (!enabled) { + return ret; + } + + // when kafka enabled, request metadata when startup. + if ((ret = request_metadata()) != ERROR_SUCCESS) { + srs_error("request kafka metadata failed. ret=%d", ret); + return ret; + } + + return ret; +} + int SrsKafkaProducer::request_metadata() { int ret = ERROR_SUCCESS; @@ -130,9 +196,11 @@ int SrsKafkaProducer::request_metadata() std::string senabled = srs_bool2switch(enabled); std::string sbrokers = srs_join_vector_string(brokers->args, ","); srs_trace("kafka ok, enabled:%s, brokers:%s, current:[%d]%s:%d, topic:%s", - senabled.c_str(), sbrokers.c_str(), lb->current(), server.c_str(), port, topic.c_str()); + senabled.c_str(), sbrokers.c_str(), lb->current(), server.c_str(), port, topic.c_str()); } + meatadata_ok = true; + return ret; } diff --git a/trunk/src/app/srs_app_kafka.hpp b/trunk/src/app/srs_app_kafka.hpp index 81251317f..e3ec46453 100644 --- a/trunk/src/app/srs_app_kafka.hpp +++ b/trunk/src/app/srs_app_kafka.hpp @@ -34,13 +34,21 @@ class SrsAsyncCallWorker; class SrsTcpClient; class SrsKafkaClient; +#include + #ifdef SRS_AUTO_KAFKA /** * the kafka producer used to save log to kafka cluster. */ -class SrsKafkaProducer +class SrsKafkaProducer : public ISrsReusableThreadHandler { +private: + st_mutex_t lock; + SrsReusableThread* pthread; +private: + bool meatadata_ok; + st_cond_t metadata_expired; private: SrsLbRoundRobin* lb; SrsAsyncCallWorker* worker; @@ -53,7 +61,13 @@ public: virtual int initialize(); virtual int start(); virtual void stop(); +// interface ISrsReusableThreadHandler +public: + virtual int cycle(); + virtual int on_before_cycle(); + virtual int on_end_cycle(); private: + virtual int do_cycle(); virtual int request_metadata(); }; From 84b3981d47d300726890a7c63a4317dad4bcae26 Mon Sep 17 00:00:00 2001 From: winlin Date: Thu, 22 Oct 2015 11:47:38 +0800 Subject: [PATCH 28/46] refs #1670: support decode the metadata response. --- trunk/src/kernel/srs_kernel_error.hpp | 1 + trunk/src/protocol/srs_kafka_stack.cpp | 219 ++++++++++++++++++++++++- trunk/src/protocol/srs_kafka_stack.hpp | 129 ++++++++++++++- 3 files changed, 344 insertions(+), 5 deletions(-) diff --git a/trunk/src/kernel/srs_kernel_error.hpp b/trunk/src/kernel/srs_kernel_error.hpp index 354dcd3cd..eb5149b06 100755 --- a/trunk/src/kernel/srs_kernel_error.hpp +++ b/trunk/src/kernel/srs_kernel_error.hpp @@ -273,6 +273,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #define ERROR_KAFKA_CODEC_REQUEST 4032 #define ERROR_KAFKA_CODEC_RESPONSE 4033 #define ERROR_KAFKA_CODEC_ARRAY 4034 +#define ERROR_KAFKA_CODEC_METADATA 4035 /////////////////////////////////////////////////////// // HTTP API error. diff --git a/trunk/src/protocol/srs_kafka_stack.cpp b/trunk/src/protocol/srs_kafka_stack.cpp index 73ddec2cd..0ae7384e9 100644 --- a/trunk/src/protocol/srs_kafka_stack.cpp +++ b/trunk/src/protocol/srs_kafka_stack.cpp @@ -601,6 +601,200 @@ int SrsKafkaTopicMetadataRequest::decode(SrsBuffer* buf) return ret; } +SrsKafkaBroker::SrsKafkaBroker() +{ + node_id = port = 0; +} + +SrsKafkaBroker::~SrsKafkaBroker() +{ +} + +int SrsKafkaBroker::size() +{ + return 4 + host.size() + 4; +} + +int SrsKafkaBroker::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_METADATA; + srs_error("kafka encode broker node_id failed. ret=%d", ret); + return ret; + } + buf->write_4bytes(node_id); + + if ((ret = host.encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode broker host failed. ret=%d", ret); + return ret; + } + + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_METADATA; + srs_error("kafka encode broker port failed. ret=%d", ret); + return ret; + } + buf->write_4bytes(port); + + return ret; +} + +int SrsKafkaBroker::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_METADATA; + srs_error("kafka decode broker node_id failed. ret=%d", ret); + return ret; + } + node_id = buf->read_4bytes(); + + if ((ret = host.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode broker host failed. ret=%d", ret); + return ret; + } + + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_METADATA; + srs_error("kafka decode broker port failed. ret=%d", ret); + return ret; + } + port = buf->read_4bytes(); + + return ret; +} + +SrsKafkaPartitionMetadata::SrsKafkaPartitionMetadata() +{ + error_code = 0; + partition_id = 0; + leader = 0; +} + +SrsKafkaPartitionMetadata::~SrsKafkaPartitionMetadata() +{ +} + +int SrsKafkaPartitionMetadata::size() +{ + return 2 + 4 + 4 + replicas.size() + isr.size(); +} + +int SrsKafkaPartitionMetadata::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(2 + 4 + 4)) { + ret = ERROR_KAFKA_CODEC_METADATA; + srs_error("kafka encode partition metadata failed. ret=%d", ret); + return ret; + } + buf->write_2bytes(error_code); + buf->write_4bytes(partition_id); + buf->write_4bytes(leader); + + if ((ret = replicas.encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode partition metadata replicas failed. ret=%d", ret); + return ret; + } + if ((ret = isr.encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode partition metadata isr failed. ret=%d", ret); + return ret; + } + + return ret; +} + +int SrsKafkaPartitionMetadata::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(2 + 4 + 4)) { + ret = ERROR_KAFKA_CODEC_METADATA; + srs_error("kafka decode partition metadata failed. ret=%d", ret); + return ret; + } + error_code = buf->read_2bytes(); + partition_id = buf->read_4bytes(); + leader = buf->read_4bytes(); + + if ((ret = replicas.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode partition metadata replicas failed. ret=%d", ret); + return ret; + } + if ((ret = isr.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode partition metadata isr failed. ret=%d", ret); + return ret; + } + + return ret; +} + +SrsKafkaTopicMetadata::SrsKafkaTopicMetadata() +{ + error_code = 0; +} + +SrsKafkaTopicMetadata::~SrsKafkaTopicMetadata() +{ +} + +int SrsKafkaTopicMetadata::size() +{ + return 2 + name.size() + metadatas.size(); +} + +int SrsKafkaTopicMetadata::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(2)) { + ret = ERROR_KAFKA_CODEC_METADATA; + srs_error("kafka encode topic metadata failed. ret=%d", ret); + return ret; + } + buf->write_2bytes(error_code); + + if ((ret = name.encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode topic name failed. ret=%d", ret); + return ret; + } + + if ((ret = metadatas.encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode topic metadatas failed. ret=%d", ret); + return ret; + } + + return ret; +} + +int SrsKafkaTopicMetadata::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(2)) { + ret = ERROR_KAFKA_CODEC_METADATA; + srs_error("kafka decode topic metadata failed. ret=%d", ret); + return ret; + } + error_code = buf->read_2bytes(); + + if ((ret = name.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode topic name failed. ret=%d", ret); + return ret; + } + + if ((ret = metadatas.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode topic metadatas failed. ret=%d", ret); + return ret; + } + + return ret; +} + SrsKafkaTopicMetadataResponse::SrsKafkaTopicMetadataResponse() { } @@ -611,8 +805,7 @@ SrsKafkaTopicMetadataResponse::~SrsKafkaTopicMetadataResponse() int SrsKafkaTopicMetadataResponse::size() { - // TODO: FIXME: implements it. - return SrsKafkaResponse::size(); + return SrsKafkaResponse::size() + brokers.size() + metadatas.size(); } int SrsKafkaTopicMetadataResponse::encode(SrsBuffer* buf) @@ -624,7 +817,16 @@ int SrsKafkaTopicMetadataResponse::encode(SrsBuffer* buf) return ret; } - // TODO: FIXME: implements it. + if ((ret = brokers.encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode metadata brokers failed. ret=%d", ret); + return ret; + } + + if ((ret = metadatas.encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode metadatas failed. ret=%d", ret); + return ret; + } + return ret; } @@ -637,7 +839,16 @@ int SrsKafkaTopicMetadataResponse::decode(SrsBuffer* buf) return ret; } - // TODO: FIXME: implements it. + if ((ret = brokers.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode metadata brokers failed. ret=%d", ret); + return ret; + } + + if ((ret = metadatas.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode metadatas failed. ret=%d", ret); + return ret; + } + return ret; } diff --git a/trunk/src/protocol/srs_kafka_stack.hpp b/trunk/src/protocol/srs_kafka_stack.hpp index 36c5e7fe7..714f8220d 100644 --- a/trunk/src/protocol/srs_kafka_stack.hpp +++ b/trunk/src/protocol/srs_kafka_stack.hpp @@ -190,7 +190,7 @@ public: srs_error("kafka decode array failed. ret=%d", ret); return ret; } - length = buf->read_2bytes(); + length = buf->read_4bytes(); for (int i = 0; i < length; i++) { T* elem = new T(); @@ -206,6 +206,78 @@ public: return ret; } }; +template<> +class SrsKafkaArray : public ISrsCodec +{ +private: + int32_t length; + std::vector elems; + typedef std::vector::iterator SrsIterator; +public: + SrsKafkaArray() + { + length = 0; + } + virtual ~SrsKafkaArray() + { + elems.clear(); + } +public: + virtual void append(int32_t elem) + { + length++; + elems.push_back(elem); + } + // interface ISrsCodec +public: + virtual int size() + { + return 4 + sizeof(int32_t) * (int)elems.size(); + } + virtual int encode(SrsBuffer* buf) + { + int ret = ERROR_SUCCESS; + + if (!buf->require(4 + sizeof(int32_t) * (int)elems.size())) { + ret = ERROR_KAFKA_CODEC_ARRAY; + srs_error("kafka encode array failed. ret=%d", ret); + return ret; + } + buf->write_4bytes(length); + + for (SrsIterator it = elems.begin(); it != elems.end(); ++it) { + int32_t elem = *it; + buf->write_4bytes(elem); + } + + return ret; + } + virtual int decode(SrsBuffer* buf) + { + int ret = ERROR_SUCCESS; + + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_ARRAY; + srs_error("kafka decode array failed. ret=%d", ret); + return ret; + } + length = buf->read_4bytes(); + + for (int i = 0; i < length; i++) { + if (!buf->require(sizeof(int32_t))) { + ret = ERROR_KAFKA_CODEC_ARRAY; + srs_error("kafka decode array elem failed. ret=%d", ret); + return ret; + + } + + int32_t elem = buf->read_4bytes(); + elems.push_back(elem); + } + + return ret; + } +}; /** * the header of request, includes the size of request. @@ -532,6 +604,58 @@ public: virtual int decode(SrsBuffer* buf); }; +/** + * the metadata response data. + * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-MetadataResponse + */ +struct SrsKafkaBroker : public ISrsCodec +{ +public: + int32_t node_id; + SrsKafkaString host; + int32_t port; +public: + SrsKafkaBroker(); + virtual ~SrsKafkaBroker(); +// interface ISrsCodec +public: + virtual int size(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); +}; +struct SrsKafkaPartitionMetadata : public ISrsCodec +{ +public: + int16_t error_code; + int32_t partition_id; + int32_t leader; + SrsKafkaArray replicas; + SrsKafkaArray isr; +public: + SrsKafkaPartitionMetadata(); + virtual ~SrsKafkaPartitionMetadata(); +// interface ISrsCodec +public: + virtual int size(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); +}; +struct SrsKafkaTopicMetadata : public ISrsCodec +{ +public: + int16_t error_code; + SrsKafkaString name; + SrsKafkaArray metadatas; +public: + SrsKafkaTopicMetadata(); + virtual ~SrsKafkaTopicMetadata(); +// interface ISrsCodec +public: + virtual int size(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); +}; + /** * response for the metadata request from broker. * The response contains metadata for each partition, @@ -542,6 +666,9 @@ public: */ class SrsKafkaTopicMetadataResponse : public SrsKafkaResponse { +private: + SrsKafkaArray brokers; + SrsKafkaArray metadatas; public: SrsKafkaTopicMetadataResponse(); virtual ~SrsKafkaTopicMetadataResponse(); From 33a015329d28c06740316e1353edb9da08fdbea2 Mon Sep 17 00:00:00 2001 From: winlin Date: Thu, 22 Oct 2015 12:15:30 +0800 Subject: [PATCH 29/46] add function to convert kafka array to vector --- trunk/src/app/srs_app_kafka.cpp | 22 ++++++++------- trunk/src/protocol/srs_kafka_stack.cpp | 18 ++++++++----- trunk/src/protocol/srs_kafka_stack.hpp | 37 ++++++++++++++++++++++++-- trunk/src/protocol/srs_rtmp_stack.hpp | 2 +- 4 files changed, 60 insertions(+), 19 deletions(-) diff --git a/trunk/src/app/srs_app_kafka.cpp b/trunk/src/app/srs_app_kafka.cpp index 89da50e5f..9dd1f3def 100644 --- a/trunk/src/app/srs_app_kafka.cpp +++ b/trunk/src/app/srs_app_kafka.cpp @@ -34,6 +34,7 @@ using namespace std; #include #include #include +#include #ifdef SRS_AUTO_KAFKA @@ -178,6 +179,14 @@ int SrsKafkaProducer::request_metadata() srs_parse_endpoint(broker, server, port); } + std::string topic = _srs_config->get_kafka_topic(); + if (true) { + std::string senabled = srs_bool2switch(enabled); + std::string sbrokers = srs_join_vector_string(brokers->args, ","); + srs_trace("kafka request enabled:%s, brokers:%s, current:[%d]%s:%d, topic:%s", + senabled.c_str(), sbrokers.c_str(), lb->current(), server.c_str(), port, topic.c_str()); + } + // connect to kafka server. if ((ret = transport->connect(server, port, SRS_CONSTS_KAFKA_TIMEOUT_US)) != ERROR_SUCCESS) { srs_error("kafka connect %s:%d failed. ret=%d", server.c_str(), port, ret); @@ -185,19 +194,12 @@ int SrsKafkaProducer::request_metadata() } // do fetch medata from broker. - std::string topic = _srs_config->get_kafka_topic(); - if ((ret = kafka->fetch_metadata(topic)) != ERROR_SUCCESS) { + SrsKafkaTopicMetadataResponse* metadata = NULL; + if ((ret = kafka->fetch_metadata(topic, &metadata)) != ERROR_SUCCESS) { srs_error("kafka fetch metadata failed. ret=%d", ret); return ret; } - - // log when completed. - if (true) { - std::string senabled = srs_bool2switch(enabled); - std::string sbrokers = srs_join_vector_string(brokers->args, ","); - srs_trace("kafka ok, enabled:%s, brokers:%s, current:[%d]%s:%d, topic:%s", - senabled.c_str(), sbrokers.c_str(), lb->current(), server.c_str(), port, topic.c_str()); - } + SrsAutoFree(SrsKafkaTopicMetadataResponse, metadata); meatadata_ok = true; diff --git a/trunk/src/protocol/srs_kafka_stack.cpp b/trunk/src/protocol/srs_kafka_stack.cpp index 0ae7384e9..87dabe6a3 100644 --- a/trunk/src/protocol/srs_kafka_stack.cpp +++ b/trunk/src/protocol/srs_kafka_stack.cpp @@ -1051,8 +1051,10 @@ SrsKafkaClient::~SrsKafkaClient() srs_freep(protocol); } -int SrsKafkaClient::fetch_metadata(string topic) +int SrsKafkaClient::fetch_metadata(string topic, SrsKafkaTopicMetadataResponse** pmsg) { + *pmsg = NULL; + int ret = ERROR_SUCCESS; SrsKafkaTopicMetadataRequest* req = new SrsKafkaTopicMetadataRequest(); @@ -1064,17 +1066,21 @@ int SrsKafkaClient::fetch_metadata(string topic) return ret; } - SrsKafkaResponse* res = NULL; - if ((ret = protocol->recv_message(&res)) != ERROR_SUCCESS) { + if ((ret = protocol->expect_message(pmsg)) != ERROR_SUCCESS) { srs_error("kafka recv response failed. ret=%d", ret); return ret; } - SrsAutoFree(SrsKafkaResponse, res); - - // TODO: FIXME: implements it. return ret; } +vector srs_kafka_array2vector(SrsKafkaArray* arr) +{ + vector strs; + for (int i = 0; i < arr->size(); i++) { + } + return strs; +} + #endif diff --git a/trunk/src/protocol/srs_kafka_stack.hpp b/trunk/src/protocol/srs_kafka_stack.hpp index 714f8220d..6e253e744 100644 --- a/trunk/src/protocol/srs_kafka_stack.hpp +++ b/trunk/src/protocol/srs_kafka_stack.hpp @@ -666,7 +666,7 @@ public: */ class SrsKafkaTopicMetadataResponse : public SrsKafkaResponse { -private: +public: SrsKafkaArray brokers; SrsKafkaArray metadatas; public: @@ -741,6 +741,36 @@ public: * @param pmsg output the received message. user must free it. */ virtual int recv_message(SrsKafkaResponse** pmsg); +public: + /** + * expect specified message. + */ + template + int expect_message(T** pmsg) + { + int ret = ERROR_SUCCESS; + + while (true) { + SrsKafkaResponse* res = NULL; + if ((ret = recv_message(&res)) != ERROR_SUCCESS) { + srs_error("recv response failed. ret=%d", ret); + return ret; + } + + // drop not matched. + T* msg = dynamic_cast(res); + if (!msg) { + srs_info("kafka drop response."); + srs_freep(res); + continue; + } + + *pmsg = msg; + break; + } + + return ret; + } }; /** @@ -757,9 +787,12 @@ public: /** * fetch the metadata from broker for topic. */ - virtual int fetch_metadata(std::string topic); + virtual int fetch_metadata(std::string topic, SrsKafkaTopicMetadataResponse** pmsg); }; +// convert kafka array[string] to vector[string] +extern std::vector srs_kafka_array2vector(SrsKafkaArray* arr); + #endif #endif diff --git a/trunk/src/protocol/srs_rtmp_stack.hpp b/trunk/src/protocol/srs_rtmp_stack.hpp index e895fc3f3..12dbb574d 100644 --- a/trunk/src/protocol/srs_rtmp_stack.hpp +++ b/trunk/src/protocol/srs_rtmp_stack.hpp @@ -997,7 +997,7 @@ public: * for example: * SrsCommonMessage* msg = NULL; * SrsConnectAppResPacket* pkt = NULL; - * if ((ret = server->expect_message(protocol, &msg, &pkt)) != ERROR_SUCCESS) { + * if ((ret = server->expect_message(&msg, &pkt)) != ERROR_SUCCESS) { * return ret; * } * // use then free msg and pkt From d0133748718444ebd8d95513f7649f300a60d985 Mon Sep 17 00:00:00 2001 From: winlin Date: Thu, 22 Oct 2015 12:20:31 +0800 Subject: [PATCH 30/46] rename ISrsCodec size to nb_bytes. --- trunk/src/kernel/srs_kernel_buffer.hpp | 4 +-- trunk/src/protocol/srs_kafka_stack.cpp | 44 +++++++++++++------------- trunk/src/protocol/srs_kafka_stack.hpp | 30 +++++++++--------- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/trunk/src/kernel/srs_kernel_buffer.hpp b/trunk/src/kernel/srs_kernel_buffer.hpp index f79e08f33..6b85aa537 100644 --- a/trunk/src/kernel/srs_kernel_buffer.hpp +++ b/trunk/src/kernel/srs_kernel_buffer.hpp @@ -67,9 +67,9 @@ public: virtual ~ISrsCodec(); public: /** - * get the size of object to encode object to bytes. + * get the number of bytes to code to. */ - virtual int size() = 0; + virtual int nb_bytes() = 0; /** * encode object to bytes in SrsBuffer. */ diff --git a/trunk/src/protocol/srs_kafka_stack.cpp b/trunk/src/protocol/srs_kafka_stack.cpp index 87dabe6a3..9b966a2dc 100644 --- a/trunk/src/protocol/srs_kafka_stack.cpp +++ b/trunk/src/protocol/srs_kafka_stack.cpp @@ -64,7 +64,7 @@ bool SrsKafkaString::empty() return _size <= 0; } -int SrsKafkaString::size() +int SrsKafkaString::nb_bytes() { return _size == -1? 2 : 2 + _size; } @@ -159,7 +159,7 @@ bool SrsKafkaBytes::empty() return _size <= 0; } -int SrsKafkaBytes::size() +int SrsKafkaBytes::nb_bytes() { return 4 + (_size == -1? 0 : _size); } @@ -234,7 +234,7 @@ SrsKafkaRequestHeader::~SrsKafkaRequestHeader() int SrsKafkaRequestHeader::header_size() { - return 2 + 2 + 4 + client_id->size(); + return 2 + 2 + 4 + client_id->nb_bytes(); } int SrsKafkaRequestHeader::message_size() @@ -307,7 +307,7 @@ bool SrsKafkaRequestHeader::is_consumer_metadata_request() return _api_key == SrsKafkaApiKeyConsumerMetadataRequest; } -int SrsKafkaRequestHeader::size() +int SrsKafkaRequestHeader::nb_bytes() { return 4 + header_size(); } @@ -403,7 +403,7 @@ int32_t SrsKafkaResponseHeader::correlation_id() return _correlation_id; } -int SrsKafkaResponseHeader::size() +int SrsKafkaResponseHeader::nb_bytes() { return 4 + header_size(); } @@ -505,9 +505,9 @@ SrsKafkaApiKey SrsKafkaRequest::api_key() return header.api_key(); } -int SrsKafkaRequest::size() +int SrsKafkaRequest::nb_bytes() { - return header.size(); + return header.nb_bytes(); } int SrsKafkaRequest::encode(SrsBuffer* buf) @@ -533,9 +533,9 @@ void SrsKafkaResponse::update_header(int s) header.set_total_size(s); } -int SrsKafkaResponse::size() +int SrsKafkaResponse::nb_bytes() { - return header.size(); + return header.nb_bytes(); } int SrsKafkaResponse::encode(SrsBuffer* buf) @@ -562,9 +562,9 @@ void SrsKafkaTopicMetadataRequest::add_topic(string topic) topics.append(new SrsKafkaString(topic)); } -int SrsKafkaTopicMetadataRequest::size() +int SrsKafkaTopicMetadataRequest::nb_bytes() { - return SrsKafkaRequest::size() + topics.size(); + return SrsKafkaRequest::nb_bytes() + topics.nb_bytes(); } int SrsKafkaTopicMetadataRequest::encode(SrsBuffer* buf) @@ -610,9 +610,9 @@ SrsKafkaBroker::~SrsKafkaBroker() { } -int SrsKafkaBroker::size() +int SrsKafkaBroker::nb_bytes() { - return 4 + host.size() + 4; + return 4 + host.nb_bytes() + 4; } int SrsKafkaBroker::encode(SrsBuffer* buf) @@ -678,9 +678,9 @@ SrsKafkaPartitionMetadata::~SrsKafkaPartitionMetadata() { } -int SrsKafkaPartitionMetadata::size() +int SrsKafkaPartitionMetadata::nb_bytes() { - return 2 + 4 + 4 + replicas.size() + isr.size(); + return 2 + 4 + 4 + replicas.nb_bytes() + isr.nb_bytes(); } int SrsKafkaPartitionMetadata::encode(SrsBuffer* buf) @@ -742,9 +742,9 @@ SrsKafkaTopicMetadata::~SrsKafkaTopicMetadata() { } -int SrsKafkaTopicMetadata::size() +int SrsKafkaTopicMetadata::nb_bytes() { - return 2 + name.size() + metadatas.size(); + return 2 + name.nb_bytes() + metadatas.nb_bytes(); } int SrsKafkaTopicMetadata::encode(SrsBuffer* buf) @@ -803,9 +803,9 @@ SrsKafkaTopicMetadataResponse::~SrsKafkaTopicMetadataResponse() { } -int SrsKafkaTopicMetadataResponse::size() +int SrsKafkaTopicMetadataResponse::nb_bytes() { - return SrsKafkaResponse::size() + brokers.size() + metadatas.size(); + return SrsKafkaResponse::nb_bytes() + brokers.nb_bytes() + metadatas.nb_bytes(); } int SrsKafkaTopicMetadataResponse::encode(SrsBuffer* buf) @@ -928,7 +928,7 @@ int SrsKafkaProtocol::send_and_free_message(SrsKafkaRequest* msg) // TODO: FIXME: refine for performance issue. SrsAutoFree(SrsKafkaRequest, msg); - int size = msg->size(); + int size = msg->nb_bytes(); if (size <= 0) { return ret; } @@ -976,7 +976,7 @@ int SrsKafkaProtocol::recv_message(SrsKafkaResponse** pmsg) SrsKafkaResponseHeader header; // ensure enough bytes for response header. - if ((ret = reader->grow(skt, header.size())) != ERROR_SUCCESS) { + if ((ret = reader->grow(skt, header.nb_bytes())) != ERROR_SUCCESS) { srs_error("kafka recv message failed. ret=%d", ret); return ret; } @@ -1077,7 +1077,7 @@ int SrsKafkaClient::fetch_metadata(string topic, SrsKafkaTopicMetadataResponse** vector srs_kafka_array2vector(SrsKafkaArray* arr) { vector strs; - for (int i = 0; i < arr->size(); i++) { + for (int i = 0; i < arr->nb_bytes(); i++) { } return strs; } diff --git a/trunk/src/protocol/srs_kafka_stack.hpp b/trunk/src/protocol/srs_kafka_stack.hpp index 6e253e744..4e25f2f9e 100644 --- a/trunk/src/protocol/srs_kafka_stack.hpp +++ b/trunk/src/protocol/srs_kafka_stack.hpp @@ -79,7 +79,7 @@ public: virtual bool empty(); // interface ISrsCodec public: - virtual int size(); + virtual int nb_bytes(); virtual int encode(SrsBuffer* buf); virtual int decode(SrsBuffer* buf); }; @@ -103,7 +103,7 @@ public: virtual bool empty(); // interface ISrsCodec public: - virtual int size(); + virtual int nb_bytes(); virtual int encode(SrsBuffer* buf); virtual int decode(SrsBuffer* buf); }; @@ -149,13 +149,13 @@ public: } // interface ISrsCodec public: - virtual int size() + virtual int nb_bytes() { int s = 4; for (SrsIterator it = elems.begin(); it != elems.end(); ++it) { T* elem = *it; - s += elem->size(); + s += elem->nb_bytes(); } return s; @@ -230,9 +230,9 @@ public: } // interface ISrsCodec public: - virtual int size() + virtual int nb_bytes() { - return 4 + sizeof(int32_t) * (int)elems.size(); + return 4 + 4 * (int)elems.size(); } virtual int encode(SrsBuffer* buf) { @@ -384,7 +384,7 @@ public: virtual bool is_consumer_metadata_request(); // interface ISrsCodec public: - virtual int size(); + virtual int nb_bytes(); virtual int encode(SrsBuffer* buf); virtual int decode(SrsBuffer* buf); }; @@ -450,7 +450,7 @@ public: virtual int32_t correlation_id(); // interface ISrsCodec public: - virtual int size(); + virtual int nb_bytes(); virtual int encode(SrsBuffer* buf); virtual int decode(SrsBuffer* buf); }; @@ -546,7 +546,7 @@ public: virtual SrsKafkaApiKey api_key(); // interface ISrsCodec public: - virtual int size(); + virtual int nb_bytes(); virtual int encode(SrsBuffer* buf); virtual int decode(SrsBuffer* buf); }; @@ -569,7 +569,7 @@ public: virtual void update_header(int s); // interface ISrsCodec public: - virtual int size(); + virtual int nb_bytes(); virtual int encode(SrsBuffer* buf); virtual int decode(SrsBuffer* buf); }; @@ -599,7 +599,7 @@ public: virtual void add_topic(std::string topic); // interface ISrsCodec public: - virtual int size(); + virtual int nb_bytes(); virtual int encode(SrsBuffer* buf); virtual int decode(SrsBuffer* buf); }; @@ -619,7 +619,7 @@ public: virtual ~SrsKafkaBroker(); // interface ISrsCodec public: - virtual int size(); + virtual int nb_bytes(); virtual int encode(SrsBuffer* buf); virtual int decode(SrsBuffer* buf); }; @@ -636,7 +636,7 @@ public: virtual ~SrsKafkaPartitionMetadata(); // interface ISrsCodec public: - virtual int size(); + virtual int nb_bytes(); virtual int encode(SrsBuffer* buf); virtual int decode(SrsBuffer* buf); }; @@ -651,7 +651,7 @@ public: virtual ~SrsKafkaTopicMetadata(); // interface ISrsCodec public: - virtual int size(); + virtual int nb_bytes(); virtual int encode(SrsBuffer* buf); virtual int decode(SrsBuffer* buf); }; @@ -674,7 +674,7 @@ public: virtual ~SrsKafkaTopicMetadataResponse(); // interface ISrsCodec public: - virtual int size(); + virtual int nb_bytes(); virtual int encode(SrsBuffer* buf); virtual int decode(SrsBuffer* buf); }; From f0e39cc33047c3c2cb30e302680d2dcf29130e4c Mon Sep 17 00:00:00 2001 From: winlin Date: Thu, 22 Oct 2015 13:43:38 +0800 Subject: [PATCH 31/46] support show the summary of kafka metadata. --- trunk/src/app/srs_app_config.cpp | 4 +- trunk/src/app/srs_app_kafka.cpp | 43 +++++++++++++++++++++ trunk/src/app/srs_app_utility.cpp | 14 ------- trunk/src/app/srs_app_utility.hpp | 3 -- trunk/src/protocol/srs_kafka_stack.cpp | 31 ++++++++++++++- trunk/src/protocol/srs_kafka_stack.hpp | 18 +++++++++ trunk/src/protocol/srs_protocol_utility.cpp | 14 +++++++ trunk/src/protocol/srs_protocol_utility.hpp | 4 ++ 8 files changed, 111 insertions(+), 20 deletions(-) diff --git a/trunk/src/app/srs_app_config.cpp b/trunk/src/app/srs_app_config.cpp index 12e6525ee..477a3eca6 100644 --- a/trunk/src/app/srs_app_config.cpp +++ b/trunk/src/app/srs_app_config.cpp @@ -2128,6 +2128,8 @@ int SrsConfig::global_to_json(SrsJsonObject* obj) sobj->set(sdir->name, sdir->dumps_arg0_to_boolean()); } else if (sdir->name == "brokers") { sobj->set(sdir->name, sdir->dumps_args()); + } else if (sdir->name == "topic") { + sobj->set(sdir->name, sdir->dumps_arg0_to_str()); } } obj->set(dir->name, sobj); @@ -3546,7 +3548,7 @@ int SrsConfig::check_config() SrsConfDirective* conf = root->get("kafka"); for (int i = 0; conf && i < (int)conf->directives.size(); i++) { string n = conf->at(i)->name; - if (n != "enabled" && n != "brokers") { + if (n != "enabled" && n != "brokers" && n != "topic") { ret = ERROR_SYSTEM_CONFIG_INVALID; srs_error("unsupported kafka directive %s, ret=%d", n.c_str(), ret); return ret; diff --git a/trunk/src/app/srs_app_kafka.cpp b/trunk/src/app/srs_app_kafka.cpp index 9dd1f3def..3c7b01c99 100644 --- a/trunk/src/app/srs_app_kafka.cpp +++ b/trunk/src/app/srs_app_kafka.cpp @@ -32,6 +32,7 @@ using namespace std; #include #include #include +#include #include #include #include @@ -201,6 +202,48 @@ int SrsKafkaProducer::request_metadata() } SrsAutoFree(SrsKafkaTopicMetadataResponse, metadata); + // show kafka metadata. + string summary; + if (true) { + vector bs; + for (int i = 0; i < metadata->brokers.size(); i++) { + SrsKafkaBroker* broker = metadata->brokers.at(i); + + string hostport = srs_int2str(broker->node_id) + "/" + broker->host.to_str(); + if (broker->port > 0) { + hostport += ":" + srs_int2str(broker->port); + } + + bs.push_back(hostport); + } + + vector ps; + for (int i = 0; i < metadata->metadatas.size(); i++) { + SrsKafkaTopicMetadata* topic = metadata->metadatas.at(i); + + string desc = "topic=" + topic->name.to_str(); + + for (int j = 0; j < topic->metadatas.size(); j++) { + SrsKafkaPartitionMetadata* partition = topic->metadatas.at(j); + + desc += ", partition" + srs_int2str(partition->partition_id) +"="; + desc += srs_int2str(partition->leader) + "/"; + + vector replicas = srs_kafka_array2vector(&partition->replicas); + desc += srs_join_vector_string(replicas, ","); + } + + ps.push_back(desc); + } + + std::stringstream ss; + ss << "brokers=" << srs_join_vector_string(bs, ","); + ss << ", " << srs_join_vector_string(ps, ","); + + summary = ss.str(); + } + srs_trace("kafka metadata: %s", summary.c_str()); + meatadata_ok = true; return ret; diff --git a/trunk/src/app/srs_app_utility.cpp b/trunk/src/app/srs_app_utility.cpp index 490cff7ee..f978909dc 100644 --- a/trunk/src/app/srs_app_utility.cpp +++ b/trunk/src/app/srs_app_utility.cpp @@ -1467,17 +1467,3 @@ void srs_api_dump_summaries(SrsJsonObject* obj) sys->set("conn_srs", SrsJsonAny::integer(nrs->nb_conn_srs)); } -string srs_join_vector_string(vector& vs, string separator) -{ - string str = ""; - - for (int i = 0; i < (int)vs.size(); i++) { - str += vs.at(i); - if (i != (int)vs.size() - 1) { - str += separator; - } - } - - return str; -} - diff --git a/trunk/src/app/srs_app_utility.hpp b/trunk/src/app/srs_app_utility.hpp index 2ef07d330..971bdbe99 100644 --- a/trunk/src/app/srs_app_utility.hpp +++ b/trunk/src/app/srs_app_utility.hpp @@ -677,8 +677,5 @@ extern bool srs_is_boolean(const std::string& str); // dump summaries for /api/v1/summaries. extern void srs_api_dump_summaries(SrsJsonObject* obj); -// join string in vector with indicated separator -extern std::string srs_join_vector_string(std::vector& vs, std::string separator); - #endif diff --git a/trunk/src/protocol/srs_kafka_stack.cpp b/trunk/src/protocol/srs_kafka_stack.cpp index 9b966a2dc..ed66d76f5 100644 --- a/trunk/src/protocol/srs_kafka_stack.cpp +++ b/trunk/src/protocol/srs_kafka_stack.cpp @@ -23,7 +23,7 @@ #include -#include +#include using namespace std; #include @@ -31,6 +31,8 @@ using namespace std; #include #include #include +#include +#include #ifdef SRS_AUTO_KAFKA @@ -64,6 +66,15 @@ bool SrsKafkaString::empty() return _size <= 0; } +string SrsKafkaString::to_str() +{ + string ret; + if (_size > 0) { + ret.append(data, _size); + } + return ret; +} + int SrsKafkaString::nb_bytes() { return _size == -1? 2 : 2 + _size; @@ -1077,8 +1088,24 @@ int SrsKafkaClient::fetch_metadata(string topic, SrsKafkaTopicMetadataResponse** vector srs_kafka_array2vector(SrsKafkaArray* arr) { vector strs; - for (int i = 0; i < arr->nb_bytes(); i++) { + + for (int i = 0; i < arr->size(); i++) { + SrsKafkaString* elem = arr->at(i); + strs.push_back(elem->to_str()); } + + return strs; +} + +vector srs_kafka_array2vector(SrsKafkaArray* arr) +{ + vector strs; + + for (int i = 0; i < arr->size(); i++) { + int32_t elem = arr->at(i); + strs.push_back(srs_int2str(elem)); + } + return strs; } diff --git a/trunk/src/protocol/srs_kafka_stack.hpp b/trunk/src/protocol/srs_kafka_stack.hpp index 4e25f2f9e..f181d04dc 100644 --- a/trunk/src/protocol/srs_kafka_stack.hpp +++ b/trunk/src/protocol/srs_kafka_stack.hpp @@ -77,6 +77,7 @@ public: public: virtual bool null(); virtual bool empty(); + virtual std::string to_str(); // interface ISrsCodec public: virtual int nb_bytes(); @@ -147,6 +148,14 @@ public: length++; elems.push_back(elem); } + virtual int size() + { + return length; + } + virtual T* at(int index) + { + return elems.at(index); + } // interface ISrsCodec public: virtual int nb_bytes() @@ -228,6 +237,14 @@ public: length++; elems.push_back(elem); } + virtual int size() + { + return length; + } + virtual int32_t at(int index) + { + return elems.at(index); + } // interface ISrsCodec public: virtual int nb_bytes() @@ -792,6 +809,7 @@ public: // convert kafka array[string] to vector[string] extern std::vector srs_kafka_array2vector(SrsKafkaArray* arr); +extern std::vector srs_kafka_array2vector(SrsKafkaArray* arr); #endif diff --git a/trunk/src/protocol/srs_protocol_utility.cpp b/trunk/src/protocol/srs_protocol_utility.cpp index 09cfd6b5c..be36509b2 100644 --- a/trunk/src/protocol/srs_protocol_utility.cpp +++ b/trunk/src/protocol/srs_protocol_utility.cpp @@ -312,3 +312,17 @@ int srs_write_large_iovs(ISrsProtocolReaderWriter* skt, iovec* iovs, int size, s return ret; } +string srs_join_vector_string(vector& vs, string separator) +{ + string str = ""; + + for (int i = 0; i < (int)vs.size(); i++) { + str += vs.at(i); + if (i != (int)vs.size() - 1) { + str += separator; + } + } + + return str; +} + diff --git a/trunk/src/protocol/srs_protocol_utility.hpp b/trunk/src/protocol/srs_protocol_utility.hpp index d2ce19068..41893a34c 100644 --- a/trunk/src/protocol/srs_protocol_utility.hpp +++ b/trunk/src/protocol/srs_protocol_utility.hpp @@ -35,6 +35,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #endif #include +#include #include @@ -130,5 +131,8 @@ extern int srs_write_large_iovs( ssize_t* pnwrite = NULL ); +// join string in vector with indicated separator +extern std::string srs_join_vector_string(std::vector& vs, std::string separator); + #endif From 31a77a83edac6414a791c210e52098d46ad87caa Mon Sep 17 00:00:00 2001 From: winlin Date: Thu, 22 Oct 2015 14:22:10 +0800 Subject: [PATCH 32/46] convert metadata to partitions --- trunk/src/app/srs_app_kafka.cpp | 137 ++++++++++++++++++++++++-------- trunk/src/app/srs_app_kafka.hpp | 24 ++++++ 2 files changed, 128 insertions(+), 33 deletions(-) diff --git a/trunk/src/app/srs_app_kafka.cpp b/trunk/src/app/srs_app_kafka.cpp index 3c7b01c99..c59a1dce1 100644 --- a/trunk/src/app/srs_app_kafka.cpp +++ b/trunk/src/app/srs_app_kafka.cpp @@ -41,6 +41,83 @@ using namespace std; #define SRS_KAKFA_CYCLE_INTERVAL_MS 3000 +std::string srs_kafka_metadata_summary(SrsKafkaTopicMetadataResponse* metadata) +{ + vector bs; + for (int i = 0; i < metadata->brokers.size(); i++) { + SrsKafkaBroker* broker = metadata->brokers.at(i); + + string hostport = srs_int2str(broker->node_id) + "/" + broker->host.to_str(); + if (broker->port > 0) { + hostport += ":" + srs_int2str(broker->port); + } + + bs.push_back(hostport); + } + + vector ps; + for (int i = 0; i < metadata->metadatas.size(); i++) { + SrsKafkaTopicMetadata* topic = metadata->metadatas.at(i); + + for (int j = 0; j < topic->metadatas.size(); j++) { + string desc = "topic=" + topic->name.to_str(); + + SrsKafkaPartitionMetadata* partition = topic->metadatas.at(j); + + desc += "?partition=" + srs_int2str(partition->partition_id); + desc += "&leader=" + srs_int2str(partition->leader); + + vector replicas = srs_kafka_array2vector(&partition->replicas); + desc += "&replicas=" + srs_join_vector_string(replicas, ","); + + ps.push_back(desc); + } + } + + std::stringstream ss; + ss << "brokers=" << srs_join_vector_string(bs, ","); + ss << ", " << srs_join_vector_string(ps, ", "); + + return ss.str(); +} + +std::string srs_kafka_summary_partitions(const vector& partitions) +{ + vector ret; + + vector::const_iterator it; + for (it = partitions.begin(); it != partitions.end(); ++it) { + SrsKafkaPartition* partition = *it; + + string desc = "tcp://"; + desc += partition->host + ":" + srs_int2str(partition->port); + desc += "?broker=" + srs_int2str(partition->broker); + desc += "&partition=" + srs_int2str(partition->id); + ret.push_back(desc); + } + + return srs_join_vector_string(ret, ", "); +} + +SrsKafkaPartition::SrsKafkaPartition() +{ + id = broker = 0; + port = SRS_CONSTS_KAFKA_DEFAULT_PORT; +} + +SrsKafkaPartition::~SrsKafkaPartition() +{ +} + +string SrsKafkaPartition::hostport() +{ + if (ep.empty()) { + ep = host + ":" + srs_int2str(port); + } + + return ep; +} + SrsKafkaProducer::SrsKafkaProducer() { meatadata_ok = false; @@ -57,6 +134,13 @@ SrsKafkaProducer::SrsKafkaProducer() SrsKafkaProducer::~SrsKafkaProducer() { + vector::iterator it; + for (it = partitions.begin(); it != partitions.end(); ++it) { + SrsKafkaPartition* partition = *it; + srs_freep(partition); + } + partitions.clear(); + srs_freep(lb); srs_freep(kafka); srs_freep(transport); @@ -203,46 +287,33 @@ int SrsKafkaProducer::request_metadata() SrsAutoFree(SrsKafkaTopicMetadataResponse, metadata); // show kafka metadata. - string summary; - if (true) { - vector bs; - for (int i = 0; i < metadata->brokers.size(); i++) { - SrsKafkaBroker* broker = metadata->brokers.at(i); - - string hostport = srs_int2str(broker->node_id) + "/" + broker->host.to_str(); - if (broker->port > 0) { - hostport += ":" + srs_int2str(broker->port); - } - - bs.push_back(hostport); - } + string summary = srs_kafka_metadata_summary(metadata); + srs_trace("kafka metadata: %s", summary.c_str()); + + // generate the partition info. + for (int i = 0; i < metadata->metadatas.size(); i++) { + SrsKafkaTopicMetadata* topic = metadata->metadatas.at(i); - vector ps; - for (int i = 0; i < metadata->metadatas.size(); i++) { - SrsKafkaTopicMetadata* topic = metadata->metadatas.at(i); + for (int j = 0; j < topic->metadatas.size(); j++) { + SrsKafkaPartitionMetadata* partition = topic->metadatas.at(j); - string desc = "topic=" + topic->name.to_str(); + SrsKafkaPartition* p = new SrsKafkaPartition(); + p->id = partition->partition_id; + p->broker = partition->leader; - for (int j = 0; j < topic->metadatas.size(); j++) { - SrsKafkaPartitionMetadata* partition = topic->metadatas.at(j); - - desc += ", partition" + srs_int2str(partition->partition_id) +"="; - desc += srs_int2str(partition->leader) + "/"; - - vector replicas = srs_kafka_array2vector(&partition->replicas); - desc += srs_join_vector_string(replicas, ","); + for (int i = 0; i < metadata->brokers.size(); i++) { + SrsKafkaBroker* broker = metadata->brokers.at(i); + if (broker->node_id == p->broker) { + p->host = broker->host.to_str(); + p->port = broker->port; + break; + } } - ps.push_back(desc); + partitions.push_back(p); } - - std::stringstream ss; - ss << "brokers=" << srs_join_vector_string(bs, ","); - ss << ", " << srs_join_vector_string(ps, ","); - - summary = ss.str(); } - srs_trace("kafka metadata: %s", summary.c_str()); + srs_trace("kafka connector: %s", srs_kafka_summary_partitions(partitions).c_str()); meatadata_ok = true; diff --git a/trunk/src/app/srs_app_kafka.hpp b/trunk/src/app/srs_app_kafka.hpp index e3ec46453..dfd28f954 100644 --- a/trunk/src/app/srs_app_kafka.hpp +++ b/trunk/src/app/srs_app_kafka.hpp @@ -29,6 +29,8 @@ */ #include +#include + class SrsLbRoundRobin; class SrsAsyncCallWorker; class SrsTcpClient; @@ -38,6 +40,26 @@ class SrsKafkaClient; #ifdef SRS_AUTO_KAFKA +/** + * the kafka partition info. + */ +struct SrsKafkaPartition +{ +private: + std::string ep; +public: + int id; + // leader. + int broker; + std::string host; + int port; +public: + SrsKafkaPartition(); + virtual ~SrsKafkaPartition(); +public: + virtual std::string hostport(); +}; + /** * the kafka producer used to save log to kafka cluster. */ @@ -49,6 +71,8 @@ private: private: bool meatadata_ok; st_cond_t metadata_expired; +public: + std::vector partitions; private: SrsLbRoundRobin* lb; SrsAsyncCallWorker* worker; From de41c1c9d2b8600d8d7484ec19b456b63154d4a8 Mon Sep 17 00:00:00 2001 From: winlin Date: Thu, 22 Oct 2015 14:29:37 +0800 Subject: [PATCH 33/46] kafka refine comments. --- trunk/src/app/srs_app_kafka.cpp | 67 +++++++++++++++----------- trunk/src/app/srs_app_kafka.hpp | 4 +- trunk/src/protocol/srs_kafka_stack.cpp | 2 +- 3 files changed, 42 insertions(+), 31 deletions(-) diff --git a/trunk/src/app/srs_app_kafka.cpp b/trunk/src/app/srs_app_kafka.cpp index c59a1dce1..03f832429 100644 --- a/trunk/src/app/srs_app_kafka.cpp +++ b/trunk/src/app/srs_app_kafka.cpp @@ -99,6 +99,32 @@ std::string srs_kafka_summary_partitions(const vector& parti return srs_join_vector_string(ret, ", "); } +void srs_kafka_metadata2connector(SrsKafkaTopicMetadataResponse* metadata, vector& partitions) +{ + for (int i = 0; i < metadata->metadatas.size(); i++) { + SrsKafkaTopicMetadata* topic = metadata->metadatas.at(i); + + for (int j = 0; j < topic->metadatas.size(); j++) { + SrsKafkaPartitionMetadata* partition = topic->metadatas.at(j); + + SrsKafkaPartition* p = new SrsKafkaPartition(); + p->id = partition->partition_id; + p->broker = partition->leader; + + for (int i = 0; i < metadata->brokers.size(); i++) { + SrsKafkaBroker* broker = metadata->brokers.at(i); + if (broker->node_id == p->broker) { + p->host = broker->host.to_str(); + p->port = broker->port; + break; + } + } + + partitions.push_back(p); + } + } +} + SrsKafkaPartition::SrsKafkaPartition() { id = broker = 0; @@ -120,7 +146,7 @@ string SrsKafkaPartition::hostport() SrsKafkaProducer::SrsKafkaProducer() { - meatadata_ok = false; + metadata_ok = false; metadata_expired = st_cond_new(); lock = st_mutex_new(); @@ -156,7 +182,6 @@ int SrsKafkaProducer::initialize() { int ret = ERROR_SUCCESS; - meatadata_ok = false; srs_info("initialize kafka producer ok."); return ret; @@ -175,9 +200,7 @@ int SrsKafkaProducer::start() srs_error("start kafka thread failed. ret=%d", ret); } - meatadata_ok = false; - st_cond_signal(metadata_expired); - srs_trace("kafka work in background"); + refresh_metadata(); return ret; } @@ -203,7 +226,7 @@ int SrsKafkaProducer::on_before_cycle() { // wait for the metadata expired. // when metadata is ok, wait for it expired. - if (meatadata_ok) { + if (metadata_ok) { st_cond_wait(metadata_expired); } @@ -291,34 +314,20 @@ int SrsKafkaProducer::request_metadata() srs_trace("kafka metadata: %s", summary.c_str()); // generate the partition info. - for (int i = 0; i < metadata->metadatas.size(); i++) { - SrsKafkaTopicMetadata* topic = metadata->metadatas.at(i); - - for (int j = 0; j < topic->metadatas.size(); j++) { - SrsKafkaPartitionMetadata* partition = topic->metadatas.at(j); - - SrsKafkaPartition* p = new SrsKafkaPartition(); - p->id = partition->partition_id; - p->broker = partition->leader; - - for (int i = 0; i < metadata->brokers.size(); i++) { - SrsKafkaBroker* broker = metadata->brokers.at(i); - if (broker->node_id == p->broker) { - p->host = broker->host.to_str(); - p->port = broker->port; - break; - } - } - - partitions.push_back(p); - } - } + srs_kafka_metadata2connector(metadata, partitions); srs_trace("kafka connector: %s", srs_kafka_summary_partitions(partitions).c_str()); - meatadata_ok = true; + metadata_ok = true; return ret; } +void SrsKafkaProducer::refresh_metadata() +{ + metadata_ok = false; + st_cond_signal(metadata_expired); + srs_trace("kafka async refresh metadata in background"); +} + #endif diff --git a/trunk/src/app/srs_app_kafka.hpp b/trunk/src/app/srs_app_kafka.hpp index dfd28f954..f7edaa47f 100644 --- a/trunk/src/app/srs_app_kafka.hpp +++ b/trunk/src/app/srs_app_kafka.hpp @@ -69,7 +69,7 @@ private: st_mutex_t lock; SrsReusableThread* pthread; private: - bool meatadata_ok; + bool metadata_ok; st_cond_t metadata_expired; public: std::vector partitions; @@ -93,6 +93,8 @@ public: private: virtual int do_cycle(); virtual int request_metadata(); + // set the metadata to invalid and refresh it. + virtual void refresh_metadata(); }; #endif diff --git a/trunk/src/protocol/srs_kafka_stack.cpp b/trunk/src/protocol/srs_kafka_stack.cpp index ed66d76f5..c0b0eaa16 100644 --- a/trunk/src/protocol/srs_kafka_stack.cpp +++ b/trunk/src/protocol/srs_kafka_stack.cpp @@ -1010,7 +1010,7 @@ int SrsKafkaProtocol::recv_message(SrsKafkaResponse** pmsg) // fetch cached api key. SrsKafkaCorrelationPool* pool = SrsKafkaCorrelationPool::instance(); SrsKafkaApiKey key = pool->unset(header.correlation_id()); - srs_trace("kafka got %d bytes response, key=%d", header.total_size(), header.correlation_id()); + srs_info("kafka got %d bytes response, key=%d", header.total_size(), header.correlation_id()); // create message by cached api key. SrsKafkaResponse* res = NULL; From 7013993c7aba12905bc41120e9fb6e7b9154fcc8 Mon Sep 17 00:00:00 2001 From: winlin Date: Thu, 22 Oct 2015 15:26:57 +0800 Subject: [PATCH 34/46] kafka send the accept message. --- trunk/src/app/srs_app_kafka.cpp | 71 ++++++++++++++++++++++++++++++++ trunk/src/app/srs_app_kafka.hpp | 35 ++++++++++++++++ trunk/src/app/srs_app_server.cpp | 8 ++++ 3 files changed, 114 insertions(+) diff --git a/trunk/src/app/srs_app_kafka.cpp b/trunk/src/app/srs_app_kafka.cpp index 03f832429..5c15f1604 100644 --- a/trunk/src/app/srs_app_kafka.cpp +++ b/trunk/src/app/srs_app_kafka.cpp @@ -36,10 +36,12 @@ using namespace std; #include #include #include +#include #ifdef SRS_AUTO_KAFKA #define SRS_KAKFA_CYCLE_INTERVAL_MS 3000 +#define SRS_KAFKA_PRODUCER_AGGREGATE_SIZE 10 std::string srs_kafka_metadata_summary(SrsKafkaTopicMetadataResponse* metadata) { @@ -144,6 +146,33 @@ string SrsKafkaPartition::hostport() return ep; } +SrsKafkaMessageOnClient::SrsKafkaMessageOnClient(SrsKafkaProducer* p, SrsListenerType t, string i) +{ + producer = p; + type = t; + ip = i; +} + +SrsKafkaMessageOnClient::~SrsKafkaMessageOnClient() +{ +} + +int SrsKafkaMessageOnClient::call() +{ + SrsJsonObject* obj = SrsJsonAny::object(); + + obj->set("msg", SrsJsonAny::str("accept")); + obj->set("type", SrsJsonAny::integer(type)); + obj->set("ip", SrsJsonAny::str(ip.c_str())); + + return producer->send(obj); +} + +string SrsKafkaMessageOnClient::to_string() +{ + return ip; +} + SrsKafkaProducer::SrsKafkaProducer() { metadata_ok = false; @@ -211,6 +240,41 @@ void SrsKafkaProducer::stop() worker->stop(); } +int SrsKafkaProducer::on_client(SrsListenerType type, st_netfd_t stfd) +{ + return worker->execute(new SrsKafkaMessageOnClient(this, type, srs_get_peer_ip(st_netfd_fileno(stfd)))); +} + +int SrsKafkaProducer::send(SrsJsonObject* obj) +{ + int ret = ERROR_SUCCESS; + + // cache the json object. + objects.push_back(obj); + + // too few messages, ignore. + if (objects.size() < SRS_KAFKA_PRODUCER_AGGREGATE_SIZE) { + return ret; + } + + // too many messages, warn user. + if (objects.size() > SRS_KAFKA_PRODUCER_AGGREGATE_SIZE * 10) { + srs_warn("kafka cache too many messages: %d", objects.size()); + } + + // sync with backgound metadata worker. + st_mutex_lock(lock); + + // flush message when metadata is ok. + if (metadata_ok) { + ret = flush(); + } + + st_mutex_unlock(lock); + + return ret; +} + int SrsKafkaProducer::cycle() { int ret = ERROR_SUCCESS; @@ -329,5 +393,12 @@ void SrsKafkaProducer::refresh_metadata() srs_trace("kafka async refresh metadata in background"); } +int SrsKafkaProducer::flush() +{ + int ret = ERROR_SUCCESS; + // TODO: FIXME: implements it. + return ret; +} + #endif diff --git a/trunk/src/app/srs_app_kafka.hpp b/trunk/src/app/srs_app_kafka.hpp index f7edaa47f..f4ba4231d 100644 --- a/trunk/src/app/srs_app_kafka.hpp +++ b/trunk/src/app/srs_app_kafka.hpp @@ -35,8 +35,12 @@ class SrsLbRoundRobin; class SrsAsyncCallWorker; class SrsTcpClient; class SrsKafkaClient; +class SrsJsonObject; +class SrsKafkaProducer; #include +#include +#include #ifdef SRS_AUTO_KAFKA @@ -60,6 +64,24 @@ public: virtual std::string hostport(); }; +/** + * the following is all types of kafka messages. + */ +struct SrsKafkaMessageOnClient : public ISrsAsyncCallTask +{ +public: + SrsKafkaProducer* producer; + SrsListenerType type; + std::string ip; +public: + SrsKafkaMessageOnClient(SrsKafkaProducer* p, SrsListenerType t, std::string i); + virtual ~SrsKafkaMessageOnClient(); +// interface ISrsAsyncCallTask +public: + virtual int call(); + virtual std::string to_string(); +}; + /** * the kafka producer used to save log to kafka cluster. */ @@ -73,6 +95,7 @@ private: st_cond_t metadata_expired; public: std::vector partitions; + std::vector objects; private: SrsLbRoundRobin* lb; SrsAsyncCallWorker* worker; @@ -85,6 +108,17 @@ public: virtual int initialize(); virtual int start(); virtual void stop(); +public: + /** + * when got any client connect to SRS, notify kafka. + */ + virtual int on_client(SrsListenerType type, st_netfd_t stfd); + /** + * send json object to kafka cluster. + * the producer will aggregate message and send in kafka message set. + * @param obj the json object; user must never free it again. + */ + virtual int send(SrsJsonObject* obj); // interface ISrsReusableThreadHandler public: virtual int cycle(); @@ -95,6 +129,7 @@ private: virtual int request_metadata(); // set the metadata to invalid and refresh it. virtual void refresh_metadata(); + virtual int flush(); }; #endif diff --git a/trunk/src/app/srs_app_server.cpp b/trunk/src/app/srs_app_server.cpp index 89d568c20..6fc8aec49 100755 --- a/trunk/src/app/srs_app_server.cpp +++ b/trunk/src/app/srs_app_server.cpp @@ -1289,6 +1289,14 @@ int SrsServer::accept_client(SrsListenerType type, st_netfd_t client_stfd) } srs_assert(conn); +#ifdef SRS_AUTO_KAFKA + // notify kafka cluster. + if ((ret = kafka->on_client(type, client_stfd)) != ERROR_SUCCESS) { + srs_error("kafka handler on_client failed. ret=%d", ret); + return ret; + } +#endif + // directly enqueue, the cycle thread will remove the client. conns.push_back(conn); srs_verbose("add conn to vector."); From 61486a82aaa76aec9040ff24fd8b251766440b0b Mon Sep 17 00:00:00 2001 From: winlin Date: Thu, 22 Oct 2015 16:24:10 +0800 Subject: [PATCH 35/46] kafka use topic and partition cache --- trunk/src/app/srs_app_kafka.cpp | 156 +++++++++++++++++++++++++--- trunk/src/app/srs_app_kafka.hpp | 81 +++++++++++++-- trunk/src/app/srs_app_rtmp_conn.cpp | 17 +++ trunk/src/app/srs_app_rtmp_conn.hpp | 11 ++ trunk/src/app/srs_app_server.cpp | 10 +- 5 files changed, 244 insertions(+), 31 deletions(-) diff --git a/trunk/src/app/srs_app_kafka.cpp b/trunk/src/app/srs_app_kafka.cpp index 5c15f1604..00abb442c 100644 --- a/trunk/src/app/srs_app_kafka.cpp +++ b/trunk/src/app/srs_app_kafka.cpp @@ -146,7 +146,17 @@ string SrsKafkaPartition::hostport() return ep; } -SrsKafkaMessageOnClient::SrsKafkaMessageOnClient(SrsKafkaProducer* p, SrsListenerType t, string i) +SrsKafkaMessage::SrsKafkaMessage(int k) +{ + key = k; +} + +SrsKafkaMessage::~SrsKafkaMessage() +{ +} + +SrsKafkaMessageOnClient::SrsKafkaMessageOnClient(SrsKafkaProducer* p, int k, SrsListenerType t, string i) + : SrsKafkaMessage(k) { producer = p; type = t; @@ -165,7 +175,7 @@ int SrsKafkaMessageOnClient::call() obj->set("type", SrsJsonAny::integer(type)); obj->set("ip", SrsJsonAny::str(ip.c_str())); - return producer->send(obj); + return producer->send(key, obj); } string SrsKafkaMessageOnClient::to_string() @@ -173,6 +183,87 @@ string SrsKafkaMessageOnClient::to_string() return ip; } +SrsKafkaCache::SrsKafkaCache() +{ + count = 0; + nb_partitions = 0; +} + +SrsKafkaCache::~SrsKafkaCache() +{ + map::iterator it; + for (it = cache.begin(); it != cache.end(); ++it) { + SrsKafkaPartitionCache* pc = it->second; + + for (vector::iterator it2 = pc->begin(); it2 != pc->end(); ++it2) { + SrsJsonObject* obj = *it2; + srs_freep(obj); + } + pc->clear(); + + srs_freep(pc); + } + cache.clear(); +} + +void SrsKafkaCache::append(int key, SrsJsonObject* obj) +{ + count++; + + int partition = 0; + if (nb_partitions > 0) { + partition = key % nb_partitions; + } + + SrsKafkaPartitionCache* pc = NULL; + map::iterator it = cache.find(partition); + if (it == cache.end()) { + pc = new SrsKafkaPartitionCache(); + cache[partition] = pc; + } else { + pc = it->second; + } + + pc->push_back(obj); +} + +int SrsKafkaCache::size() +{ + return count; +} + +bool SrsKafkaCache::fetch(int* pkey, SrsKafkaPartitionCache** ppc) +{ + map::iterator it; + for (it = cache.begin(); it != cache.end(); ++it) { + int32_t key = it->first; + SrsKafkaPartitionCache* pc = it->second; + + if (!pc->empty()) { + *pkey = (int)key; + *ppc = pc; + return true; + } + } + + return false; +} + +int SrsKafkaCache::flush(SrsKafkaPartition* partition, int key, SrsKafkaPartitionCache* pc) +{ + int ret = ERROR_SUCCESS; + // TODO: FIXME: implements it. + return ret; +} + +ISrsKafkaCluster::ISrsKafkaCluster() +{ +} + +ISrsKafkaCluster::~ISrsKafkaCluster() +{ +} + SrsKafkaProducer::SrsKafkaProducer() { metadata_ok = false; @@ -181,6 +272,7 @@ SrsKafkaProducer::SrsKafkaProducer() lock = st_mutex_new(); pthread = new SrsReusableThread("kafka", this, SRS_KAKFA_CYCLE_INTERVAL_MS * 1000); worker = new SrsAsyncCallWorker(); + cache = new SrsKafkaCache(); lb = new SrsLbRoundRobin(); transport = new SrsTcpClient(); @@ -189,12 +281,7 @@ SrsKafkaProducer::SrsKafkaProducer() SrsKafkaProducer::~SrsKafkaProducer() { - vector::iterator it; - for (it = partitions.begin(); it != partitions.end(); ++it) { - SrsKafkaPartition* partition = *it; - srs_freep(partition); - } - partitions.clear(); + clear_metadata(); srs_freep(lb); srs_freep(kafka); @@ -202,6 +289,7 @@ SrsKafkaProducer::~SrsKafkaProducer() srs_freep(worker); srs_freep(pthread); + srs_freep(cache); st_mutex_destroy(lock); st_cond_destroy(metadata_expired); @@ -240,26 +328,26 @@ void SrsKafkaProducer::stop() worker->stop(); } -int SrsKafkaProducer::on_client(SrsListenerType type, st_netfd_t stfd) +int SrsKafkaProducer::on_client(int key, SrsListenerType type, string ip) { - return worker->execute(new SrsKafkaMessageOnClient(this, type, srs_get_peer_ip(st_netfd_fileno(stfd)))); + return worker->execute(new SrsKafkaMessageOnClient(this, key, type, ip)); } -int SrsKafkaProducer::send(SrsJsonObject* obj) +int SrsKafkaProducer::send(int key, SrsJsonObject* obj) { int ret = ERROR_SUCCESS; // cache the json object. - objects.push_back(obj); + cache->append(key, obj); // too few messages, ignore. - if (objects.size() < SRS_KAFKA_PRODUCER_AGGREGATE_SIZE) { + if (cache->size() < SRS_KAFKA_PRODUCER_AGGREGATE_SIZE) { return ret; } // too many messages, warn user. - if (objects.size() > SRS_KAFKA_PRODUCER_AGGREGATE_SIZE * 10) { - srs_warn("kafka cache too many messages: %d", objects.size()); + if (cache->size() > SRS_KAFKA_PRODUCER_AGGREGATE_SIZE * 10) { + srs_warn("kafka cache too many messages: %d", cache->size()); } // sync with backgound metadata worker. @@ -307,6 +395,18 @@ int SrsKafkaProducer::on_end_cycle() return ERROR_SUCCESS; } +void SrsKafkaProducer::clear_metadata() +{ + vector::iterator it; + + for (it = partitions.begin(); it != partitions.end(); ++it) { + SrsKafkaPartition* partition = *it; + srs_freep(partition); + } + + partitions.clear(); +} + int SrsKafkaProducer::do_cycle() { int ret = ERROR_SUCCESS; @@ -381,6 +481,9 @@ int SrsKafkaProducer::request_metadata() srs_kafka_metadata2connector(metadata, partitions); srs_trace("kafka connector: %s", srs_kafka_summary_partitions(partitions).c_str()); + // update the total partition for cache. + cache->nb_partitions = (int)partitions.size(); + metadata_ok = true; return ret; @@ -388,6 +491,8 @@ int SrsKafkaProducer::request_metadata() void SrsKafkaProducer::refresh_metadata() { + clear_metadata(); + metadata_ok = false; st_cond_signal(metadata_expired); srs_trace("kafka async refresh metadata in background"); @@ -396,7 +501,26 @@ void SrsKafkaProducer::refresh_metadata() int SrsKafkaProducer::flush() { int ret = ERROR_SUCCESS; - // TODO: FIXME: implements it. + + // flush all available partition caches. + while (true) { + int key = 0; + SrsKafkaPartitionCache* pc = NULL; + + // all flushed, or no kafka partition to write to. + if (!cache->fetch(&key, &pc) || partitions.empty()) { + break; + } + + // flush specified partition. + srs_assert(key && pc); + SrsKafkaPartition* partition = partitions.at(key % partitions.size()); + if ((ret = cache->flush(partition, key, pc)) != ERROR_SUCCESS) { + srs_error("flush partition failed. ret=%d", ret); + return ret; + } + } + return ret; } diff --git a/trunk/src/app/srs_app_kafka.hpp b/trunk/src/app/srs_app_kafka.hpp index f4ba4231d..a36942ee8 100644 --- a/trunk/src/app/srs_app_kafka.hpp +++ b/trunk/src/app/srs_app_kafka.hpp @@ -29,6 +29,7 @@ */ #include +#include #include class SrsLbRoundRobin; @@ -67,14 +68,22 @@ public: /** * the following is all types of kafka messages. */ -struct SrsKafkaMessageOnClient : public ISrsAsyncCallTask +class SrsKafkaMessage : public ISrsAsyncCallTask +{ +protected: + int key; +public: + SrsKafkaMessage(int k); + virtual ~SrsKafkaMessage(); +}; +struct SrsKafkaMessageOnClient : public SrsKafkaMessage { public: SrsKafkaProducer* producer; SrsListenerType type; std::string ip; public: - SrsKafkaMessageOnClient(SrsKafkaProducer* p, SrsListenerType t, std::string i); + SrsKafkaMessageOnClient(SrsKafkaProducer* p, int k, SrsListenerType t, std::string i); virtual ~SrsKafkaMessageOnClient(); // interface ISrsAsyncCallTask public: @@ -82,10 +91,66 @@ public: virtual std::string to_string(); }; +/** + * the partition messages cache. + */ +typedef std::vector SrsKafkaPartitionCache; + +/** + * a message cache for kafka. + */ +class SrsKafkaCache +{ +public: + // the total partitions, + // for the key to map to the parition by key%nb_partitions. + int nb_partitions; +private: + // total messages for all partitions. + int count; + // key is the partition id, value is the message set to write to this partition. + // @remark, when refresh metadata, the partition will increase, + // so maybe some message will dispatch to new partition. + std::map< int32_t, SrsKafkaPartitionCache*> cache; +public: + SrsKafkaCache(); + virtual ~SrsKafkaCache(); +public: + virtual void append(int key, SrsJsonObject* obj); + virtual int size(); + /** + * fetch out a available partition cache. + * @return true when got a key and pc; otherwise, false. + */ + virtual bool fetch(int* pkey, SrsKafkaPartitionCache** ppc); + /** + * flush the specified partition cache. + */ + virtual int flush(SrsKafkaPartition* partition, int key, SrsKafkaPartitionCache* pc); +}; + +/** + * the kafka cluster interface. + */ +class ISrsKafkaCluster +{ +public: + ISrsKafkaCluster(); + virtual ~ISrsKafkaCluster(); +public: + /** + * when got any client connect to SRS, notify kafka. + * @param key the partition map key, a id or hash. + * @param type the type of client. + * @param ip the peer ip of client. + */ + virtual int on_client(int key, SrsListenerType type, std::string ip) = 0; +}; + /** * the kafka producer used to save log to kafka cluster. */ -class SrsKafkaProducer : public ISrsReusableThreadHandler +class SrsKafkaProducer : virtual public ISrsReusableThreadHandler, virtual public ISrsKafkaCluster { private: st_mutex_t lock; @@ -95,7 +160,7 @@ private: st_cond_t metadata_expired; public: std::vector partitions; - std::vector objects; + SrsKafkaCache* cache; private: SrsLbRoundRobin* lb; SrsAsyncCallWorker* worker; @@ -112,19 +177,23 @@ public: /** * when got any client connect to SRS, notify kafka. */ - virtual int on_client(SrsListenerType type, st_netfd_t stfd); + virtual int on_client(int key, SrsListenerType type, std::string ip); +// for worker to call task to send object. +public: /** * send json object to kafka cluster. * the producer will aggregate message and send in kafka message set. + * @param key the key to map to the partition, user can use cid or hash. * @param obj the json object; user must never free it again. */ - virtual int send(SrsJsonObject* obj); + virtual int send(int key, SrsJsonObject* obj); // interface ISrsReusableThreadHandler public: virtual int cycle(); virtual int on_before_cycle(); virtual int on_end_cycle(); private: + virtual void clear_metadata(); virtual int do_cycle(); virtual int request_metadata(); // set the metadata to invalid and refresh it. diff --git a/trunk/src/app/srs_app_rtmp_conn.cpp b/trunk/src/app/srs_app_rtmp_conn.cpp index a9229611a..0ab1b58df 100644 --- a/trunk/src/app/srs_app_rtmp_conn.cpp +++ b/trunk/src/app/srs_app_rtmp_conn.cpp @@ -55,6 +55,7 @@ using namespace std; #include #include #include +#include // when stream is busy, for example, streaming is already // publishing, when a new client to request to publish, @@ -310,10 +311,18 @@ void SrsSimpleRtmpClient::set_recv_timeout(int64_t timeout) transport->set_recv_timeout(timeout); } +#ifdef SRS_AUTO_KAFKA +SrsRtmpConn::SrsRtmpConn(SrsServer* svr, ISrsKafkaCluster* k, st_netfd_t c) +#else SrsRtmpConn::SrsRtmpConn(SrsServer* svr, st_netfd_t c) +#endif : SrsConnection(svr, c) { server = svr; +#ifdef SRS_AUTO_KAFKA + kafka = k; +#endif + req = new SrsRequest(); res = new SrsResponse(); skt = new SrsStSocket(c); @@ -365,6 +374,14 @@ int SrsRtmpConn::do_cycle() int ret = ERROR_SUCCESS; srs_trace("RTMP client ip=%s", ip.c_str()); + + // notify kafka cluster. +#ifdef SRS_AUTO_KAFKA + if ((ret = kafka->on_client(srs_id(), SrsListenerRtmpStream, ip)) != ERROR_SUCCESS) { + srs_error("kafka handler on_client failed. ret=%d", ret); + return ret; + } +#endif rtmp->set_recv_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); rtmp->set_send_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); diff --git a/trunk/src/app/srs_app_rtmp_conn.hpp b/trunk/src/app/srs_app_rtmp_conn.hpp index 5cfc0624f..29fea6ee4 100644 --- a/trunk/src/app/srs_app_rtmp_conn.hpp +++ b/trunk/src/app/srs_app_rtmp_conn.hpp @@ -58,6 +58,9 @@ class SrsSecurity; class ISrsWakable; class SrsCommonMessage; class SrsPacket; +#ifdef SRS_AUTO_KAFKA +class ISrsKafkaCluster; +#endif /** * the simple rtmp client stub, use SrsRtmpClient and provides high level APIs. @@ -135,8 +138,16 @@ private: int publish_normal_timeout; // whether enable the tcp_nodelay. bool tcp_nodelay; + // the kafka cluster +#ifdef SRS_AUTO_KAFKA + ISrsKafkaCluster* kafka; +#endif public: +#ifdef SRS_AUTO_KAFKA + SrsRtmpConn(SrsServer* svr, ISrsKafkaCluster* k, st_netfd_t c); +#else SrsRtmpConn(SrsServer* svr, st_netfd_t c); +#endif virtual ~SrsRtmpConn(); public: virtual void dispose(); diff --git a/trunk/src/app/srs_app_server.cpp b/trunk/src/app/srs_app_server.cpp index 6fc8aec49..896f8622c 100755 --- a/trunk/src/app/srs_app_server.cpp +++ b/trunk/src/app/srs_app_server.cpp @@ -1267,7 +1267,7 @@ int SrsServer::accept_client(SrsListenerType type, st_netfd_t client_stfd) SrsConnection* conn = NULL; if (type == SrsListenerRtmpStream) { - conn = new SrsRtmpConn(this, client_stfd); + conn = new SrsRtmpConn(this, kafka, client_stfd); } else if (type == SrsListenerHttpApi) { #ifdef SRS_AUTO_HTTP_API conn = new SrsHttpApi(this, client_stfd, http_api_mux); @@ -1289,14 +1289,6 @@ int SrsServer::accept_client(SrsListenerType type, st_netfd_t client_stfd) } srs_assert(conn); -#ifdef SRS_AUTO_KAFKA - // notify kafka cluster. - if ((ret = kafka->on_client(type, client_stfd)) != ERROR_SUCCESS) { - srs_error("kafka handler on_client failed. ret=%d", ret); - return ret; - } -#endif - // directly enqueue, the cycle thread will remove the client. conns.push_back(conn); srs_verbose("add conn to vector."); From 531b658f8e653103708f913f52ad09f4f4206ef5 Mon Sep 17 00:00:00 2001 From: winlin Date: Thu, 22 Oct 2015 16:53:54 +0800 Subject: [PATCH 36/46] retry when metadata empty --- trunk/src/app/srs_app_kafka.cpp | 13 ++++++++++++- trunk/src/protocol/srs_kafka_stack.hpp | 8 ++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/trunk/src/app/srs_app_kafka.cpp b/trunk/src/app/srs_app_kafka.cpp index 00abb442c..d94af3193 100644 --- a/trunk/src/app/srs_app_kafka.cpp +++ b/trunk/src/app/srs_app_kafka.cpp @@ -459,7 +459,8 @@ int SrsKafkaProducer::request_metadata() senabled.c_str(), sbrokers.c_str(), lb->current(), server.c_str(), port, topic.c_str()); } - // connect to kafka server. + // reconnect to kafka server. + transport->close(); if ((ret = transport->connect(server, port, SRS_CONSTS_KAFKA_TIMEOUT_US)) != ERROR_SUCCESS) { srs_error("kafka connect %s:%d failed. ret=%d", server.c_str(), port, ret); return ret; @@ -473,6 +474,16 @@ int SrsKafkaProducer::request_metadata() } SrsAutoFree(SrsKafkaTopicMetadataResponse, metadata); + // we may need to request multiple times. + // for example, the first time to create a none-exists topic, then query metadata. + if (!metadata->metadatas.empty()) { + SrsKafkaTopicMetadata* topic = metadata->metadatas.at(0); + if (topic->metadatas.empty()) { + srs_warn("topic %s metadata empty, retry.", topic->name.to_str().c_str()); + return ret; + } + } + // show kafka metadata. string summary = srs_kafka_metadata_summary(metadata); srs_trace("kafka metadata: %s", summary.c_str()); diff --git a/trunk/src/protocol/srs_kafka_stack.hpp b/trunk/src/protocol/srs_kafka_stack.hpp index f181d04dc..4cb5c676e 100644 --- a/trunk/src/protocol/srs_kafka_stack.hpp +++ b/trunk/src/protocol/srs_kafka_stack.hpp @@ -152,6 +152,10 @@ public: { return length; } + virtual bool empty() + { + return elems.empty(); + } virtual T* at(int index) { return elems.at(index); @@ -241,6 +245,10 @@ public: { return length; } + virtual bool empty() + { + return elems.empty(); + } virtual int32_t at(int index) { return elems.at(index); From 22fa9a0f1f5778af32c156157be078db52ebb933 Mon Sep 17 00:00:00 2001 From: winlin Date: Thu, 22 Oct 2015 16:57:27 +0800 Subject: [PATCH 37/46] kafka use temp transport to fetch metadata. --- trunk/src/app/srs_app_kafka.cpp | 11 ++++++----- trunk/src/app/srs_app_kafka.hpp | 2 -- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/trunk/src/app/srs_app_kafka.cpp b/trunk/src/app/srs_app_kafka.cpp index d94af3193..52236e9cd 100644 --- a/trunk/src/app/srs_app_kafka.cpp +++ b/trunk/src/app/srs_app_kafka.cpp @@ -275,8 +275,6 @@ SrsKafkaProducer::SrsKafkaProducer() cache = new SrsKafkaCache(); lb = new SrsLbRoundRobin(); - transport = new SrsTcpClient(); - kafka = new SrsKafkaClient(transport); } SrsKafkaProducer::~SrsKafkaProducer() @@ -284,8 +282,6 @@ SrsKafkaProducer::~SrsKafkaProducer() clear_metadata(); srs_freep(lb); - srs_freep(kafka); - srs_freep(transport); srs_freep(worker); srs_freep(pthread); @@ -443,6 +439,12 @@ int SrsKafkaProducer::request_metadata() return ret; } + SrsTcpClient* transport = new SrsTcpClient(); + SrsAutoFree(SrsTcpClient, transport); + + SrsKafkaClient* kafka = new SrsKafkaClient(transport); + SrsAutoFree(SrsKafkaClient, kafka); + std::string server; int port = SRS_CONSTS_KAFKA_DEFAULT_PORT; if (true) { @@ -460,7 +462,6 @@ int SrsKafkaProducer::request_metadata() } // reconnect to kafka server. - transport->close(); if ((ret = transport->connect(server, port, SRS_CONSTS_KAFKA_TIMEOUT_US)) != ERROR_SUCCESS) { srs_error("kafka connect %s:%d failed. ret=%d", server.c_str(), port, ret); return ret; diff --git a/trunk/src/app/srs_app_kafka.hpp b/trunk/src/app/srs_app_kafka.hpp index a36942ee8..4300a9db7 100644 --- a/trunk/src/app/srs_app_kafka.hpp +++ b/trunk/src/app/srs_app_kafka.hpp @@ -164,8 +164,6 @@ public: private: SrsLbRoundRobin* lb; SrsAsyncCallWorker* worker; - SrsTcpClient* transport; - SrsKafkaClient* kafka; public: SrsKafkaProducer(); virtual ~SrsKafkaProducer(); From f9f5b56951e87c993c40c4122a1eb8468cb821de Mon Sep 17 00:00:00 2001 From: winlin Date: Thu, 22 Oct 2015 17:25:48 +0800 Subject: [PATCH 38/46] kakfa erase messages when wrote. --- trunk/src/app/srs_app_kafka.cpp | 58 +++++++++++++++++++++++++++++++-- trunk/src/app/srs_app_kafka.hpp | 3 ++ 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/trunk/src/app/srs_app_kafka.cpp b/trunk/src/app/srs_app_kafka.cpp index 52236e9cd..ac4e96041 100644 --- a/trunk/src/app/srs_app_kafka.cpp +++ b/trunk/src/app/srs_app_kafka.cpp @@ -41,7 +41,8 @@ using namespace std; #ifdef SRS_AUTO_KAFKA #define SRS_KAKFA_CYCLE_INTERVAL_MS 3000 -#define SRS_KAFKA_PRODUCER_AGGREGATE_SIZE 10 +#define SRS_KAFKA_PRODUCER_TIMEOUT 30000 +#define SRS_KAFKA_PRODUCER_AGGREGATE_SIZE 1 std::string srs_kafka_metadata_summary(SrsKafkaTopicMetadataResponse* metadata) { @@ -131,10 +132,15 @@ SrsKafkaPartition::SrsKafkaPartition() { id = broker = 0; port = SRS_CONSTS_KAFKA_DEFAULT_PORT; + + transport = new SrsTcpClient(); + kafka = new SrsKafkaClient(transport); } SrsKafkaPartition::~SrsKafkaPartition() { + srs_freep(kafka); + srs_freep(transport); } string SrsKafkaPartition::hostport() @@ -146,6 +152,25 @@ string SrsKafkaPartition::hostport() return ep; } +int SrsKafkaPartition::connect() +{ + int ret = ERROR_SUCCESS; + + if (transport->connected()) { + return ret; + } + + int64_t timeout = SRS_KAFKA_PRODUCER_TIMEOUT * 1000; + if ((ret = transport->connect(host, port, timeout)) != ERROR_SUCCESS) { + srs_error("connect to %s partition=%d failed, timeout=%"PRId64". ret=%d", hostport().c_str(), id, timeout, ret); + return ret; + } + + srs_trace("connect at %s, partition=%d, broker=%d", hostport().c_str(), id, broker); + + return ret; +} + SrsKafkaMessage::SrsKafkaMessage(int k) { key = k; @@ -252,7 +277,34 @@ bool SrsKafkaCache::fetch(int* pkey, SrsKafkaPartitionCache** ppc) int SrsKafkaCache::flush(SrsKafkaPartition* partition, int key, SrsKafkaPartitionCache* pc) { int ret = ERROR_SUCCESS; + + // ensure the key exists. + srs_assert (cache.find(key) != cache.end()); + + // connect transport. + if ((ret = partition->connect()) != ERROR_SUCCESS) { + srs_error("connect to partition failed. ret=%d", ret); + return ret; + } + + // copy the messages to a temp cache. + SrsKafkaPartitionCache tpc(*pc); + // TODO: FIXME: implements it. + + // free all wrote messages. + for (vector::iterator it = tpc.begin(); it != tpc.end(); ++it) { + SrsJsonObject* obj = *it; + srs_freep(obj); + } + + // remove the messages from cache. + if (pc->size() == tpc.size()) { + pc->clear(); + } else { + pc->erase(pc->begin(), pc->begin() + tpc.size()); + } + return ret; } @@ -516,7 +568,7 @@ int SrsKafkaProducer::flush() // flush all available partition caches. while (true) { - int key = 0; + int key = -1; SrsKafkaPartitionCache* pc = NULL; // all flushed, or no kafka partition to write to. @@ -525,7 +577,7 @@ int SrsKafkaProducer::flush() } // flush specified partition. - srs_assert(key && pc); + srs_assert(key >= 0 && pc); SrsKafkaPartition* partition = partitions.at(key % partitions.size()); if ((ret = cache->flush(partition, key, pc)) != ERROR_SUCCESS) { srs_error("flush partition failed. ret=%d", ret); diff --git a/trunk/src/app/srs_app_kafka.hpp b/trunk/src/app/srs_app_kafka.hpp index 4300a9db7..7f777e812 100644 --- a/trunk/src/app/srs_app_kafka.hpp +++ b/trunk/src/app/srs_app_kafka.hpp @@ -52,6 +52,8 @@ struct SrsKafkaPartition { private: std::string ep; + SrsTcpClient* transport; + SrsKafkaClient* kafka; public: int id; // leader. @@ -63,6 +65,7 @@ public: virtual ~SrsKafkaPartition(); public: virtual std::string hostport(); + virtual int connect(); }; /** From 7a0aaf5900604de8e600b29b8de270a72fb18f57 Mon Sep 17 00:00:00 2001 From: winlin Date: Thu, 22 Oct 2015 17:43:15 +0800 Subject: [PATCH 39/46] kafka refine code --- trunk/src/app/srs_app_kafka.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/trunk/src/app/srs_app_kafka.cpp b/trunk/src/app/srs_app_kafka.cpp index ac4e96041..01e99fab8 100644 --- a/trunk/src/app/srs_app_kafka.cpp +++ b/trunk/src/app/srs_app_kafka.cpp @@ -281,28 +281,32 @@ int SrsKafkaCache::flush(SrsKafkaPartition* partition, int key, SrsKafkaPartitio // ensure the key exists. srs_assert (cache.find(key) != cache.end()); + // the cache is vector, which is continous store. + // we remember the messages we have written and clear it when completed. + int nb_msgs = (int)pc->size(); + if (pc->empty()) { + return ret; + } + // connect transport. if ((ret = partition->connect()) != ERROR_SUCCESS) { srs_error("connect to partition failed. ret=%d", ret); return ret; } - // copy the messages to a temp cache. - SrsKafkaPartitionCache tpc(*pc); - // TODO: FIXME: implements it. // free all wrote messages. - for (vector::iterator it = tpc.begin(); it != tpc.end(); ++it) { + for (vector::iterator it = pc->begin(); it != pc->end(); ++it) { SrsJsonObject* obj = *it; srs_freep(obj); } // remove the messages from cache. - if (pc->size() == tpc.size()) { + if (pc->size() == nb_msgs) { pc->clear(); } else { - pc->erase(pc->begin(), pc->begin() + tpc.size()); + pc->erase(pc->begin(), pc->begin() + nb_msgs); } return ret; From 8e344f1c26559e21a4c526160e632b54909fa366 Mon Sep 17 00:00:00 2001 From: winlin Date: Fri, 23 Oct 2015 09:55:29 +0800 Subject: [PATCH 40/46] add producer api messages. --- trunk/src/app/srs_app_kafka.cpp | 19 +- trunk/src/app/srs_app_kafka.hpp | 12 +- trunk/src/kernel/srs_kernel_error.hpp | 2 + trunk/src/protocol/srs_kafka_stack.cpp | 271 +++++++++++++++++++++++++ trunk/src/protocol/srs_kafka_stack.hpp | 110 +++++++++- 5 files changed, 403 insertions(+), 11 deletions(-) diff --git a/trunk/src/app/srs_app_kafka.cpp b/trunk/src/app/srs_app_kafka.cpp index 01e99fab8..0f75217f0 100644 --- a/trunk/src/app/srs_app_kafka.cpp +++ b/trunk/src/app/srs_app_kafka.cpp @@ -102,7 +102,7 @@ std::string srs_kafka_summary_partitions(const vector& parti return srs_join_vector_string(ret, ", "); } -void srs_kafka_metadata2connector(SrsKafkaTopicMetadataResponse* metadata, vector& partitions) +void srs_kafka_metadata2connector(string topic_name, SrsKafkaTopicMetadataResponse* metadata, vector& partitions) { for (int i = 0; i < metadata->metadatas.size(); i++) { SrsKafkaTopicMetadata* topic = metadata->metadatas.at(i); @@ -111,6 +111,8 @@ void srs_kafka_metadata2connector(SrsKafkaTopicMetadataResponse* metadata, vecto SrsKafkaPartitionMetadata* partition = topic->metadatas.at(j); SrsKafkaPartition* p = new SrsKafkaPartition(); + + p->topic = topic_name; p->id = partition->partition_id; p->broker = partition->leader; @@ -171,6 +173,11 @@ int SrsKafkaPartition::connect() return ret; } +int SrsKafkaPartition::flush(SrsKafkaPartitionCache* pc) +{ + return kafka->write_messages(topic, id, *pc); +} + SrsKafkaMessage::SrsKafkaMessage(int k) { key = k; @@ -290,11 +297,15 @@ int SrsKafkaCache::flush(SrsKafkaPartition* partition, int key, SrsKafkaPartitio // connect transport. if ((ret = partition->connect()) != ERROR_SUCCESS) { - srs_error("connect to partition failed. ret=%d", ret); + srs_error("kafka connect to partition failed. ret=%d", ret); return ret; } - // TODO: FIXME: implements it. + // write the json objects. + if ((ret = partition->flush(pc)) != ERROR_SUCCESS) { + srs_error("kafka write messages failed. ret=%d", ret); + return ret; + } // free all wrote messages. for (vector::iterator it = pc->begin(); it != pc->end(); ++it) { @@ -546,7 +557,7 @@ int SrsKafkaProducer::request_metadata() srs_trace("kafka metadata: %s", summary.c_str()); // generate the partition info. - srs_kafka_metadata2connector(metadata, partitions); + srs_kafka_metadata2connector(topic, metadata, partitions); srs_trace("kafka connector: %s", srs_kafka_summary_partitions(partitions).c_str()); // update the total partition for cache. diff --git a/trunk/src/app/srs_app_kafka.hpp b/trunk/src/app/srs_app_kafka.hpp index 7f777e812..457470f0e 100644 --- a/trunk/src/app/srs_app_kafka.hpp +++ b/trunk/src/app/srs_app_kafka.hpp @@ -45,6 +45,11 @@ class SrsKafkaProducer; #ifdef SRS_AUTO_KAFKA +/** + * the partition messages cache. + */ +typedef std::vector SrsKafkaPartitionCache; + /** * the kafka partition info. */ @@ -56,6 +61,7 @@ private: SrsKafkaClient* kafka; public: int id; + std::string topic; // leader. int broker; std::string host; @@ -66,6 +72,7 @@ public: public: virtual std::string hostport(); virtual int connect(); + virtual int flush(SrsKafkaPartitionCache* pc); }; /** @@ -94,11 +101,6 @@ public: virtual std::string to_string(); }; -/** - * the partition messages cache. - */ -typedef std::vector SrsKafkaPartitionCache; - /** * a message cache for kafka. */ diff --git a/trunk/src/kernel/srs_kernel_error.hpp b/trunk/src/kernel/srs_kernel_error.hpp index eb5149b06..4435450f9 100755 --- a/trunk/src/kernel/srs_kernel_error.hpp +++ b/trunk/src/kernel/srs_kernel_error.hpp @@ -274,6 +274,8 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #define ERROR_KAFKA_CODEC_RESPONSE 4033 #define ERROR_KAFKA_CODEC_ARRAY 4034 #define ERROR_KAFKA_CODEC_METADATA 4035 +#define ERROR_KAFKA_CODEC_MESSAGE 4036 +#define ERROR_KAFKA_CODEC_PRODUCER 4037 /////////////////////////////////////////////////////// // HTTP API error. diff --git a/trunk/src/protocol/srs_kafka_stack.cpp b/trunk/src/protocol/srs_kafka_stack.cpp index c0b0eaa16..5cf1e7b20 100644 --- a/trunk/src/protocol/srs_kafka_stack.cpp +++ b/trunk/src/protocol/srs_kafka_stack.cpp @@ -33,6 +33,7 @@ using namespace std; #include #include #include +#include #ifdef SRS_AUTO_KAFKA @@ -478,6 +479,67 @@ SrsKafkaRawMessage::~SrsKafkaRawMessage() srs_freep(value); } +int SrsKafkaRawMessage::nb_bytes() +{ + return 8 + 4 + 4 + 1 + 1 + key->nb_bytes() + value->nb_bytes(); +} + +int SrsKafkaRawMessage::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(8 + 4 + 4 + 1 + 1)) { + ret = ERROR_KAFKA_CODEC_MESSAGE; + srs_error("kafka encode message failed. ret=%d", ret); + return ret; + } + buf->write_8bytes(offset); + buf->write_4bytes(message_size); + buf->write_4bytes(crc); + buf->write_1bytes(magic_byte); + buf->write_1bytes(attributes); + + if ((ret = key->encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode message key failed. ret=%d", ret); + return ret; + } + + if ((ret = value->encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode message value failed. ret=%d", ret); + return ret; + } + + return ret; +} + +int SrsKafkaRawMessage::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(8 + 4 + 4 + 1 + 1)) { + ret = ERROR_KAFKA_CODEC_MESSAGE; + srs_error("kafka decode message failed. ret=%d", ret); + return ret; + } + offset = buf->read_8bytes(); + message_size = buf->read_4bytes(); + crc = buf->read_4bytes(); + magic_byte = buf->read_1bytes(); + attributes = buf->read_1bytes(); + + if ((ret = key->decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode message key failed. ret=%d", ret); + return ret; + } + + if ((ret = value->decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode message value failed. ret=%d", ret); + return ret; + } + + return ret; +} + SrsKafkaRawMessageSet::SrsKafkaRawMessageSet() { } @@ -492,6 +554,59 @@ SrsKafkaRawMessageSet::~SrsKafkaRawMessageSet() messages.clear(); } +void SrsKafkaRawMessageSet::append(SrsKafkaRawMessage* msg) +{ + messages.push_back(msg); +} + +int SrsKafkaRawMessageSet::nb_bytes() +{ + int s = 0; + + vector::iterator it; + for (it = messages.begin(); it != messages.end(); ++it) { + SrsKafkaRawMessage* message = *it; + s += message->nb_bytes(); + } + + return s; +} + +int SrsKafkaRawMessageSet::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + vector::iterator it; + for (it = messages.begin(); it != messages.end(); ++it) { + SrsKafkaRawMessage* message = *it; + if ((ret = message->encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode message set failed. ret=%d", ret); + return ret; + } + } + + return ret; +} + +int SrsKafkaRawMessageSet::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + while (!buf->empty()) { + SrsKafkaRawMessage* message = new SrsKafkaRawMessage(); + + if ((ret = message->decode(buf)) != ERROR_SUCCESS) { + srs_freep(message); + srs_error("kafka decode message set elem failed. ret=%d", ret); + return ret; + } + + messages.push_back(message); + } + + return ret; +} + SrsKafkaRequest::SrsKafkaRequest() { header.set_correlation_id(SrsKafkaCorrelationPool::instance()->generate_correlation_id()); @@ -863,6 +978,155 @@ int SrsKafkaTopicMetadataResponse::decode(SrsBuffer* buf) return ret; } +int SrsKafkaProducerPartitionMessages::nb_bytes() +{ + return 4 + 4 + messages.nb_bytes(); +} + +int SrsKafkaProducerPartitionMessages::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(4 + 4)) { + ret = ERROR_KAFKA_CODEC_PRODUCER; + srs_error("kafka encode producer failed. ret=%d", ret); + return ret; + } + buf->write_4bytes(partition); + buf->write_4bytes(message_set_size); + + if ((ret = messages.encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode producer messages failed. ret=%d", ret); + return ret; + } + + return ret; +} + +int SrsKafkaProducerPartitionMessages::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(4 + 4)) { + ret = ERROR_KAFKA_CODEC_PRODUCER; + srs_error("kafka decode producer failed. ret=%d", ret); + return ret; + } + partition = buf->read_4bytes(); + message_set_size = buf->read_4bytes(); + + // for the message set decode util empty, we must create a new buffer when + // there exists other objects after message set. + if (buf->size() - buf->pos() != message_set_size) { + SrsBuffer* tbuf = new SrsBuffer(); + SrsAutoFree(SrsBuffer, tbuf); + + if ((ret = tbuf->initialize(buf->data() + buf->pos(), message_set_size)) != ERROR_SUCCESS) { + return ret; + } + + if ((ret = messages.decode(tbuf)) != ERROR_SUCCESS) { + srs_error("kafka decode procuder messages failed. ret=%d", ret); + return ret; + } + } else { + if ((ret = messages.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode procuder messages failed. ret=%d", ret); + return ret; + } + } + + return ret; +} + +int SrsKafkaProducerTopicMessages::nb_bytes() +{ + return topic_name.nb_bytes() + partitions.nb_bytes(); +} + +int SrsKafkaProducerTopicMessages::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if ((ret = topic_name.encode(buf)) != ERROR_SUCCESS) { + return ret; + } + + if ((ret = partitions.encode(buf)) != ERROR_SUCCESS) { + return ret; + } + + return ret; +} + +int SrsKafkaProducerTopicMessages::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if ((ret = topic_name.decode(buf)) != ERROR_SUCCESS) { + return ret; + } + + if ((ret = partitions.decode(buf)) != ERROR_SUCCESS) { + return ret; + } + + return ret; +} + +SrsKafkaProducerRequest::SrsKafkaProducerRequest() +{ + required_acks = 0; + timeout = 0; +} + +SrsKafkaProducerRequest::~SrsKafkaProducerRequest() +{ +} + +int SrsKafkaProducerRequest::nb_bytes() +{ + return 2 + 4 + topics.nb_bytes(); +} + +int SrsKafkaProducerRequest::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(2 + 4)) { + ret = ERROR_KAFKA_CODEC_PRODUCER; + srs_error("kafka encode producer failed. ret=%d", ret); + return ret; + } + buf->write_2bytes(required_acks); + buf->write_4bytes(timeout); + + if ((ret = topics.encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode producer topics failed. ret=%d", ret); + } + + return ret; +} + +int SrsKafkaProducerRequest::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(2 + 4)) { + ret = ERROR_KAFKA_CODEC_PRODUCER; + srs_error("kafka decode producer failed. ret=%d", ret); + return ret; + } + required_acks = buf->read_2bytes(); + timeout = buf->read_4bytes(); + + if ((ret = topics.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode producer topics failed. ret=%d", ret); + } + + return ret; +} + SrsKafkaCorrelationPool* SrsKafkaCorrelationPool::_instance = new SrsKafkaCorrelationPool(); SrsKafkaCorrelationPool* SrsKafkaCorrelationPool::instance() @@ -1085,6 +1349,13 @@ int SrsKafkaClient::fetch_metadata(string topic, SrsKafkaTopicMetadataResponse** return ret; } +int SrsKafkaClient::write_messages(std::string topic, int32_t partition, vector& msgs) +{ + int ret = ERROR_SUCCESS; + // TODO: FIXME: implements it. + return ret; +} + vector srs_kafka_array2vector(SrsKafkaArray* arr) { vector strs; diff --git a/trunk/src/protocol/srs_kafka_stack.hpp b/trunk/src/protocol/srs_kafka_stack.hpp index 4cb5c676e..6764a6e35 100644 --- a/trunk/src/protocol/srs_kafka_stack.hpp +++ b/trunk/src/protocol/srs_kafka_stack.hpp @@ -39,6 +39,7 @@ class SrsFastStream; class ISrsProtocolReaderWriter; +class SrsJsonObject; #ifdef SRS_AUTO_KAFKA @@ -484,7 +485,7 @@ public: * the kafka message in message set. * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets */ -struct SrsKafkaRawMessage +struct SrsKafkaRawMessage : public ISrsCodec { // metadata. public: @@ -530,19 +531,32 @@ public: public: SrsKafkaRawMessage(); virtual ~SrsKafkaRawMessage(); +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); }; /** * a set of kafka message. * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets + * @remark because the message set are not preceded by int32, so we decode the buffer util empty. */ -class SrsKafkaRawMessageSet +class SrsKafkaRawMessageSet : public ISrsCodec { private: std::vector messages; public: SrsKafkaRawMessageSet(); virtual ~SrsKafkaRawMessageSet(); +public: + virtual void append(SrsKafkaRawMessage* msg); +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); }; /** @@ -704,6 +718,94 @@ public: virtual int decode(SrsBuffer* buf); }; + +/** + * the messages for producer to send. + * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ProduceRequest + */ +struct SrsKafkaProducerPartitionMessages : public ISrsCodec +{ +public: + /** + * The partition that data is being published to. + */ + int32_t partition; + /** + * The size, in bytes, of the message set that follows. + */ + int32_t message_set_size; + /** + * messages in set. + */ + SrsKafkaRawMessageSet messages; +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); +}; +struct SrsKafkaProducerTopicMessages : public ISrsCodec +{ +public: + /** + * The topic that data is being published to. + */ + SrsKafkaString topic_name; + /** + * messages of partitions. + */ + SrsKafkaArray partitions; +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); +}; + +/** + * the request for producer to send message. + * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ProduceRequest + */ +class SrsKafkaProducerRequest : public SrsKafkaRequest +{ +private: + /** + * This field indicates how many acknowledgements the servers should receive + * before responding to the request. If it is 0 the server will not send any + * response (this is the only case where the server will not reply to a request). + * If it is 1, the server will wait the data is written to the local log + * before sending a response. If it is -1 the server will block until the + * message is committed by all in sync replicas before sending a response. + * For any number > 1 the server will block waiting for this number of + * acknowledgements to occur (but the server will never wait for more + * acknowledgements than there are in-sync replicas). + */ + int16_t required_acks; + /** + * This provides a maximum time in milliseconds the server can await the receipt + * of the number of acknowledgements in RequiredAcks. The timeout is not an exact + * limit on the request time for a few reasons: (1) it does not include network + * latency, (2) the timer begins at the beginning of the processing of this request + * so if many requests are queued due to server overload that wait time will not + * be included, (3) we will not terminate a local write so if the local write + * time exceeds this timeout it will not be respected. To get a hard timeout of + * this type the client should use the socket timeout. + */ + int32_t timeout; + /** + * messages of topics. + */ + SrsKafkaArray topics; +public: + SrsKafkaProducerRequest(); + virtual ~SrsKafkaProducerRequest(); +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); +}; + /** * the poll to discovery reponse. * @param CorrelationId This is a user-supplied integer. It will be passed back @@ -813,6 +915,10 @@ public: * fetch the metadata from broker for topic. */ virtual int fetch_metadata(std::string topic, SrsKafkaTopicMetadataResponse** pmsg); + /** + * write the messages to partition of topic. + */ + virtual int write_messages(std::string topic, int32_t partition, std::vector& msgs); }; // convert kafka array[string] to vector[string] From 76cd3f874972d06d10fd918dff09b3a30b0f5ff7 Mon Sep 17 00:00:00 2001 From: winlin Date: Fri, 23 Oct 2015 10:16:53 +0800 Subject: [PATCH 41/46] kafka convert json to producer message. --- trunk/src/protocol/srs_kafka_stack.cpp | 109 +++++++++++++++++++++++-- trunk/src/protocol/srs_kafka_stack.hpp | 15 +++- 2 files changed, 115 insertions(+), 9 deletions(-) diff --git a/trunk/src/protocol/srs_kafka_stack.cpp b/trunk/src/protocol/srs_kafka_stack.cpp index 5cf1e7b20..6ecda777a 100644 --- a/trunk/src/protocol/srs_kafka_stack.cpp +++ b/trunk/src/protocol/srs_kafka_stack.cpp @@ -37,6 +37,8 @@ using namespace std; #ifdef SRS_AUTO_KAFKA +#define SRS_KAFKA_PRODUCER_MESSAGE_TIMEOUT_MS 300000 + SrsKafkaString::SrsKafkaString() { _size = -1; @@ -45,11 +47,10 @@ SrsKafkaString::SrsKafkaString() SrsKafkaString::SrsKafkaString(string v) { - _size = (int16_t)v.length(); + _size = -1; + data = NULL; - srs_assert(_size > 0); - data = new char[_size]; - memcpy(data, v.data(), _size); + set_value(v); } SrsKafkaString::~SrsKafkaString() @@ -76,6 +77,19 @@ string SrsKafkaString::to_str() return ret; } +void SrsKafkaString::set_value(string v) +{ + // free previous data. + srs_freep(data); + + // copy new value to data. + _size = (int16_t)v.length(); + + srs_assert(_size > 0); + data = new char[_size]; + memcpy(data, v.data(), _size); +} + int SrsKafkaString::nb_bytes() { return _size == -1? 2 : 2 + _size; @@ -149,11 +163,10 @@ SrsKafkaBytes::SrsKafkaBytes() SrsKafkaBytes::SrsKafkaBytes(const char* v, int nb_v) { - _size = (int16_t)nb_v; + _size = -1; + data = NULL; - srs_assert(_size > 0); - data = new char[_size]; - memcpy(data, v, _size); + set_value(v, nb_v); } SrsKafkaBytes::~SrsKafkaBytes() @@ -171,6 +184,24 @@ bool SrsKafkaBytes::empty() return _size <= 0; } +void SrsKafkaBytes::set_value(string v) +{ + set_value(v.data(), (int)v.length()); +} + +void SrsKafkaBytes::set_value(const char* v, int nb_v) +{ + // free previous data. + srs_freep(data); + + // copy new value to data. + _size = (int16_t)nb_v; + + srs_assert(_size > 0); + data = new char[_size]; + memcpy(data, v, _size); +} + int SrsKafkaBytes::nb_bytes() { return 4 + (_size == -1? 0 : _size); @@ -479,6 +510,32 @@ SrsKafkaRawMessage::~SrsKafkaRawMessage() srs_freep(value); } +int SrsKafkaRawMessage::create(SrsJsonObject* obj) +{ + int ret = ERROR_SUCCESS; + + // current must be 0. + magic_byte = 0; + + // no compression codec. + attributes = 0; + + // dumps the json to string. + value->set_value(obj->dumps()); + + // TODO: FIXME: implements it. + crc = 0; + + message_size = raw_message_size(); + + return ret; +} + +int SrsKafkaRawMessage::raw_message_size() +{ + return 4 + 1 + 1 + key->nb_bytes() + value->nb_bytes(); +} + int SrsKafkaRawMessage::nb_bytes() { return 8 + 4 + 4 + 1 + 1 + key->nb_bytes() + value->nb_bytes(); @@ -1352,6 +1409,42 @@ int SrsKafkaClient::fetch_metadata(string topic, SrsKafkaTopicMetadataResponse** int SrsKafkaClient::write_messages(std::string topic, int32_t partition, vector& msgs) { int ret = ERROR_SUCCESS; + + SrsKafkaProducerRequest* req = new SrsKafkaProducerRequest(); + + // 0 the server will not send any response. + req->required_acks = 0; + // timeout of producer message. + req->timeout = SRS_KAFKA_PRODUCER_MESSAGE_TIMEOUT_MS; + + // create the topic and partition to write message to. + SrsKafkaProducerTopicMessages* topics = new SrsKafkaProducerTopicMessages(); + SrsKafkaProducerPartitionMessages* partitions = new SrsKafkaProducerPartitionMessages(); + + topics->partitions.append(partitions); + req->topics.append(topics); + + topics->topic_name.set_value(topic); + partitions->partition = partition; + + // convert json objects to kafka raw messages. + vector::iterator it; + for (it = msgs.begin(); it != msgs.end(); ++it) { + SrsJsonObject* obj = *it; + SrsKafkaRawMessage* msg = new SrsKafkaRawMessage(); + + if ((ret = msg->create(obj)) != ERROR_SUCCESS) { + srs_freep(msg); + srs_freep(req); + srs_error("kafka write messages failed. ret=%d", ret); + return ret; + } + + partitions->messages.append(msg); + } + + partitions->message_set_size = partitions->messages.nb_bytes(); + // TODO: FIXME: implements it. return ret; } diff --git a/trunk/src/protocol/srs_kafka_stack.hpp b/trunk/src/protocol/srs_kafka_stack.hpp index 6764a6e35..f08a5450c 100644 --- a/trunk/src/protocol/srs_kafka_stack.hpp +++ b/trunk/src/protocol/srs_kafka_stack.hpp @@ -79,6 +79,7 @@ public: virtual bool null(); virtual bool empty(); virtual std::string to_str(); + virtual void set_value(std::string v); // interface ISrsCodec public: virtual int nb_bytes(); @@ -103,6 +104,8 @@ public: public: virtual bool null(); virtual bool empty(); + virtual void set_value(std::string v); + virtual void set_value(const char* v, int nb_v); // interface ISrsCodec public: virtual int nb_bytes(); @@ -531,6 +534,16 @@ public: public: SrsKafkaRawMessage(); virtual ~SrsKafkaRawMessage(); +public: + /** + * create message from json object. + */ + virtual int create(SrsJsonObject* obj); +private: + /** + * get the raw message, bytes after the message_size. + */ + virtual int raw_message_size(); // interface ISrsCodec public: virtual int nb_bytes(); @@ -768,7 +781,7 @@ public: */ class SrsKafkaProducerRequest : public SrsKafkaRequest { -private: +public: /** * This field indicates how many acknowledgements the servers should receive * before responding to the request. If it is 0 the server will not send any From 9a473902530e35b41f6e0acfc4fe364d4d74d2d3 Mon Sep 17 00:00:00 2001 From: winlin Date: Fri, 23 Oct 2015 14:21:10 +0800 Subject: [PATCH 42/46] fix #467, support write log to kafka. 3.0.6 --- README.md | 4 +- trunk/src/core/srs_core.hpp | 2 +- trunk/src/kernel/srs_kernel_buffer.cpp | 20 ++- trunk/src/kernel/srs_kernel_buffer.hpp | 3 + trunk/src/kernel/srs_kernel_ts.cpp | 4 +- trunk/src/kernel/srs_kernel_utility.cpp | 222 ++++++++++++++++-------- trunk/src/kernel/srs_kernel_utility.hpp | 11 +- trunk/src/protocol/srs_kafka_stack.cpp | 71 ++++++-- trunk/src/protocol/srs_kafka_stack.hpp | 5 +- 9 files changed, 243 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index 86c530644..c5596ff13 100755 --- a/README.md +++ b/README.md @@ -259,7 +259,7 @@ The `features`, `compare`, `release` and `performance` of SRS. 1. Support NGINX-RTMP style EXEC, read [#367][bug #367]. 1. Support NGINX-RTMP style dvr control module, read [#459][bug #459]. 1. Support HTTP Security Raw Api, read [#459][bug #459], [#470][bug #470], [#319][bug #319]. -1. [dev]Support Integration with Kafka/Spark Big-Data system, read [#467][bug #467]. +1. Support Integration with Kafka/Spark Big-Data system, read [#467][bug #467]. 1. [plan]Support Origin Cluster for Load Balance and Fault Tolarence, read [#464][bug #464], [RTMP 302][bug #92]. 1. [plan]Support H.265, push RTMP with H.265, delivery in HLS, read [#465][bug #465]. 1. [plan]Support MPEG-DASH, the future streaming protocol, read [#299][bug #299]. @@ -386,6 +386,7 @@ Remark: ### History +* v3.0, 2015-10-23, fix [#467][bug #467], support write log to kafka. 3.0.6 * v3.0, 2015-10-20, fix [#502][bug #502], support snapshot with http-callback or transcoder. 3.0.5 * v3.0, 2015-09-19, support amf0 and json to convert with each other. * v3.0, 2015-09-19, json objects support dumps to string. @@ -1279,6 +1280,7 @@ Winlin [bug #466]: https://github.com/simple-rtmp-server/srs/issues/466 [bug #468]: https://github.com/simple-rtmp-server/srs/issues/468 [bug #502]: https://github.com/simple-rtmp-server/srs/issues/502 +[bug #467]: https://github.com/simple-rtmp-server/srs/issues/467 [bug #xxxxxxx]: https://github.com/simple-rtmp-server/srs/issues/xxxxxxx [r2.0a2]: https://github.com/simple-rtmp-server/srs/releases/tag/v2.0-a2 diff --git a/trunk/src/core/srs_core.hpp b/trunk/src/core/srs_core.hpp index 56e43471b..aa933d018 100644 --- a/trunk/src/core/srs_core.hpp +++ b/trunk/src/core/srs_core.hpp @@ -31,7 +31,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // current release version #define VERSION_MAJOR 3 #define VERSION_MINOR 0 -#define VERSION_REVISION 5 +#define VERSION_REVISION 6 // server info. #define RTMP_SIG_SRS_KEY "SRS" diff --git a/trunk/src/kernel/srs_kernel_buffer.cpp b/trunk/src/kernel/srs_kernel_buffer.cpp index a4b58e197..96a04e8b0 100644 --- a/trunk/src/kernel/srs_kernel_buffer.cpp +++ b/trunk/src/kernel/srs_kernel_buffer.cpp @@ -39,17 +39,27 @@ ISrsCodec::~ISrsCodec() SrsBuffer::SrsBuffer() { - p = bytes = NULL; - nb_bytes = 0; - - // TODO: support both little and big endian. - srs_assert(srs_is_little_endian()); + set_value(NULL, 0); +} + +SrsBuffer::SrsBuffer(char* b, int nb_b) +{ + set_value(b, nb_b); } SrsBuffer::~SrsBuffer() { } +void SrsBuffer::set_value(char* b, int nb_b) +{ + p = bytes = b; + nb_bytes = nb_b; + + // TODO: support both little and big endian. + srs_assert(srs_is_little_endian()); +} + int SrsBuffer::initialize(char* b, int nb) { int ret = ERROR_SUCCESS; diff --git a/trunk/src/kernel/srs_kernel_buffer.hpp b/trunk/src/kernel/srs_kernel_buffer.hpp index 6b85aa537..46ce3f012 100644 --- a/trunk/src/kernel/srs_kernel_buffer.hpp +++ b/trunk/src/kernel/srs_kernel_buffer.hpp @@ -98,7 +98,10 @@ private: int nb_bytes; public: SrsBuffer(); + SrsBuffer(char* b, int nb_b); virtual ~SrsBuffer(); +private: + virtual void set_value(char* b, int nb_b); public: /** * initialize the stream from bytes. diff --git a/trunk/src/kernel/srs_kernel_ts.cpp b/trunk/src/kernel/srs_kernel_ts.cpp index 1b3f144a1..a532ae821 100644 --- a/trunk/src/kernel/srs_kernel_ts.cpp +++ b/trunk/src/kernel/srs_kernel_ts.cpp @@ -2141,7 +2141,7 @@ int SrsTsPayloadPSI::decode(SrsBuffer* stream, SrsTsMessage** /*ppmsg*/) CRC_32 = stream->read_4bytes(); // verify crc32. - int32_t crc32 = srs_crc32(ppat, stream->pos() - pat_pos - 4); + int32_t crc32 = srs_crc32_mpegts(ppat, stream->pos() - pat_pos - 4); if (crc32 != CRC_32) { ret = ERROR_STREAM_CASTER_TS_CRC32; srs_error("ts: verify PSI crc32 failed. ret=%d", ret); @@ -2238,7 +2238,7 @@ int SrsTsPayloadPSI::encode(SrsBuffer* stream) srs_error("ts: mux PSI crc32 failed. ret=%d", ret); return ret; } - CRC_32 = srs_crc32(ppat, stream->pos() - pat_pos); + CRC_32 = srs_crc32_mpegts(ppat, stream->pos() - pat_pos); stream->write_4bytes(CRC_32); return ret; diff --git a/trunk/src/kernel/srs_kernel_utility.cpp b/trunk/src/kernel/srs_kernel_utility.cpp index 13f863979..76cb1ff63 100644 --- a/trunk/src/kernel/srs_kernel_utility.cpp +++ b/trunk/src/kernel/srs_kernel_utility.cpp @@ -535,85 +535,163 @@ bool srs_aac_startswith_adts(SrsBuffer* stream) return true; } -/* - * MPEG2 transport stream (aka DVB) mux - * Copyright (c) 2003 Fabrice Bellard. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ -static const u_int32_t crc_table[256] = { - 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, 0x130476dc, 0x17c56b6b, - 0x1a864db2, 0x1e475005, 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, - 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, 0x4c11db70, 0x48d0c6c7, - 0x4593e01e, 0x4152fda9, 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75, - 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, 0x791d4014, 0x7ddc5da3, - 0x709f7b7a, 0x745e66cd, 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039, - 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5, 0xbe2b5b58, 0xbaea46ef, - 0xb7a96036, 0xb3687d81, 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d, - 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, 0xc7361b4c, 0xc3f706fb, - 0xceb42022, 0xca753d95, 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, - 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, 0x34867077, 0x30476dc0, - 0x3d044b19, 0x39c556ae, 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072, - 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, 0x018aeb13, 0x054bf6a4, - 0x0808d07d, 0x0cc9cdca, 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, - 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02, 0x5e9f46bf, 0x5a5e5b08, - 0x571d7dd1, 0x53dc6066, 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba, - 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, 0xbfa1b04b, 0xbb60adfc, - 0xb6238b25, 0xb2e29692, 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6, - 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a, 0xe0b41de7, 0xe4750050, - 0xe9362689, 0xedf73b3e, 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2, - 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, 0xd5b88683, 0xd1799b34, - 0xdc3abded, 0xd8fba05a, 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637, - 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb, 0x4f040d56, 0x4bc510e1, - 0x46863638, 0x42472b8f, 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53, - 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, 0x36194d42, 0x32d850f5, - 0x3f9b762c, 0x3b5a6b9b, 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff, - 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623, 0xf12f560e, 0xf5ee4bb9, - 0xf8ad6d60, 0xfc6c70d7, 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b, - 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, 0xc423cd6a, 0xc0e2d0dd, - 0xcda1f604, 0xc960ebb3, 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, - 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, 0x9b3660c6, 0x9ff77d71, - 0x92b45ba8, 0x9675461f, 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3, - 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, 0x4e8ee645, 0x4a4ffbf2, - 0x470cdd2b, 0x43cdc09c, 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, - 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24, 0x119b4be9, 0x155a565e, - 0x18197087, 0x1cd86d30, 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec, - 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, 0x2497d08d, 0x2056cd3a, - 0x2d15ebe3, 0x29d4f654, 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, - 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c, 0xe3a1cbc1, 0xe760d676, - 0xea23f0af, 0xeee2ed18, 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4, - 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, 0x9abc8bd5, 0x9e7d9662, - 0x933eb0bb, 0x97ffad0c, 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668, - 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4 -}; - // @see http://www.stmc.edu.hk/~vincent/ffmpeg_0.4.9-pre1/libavformat/mpegtsenc.c -unsigned int mpegts_crc32(const u_int8_t *data, int len) +unsigned int __mpegts_crc32(const u_int8_t *data, int len) { - register int i; - unsigned int crc = 0xffffffff; - - for (i=0; i> 24) ^ *data++) & 0xff]; + /* + * MPEG2 transport stream (aka DVB) mux + * Copyright (c) 2003 Fabrice Bellard. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + static const u_int32_t table[256] = { + 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, 0x130476dc, 0x17c56b6b, + 0x1a864db2, 0x1e475005, 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, + 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, 0x4c11db70, 0x48d0c6c7, + 0x4593e01e, 0x4152fda9, 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75, + 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, 0x791d4014, 0x7ddc5da3, + 0x709f7b7a, 0x745e66cd, 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039, + 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5, 0xbe2b5b58, 0xbaea46ef, + 0xb7a96036, 0xb3687d81, 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d, + 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, 0xc7361b4c, 0xc3f706fb, + 0xceb42022, 0xca753d95, 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, + 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, 0x34867077, 0x30476dc0, + 0x3d044b19, 0x39c556ae, 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072, + 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, 0x018aeb13, 0x054bf6a4, + 0x0808d07d, 0x0cc9cdca, 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, + 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02, 0x5e9f46bf, 0x5a5e5b08, + 0x571d7dd1, 0x53dc6066, 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba, + 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, 0xbfa1b04b, 0xbb60adfc, + 0xb6238b25, 0xb2e29692, 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6, + 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a, 0xe0b41de7, 0xe4750050, + 0xe9362689, 0xedf73b3e, 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2, + 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, 0xd5b88683, 0xd1799b34, + 0xdc3abded, 0xd8fba05a, 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637, + 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb, 0x4f040d56, 0x4bc510e1, + 0x46863638, 0x42472b8f, 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53, + 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, 0x36194d42, 0x32d850f5, + 0x3f9b762c, 0x3b5a6b9b, 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff, + 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623, 0xf12f560e, 0xf5ee4bb9, + 0xf8ad6d60, 0xfc6c70d7, 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b, + 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, 0xc423cd6a, 0xc0e2d0dd, + 0xcda1f604, 0xc960ebb3, 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, + 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, 0x9b3660c6, 0x9ff77d71, + 0x92b45ba8, 0x9675461f, 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3, + 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, 0x4e8ee645, 0x4a4ffbf2, + 0x470cdd2b, 0x43cdc09c, 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, + 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24, 0x119b4be9, 0x155a565e, + 0x18197087, 0x1cd86d30, 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec, + 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, 0x2497d08d, 0x2056cd3a, + 0x2d15ebe3, 0x29d4f654, 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, + 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c, 0xe3a1cbc1, 0xe760d676, + 0xea23f0af, 0xeee2ed18, 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4, + 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, 0x9abc8bd5, 0x9e7d9662, + 0x933eb0bb, 0x97ffad0c, 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668, + 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4 + }; + + u_int32_t crc = 0xffffffff; + + for (int i=0; i> 24) ^ *data++) & 0xff]; + } return crc; } -u_int32_t srs_crc32(const void* buf, int size) +// @see https://github.com/ETrun/crc32/blob/master/crc32.c +u_int32_t __crc32_ieee(u_int32_t init, const u_int8_t* buf, size_t nb_buf) +{ + /*----------------------------------------------------------------------------*\ + * CRC-32 version 2.0.0 by Craig Bruce, 2006-04-29. + * + * This program generates the CRC-32 values for the files named in the + * command-line arguments. These are the same CRC-32 values used by GZIP, + * PKZIP, and ZMODEM. The Crc32_ComputeBuf() can also be detached and + * used independently. + * + * THIS PROGRAM IS PUBLIC-DOMAIN SOFTWARE. + * + * Based on the byte-oriented implementation "File Verification Using CRC" + * by Mark R. Nelson in Dr. Dobb's Journal, May 1992, pp. 64-67. + * + * v1.0.0: original release. + * v1.0.1: fixed printf formats. + * v1.0.2: fixed something else. + * v1.0.3: replaced CRC constant table by generator function. + * v1.0.4: reformatted code, made ANSI C. 1994-12-05. + * v2.0.0: rewrote to use memory buffer & static table, 2006-04-29. + * v2.1.0: modified by Nico, 2013-04-20 + \*----------------------------------------------------------------------------*/ + static const u_int32_t table[256] = { + 0x00000000,0x77073096,0xEE0E612C,0x990951BA,0x076DC419,0x706AF48F,0xE963A535, + 0x9E6495A3,0x0EDB8832,0x79DCB8A4,0xE0D5E91E,0x97D2D988,0x09B64C2B,0x7EB17CBD, + 0xE7B82D07,0x90BF1D91,0x1DB71064,0x6AB020F2,0xF3B97148,0x84BE41DE,0x1ADAD47D, + 0x6DDDE4EB,0xF4D4B551,0x83D385C7,0x136C9856,0x646BA8C0,0xFD62F97A,0x8A65C9EC, + 0x14015C4F,0x63066CD9,0xFA0F3D63,0x8D080DF5,0x3B6E20C8,0x4C69105E,0xD56041E4, + 0xA2677172,0x3C03E4D1,0x4B04D447,0xD20D85FD,0xA50AB56B,0x35B5A8FA,0x42B2986C, + 0xDBBBC9D6,0xACBCF940,0x32D86CE3,0x45DF5C75,0xDCD60DCF,0xABD13D59,0x26D930AC, + 0x51DE003A,0xC8D75180,0xBFD06116,0x21B4F4B5,0x56B3C423,0xCFBA9599,0xB8BDA50F, + 0x2802B89E,0x5F058808,0xC60CD9B2,0xB10BE924,0x2F6F7C87,0x58684C11,0xC1611DAB, + 0xB6662D3D,0x76DC4190,0x01DB7106,0x98D220BC,0xEFD5102A,0x71B18589,0x06B6B51F, + 0x9FBFE4A5,0xE8B8D433,0x7807C9A2,0x0F00F934,0x9609A88E,0xE10E9818,0x7F6A0DBB, + 0x086D3D2D,0x91646C97,0xE6635C01,0x6B6B51F4,0x1C6C6162,0x856530D8,0xF262004E, + 0x6C0695ED,0x1B01A57B,0x8208F4C1,0xF50FC457,0x65B0D9C6,0x12B7E950,0x8BBEB8EA, + 0xFCB9887C,0x62DD1DDF,0x15DA2D49,0x8CD37CF3,0xFBD44C65,0x4DB26158,0x3AB551CE, + 0xA3BC0074,0xD4BB30E2,0x4ADFA541,0x3DD895D7,0xA4D1C46D,0xD3D6F4FB,0x4369E96A, + 0x346ED9FC,0xAD678846,0xDA60B8D0,0x44042D73,0x33031DE5,0xAA0A4C5F,0xDD0D7CC9, + 0x5005713C,0x270241AA,0xBE0B1010,0xC90C2086,0x5768B525,0x206F85B3,0xB966D409, + 0xCE61E49F,0x5EDEF90E,0x29D9C998,0xB0D09822,0xC7D7A8B4,0x59B33D17,0x2EB40D81, + 0xB7BD5C3B,0xC0BA6CAD,0xEDB88320,0x9ABFB3B6,0x03B6E20C,0x74B1D29A,0xEAD54739, + 0x9DD277AF,0x04DB2615,0x73DC1683,0xE3630B12,0x94643B84,0x0D6D6A3E,0x7A6A5AA8, + 0xE40ECF0B,0x9309FF9D,0x0A00AE27,0x7D079EB1,0xF00F9344,0x8708A3D2,0x1E01F268, + 0x6906C2FE,0xF762575D,0x806567CB,0x196C3671,0x6E6B06E7,0xFED41B76,0x89D32BE0, + 0x10DA7A5A,0x67DD4ACC,0xF9B9DF6F,0x8EBEEFF9,0x17B7BE43,0x60B08ED5,0xD6D6A3E8, + 0xA1D1937E,0x38D8C2C4,0x4FDFF252,0xD1BB67F1,0xA6BC5767,0x3FB506DD,0x48B2364B, + 0xD80D2BDA,0xAF0A1B4C,0x36034AF6,0x41047A60,0xDF60EFC3,0xA867DF55,0x316E8EEF, + 0x4669BE79,0xCB61B38C,0xBC66831A,0x256FD2A0,0x5268E236,0xCC0C7795,0xBB0B4703, + 0x220216B9,0x5505262F,0xC5BA3BBE,0xB2BD0B28,0x2BB45A92,0x5CB36A04,0xC2D7FFA7, + 0xB5D0CF31,0x2CD99E8B,0x5BDEAE1D,0x9B64C2B0,0xEC63F226,0x756AA39C,0x026D930A, + 0x9C0906A9,0xEB0E363F,0x72076785,0x05005713,0x95BF4A82,0xE2B87A14,0x7BB12BAE, + 0x0CB61B38,0x92D28E9B,0xE5D5BE0D,0x7CDCEFB7,0x0BDBDF21,0x86D3D2D4,0xF1D4E242, + 0x68DDB3F8,0x1FDA836E,0x81BE16CD,0xF6B9265B,0x6FB077E1,0x18B74777,0x88085AE6, + 0xFF0F6A70,0x66063BCA,0x11010B5C,0x8F659EFF,0xF862AE69,0x616BFFD3,0x166CCF45, + 0xA00AE278,0xD70DD2EE,0x4E048354,0x3903B3C2,0xA7672661,0xD06016F7,0x4969474D, + 0x3E6E77DB,0xAED16A4A,0xD9D65ADC,0x40DF0B66,0x37D83BF0,0xA9BCAE53,0xDEBB9EC5, + 0x47B2CF7F,0x30B5FFE9,0xBDBDF21C,0xCABAC28A,0x53B39330,0x24B4A3A6,0xBAD03605, + 0xCDD70693,0x54DE5729,0x23D967BF,0xB3667A2E,0xC4614AB8,0x5D681B02,0x2A6F2B94, + 0xB40BBE37,0xC30C8EA1,0x5A05DF1B,0x2D02EF8D + }; + + u_int32_t crc = init ^ 0xFFFFFFFF; + + for (int i = 0; i < nb_buf; i++) { + crc = table[(crc ^ buf[i]) & 0xff] ^ (crc >> 8); + } + + return crc^0xFFFFFFFF; +} + +u_int32_t srs_crc32_mpegts(const void* buf, int size) +{ + return __mpegts_crc32((const u_int8_t*)buf, size); +} + +u_int32_t srs_crc32_ieee(const void* buf, int size, u_int32_t previous) { - return mpegts_crc32((const u_int8_t*)buf, size); + return __crc32_ieee(previous, (const u_int8_t*)buf, size); } /* diff --git a/trunk/src/kernel/srs_kernel_utility.hpp b/trunk/src/kernel/srs_kernel_utility.hpp index 4a5bc5cb4..efaf5e876 100644 --- a/trunk/src/kernel/srs_kernel_utility.hpp +++ b/trunk/src/kernel/srs_kernel_utility.hpp @@ -122,9 +122,14 @@ extern bool srs_avc_startswith_annexb(SrsBuffer* stream, int* pnb_start_code = N extern bool srs_aac_startswith_adts(SrsBuffer* stream); /** -* cacl the crc32 of bytes in buf. -*/ -extern u_int32_t srs_crc32(const void* buf, int size); +* cacl the crc32 of bytes in buf, for ffmpeg. + */ +extern u_int32_t srs_crc32_mpegts(const void* buf, int size); + +/** + * calc the crc32 of bytes in buf by IEEE, for zip. + */ +extern u_int32_t srs_crc32_ieee(const void* buf, int size, u_int32_t previous = 0); /** * Decode a base64-encoded string. diff --git a/trunk/src/protocol/srs_kafka_stack.cpp b/trunk/src/protocol/srs_kafka_stack.cpp index 6ecda777a..be5dc365e 100644 --- a/trunk/src/protocol/srs_kafka_stack.cpp +++ b/trunk/src/protocol/srs_kafka_stack.cpp @@ -158,20 +158,30 @@ int SrsKafkaString::decode(SrsBuffer* buf) SrsKafkaBytes::SrsKafkaBytes() { _size = -1; - data = NULL; + _data = NULL; } SrsKafkaBytes::SrsKafkaBytes(const char* v, int nb_v) { _size = -1; - data = NULL; + _data = NULL; set_value(v, nb_v); } SrsKafkaBytes::~SrsKafkaBytes() { - srs_freep(data); + srs_freep(_data); +} + +char* SrsKafkaBytes::data() +{ + return _data; +} + +int SrsKafkaBytes::size() +{ + return _size; } bool SrsKafkaBytes::null() @@ -192,14 +202,29 @@ void SrsKafkaBytes::set_value(string v) void SrsKafkaBytes::set_value(const char* v, int nb_v) { // free previous data. - srs_freep(data); + srs_freep(_data); // copy new value to data. _size = (int16_t)nb_v; srs_assert(_size > 0); - data = new char[_size]; - memcpy(data, v, _size); + _data = new char[_size]; + memcpy(_data, v, _size); +} + +u_int32_t SrsKafkaBytes::crc32(u_int32_t previous) +{ + char bsize[4]; + SrsBuffer(bsize, 4).write_4bytes(_size); + + if (_size <= 0) { + return srs_crc32_ieee(bsize, 4, previous); + } + + u_int32_t crc = srs_crc32_ieee(bsize, 4, previous); + crc = srs_crc32_ieee(_data, _size, crc); + + return crc; } int SrsKafkaBytes::nb_bytes() @@ -227,7 +252,7 @@ int SrsKafkaBytes::encode(SrsBuffer* buf) srs_error("kafka encode bytes data failed. ret=%d", ret); return ret; } - buf->write_bytes(data, _size); + buf->write_bytes(_data, _size); return ret; } @@ -255,9 +280,9 @@ int SrsKafkaBytes::decode(SrsBuffer* buf) return ret; } - srs_freep(data); - data = new char[_size]; - buf->read_bytes(data, _size); + srs_freep(_data); + _data = new char[_size]; + buf->read_bytes(_data, _size); return ret; } @@ -523,8 +548,13 @@ int SrsKafkaRawMessage::create(SrsJsonObject* obj) // dumps the json to string. value->set_value(obj->dumps()); - // TODO: FIXME: implements it. - crc = 0; + // crc32 message. + crc = srs_crc32_ieee(&magic_byte, 1); + crc = srs_crc32_ieee(&attributes, 1, crc); + crc = key->crc32(crc); + crc = value->crc32(crc); + + srs_info("crc32 message is %#x", crc); message_size = raw_message_size(); @@ -1143,13 +1173,17 @@ SrsKafkaProducerRequest::~SrsKafkaProducerRequest() int SrsKafkaProducerRequest::nb_bytes() { - return 2 + 4 + topics.nb_bytes(); + return SrsKafkaRequest::nb_bytes() + 2 + 4 + topics.nb_bytes(); } int SrsKafkaProducerRequest::encode(SrsBuffer* buf) { int ret = ERROR_SUCCESS; + if ((ret = SrsKafkaRequest::encode(buf)) != ERROR_SUCCESS) { + return ret; + } + if (!buf->require(2 + 4)) { ret = ERROR_KAFKA_CODEC_PRODUCER; srs_error("kafka encode producer failed. ret=%d", ret); @@ -1169,6 +1203,10 @@ int SrsKafkaProducerRequest::decode(SrsBuffer* buf) { int ret = ERROR_SUCCESS; + if ((ret = SrsKafkaRequest::decode(buf)) != ERROR_SUCCESS) { + return ret; + } + if (!buf->require(2 + 4)) { ret = ERROR_KAFKA_CODEC_PRODUCER; srs_error("kafka decode producer failed. ret=%d", ret); @@ -1445,7 +1483,12 @@ int SrsKafkaClient::write_messages(std::string topic, int32_t partition, vector< partitions->message_set_size = partitions->messages.nb_bytes(); - // TODO: FIXME: implements it. + // write to kafka cluster. + if ((ret = protocol->send_and_free_message(req)) != ERROR_SUCCESS) { + srs_error("kafka write producer message failed. ret=%d", ret); + return ret; + } + return ret; } diff --git a/trunk/src/protocol/srs_kafka_stack.hpp b/trunk/src/protocol/srs_kafka_stack.hpp index f08a5450c..773ac0de0 100644 --- a/trunk/src/protocol/srs_kafka_stack.hpp +++ b/trunk/src/protocol/srs_kafka_stack.hpp @@ -96,16 +96,19 @@ class SrsKafkaBytes : public ISrsCodec { private: int32_t _size; - char* data; + char* _data; public: SrsKafkaBytes(); SrsKafkaBytes(const char* v, int nb_v); virtual ~SrsKafkaBytes(); public: + virtual char* data(); + virtual int size(); virtual bool null(); virtual bool empty(); virtual void set_value(std::string v); virtual void set_value(const char* v, int nb_v); + virtual u_int32_t crc32(u_int32_t previous); // interface ISrsCodec public: virtual int nb_bytes(); From 71451878c987402642432bd4d18c1d3400fe5611 Mon Sep 17 00:00:00 2001 From: winlin Date: Fri, 23 Oct 2015 14:30:16 +0800 Subject: [PATCH 43/46] refine kafka, simplify code. --- trunk/src/app/srs_app_kafka.cpp | 40 +++++++++++++++------------------ trunk/src/app/srs_app_kafka.hpp | 16 ++++--------- 2 files changed, 22 insertions(+), 34 deletions(-) diff --git a/trunk/src/app/srs_app_kafka.cpp b/trunk/src/app/srs_app_kafka.cpp index 0f75217f0..287751585 100644 --- a/trunk/src/app/srs_app_kafka.cpp +++ b/trunk/src/app/srs_app_kafka.cpp @@ -178,41 +178,31 @@ int SrsKafkaPartition::flush(SrsKafkaPartitionCache* pc) return kafka->write_messages(topic, id, *pc); } -SrsKafkaMessage::SrsKafkaMessage(int k) +SrsKafkaMessage::SrsKafkaMessage(SrsKafkaProducer* p, int k, SrsJsonObject* j) { + producer = p; key = k; + obj = j; } SrsKafkaMessage::~SrsKafkaMessage() { + srs_freep(obj); } -SrsKafkaMessageOnClient::SrsKafkaMessageOnClient(SrsKafkaProducer* p, int k, SrsListenerType t, string i) - : SrsKafkaMessage(k) -{ - producer = p; - type = t; - ip = i; -} - -SrsKafkaMessageOnClient::~SrsKafkaMessageOnClient() -{ -} - -int SrsKafkaMessageOnClient::call() +int SrsKafkaMessage::call() { - SrsJsonObject* obj = SrsJsonAny::object(); + int ret = producer->send(key, obj); - obj->set("msg", SrsJsonAny::str("accept")); - obj->set("type", SrsJsonAny::integer(type)); - obj->set("ip", SrsJsonAny::str(ip.c_str())); + // the obj is manged by producer now. + obj = NULL; - return producer->send(key, obj); + return ret; } -string SrsKafkaMessageOnClient::to_string() +string SrsKafkaMessage::to_string() { - return ip; + return "kafka"; } SrsKafkaCache::SrsKafkaCache() @@ -393,7 +383,13 @@ void SrsKafkaProducer::stop() int SrsKafkaProducer::on_client(int key, SrsListenerType type, string ip) { - return worker->execute(new SrsKafkaMessageOnClient(this, key, type, ip)); + SrsJsonObject* obj = SrsJsonAny::object(); + + obj->set("msg", SrsJsonAny::str("accept")); + obj->set("type", SrsJsonAny::integer(type)); + obj->set("ip", SrsJsonAny::str(ip.c_str())); + + return worker->execute(new SrsKafkaMessage(this, key, obj)); } int SrsKafkaProducer::send(int key, SrsJsonObject* obj) diff --git a/trunk/src/app/srs_app_kafka.hpp b/trunk/src/app/srs_app_kafka.hpp index 457470f0e..77a4c217c 100644 --- a/trunk/src/app/srs_app_kafka.hpp +++ b/trunk/src/app/srs_app_kafka.hpp @@ -80,21 +80,13 @@ public: */ class SrsKafkaMessage : public ISrsAsyncCallTask { -protected: +private: + SrsKafkaProducer* producer; int key; + SrsJsonObject* obj; public: - SrsKafkaMessage(int k); + SrsKafkaMessage(SrsKafkaProducer* p, int k, SrsJsonObject* j); virtual ~SrsKafkaMessage(); -}; -struct SrsKafkaMessageOnClient : public SrsKafkaMessage -{ -public: - SrsKafkaProducer* producer; - SrsListenerType type; - std::string ip; -public: - SrsKafkaMessageOnClient(SrsKafkaProducer* p, int k, SrsListenerType t, std::string i); - virtual ~SrsKafkaMessageOnClient(); // interface ISrsAsyncCallTask public: virtual int call(); From d2ca51ac50afd9cbb618072015d9ac33de737267 Mon Sep 17 00:00:00 2001 From: winlin Date: Fri, 23 Oct 2015 14:36:55 +0800 Subject: [PATCH 44/46] notify kafka when client close --- trunk/src/app/srs_app_kafka.cpp | 23 +++++++++++++++++++++++ trunk/src/app/srs_app_kafka.hpp | 12 ++++++++---- trunk/src/app/srs_app_rtmp_conn.cpp | 7 +++++++ 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/trunk/src/app/srs_app_kafka.cpp b/trunk/src/app/srs_app_kafka.cpp index 287751585..b4c96129a 100644 --- a/trunk/src/app/srs_app_kafka.cpp +++ b/trunk/src/app/srs_app_kafka.cpp @@ -383,6 +383,13 @@ void SrsKafkaProducer::stop() int SrsKafkaProducer::on_client(int key, SrsListenerType type, string ip) { + int ret = ERROR_SUCCESS; + + bool enabled = _srs_config->get_kafka_enabled(); + if (!enabled) { + return ret; + } + SrsJsonObject* obj = SrsJsonAny::object(); obj->set("msg", SrsJsonAny::str("accept")); @@ -392,6 +399,22 @@ int SrsKafkaProducer::on_client(int key, SrsListenerType type, string ip) return worker->execute(new SrsKafkaMessage(this, key, obj)); } +int SrsKafkaProducer::on_close(int key) +{ + int ret = ERROR_SUCCESS; + + bool enabled = _srs_config->get_kafka_enabled(); + if (!enabled) { + return ret; + } + + SrsJsonObject* obj = SrsJsonAny::object(); + + obj->set("msg", SrsJsonAny::str("close")); + + return worker->execute(new SrsKafkaMessage(this, key, obj)); +} + int SrsKafkaProducer::send(int key, SrsJsonObject* obj) { int ret = ERROR_SUCCESS; diff --git a/trunk/src/app/srs_app_kafka.hpp b/trunk/src/app/srs_app_kafka.hpp index 77a4c217c..b42f1c9d4 100644 --- a/trunk/src/app/srs_app_kafka.hpp +++ b/trunk/src/app/srs_app_kafka.hpp @@ -137,11 +137,16 @@ public: public: /** * when got any client connect to SRS, notify kafka. - * @param key the partition map key, a id or hash. + * @param key the partition map key, the client id or hash(ip). * @param type the type of client. * @param ip the peer ip of client. */ virtual int on_client(int key, SrsListenerType type, std::string ip) = 0; + /** + * when client close or disconnect for error. + * @param key the partition map key, the client id or hash(ip). + */ + virtual int on_close(int key) = 0; }; /** @@ -168,11 +173,10 @@ public: virtual int initialize(); virtual int start(); virtual void stop(); +// interface ISrsKafkaCluster public: - /** - * when got any client connect to SRS, notify kafka. - */ virtual int on_client(int key, SrsListenerType type, std::string ip); + virtual int on_close(int key); // for worker to call task to send object. public: /** diff --git a/trunk/src/app/srs_app_rtmp_conn.cpp b/trunk/src/app/srs_app_rtmp_conn.cpp index 0ab1b58df..777f50c61 100644 --- a/trunk/src/app/srs_app_rtmp_conn.cpp +++ b/trunk/src/app/srs_app_rtmp_conn.cpp @@ -1555,6 +1555,13 @@ int SrsRtmpConn::on_disconnect() int ret = ERROR_SUCCESS; http_hooks_on_close(); + +#ifdef SRS_AUTO_KAFKA + if ((ret = kafka->on_close(srs_id())) != ERROR_SUCCESS) { + srs_error("notify kafka failed. ret=%d", ret); + return ret; + } +#endif // TODO: implements it. From cbe4252b4d08f3e1fdf0fa14ed9ffe275d4780dd Mon Sep 17 00:00:00 2001 From: winlin Date: Fri, 23 Oct 2015 14:37:34 +0800 Subject: [PATCH 45/46] refine code. --- trunk/src/app/srs_app_kafka.cpp | 60 ++++++++++++++++----------------- trunk/src/app/srs_app_kafka.hpp | 10 +++--- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/trunk/src/app/srs_app_kafka.cpp b/trunk/src/app/srs_app_kafka.cpp index b4c96129a..ed63d9748 100644 --- a/trunk/src/app/srs_app_kafka.cpp +++ b/trunk/src/app/srs_app_kafka.cpp @@ -381,25 +381,37 @@ void SrsKafkaProducer::stop() worker->stop(); } -int SrsKafkaProducer::on_client(int key, SrsListenerType type, string ip) +int SrsKafkaProducer::send(int key, SrsJsonObject* obj) { int ret = ERROR_SUCCESS; - bool enabled = _srs_config->get_kafka_enabled(); - if (!enabled) { + // cache the json object. + cache->append(key, obj); + + // too few messages, ignore. + if (cache->size() < SRS_KAFKA_PRODUCER_AGGREGATE_SIZE) { return ret; } - SrsJsonObject* obj = SrsJsonAny::object(); + // too many messages, warn user. + if (cache->size() > SRS_KAFKA_PRODUCER_AGGREGATE_SIZE * 10) { + srs_warn("kafka cache too many messages: %d", cache->size()); + } - obj->set("msg", SrsJsonAny::str("accept")); - obj->set("type", SrsJsonAny::integer(type)); - obj->set("ip", SrsJsonAny::str(ip.c_str())); + // sync with backgound metadata worker. + st_mutex_lock(lock); - return worker->execute(new SrsKafkaMessage(this, key, obj)); + // flush message when metadata is ok. + if (metadata_ok) { + ret = flush(); + } + + st_mutex_unlock(lock); + + return ret; } -int SrsKafkaProducer::on_close(int key) +int SrsKafkaProducer::on_client(int key, SrsListenerType type, string ip) { int ret = ERROR_SUCCESS; @@ -410,39 +422,27 @@ int SrsKafkaProducer::on_close(int key) SrsJsonObject* obj = SrsJsonAny::object(); - obj->set("msg", SrsJsonAny::str("close")); + obj->set("msg", SrsJsonAny::str("accept")); + obj->set("type", SrsJsonAny::integer(type)); + obj->set("ip", SrsJsonAny::str(ip.c_str())); return worker->execute(new SrsKafkaMessage(this, key, obj)); } -int SrsKafkaProducer::send(int key, SrsJsonObject* obj) +int SrsKafkaProducer::on_close(int key) { int ret = ERROR_SUCCESS; - // cache the json object. - cache->append(key, obj); - - // too few messages, ignore. - if (cache->size() < SRS_KAFKA_PRODUCER_AGGREGATE_SIZE) { + bool enabled = _srs_config->get_kafka_enabled(); + if (!enabled) { return ret; } - // too many messages, warn user. - if (cache->size() > SRS_KAFKA_PRODUCER_AGGREGATE_SIZE * 10) { - srs_warn("kafka cache too many messages: %d", cache->size()); - } - - // sync with backgound metadata worker. - st_mutex_lock(lock); - - // flush message when metadata is ok. - if (metadata_ok) { - ret = flush(); - } + SrsJsonObject* obj = SrsJsonAny::object(); - st_mutex_unlock(lock); + obj->set("msg", SrsJsonAny::str("close")); - return ret; + return worker->execute(new SrsKafkaMessage(this, key, obj)); } int SrsKafkaProducer::cycle() diff --git a/trunk/src/app/srs_app_kafka.hpp b/trunk/src/app/srs_app_kafka.hpp index b42f1c9d4..3b4129262 100644 --- a/trunk/src/app/srs_app_kafka.hpp +++ b/trunk/src/app/srs_app_kafka.hpp @@ -173,11 +173,7 @@ public: virtual int initialize(); virtual int start(); virtual void stop(); -// interface ISrsKafkaCluster -public: - virtual int on_client(int key, SrsListenerType type, std::string ip); - virtual int on_close(int key); -// for worker to call task to send object. +// internal: for worker to call task to send object. public: /** * send json object to kafka cluster. @@ -186,6 +182,10 @@ public: * @param obj the json object; user must never free it again. */ virtual int send(int key, SrsJsonObject* obj); +// interface ISrsKafkaCluster +public: + virtual int on_client(int key, SrsListenerType type, std::string ip); + virtual int on_close(int key); // interface ISrsReusableThreadHandler public: virtual int cycle(); From b8f2ba4f0d7743f6d65962c4f30aea5a176244c9 Mon Sep 17 00:00:00 2001 From: winlin Date: Fri, 23 Oct 2015 14:40:14 +0800 Subject: [PATCH 46/46] complete kafka --- trunk/src/app/srs_app_kafka.cpp | 7 ++----- trunk/src/app/srs_app_kafka.hpp | 2 ++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/trunk/src/app/srs_app_kafka.cpp b/trunk/src/app/srs_app_kafka.cpp index ed63d9748..89491f742 100644 --- a/trunk/src/app/srs_app_kafka.cpp +++ b/trunk/src/app/srs_app_kafka.cpp @@ -352,7 +352,8 @@ int SrsKafkaProducer::initialize() { int ret = ERROR_SUCCESS; - srs_info("initialize kafka producer ok."); + enabled = _srs_config->get_kafka_enabled(); + srs_info("initialize kafka ok, enabled=%d.", enabled); return ret; } @@ -415,7 +416,6 @@ int SrsKafkaProducer::on_client(int key, SrsListenerType type, string ip) { int ret = ERROR_SUCCESS; - bool enabled = _srs_config->get_kafka_enabled(); if (!enabled) { return ret; } @@ -433,7 +433,6 @@ int SrsKafkaProducer::on_close(int key) { int ret = ERROR_SUCCESS; - bool enabled = _srs_config->get_kafka_enabled(); if (!enabled) { return ret; } @@ -494,7 +493,6 @@ int SrsKafkaProducer::do_cycle() int ret = ERROR_SUCCESS; // ignore when disabled. - bool enabled = _srs_config->get_kafka_enabled(); if (!enabled) { return ret; } @@ -513,7 +511,6 @@ int SrsKafkaProducer::request_metadata() int ret = ERROR_SUCCESS; // ignore when disabled. - bool enabled = _srs_config->get_kafka_enabled(); if (!enabled) { return ret; } diff --git a/trunk/src/app/srs_app_kafka.hpp b/trunk/src/app/srs_app_kafka.hpp index 3b4129262..e7508b249 100644 --- a/trunk/src/app/srs_app_kafka.hpp +++ b/trunk/src/app/srs_app_kafka.hpp @@ -155,6 +155,8 @@ public: class SrsKafkaProducer : virtual public ISrsReusableThreadHandler, virtual public ISrsKafkaCluster { private: + // TODO: FIXME: support reload. + bool enabled; st_mutex_t lock; SrsReusableThread* pthread; private: