Kernel: Support grab backtrace stack when assert fail. v5.0.80

trunk/auto/auto_headers.sh

@@ -68,6 +68,12 @@ else
     srs_undefine_macro "SRS_CXX14" $SRS_AUTO_HEADERS_H
 fi
 
+if [ $SRS_BACKTRACE = YES ]; then
+    srs_define_macro "SRS_BACKTRACE" $SRS_AUTO_HEADERS_H
+else
+    srs_undefine_macro "SRS_BACKTRACE" $SRS_AUTO_HEADERS_H
+fi
+
 if [ $SRS_RTC = YES ]; then
     srs_define_macro "SRS_RTC" $SRS_AUTO_HEADERS_H
 else

trunk/auto/options.sh

@@ -9,6 +9,7 @@ SRS_RTC=YES
 SRS_GB28181=NO
 SRS_CXX11=YES
 SRS_CXX14=NO
+SRS_BACKTRACE=YES
 SRS_NGINX=NO
 SRS_UTEST=NO
 # Always enable the bellow features.
@@ -128,6 +129,7 @@ Features:
   --gb28181=on|off          Whether build the GB28181. Default: $(value2switch $SRS_GB28181)
   --cxx11=on|off            Whether enable the C++11. Default: $(value2switch $SRS_CXX11)
   --cxx14=on|off            Whether enable the C++14. Default: $(value2switch $SRS_CXX14)
+  --backtrace=on|off        Whether show backtrace when crashing. Default: $(value2switch $SRS_BACKTRACE)
   --ffmpeg-fit=on|off       Whether enable the FFmpeg fit(source code). Default: $(value2switch $SRS_FFMPEG_FIT)
 
   --prefix=<path>           The absolute installation path. Default: $SRS_PREFIX
@@ -298,6 +300,7 @@ function parse_user_option() {
 
         --cxx11)                        SRS_CXX11=$(switch2value $value) ;;
         --cxx14)                        SRS_CXX14=$(switch2value $value) ;;
+        --backtrace)                    SRS_BACKTRACE=$(switch2value $value) ;;
 
         --with-clean)                   SRS_CLEAN=YES               ;;
         --without-clean)                SRS_CLEAN=NO                ;;
@@ -525,6 +528,7 @@ function regenerate_options() {
     SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --simulator=$(value2switch $SRS_SIMULATOR)"
     SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --cxx11=$(value2switch $SRS_CXX11)"
     SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --cxx14=$(value2switch $SRS_CXX14)"
+    SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --backtrace=$(value2switch $SRS_BACKTRACE)"
     SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --ffmpeg-fit=$(value2switch $SRS_FFMPEG_FIT)"
     SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --nasm=$(value2switch $SRS_NASM)"
     SRS_AUTO_CONFIGURE="${SRS_AUTO_CONFIGURE} --srtp-nasm=$(value2switch $SRS_SRTP_ASM)"

trunk/configure

@@ -193,6 +193,11 @@ if [[ $SRS_OSX != YES && $SRS_RTC == YES && $SRS_FFMPEG_FIT == YES ]]; then
     SrsLinkOptions="${SrsLinkOptions} -lrt";
 fi
 
+# The backtrace symbol option.
+if [[ $SRS_BACKTRACE == YES ]]; then
+    SrsLinkOptions="${SrsLinkOptions} -rdynamic";
+fi
+
 #####################################################################################
 # Modules, compile each module, then link to binary
 #

trunk/doc/CHANGELOG.md

@@ -7,6 +7,7 @@ The changelog for SRS.
 
 ## SRS 5.0 Changelog
 
+* v5.0, 2022-10-21, Kernel: Support grab backtrace stack when assert fail. v5.0.80
 * v5.0, 2022-10-21, ST: Refine tools and CMakeLists.txt. Add backtrace example. v5.0.79
 * v5.0, 2022-10-10, For [#2901](https://github.com/ossrs/srs/issues/2901): Edge: Fast disconnect and reconnect. v5.0.78
 * v5.0, 2022-10-09, Fix [#3198](https://github.com/ossrs/srs/issues/3198): SRT: Support PUSH SRT by IP and optional port. v5.0.76

trunk/ide/srs_clion/CMakeLists.txt

@@ -87,6 +87,7 @@ ADD_EXECUTABLE(srs ${SRS_SOURCE_FILES})
 TARGET_LINK_LIBRARIES(srs dl)
 TARGET_LINK_LIBRARIES(srs ${DEPS_LIBS})
 TARGET_LINK_LIBRARIES(srs -ldl -pthread)
+TARGET_LINK_LIBRARIES(srs -rdynamic)
 
 ###########################################################
 # For utest.

trunk/src/app/srs_app_rtmp_conn.cpp

@@ -66,7 +66,7 @@ SrsSimpleRtmpClient::~SrsSimpleRtmpClient()
 srs_error_t SrsSimpleRtmpClient::connect_app()
 {
     std::vector<SrsIPAddress*>& ips = srs_get_local_ips();
-    assert(_srs_config->get_stats_network() < (int)ips.size());
+    srs_assert(_srs_config->get_stats_network() < (int)ips.size());
     SrsIPAddress* local_ip = ips[_srs_config->get_stats_network()];
     
     bool debug_srs_upnode = _srs_config->get_debug_srs_upnode(req->vhost);

trunk/src/core/srs_core_platform.hpp

@@ -28,11 +28,6 @@
 #include <inttypes.h>
 #endif
 
-#include <assert.h>
-#ifndef srs_assert
-#define srs_assert(expression) assert(expression)
-#endif
-
 #include <stddef.h>
 #include <sys/types.h>
 

trunk/src/core/srs_core_version5.hpp

@@ -9,6 +9,6 @@
 
 #define VERSION_MAJOR       5
 #define VERSION_MINOR       0
-#define VERSION_REVISION    79
+#define VERSION_REVISION    80
 
 #endif

trunk/src/kernel/srs_kernel_balance.cpp

@@ -6,6 +6,8 @@
 
 #include <srs_kernel_balance.hpp>
 
+#include <srs_kernel_error.hpp>
+
 using namespace std;
 
 SrsLbRoundRobin::SrsLbRoundRobin()

trunk/src/kernel/srs_kernel_error.cpp

@@ -12,10 +12,112 @@
 #include <sstream>
 #include <stdarg.h>
 #include <unistd.h>
+#include <assert.h>
 
 #include <map>
 using namespace std;
 
+#if defined(SRS_BACKTRACE) && defined(__linux)
+#include <execinfo.h>
+#include <dlfcn.h>
+#include <string.h>
+#include <stdlib.h>
+
+void* parse_symbol_offset(char* frame)
+{
+    char* p = NULL;
+    char* p_symbol = NULL;
+    int nn_symbol = 0;
+    char* p_offset = NULL;
+    int nn_offset = 0;
+
+    // Read symbol and offset, for example:
+    //      /tools/backtrace(foo+0x1820) [0x555555555820]
+    for (p = frame; *p; p++) {
+        if (*p == '(') {
+            p_symbol = p + 1;
+        } else if (*p == '+') {
+            if (p_symbol) nn_symbol = p - p_symbol;
+            p_offset = p + 1;
+        } else if (*p == ')') {
+            if (p_offset) nn_offset = p - p_offset;
+        }
+    }
+    if (!nn_symbol && !nn_offset) {
+        return NULL;
+    }
+
+    // Convert offset(0x1820) to pointer, such as 0x1820.
+    char tmp[128];
+    if (!nn_offset || nn_offset >= (int)sizeof(tmp)) {
+        return NULL;
+    }
+
+    int r0 = EOF;
+    void* offset = NULL;
+    tmp[nn_offset] = 0;
+    if ((r0 = sscanf(strncpy(tmp, p_offset, nn_offset), "%p", &offset)) == EOF) {
+        return NULL;
+    }
+
+    // Covert symbol(foo) to offset, such as 0x2fba.
+    if (!nn_symbol || nn_symbol >= (int)sizeof(tmp)) {
+        return offset;
+    }
+
+    void* object_file;
+    if ((object_file = dlopen(NULL, RTLD_LAZY)) == NULL) {
+        return offset;
+    }
+
+    void* address;
+    tmp[nn_symbol] = 0;
+    if ((address = dlsym(object_file, strncpy(tmp, p_symbol, nn_symbol))) == NULL) {
+        dlclose(object_file);
+        return offset;
+    }
+
+    Dl_info symbol_info;
+    if ((r0 = dladdr(address, &symbol_info)) == 0) {
+        dlclose(object_file);
+        return offset;
+    }
+
+    dlclose(object_file);
+    return (char*)symbol_info.dli_saddr - (char*)symbol_info.dli_fbase + (char*)offset;
+}
+
+extern const char* _srs_binary;
+
+char* addr2line_format(void* addr, char* symbol, char* buffer, int nn_buffer)
+{
+    char cmd[512] = {0};
+    int r0 = snprintf(cmd, sizeof(cmd), "addr2line -C -p -s -f -a -e %s %p", _srs_binary, (void*)((char*)addr - 1));
+    if (r0 < 0 || r0 >= (int)sizeof(cmd)) return symbol;
+
+    FILE* fp = popen(cmd, "r");
+    if (!fp) return symbol;
+
+    char* p = fgets(buffer, nn_buffer, fp);
+    pclose(fp);
+
+    if (p == NULL) return symbol;
+    if ((r0 = strlen(p)) == 0) return symbol;
+
+    // Trait the last newline if exists.
+    if (p[r0 - 1] == '\n') p[r0 - 1] = '\0';
+
+    // Find symbol not match by addr2line, like
+    //      0x0000000000021c87: ?? ??:0
+    //      0x0000000000002ffa: _start at ??:?
+    for (p = buffer; p < buffer + r0 - 1; p++) {
+        if (p[0] == '?' && p[1] == '?') return symbol;
+    }
+
+    return buffer;
+}
+#endif
+
 bool srs_is_system_control_error(srs_error_t err)
 {
     int error_code = srs_error_code(err);
@@ -266,3 +368,30 @@ std::string SrsCplxError::error_code_longstr(SrsCplxError* err)
     return it->second;
 }
 
+void SrsCplxError::srs_assert(bool expression)
+{
+#if defined(SRS_BACKTRACE) && defined(__linux)
+    if (!expression) {
+        void* addresses[64];
+        int nn_addresses = backtrace(addresses, sizeof(addresses) / sizeof(void*));
+        char** symbols = backtrace_symbols(addresses, nn_addresses);
+
+        // No error code for assert failed.
+        errno = 0;
+
+        char buffer[128];
+        srs_error("backtrace %d frames of %s %s", nn_addresses, _srs_binary, RTMP_SIG_SRS_SERVER);
+        for (int i = 0; i < nn_addresses; i++) {
+            void* frame = parse_symbol_offset(symbols[i]);
+            char* fmt = addr2line_format(frame, symbols[i], buffer, sizeof(buffer));
+            int parsed = (fmt == buffer);
+            srs_error("#%d %p %d %s", i, frame, parsed, fmt);
+        }
+
+        free(symbols);
+    }
+#endif
+
+    assert(expression);
+}
+

trunk/src/kernel/srs_kernel_error.hpp

@@ -439,6 +439,8 @@ public:
     static int error_code(SrsCplxError* err);
     static std::string error_code_str(SrsCplxError* err);
     static std::string error_code_longstr(SrsCplxError* err);
+public:
+    static void srs_assert(bool expression);
 };
 
 // Error helpers, should use these functions to new or wrap an error.
@@ -453,5 +455,9 @@ public:
 #define srs_error_code_longstr(err) SrsCplxError::error_code_longstr(err)
 #define srs_error_reset(err) srs_freep(err); err = srs_success
 
+#ifndef srs_assert
+#define srs_assert(expression) SrsCplxError::srs_assert(expression)
+#endif
+
 #endif
 

trunk/src/kernel/srs_kernel_kbps.cpp

@@ -7,6 +7,7 @@
 #include <srs_kernel_kbps.hpp>
 
 #include <srs_kernel_utility.hpp>
+#include <srs_kernel_error.hpp>
 
 SrsRateSample::SrsRateSample()
 {

trunk/src/main/srs_main_ingest_hls.cpp

@@ -44,6 +44,9 @@ SrsConfig* _srs_config = new SrsConfig();
 // @global Other variables.
 bool _srs_in_docker = false;
 
+// The binary name of SRS.
+const char* _srs_binary = NULL;
+
 /**
  * main entrance.
  */
@@ -52,6 +55,8 @@ int main(int argc, char** argv)
     // TODO: support both little and big endian.
     srs_assert(srs_is_little_endian());
 
+    _srs_binary = argv[0];
+    
     // directly failed when compile limited.
 #if defined(SRS_GPERF_MP) || defined(SRS_GPERF_MP) \
     || defined(SRS_GPERF_MC) || defined(SRS_GPERF_MP)

trunk/src/main/srs_main_mp4_parser.cpp

@@ -30,6 +30,9 @@ SrsConfig* _srs_config = new SrsConfig();
 // @global Other variables.
 bool _srs_in_docker = false;
 
+// The binary name of SRS.
+const char* _srs_binary = NULL;
+
 srs_error_t parse(std::string mp4_file, bool verbose)
 {
     srs_error_t err = srs_success;
@@ -88,6 +91,8 @@ int main(int argc, char** argv)
     printf("SRS MP4 parser/%d.%d.%d, parse and show the mp4 boxes structure.\n",
            VERSION_MAJOR, VERSION_MINOR, VERSION_REVISION);
 
+    _srs_binary = argv[0];
+    
     if (argc < 2) {
         printf("Usage: %s <mp4_file> [verbose]\n"
                "        mp4_file The MP4 file path to parse.\n"

trunk/src/main/srs_main_server.cpp

@@ -71,6 +71,9 @@ SrsServer* _srs_server = NULL;
 // Whether setup config by environment variables, see https://github.com/ossrs/srs/issues/2277
 bool _srs_config_by_env = false;
 
+// The binary name of SRS.
+const char* _srs_binary = NULL;
+
 /**
  * main entrance.
  */
@@ -78,6 +81,9 @@ srs_error_t do_main(int argc, char** argv, char** envp)
 {
     srs_error_t err = srs_success;
 
+    // TODO: Might fail if change working directory.
+    _srs_binary = argv[0];
+
     // Initialize global and thread-local variables.
     if ((err = srs_global_initialize()) != srs_success) {
         return srs_error_wrap(err, "global init");

trunk/src/protocol/srs_protocol_conn.cpp

@@ -6,6 +6,8 @@
 
 #include <srs_protocol_conn.hpp>
 
+#include <srs_kernel_error.hpp>
+
 #include <algorithm>
 using namespace std;
 

trunk/src/protocol/srs_protocol_json.cpp

@@ -6,6 +6,8 @@
 
 #include <srs_core.hpp>
 
+#include <srs_kernel_error.hpp>
+
 // LCOV_EXCL_START
 /* vim: set et ts=3 sw=3 sts=3 ft=c:
  *

trunk/src/protocol/srs_protocol_st.hpp

@@ -12,6 +12,7 @@
 #include <string>
 
 #include <srs_protocol_io.hpp>
+#include <srs_kernel_error.hpp>
 
 // Wrap for coroutine.
 typedef void* srs_netfd_t;

trunk/src/utest/srs_utest.cpp

@@ -37,6 +37,9 @@ SrsServer* _srs_server = NULL;
 bool _srs_in_docker = false;
 bool _srs_config_by_env = false;
 
+// The binary name of SRS.
+const char* _srs_binary = NULL;
+
 #include <srs_app_st.hpp>
 
 // Initialize global settings.
@@ -83,6 +86,8 @@ srs_error_t prepare_main() {
 GTEST_API_ int main(int argc, char **argv) {
     srs_error_t err = srs_success;
 
+    _srs_binary = argv[0];
+
     if ((err = prepare_main()) != srs_success) {
         fprintf(stderr, "Failed, %s\n", srs_error_desc(err).c_str());