From 8f48a0e2d19e727b42c2c9cb004dad25b5905d14 Mon Sep 17 00:00:00 2001 From: Winlin Date: Thu, 22 Aug 2024 17:12:39 +0800 Subject: [PATCH] ASAN: Support coroutine context switching and stack tracing (#4153) For coroutine, we should use `__sanitizer_start_switch_fiber` which similar to`VALGRIND_STACK_REGISTER`, see https://github.com/google/sanitizers/issues/189#issuecomment-1346243598 for details. If not fix this, asan will output warning: ``` ==72269==WARNING: ASan is ignoring requested __asan_handle_no_return: stack type: default top: 0x00016f638000; bottom 0x000106bec000; size: 0x000068a4c000 (1755627520) False positive error reports may follow For details see https://github.com/google/sanitizers/issues/189 ``` It will cause asan failed to get the stack, see `research/st/asan-switch.cpp` for example: ``` ==71611==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x000103600733 at pc 0x0001009d3d7c bp 0x000100b4bd40 sp 0x000100b4bd38 WRITE of size 1 at 0x000103600733 thread T0 #0 0x1009d3d78 in foo(void*) asan-switch.cpp:13 ``` After fix this issue, it should provide the full stack when crashing: ``` ==73437==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x000103300733 at pc 0x000100693d7c bp 0x00016f76f550 sp 0x00016f76f548 WRITE of size 1 at 0x000103300733 thread T0 #0 0x100693d78 in foo(void*) asan-switch.cpp:13 #1 0x100693df4 in main asan-switch.cpp:23 #2 0x195aa20dc () ``` For primordial coroutine, if not set the stack by `st_set_primordial_stack`, then the stack is NULL and asan can't get the stack tracing. Note that it's optional and only make it fail to display the stack information, no other errors. --- Co-authored-by: john --- trunk/3rdparty/st-srs/Makefile | 6 ++ trunk/3rdparty/st-srs/README.md | 6 ++ trunk/3rdparty/st-srs/common.c | 9 +++ trunk/3rdparty/st-srs/common.h | 76 +++++++++++++++++++++++++- trunk/3rdparty/st-srs/public.h | 2 + trunk/3rdparty/st-srs/sched.c | 20 ++++--- trunk/auto/depends.sh | 25 ++++++++- trunk/configure | 4 +- trunk/research/st/.gitignore | 2 + trunk/research/st/asan-switch.cpp | 40 ++++++++++++++ trunk/research/st/hello-thread.cpp | 21 +++++++ trunk/research/st/hello.c | 9 +++ trunk/src/main/srs_main_server.cpp | 7 +++ trunk/src/protocol/srs_protocol_st.cpp | 18 ++++++ trunk/src/protocol/srs_protocol_st.hpp | 6 ++ 15 files changed, 236 insertions(+), 15 deletions(-) create mode 100644 trunk/research/st/asan-switch.cpp create mode 100644 trunk/research/st/hello-thread.cpp create mode 100644 trunk/research/st/hello.c diff --git a/trunk/3rdparty/st-srs/Makefile b/trunk/3rdparty/st-srs/Makefile index c5590380d..11fdc95c9 100644 --- a/trunk/3rdparty/st-srs/Makefile +++ b/trunk/3rdparty/st-srs/Makefile @@ -195,6 +195,12 @@ endif # or cache the stack and reuse it: # make EXTRA_CFLAGS=-DMD_CACHE_STACK # +# or enable support for valgrind: +# make EXTRA_CFLAGS="-DMD_VALGRIND" +# +# or enable support for asan: +# make EXTRA_CFLAGS="-DMD_ASAN -fsanitize=address -fno-omit-frame-pointer" +# # or enable the coverage for utest: # make UTEST_FLAGS="-fprofile-arcs -ftest-coverage" # diff --git a/trunk/3rdparty/st-srs/README.md b/trunk/3rdparty/st-srs/README.md index e1959a734..f033d8eae 100644 --- a/trunk/3rdparty/st-srs/README.md +++ b/trunk/3rdparty/st-srs/README.md @@ -51,6 +51,12 @@ Linux with valgrind and epoll: make linux-debug EXTRA_CFLAGS="-DMD_HAVE_EPOLL -DMD_VALGRIND" ``` +Linux with ASAN(Google Address Sanitizer): + +```bash +make linux-debug EXTRA_CFLAGS="-DMD_ASAN" +``` + ## Mac: Usage Get code: diff --git a/trunk/3rdparty/st-srs/common.c b/trunk/3rdparty/st-srs/common.c index 1b71b1755..e241ec5f3 100644 --- a/trunk/3rdparty/st-srs/common.c +++ b/trunk/3rdparty/st-srs/common.c @@ -3,3 +3,12 @@ #include "common.h" +void *_st_primordial_stack_bottom = NULL; +size_t _st_primordial_stack_size = 0; + +void st_set_primordial_stack(void *top, void *bottom) +{ + _st_primordial_stack_bottom = bottom; + _st_primordial_stack_size = (char *)top - (char *)bottom; +} + diff --git a/trunk/3rdparty/st-srs/common.h b/trunk/3rdparty/st-srs/common.h index ed9ecfee7..74b6afe21 100644 --- a/trunk/3rdparty/st-srs/common.h +++ b/trunk/3rdparty/st-srs/common.h @@ -162,6 +162,10 @@ struct _st_thread { _st_clist_t tlink; /* For putting on thread queue */ #endif +#ifdef MD_ASAN + void *fake_stack; /* Fake stack for ASAN */ +#endif + st_utime_t due; /* Wakeup time when thread is sleeping */ _st_thread_t *left; /* For putting in timeout heap */ _st_thread_t *right; /* -- see docs/timeout_heap.txt for details */ @@ -314,7 +318,7 @@ extern __thread _st_eventsys_t *_st_eventsys; * Forward declarations */ -void _st_vp_schedule(void); +void _st_vp_schedule(_st_thread_t *from); void _st_vp_check_clock(void); void *_st_idle_thread_start(void *arg); void _st_thread_main(void); @@ -365,6 +369,64 @@ _st_thread_t *st_thread_create(void *(*start)(void *arg), void *arg, int joinabl #define ST_SWITCH_IN_CB(_thread) #endif +#ifdef MD_ASAN +/* + * Fiber annotation interface. + * + * Before switching to a different stack, one must call + * __sanitizer_start_switch_fiber with a pointer to the bottom of the + * destination stack and its size. When code starts running on the new stack, + * it must call __sanitizer_finish_switch_fiber to finalize the switch. + * The start_switch function takes a void** to store the current fake stack if + * there is one (it is needed when detect_stack_use_after_return is enabled). + * When restoring a stack, this pointer must be given to the finish_switch + * function. In most cases, this void* can be stored on the stack just before + * switching. When leaving a fiber definitely, null must be passed as first + * argument to the start_switch function so that the fake stack is destroyed. + * If you do not want support for stack use-after-return detection, you can + * always pass null to these two functions. + * Note that the fake stack mechanism is disabled during fiber switch, so if a + * signal callback runs during the switch, it will not benefit from the stack + * use-after-return detection. + * + * See https://github.com/google/sanitizers/issues/189#issuecomment-1346243598 + */ +extern void __sanitizer_start_switch_fiber(void **fake_stack_save, + const void *bottom, size_t size); + +extern void __sanitizer_finish_switch_fiber(void *fake_stack_save, + const void **bottom_old, + size_t *size_old); + +/* The stack for primoridal thread. */ +extern void *_st_primordial_stack_bottom; +extern size_t _st_primordial_stack_size; + +static inline void _st_asan_start_switch(_st_thread_t *from, _st_thread_t *thread) +{ + /* For primordial thread, the stack is NULL, so asan can not capture it. */ + const void *stk_bottom = thread->stack ? thread->stack->stk_bottom : NULL; + size_t stk_size = thread->stack ? thread->stack->stk_size : 0; + + /* For primordial thread, user should setup the stack information. */ + if (!stk_bottom && (thread->flags & _ST_FL_PRIMORDIAL)) { + stk_bottom = _st_primordial_stack_bottom; + stk_size = _st_primordial_stack_size; + } + + /* + * Save the current stack to fake_stack of from, tell asan the target stack + * we are targeting to switch to. + */ + __sanitizer_start_switch_fiber(&from->fake_stack, stk_bottom, stk_size); +} + +static inline void _st_asan_finish_switch(_st_thread_t *thread) +{ + __sanitizer_finish_switch_fiber(thread->fake_stack, NULL, NULL); +} +#endif + /* * Switch away from the current thread context by saving its state and * calling the thread scheduler @@ -374,9 +436,14 @@ static inline void _st_switch_context(_st_thread_t *thread) ST_SWITCH_OUT_CB(thread); if (!_st_md_cxt_save(thread->context)) { - _st_vp_schedule(); + _st_vp_schedule(thread); } +#ifdef MD_ASAN + /* Switch from other thread to this running thread. */ + _st_asan_finish_switch(thread); +#endif + ST_DEBUG_ITERATE_THREADS(); ST_SWITCH_IN_CB(thread); } @@ -385,8 +452,11 @@ static inline void _st_switch_context(_st_thread_t *thread) * Restore a thread context that was saved by _st_switch_context or * initialized by _ST_INIT_CONTEXT */ -static inline void _st_restore_context(_st_thread_t *thread) +static inline void _st_restore_context(_st_thread_t *from, _st_thread_t *thread) { +#ifdef MD_ASAN + _st_asan_start_switch(from, thread); +#endif _st_this_thread = thread; _st_md_cxt_restore(thread->context, 1); } diff --git a/trunk/3rdparty/st-srs/public.h b/trunk/3rdparty/st-srs/public.h index 97da14642..6555c02c0 100644 --- a/trunk/3rdparty/st-srs/public.h +++ b/trunk/3rdparty/st-srs/public.h @@ -159,6 +159,8 @@ extern st_netfd_t st_open(const char *path, int oflags, mode_t mode); extern void st_destroy(void); extern int st_thread_setspecific2(st_thread_t thread, int key, void *value); +extern void st_set_primordial_stack(void *top, void *bottom); + #ifdef DEBUG extern void _st_show_thread_stack(st_thread_t thread, const char *messg); extern void _st_iterate_threads(void); diff --git a/trunk/3rdparty/st-srs/sched.c b/trunk/3rdparty/st-srs/sched.c index e26c327eb..691f2a2aa 100644 --- a/trunk/3rdparty/st-srs/sched.c +++ b/trunk/3rdparty/st-srs/sched.c @@ -135,7 +135,7 @@ int st_poll(struct pollfd *pds, int npds, st_utime_t timeout) } -void _st_vp_schedule(void) +void _st_vp_schedule(_st_thread_t *from) { _st_thread_t *thread; @@ -159,7 +159,7 @@ void _st_vp_schedule(void) /* Resume the thread */ thread->state = _ST_ST_RUNNING; - _st_restore_context(thread); + _st_restore_context(from, thread); } @@ -368,6 +368,11 @@ int st_thread_join(_st_thread_t *thread, void **retvalp) void _st_thread_main(void) { _st_thread_t *thread = _st_this_thread; + +#ifdef MD_ASAN + /* Switch from other thread to this new created thread. */ + _st_asan_finish_switch(thread); +#endif /* * Cap the stack by zeroing out the saved return address register @@ -572,7 +577,7 @@ void _st_vp_check_clock(void) /* Make thread runnable */ ST_ASSERT(!(thread->flags & _ST_FL_IDLE_THREAD)); thread->state = _ST_ST_RUNNABLE; - // Insert at the head of RunQ, to execute timer first. + /* Insert at the head of RunQ, to execute timer first. */ st_clist_insert_after(&thread->links, &_st_this_vp.run_q); } } @@ -589,7 +594,7 @@ void st_thread_yield() /* Check sleep queue for expired threads */ _st_vp_check_clock(); - // If not thread in RunQ to yield to, ignore and continue to run. + /* If not thread in RunQ to yield to, ignore and continue to run. */ if (_st_this_vp.run_q.next == &_st_this_vp.run_q) { return; } @@ -598,11 +603,11 @@ void st_thread_yield() ++_st_stat_thread_yield2; #endif - // Append thread to the tail of RunQ, we will back after all threads executed. + /* Append thread to the tail of RunQ, we will back after all threads executed. */ me->state = _ST_ST_RUNNABLE; st_clist_insert_before(&me->links, &_st_this_vp.run_q); - // Yield to other threads in the RunQ. + /* Yield to other threads in the RunQ. */ _st_switch_context(me); } @@ -664,8 +669,9 @@ _st_thread_t *st_thread_create(void *(*start)(void *arg), void *arg, int joinabl thread->arg = arg; /* Note that we must directly call rather than call any functions. */ - if (_st_md_cxt_save(thread->context)) + if (_st_md_cxt_save(thread->context)) { _st_thread_main(); + } MD_GET_SP(thread) = (long)(stack->sp); /* If thread is joinable, allocate a termination condition variable */ diff --git a/trunk/auto/depends.sh b/trunk/auto/depends.sh index 38d09bf83..a73b6b8b5 100755 --- a/trunk/auto/depends.sh +++ b/trunk/auto/depends.sh @@ -205,7 +205,7 @@ fi ##################################################################################### # Check for address sanitizer, see https://github.com/google/sanitizers ##################################################################################### -if [[ $SRS_SANITIZER == YES && $OS_IS_X86_64 == YES ]]; then +if [[ $SRS_SANITIZER == YES ]]; then echo 'int main() { return 0; }' > ${SRS_OBJS}/test_sanitizer.c && gcc -fsanitize=address -fno-omit-frame-pointer -g -O0 ${SRS_OBJS}/test_sanitizer.c \ -o ${SRS_OBJS}/test_sanitizer 1>/dev/null 2>&1; @@ -217,7 +217,7 @@ if [[ $SRS_SANITIZER == YES && $OS_IS_X86_64 == YES ]]; then fi fi -if [[ $SRS_SANITIZER == YES && $OS_IS_X86_64 == YES && $SRS_SANITIZER_STATIC == NO ]]; then +if [[ $SRS_SANITIZER == YES && $SRS_SANITIZER_STATIC == NO ]]; then echo 'int main() { return 0; }' > ${SRS_OBJS}/test_sanitizer.c && gcc -fsanitize=address -fno-omit-frame-pointer -static-libasan -g -O0 ${SRS_OBJS}/test_sanitizer.c \ -o ${SRS_OBJS}/test_sanitizer 1>/dev/null 2>&1; @@ -228,7 +228,7 @@ if [[ $SRS_SANITIZER == YES && $OS_IS_X86_64 == YES && $SRS_SANITIZER_STATIC == fi fi -if [[ $SRS_SANITIZER == YES && $OS_IS_X86_64 == YES && $SRS_SANITIZER_LOG == NO ]]; then +if [[ $SRS_SANITIZER == YES && $SRS_SANITIZER_LOG == NO ]]; then echo "#include " > ${SRS_OBJS}/test_sanitizer.c && echo "int main() { return 0; }" >> ${SRS_OBJS}/test_sanitizer.c && gcc -fsanitize=address -fno-omit-frame-pointer -g -O0 ${SRS_OBJS}/test_sanitizer.c \ @@ -240,6 +240,18 @@ if [[ $SRS_SANITIZER == YES && $OS_IS_X86_64 == YES && $SRS_SANITIZER_LOG == NO fi fi +if [[ $SRS_SANITIZER == YES ]]; then + echo "#include " > ${SRS_OBJS}/test_sanitizer.c && + echo "int main() { __sanitizer_start_switch_fiber(NULL, NULL, 0); }" >> ${SRS_OBJS}/test_sanitizer.c && + gcc -fsanitize=address -fno-omit-frame-pointer -g -O0 ${SRS_OBJS}/test_sanitizer.c \ + -o ${SRS_OBJS}/test_sanitizer 1>/dev/null 2>&1; + ret=$?; rm -rf ${SRS_OBJS}/test_sanitizer* + if [[ $ret -eq 0 ]]; then + echo "libasan fiber switch api found ok!"; + SRS_SANITIZER_FIBER_SWITCH=YES + fi +fi + ##################################################################################### # state-threads ##################################################################################### @@ -267,6 +279,13 @@ fi if [[ $SRS_DEBUG_STATS == YES ]]; then _ST_EXTRA_CFLAGS="$_ST_EXTRA_CFLAGS -DDEBUG_STATS" fi +# Whether to enable asan. +if [[ $SRS_SANITIZER_FIBER_SWITCH == YES ]]; then + _ST_EXTRA_CFLAGS="$_ST_EXTRA_CFLAGS -DMD_ASAN" +fi +if [[ $SRS_SANITIZER == YES ]]; then + _ST_EXTRA_CFLAGS="$_ST_EXTRA_CFLAGS -fsanitize=address -fno-omit-frame-pointer" +fi # Pass the global extra flags. if [[ $SRS_EXTRA_FLAGS != '' ]]; then _ST_EXTRA_CFLAGS="$_ST_EXTRA_CFLAGS $SRS_EXTRA_FLAGS" diff --git a/trunk/configure b/trunk/configure index a19be60ad..b9475493e 100755 --- a/trunk/configure +++ b/trunk/configure @@ -122,7 +122,7 @@ fi # For Sanitizer # @doc: https://github.com/google/sanitizers/wiki/AddressSanitizer -if [[ $SRS_SANITIZER == YES && $OS_IS_X86_64 == YES ]]; then +if [[ $SRS_SANITIZER == YES ]]; then CXXFLAGS="${CXXFLAGS} -fsanitize=address -fno-omit-frame-pointer"; fi @@ -244,7 +244,7 @@ fi # For address sanitizer # @doc: https://github.com/google/sanitizers/wiki/AddressSanitizer -if [[ $SRS_SANITIZER == YES && $OS_IS_X86_64 == YES ]]; then +if [[ $SRS_SANITIZER == YES ]]; then SrsLinkOptions="${SrsLinkOptions} -fsanitize=address -fno-omit-frame-pointer"; if [[ $SRS_SANITIZER_STATIC == YES ]]; then SrsLinkOptions="${SrsLinkOptions} -static-libasan"; diff --git a/trunk/research/st/.gitignore b/trunk/research/st/.gitignore index 69664733d..d24fbd32a 100644 --- a/trunk/research/st/.gitignore +++ b/trunk/research/st/.gitignore @@ -10,3 +10,5 @@ huge-threads exceptions hello pthreads +asan-switch +hello-thread diff --git a/trunk/research/st/asan-switch.cpp b/trunk/research/st/asan-switch.cpp new file mode 100644 index 000000000..4611b5434 --- /dev/null +++ b/trunk/research/st/asan-switch.cpp @@ -0,0 +1,40 @@ +/* +g++ asan-switch.cpp ../../objs/st/libst.a -fsanitize=address -fno-omit-frame-pointer -g -O0 -o asan-switch && ./asan-switch +*/ +#include +#include +#include +#include "../../objs/st/st.h" + +void* foo(void *args) { + for (int i = 0; ; i++) { + st_sleep(1); + if (i && (i % 2) == 0) { + char *p = new char[3]; + p[3] = 'H'; + } + printf("#%d: main: working\n", i); + } + return NULL; +} + +int main(int argc, char **argv) { + register void* stack_top asm ("sp"); + struct rlimit limit; + if (getrlimit (RLIMIT_STACK, &limit) == 0) { + void* stack_bottom = (char*)stack_top - limit.rlim_cur; + st_set_primordial_stack(stack_top, stack_bottom); + } + + st_init(); + if (argc > 1) { + // Directly call foo() to trigger ASAN, call the function in the primordial thread, + // note that asan can not capther the stack of primordial thread. + foo(NULL); + } else { + st_thread_create(foo, NULL, 0, 0); + st_thread_exit(NULL); + } + return 0; +} + diff --git a/trunk/research/st/hello-thread.cpp b/trunk/research/st/hello-thread.cpp new file mode 100644 index 000000000..3128ef80a --- /dev/null +++ b/trunk/research/st/hello-thread.cpp @@ -0,0 +1,21 @@ +/* +g++ hello-thread.cpp ../../objs/st/libst.a -g -O0 -o hello-thread && ./hello-thread +*/ +#include +#include "../../objs/st/st.h" + +void* foo(void *args) { + for (int i = 0; ; i++) { + st_sleep(1); + printf("#%d: main: working\n", i); + } + + return NULL; +} + +int main() { + st_init(); + st_thread_create(foo, NULL, 0, 0); + st_thread_exit(NULL); + return 0; +} diff --git a/trunk/research/st/hello.c b/trunk/research/st/hello.c new file mode 100644 index 000000000..8000cb131 --- /dev/null +++ b/trunk/research/st/hello.c @@ -0,0 +1,9 @@ +#include +#include +int main (void) +{ + struct rlimit limit; + + getrlimit (RLIMIT_STACK, &limit); + printf ("\nStack Limit = %ld and %ld max\n", limit.rlim_cur, limit.rlim_max); +} diff --git a/trunk/src/main/srs_main_server.cpp b/trunk/src/main/srs_main_server.cpp index c34214759..01e9b249f 100644 --- a/trunk/src/main/srs_main_server.cpp +++ b/trunk/src/main/srs_main_server.cpp @@ -253,6 +253,13 @@ srs_error_t do_main(int argc, char** argv, char** envp) int main(int argc, char** argv, char** envp) { +#ifdef SRS_SANITIZER + // Setup the primordial stack for st. Use the current variable address as the stack top. + // This is not very accurate but sufficient. + void* p = NULL; + srs_set_primordial_stack(&p); +#endif + srs_error_t err = do_main(argc, argv, envp); if (err != srs_success) { diff --git a/trunk/src/protocol/srs_protocol_st.cpp b/trunk/src/protocol/srs_protocol_st.cpp index f15f7252c..4e45430f2 100644 --- a/trunk/src/protocol/srs_protocol_st.cpp +++ b/trunk/src/protocol/srs_protocol_st.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include using namespace std; @@ -38,6 +39,23 @@ bool srs_st_epoll_is_supported(void) } #endif +#ifdef SRS_SANITIZER +void srs_set_primordial_stack(void* stack_top) +{ + if (!stack_top) { + return; + } + + struct rlimit limit; + if (getrlimit (RLIMIT_STACK, &limit) != 0) { + return; + } + + void* stack_bottom = (char*)stack_top - (uint64_t)limit.rlim_cur; + st_set_primordial_stack(stack_top, stack_bottom); +} +#endif + srs_error_t srs_st_init() { #ifdef __linux__ diff --git a/trunk/src/protocol/srs_protocol_st.hpp b/trunk/src/protocol/srs_protocol_st.hpp index c893db077..de2e2af00 100644 --- a/trunk/src/protocol/srs_protocol_st.hpp +++ b/trunk/src/protocol/srs_protocol_st.hpp @@ -20,6 +20,12 @@ typedef void* srs_thread_t; typedef void* srs_cond_t; typedef void* srs_mutex_t; + +#ifdef SRS_SANITIZER +// Setup the primordial stack for asan detecting. +void srs_set_primordial_stack(void* stack_top); +#endif + // Initialize ST, requires epoll for linux. extern srs_error_t srs_st_init(); // Destroy ST, free resources for asan detecting.