mirror of https://github.com/ossrs/srs.git
You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
368 lines
12 KiB
Diff
368 lines
12 KiB
Diff
Michael Abd-El-Malek contributed this patch. He wrote:
|
|
----------------------------------------
|
|
Hello,
|
|
|
|
This is a patch that enables programmatically dumping the stack of
|
|
every thread. This has been useful in debugging deadlocks, etc...
|
|
Our usage model is that the SIGUSR2 handler calls the new
|
|
_st_print_thread_stacks function, which dumps the stack for all
|
|
threads. A convenient feature is that for thread stacks that are the
|
|
same (which is common for application with a lot of worker threads
|
|
waiting for work), only one stack trace is printed, along with a
|
|
count of how many threads have that same stack.
|
|
|
|
I use the glibc backtrace function to get the backtrace, and then use
|
|
popen to execute addr2line and convert memory addresses to file
|
|
names, function names, and line numbers. If glibc isn't available,
|
|
_st_print_thread_stacks just prints a warning. And this feature is
|
|
only available if DEBUG is turned on.
|
|
|
|
We've found this feature extremely helpful when debugging.
|
|
|
|
The patch can be a bit more robust (it assumes addr2line exists).
|
|
But I didn't want to go through the hassle of doing this, if the
|
|
StateThreads community doesn't want to use this patch. (In our
|
|
environment, addr2line will always be there.)
|
|
|
|
Cheers,
|
|
Mike
|
|
----------------------------------------
|
|
Invoking complex functions from a signal handler is not recommended,
|
|
plus this patch changes the behavior of existing API hooks. It will
|
|
not become part of State Threads proper but you may find it useful
|
|
nonetheless. This patch applies to st-1.5.2.
|
|
|
|
diff -Nur Makefile.1.5.2 Makefile
|
|
--- Makefile.1.5.2 Wed Sep 7 14:19:50 2005
|
|
+++ Makefile Wed Sep 7 14:33:08 2005
|
|
@@ -255,7 +255,8 @@
|
|
$(TARGETDIR)/stk.o \
|
|
$(TARGETDIR)/sync.o \
|
|
$(TARGETDIR)/key.o \
|
|
- $(TARGETDIR)/io.o
|
|
+ $(TARGETDIR)/io.o \
|
|
+ $(TARGETDIR)/backtrace.o
|
|
OBJS += $(EXTRA_OBJS)
|
|
HEADER = $(TARGETDIR)/st.h
|
|
SLIBRARY = $(TARGETDIR)/libst.a
|
|
diff -Nur backtrace.c.1.5.2 backtrace.c
|
|
--- backtrace.c.1.5.2 Wed Dec 31 16:00:00 1969
|
|
+++ backtrace.c Wed Sep 7 13:40:21 2005
|
|
@@ -0,0 +1,211 @@
|
|
+/*
|
|
+ * The contents of this file are subject to the Mozilla Public
|
|
+ * License Version 1.1 (the "License"); you may not use this file
|
|
+ * except in compliance with the License. You may obtain a copy of
|
|
+ * the License at http://www.mozilla.org/MPL/
|
|
+ *
|
|
+ * Software distributed under the License is distributed on an "AS
|
|
+ * IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
|
|
+ * implied. See the License for the specific language governing
|
|
+ * rights and limitations under the License.
|
|
+ *
|
|
+ * Contributor(s): Michael Abd-El-Malek (mabdelmalek@cmu.edu)
|
|
+ * Carnegie Mellon University
|
|
+ *
|
|
+ * Alternatively, the contents of this file may be used under the
|
|
+ * terms of the GNU General Public License Version 2 or later (the
|
|
+ * "GPL"), in which case the provisions of the GPL are applicable
|
|
+ * instead of those above. If you wish to allow use of your
|
|
+ * version of this file only under the terms of the GPL and not to
|
|
+ * allow others to use your version of this file under the MPL,
|
|
+ * indicate your decision by deleting the provisions above and
|
|
+ * replace them with the notice and other provisions required by
|
|
+ * the GPL. If you do not delete the provisions above, a recipient
|
|
+ * may use your version of this file under either the MPL or the
|
|
+ * GPL.
|
|
+ */
|
|
+
|
|
+
|
|
+
|
|
+/*
|
|
+ * This file contains routines for printing a stack trace of all threads.
|
|
+ * Only works when DEBUG is defined and where glibc is available, since it
|
|
+ * provides the backtrace() function.
|
|
+ */
|
|
+
|
|
+#define _GNU_SOURCE /* to get program_invocation_name */
|
|
+
|
|
+#include <stdio.h>
|
|
+#include <stdlib.h>
|
|
+
|
|
+
|
|
+#if defined(DEBUG) && defined(__GLIBC__)
|
|
+
|
|
+#include <errno.h>
|
|
+#include "common.h"
|
|
+#include <execinfo.h>
|
|
+#include <inttypes.h>
|
|
+#include <string.h>
|
|
+
|
|
+
|
|
+/* The maximum number of frames to get a stack trace for. If a thread has more
|
|
+ * frames than this, then we only show the latest X frames. */
|
|
+#define MAX_NUM_FRAMES 64
|
|
+
|
|
+
|
|
+typedef struct thread_stack_s {
|
|
+ uint32_t num_frames;
|
|
+ void* addresses[MAX_NUM_FRAMES]; /* frame pointers */
|
|
+ char* locations[MAX_NUM_FRAMES]; /* file/function/line numbers */
|
|
+ uint32_t num_matches;
|
|
+
|
|
+ struct thread_stack_s* next;
|
|
+} thread_stack_t;
|
|
+
|
|
+static thread_stack_t* stacks = NULL;
|
|
+
|
|
+
|
|
+/* Converts the function's memory addresses to function names, file names, and
|
|
+ * line numbers. Calls binutil's addr2line program. */
|
|
+static void get_symbol_names(thread_stack_t *stack)
|
|
+{
|
|
+ char program_to_run[1024], function[256], filename_lineno[256], temp[19];
|
|
+ FILE* output;
|
|
+ int num_bytes_left;
|
|
+ uint32_t i;
|
|
+
|
|
+ /* Construct the arguments to addr2line */
|
|
+ num_bytes_left = sizeof(program_to_run);
|
|
+ num_bytes_left -= snprintf(program_to_run, sizeof(program_to_run),
|
|
+ "addr2line -fCe %s", program_invocation_name);
|
|
+ for (i = 0; i < stack->num_frames && num_bytes_left > 0; ++i) {
|
|
+ num_bytes_left -= snprintf(temp, sizeof(temp), " %p", stack->addresses[i]);
|
|
+ strncat(program_to_run, temp, num_bytes_left);
|
|
+ }
|
|
+
|
|
+ /* Use popen to execute addr2line and read its ouput */
|
|
+ output = popen(program_to_run, "r");
|
|
+ for (i = 0; i < stack->num_frames; ++i) {
|
|
+ char* function_listing = (char*) malloc(512);
|
|
+ fscanf(output, "%255s\n", function);
|
|
+ fscanf(output, "%255s\n", filename_lineno);
|
|
+ snprintf(function_listing, 512, "%s at %s", function, filename_lineno);
|
|
+ stack->locations[i] = function_listing;
|
|
+ }
|
|
+ pclose(output);
|
|
+}
|
|
+
|
|
+
|
|
+static void print_stack(thread_stack_t* stack)
|
|
+{
|
|
+ int skip_offset = 0, cmp_len;
|
|
+ uint32_t i;
|
|
+
|
|
+ /* Get the function names/filenames/line numbers */
|
|
+ get_symbol_names(stack);
|
|
+
|
|
+ cmp_len = strlen("_st_iterate_threads_helper");
|
|
+
|
|
+ /* Print the backtrace */
|
|
+ for (i = 0; i < stack->num_frames; ++i) {
|
|
+ /* Skip frames we don't have location info for */
|
|
+ if (!strncmp(stack->locations[i], "??", 2)) {
|
|
+ continue;
|
|
+ }
|
|
+
|
|
+ /* Skip the frames that are used for printing the stack trace */
|
|
+ if (skip_offset) {
|
|
+ printf("\t#%2d %s %p\n", i - skip_offset, stack->locations[i],
|
|
+ stack->addresses[i]);
|
|
+ } else if (!strncmp(stack->locations[i], "_st_iterate_threads_helper",
|
|
+ cmp_len)) {
|
|
+ skip_offset = i + 1;
|
|
+ }
|
|
+ }
|
|
+}
|
|
+
|
|
+
|
|
+static void add_current_thread_stack(void)
|
|
+{
|
|
+ thread_stack_t *new_stack = malloc(sizeof(thread_stack_t));
|
|
+ thread_stack_t *search;
|
|
+
|
|
+ /* Call glibc function to get the backtrace */
|
|
+ new_stack->num_frames = backtrace(new_stack->addresses, MAX_NUM_FRAMES);
|
|
+
|
|
+ /* Check if we have another stacks that is equivalent. If so, then coaelsce
|
|
+ * two stacks into one, to minimize output to user. */
|
|
+ search = stacks;
|
|
+ while (search) {
|
|
+ if (search->num_frames == new_stack->num_frames &&
|
|
+ !memcmp(search->addresses, new_stack->addresses,
|
|
+ search->num_frames * sizeof(void*))) {
|
|
+ /* Found an existing stack that is the same as this thread's stack */
|
|
+ ++search->num_matches;
|
|
+ free(new_stack);
|
|
+ return;
|
|
+ } else {
|
|
+ search = search->next;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ /* This is a new stack. Add it to the list of stacks. */
|
|
+ new_stack->num_matches = 1;
|
|
+ new_stack->next = stacks;
|
|
+ stacks = new_stack;
|
|
+}
|
|
+
|
|
+static void print_stack_frames(void)
|
|
+{
|
|
+ while (stacks) {
|
|
+ printf("\n%u thread(s) with this backtrace:\n", stacks->num_matches);
|
|
+ print_stack(stacks);
|
|
+ stacks = stacks->next;
|
|
+ }
|
|
+ printf("\n");
|
|
+}
|
|
+
|
|
+static void free_stacks(void)
|
|
+{
|
|
+ uint32_t i;
|
|
+ while (stacks) {
|
|
+ thread_stack_t *next = stacks->next;
|
|
+ for (i = 0; i < stacks->num_frames; ++i) {
|
|
+ free(stacks->locations[i]);
|
|
+ }
|
|
+ free(stacks);
|
|
+ stacks = next;
|
|
+ }
|
|
+ stacks = NULL;
|
|
+}
|
|
+
|
|
+
|
|
+static void st_print_thread_stack(_st_thread_t *thread, int start_flag,
|
|
+ int end_flag)
|
|
+{
|
|
+ if (end_flag == 0) {
|
|
+ add_current_thread_stack();
|
|
+ } else {
|
|
+ print_stack_frames();
|
|
+ }
|
|
+}
|
|
+
|
|
+
|
|
+void _st_print_thread_stacks(int ignore)
|
|
+{
|
|
+ _st_iterate_threads_flag = 1;
|
|
+ _st_iterate_threads_helper(st_print_thread_stack);
|
|
+ _st_iterate_threads_flag = 0;
|
|
+
|
|
+ /* Deallocate memory */
|
|
+ free_stacks();
|
|
+}
|
|
+
|
|
+#else /* defined(DEBUG) && defined(__GLIBC__) */
|
|
+
|
|
+void _st_print_thread_stacks(int ignore)
|
|
+{
|
|
+ printf("%s: need DEBUG mode and glibc-specific functions to read stack.\n",
|
|
+ __FUNCTION__);
|
|
+}
|
|
+#endif /* defined(DEBUG) && defined(__GLIBC__) */
|
|
diff -Nur common.h.1.5.2 common.h
|
|
--- common.h.1.5.2 Wed Sep 7 14:18:37 2005
|
|
+++ common.h Wed Sep 7 14:35:36 2005
|
|
@@ -371,8 +371,18 @@
|
|
*/
|
|
|
|
#ifdef DEBUG
|
|
-void _st_iterate_threads(void);
|
|
-#define ST_DEBUG_ITERATE_THREADS() _st_iterate_threads()
|
|
+typedef void(*_st_func_ptr_t)(_st_thread_t *thread,
|
|
+ int start_flag,
|
|
+ int end_flag);
|
|
+/* Pointer to function that will be called on thread switch */
|
|
+extern _st_func_ptr_t _st_iterate_func_ptr;
|
|
+extern int _st_iterate_threads_flag;
|
|
+/* Thread iteration function that will call an arbitrary function */
|
|
+extern void _st_iterate_threads_helper(_st_func_ptr_t func);
|
|
+#define ST_DEBUG_ITERATE_THREADS() \
|
|
+ if (_st_iterate_func_ptr) { \
|
|
+ _st_iterate_threads_helper(_st_iterate_func_ptr); \
|
|
+ }
|
|
#else
|
|
#define ST_DEBUG_ITERATE_THREADS()
|
|
#endif
|
|
diff -Nur public.h.1.5.2 public.h
|
|
--- public.h.1.5.2 Wed Sep 7 11:46:58 2005
|
|
+++ public.h Wed Sep 7 13:38:46 2005
|
|
@@ -171,8 +171,10 @@
|
|
extern st_netfd_t st_open(const char *path, int oflags, mode_t mode);
|
|
|
|
#ifdef DEBUG
|
|
-extern void _st_show_thread_stack(st_thread_t thread, const char *messg);
|
|
+extern void _st_show_thread_stack(st_thread_t thread, int start_flag,
|
|
+ int end_flag);
|
|
extern void _st_iterate_threads(void);
|
|
+extern void _st_print_thread_stacks(int ignore);
|
|
#endif
|
|
|
|
#ifdef __cplusplus
|
|
diff -Nur sched.c.1.5.2 sched.c
|
|
--- sched.c.1.5.2 Wed Sep 7 10:48:05 2005
|
|
+++ sched.c Wed Sep 7 13:38:46 2005
|
|
@@ -919,16 +919,13 @@
|
|
|
|
|
|
#ifdef DEBUG
|
|
-/* ARGSUSED */
|
|
-void _st_show_thread_stack(_st_thread_t *thread, const char *messg)
|
|
-{
|
|
-
|
|
-}
|
|
-
|
|
/* To be set from debugger */
|
|
int _st_iterate_threads_flag = 0;
|
|
+/* Thread iteration function that will call an arbitrary function */
|
|
+_st_func_ptr_t _st_iterate_func_ptr = NULL;
|
|
|
|
-void _st_iterate_threads(void)
|
|
+/* This function iterates over all threads, calling "func" for each thread. */
|
|
+void _st_iterate_threads_helper(_st_func_ptr_t func)
|
|
{
|
|
static _st_thread_t *thread = NULL;
|
|
static jmp_buf orig_jb, save_jb;
|
|
@@ -944,16 +941,20 @@
|
|
|
|
if (thread) {
|
|
memcpy(thread->context, save_jb, sizeof(jmp_buf));
|
|
- _st_show_thread_stack(thread, NULL);
|
|
+ func(thread, 0, 0);
|
|
} else {
|
|
if (MD_SETJMP(orig_jb)) {
|
|
_st_iterate_threads_flag = 0;
|
|
+ _st_iterate_func_ptr = NULL;
|
|
thread = NULL;
|
|
- _st_show_thread_stack(thread, "Iteration completed");
|
|
+ /* Last thread to iterate through */
|
|
+ func(thread, 0, 1);
|
|
return;
|
|
}
|
|
+ /* First thread to iterate through */
|
|
thread = _ST_CURRENT_THREAD();
|
|
- _st_show_thread_stack(thread, "Iteration started");
|
|
+ _st_iterate_func_ptr = func;
|
|
+ func(thread, 1, 0);
|
|
}
|
|
|
|
q = thread->tlink.next;
|
|
@@ -966,5 +967,17 @@
|
|
memcpy(save_jb, thread->context, sizeof(jmp_buf));
|
|
MD_LONGJMP(thread->context, 1);
|
|
}
|
|
+
|
|
+/* ARGSUSED */
|
|
+void _st_show_thread_stack(_st_thread_t *thread, int start_flag, int end_flag)
|
|
+{
|
|
+}
|
|
+
|
|
+/* Iterate over threads inside debugger; see st/README */
|
|
+void _st_iterate_threads(void)
|
|
+{
|
|
+ _st_iterate_threads_helper(_st_show_thread_stack);
|
|
+}
|
|
+
|
|
#endif /* DEBUG */
|
|
|