[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
Hung threads
|
From: |
Linas Vepstas |
|
Subject: |
Hung threads |
|
Date: |
Fri, 14 Nov 2008 11:31:37 -0600 |
Here's a deadlock I saw today. It appears to be a case of
improperly-nested locks.
Thread 16 is trying to exit.
CRITICAL SECTION -- threads 16
scm_i_port_table_mutex -- threads 18,24,27,28,29,30,32,33,37,39
scm_i_sweep_mutex -- threads 19,21, 35, 36, 38
t->heap_mutex -- threads 23
thread_admin_mutex -- threads 34
thread 16 -- SCM_CRITICAL_SECTION_START -throw.c 201
scm_c_catch throw.c:201
thread 18 -- scm_i_scm_pthread_mutex_lock (&scm_i_port_table_mutex);
ports.c 764
thread 19 -- scm_pthread_mutex_lock (&scm_i_sweep_mutex);
in scm_gc_for_newcell gc.c:486
thread 21 -- scm_pthread_mutex_lock (&scm_i_sweep_mutex);
increase_mtrigger gc-malloc.c:234
thread 23 -- pthread_mutex_lock (&t->heap_mutex);
scm_i_thread_put_to_sleep in scm_i_gc gc.c:552
thread 24 -- scm_pthread_mutex_lock (&scm_i_port_table_mutex);
scm_mkstrport in strports.c:321
thread 27 -- scm_pthread_mutex_lock
scm_mkstrport in strports.c:321
thread 28 -- scm_pthread_mutex_lock (&scm_i_port_table_mutex)
scm_close_port ports.c:764
thread 29 -- scm_pthread_mutex_lock
scm_mkstrport strports.c:321
thread 30 -- scm_pthread_mutex_lock
scm_mkstrport strports.c:321
thread 32 -- scm_pthread_mutex_lock
scm_close_port ports.c:764
thread 33 -- scm_pthread_mutex_lock
scm_mkstrport strports.c:321
thread 34 -- scm_pthread_mutex_lock (&thread_admin_mutex);
do_thread_exit threads.c:483
thread 35 -- scm_pthread_mutex_lock
scm_gc_for_newcell gc.c:486
thread 36 -- scm_pthread_mutex_lock
scm_gc_for_newcell gc.c:486
thread 37 -- scm_pthread_mutex_lock
scm_close_port ports.c:764
thread 38 -- scm_pthread_mutex_lock
scm_gc_for_newcell gc.c:486
thread 39 -- scm_pthread_mutex_lock
scm_mkstrport strports.c:321
(gdb) info threads
39 Thread 0xe8c66b90 (LWP 22581) 0xffffe425 in __kernel_vsyscall ()
38 Thread 0xe9467b90 (LWP 22580) 0xffffe425 in __kernel_vsyscall ()
37 Thread 0xe9c68b90 (LWP 22579) 0xffffe425 in __kernel_vsyscall ()
36 Thread 0xea469b90 (LWP 22578) 0xffffe425 in __kernel_vsyscall ()
35 Thread 0xeac6ab90 (LWP 22577) 0xffffe425 in __kernel_vsyscall ()
34 Thread 0xeb46bb90 (LWP 22576) 0xffffe425 in __kernel_vsyscall ()
33 Thread 0xebc6cb90 (LWP 22575) 0xffffe425 in __kernel_vsyscall ()
32 Thread 0xec46db90 (LWP 22561) 0xffffe425 in __kernel_vsyscall ()
30 Thread 0xee0d4b90 (LWP 22553) 0xffffe425 in __kernel_vsyscall ()
29 Thread 0xee8d5b90 (LWP 22552) 0xffffe425 in __kernel_vsyscall ()
28 Thread 0xef0d6b90 (LWP 22551) 0xffffe425 in __kernel_vsyscall ()
27 Thread 0xef9d2b90 (LWP 22550) 0xffffe425 in __kernel_vsyscall ()
24 Thread 0xf28cdb90 (LWP 22547) 0xffffe425 in __kernel_vsyscall ()
23 Thread 0xf42feb90 (LWP 22533) 0xffffe425 in __kernel_vsyscall ()
21 Thread 0xf30ceb90 (LWP 22525) 0xffffe425 in __kernel_vsyscall ()
19 Thread 0xf5e1db90 (LWP 22523) 0xffffe425 in __kernel_vsyscall ()
* 18 Thread 0xf6e1fb90 (LWP 22522) 0xffffe425 in __kernel_vsyscall ()
16 Thread 0xf561cb90 (LWP 22516) 0xffffe425 in __kernel_vsyscall ()
2 Thread 0xf7620b90 (LWP 22466) 0xffffe425 in __kernel_vsyscall ()
---Type <return> to continue, or q <return> to quit---
1 Thread 0xf794f6c0 (LWP 22465) 0xffffe425 in __kernel_vsyscall ()
(gdb) thread 16
[Switching to thread 16 (Thread 0xf561cb90 (LWP 22516))]#0 0xffffe425
in __kernel_vsyscall ()
(gdb) bt
#0 0xffffe425 in __kernel_vsyscall ()
#1 0xf7d95589 in __lll_lock_wait () from
/lib/tls/i686/cmov/libpthread.so.0
#2 0xf7d90bb4 in _L_lock_236 () from /lib/tls/i686/cmov/libpthread.so.0
#3 0xf7d9060b in pthread_mutex_lock () from
/lib/tls/i686/cmov/libpthread.so.0
this is in SCM_CRITICAL_SECTION_START
#4 0xf773dd7d in scm_c_catch (tag=0x104, body=0xf76c89d0 <c_body>,
body_data=0xf561c328, handler=0xf76c89f0 <c_handler>,
handler_data=0xf561c328,
pre_unwind_handler=0xf773d630 <scm_handle_by_message_noexit>,
pre_unwind_handler_data=0x0) at throw.c:201
#5 0xf76c8e92 in scm_i_with_continuation_barrier (body=0xf76c89d0
<c_body>,
body_data=0xf561c328, handler=0xf76c89f0 <c_handler>,
handler_data=0xf561c328,
pre_unwind_handler=0xf773d630 <scm_handle_by_message_noexit>,
pre_unwind_handler_data=0x0) at continuations.c:326
#6 0xf76c8f73 in scm_c_with_continuation_barrier (
func=0xf773cd00 <do_thread_exit>, data=0x9cc5900) at
continuations.c:368
#7 0xf773cb49 in scm_i_with_guile_and_parent (func=0xf773cd00
<do_thread_exit>,
data=0x9cc5900, parent=0xf4da9e90) at threads.c:710
#8 0xf773cc3e in scm_with_guile (func=0xf773cd00 <do_thread_exit>,
---Type <return> to continue, or q <return> to quit---
data=0x9cc5900) at threads.c:698
#9 0xf773cc93 in on_thread_exit (v=0x9cc5900) at threads.c:505
#10 0xf7d8dbb0 in __nptl_deallocate_tsd ()
from /lib/tls/i686/cmov/libpthread.so.0
#11 0xf7d8e509 in start_thread () from
/lib/tls/i686/cmov/libpthread.so.0
#12 0xf7b8ce5e in clone () from /lib/tls/i686/cmov/libc.so.6
(gdb)
(gdb)
(gdb)
(gdb) thread 18
[Switching to thread 18 (Thread 0xf6e1fb90 (LWP 22522))]#0 0xffffe425
in __kernel_vsyscall ()
(gdb) bt
#0 0xffffe425 in __kernel_vsyscall ()
#1 0xf7d95589 in __lll_lock_wait () from
/lib/tls/i686/cmov/libpthread.so.0
#2 0xf7d90ba6 in _L_lock_95 () from /lib/tls/i686/cmov/libpthread.so.0
#3 0xf7d9058a in pthread_mutex_lock () from
/lib/tls/i686/cmov/libpthread.so.0
#4 0xf773b4dc in scm_pthread_mutex_lock (mutex=0xf776dc6c) at
threads.c:1459
#5 0xf770c2b6 in scm_close_port (port=0xd9bc4cf0) at ports.c:764
#6 0xf770c358 in scm_close_output_port (port=0x80) at ports.c:798
#7 0xf771419a in scm_i_input_error (function=0xf7760898
"scm_i_lreadparen",
port=0xd9bc4c80, message=0xf77607dc "end of file", arg=0x404) at
read.c:109
#8 0xf7714529 in flush_ws (port=0xd9bc4c80,
eoferr=0xf7760898 "scm_i_lreadparen") at read.c:261
#9 0xf771728c in scm_read_sexp (chr=<value optimized out>,
port=0xd9bc4c80)
at read.c:357
#10 0xf7715dd1 in scm_read_expression (port=0xd9bc4c80) at read.c:1079
#11 0xf7737e02 in inner_eval_string (data=0xd9bc4c80) at strports.c:499
#12 0xf76e317e in scm_c_with_fluid (fluid=0x9cdc920, value=0xf4db0ea0,
cproc=0xf7737dd0 <inner_eval_string>, cdata=0xd9bc4c80) at
fluids.c:459
#13 0xf76fc575 in scm_c_call_with_current_module (module=0xf4db0ea0,
func=0xf7737dd0 <inner_eval_string>, data=0xd9bc4c80) at
modules.c:104
---Type <return> to continue, or q <return> to quit---
#14 0xf7738061 in scm_eval_string_in_module (string=0xdb01d740,
module=0xf4db0ea0) at strports.c:527
#15 0xf7738095 in scm_eval_string (string=0xdb01d740) at strports.c:535
#16 0xf77380c5 in scm_c_eval_string (
expr=0x9d04fe4 "(define (wire-fan-in a-wire b-wire c-wire)\n")
at strports.c:481
#17 0xf773dd69 in scm_c_catch (tag=0x104, body=0xf77380a0
<scm_c_eval_string>,
body_data=0x9d04fe4,
handler=0xf77d0638
<opencog::SchemeEval::catch_handler_wrapper(void*, scm_unused_struct*,
scm_unused_struct*)>, handler_data=0x9cd7758,
pre_unwind_handler=0xf77d06aa
<opencog::SchemeEval::preunwind_handler_wrapper(void*,
scm_unused_struct*, scm_unused_struct*)>,
pre_unwind_handler_data=0x9cd7758) at throw.c:200
#18 0xf77d0a7d in opencog::SchemeEval::do_eval (this=0x9cd7758,
address@hidden)
at
/home/linas/src/novamente/src/opencog-stage4/staging/opencog/guile/SchemeEval.cc:364
#19 0xf77d0d37 in opencog::SchemeEval::c_wrap_eval (p=0x9cd7758)
at
/home/linas/src/novamente/src/opencog-stage4/staging/opencog/guile/SchemeEval.cc:341
---Type <return> to continue, or q <return> to quit---
#20 0xf76c89e2 in c_body (d=0xf6e1ea08) at continuations.c:350
#21 0xf773dd69 in scm_c_catch (tag=0x104, body=0xf76c89d0 <c_body>,
body_data=0xf6e1ea08, handler=0xf76c89f0 <c_handler>,
handler_data=0xf6e1ea08,
pre_unwind_handler=0xf773d630 <scm_handle_by_message_noexit>,
pre_unwind_handler_data=0x0) at throw.c:200
#22 0xf76c8e92 in scm_i_with_continuation_barrier (body=0xf76c89d0
<c_body>,
body_data=0xf6e1ea08, handler=0xf76c89f0 <c_handler>,
handler_data=0xf6e1ea08,
pre_unwind_handler=0xf773d630 <scm_handle_by_message_noexit>,
pre_unwind_handler_data=0x0) at continuations.c:326
#23 0xf76c8f73 in scm_c_with_continuation_barrier (
func=0xf77d0d02 <opencog::SchemeEval::c_wrap_eval(void*)>,
data=0x9cd7758)
at continuations.c:368
#24 0xf773cb49 in scm_i_with_guile_and_parent (
func=0xf77d0d02 <opencog::SchemeEval::c_wrap_eval(void*)>,
data=0x9cd7758,
parent=0xf4da9e90) at threads.c:710
#25 0xf773cc3e in scm_with_guile (
func=0xf77d0d02 <opencog::SchemeEval::c_wrap_eval(void*)>,
data=0x9cd7758)
---Type <return> to continue, or q <return> to quit---
at threads.c:698
#26 0xf77d03ac in opencog::SchemeEval::eval (this=0x9cd7758,
address@hidden)
at
/home/linas/src/novamente/src/opencog-stage4/staging/opencog/guile/SchemeEval.cc:334
#27 0xf78b1dd2 in opencog::SchemeShell::do_eval (this=0x9cc6238,
address@hidden)
at
/home/linas/src/novamente/src/opencog-stage4/staging/opencog/shell/SchemeShell.cc:232
#28 0xf78b1ffd in opencog::SchemeShell::eval (this=0x9cc6238,
address@hidden,
h=0x9cbff70)
at
/home/linas/src/novamente/src/opencog-stage4/staging/opencog/shell/SchemeShell.cc:143
#29 0xf7ef5faf in opencog::ConsoleSocket::OnLine (this=0x9cd7508,
address@hidden)
at
/home/linas/src/novamente/src/opencog-stage4/staging/opencog/server/ConsoleSocket.cc:88
#30 0x0806a738 in TcpSocket::OnRead ()
#31 0x0806e68d in TcpSocket::OnRead ()
#32 0x08066c0d in SocketHandler::Select ()
#33 0x080631c4 in SocketHandler::Select ()
---Type <return> to continue, or q <return> to quit---
#34 0x0805b5fa in Socket::SocketThread::Run ()
#35 0x08072073 in Thread::StartThread ()
#36 0xf7d8e4fb in start_thread () from
/lib/tls/i686/cmov/libpthread.so.0
#37 0xf7b8ce5e in clone () from /lib/tls/i686/cmov/libc.so.6
(gdb)
(gdb) thread 34
[Switching to thread 34 (Thread 0xeb46bb90 (LWP 22576))]#0 0xffffe425
in __kernel_vsyscall ()
(gdb) bt
#0 0xffffe425 in __kernel_vsyscall ()
#1 0xf7d95589 in __lll_lock_wait () from
/lib/tls/i686/cmov/libpthread.so.0
#2 0xf7d90ba6 in _L_lock_95 () from /lib/tls/i686/cmov/libpthread.so.0
#3 0xf7d9058a in pthread_mutex_lock () from
/lib/tls/i686/cmov/libpthread.so.0
#4 0xf773b4dc in scm_pthread_mutex_lock (mutex=0xf7770148) at
threads.c:1459
#5 0xf773cd25 in do_thread_exit (v=0x9cc7cd0) at threads.c:483
#6 0xf76c89e2 in c_body (d=0xeb46b328) at continuations.c:350
#7 0xf773dd69 in scm_c_catch (tag=0x104, body=0xf76c89d0 <c_body>,
body_data=0xeb46b328, handler=0xf76c89f0 <c_handler>,
handler_data=0xeb46b328,
pre_unwind_handler=0xf773d630 <scm_handle_by_message_noexit>,
pre_unwind_handler_data=0x0) at throw.c:200
#8 0xf76c8e92 in scm_i_with_continuation_barrier (body=0xf76c89d0
<c_body>,
body_data=0xeb46b328, handler=0xf76c89f0 <c_handler>,
handler_data=0xeb46b328,
pre_unwind_handler=0xf773d630 <scm_handle_by_message_noexit>,
pre_unwind_handler_data=0x0) at continuations.c:326
#9 0xf76c8f73 in scm_c_with_continuation_barrier (
func=0xf773cd00 <do_thread_exit>, data=0x9cc7cd0) at
continuations.c:368
---Type <return> to continue, or q <return> to quit---
#10 0xf773cb49 in scm_i_with_guile_and_parent (func=0xf773cd00
<do_thread_exit>,
data=0x9cc7cd0, parent=0xf4da9e90) at threads.c:710
#11 0xf773cc3e in scm_with_guile (func=0xf773cd00 <do_thread_exit>,
data=0x9cc7cd0) at threads.c:698
#12 0xf773cc93 in on_thread_exit (v=0x9cc7cd0) at threads.c:505
#13 0xf7d8dbb0 in __nptl_deallocate_tsd ()
from /lib/tls/i686/cmov/libpthread.so.0
#14 0xf7d8e509 in start_thread () from
/lib/tls/i686/cmov/libpthread.so.0
#15 0xf7b8ce5e in clone () from /lib/tls/i686/cmov/libc.so.6
(gdb)
(gdb) thread 23
[Switching to thread 23 (Thread 0xf42feb90 (LWP 22533))]#0 0xffffe425
in __kernel_vsyscall ()
(gdb) bt
#0 0xffffe425 in __kernel_vsyscall ()
#1 0xf7d95589 in __lll_lock_wait () from
/lib/tls/i686/cmov/libpthread.so.0
#2 0xf7d90ba6 in _L_lock_95 () from /lib/tls/i686/cmov/libpthread.so.0
#3 0xf7d9058a in pthread_mutex_lock () from
/lib/tls/i686/cmov/libpthread.so.0
#4 0xf773b44b in scm_i_thread_put_to_sleep () at threads.c:1603
#5 0xf76e5da9 in scm_i_gc (what=0xf7761831 "string") at gc.c:552
#6 0xf76e7c9e in increase_mtrigger (size=<value optimized out>,
what=0xf7761831 "string") at gc-malloc.c:238
#7 0xf76e7dea in scm_gc_register_collectable_memory (mem=0xf4b9d250,
size=81,
what=0xf7761831 "string") at gc-malloc.c:288
#8 0xf76e8220 in scm_gc_malloc (size=81, what=0xf7761831 "string")
at gc-malloc.c:321
#9 0xf7720e68 in make_stringbuf (len=80) at strings.c:118
#10 0xf7721115 in scm_i_make_string (len=80, charsp=0xf42fd1b8) at
strings.c:185
#11 0xf77378dc in st_resize_port (pt=0xf4b07e50, new_size=80) at
strports.c:115
#12 0xf77379fc in st_flush (port=0xd9bd5200) at strports.c:158
#13 0xf7737f7b in scm_mkstrport (pos=0x2, str=<value optimized out>,
modes=328061, caller=0xf7763c8b "open-output-string") at
strports.c:338
#14 0xf77381c8 in scm_open_output_string () at strports.c:437
---Type <return> to continue, or q <return> to quit---
#15 0xf7714111 in scm_i_input_error (function=0xf7760898
"scm_i_lreadparen",
port=0xd9bd51a8, message=0xf77607dc "end of file", arg=0x404) at
read.c:99
#16 0xf7714529 in flush_ws (port=0xd9bd51a8,
eoferr=0xf7760898 "scm_i_lreadparen") at read.c:261
#17 0xf771728c in scm_read_sexp (chr=<value optimized out>,
port=0xd9bd51a8)
at read.c:357
#18 0xf7715dd1 in scm_read_expression (port=0xd9bd51a8) at read.c:1079
#19 0xf7737e02 in inner_eval_string (data=0xd9bd51a8) at strports.c:499
#20 0xf76e317e in scm_c_with_fluid (fluid=0x9cdc920, value=0xf4db0ea0,
cproc=0xf7737dd0 <inner_eval_string>, cdata=0xd9bd51a8) at
fluids.c:459
#21 0xf76fc575 in scm_c_call_with_current_module (module=0xf4db0ea0,
func=0xf7737dd0 <inner_eval_string>, data=0xd9bd51a8) at
modules.c:104
#22 0xf7738061 in scm_eval_string_in_module (string=0xdb0257a0,
module=0xf4db0ea0) at strports.c:527
#23 0xf7738095 in scm_eval_string (string=0xdb0257a0) at strports.c:535
#24 0xf77380c5 in scm_c_eval_string (
- Hung threads,
Linas Vepstas <=