[Qemu-devel] [Bug 1671876] Re: qemu 2.7.0 segfaults in qemu_co_queue_run_restart()

[Mohammed Gamal]

A third stack trace

It generates the following stack trace
---------------------------------------------------------------------
(gdb) bt
#0  qemu_co_queue_run_restart (co=0x7f75ed30dbc0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:59
#1  0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75ed30dbc0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#2  0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75f1c0f200) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#3  0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75f1c0f200) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#4  0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75ed304870) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#5  0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75ed304870) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#6  0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75e800fcd0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#7  0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75e800fcd0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#8  0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75e800fac0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#9  0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75e800fac0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#10 0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75e800f8b0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#11 0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75e800f8b0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#12 0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75fbf05570) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#13 0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75fbf05570) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#14 0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75e8009b70) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#15 0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75e8009b70) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#16 0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75e800b5d0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#17 0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75e800b5d0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#18 0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75e8008910) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#19 0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75e8008910) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#20 0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75e800f6a0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#21 0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75e800f6a0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#22 0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75fbf05100) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#23 0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75fbf05100) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#24 0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75fbf04ee0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#25 0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75fbf04ee0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#26 0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75ed301c50) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#27 0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75ed301c50) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#28 0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75ed315270) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#29 0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75ed315270) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#30 0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75ed31cf10) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#31 0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75ed31cf10) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#32 0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75e800a970) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#33 0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75e800a970) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#34 0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75e8007df0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#35 0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75e8007df0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#36 0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75e8005960) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#37 0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75e8005960) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#38 0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75e800e1b0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#39 0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75e800e1b0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#40 0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75e8000a00) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#41 0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75e8000a00) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#42 0x0000561927482e74 in qemu_co_queue_run_restart (co=0x7f75e8007900) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
#43 0x00005619274829a9 in qemu_coroutine_enter (co=0x7f75e8007900) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
#44 0x0000561927482fa0 in qemu_co_enter_next (address@hidden) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:106
#45 0x0000561927421060 in timer_cb (blk=0x5619288d1590, is_write=<optimized 
out>) at /build/pb-qemu-pssKUp/pb-qemu-2.7.0/block/throttle-groups.c:400
#46 0x00005619273de615 in timerlist_run_timers (timer_list=0x56192893de80) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/qemu-timer.c:528
#47 0x00005619273de679 in timerlistgroup_run_timers (address@hidden) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/qemu-timer.c:564
#48 0x00005619273def47 in aio_dispatch (address@hidden) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/aio-posix.c:357
#49 0x00005619273df0e8 in aio_poll (ctx=0x561929b6bbb0, blocking=<optimized 
out>) at /build/pb-qemu-pssKUp/pb-qemu-2.7.0/aio-posix.c:479
#50 0x0000561927240c79 in iothread_run (opaque=0x561929b6b960) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/iothread.c:46
#51 0x00007f77b32160a4 in start_thread (arg=0x7f77997ff700) at 
pthread_create.c:403
#52 0x00007f77b2f4b62d in clone () at 
../sysdeps/unix/sysv/linux/x86_64/clone.S:111
---------------------------------------------------------------------

It's also crashing in list traversal. Looking at the contained data we
see:

---------------------------------------------------------------------
(gdb) print *(&co->co_queue_wakeup->sqh_first)
$1 = (struct Coroutine *) 0x1
(gdb) print *(&co->co_queue_wakeup->sqh_first->co_queue_next)
Cannot access memory at address 0x31
---------------------------------------------------------------------

So again. Segfault is caused by apparently invalid addresses. And this
time it occurs after so many invocations of qemu_co_queue_run_restart()

-- 
You received this bug notification because you are a member of qemu-
devel-ml, which is subscribed to QEMU.
https://bugs.launchpad.net/bugs/1671876

Title:
  qemu 2.7.0 segfaults in qemu_co_queue_run_restart()

Status in QEMU:
  New

Bug description:
  Hi,

  I've been experiencing frequent segfaults lately with qemu 2.7.0
  running Ubuntu 16.04 guests. The crash usually happens in
  qemu_co_queue_run_restart(). I haven't seen this so far with any other
  guests or distros.

  Here is one back trace I obtained from one of the crashing VMs.

  --------------------------------------------------------------------------
  (gdb) bt
  #0  qemu_co_queue_run_restart (co=0x7fba8ff05aa0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:59
  #1  0x000055c1656f39a9 in qemu_coroutine_enter (co=0x7fba8ff05aa0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
  #2  0x000055c1656f3e74 in qemu_co_queue_run_restart (co=0x7fba8dd20430) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
  #3  0x000055c1656f39a9 in qemu_coroutine_enter (co=0x7fba8dd20430) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
  #4  0x000055c1656f3e74 in qemu_co_queue_run_restart (co=0x7fba8dd14ea0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
  #5  0x000055c1656f39a9 in qemu_coroutine_enter (co=0x7fba8dd14ea0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
  #6  0x000055c1656f3e74 in qemu_co_queue_run_restart (co=0x7fba80c11dc0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
  #7  0x000055c1656f39a9 in qemu_coroutine_enter (co=0x7fba80c11dc0) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
  #8  0x000055c1656f3e74 in qemu_co_queue_run_restart (co=0x7fba8dd0bd70) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:60
  #9  0x000055c1656f39a9 in qemu_coroutine_enter (co=0x7fba8dd0bd70) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine.c:119
  #10 0x000055c1656f3fa0 in qemu_co_enter_next (address@hidden) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/util/qemu-coroutine-lock.c:106
  #11 0x000055c165692060 in timer_cb (blk=0x55c1669e7590, is_write=<optimized 
out>) at /build/pb-qemu-pssKUp/pb-qemu-2.7.0/block/throttle-groups.c:400
  #12 0x000055c16564f615 in timerlist_run_timers (timer_list=0x55c166a53e80) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/qemu-timer.c:528
  #13 0x000055c16564f679 in timerlistgroup_run_timers (address@hidden) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/qemu-timer.c:564
  #14 0x000055c16564ff47 in aio_dispatch (address@hidden) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/aio-posix.c:357
  #15 0x000055c1656500e8 in aio_poll (ctx=0x55c167c81bb0, blocking=<optimized 
out>) at /build/pb-qemu-pssKUp/pb-qemu-2.7.0/aio-posix.c:479
  #16 0x000055c1654b1c79 in iothread_run (opaque=0x55c167c81960) at 
/build/pb-qemu-pssKUp/pb-qemu-2.7.0/iothread.c:46
  #17 0x00007fbc4b64f0a4 in allocate_stack (stack=<synthetic pointer>, 
pdp=<synthetic pointer>, attr=0x0) at allocatestack.c:416
  #18 __pthread_create_2_1 (newthread=<error reading variable: Cannot access 
memory at address 0xffffffffffffff48>, attr=<error reading variable: Cannot 
access memory at address 0xffffffffffffff40>,
      start_routine=<error reading variable: Cannot access memory at address 
0xffffffffffffff58>, arg=<error reading variable: Cannot access memory at 
address 0xffffffffffffff50>) at pthread_create.c:539
  Backtrace stopped: Cannot access memory at address 0x8
  --------------------------------------------------------------------------

  The code that crashes is this
  --------------------------------------------------------------------------
  void qemu_co_queue_run_restart(Coroutine *co)
  {
      Coroutine *next;

      trace_qemu_co_queue_run_restart(co);
      while ((next = QSIMPLEQ_FIRST(&co->co_queue_wakeup))) {
          QSIMPLEQ_REMOVE_HEAD(&co->co_queue_wakeup, co_queue_next);       <--- 
Crash occurs here this time
          qemu_coroutine_enter(next);
      }
  }
  --------------------------------------------------------------------------

  Expanding the macro QSIMPLEQ_REMOVE_HEAD gives us
  --------------------------------------------------------------------------
  #define QSIMPLEQ_REMOVE_HEAD(head, field) do {                          \
      if (((head)->sqh_first = (head)->sqh_first->field.sqe_next) == NULL)\
          (head)->sqh_last = &(head)->sqh_first;                          \
  } while (/*CONSTCOND*/0)
  --------------------------------------------------------------------------

  which corrsponds to
  --------------------------------------------------------------------------
  if (((&co->co_queue_wakeup)->sqh_first = 
(&co->co_queue_wakeup)->sqh_first->co_queue_next.sqe_next) == NULL)\
          (&co->co_queue_wakeup)->sqh_last = &(&co->co_queue_wakeup)->sqh_first;
  --------------------------------------------------------------------------

  Debugging the list we see
  --------------------------------------------------------------------------
  (gdb) print *(&co->co_queue_wakeup->sqh_first)
  $6 = (struct Coroutine *) 0x1000
  (gdb) print *(&co->co_queue_wakeup->sqh_first->co_queue_next)
  Cannot access memory at address 0x1030
  --------------------------------------------------------------------------

  So the data in co->co_queue_wakeup->sqh_first is corrupted and
  represents an invalid address. Any idea why is that?

To manage notifications about this bug go to:
https://bugs.launchpad.net/qemu/+bug/1671876/+subscriptions