bug#37757: Kernel panic upon shutdown

[Ludovic Courtès]

Hi!

Ludovic Courtès <address@hidden> skribis:

> Jesse (and anyone else experiencing this!), could you try to (1)
> reconfigure with this patch, (2) reboot, (3) try to halt the system to
> reproduce the crash, and (4) retrieve a backtrace from the ‘core’ file?
>
> For #4, you’ll have to do something along these lines once you’ve
> rebooted after the crash:
>
>   sudo gdb /run/current-system/profile/bin/guile /core
>
> and then type “thread apply all bt” at the GDB prompt.

It turns out the previous patch didn’t work; in short, we really have to
use async-signal-safe functions only from the signal handler, so this
has to be done in C.

The attached patch does that.  I’ve tried it with ‘guix system
container’ and it seems to dump core as expected, from what I can see.

Let me know if you manage to reproduce the bug and to get a core dumped
with this patch.

To everyone reading this: if you’re experiencing shepherd crashes,
please raise your hand :-) and consider applying this patch so we can
gather debugging info!

Thanks,
Ludo’.

diff --git a/gnu/services/shepherd.scm b/gnu/services/shepherd.scm
index 08bb33039c..cf82ef0a4c 100644
--- a/gnu/services/shepherd.scm
+++ b/gnu/services/shepherd.scm
@@ -271,6 +271,23 @@ and return the resulting '.go' file."
                          (compile-file #$file #:output-file #$output
                                        #:env env))))))
 
+(define (crash-handler)
+  (define gcc-toolchain
+    (module-ref (resolve-interface '(gnu packages commencement))
+                'gcc-toolchain))
+
+  (define source
+    (local-file "../system/aux-files/shepherd-crash-handler.c"))
+
+  (computed-file "crash-handler.so"
+                 #~(begin
+                     (setenv "PATH" #+(file-append gcc-toolchain "/bin"))
+                     (setenv "CPATH" #+(file-append gcc-toolchain "/include"))
+                     (setenv "LIBRARY_PATH"
+                             #+(file-append gcc-toolchain "/lib"))
+                     (system* "gcc" "-Wall" "-g" "-O3" "-fPIC"
+                              "-shared" "-o" #$output #$source))))
+
 (define (shepherd-configuration-file services)
   "Return the shepherd configuration file for SERVICES."
   (assert-valid-graph services)
@@ -281,6 +298,9 @@ and return the resulting '.go' file."
           (use-modules (srfi srfi-34)
                        (system repl error-handling))
 
+          ;; Load the crash handler, which allows shepherd to dump core.
+          (dynamic-link #$(crash-handler))
+
           ;; Arrange to spawn a REPL if something goes wrong.  This is better
           ;; than a kernel panic.
           (call-with-error-handling
diff --git a/gnu/system/aux-files/shepherd-crash-handler.c 
b/gnu/system/aux-files/shepherd-crash-handler.c
new file mode 100644
index 0000000000..6b2db10866
--- /dev/null
+++ b/gnu/system/aux-files/shepherd-crash-handler.c
@@ -0,0 +1,70 @@
+#define _GNU_SOURCE
+
+#include <stdlib.h>
+#include <unistd.h>
+#include <sched.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <sys/syscall.h>   /* For SYS_xxx definitions */
+#include <signal.h>
+
+static void
+handle_crash (int sig)
+{
+  static const char msg[] = "Shepherd crashed!\n";
+  write (2, msg, sizeof msg);
+
+#ifdef __sparc__
+  /* See 'raw_clone' in systemd.  */
+# error "SPARC uses a different 'clone' syscall convention"
+#endif
+
+  pid_t pid = syscall (SYS_clone, SIGCHLD, NULL);
+  if (pid < 0)
+    abort ();
+
+  if (pid == 0)
+    {
+      /* Restore the default signal handler to get a core dump.  */
+      signal (sig, SIG_DFL);
+
+      const struct rlimit infinity = { RLIM_INFINITY, RLIM_INFINITY };
+      setrlimit (RLIMIT_CORE, &infinity);
+      chdir ("/");
+
+      int pid = syscall (SYS_getpid);
+      kill (pid, sig);
+
+      /* As it turns out, 'kill' simply returns without doing anything, which
+        is consistent with the "Notes" section of kill(2).  Thus, force a
+        crash.  */
+      * (int *) 0 = 42;
+
+      _exit (254);
+    }
+  else
+    {
+      signal (sig, SIG_IGN);
+
+      int status;
+      waitpid (pid, &status, 0);
+
+      sync ();
+
+      _exit (255);
+    }
+
+  _exit (253);
+}
+
+static void initialize_crash_handler (void)
+  __attribute__ ((constructor));
+
+static void
+initialize_crash_handler (void)
+{
+  signal (SIGSEGV, handle_crash);
+  signal (SIGABRT, handle_crash);
+}