In user-mode emulation there is a small race between preexit_cleanup
and exit_group() which means we may end up calling instrumented
instructions before the kernel reaps child threads. To solve this we
implement a new helper which ensures the callbacks are flushed along
with any translations before we let the host do it's a thing.
While we are at it make the documentation of
qemu_plugin_register_atexit_cb clearer as to what the user can expect.
Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
Looks great to me and I cannot reproduce the race with it installed in.
With calling `qemu_plugin_disable_mem_helpers`:
---
include/qemu/plugin.h | 12 ++++++++++++
include/qemu/qemu-plugin.h | 13 +++++++++++++
bsd-user/syscall.c | 6 +++---
linux-user/exit.c | 2 +-
plugins/core.c | 33 +++++++++++++++++++++++++++++++++
5 files changed, 62 insertions(+), 4 deletions(-)
diff --git a/include/qemu/plugin.h b/include/qemu/plugin.h
index 0fefbc6084..9a8438f683 100644
--- a/include/qemu/plugin.h
+++ b/include/qemu/plugin.h
@@ -190,6 +190,16 @@ void qemu_plugin_add_dyn_cb_arr(GArray *arr);
void qemu_plugin_disable_mem_helpers(CPUState *cpu);
+/**
+ * qemu_plugin_user_exit(): clean-up callbacks before calling exit callbacks
+ *
+ * This is a user-mode only helper that ensure we have fully cleared
+ * callbacks from all threads before calling the exit callbacks. This
+ * is so the plugins themselves don't have to jump through hoops to
+ * guard against race conditions.
+ */
+void qemu_plugin_user_exit(void);
+
#else /* !CONFIG_PLUGIN */
static inline void qemu_plugin_add_opts(void)
@@ -250,6 +260,8 @@ void qemu_plugin_add_dyn_cb_arr(GArray *arr)
static inline void qemu_plugin_disable_mem_helpers(CPUState *cpu)
{ }
+static inline void qemu_plugin_user_exit(void)
+{ }
#endif /* !CONFIG_PLUGIN */
#endif /* QEMU_PLUGIN_H */
diff --git a/include/qemu/qemu-plugin.h b/include/qemu/qemu-plugin.h
index dc3496f36c..e6e815abc5 100644
--- a/include/qemu/qemu-plugin.h
+++ b/include/qemu/qemu-plugin.h
@@ -549,6 +549,19 @@ void qemu_plugin_vcpu_for_each(qemu_plugin_id_t id,
void qemu_plugin_register_flush_cb(qemu_plugin_id_t id,
qemu_plugin_simple_cb_t cb);
+/**
+ * qemu_plugin_register_atexit_cb() - register exit callback
+ * @id: plugin ID
+ * @cb: callback
+ * @userdata: user data for callback
+ *
+ * The @cb function is called once execution has finished. Plugins
+ * should be able to free all their resources at this point much like
+ * after a reset/uninstall callback is called.
+ *
+ * In user-mode it is possible a few un-instrumented instructions from
+ * child threads may run before the host kernel reaps the threads.
+ */
void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id,
qemu_plugin_udata_cb_t cb, void *userdata);
diff --git a/bsd-user/syscall.c b/bsd-user/syscall.c
index 7d986e9700..3f44311396 100644
--- a/bsd-user/syscall.c
+++ b/bsd-user/syscall.c
@@ -335,7 +335,7 @@ abi_long do_freebsd_syscall(void *cpu_env, int num, abi_long arg1,
_mcleanup();
#endif
gdb_exit(arg1);
- qemu_plugin_atexit_cb();
+ qemu_plugin_user_exit();
/* XXX: should free thread stack and CPU env */
_exit(arg1);
ret = 0; /* avoid warning */
@@ -437,7 +437,7 @@ abi_long do_netbsd_syscall(void *cpu_env, int num, abi_long arg1,
_mcleanup();
#endif
gdb_exit(arg1);
- qemu_plugin_atexit_cb();
+ qemu_plugin_user_exit();
/* XXX: should free thread stack and CPU env */
_exit(arg1);
ret = 0; /* avoid warning */
@@ -516,7 +516,7 @@ abi_long do_openbsd_syscall(void *cpu_env, int num, abi_long arg1,
_mcleanup();
#endif
gdb_exit(arg1);
- qemu_plugin_atexit_cb();
+ qemu_plugin_user_exit();
/* XXX: should free thread stack and CPU env */
_exit(arg1);
ret = 0; /* avoid warning */
diff --git a/linux-user/exit.c b/linux-user/exit.c
index 70b344048c..527e29cbc1 100644
--- a/linux-user/exit.c
+++ b/linux-user/exit.c
@@ -35,5 +35,5 @@ void preexit_cleanup(CPUArchState *env, int code)
__gcov_dump();
#endif
gdb_exit(code);
- qemu_plugin_atexit_cb();
+ qemu_plugin_user_exit();
}
diff --git a/plugins/core.c b/plugins/core.c
index e1bcdb570d..c573b81a96 100644
--- a/plugins/core.c
+++ b/plugins/core.c
@@ -487,6 +487,39 @@ void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id,
plugin_register_cb_udata(id, QEMU_PLUGIN_EV_ATEXIT, cb, udata);
}
+/*
+ * Handle exit from linux-user. Unlike the normal atexit() mechanism
+ * we need to handle the clean-up manually as it's possible threads
+ * are still running. We need to remove all callbacks from code
+ * generation, flush the current translations and then we can safely
+ * trigger the exit callbacks.
+ */
+
+void qemu_plugin_user_exit(void)
+{
+ enum qemu_plugin_event ev;
+
+ QEMU_LOCK_GUARD(&plugin.lock);
+
+ start_exclusive();
+
+ /* un-register all callbacks except the final AT_EXIT one */
+ for (ev = 0; ev < QEMU_PLUGIN_EV_MAX; ev++) {
+ if (ev != QEMU_PLUGIN_EV_ATEXIT) {
+ struct qemu_plugin_ctx *ctx;
+ QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) {
+ plugin_unregister_cb__locked(ctx, ev);
+ }
+ }
+ }
+
+ tb_flush(current_cpu);
+ end_exclusive();
+
+ /* now it's safe to handle the exit case */
+ qemu_plugin_atexit_cb();
+}
+
/*
* Call this function after longjmp'ing to the main loop. It's possible that the
* last instruction of a TB might have used helpers, and therefore the
--
2.32.0.264.g75ae10bc75