[Qemu-devel] [PATCH] qqq: module for synchronizing with a simulation clock

[James J. Nutaro]

  This patch adds an interface for pacing the execution of QEMU to match
    an external simulation clock. Its aim is to permit QEMU to be used
    as a module within a larger simulation system.

 Makefile.target          |  2 +-
 cpus.c                   | 13 ++++++-
 docs/simulation-sync.txt | 61 ++++++++++++++++++++++++++++++
 include/qemu/thread.h    | 11 ++++++
 qemu-options.hx          | 17 +++++++++
 qqq.c                    | 98 ++++++++++++++++++++++++++++++++++++++++++++++++
 qqq.h                    | 39 +++++++++++++++++++
 util/qemu-thread-posix.c | 16 ++++++++
 vl.c                     | 35 ++++++++++++++++-
 9 files changed, 288 insertions(+), 4 deletions(-)
 create mode 100644 docs/simulation-sync.txt
 create mode 100644 qqq.c
 create mode 100644 qqq.h

-- 
2.5.0

>From ef2136cf357c111a4c00bbadf78e3067cc43cab6 Mon Sep 17 00:00:00 2001
From: "James J. Nutaro" <address@hidden>
Date: Thu, 4 Feb 2016 11:12:02 -0500
Subject: [PATCH] This patch adds an interface for pacing the execution of QEMU
 to match an external simulation clock. Its aim is to permit QEMU to be used
 as a module within a larger simulation system.

Signed-off-by: James J. Nutaro <address@hidden>
---
 Makefile.target          |  2 +-
 cpus.c                   | 13 ++++++-
 docs/simulation-sync.txt | 61 ++++++++++++++++++++++++++++++
 include/qemu/thread.h    | 11 ++++++
 qemu-options.hx          | 17 +++++++++
 qqq.c                    | 98 ++++++++++++++++++++++++++++++++++++++++++++++++
 qqq.h                    | 39 +++++++++++++++++++
 util/qemu-thread-posix.c | 16 ++++++++
 vl.c                     | 35 ++++++++++++++++-
 9 files changed, 288 insertions(+), 4 deletions(-)
 create mode 100644 docs/simulation-sync.txt
 create mode 100644 qqq.c
 create mode 100644 qqq.h

diff --git a/Makefile.target b/Makefile.target
index 34ddb7e..a158877 100644
--- a/Makefile.target
+++ b/Makefile.target
@@ -131,7 +131,7 @@ endif #CONFIG_BSD_USER
 #########################################################
 # System emulator target
 ifdef CONFIG_SOFTMMU
-obj-y += arch_init.o cpus.o monitor.o gdbstub.o balloon.o ioport.o numa.o
+obj-y += arch_init.o qqq.o cpus.o monitor.o gdbstub.o balloon.o ioport.o numa.o
 obj-y += qtest.o bootdevice.o
 obj-y += hw/
 obj-$(CONFIG_KVM) += kvm-all.o
diff --git a/cpus.c b/cpus.c
index 882b618..09ec596 100644
--- a/cpus.c
+++ b/cpus.c
@@ -44,6 +44,8 @@
 #include "hw/nmi.h"
 #include "sysemu/replay.h"
 
+#include "qqq.h"
+
 #ifndef _WIN32
 #include "qemu/compatfd.h"
 #endif
@@ -995,10 +997,17 @@ static void qemu_wait_io_event_common(CPUState *cpu)
 static void qemu_tcg_wait_io_event(CPUState *cpu)
 {
     while (all_cpu_threads_idle()) {
-       /* Start accounting real time to the virtual clock if the CPUs
+        /* Start accounting real time to the virtual clock if the CPUs
           are idle.  */
         qemu_clock_warp(QEMU_CLOCK_VIRTUAL);
-        qemu_cond_wait(cpu->halt_cond, &qemu_global_mutex);
+        /* If qqq is running then this wait must timeout so that the
+         * simulation does not become stuck when the cpu idles */
+        if (qqq_enabled()) {
+            qemu_cond_wait_timeout_ns(cpu->halt_cond,
+                                      &qemu_global_mutex, 10000);
+        } else {
+            qemu_cond_wait(cpu->halt_cond, &qemu_global_mutex);
+        }
     }
 
     while (iothread_requesting_mutex) {
diff --git a/docs/simulation-sync.txt b/docs/simulation-sync.txt
new file mode 100644
index 0000000..0f053b3
--- /dev/null
+++ b/docs/simulation-sync.txt
@@ -0,0 +1,61 @@
+= Synchronizing the virtual clock with an external source =
+
+QEMU has a protocol for synchronizing its virtual clock
+with the clock of a simulator in which QEMU is embedded
+as a component. This options is enabled with the -qqq
+argument, and it should generally be accompanied by the
+following additional command line arguments:
+
+-icount 1,sleep=off -rtc clock=vm
+
+The -qqq argument is used to supply file descriptors
+for two Unix pipes. The read pipe is used by QEMU to
+receive synchronization data from the external simulator.
+The write pipe is used by QEMU to supply synchronization
+data to the external emulator. The typical procedure for
+launching QEMU in is synchronization mode has three steps:
+
+(1) Create two pairs of pipes with the Linux pipe function.
+    The code segment that does this might look like
+
+       int pipefd1[2];
+       int pipefd2[2];
+       pipe(pipefd1);
+       pipe(pipefd2);
+
+(2) Fork QEMU with the appropriate command line arguments.
+    The -qqq part of the argument will look something like
+
+       -qqq write=pipefd1[1],read=pipefd2[0]
+
+(3) After forking QEMU, close pipefd1[1] and pipefd2[0].
+    Retain the other pair of pipes for communicating
+       with QEMU.
+
+The synchronization protocol is very simple. To start, the
+external simulator writes an integer to its write pipe with
+the amount of time in microseconds that QEMU is allowed to
+advance. The code segment that does this might look like:
+
+    int ta = 1000; // Advance by 1 millisecond
+       write(pipefd2[1],&ta,sizeof(int));
+
+The external simulator can then advance its clock by this
+same amount. During this time, QEMU and the external simulator
+will be executing in parallel. When the external simulator
+completes its time advance, it waits for QEMU by reading from
+its read pipe. The value read will be the actual number of
+virtual microseconds by which QEMU has advanced its virtual clock.
+This will be greater than or equal to the requested advance.
+The code that does this might look like:
+
+   read(pipefd1[0],&ta,sizeof(int));
+
+These steps are repeated until either (1) the external simulator
+closes its pipes thereby causing QEMU to terminate or (2) QEMU
+stops executing (e.g., if the emulated computer is shutdown) and
+causes SIGPIPE to be generated by the closing of its pipes.
+
+You can find an example of a simulator using this protocol in
+the adevs simulation package at http://sourceforge.net/projects/adevs/
+
diff --git a/include/qemu/thread.h b/include/qemu/thread.h
index 5114ec8..5b4beb9 100644
--- a/include/qemu/thread.h
+++ b/include/qemu/thread.h
@@ -36,6 +36,17 @@ void qemu_cond_destroy(QemuCond *cond);
 void qemu_cond_signal(QemuCond *cond);
 void qemu_cond_broadcast(QemuCond *cond);
 void qemu_cond_wait(QemuCond *cond, QemuMutex *mutex);
+/* This defaults to qemu_cond_wait() on Windows */
+#ifndef _WIN32
+void qemu_cond_wait_timeout_ns(QemuCond *cond, QemuMutex *mutex,
+                               int64_t timeout_ns);
+#else
+void qemu_cond_wait_timeout_ns(QemuCond *cond, QemuMutex *mutex,
+                               int64_t timeout_ns)
+{
+    qemu_cond_wait(cond, mutex);
+}
+#endif
 
 void qemu_sem_init(QemuSemaphore *sem, int init);
 void qemu_sem_post(QemuSemaphore *sem);
diff --git a/qemu-options.hx b/qemu-options.hx
index f31a240..2d4f06e 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -3225,6 +3225,23 @@ many timer interrupts were not processed by the Windows 
guest and will
 re-inject them.
 ETEXI
 
+DEF("qqq", HAS_ARG, QEMU_OPTION_qqq, \
+    "-qqq read=fd,write=fd\n" \
+    "                enable synchronization of the virtual clock \n" \
+    "                with an external simulation clock\n", QEMU_ARCH_ALL)
+STEXI
address@hidden -qqq address@hidden,address@hidden
address@hidden -qqq
+Qemu will use the supplied pipes to synchronize its virtual clock with
+an external simulation clock. This requires the option
+'icount 1,sleep=off'. Qemu will wait until a time slice size in
+microseconds is supplied on the read pipe. Then it will execute for at
+least that number of virtual microseconds before writing the actual
+virtual time that has elapsed in microseconds to the write pipe. This
+cycle will repeat until a zero is elaspsed time is requested, which
+will cause qemu to exit.
+ETEXI
+
 DEF("icount", HAS_ARG, QEMU_OPTION_icount, \
     "-icount 
[shift=N|auto][,align=on|off][,sleep=no,rr=record|replay,rrfile=<filename>]\n" \
     "                enable virtual instruction counter with 2^N clock ticks 
per\n" \
diff --git a/qqq.c b/qqq.c
new file mode 100644
index 0000000..bc41e70
--- /dev/null
+++ b/qqq.c
@@ -0,0 +1,98 @@
+
+#include "qemu/osdep.h"
+#include "qemu/timer.h"
+#include "qqq.h"
+
+/* This is a Linux only feature */
+
+#ifndef _WIN32
+
+#include <unistd.h>
+#include <assert.h>
+
+static int read_fd = -1, write_fd = -1;
+static int64_t t;
+static QEMUTimer *sync_timer;
+
+static void cleanup_and_exit(void)
+{
+    close(read_fd);
+    close(write_fd);
+    exit(0);
+}
+
+static void write_mem_value(int val)
+{
+    if (write(write_fd, &val, sizeof(int)) != sizeof(int)) {
+        /* If the pipe is no good, then assume this is an
+         * indication that we should exit.
+         */
+        cleanup_and_exit();
+    }
+}
+
+static int read_mem_value(void)
+{
+    int tmp;
+    if (read(read_fd, &tmp, sizeof(int)) != sizeof(int)) {
+        /* If the pipe is no good, then assume this is an
+         * indication that we should exit.
+         */
+        cleanup_and_exit();
+    }
+    return tmp;
+}
+
+static void schedule_next_event(void)
+{
+    int time_advance;
+    /* Get the time advance allowed by the simulator. */
+    time_advance = read_mem_value();
+    /* Schedule the next synchronization point */
+    timer_mod(sync_timer, t + time_advance);
+}
+
+static void sync_func(void *data)
+{
+    /* Report the actual elapsed time. */
+    int64_t tnow = qemu_clock_get_us(QEMU_CLOCK_VIRTUAL);
+    int usecs = tnow - t;
+    write_mem_value(usecs);
+    /* Update our time of last event */
+    t = tnow;
+    /* Schedule the next event */
+    schedule_next_event();
+}
+
+bool qqq_enabled(void)
+{
+    return (read_fd > 0 && write_fd > 0);
+}
+
+void setup_qqq(QemuOpts *opts)
+{
+    /* Initialize the simulation clock */
+    t = 0;
+    /* Get the communication pipes */
+    read_fd = qemu_opt_get_number(opts, "read", 0);
+    write_fd = qemu_opt_get_number(opts, "write", 0);
+    /* Start the timer to ensure time warps advance the clock */
+    sync_timer = timer_new_us(QEMU_CLOCK_VIRTUAL, sync_func, NULL);
+    /* Get the time advance that is requested by the simulation */
+    schedule_next_event();
+}
+
+#else
+
+bool qqq_enabled(void)
+{
+    return false;
+}
+
+void setup_qqq(QemuOpts *opts)
+{
+    fprintf(stderr, "-qqq is not supported on Windows, exiting\n");
+    exit(0);
+}
+
+#endif
diff --git a/qqq.h b/qqq.h
new file mode 100644
index 0000000..50e67bc
--- /dev/null
+++ b/qqq.h
@@ -0,0 +1,39 @@
+/*
+ * This work is licensed under the terms of the GNU GPL
+ * version 2. Seethe COPYING file in the top-level directory.
+ *
+ * A module for pacing the rate of advance of the computer
+ * clock in reference to an external simulation clock. The
+ * basic approach used here is adapted from QBox from Green
+ * Socs. The mode of operation is as follows:
+ *
+ * The simulator uses pipes to exchange time advance data.
+ * The external simulator starts the exchange by forking a
+ * QEMU process and passing is descriptors for a read and
+ * write pipe. Then the external simulator writes an integer
+ * (native endian) to the pipe to indicate the number of
+ * microseconds that QEMU should advance. QEMU advances its
+ * virtual clock by this amount and writes to its write pipe
+ * the actual number of microseconds that have advanced.
+ * This process continues until a pipe on either side is
+ * closed, which will either cause QEMU to exit (if the
+ * external simulator closes a pipe) or raise SIGPIPE in the
+ * external simulator (if QEMU closes a pipe).
+ *
+ * Authors:
+ *   James Nutaro <address@hidden>
+ *
+ */
+#ifndef QQQ_H
+#define QQQ_H
+
+#include "qemu/osdep.h"
+#include "qemu-options.h"
+
+void setup_qqq(QemuOpts *opts);
+/* Returns true if qqq is enabled and false otherwise.
+ * It will be enabled only if setup_qqq() is able to
+ * attach to the shared memory segment. */
+bool qqq_enabled(void);
+
+#endif
diff --git a/util/qemu-thread-posix.c b/util/qemu-thread-posix.c
index dbd8094..8d95395 100644
--- a/util/qemu-thread-posix.c
+++ b/util/qemu-thread-posix.c
@@ -134,6 +134,22 @@ void qemu_cond_wait(QemuCond *cond, QemuMutex *mutex)
         error_exit(err, __func__);
 }
 
+void qemu_cond_wait_timeout_ns(QemuCond *cond, QemuMutex *mutex,
+                               int64_t timeout_ns)
+{
+    static const long ns_sec = 1000000000;
+    struct timeval tv;
+    struct timespec ts;
+    int err;
+    gettimeofday(&tv, NULL);
+    ts.tv_sec = tv.tv_sec + (timeout_ns + (tv.tv_usec * 1000)) / ns_sec;
+    ts.tv_nsec = (timeout_ns + (tv.tv_usec * 1000)) % ns_sec;
+    err = pthread_cond_timedwait(&cond->cond, &mutex->lock, &ts);
+    if (err != 0 && err != ETIMEDOUT) {
+        error_exit(err, __func__);
+    }
+}
+
 void qemu_sem_init(QemuSemaphore *sem, int init)
 {
     int rc;
diff --git a/vl.c b/vl.c
index f043009..8c36636 100644
--- a/vl.c
+++ b/vl.c
@@ -125,6 +125,8 @@ int main(int argc, char **argv)
 #include "sysemu/replay.h"
 #include "qapi/qmp/qerror.h"
 
+#include "qqq.h"
+
 #define MAX_VIRTIO_CONSOLES 1
 #define MAX_SCLP_CONSOLES 1
 
@@ -486,6 +488,23 @@ static QemuOptsList qemu_icount_opts = {
     },
 };
 
+static QemuOptsList qemu_qqq_opts = {
+    .name = "qqq",
+    .implied_opt_name = "",
+    .merge_lists = true,
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_qqq_opts.head),
+    .desc = {
+        {
+            .name = "read",
+            .type = QEMU_OPT_NUMBER,
+        }, {
+            .name = "write",
+            .type = QEMU_OPT_NUMBER,
+        },
+        { /* end of list */ }
+    },
+};
+
 static QemuOptsList qemu_semihosting_config_opts = {
     .name = "semihosting-config",
     .implied_opt_name = "enable",
@@ -2969,7 +2988,8 @@ int main(int argc, char **argv, char **envp)
     const char *boot_once = NULL;
     DisplayState *ds;
     int cyls, heads, secs, translation;
-    QemuOpts *hda_opts = NULL, *opts, *machine_opts, *icount_opts = NULL;
+    QemuOpts *hda_opts = NULL, *opts, *machine_opts,
+             *icount_opts = NULL, *qqq_opts = NULL;
     QemuOptsList *olist;
     int optind;
     const char *optarg;
@@ -3031,6 +3051,7 @@ int main(int argc, char **argv, char **envp)
     qemu_add_opts(&qemu_name_opts);
     qemu_add_opts(&qemu_numa_opts);
     qemu_add_opts(&qemu_icount_opts);
+    qemu_add_opts(&qemu_qqq_opts);
     qemu_add_opts(&qemu_semihosting_config_opts);
     qemu_add_opts(&qemu_fw_cfg_opts);
 
@@ -3868,6 +3889,13 @@ int main(int argc, char **argv, char **envp)
                     exit(1);
                 }
                 break;
+            case QEMU_OPTION_qqq:
+                qqq_opts = qemu_opts_parse_noisily(qemu_find_opts("qqq"),
+                                                      optarg, true);
+                if (!qqq_opts) {
+                    exit(1);
+                }
+                break;
             case QEMU_OPTION_incoming:
                 if (!incoming) {
                     runstate_set(RUN_STATE_INMIGRATE);
@@ -4385,6 +4413,11 @@ int main(int argc, char **argv, char **envp)
     /* spice needs the timers to be initialized by this point */
     qemu_spice_init();
 #endif
+    /* try to setup the qqq interface for syncing advance of the
+     * virtual clock with an external simulator */
+    if (qqq_opts && icount_opts) {
+        setup_qqq(qqq_opts);
+    }
 
     cpu_ticks_init();
     if (icount_opts) {
-- 
2.5.0