[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
Re: [Qemu-devel] [RFC][PATCH v2] add realtime option
From: |
Satoru Moriya |
Subject: |
Re: [Qemu-devel] [RFC][PATCH v2] add realtime option |
Date: |
Fri, 11 Jan 2013 21:44:39 +0000 |
Could anybody review it?
On 12/21/2012 05:46 PM, Satoru Moriya wrote:
> Changelog
> v1 -> v2
> - add RFC tag again
> - change semantics as follows
> - set event threads' priority to maxprio
> - set vcpu threads' priority to maxprio - 1
> - isolate all the posix stuff and put them into os_prioritize() in
> os-posix.c/qemu-os-win32.h to avoid breaking win32 build
> - introduce qemu_init_realtime(), qemu_realtime_is_enable and
> qemu_realtime_get_parameters() and struct QemuRealtimeInfo to
> keep realtime option and remove related global variables in vl.c
> - add other benchmark(qpid-latency-test) result
>
> We have some plans to migrate old enterprise/control systems which
> require low latency (msec order) to kvm virtualized environment. In
> order to satify the requirements, this patch adds realtime option to qemu:
>
> -realtime maxprio=<prio>,policy=<pol>
>
> This option change the scheduling policy and priority to realtime one
> (event threads: maxprio, vcpu threads: maxprio - 1) and mlock all qemu
> and guest memory.
>
> Of course, we need more improvements to keep latency low in qemu
> virtualized environment and this is a first step. OTOH, we can meet
> the requirement of our first migration project with this patch.
>
> [ Note ]
> This version doesn't support vhost, vpnc and linux-aio.
>
> These are some basic performance results:
>
> Host : 4 core, 4GB
> Guest: 1 core, 512MB
>
> Benchmark: qpid-latency-test
> http://qpid.apache.org/
> https://access.redhat.com/knowledge/docs/en-US/Red_Hat_Enterprise_MRG/2/html/Messaging_Installation_and_Configuration_Guide/qpid_latency_test.html
>
> Command:
> - qemu
> $ qemu -smp 1 -m 512 -enable-kvm -netdev tap,id=hostnet1 -device
> virtio-net-pci,netdev=hostnet1 -drive file=vm.img,if=virtio
> (-realtime maxprio=99,policy=fifo)
>
> - benchmark
> $ chrt -f 99 qpid-latency-test --tcp-nodelay --rate 10000 -b
> <server>
>
> Results: worst latency (msec) from 100 run
> - no load
> 1. normal qemu : 17.468400
> 2. chrt qemu(*) : 10.019900
> 3. realtime qemu: 8.048370
>
> - load (iperf, server:vm, client:other physical sercer)
> 4. normal qemu : 26.711100
> 5. chrt qemu : 8.485140
> 6. realtime qemu: 10.176700
>
> (*) $ chrt -f -p 99 <event_thread_tid>
> $ chrt -f -p 98 <vcpu_thread_tid>
>
> Any comments are welcome.
>
> Regards,
> Satoru
>
> Signed-off-by: Satoru Moriya <address@hidden>
> ---
> cpus.c | 17 +++++++++++++++++
> include/qemu/thread.h | 4 ++++
> include/sysemu/os-posix.h | 1 +
> include/sysemu/os-win32.h | 1 +
> os-posix.c | 48
> +++++++++++++++++++++++++++++++++++++++++++++++
> qemu-config.c | 16 ++++++++++++++++
> qemu-options.hx | 9 +++++++++
> qemu-thread-posix.c | 27 ++++++++++++++++++++++++++
> qemu-thread-win32.c | 13 +++++++++++++
> vl.c | 33 ++++++++++++++++++++++++++++++++
> 10 files changed, 169 insertions(+)
>
> diff --git a/cpus.c b/cpus.c
> index 4a7782a..a049970 100644
> --- a/cpus.c
> +++ b/cpus.c
> @@ -734,6 +734,9 @@ static void *qemu_kvm_cpu_thread_fn(void *arg)
> CPUArchState *env = arg;
> CPUState *cpu = ENV_GET_CPU(env);
> int r;
> + int rt_policy, rt_priority;
> + struct sched_param sp;
> +
>
> qemu_mutex_lock(&qemu_global_mutex);
> qemu_thread_get_self(cpu->thread);
> @@ -746,6 +749,20 @@ static void *qemu_kvm_cpu_thread_fn(void *arg)
> exit(1);
> }
>
> + if (qemu_realtime_is_enabled()) {
> + qemu_realtime_get_parameters(&rt_policy, &rt_priority);
> + /*
> + * vcpu threads' priority must be set to event thread priority -1
> + * to avoid starvation.
> + */
> + sp.sched_priority = rt_priority - 1;
> + r = sched_setscheduler(0, rt_policy, &sp);
> + if (r < 0) {
> + perror("Setting realtime policy failed");
> + exit(1);
> + }
> + }
> +
> qemu_kvm_init_cpu_signals(env);
>
> /* signal CPU creation */
> diff --git a/include/qemu/thread.h b/include/qemu/thread.h index
> c02404b..3d8b3d2 100644
> --- a/include/qemu/thread.h
> +++ b/include/qemu/thread.h
> @@ -53,4 +53,8 @@ void qemu_thread_get_self(QemuThread *thread); bool
> qemu_thread_is_self(QemuThread *thread); void qemu_thread_exit(void
> *retval);
>
> +void qemu_init_realtime(int, int);
> +bool qemu_realtime_is_enabled(void);
> +void qemu_realtime_get_parameters(int *, int *);
> +
> #endif
> diff --git a/include/sysemu/os-posix.h b/include/sysemu/os-posix.h
> index 7f198e4..e5995b0 100644
> --- a/include/sysemu/os-posix.h
> +++ b/include/sysemu/os-posix.h
> @@ -31,6 +31,7 @@ void os_set_proc_name(const char *s); void
> os_setup_signal_handling(void); void os_daemonize(void); void
> os_setup_post(void);
> +void os_prioritize(const char *, int);
>
> typedef struct timeval qemu_timeval;
> #define qemu_gettimeofday(tp) gettimeofday(tp, NULL) diff --git
> a/include/sysemu/os-win32.h b/include/sysemu/os-win32.h index
> d0e9234..946b566 100644
> --- a/include/sysemu/os-win32.h
> +++ b/include/sysemu/os-win32.h
> @@ -78,6 +78,7 @@ static inline void os_daemonize(void) {} static
> inline void os_setup_post(void) {} void os_set_line_buffering(void);
> static inline void os_set_proc_name(const char *dummy) {}
> +static inline void os_prioritize(const char *pol, int prio) {}
>
> #if !defined(EPROTONOSUPPORT)
> # define EPROTONOSUPPORT EINVAL
> diff --git a/os-posix.c b/os-posix.c
> index 5c64518..8fe0fa2 100644
> --- a/os-posix.c
> +++ b/os-posix.c
> @@ -33,12 +33,14 @@
> #include <pwd.h>
> #include <grp.h>
> #include <libgen.h>
> +#include <sched.h>
>
> /* Needed early for CONFIG_BSD etc. */ #include "config-host.h"
> #include "sysemu/sysemu.h"
> #include "net/slirp.h"
> #include "qemu-options.h"
> +#include "qemu-thread.h"
>
> #ifdef CONFIG_LINUX
> #include <sys/prctl.h>
> @@ -363,3 +365,49 @@ bool is_daemonized(void) {
> return daemonize;
> }
> +
> +void os_prioritize(const char *rt_sched_policy, int
> +max_sched_priority) {
> + int rt_pol, sys_min_prio, sys_max_prio;
> +
> + if (rt_sched_policy) {
> + if (!strcmp(rt_sched_policy, "rr")) {
> + rt_pol = SCHED_RR;
> + } else if (!strcmp(rt_sched_policy, "fifo")) {
> + rt_pol = SCHED_FIFO;
> + } else {
> + fprintf(stderr, "qemu: invalid option value '%s'\n",
> + rt_sched_policy);
> + exit(1);
> + }
> + } else {
> + rt_pol = SCHED_RR;
> + }
> +
> + sys_min_prio = sched_get_priority_min(rt_pol);
> + sys_max_prio = sched_get_priority_max(rt_pol);
> +
> + if (max_sched_priority < sys_min_prio + 1) {
> + /*
> + * We set event threads' priority to max_sched_priorty and
> + * vcpu threads' to max_sched_priority - 1 in order to avoid
> + * starvation. So, it must be > sys_min_prio + 1.
> + */
> + fprintf(stderr, "qemu: invalid option maxprio=%d. It must be >=
> %d\n",
> + max_sched_priority, sys_min_prio + 1);
> + exit(1);
> + }
> +
> + if (sys_max_prio < max_sched_priority) {
> + fprintf(stderr, "qemu: invalid option maxprio=%d. It must be <=
> %d\n",
> + max_sched_priority, sys_max_prio);
> + exit(1);
> + }
> +
> + qemu_init_realtime(rt_pol, max_sched_priority);
> +
> + if (mlockall(MCL_CURRENT | MCL_FUTURE)) {
> + perror("mlockall");
> + exit(1);
> + }
> +}
> diff --git a/qemu-config.c b/qemu-config.c index 2188c3e..b945d07
> 100644
> --- a/qemu-config.c
> +++ b/qemu-config.c
> @@ -647,6 +647,21 @@ static QemuOptsList qemu_object_opts = {
> },
> };
>
> +static QemuOptsList qemu_realtime_opts = {
> + .name = "realtime",
> + .head = QTAILQ_HEAD_INITIALIZER(qemu_realtime_opts.head),
> + .desc = {
> + {
> + .name = "maxprio",
> + .type = QEMU_OPT_NUMBER,
> + }, {
> + .name = "policy",
> + .type = QEMU_OPT_STRING,
> + },
> + { /* end of list */ }
> + },
> +};
> +
> static QemuOptsList *vm_config_groups[32] = {
> &qemu_drive_opts,
> &qemu_chardev_opts,
> @@ -664,6 +679,7 @@ static QemuOptsList *vm_config_groups[32] = {
> &qemu_sandbox_opts,
> &qemu_add_fd_opts,
> &qemu_object_opts,
> + &qemu_realtime_opts,
> NULL,
> };
>
> diff --git a/qemu-options.hx b/qemu-options.hx index 9df0cde..968a20a
> 100644
> --- a/qemu-options.hx
> +++ b/qemu-options.hx
> @@ -2447,6 +2447,15 @@ STEXI
> Do not start CPU at startup (you must type 'c' in the monitor).
> ETEXI
>
> +DEF("realtime", HAS_ARG, QEMU_OPTION_realtime,
> + "-realtime maxprio=prio[,policy=pol]\n",
> + QEMU_ARCH_ALL)
> +STEXI
> address@hidden -realtime address@hidden,address@hidden
> address@hidden -realtime
> +run qemu as realtime process with priority @var{prio} and policy @var{pol}.
> +ETEXI
> +
> DEF("gdb", HAS_ARG, QEMU_OPTION_gdb, \
> "-gdb dev wait for gdb connection on 'dev'\n", QEMU_ARCH_ALL)
> STEXI
> diff --git a/qemu-thread-posix.c b/qemu-thread-posix.c index
> 7be292e..10a97cc 100644
> --- a/qemu-thread-posix.c
> +++ b/qemu-thread-posix.c
> @@ -22,6 +22,15 @@
> #include <sys/time.h>
> #include "qemu/thread.h"
>
> +struct QemuRealtimeInfo {
> + bool is_realtime;
> + int policy;
> + int max_priority;
> +};
> +typedef struct QemuRealtimeInfo QemuRealtimeInfo;
> +
> +static QemuRealtimeInfo rt_info;
> +
> static void error_exit(int err, const char *msg) {
> fprintf(stderr, "qemu: %s: %s\n", msg, strerror(err)); @@ -324,3
> +333,21 @@ void *qemu_thread_join(QemuThread *thread)
> }
> return ret;
> }
> +
> +void qemu_init_realtime(int rt_sched_policy, int max_sched_priority)
> +{
> + rt_info.is_realtime = true;
> + rt_info.policy = rt_sched_policy;
> + rt_info.max_priority = max_sched_priority; }
> +
> +bool qemu_realtime_is_enabled(void)
> +{
> + return rt_info.is_realtime;
> +}
> +
> +void qemu_realtime_get_parameters(int *policy, int *max_priority) {
> + *policy = rt_info.policy;
> + *max_priority = rt_info.max_priority; }
> diff --git a/qemu-thread-win32.c b/qemu-thread-win32.c index
> 8037b39..3beebcf 100644
> --- a/qemu-thread-win32.c
> +++ b/qemu-thread-win32.c
> @@ -369,3 +369,16 @@ bool qemu_thread_is_self(QemuThread *thread) {
> return GetCurrentThreadId() == thread->tid; }
> +
> +void qemu_init_realtime(int rt_sched_policy, int max_sched_priority)
> +{ }
> +
> +bool qemu_realtime_is_enabled(void)
> +{
> + return false;
> +}
> +
> +void qemu_realtime_get_parameters(int *policy, int *max_priority) { }
> diff --git a/vl.c b/vl.c
> index e6a8d89..c310587 100644
> --- a/vl.c
> +++ b/vl.c
> @@ -29,6 +29,7 @@
> #include <sys/time.h>
> #include <zlib.h>
> #include "qemu/bitmap.h"
> +#include "qemu-thread.h"
>
> /* Needed early for CONFIG_BSD etc. */ #include "config-host.h"
> @@ -1148,6 +1149,17 @@ static void smp_parse(const char *optarg)
> max_cpus = smp_cpus;
> }
>
> +static void configure_realtime(QemuOpts *opts) {
> + const char *pol;
> + int prio;
> +
> + pol = qemu_opt_get(opts, "policy");
> + prio = qemu_opt_get_number(opts, "maxprio", 1);
> +
> + os_prioritize(pol, prio);
> +}
> +
> /***********************************************************/
> /* USB devices */
>
> @@ -1754,9 +1766,22 @@ static void main_loop(void) {
> bool nonblocking;
> int last_io = 0;
> + int rt_policy, rt_priority;
> + struct sched_param sp;
> #ifdef CONFIG_PROFILER
> int64_t ti;
> #endif
> +
> + if (qemu_realtime_is_enabled()) {
> + qemu_realtime_get_parameters(&rt_policy, &rt_priority);
> +
> + sp.sched_priority = rt_priority;;
> + if (sched_setscheduler(0, rt_policy, &sp) < 0) {
> + perror("Setting realtime policy failed");
> + exit(1);
> + }
> + }
> +
> do {
> nonblocking = !kvm_enabled() && last_io > 0; #ifdef
> CONFIG_PROFILER @@ -2758,6 +2783,14 @@ int main(int argc, char **argv,
> char **envp)
> }
> numa_add(optarg);
> break;
> + case QEMU_OPTION_realtime:
> + opts = qemu_opts_parse(qemu_find_opts("realtime"), optarg,
> 0);
> + if (!opts) {
> + fprintf(stderr, "parse error: %s\n", optarg);
> + exit(1);
> + }
> + configure_realtime(opts);
> + break;
> case QEMU_OPTION_display:
> display_type = select_display(optarg);
> break;
>
[Prev in Thread] |
Current Thread |
[Next in Thread] |
- Re: [Qemu-devel] [RFC][PATCH v2] add realtime option,
Satoru Moriya <=