[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Qemu-devel] [PATCH V4 07/10] NUMA: set guest numa nodes memory policy
From: |
Wanlong Gao |
Subject: |
[Qemu-devel] [PATCH V4 07/10] NUMA: set guest numa nodes memory policy |
Date: |
Thu, 4 Jul 2013 17:53:14 +0800 |
Set the guest numa nodes memory policies using the mbind(2)
system call node by node.
After this patch, we are able to set guest nodes memory policies
through the QEMU options, this arms to solve the guest cross
nodes memory access performance issue.
And as you all know, if PCI-passthrough is used,
direct-attached-device uses DMA transfer between device and qemu process.
All pages of the guest will be pinned by get_user_pages().
KVM_ASSIGN_PCI_DEVICE ioctl
kvm_vm_ioctl_assign_device()
=>kvm_assign_device()
=> kvm_iommu_map_memslots()
=> kvm_iommu_map_pages()
=> kvm_pin_pages()
So, with direct-attached-device, all guest page's page count will be +1 and
any page migration will not work. AutoNUMA won't too.
So, we should set the guest nodes memory allocation policies before
the pages are really mapped.
Signed-off-by: Andre Przywara <address@hidden>
Signed-off-by: Wanlong Gao <address@hidden>
---
cpus.c | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 87 insertions(+)
diff --git a/cpus.c b/cpus.c
index 496d5ce..7240de7 100644
--- a/cpus.c
+++ b/cpus.c
@@ -60,6 +60,15 @@
#endif /* CONFIG_LINUX */
+#ifdef CONFIG_NUMA
+#include <numa.h>
+#include <numaif.h>
+#ifndef MPOL_F_RELATIVE_NODES
+#define MPOL_F_RELATIVE_NODES (1 << 14)
+#define MPOL_F_STATIC_NODES (1 << 15)
+#endif
+#endif
+
static CPUArchState *next_cpu;
static bool cpu_thread_is_idle(CPUState *cpu)
@@ -1171,6 +1180,75 @@ static void tcg_exec_all(void)
exit_request = 0;
}
+#ifdef CONFIG_NUMA
+static int node_parse_bind_mode(unsigned int nodeid)
+{
+ int bind_mode;
+
+ switch (numa_info[nodeid].flags & NODE_HOST_POLICY_MASK) {
+ case NODE_HOST_BIND:
+ bind_mode = MPOL_BIND;
+ break;
+ case NODE_HOST_INTERLEAVE:
+ bind_mode = MPOL_INTERLEAVE;
+ break;
+ case NODE_HOST_PREFERRED:
+ bind_mode = MPOL_PREFERRED;
+ break;
+ default:
+ bind_mode = MPOL_DEFAULT;
+ return bind_mode;
+ }
+
+ bind_mode |= (numa_info[nodeid].flags & NODE_HOST_RELATIVE) ?
+ MPOL_F_RELATIVE_NODES : MPOL_F_STATIC_NODES;
+
+ return bind_mode;
+}
+#endif
+
+static int set_node_mpol(unsigned int nodeid)
+{
+#ifdef CONFIG_NUMA
+ void *ram_ptr;
+ RAMBlock *block;
+ ram_addr_t len, ram_offset = 0;
+ int bind_mode;
+ int i;
+
+ QTAILQ_FOREACH(block, &ram_list.blocks, next) {
+ if (!strcmp(block->mr->name, "pc.ram")) {
+ break;
+ }
+ }
+
+ if (block->host == NULL)
+ return -1;
+
+ ram_ptr = block->host;
+ for (i = 0; i < nodeid; i++) {
+ len = numa_info[i].node_mem;
+ ram_offset += len;
+ }
+
+ len = numa_info[i].node_mem;
+ bind_mode = node_parse_bind_mode(i);
+
+ /* This is a workaround for a long standing bug in Linux'
+ * mbind implementation, which cuts off the last specified
+ * node. To stay compatible should this bug be fixed, we
+ * specify one more node and zero this one out.
+ */
+ clear_bit(numa_num_configured_nodes() + 1, numa_info[i].host_mem);
+ if (mbind(ram_ptr + ram_offset, len, bind_mode,
+ numa_info[i].host_mem, numa_num_configured_nodes() + 1, 0)) {
+ perror("mbind");
+ return -1;
+ }
+#endif
+ return 0;
+}
+
void set_numa_modes(void)
{
CPUArchState *env;
@@ -1185,6 +1263,15 @@ void set_numa_modes(void)
}
}
}
+
+#ifdef CONFIG_NUMA
+ for (i = 0; i < nb_numa_nodes; i++) {
+ if (set_node_mpol(i) == -1) {
+ fprintf(stderr,
+ "qemu: can't set host memory policy for node%d\n", i);
+ }
+ }
+#endif
}
void list_cpus(FILE *f, fprintf_function cpu_fprintf, const char *optarg)
--
1.8.3.2.634.g7a3187e
- [Qemu-devel] [PATCH V4 02/10] NUMA: Add numa_info structure to contain numa nodes info, (continued)
- [Qemu-devel] [PATCH V4 02/10] NUMA: Add numa_info structure to contain numa nodes info, Wanlong Gao, 2013/07/04
- [Qemu-devel] [PATCH V4 04/10] NUMA: parse guest numa nodes memory policy, Wanlong Gao, 2013/07/04
- [Qemu-devel] [PATCH V4 09/10] NUMA: add hmp command set-mpol, Wanlong Gao, 2013/07/04
- [Qemu-devel] [PATCH V4 05/10] NUMA: handle Error in cpus, mpol and hostnode parser, Wanlong Gao, 2013/07/04
- [Qemu-devel] [PATCH V4 10/10] NUMA: show host memory policy info in info numa command, Wanlong Gao, 2013/07/04
- [Qemu-devel] [PATCH V4 07/10] NUMA: set guest numa nodes memory policy,
Wanlong Gao <=
- [Qemu-devel] [PATCH V4 08/10] NUMA: add qmp command set-mpol to set memory policy for NUMA node, Wanlong Gao, 2013/07/04
- Re: [Qemu-devel] [PATCH V4 08/10] NUMA: add qmp command set-mpol to set memory policy for NUMA node, Wanlong Gao, 2013/07/15
- Re: [Qemu-devel] [PATCH V4 08/10] NUMA: add qmp command set-mpol to set memory policy for NUMA node, Eric Blake, 2013/07/08
Re: [Qemu-devel] [PATCH V4 00/10] Add support for binding guest numa nodes to host numa nodes, Paolo Bonzini, 2013/07/04