[Qemu-devel] [PATCH 5/7 v2] kvm: improve handling of overlapping slots

[Jan Kiszka]

This reworks the slot management to handle all patterns of
cpu_register_physical_memory*, finally allowing to reset KVM guests (so
far address remapping on reset broke the slot management).

Nevertheless, KVM kernel bug in older versions forces us to track
previousfragmentations and maintain them (as that bug prevents
registering larger slots that overlap also deleted ones). When affected
KVM versions are detected, we apply a workaround that covers all
currently used patterns.

Changes in v2:
 - detect KVM kernel bug and apply the workaround only when needed

Signed-off-by: Jan Kiszka <address@hidden>
---

 kvm-all.c |  176 +++++++++++++++++++++++++++++++++++++++++++------------------
 1 files changed, 124 insertions(+), 52 deletions(-)

diff --git a/kvm-all.c b/kvm-all.c
index 3e4e421..9e9b462 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -57,6 +57,7 @@ struct KVMState
     int fd;
     int vmfd;
     int coalesced_mmio;
+    int broken_set_mem_region;
 #ifdef KVM_CAP_SET_GUEST_DEBUG
     struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
 #endif
@@ -98,19 +99,31 @@ static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
     return NULL;
 }
 
-static KVMSlot *kvm_lookup_slot(KVMState *s, target_phys_addr_t start_addr)
+/*
+ * Find overlapping slot with lowest start address
+ */
+static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
+                                            target_phys_addr_t start_addr,
+                                            target_phys_addr_t end_addr)
 {
+    KVMSlot *found = NULL;
     int i;
 
     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
         KVMSlot *mem = &s->slots[i];
 
-        if (start_addr >= mem->start_addr &&
-            start_addr < (mem->start_addr + mem->memory_size))
-            return mem;
+        if (mem->memory_size == 0 ||
+            (found && found->start_addr < mem->start_addr)) {
+            continue;
+        }
+
+        if (end_addr > mem->start_addr &&
+            start_addr < mem->start_addr + mem->memory_size) {
+            found = mem;
+        }
     }
 
-    return NULL;
+    return found;
 }
 
 static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
@@ -386,6 +399,14 @@ int kvm_init(int smp_cpus)
         s->coalesced_mmio = ret;
 #endif
 
+    s->broken_set_mem_region = 1;
+#ifdef KVM_CAP_JOIN_MEMORY_REGIONS_WORKS
+    ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
+    if (ret > 0) {
+        s->broken_set_mem_region = 0;
+    }
+#endif
+
     ret = kvm_arch_init(s, smp_cpus);
     if (ret < 0)
         goto err;
@@ -567,7 +588,8 @@ void kvm_set_phys_mem(target_phys_addr_t start_addr,
 {
     KVMState *s = kvm_state;
     ram_addr_t flags = phys_offset & ~TARGET_PAGE_MASK;
-    KVMSlot *mem;
+    KVMSlot *mem, old;
+    int err;
 
     if (start_addr & ~TARGET_PAGE_MASK) {
         fprintf(stderr, "Only page-aligned memory slots supported\n");
@@ -577,55 +599,101 @@ void kvm_set_phys_mem(target_phys_addr_t start_addr,
     /* KVM does not support read-only slots */
     phys_offset &= ~IO_MEM_ROM;
 
-    mem = kvm_lookup_slot(s, start_addr);
-    if (mem) {
-        if (flags >= IO_MEM_UNASSIGNED) {
-            mem->memory_size = 0;
-            mem->start_addr = start_addr;
-            mem->phys_offset = 0;
-            mem->flags = 0;
-
-            kvm_set_user_memory_region(s, mem);
-        } else if (start_addr >= mem->start_addr &&
-                   (start_addr + size) <= (mem->start_addr +
-                                           mem->memory_size)) {
-            KVMSlot slot;
-            target_phys_addr_t mem_start;
-            ram_addr_t mem_size, mem_offset;
-
-            /* Not splitting */
-            if ((phys_offset - (start_addr - mem->start_addr)) == 
-                mem->phys_offset)
-                return;
-
-            /* unregister whole slot */
-            memcpy(&slot, mem, sizeof(slot));
-            mem->memory_size = 0;
-            kvm_set_user_memory_region(s, mem);
-
-            /* register prefix slot */
-            mem_start = slot.start_addr;
-            mem_size = start_addr - slot.start_addr;
-            mem_offset = slot.phys_offset;
-            if (mem_size)
-                kvm_set_phys_mem(mem_start, mem_size, mem_offset);
-
-            /* register new slot */
-            kvm_set_phys_mem(start_addr, size, phys_offset);
-
-            /* register suffix slot */
-            mem_start = start_addr + size;
-            mem_offset += mem_size + size;
-            mem_size = slot.memory_size - mem_size - size;
-            if (mem_size)
-                kvm_set_phys_mem(mem_start, mem_size, mem_offset);
+    while (1) {
+        mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
+        if (!mem) {
+            break;
+        }
 
+        if (flags < IO_MEM_UNASSIGNED && start_addr >= mem->start_addr &&
+            (start_addr + size <= mem->start_addr + mem->memory_size) &&
+            (phys_offset - start_addr == mem->phys_offset - mem->start_addr)) {
+            /* The new slot fits into the existing one and comes with
+             * identical parameters - nothing to be done. */
             return;
-        } else {
-            printf("Registering overlapping slot\n");
+        }
+
+        old = *mem;
+
+        /* unregister the overlapping slot */
+        mem->memory_size = 0;
+        err = kvm_set_user_memory_region(s, mem);
+        if (err) {
+            fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
+                    __func__, strerror(-err));
             abort();
         }
+
+        /* Workaround for older KVM versions: we can't join slots, even not by
+         * unregistering the previous ones and then registering the larger
+         * slot. We have to maintain the existing fragmentation. Sigh.
+         *
+         * This workaround assumes that the new slot starts at the same
+         * address as the first existing one. If not or if some overlapping
+         * slot comes around later, we will fail (not seen in practice so far)
+         * - and actually require a recent KVM version. */
+        if (s->broken_set_mem_region &&
+            old.start_addr == start_addr && old.memory_size < size &&
+            flags < IO_MEM_UNASSIGNED) {
+            mem = kvm_alloc_slot(s);
+            mem->memory_size = old.memory_size;
+            mem->start_addr = old.start_addr;
+            mem->phys_offset = old.phys_offset;
+            mem->flags = 0;
+
+            err = kvm_set_user_memory_region(s, mem);
+            if (err) {
+                fprintf(stderr, "%s: error updating slot: %s\n", __func__,
+                        strerror(-err));
+                abort();
+            }
+
+            start_addr += old.memory_size;
+            phys_offset += old.memory_size;
+            size -= old.memory_size;
+            continue;
+        }
+
+        /* register prefix slot */
+        if (old.start_addr < start_addr) {
+            mem = kvm_alloc_slot(s);
+            mem->memory_size = start_addr - old.start_addr;
+            mem->start_addr = old.start_addr;
+            mem->phys_offset = old.phys_offset;
+            mem->flags = 0;
+
+            err = kvm_set_user_memory_region(s, mem);
+            if (err) {
+                fprintf(stderr, "%s: error registering prefix slot: %s\n",
+                        __func__, strerror(-err));
+                abort();
+            }
+        }
+
+        /* register suffix slot */
+        if (old.start_addr + old.memory_size > start_addr + size) {
+            ram_addr_t size_delta;
+
+            mem = kvm_alloc_slot(s);
+            mem->start_addr = start_addr + size;
+            size_delta = mem->start_addr - old.start_addr;
+            mem->memory_size = old.memory_size - size_delta;
+            mem->phys_offset = old.phys_offset + size_delta;
+            mem->flags = 0;
+
+            err = kvm_set_user_memory_region(s, mem);
+            if (err) {
+                fprintf(stderr, "%s: error registering suffix slot: %s\n",
+                        __func__, strerror(-err));
+                abort();
+            }
+        }
     }
+
+    /* in case the KVM bug workaround already "consumed" the new slot */
+    if (!size)
+        return;
+
     /* KVM does not need to know about this memory */
     if (flags >= IO_MEM_UNASSIGNED)
         return;
@@ -636,8 +704,12 @@ void kvm_set_phys_mem(target_phys_addr_t start_addr,
     mem->phys_offset = phys_offset;
     mem->flags = 0;
 
-    kvm_set_user_memory_region(s, mem);
-    /* FIXME deal with errors */
+    err = kvm_set_user_memory_region(s, mem);
+    if (err) {
+        fprintf(stderr, "%s: error registering slot: %s\n", __func__,
+                strerror(-err));
+        abort();
+    }
 }
 
 int kvm_ioctl(KVMState *s, int type, ...)

signature.asc
Description: OpenPGP digital signature