[Qemu-devel] [PATCH v3 2/4] balloon: add balloon bitmap migration capability and setup bitmap migration status.

[Jitendra Kolhe]

Add “skip-balloon” migration capability. The balloon bitmap optimization
will get disabled as a part of migration setup if the “skip-balloon” migration
capability is disabled or if the guest balloon driver has not set
VIRTIO_BALLOON_F_MUST_TELL_HOST feature. In case the balloon bitmap
optimization is disabled, migration setup will resize balloon bitmap ramblock
size to zero to avoid overhead of bitmap migration.

Signed-off-by: Jitendra Kolhe <address@hidden>
---
 balloon.c                     | 58 +++++++++++++++++++++++++++++++++++++++++--
 hw/virtio/virtio-balloon.c    | 10 ++++++++
 include/migration/migration.h |  1 +
 include/sysemu/balloon.h      |  3 +++
 migration/migration.c         |  9 +++++++
 migration/ram.c               |  3 +++
 qapi-schema.json              |  5 +++-
 7 files changed, 86 insertions(+), 3 deletions(-)

diff --git a/balloon.c b/balloon.c
index c814102..5b98aa7 100644
--- a/balloon.c
+++ b/balloon.c
@@ -35,17 +35,29 @@
 #include "qapi/qmp/qjson.h"
 #include "exec/ram_addr.h"
 #include "migration/vmstate.h"
+#include "migration/migration.h"
+#include "qemu/error-report.h"
 
 #define BALLOON_BMAP_NAME       "balloon.bmap"
 #define BALLOON_BMAP_SIZE(nr)   (nr / (((1UL << balloon_bitmap_pfn_shift) * \
                                          sizeof(unsigned long)) - 1))
 
+typedef enum {
+    BALLOON_BITMAP_NONE = 0,
+    BALLOON_BITMAP_INIT,
+    BALLOON_BITMAP_ENABLE,
+    BALLOON_BITMAP_DISABLE,
+    BALLOON_BITMAP_DISABLE_FROM_GUEST,
+} BalloonBitmapState;
+
 static QEMUBalloonEvent *balloon_event_fn;
 static QEMUBalloonStatus *balloon_stat_fn;
+static QEMUBalloonTellHost *balloon_tellhost_fn;
 static QemuMutex balloon_bmap_mutex;
 static MemoryRegion *bmap_mr;
 static unsigned long *bmap;
 static unsigned int balloon_bitmap_pfn_shift;
+static BalloonBitmapState balloon_bitmap_state;
 static void *balloon_opaque;
 static bool balloon_inhibited;
 
@@ -86,9 +98,11 @@ static bool have_balloon(Error **errp)
 
 int qemu_add_balloon_handler(QEMUBalloonEvent *event_func,
                              QEMUBalloonStatus *stat_func,
+                             QEMUBalloonTellHost *tellhost_func,
                              void *opaque, int balloon_pfn_shift)
 {
-    if (balloon_event_fn || balloon_stat_fn || balloon_opaque) {
+    if (balloon_event_fn || balloon_stat_fn ||
+        balloon_tellhost_fn || balloon_opaque) {
         /* We're already registered one balloon handler.  How many can
          * a guest really have?
          */
@@ -96,6 +110,7 @@ int qemu_add_balloon_handler(QEMUBalloonEvent *event_func,
     }
     balloon_event_fn = event_func;
     balloon_stat_fn = stat_func;
+    balloon_tellhost_fn = tellhost_func;
     balloon_bitmap_pfn_shift = balloon_pfn_shift;
     balloon_opaque = opaque;
 
@@ -108,6 +123,7 @@ int qemu_add_balloon_handler(QEMUBalloonEvent *event_func,
     vmstate_register_ram_global(bmap_mr);
     bmap = memory_region_get_ram_ptr(bmap_mr);
     bitmap_clear(bmap, 0, (last_ram_offset() >> balloon_bitmap_pfn_shift));
+    balloon_bitmap_state = BALLOON_BITMAP_INIT;
     return 0;
 }
 
@@ -118,8 +134,11 @@ void qemu_remove_balloon_handler(void *opaque)
     }
     object_unref(OBJECT(bmap_mr));
     bmap = NULL;
+    balloon_bitmap_state = BALLOON_BITMAP_NONE;
+
     balloon_event_fn = NULL;
     balloon_stat_fn = NULL;
+    balloon_tellhost_fn = NULL;
     balloon_opaque = NULL;
 }
 
@@ -154,7 +173,7 @@ void qmp_balloon(int64_t target, Error **errp)
 /* Should be called with balloon bitmap mutex lock held */
 void qemu_balloon_bitmap_update(ram_addr_t addr, int deflate)
 {
-    unsigned long offset = 0;
+    unsigned long offset = 0, byte_offset;
 
     if (!bmap) {
         return;
@@ -165,6 +184,11 @@ void qemu_balloon_bitmap_update(ram_addr_t addr, int 
deflate)
     } else {
         clear_bit(offset, bmap);
     }
+
+    if (balloon_bitmap_state == BALLOON_BITMAP_ENABLE) {
+        byte_offset = offset / BITS_PER_BYTE;
+        memory_region_set_dirty(bmap_mr, byte_offset, BITS_PER_BYTE);
+    }
 }
 
 /* Handle Ram hotplug case, only called in case old < new */
@@ -205,3 +229,33 @@ void qemu_balloon_bitmap_extend(RAMBlock *new_block,
     qemu_mutex_unlock_balloon_bitmap();
     g_free(old_bitmap);
 }
+
+void qemu_balloon_bitmap_setup(void)
+{
+    Error *err = NULL;
+    Error **errp = &err;
+    int ret = -1;
+
+    if (!bmap || balloon_bitmap_state != BALLOON_BITMAP_INIT) {
+        return;
+    }
+
+    if (!have_balloon(errp) || !migrate_skip_balloon()) {
+        balloon_bitmap_state = BALLOON_BITMAP_DISABLE;
+    } else {
+        balloon_tellhost_fn(balloon_opaque, &ret);
+        if (ret != 1) {
+            error_report("Guest balloon driver does not support "
+                         "MUST_TELL_HOST feature, enabling skip-balloon "
+                         "may not have any effect");
+            balloon_bitmap_state = BALLOON_BITMAP_DISABLE_FROM_GUEST;
+        } else {
+            balloon_bitmap_state = BALLOON_BITMAP_ENABLE;
+        }
+    }
+
+    if (bmap && (balloon_bitmap_state == BALLOON_BITMAP_DISABLE_FROM_GUEST ||
+                 balloon_bitmap_state == BALLOON_BITMAP_DISABLE)) {
+        memory_region_ram_resize(bmap_mr, 0, &error_fatal);
+    }
+}
diff --git a/hw/virtio/virtio-balloon.c b/hw/virtio/virtio-balloon.c
index 33750f7..5031fa8 100644
--- a/hw/virtio/virtio-balloon.c
+++ b/hw/virtio/virtio-balloon.c
@@ -388,6 +388,7 @@ static uint64_t virtio_balloon_get_features(VirtIODevice 
*vdev, uint64_t f,
     VirtIOBalloon *dev = VIRTIO_BALLOON(vdev);
     f |= dev->host_features;
     virtio_add_feature(&f, VIRTIO_BALLOON_F_STATS_VQ);
+    virtio_add_feature(&f, VIRTIO_BALLOON_F_MUST_TELL_HOST);
     return f;
 }
 
@@ -398,6 +399,14 @@ static void virtio_balloon_stat(void *opaque, BalloonInfo 
*info)
                                              VIRTIO_BALLOON_PFN_SHIFT);
 }
 
+static void virtio_balloon_tellhost_supported(void *opaque, int *status)
+{
+    VirtIOBalloon *dev = VIRTIO_BALLOON(opaque);
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+
+    *status = virtio_vdev_has_feature(vdev, VIRTIO_BALLOON_F_MUST_TELL_HOST);
+}
+
 static void virtio_balloon_to_target(void *opaque, ram_addr_t target)
 {
     VirtIOBalloon *dev = VIRTIO_BALLOON(opaque);
@@ -460,6 +469,7 @@ static void virtio_balloon_device_realize(DeviceState *dev, 
Error **errp)
 
     ret = qemu_add_balloon_handler(virtio_balloon_to_target,
                                    virtio_balloon_stat,
+                                   virtio_balloon_tellhost_supported,
                                    s, VIRTIO_BALLOON_PFN_SHIFT);
 
     if (ret < 0) {
diff --git a/include/migration/migration.h b/include/migration/migration.h
index ac2c12c..6c1d1af 100644
--- a/include/migration/migration.h
+++ b/include/migration/migration.h
@@ -267,6 +267,7 @@ void migrate_del_blocker(Error *reason);
 
 bool migrate_postcopy_ram(void);
 bool migrate_zero_blocks(void);
+bool migrate_skip_balloon(void);
 
 bool migrate_auto_converge(void);
 
diff --git a/include/sysemu/balloon.h b/include/sysemu/balloon.h
index ebaa292..8cf5a2f 100644
--- a/include/sysemu/balloon.h
+++ b/include/sysemu/balloon.h
@@ -18,9 +18,11 @@
 
 typedef void (QEMUBalloonEvent)(void *opaque, ram_addr_t target);
 typedef void (QEMUBalloonStatus)(void *opaque, BalloonInfo *info);
+typedef void (QEMUBalloonTellHost) (void *opaque, int *status);
 
 int qemu_add_balloon_handler(QEMUBalloonEvent *event_func,
                              QEMUBalloonStatus *stat_func,
+                             QEMUBalloonTellHost *tellhost_func,
                              void *opaque, int balloon_pfn_shift);
 void qemu_remove_balloon_handler(void *opaque);
 bool qemu_balloon_is_inhibited(void);
@@ -30,5 +32,6 @@ void qemu_mutex_unlock_balloon_bitmap(void);
 void qemu_balloon_bitmap_update(ram_addr_t addr, int deflate);
 void qemu_balloon_bitmap_extend(RAMBlock *new_block,
                                 ram_addr_t old, ram_addr_t new);
+void qemu_balloon_bitmap_setup(void);
 
 #endif
diff --git a/migration/migration.c b/migration/migration.c
index 991313a..25678cc 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -1200,6 +1200,15 @@ int migrate_use_xbzrle(void)
     return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE];
 }
 
+bool migrate_skip_balloon(void)
+{
+    MigrationState *s;
+
+    s = migrate_get_current();
+
+    return s->enabled_capabilities[MIGRATION_CAPABILITY_SKIP_BALLOON];
+}
+
 int64_t migrate_xbzrle_cache_size(void)
 {
     MigrationState *s;
diff --git a/migration/ram.c b/migration/ram.c
index 3f05738..a4c3582 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -41,6 +41,7 @@
 #include "trace.h"
 #include "exec/ram_addr.h"
 #include "qemu/rcu_queue.h"
+#include "sysemu/balloon.h"
 
 #ifdef DEBUG_MIGRATION_RAM
 #define DPRINTF(fmt, ...) \
@@ -1921,6 +1922,8 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
         acct_clear();
     }
 
+    qemu_balloon_bitmap_setup();
+
     /* For memory_global_dirty_log_start below.  */
     qemu_mutex_lock_iothread();
 
diff --git a/qapi-schema.json b/qapi-schema.json
index 54634c4..d2002c2 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -544,11 +544,14 @@
 #          been migrated, pulling the remaining pages along as needed. NOTE: If
 #          the migration fails during postcopy the VM will fail.  (since 2.6)
 #
+# @skip-balloon: Skip scanning ram pages released by virtio-balloon driver.
+#          (since 2.7)
+#
 # Since: 1.2
 ##
 { 'enum': 'MigrationCapability',
   'data': ['xbzrle', 'rdma-pin-all', 'auto-converge', 'zero-blocks',
-           'compress', 'events', 'postcopy-ram'] }
+           'compress', 'events', 'postcopy-ram', 'skip-balloon'] }
 
 ##
 # @MigrationCapabilityStatus
-- 
1.8.3.1