[Qemu-devel] [PATCH v2 5/5] sclp-s390: Add memory hotplug SCLPs

[Matthew Rosato]

Add memory information to read SCP info and add handlers for
Read Storage Element Information, Attach Storage Element,
Assign Storage and Unassign Storage.

Signed-off-by: Matthew Rosato <address@hidden>
---
 hw/s390x/sclp.c    |  244 ++++++++++++++++++++++++++++++++++++++++++++++++++--
 target-s390x/cpu.h |   15 ++++
 target-s390x/kvm.c |    5 ++
 3 files changed, 257 insertions(+), 7 deletions(-)

diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c
index 338dbdf..ea6cf60 100644
--- a/hw/s390x/sclp.c
+++ b/hw/s390x/sclp.c
@@ -16,7 +16,8 @@
 #include "sysemu/kvm.h"
 #include "exec/memory.h"
 #include "sysemu/sysemu.h"
-
+#include "exec/address-spaces.h"
+#include "qemu/config-file.h"
 #include "hw/s390x/sclp.h"
 #include "hw/s390x/event-facility.h"
 
@@ -33,10 +34,18 @@ static inline SCLPEventFacility *get_event_facility(void)
 static void read_SCP_info(SCCB *sccb)
 {
     ReadInfo *read_info = (ReadInfo *) sccb;
+    sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
     CPUState *cpu;
-    int shift = 0;
     int cpu_count = 0;
     int i = 0;
+    int rnsize, rnmax;
+    QemuOpts *opts = qemu_opts_find(qemu_find_opts("memory-opts"), NULL);
+    int slots = qemu_opt_get_number(opts, "slots", 0);
+    int max_avail_slots = s390_get_memslot_count(kvm_state);
+
+    if (slots > max_avail_slots) {
+        slots = max_avail_slots;
+    }
 
     CPU_FOREACH(cpu) {
         cpu_count++;
@@ -52,16 +61,221 @@ static void read_SCP_info(SCCB *sccb)
         read_info->entries[i].type = 0;
     }
 
-    read_info->facilities = cpu_to_be64(SCLP_HAS_CPU_INFO);
+    /*
+     * The storage increment size is a multiple of 1M and is a power of 2.
+     * The number of storage increments must be 1020 or fewer.
+     */
+    while ((ram_size >> (20 + mhd->shift)) > 1020) {
+        mhd->shift++;
+    }
+    while ((mhd->standby_mem_size >> (20 + mhd->shift)) > 1020) {
+        mhd->shift++;
+    }
+
+    mhd->standby_subregion_size = MEM_SECTION_SIZE;
+    /* Deduct the memory slot already used for core */
+    if (slots > 0) {
+        while ((mhd->standby_subregion_size * (slots - 1)
+                < mhd->standby_mem_size)) {
+            mhd->standby_subregion_size = mhd->standby_subregion_size << 1;
+        }
+    }
+    /*
+     * Initialize mapping of guest standby memory sections indicating which
+     * are and are not online. Assume all standby memory begins offline.
+     */
+    if (mhd->standby_state_map == 0) {
+        if (mhd->standby_mem_size % mhd->standby_subregion_size) {
+            mhd->standby_state_map = g_malloc0((mhd->standby_mem_size /
+                                           mhd->standby_subregion_size + 1) *
+                                          (mhd->standby_subregion_size /
+                                           MEM_SECTION_SIZE));
+        } else {
+            mhd->standby_state_map = g_malloc0(mhd->standby_mem_size /
+                                               MEM_SECTION_SIZE);
+        }
+    }
+
+    mhd->padded_ram_size = ram_size + mhd->pad_size;
+    mhd->rzm = 1 << (20 + mhd->shift);
+    rnsize = 1 << mhd->shift;
+    if (rnsize <= 128) {
+        read_info->rnsize = rnsize;
+    } else {
+        read_info->rnsize = 0;
+        read_info->rnsize2 = cpu_to_be32(rnsize);
+    }
 
-    while ((ram_size >> (20 + shift)) > 65535) {
-        shift++;
+    rnmax = ((ram_size + mhd->standby_mem_size + mhd->pad_size)
+             >> (20 + mhd->shift));
+    if (rnmax < 0x10000) {
+        read_info->rnmax = cpu_to_be16(rnmax);
+    } else {
+        read_info->rnmax = cpu_to_be16(0);
+        read_info->rnmax2 = cpu_to_be64(rnmax);
     }
-    read_info->rnmax = cpu_to_be16(ram_size >> (20 + shift));
-    read_info->rnsize = 1 << shift;
+
+    read_info->facilities = cpu_to_be64(SCLP_HAS_CPU_INFO |
+                                        SCLP_FC_ASSIGN_ATTACH_READ_STOR);
+
     sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_READ_COMPLETION);
 }
 
+static void read_storage_element0_info(SCCB *sccb)
+{
+    int i, assigned;
+    int subincrement_id = SCLP_STARTING_SUBINCREMENT_ID;
+    ReadStorageElementInfo *storage_info = (ReadStorageElementInfo *) sccb;
+    sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
+
+    if ((ram_size >> (20 + mhd->shift)) >= 0x10000) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_SCCB_BOUNDARY_VIOLATION);
+        return;
+    }
+
+    /* Return information regarding core memory */
+    storage_info->max_id = cpu_to_be16(mhd->standby_mem_size ? 1 : 0);
+    assigned = ram_size >> (20 + mhd->shift);
+    storage_info->assigned = cpu_to_be16(assigned);
+
+    for (i = 0; i < assigned; i++) {
+        storage_info->entries[i] = cpu_to_be32(subincrement_id);
+        subincrement_id += SCLP_INCREMENT_UNIT;
+    }
+    sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_READ_COMPLETION);
+}
+
+static void read_storage_element1_info(SCCB *sccb)
+{
+    ReadStorageElementInfo *storage_info = (ReadStorageElementInfo *) sccb;
+    sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
+
+    if ((mhd->standby_mem_size >> (20 + mhd->shift)) >= 0x10000) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_SCCB_BOUNDARY_VIOLATION);
+        return;
+    }
+
+    /* Return information regarding standby memory */
+    storage_info->max_id = cpu_to_be16(mhd->standby_mem_size ? 1 : 0);
+    storage_info->assigned = cpu_to_be16(mhd->standby_mem_size >>
+                                         (20 + mhd->shift));
+    storage_info->standby = cpu_to_be16(mhd->standby_mem_size >>
+                                        (20 + mhd->shift));
+    sccb->h.response_code = cpu_to_be16(SCLP_RC_STANDBY_READ_COMPLETION);
+}
+
+static void attach_storage_element(SCCB *sccb, uint16_t element)
+{
+    int i, assigned, subincrement_id;
+    AttachStorageElement *attach_info = (AttachStorageElement *) sccb;
+    sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
+
+    if (element != 1) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
+        return;
+    }
+
+    assigned = mhd->standby_mem_size >> (20 + mhd->shift);
+    attach_info->assigned = cpu_to_be16(assigned);
+    subincrement_id = ((ram_size >> (20 + mhd->shift)) << 16)
+                      + SCLP_STARTING_SUBINCREMENT_ID;
+    for (i = 0; i < assigned; i++) {
+        attach_info->entries[i] = cpu_to_be32(subincrement_id);
+        subincrement_id += SCLP_INCREMENT_UNIT;
+    }
+    sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_COMPLETION);
+}
+
+static void assign_storage(SCCB *sccb)
+{
+    MemoryRegion *mr = NULL;
+    int this_subregion_size;
+    AssignStorage *assign_info = (AssignStorage *) sccb;
+    sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
+    ram_addr_t assign_addr = (assign_info->rn - 1) * mhd->rzm;
+    MemoryRegion *sysmem = get_system_memory();
+
+    if ((assign_addr % MEM_SECTION_SIZE == 0) &&
+        (assign_addr >= mhd->padded_ram_size)) {
+        /* Re-use existing memory region if found */
+        mr = memory_region_find(sysmem, assign_addr, 1).mr;
+        if (!mr) {
+
+            MemoryRegion *standby_ram = g_new(MemoryRegion, 1);
+
+            /* offset to align to standby_subregion_size for allocation */
+            ram_addr_t offset = assign_addr -
+                                (assign_addr - mhd->padded_ram_size)
+                                % mhd->standby_subregion_size;
+
+            /* strlen("standby.ram") + 4 (Max of KVM_MEMORY_SLOTS) +  NULL */
+            char id[16];
+            snprintf(id, 16, "standby.ram%d",
+                     (int)((offset - mhd->padded_ram_size) /
+                     mhd->standby_subregion_size) + 1);
+
+            /* Allocate a subregion of the calculated standby_subregion_size */
+            if (offset + mhd->standby_subregion_size >
+                mhd->padded_ram_size + mhd->standby_mem_size) {
+                this_subregion_size = mhd->padded_ram_size +
+                  mhd->standby_mem_size - offset;
+            } else {
+                this_subregion_size = mhd->standby_subregion_size;
+            }
+
+            memory_region_init_ram(standby_ram, NULL, id, this_subregion_size);
+            vmstate_register_ram_global(standby_ram);
+            memory_region_add_subregion(sysmem, offset, standby_ram);
+        }
+        /* The specified subregion is no longer in standby */
+        mhd->standby_state_map[(assign_addr - mhd->padded_ram_size)
+                               / MEM_SECTION_SIZE] = 1;
+    }
+    sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_COMPLETION);
+}
+
+static void unassign_storage(SCCB *sccb)
+{
+    MemoryRegion *mr = NULL;
+    AssignStorage *assign_info = (AssignStorage *) sccb;
+    sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
+    ram_addr_t unassign_addr = (assign_info->rn - 1) * mhd->rzm;
+    MemoryRegion *sysmem = get_system_memory();
+
+    /* if the addr is a multiple of 256 MB */
+    if ((unassign_addr % MEM_SECTION_SIZE == 0) &&
+        (unassign_addr >= mhd->padded_ram_size)) {
+        mhd->standby_state_map[(unassign_addr -
+                           mhd->padded_ram_size) / MEM_SECTION_SIZE] = 0;
+
+        /* find the specified memory region and destroy it */
+        mr = memory_region_find(sysmem, unassign_addr, 1).mr;
+        if (mr) {
+            int i;
+            int is_removable = 1;
+            ram_addr_t map_offset = (unassign_addr - mhd->padded_ram_size -
+                                     (unassign_addr - mhd->padded_ram_size)
+                                     % mhd->standby_subregion_size);
+            /* Mark all affected subregions as 'standby' once again */
+            for (i = 0;
+                 i < (mhd->standby_subregion_size / MEM_SECTION_SIZE);
+                 i++) {
+
+                if (mhd->standby_state_map[i + map_offset / MEM_SECTION_SIZE]) 
{
+                    is_removable = 0;
+                    break;
+                }
+            }
+            if (is_removable) {
+                memory_region_del_subregion(sysmem, mr);
+                memory_region_destroy(mr);
+                g_free(mr);
+            }
+        }
+    }
+    sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_COMPLETION);
+}
+
 /* Provide information about the CPU */
 static void sclp_read_cpu_info(SCCB *sccb)
 {
@@ -103,6 +317,22 @@ static void sclp_execute(SCCB *sccb, uint32_t code)
     case SCLP_CMDW_READ_CPU_INFO:
         sclp_read_cpu_info(sccb);
         break;
+    case SCLP_READ_STORAGE_ELEMENT_INFO:
+        if (code & 0xff00) {
+            read_storage_element1_info(sccb);
+        } else {
+            read_storage_element0_info(sccb);
+        }
+        break;
+    case SCLP_ATTACH_STORAGE_ELEMENT:
+        attach_storage_element(sccb, (code & 0xff00) >> 8);
+        break;
+    case SCLP_ASSIGN_STORAGE:
+        assign_storage(sccb);
+        break;
+    case SCLP_UNASSIGN_STORAGE:
+        unassign_storage(sccb);
+        break;
     default:
         efc->command_handler(ef, sccb, code);
         break;
diff --git a/target-s390x/cpu.h b/target-s390x/cpu.h
index 8d25fef..47aa46a 100644
--- a/target-s390x/cpu.h
+++ b/target-s390x/cpu.h
@@ -382,6 +382,7 @@ void s390_sclp_extint(uint32_t parm);
 
 /* from s390-virtio-ccw */
 #define MEM_SECTION_SIZE             0x10000000UL
+#define MAX_AVAIL_SLOTS              32
 
 /* from s390-virtio-bus */
 extern const hwaddr virtio_size;
@@ -1074,6 +1075,7 @@ void kvm_s390_enable_css_support(S390CPU *cpu);
 int kvm_s390_assign_subch_ioeventfd(EventNotifier *notifier, uint32_t sch,
                                     int vq, bool assign);
 int kvm_s390_cpu_restart(S390CPU *cpu);
+int kvm_s390_get_memslot_count(KVMState *s);
 #else
 static inline void kvm_s390_io_interrupt(S390CPU *cpu,
                                         uint16_t subchannel_id,
@@ -1098,6 +1100,10 @@ static inline int kvm_s390_cpu_restart(S390CPU *cpu)
 {
     return -ENOSYS;
 }
+static inline int kvm_s390_get_memslot_count(KVMState *s)
+{
+  return MAX_AVAIL_SLOTS;
+}
 #endif
 
 static inline int s390_cpu_restart(S390CPU *cpu)
@@ -1108,6 +1114,15 @@ static inline int s390_cpu_restart(S390CPU *cpu)
     return -ENOSYS;
 }
 
+static inline int s390_get_memslot_count(KVMState *s)
+{
+    if (kvm_enabled()) {
+        return kvm_s390_get_memslot_count(s);
+    } else {
+        return MAX_AVAIL_SLOTS;
+    }
+}
+
 static inline void s390_io_interrupt(S390CPU *cpu,
                                      uint16_t subchannel_id,
                                      uint16_t subchannel_nr,
diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
index 20c711f..4a8aa7b 100644
--- a/target-s390x/kvm.c
+++ b/target-s390x/kvm.c
@@ -917,3 +917,8 @@ int kvm_s390_assign_subch_ioeventfd(EventNotifier 
*notifier, uint32_t sch,
     }
     return kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
 }
+
+int kvm_s390_get_memslot_count(KVMState *s)
+{
+    return kvm_check_extension(s, KVM_CAP_NR_MEMSLOTS);
+}
-- 
1.7.9.5