[Qemu-ppc] [PATCH v4 03/28] spapr: introduce a new IRQ backend using fixed IRQ number ranges

[Cédric Le Goater]

The proposed layout of the IRQ number space is organized as follow :

   RANGES             DEVICES

   0x0000 - 0x0FFF    Reserved for future use (IPI = 2)

   0x1000 - 0x1000    1 EPOW
   0x1001 - 0x1001    1 HOTPLUG
   0x1002 - 0x10FF    unused
   0x1100 - 0x11FF    256 VIO devices    (1 IRQ each)
   0x1200 - 0x1283    32 PCI LSI devices (4 IRQs each)
   0x1284 - 0x13FF    unused
   0x1400 - 0x17FF    PCI MSI device 1   (1024 IRQs each)
   0x1800 - 0x1BFF    PCI MSI device 2
   0x1c00 - 0x1FFF    PCI MSI device 3

The MSI range is a bit more complex to handle as the IRQs are dynamically
allocated by the guest OS. We use a bitmap allocator under the sPAPR
IRQ backend for these.

The XICS IRQ number space is increased to 4K, which gives three MSI
ranges of 1K per PHB device. If more PHB devices are needed, the total
number of IRQs in the backend should be increased.

Signed-off-by: Cédric Le Goater <address@hidden>
---
 include/hw/ppc/spapr.h     |   2 +
 include/hw/ppc/spapr_irq.h |  12 +++
 hw/ppc/spapr.c             |  22 ++++
 hw/ppc/spapr_irq.c         | 253 ++++++++++++++++++++++++++++++++++++++++++++-
 4 files changed, 288 insertions(+), 1 deletion(-)

diff --git a/include/hw/ppc/spapr.h b/include/hw/ppc/spapr.h
index aba1892f2bd0..bd256fbc9158 100644
--- a/include/hw/ppc/spapr.h
+++ b/include/hw/ppc/spapr.h
@@ -164,6 +164,8 @@ struct sPAPRMachineState {
     /*< public >*/
     char *kvm_type;
 
+    int32_t nr_irqs;
+    unsigned long *irq_map;
     const char *icp_type;
 
     bool cmd_line_caps[SPAPR_CAP_NUM];
diff --git a/include/hw/ppc/spapr_irq.h b/include/hw/ppc/spapr_irq.h
index 67ec7ff162a6..dd9c4038d5b6 100644
--- a/include/hw/ppc/spapr_irq.h
+++ b/include/hw/ppc/spapr_irq.h
@@ -21,8 +21,16 @@
 #define SPAPR_IRQ_PCI_LSI    0x1200  /* 4 IRQs per device */
 #define SPAPR_IRQ_PCI_MSI    0x1400  /* 1K IRQs per device covered by
                                       * a bitmap allocator */
+typedef struct sPAPRPIrqRange {
+    const char   *name;
+    uint32_t     offset;
+    uint32_t     width;
+    uint32_t     max_index;
+} sPAPRPIrqRange;
+
 typedef struct sPAPRIrq {
     uint32_t    nr_irqs;
+    const sPAPRPIrqRange *ranges;
 
     void (*init)(sPAPRMachineState *spapr, Error **errp);
     int (*assign)(sPAPRMachineState *spapr, uint32_t range, uint32_t irq,
@@ -36,8 +44,12 @@ typedef struct sPAPRIrq {
     void (*print_info)(sPAPRMachineState *spapr, Monitor *mon);
 } sPAPRIrq;
 
+extern sPAPRIrq spapr_irq_legacy;
 extern sPAPRIrq spapr_irq_xics;
 
+const sPAPRPIrqRange *spapr_irq_get_range(sPAPRMachineState *spapr,
+                                          uint32_t offset);
+
 int spapr_irq_assign(sPAPRMachineState *spapr, uint32_t range, uint32_t irq,
                      Error **errp);
 int spapr_irq_alloc(sPAPRMachineState *spapr, uint32_t range, uint32_t index,
diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
index fd37ca4803eb..c097a7093e62 100644
--- a/hw/ppc/spapr.c
+++ b/hw/ppc/spapr.c
@@ -1850,6 +1850,24 @@ static const VMStateDescription vmstate_spapr_patb_entry 
= {
     },
 };
 
+static bool spapr_irq_map_needed(void *opaque)
+{
+    sPAPRMachineState *spapr = opaque;
+
+    return spapr->irq_map && !bitmap_empty(spapr->irq_map, spapr->nr_irqs);
+}
+
+static const VMStateDescription vmstate_spapr_irq_map = {
+    .name = "spapr_irq_map",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = spapr_irq_map_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_BITMAP(irq_map, sPAPRMachineState, 0, nr_irqs),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
 static const VMStateDescription vmstate_spapr = {
     .name = "spapr",
     .version_id = 3,
@@ -1877,6 +1895,7 @@ static const VMStateDescription vmstate_spapr = {
         &vmstate_spapr_cap_cfpc,
         &vmstate_spapr_cap_sbbc,
         &vmstate_spapr_cap_ibs,
+        &vmstate_spapr_irq_map,
         NULL
     }
 };
@@ -3913,7 +3932,10 @@ static void 
spapr_machine_2_12_instance_options(MachineState *machine)
 
 static void spapr_machine_2_12_class_options(MachineClass *mc)
 {
+    sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
+
     spapr_machine_3_0_class_options(mc);
+    smc->irq = &spapr_irq_legacy;
     SET_MACHINE_COMPAT(mc, SPAPR_COMPAT_2_12);
 }
 
diff --git a/hw/ppc/spapr_irq.c b/hw/ppc/spapr_irq.c
index 1efccf590899..6bd3dd7e01d7 100644
--- a/hw/ppc/spapr_irq.c
+++ b/hw/ppc/spapr_irq.c
@@ -237,7 +237,7 @@ static void spapr_irq_print_info_legacy(sPAPRMachineState 
*spapr,
     ics_pic_print_info(spapr->ics, mon);
 }
 
-sPAPRIrq spapr_irq_xics = {
+sPAPRIrq spapr_irq_legacy = {
     .nr_irqs     = XICS_IRQS_SPAPR,
     .init        = spapr_irq_init_legacy,
     .assign      = spapr_irq_assign_legacy,
@@ -249,6 +249,257 @@ sPAPRIrq spapr_irq_xics = {
 };
 
 /*
+ * IRQ range helpers for new IRQ backends
+ */
+const sPAPRPIrqRange *spapr_irq_get_range(sPAPRMachineState *spapr,
+                                          uint32_t range)
+{
+    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(spapr);
+    const sPAPRPIrqRange *irq_range = smc->irq->ranges;
+
+    if (irq_range) {
+        /*
+         * The range identifier is also its offset in the IRQ number
+         * space. May be introduce a enum instead.
+         */
+        while (irq_range->name && irq_range->offset != range) {
+            irq_range++;
+        }
+
+        if (!irq_range->name) {
+            return NULL;
+        }
+    }
+
+    return irq_range;
+}
+
+static int spapr_irq_range_base(sPAPRMachineState *spapr, uint32_t range,
+                                    uint32_t index, Error **errp)
+{
+    const sPAPRPIrqRange *irq_range = spapr_irq_get_range(spapr, range);
+
+    if (!irq_range) {
+        error_setg(errp, "Invalid IRQ range %x", range);
+        return -1;
+    }
+
+    if (index > irq_range->max_index) {
+        error_setg(errp, "Index %d too big for IRQ range %s", index,
+                   irq_range->name);
+        return -1;
+    }
+
+    return irq_range->offset + index * irq_range->width;
+}
+
+static int spapr_irq_range_alloc(sPAPRMachineState *spapr,
+                                 uint32_t range, uint32_t index, Error **errp)
+{
+    return spapr_irq_range_base(spapr, range, index, errp);
+}
+
+static int spapr_irq_range_alloc_msi(sPAPRMachineState *spapr, uint32_t range,
+                                     uint32_t index, int num, bool align,
+                                     Error **errp)
+{
+    int msi_base;
+    int irq;
+
+    msi_base = spapr_irq_range_base(spapr, SPAPR_IRQ_PCI_MSI, index, errp);
+    if (msi_base == -1) {
+        return -1;
+    }
+
+    /*
+     * The 'align_mask' parameter of bitmap_find_next_zero_area()
+     * should be one less than a power of 2; 0 means no
+     * alignment. Adapt the 'align' value of the former allocator
+     * to fit the requirements of bitmap_find_next_zero_area()
+     */
+    align -= 1;
+
+    irq = bitmap_find_next_zero_area(spapr->irq_map, spapr->nr_irqs,
+                                     msi_base, num, align);
+    if (irq == spapr->nr_irqs) {
+        error_setg(errp, "can't find a free MSI %d-IRQ block", num);
+        return -1;
+    }
+
+    bitmap_set(spapr->irq_map, irq, num);
+    return irq;
+}
+
+static void spapr_irq_range_free_msi(sPAPRMachineState *spapr, int irq, int 
num)
+{
+    bitmap_clear(spapr->irq_map, irq, num);
+}
+
+/*
+ * New XICS IRQ backend
+ *
+ * using the IRQ ranges and device indexes
+ */
+static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp)
+{
+    MachineState *machine = MACHINE(spapr);
+    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(machine);
+
+    spapr_irq_init_legacy(spapr, errp);
+
+    /*
+     * Initialize the MSI IRQ allocator. The full XICS IRQ number
+     * space is covered even though the bottow IRQ numbers below the
+     * XICS source number offset (4K) are unused and that only MSI IRQ
+     * numbers can be allocated. It does waste some bytes but it makes
+     * things easier. We will optimize later.
+     */
+    spapr->nr_irqs  = smc->irq->nr_irqs + spapr->ics->offset;
+    spapr->irq_map  = bitmap_new(spapr->nr_irqs);
+}
+
+/*
+ * The 'irq' assignment is only used by the sPAPR VIO devices and it
+ * has been deprecated in QEMU 3.0. This handler should be removed
+ * soon.
+ */
+static int spapr_irq_assign_xics(sPAPRMachineState *spapr, uint32_t range,
+                                 uint32_t irq, Error **errp)
+{
+    assert(range == SPAPR_IRQ_VIO);
+
+    error_setg(errp, "IRQ assignment is not available on the new "
+               " XICS IRQ backend");
+    return -1;
+}
+
+static int spapr_irq_alloc_xics(sPAPRMachineState *spapr, uint32_t range,
+                                uint32_t index, Error **errp)
+{
+    ICSState *ics = spapr->ics;
+    bool lsi = (range == SPAPR_IRQ_PCI_LSI);
+    int irq = spapr_irq_range_alloc(spapr, range, index, errp);
+
+    if (irq < 0) {
+        return irq;
+    }
+
+    /* Update the IRQState in the XICS source */
+    ics_set_irq_type(ics, irq - ics->offset, lsi);
+
+    return irq;
+}
+
+static int spapr_irq_alloc_block_xics(sPAPRMachineState *spapr, uint32_t range,
+                                      uint32_t index, int num, bool align,
+                                      Error **errp)
+{
+    ICSState *ics = spapr->ics;
+    bool lsi = (range == SPAPR_IRQ_PCI_LSI);
+    int irq;
+    int i;
+
+    if (range == SPAPR_IRQ_PCI_MSI) {
+        irq = spapr_irq_range_alloc_msi(spapr, range, index, num, align, errp);
+    } else {
+        /* TODO: check IRQ range width vs. required block size */
+        irq = spapr_irq_range_alloc(spapr, range, index, errp);
+    }
+
+    if (irq < 0) {
+        return irq;
+    }
+
+    /* Update the IRQState in the XICS source */
+    for (i = irq; i < irq + num; ++i) {
+        ics_set_irq_type(ics, i - ics->offset, lsi);
+    }
+
+    return irq;
+}
+
+static void spapr_irq_free_xics(sPAPRMachineState *spapr, int irq, int num,
+                                Error **errp)
+{
+    ICSState *ics = spapr->ics;
+    int msi_base = spapr_irq_range_base(spapr, SPAPR_IRQ_PCI_MSI, 0, NULL);
+    int i;
+
+    /* Any IRQ below MSI base should not be freed */
+    if (irq < msi_base) {
+        error_setg(errp, "IRQs %x-%x can not be freed", irq, irq + num);
+        return;
+    }
+
+    spapr_irq_range_free_msi(spapr, irq, num);
+
+    /* Clear out the IRQState from the XICS source */
+    for (i = irq; i < irq + num; ++i) {
+        if (ics_valid_irq(ics, i)) {
+            uint32_t srcno = i - ics->offset;
+            memset(&ics->irqs[srcno], 0, sizeof(ICSIRQState));
+        }
+    }
+}
+
+static qemu_irq spapr_qirq_xics(sPAPRMachineState *spapr, int irq)
+{
+    return spapr_qirq_legacy(spapr, irq);
+}
+
+static void spapr_irq_print_info_xics(sPAPRMachineState *spapr,
+                                      Monitor *mon)
+{
+    spapr_irq_print_info_legacy(spapr, mon);
+}
+
+/*
+ * XICS IRQ number space
+ *
+ *   RANGES             DEVICES
+ *
+ *   0x0000 - 0x0FFF    Reserved for future use (IPI = 2)
+ *
+ *   0x1000 - 0x1000    1 EPOW
+ *   0x1001 - 0x1001    1 HOTPLUG
+ *   0x1002 - 0x10FF    unused
+ *   0x1100 - 0x11FF    256 VIO devices    (1 IRQ each)
+ *   0x1200 - 0x1283    32 PCI LSI devices (4 IRQs each)
+ *   0x1284 - 0x13FF    unused
+ *   0x1400 - 0x17FF    PCI MSI device 1   (1024 IRQs each)
+ *   0x1800 - 0x1BFF    PCI MSI device 2
+ *   0x1c00 - 0x1FFF    PCI MSI device 3
+ *
+ *   0x2000    ....     not allocated. Need to increase NR_IRQS
+ */
+static const sPAPRPIrqRange spapr_irq_ranges_xics[] = {
+    /* "IPI" Not used */
+    { "EPOW",       SPAPR_IRQ_EPOW,       1,               0      },
+    { "HOTPLUG",    SPAPR_IRQ_HOTPLUG,    1,               0      },
+    { "VIO",        SPAPR_IRQ_VIO,        1,               0xFF   },
+    { "PCI LSI",    SPAPR_IRQ_PCI_LSI,    PCI_NUM_PINS,    0x1F   },
+    { "PCI MSI",    SPAPR_IRQ_PCI_MSI,    0x400,           0x1F   },
+    { NULL,         0,                    0,               0      },
+};
+
+/*
+ * Increase the XICS IRQ number space to 4K. It gives us 3 possible
+ * MSI ranges for the PHBs.
+ */
+
+sPAPRIrq spapr_irq_xics = {
+    .nr_irqs     = 0x1000,
+    .init        = spapr_irq_init_xics,
+    .ranges      = spapr_irq_ranges_xics,
+    .assign      = spapr_irq_assign_xics,
+    .alloc       = spapr_irq_alloc_xics,
+    .alloc_block = spapr_irq_alloc_block_xics,
+    .free        = spapr_irq_free_xics,
+    .qirq        = spapr_qirq_xics,
+    .print_info  = spapr_irq_print_info_xics,
+};
+
+/*
  * sPAPR IRQ frontend routines for devices
  */
 int spapr_irq_assign(sPAPRMachineState *spapr, uint32_t range, uint32_t irq,
-- 
2.13.6