[Qemu-devel] [PATCH] Add virt-v3 machine that uses GIC-500

[Pavel Fedin]

 This is an alternative version of:
https://lists.gnu.org/archive/html/qemu-devel/2015-05/msg00945.html. Code 
merged with
original "virt" implementation.
 I decided to stick to different machine name instead of adding an option 
because of
compatibility with libvirt. In order to make use of a new option we would have 
to teach
libvirt to pass it, which means extensive amount of patching. After all, there 
are
architectural differences between GIC versions, so GICv3-based machine deserves 
to have a
different name. I decided to name it "virt-v3". Feel free to change it if you 
like, i'm
just out of fantasy.
 I have tested the implementation with kernel v4.1rc2 and 16 CPUs. Works 
perfectly fine,
no lockups were observed.

Signed-off-by: Pavel Fedin <address@hidden>
---
 hw/arm/virt.c | 145 ++++++++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 120 insertions(+), 25 deletions(-)

diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 565f573..b1c0eca 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -66,6 +66,10 @@ enum {
     VIRT_CPUPERIPHS,
     VIRT_GIC_DIST,
     VIRT_GIC_CPU,
+    VIRT_GIC_DIST_SPI = VIRT_GIC_CPU,
+    VIRT_ITS_CONTROL,
+    VIRT_ITS_TRANSLATION,
+    VIRT_LPI,
     VIRT_UART,
     VIRT_MMIO,
     VIRT_RTC,
@@ -97,6 +101,7 @@ typedef struct {
 typedef struct {
     MachineState parent;
     bool secure;
+    bool gicv3;
 } VirtMachineState;
 
 #define TYPE_VIRT_MACHINE   "virt"
@@ -107,6 +112,8 @@ typedef struct {
 #define VIRT_MACHINE_CLASS(klass) \
     OBJECT_CLASS_CHECK(VirtMachineClass, klass, TYPE_VIRT_MACHINE)
 
+#define TYPE_VIRTV3_MACHINE   "virt-v3"
+
 /* Addresses and sizes of our components.
  * 0..128MB is space for a flash device so we can run bootrom code such as 
UEFI.
  * 128MB..256MB is used for miscellaneous device I/O.
@@ -121,25 +128,29 @@ typedef struct {
  */
 static const MemMapEntry a15memmap[] = {
     /* Space up to 0x8000000 is reserved for a boot ROM */
-    [VIRT_FLASH] =      {          0, 0x08000000 },
-    [VIRT_CPUPERIPHS] = { 0x08000000, 0x00020000 },
+    [VIRT_FLASH] =           {          0, 0x08000000 },
+    [VIRT_CPUPERIPHS] =      { 0x08000000, 0x00020000 },
     /* GIC distributor and CPU interfaces sit inside the CPU peripheral space 
*/
-    [VIRT_GIC_DIST] =   { 0x08000000, 0x00010000 },
-    [VIRT_GIC_CPU] =    { 0x08010000, 0x00010000 },
-    [VIRT_UART] =       { 0x09000000, 0x00001000 },
-    [VIRT_RTC] =        { 0x09010000, 0x00001000 },
-    [VIRT_FW_CFG] =     { 0x09020000, 0x0000000a },
-    [VIRT_MMIO] =       { 0x0a000000, 0x00000200 },
+    [VIRT_GIC_DIST] =        { 0x08000000, 0x00010000 },
+    [VIRT_GIC_CPU] =         { 0x08010000, 0x00010000 },
+    /* On v3 VIRT_GIC_DIST_SPI takes place of VIRT_GIC_CPU */
+    [VIRT_ITS_CONTROL] =     { 0x08020000, 0x00001000 },
+    [VIRT_ITS_TRANSLATION] = { 0x08030000, 0x00010000 },
+    [VIRT_LPI] =             { 0x08040000, 0x00800000 },
+    [VIRT_UART] =            { 0x09000000, 0x00001000 },
+    [VIRT_RTC] =             { 0x09010000, 0x00001000 },
+    [VIRT_FW_CFG] =          { 0x09020000, 0x0000000a },
+    [VIRT_MMIO] =            { 0x0a000000, 0x00000200 },
     /* ...repeating for a total of NUM_VIRTIO_TRANSPORTS, each of that size */
     /*
      * PCIE verbose map:
      *
-     * MMIO window      { 0x10000000, 0x2eff0000 },
-     * PIO window       { 0x3eff0000, 0x00010000 },
-     * ECAM             { 0x3f000000, 0x01000000 },
+     * MMIO window           { 0x10000000, 0x2eff0000 },
+     * PIO window            { 0x3eff0000, 0x00010000 },
+     * ECAM                  { 0x3f000000, 0x01000000 },
      */
-    [VIRT_PCIE] =       { 0x10000000, 0x30000000 },
-    [VIRT_MEM] =        { 0x40000000, 30ULL * 1024 * 1024 * 1024 },
+    [VIRT_PCIE] =            { 0x10000000, 0x30000000 },
+    [VIRT_MEM] =             { 0x40000000, 30ULL * 1024 * 1024 * 1024 },
 };
 
 static const int a15irqmap[] = {
@@ -299,11 +310,24 @@ static void fdt_add_cpu_nodes(const VirtBoardInfo *vbi)
 {
     int cpu;
 
+    /*
+     * From Documentation/devicetree/bindings/arm/cpus.txt
+     *  On ARM v8 64-bit systems value should be set to 2,
+     *  that corresponds to the MPIDR_EL1 register size.
+     *  If MPIDR_EL1[63:32] value is equal to 0 on all CPUs
+     *  in the system, #address-cells can be set to 1, since
+     *  MPIDR_EL1[63:32] bits are not used for CPUs
+     *  identification.
+     *
+     *  Now GIC500 doesn't support affinities 2 & 3 so currently
+     *  #address-cells can stay 1 until future GIC
+     */
     qemu_fdt_add_subnode(vbi->fdt, "/cpus");
     qemu_fdt_setprop_cell(vbi->fdt, "/cpus", "#address-cells", 0x1);
     qemu_fdt_setprop_cell(vbi->fdt, "/cpus", "#size-cells", 0x0);
 
     for (cpu = vbi->smp_cpus - 1; cpu >= 0; cpu--) {
+        int Aff1, Aff0;
         char *nodename = g_strdup_printf("/cpus/address@hidden", cpu);
         ARMCPU *armcpu = ARM_CPU(qemu_get_cpu(cpu));
 
@@ -317,12 +341,20 @@ static void fdt_add_cpu_nodes(const VirtBoardInfo *vbi)
                                         "enable-method", "psci");
         }
 
-        qemu_fdt_setprop_cell(vbi->fdt, nodename, "reg", cpu);
+        /*
+         * If cpus node's #address-cells property is set to 1
+         * The reg cell bits [23:0] must be set to bits [23:0] of MPIDR_EL1.
+         * Currently we support simple affinity scheme e.g. we can have
+         * 1 cluster with 8 cores but we can't have 8 clusters wit 1 core each.
+         */
+        Aff1 = cpu / 8;
+        Aff0 = cpu % 8;
+        qemu_fdt_setprop_cell(vbi->fdt, nodename, "reg", (Aff1 << 8) | Aff0);
         g_free(nodename);
     }
 }
 
-static uint32_t fdt_add_gic_node(const VirtBoardInfo *vbi)
+static uint32_t fdt_add_gic_node(const VirtBoardInfo *vbi, bool gicv3)
 {
     uint32_t gic_phandle;
 
@@ -330,35 +362,64 @@ static uint32_t fdt_add_gic_node(const VirtBoardInfo *vbi)
     qemu_fdt_setprop_cell(vbi->fdt, "/", "interrupt-parent", gic_phandle);
 
     qemu_fdt_add_subnode(vbi->fdt, "/intc");
-    /* 'cortex-a15-gic' means 'GIC v2' */
-    qemu_fdt_setprop_string(vbi->fdt, "/intc", "compatible",
-                            "arm,cortex-a15-gic");
     qemu_fdt_setprop_cell(vbi->fdt, "/intc", "#interrupt-cells", 3);
     qemu_fdt_setprop(vbi->fdt, "/intc", "interrupt-controller", NULL, 0);
-    qemu_fdt_setprop_sized_cells(vbi->fdt, "/intc", "reg",
+    if (gicv3) {
+        qemu_fdt_setprop_string(vbi->fdt, "/intc", "compatible",
+                                "arm,gic-v3");
+        qemu_fdt_setprop_sized_cells(vbi->fdt, "/intc", "reg",
+                                     2, vbi->memmap[VIRT_GIC_DIST].base,
+                                     2, vbi->memmap[VIRT_GIC_DIST].size,
+#if 0                                /* Currently no need for SPI & ITS */
+                                     2, vbi->memmap[VIRT_GIC_DIST_SPI].base,
+                                     2, vbi->memmap[VIRT_GIC_DIST_SPI].size,
+                                     2, vbi->memmap[VIRT_ITS_CONTROL].base,
+                                     2, vbi->memmap[VIRT_ITS_CONTROL].size,
+                                     2, vbi->memmap[VIRT_ITS_TRANSLATION].base,
+                                     2, vbi->memmap[VIRT_ITS_TRANSLATION].size,
+#endif
+                                     2, vbi->memmap[VIRT_LPI].base,
+                                     2, vbi->memmap[VIRT_LPI].size);
+    } else {
+        /* 'cortex-a15-gic' means 'GIC v2' */
+        qemu_fdt_setprop_string(vbi->fdt, "/intc", "compatible",
+                                "arm,cortex-a15-gic");
+        qemu_fdt_setprop_sized_cells(vbi->fdt, "/intc", "reg",
                                      2, vbi->memmap[VIRT_GIC_DIST].base,
                                      2, vbi->memmap[VIRT_GIC_DIST].size,
                                      2, vbi->memmap[VIRT_GIC_CPU].base,
                                      2, vbi->memmap[VIRT_GIC_CPU].size);
+    }
     qemu_fdt_setprop_cell(vbi->fdt, "/intc", "phandle", gic_phandle);
 
     return gic_phandle;
 }
 
-static uint32_t create_gic(const VirtBoardInfo *vbi, qemu_irq *pic)
+static uint32_t create_gic(const VirtBoardInfo *vbi, qemu_irq *pic, bool gicv3)
 {
     /* We create a standalone GIC v2 */
     DeviceState *gicdev;
     SysBusDevice *gicbusdev;
-    const char *gictype = "arm_gic";
+    const char *gictype;
     int i;
 
-    if (kvm_irqchip_in_kernel()) {
+    if (gicv3) {
+        gictype = "arm_gicv3";
+    } else if (kvm_irqchip_in_kernel()) {
         gictype = "kvm-arm-gic";
+    } else {
+        gictype = "arm_gic";
     }
 
     gicdev = qdev_create(NULL, gictype);
-    qdev_prop_set_uint32(gicdev, "revision", 2);
+
+    for (i = 0; i < vbi->smp_cpus; i++) {
+        CPUState *cpu = qemu_get_cpu(i);
+        CPUARMState *env = cpu->env_ptr;
+        env->nvic = gicdev;
+    }
+
+    qdev_prop_set_uint32(gicdev, "revision", gicv3 ? 3 : 2);
     qdev_prop_set_uint32(gicdev, "num-cpu", smp_cpus);
     /* Note that the num-irq property counts both internal and external
      * interrupts; there are always 32 of the former (mandated by GIC spec).
@@ -368,6 +429,11 @@ static uint32_t create_gic(const VirtBoardInfo *vbi, 
qemu_irq *pic)
     gicbusdev = SYS_BUS_DEVICE(gicdev);
     sysbus_mmio_map(gicbusdev, 0, vbi->memmap[VIRT_GIC_DIST].base);
     sysbus_mmio_map(gicbusdev, 1, vbi->memmap[VIRT_GIC_CPU].base);
+    if (gicv3) {
+        sysbus_mmio_map(gicbusdev, 2, vbi->memmap[VIRT_ITS_CONTROL].base);
+        sysbus_mmio_map(gicbusdev, 3, vbi->memmap[VIRT_ITS_TRANSLATION].base);
+        sysbus_mmio_map(gicbusdev, 4, vbi->memmap[VIRT_LPI].base);
+    }
 
     /* Wire the outputs from each CPU's generic timer to the
      * appropriate GIC PPI inputs, and the GIC's IRQ output to
@@ -392,7 +458,7 @@ static uint32_t create_gic(const VirtBoardInfo *vbi, 
qemu_irq *pic)
         pic[i] = qdev_get_gpio_in(gicdev, i);
     }
 
-    return fdt_add_gic_node(vbi);
+    return fdt_add_gic_node(vbi, gicv3);
 }
 
 static void create_uart(const VirtBoardInfo *vbi, qemu_irq *pic)
@@ -811,7 +877,7 @@ static void machvirt_init(MachineState *machine)
 
     create_flash(vbi);
 
-    gic_phandle = create_gic(vbi, pic);
+    gic_phandle = create_gic(vbi, pic, vms->gicv3);
 
     create_uart(vbi, pic);
 
@@ -886,9 +952,38 @@ static const TypeInfo machvirt_info = {
     .class_init = virt_class_init,
 };
 
+static void virtv3_instance_init(Object *obj)
+{
+    VirtMachineState *vms = VIRT_MACHINE(obj);
+
+    vms->gicv3 = true;
+    virt_instance_init(obj);
+}
+
+static void virtv3_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->name = TYPE_VIRTV3_MACHINE;
+    mc->desc = "ARM Virtual Machine with GICv3",
+    mc->init = machvirt_init;
+    /* With gic3 full implementation (with bitops) rase the lmit to 128 */
+    mc->max_cpus = 64;
+}
+
+static const TypeInfo machvirtv3_info = {
+    .name = TYPE_VIRTV3_MACHINE,
+    .parent = TYPE_VIRT_MACHINE,
+    .instance_size = sizeof(VirtMachineState),
+    .instance_init = virtv3_instance_init,
+    .class_size = sizeof(VirtMachineClass),
+    .class_init = virtv3_class_init,
+};
+
 static void machvirt_machine_init(void)
 {
     type_register_static(&machvirt_info);
+    type_register_static(&machvirtv3_info);
 }
 
 machine_init(machvirt_machine_init);
-- 
1.9.5.msysgit.0