Re: [Qemu-arm] [RFC PATCH 12/14] hw/arm/bcm2836: Add the BCM2838 which uses a GICv2

[Luc Michel]

Hi Philippe,

On 9/4/19 7:13 PM, Philippe Mathieu-Daudé wrote:
> The BCM2838 is improvement of the BCM2837:
> - Cortex-A72 instead of the A53
> - peripheral block and local soc controller are mapped differently,
> - GICv2
> - PCIe block
> - exhanced MMU to address over 4GiB of SDRAM
> 
> See https://www.raspberrypi.org/forums/viewtopic.php?t=244479&start=25
> and https://patchwork.kernel.org/patch/11053097/
> 
> This patch starts mapping the GICv2 but interrupt lines are NOT
> wired (yet).
> 
> This is enough to start running the Ubuntu kernel8.img from [1].
> 
> Extract the kernel with:
> 
>    $ mkdir bootpart
>    $ guestfish \
>        --ro \
>        -a ubuntu-18.04.3-preinstalled-server-arm64+raspi4.img \
>        -m /dev/sda1
>    Welcome to guestfish, the guest filesystem shell for
>    editing virtual machine filesystems and disk images.
> 
>    ><fs> ls /
>    COPYING.linux
>    LICENCE.broadcom
>    System.map
>    armstub8-gic.bin
>    bcm2710-rpi-3-b-plus.dtb
>    bcm2710-rpi-3-b.dtb
>    bcm2710-rpi-cm3.dtb
>    bcm2711-rpi-4-b.dtb
>    bcm2837-rpi-3-b-plus.dtb
>    bcm2837-rpi-3-b.dtb
>    cmdline.txt
>    config.txt
>    fixup4.dat
>    fixup4cd.dat
>    fixup4db.dat
>    fixup4x.dat
>    kernel8.img
>    overlays
>    start4.elf
>    start4cd.elf
>    start4db.elf
>    start4x.elf
>    ><fs> copy-out / bootpart/
>    ><fs> q
> 
> Then some progress can be noticed running:
> 
>    $ qemu-system-aarch64 -d unimp,guest_errors,int,in_asm \
>        -M raspi4 \
>        -kernel bootpart/kernel8.img \
>        -dtb bootpart/bcm2711-rpi-4-b.dtb \
>        -initrd bootpart/boot/initrd.img \
>        -append \
>           "earlycon=pl011,0xfe201000 console=ttyAMA0 console=tty1 loglevel=8"
> 
> Not very interesting, but it runs until configuring the GIC.
> (remove 'in_asm' if too verbose).
> 
> TODO:
> 
> - wire IRQs to the GIC :)
> 
> - map the SPI bootrom from [3] (boot sequence: [4])
> 
> - per [2] we could try booting without using the GIC, adding "enable_gic=0"
>    in config.txt. this variable is parsed by the firmware:
> 
>    $ fgrep -r enable_gic bootpart
>    Binary file bootpart/start4x.elf matches
>    Binary file bootpart/start4.elf matches
>    Binary file bootpart/start4db.elf matches
>    Binary file bootpart/start4cd.elf matches
>    bootpart/config.txt:enable_gic=1
> 
>    the stub [5] doesn't seem to check a register for it.
>    maybe it falls back to kernel7l?
> 
> - decompile start4.elf to check how 'enable_gic' is used
>    using vc4 toolchain from [6]
> 
> [1] https://github.com/TheRemote/Ubuntu-Server-raspi4-unofficial/releases
> [2] 
> https://jamesachambers.com/raspberry-pi-ubuntu-server-18-04-2-installation-guide/
> [3] 
> https://www.raspberrypi.org/documentation/hardware/raspberrypi/booteeprom.md
> [4] 
> https://raspberrypi.stackexchange.com/questions/10442/what-is-the-boot-sequence
> [5] 
> https://github.com/raspberrypi/tools/commit/7f4a937e1bacbc111a22552169bc890b4bb26a94#diff-8c41083e9fa0c98f1c3015e11b897444
> [6] https://github.com/christinaa/rpi-open-firmware
> 
> Signed-off-by: Philippe Mathieu-Daudé <address@hidden>
> ---
>   hw/arm/bcm2836.c         | 75 ++++++++++++++++++++++++++++++++++++++++
>   include/hw/arm/bcm2836.h |  3 ++
>   2 files changed, 78 insertions(+)
> 
> diff --git a/hw/arm/bcm2836.c b/hw/arm/bcm2836.c
> index 019e67b906..d89d7cd71d 100644
> --- a/hw/arm/bcm2836.c
> +++ b/hw/arm/bcm2836.c
> @@ -21,6 +21,7 @@ struct BCM283XInfo {
>       const char *cpu_type;
>       hwaddr peri_base; /* Peripheral base address seen by the CPU */
>       hwaddr ctrl_base; /* Interrupt controller and mailboxes etc. */
> +    hwaddr gic_base;
>       int clusterid;
>   };
>   
> @@ -40,9 +41,25 @@ static const BCM283XInfo bcm283x_socs[] = {
>           .ctrl_base = 0x40000000,
>           .clusterid = 0x0,
>       },
> +    {
> +        .name = TYPE_BCM2838,
> +        .cpu_type = ARM_CPU_TYPE_NAME("cortex-a72"),
> +        .peri_base = 0xfe000000,
> +        .ctrl_base = 0xff800000,
> +        .gic_base = 0x40000,
> +    },
>   #endif
>   };
>   
> +#define GIC_NUM_IRQS                256
> +
> +#define GIC_BASE_OFS                0x0000
> +#define GIC_DIST_OFS                0x1000
> +#define GIC_CPU_OFS                 0x2000
> +#define GIC_VIFACE_THIS_OFS         0x4000
> +#define GIC_VIFACE_OTHER_OFS(cpu)  (0x5000 + (cpu) * 0x200)
> +#define GIC_VCPU_OFS                0x6000
> +
>   static void bcm2836_init(Object *obj)
>   {
>       BCM283XState *s = BCM283X(obj);
> @@ -55,6 +72,11 @@ static void bcm2836_init(Object *obj)
>                                   info->cpu_type, &error_abort, NULL);
>       }
>   
> +    if (info->gic_base) {
> +        sysbus_init_child_obj(obj, "gic", &s->gic, sizeof(s->gic),
> +                              TYPE_ARM_GIC);
> +    }
> +
>       sysbus_init_child_obj(obj, "control", &s->control, sizeof(s->control),
>                             TYPE_BCM2836_CONTROL);
>   
> @@ -115,6 +137,59 @@ static void bcm2836_realize(DeviceState *dev, Error 
> **errp)
>   
>       sysbus_mmio_map(SYS_BUS_DEVICE(&s->control), 0, info->ctrl_base);
>   
> +    /* bcm2838 GICv2 */
> +    if (info->gic_base) {
> +        object_property_set_uint(OBJECT(&s->gic), 2, "revision", &err);
> +        if (err) {
> +            error_propagate(errp, err);
> +            return;
> +        }
> +
> +        object_property_set_uint(OBJECT(&s->gic),
> +                                 BCM283X_NCPUS, "num-cpu", &err);
> +        if (err) {
> +            error_propagate(errp, err);
> +            return;
> +        }
> +
> +        object_property_set_uint(OBJECT(&s->gic),
> +                                 32 + GIC_NUM_IRQS, "num-irq", &err);
> +        if (err) {
> +            error_propagate(errp, err);
> +            return;
> +        }
> +
> +        object_property_set_bool(OBJECT(&s->gic),
> +                                 true, "has-virtualization-extensions", 
> &err);
> +        if (err) {
> +            error_propagate(errp, err);
> +            return;
> +        }
> +
> +        object_property_set_bool(OBJECT(&s->gic), true, "realized", &err);
> +        if (err) {
> +            error_propagate(errp, err);
> +            return;
> +        }
> +
> +        sysbus_mmio_map(SYS_BUS_DEVICE(&s->gic), 0,
> +                        info->ctrl_base + info->gic_base + GIC_DIST_OFS);
> +        sysbus_mmio_map(SYS_BUS_DEVICE(&s->gic), 1,
> +                        info->ctrl_base + info->gic_base + GIC_CPU_OFS);
> +        sysbus_mmio_map(SYS_BUS_DEVICE(&s->gic), 2,
> +                        info->ctrl_base + info->gic_base + 
> GIC_VIFACE_THIS_OFS);
> +        sysbus_mmio_map(SYS_BUS_DEVICE(&s->gic), 3,
> +                        info->ctrl_base + info->gic_base + GIC_VCPU_OFS);
> +
> +        for (n = 0; n < BCM283X_NCPUS; n++) {
> +            sysbus_mmio_map(SYS_BUS_DEVICE(&s->gic), 4 + n,
> +                            info->ctrl_base + info->gic_base
> +                            + GIC_VIFACE_OTHER_OFS(n));> +        }
> +
> +        /* TODO wire IRQs!!! */

I think as a bare minimum, you must wire:
  - the ARM generic timer IRQs going out of all the CPUs, into their
respective PPI. Looking at [1], they seem to be connected to the usual
PPIs (at least we have the same mapping in the vexpress, the xynqmp and
the virt board).
  - The PMU interrupts
  - Possibly the GICv2 maintenance interrupt, but I can't find the IRQ
number in the DTS. This is related to the virtualization extension. It
should not prevent Linux from booting if it's not connected (I think KVM
does not even use the GICv2 maintenance interrupts anyway).
  - Finally, Connect the four GICv2 output (irq, fiq, virq, vfiq) to
their respective CPU inputs.


[1]
https://github.com/raspberrypi/linux/blob/rpi-5.3.y/arch/arm/boot/dts/bcm2838.dtsi


Here is a snippet of the virt board, quickly adapted foc this SoC (I
didn't test it):

#define BCM2838_ARCH_TIMER_VIRT_IRQ   11
#define BCM2838_ARCH_TIMER_S_EL1_IRQ  13
#define BCM2838_ARCH_TIMER_NS_EL1_IRQ 14
#define BCM2838_ARCH_TIMER_NS_EL2_IRQ 10

#define BCM2838_PMU_SPI_BASE          16
[...]

 for (i = 0; i < BCM283X_NCPUS; i++) {
        DeviceState *cpudev = DEVICE(qemu_get_cpu(i));
        int ppibase = GIC_NUM_IRQS + i * GIC_INTERNAL + GIC_NR_SGIS;
        int irq;

        /* Mapping from the output timer irq lines from the CPU to the
         * GIC PPI inputs.
         */
        const int timer_irq[] = {
            [GTIMER_PHYS] = BCM2838_ARCH_TIMER_NS_EL1_IRQ,
            [GTIMER_VIRT] = BCM2838_ARCH_TIMER_VIRT_IRQ,
            [GTIMER_HYP]  = BCM2838_ARCH_TIMER_NS_EL2_IRQ,
            [GTIMER_SEC]  = BCM2838_ARCH_TIMER_S_EL1_IRQ,
        };

        for (irq = 0; irq < ARRAY_SIZE(timer_irq); irq++) {
            qdev_connect_gpio_out(cpudev, irq,
                                  qdev_get_gpio_in(gicdev,
                                                   ppibase +
timer_irq[irq]));
        }

        /* I don't know the maintenance IRQ number for the this SoC */
#if 0
        qemu_irq irq = qdev_get_gpio_in(gicdev,
                                        ppibase + ARCH_GIC_MAINT_IRQ);
        sysbus_connect_irq(SYS_BUS_DEVICE(gicbusdevs->gic), i + 4 *
smp_cpus, irq);
#endif

        /* PMU interrupt */
        qdev_connect_gpio_out_named(cpudev, "pmu-interrupt", 0,
                                    qdev_get_gpio_in(gicdev,
BCM2838_PMU_SPI_BASE + i));

        /* Connect the GICv2 outputs to the CPU */
        sysbus_connect_irq(SYS_BUS_DEVICE(gicbusdevs->gic), i,
                           qdev_get_gpio_in(cpudev, ARM_CPU_IRQ));
        sysbus_connect_irq(SYS_BUS_DEVICE(gicbusdevs->gic), i + smp_cpus,
                           qdev_get_gpio_in(cpudev, ARM_CPU_FIQ));
        sysbus_connect_irq(SYS_BUS_DEVICE(gicbusdevs->gic), i + 2 *
smp_cpus,
                           qdev_get_gpio_in(cpudev, ARM_CPU_VIRQ));
        sysbus_connect_irq(SYS_BUS_DEVICE(gicbusdevs->gic), i + 3 *
smp_cpus,
                           qdev_get_gpio_in(cpudev, ARM_CPU_VFIQ));
    }



HTH!

-- 
Luc

> +    }
> +
>       sysbus_connect_irq(SYS_BUS_DEVICE(&s->peripherals), 0,
>           qdev_get_gpio_in_named(DEVICE(&s->control), "gpu-irq", 0));
>       sysbus_connect_irq(SYS_BUS_DEVICE(&s->peripherals), 1,
> diff --git a/include/hw/arm/bcm2836.h b/include/hw/arm/bcm2836.h
> index 97187f72be..31c8fb90bd 100644
> --- a/include/hw/arm/bcm2836.h
> +++ b/include/hw/arm/bcm2836.h
> @@ -13,6 +13,7 @@
>   
>   #include "hw/arm/bcm2835_peripherals.h"
>   #include "hw/intc/bcm2836_control.h"
> +#include "hw/intc/arm_gic.h"
>   #include "target/arm/cpu.h"
>   
>   #define TYPE_BCM283X "bcm283x"
> @@ -26,6 +27,7 @@
>    */
>   #define TYPE_BCM2836 "bcm2836"
>   #define TYPE_BCM2837 "bcm2837"
> +#define TYPE_BCM2838 "bcm2838"
>   
>   typedef struct BCM283XState {
>       /*< private >*/
> @@ -36,6 +38,7 @@ typedef struct BCM283XState {
>       uint32_t enabled_cpus;
>   
>       ARMCPU cpus[BCM283X_NCPUS];
> +    GICState gic;
>       BCM2836ControlState control;
>       BCM2835PeripheralState peripherals;
>   } BCM283XState;
>