Re: [RFC PATCH] hw/arm/virt: Support NMI injection

[Gavin Shan]

On 12/19/19 3:06 PM, Gavin Shan wrote:
> This supports NMI injection for virtual machine and currently it's only
> supported on GICv3 controller, which is emulated by qemu or host kernel.
> The design is highlighted as below:
>
>     * The NMI is identified by its priority (0x20). In the guest (linux)
>       kernel, the GICC_PMR is set to 0x80, to block all interrupts except
>       the NMIs when the external interrupt is disabled. It means the FIQ
>       and IRQ bit in PSTATE isn't touched when the functionality (NMI) is
>       functional.
>     * LPIs aren't considered as NMIs because of their nature. It means NMI
>       is either SPI or PPI. Besides, the NMIs are injected in round-robin
>       fashion is there are multiple NMIs existing.
>     * When the GICv3 controller is emulated by qemu, the interrupt states
>       (e.g. enabled, priority) is fetched from the corresponding data struct
>       directly. However, we have to pause all CPUs to fetch the interrupt
>       states from host in advance if the GICv3 controller is emulated by
>       host.
>
> The testing scenario is to tweak guest (linux) kernel where the pl011 SPI
> can be enabled as NMI by request_nmi(). Check "/proc/interrupts" after injecting
> several NMIs, to see if the interrupt count is increased or not. The result
> is just as expected.
>
> Signed-off-by: Gavin Shan <address@hidden>
> ---
>   hw/arm/virt.c                      | 24 ++++++++
>   hw/intc/arm_gicv3.c                | 76 ++++++++++++++++++++++++
>   hw/intc/arm_gicv3_kvm.c            | 92 ++++++++++++++++++++++++++++++
>   include/hw/intc/arm_gicv3_common.h |  2 +
>   4 files changed, 194 insertions(+)

Peter, could you please take a look when you have free time? Thanks in advance
for your comments :)

Thanks,
Gavin

> diff --git a/hw/arm/virt.c b/hw/arm/virt.c
> index 39ab5f47e0..fc58ee70b4 100644
> --- a/hw/arm/virt.c
> +++ b/hw/arm/virt.c
> @@ -71,6 +71,8 @@
>   #include "hw/mem/pc-dimm.h"
>   #include "hw/mem/nvdimm.h"
>   #include "hw/acpi/generic_event_device.h"
> +#include "hw/nmi.h"
> +#include "hw/intc/arm_gicv3.h"
>   
>   #define DEFINE_VIRT_MACHINE_LATEST(major, minor, latest) \
>       static void virt_##major##_##minor##_class_init(ObjectClass *oc, \
> @@ -1980,6 +1982,25 @@ static void 
> virt_machine_device_unplug_request_cb(HotplugHandler *hotplug_dev,
>                  " type: %s", object_get_typename(OBJECT(dev)));
>   }
>   
> +static void virt_nmi(NMIState *n, int cpu_index, Error **errp)
> +{
> +    VirtMachineState *vms = VIRT_MACHINE(n);
> +    ARMGICv3CommonClass *agcc;
> +
> +    if (vms->gic_version != 3) {
> +        error_setg(errp, "NMI is only supported by GICv3");
> +        return;
> +    }
> +
> +    agcc = ARM_GICV3_COMMON_GET_CLASS(vms->gic);
> +    if (agcc->inject_nmi) {
> +        agcc->inject_nmi(vms->gic, cpu_index, errp);
> +    } else {
> +        error_setg(errp, "NMI injection isn't supported by %s",
> +                   object_get_typename(OBJECT(vms->gic)));
> +    }
> +}
> +
>   static HotplugHandler *virt_machine_get_hotplug_handler(MachineState *machine,
>                                                           DeviceState *dev)
>   {
> @@ -2025,6 +2046,7 @@ static void virt_machine_class_init(ObjectClass *oc, void 
> *data)
>   {
>       MachineClass *mc = MACHINE_CLASS(oc);
>       HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(oc);
> +    NMIClass *nc = NMI_CLASS(oc);
>   
>       mc->init = machvirt_init;
>       /* Start with max_cpus set to 512, which is the maximum supported by KVM.
> @@ -2051,6 +2073,7 @@ static void virt_machine_class_init(ObjectClass *oc, void 
> *data)
>       hc->pre_plug = virt_machine_device_pre_plug_cb;
>       hc->plug = virt_machine_device_plug_cb;
>       hc->unplug_request = virt_machine_device_unplug_request_cb;
> +    nc->nmi_monitor_handler = virt_nmi;
>       mc->numa_mem_supported = true;
>       mc->auto_enable_numa_with_memhp = true;
>   }
> @@ -2136,6 +2159,7 @@ static const TypeInfo virt_machine_info = {
>       .instance_init = virt_instance_init,
>       .interfaces = (InterfaceInfo[]) {
>            { TYPE_HOTPLUG_HANDLER },
> +         { TYPE_NMI },
>            { }
>       },
>   };
> diff --git a/hw/intc/arm_gicv3.c b/hw/intc/arm_gicv3.c
> index 66eaa97198..d3409cb6ef 100644
> --- a/hw/intc/arm_gicv3.c
> +++ b/hw/intc/arm_gicv3.c
> @@ -338,6 +338,81 @@ static void gicv3_set_irq(void *opaque, int irq, int level)
>       }
>   }
>   
> +static bool arm_gicv3_inject_nmi_once(GICv3State*s, int start, int end)
> +{
> +    GICv3CPUState *cs;
> +    int irq_count = (s->num_irq + (GIC_INTERNAL * (s->num_cpu - 1)));
> +    int i, cpu, irq;
> +
> +    /* SPIs */
> +    for (i = start; (i < end) && (i < (s->num_irq - GIC_INTERNAL)); i++) {
> +        if (gicv3_gicd_enabled_test(s, i + GIC_INTERNAL) &&
> +            s->gicd_ipriority[i + GIC_INTERNAL] == 0x20) {
> +
> +            /*
> +             * Reset the level and toggling the pending bit will ensure
> +             * the interrupt is queued.
> +             */
> +            if (gicv3_gicd_level_test(s, i + GIC_INTERNAL)) {
> +                gicv3_set_irq(s, i, false);
> +            }
> +
> +            gicv3_gicd_pending_set(s, i + GIC_INTERNAL);
> +            gicv3_set_irq(s, i, true);
> +
> +            s->last_nmi_index = (i + 1);
> +            return true;
> +        }
> +    }
> +
> +    /* PPIs */
> +    if (start < (s->num_irq - GIC_INTERNAL)) {
> +        start = (s->num_irq - GIC_INTERNAL);
> +    }
> +
> +    for (i = start; (i < end) && (i < irq_count); i++) {
> +        cpu = (i - ((s->num_irq - GIC_INTERNAL))) / GIC_INTERNAL;
> +        irq = (i - ((s->num_irq - GIC_INTERNAL))) % GIC_INTERNAL;
> +        cs = &s->cpu[cpu];
> +
> +        if ((cs->gicr_ienabler0 & (1 << irq)) &&
> +            cs->gicr_ipriorityr[irq] == 0x20) {
> +
> +            if (extract32(cs->level, irq, 1)) {
> +                gicv3_set_irq(s, i, false);
> +            }
> +
> +            deposit32(cs->gicr_ipendr0, irq, 1, 1);
> +            gicv3_set_irq(s, i, true);
> +
> +            s->last_nmi_index = (i + 1);
> +            if (s->last_nmi_index > irq_count) {
> +                s->last_nmi_index = 0;
> +            }
> +
> +            return true;
> +        }
> +    }
> +
> +    return false;
> +}
> +
> +static void arm_gicv3_inject_nmi(DeviceState *dev, int cpu_index, Error **errp)
> +{
> +    GICv3State *s = ARM_GICV3(dev);
> +    int irq_count = (s->num_irq + (GIC_INTERNAL * (s->num_cpu - 1)));
> +    bool injected;
> +
> +    injected = arm_gicv3_inject_nmi_once(s, s->last_nmi_index, irq_count);
> +    if (!injected) {
> +        injected = arm_gicv3_inject_nmi_once(s, 0, s->last_nmi_index);
> +    }
> +
> +    if (!injected) {
> +        error_setg(errp, "No NMI found");
> +    }
> +}
> +
>   static void arm_gicv3_post_load(GICv3State *s)
>   {
>       /* Recalculate our cached idea of the current highest priority
> @@ -395,6 +470,7 @@ static void arm_gicv3_class_init(ObjectClass *klass, void 
> *data)
>       ARMGICv3CommonClass *agcc = ARM_GICV3_COMMON_CLASS(klass);
>       ARMGICv3Class *agc = ARM_GICV3_CLASS(klass);
>   
> +    agcc->inject_nmi = arm_gicv3_inject_nmi;
>       agcc->post_load = arm_gicv3_post_load;
>       device_class_set_parent_realize(dc, arm_gic_realize, 
> &agc->parent_realize);
>   }
> diff --git a/hw/intc/arm_gicv3_kvm.c b/hw/intc/arm_gicv3_kvm.c
> index 9c7f4ab871..b076d67c52 100644
> --- a/hw/intc/arm_gicv3_kvm.c
> +++ b/hw/intc/arm_gicv3_kvm.c
> @@ -31,6 +31,7 @@
>   #include "gicv3_internal.h"
>   #include "vgic_common.h"
>   #include "migration/blocker.h"
> +#include "sysemu/cpus.h"
>   
>   #ifdef DEBUG_GICV3_KVM
>   #define DPRINTF(fmt, ...) \
> @@ -506,6 +507,96 @@ static void kvm_arm_gicv3_put(GICv3State *s)
>       }
>   }
>   
> +static bool kvm_arm_gicv3_inject_nmi_once(GICv3State *s, int start, int end)
> +{
> +    GICv3CPUState *cs;
> +    int irq_count = (s->num_irq + (GIC_INTERNAL * (s->num_cpu - 1)));
> +    int i, cpu, irq;
> +
> +    /* SPIs */
> +    for (i = start; (i < end) && (i < (s->num_irq - GIC_INTERNAL)); i++) {
> +        if (gicv3_gicd_enabled_test(s, i + GIC_INTERNAL) &&
> +            s->gicd_ipriority[i + GIC_INTERNAL] == 0x20) {
> +            kvm_arm_gicv3_set_irq(s, i, true);
> +
> +            s->last_nmi_index = (i + 1);
> +            return true;
> +        }
> +    }
> +
> +    /* PPIs */
> +    if (start < (s->num_irq - GIC_INTERNAL)) {
> +        start = (s->num_irq - GIC_INTERNAL);
> +    }
> +
> +    for (i = start; (i < end) && (i < irq_count); i++) {
> +        cpu = (i - ((s->num_irq - GIC_INTERNAL))) / GIC_INTERNAL;
> +        irq = (i - ((s->num_irq - GIC_INTERNAL))) % GIC_INTERNAL;
> +        cs = &s->cpu[cpu];
> +
> +        if ((cs->gicr_ienabler0 & (1 << irq)) &&
> +            cs->gicr_ipriorityr[irq] == 0x20) {
> +            kvm_arm_gicv3_set_irq(s, i, true);
> +
> +            s->last_nmi_index = (i + 1);
> +            if (s->last_nmi_index > irq_count) {
> +                s->last_nmi_index = 0;
> +            }
> +
> +            return true;
> +        }
> +    }
> +
> +    return false;
> +}
> +
> +static void kvm_arm_gicv3_snapshot(GICv3State *s)
> +{
> +    GICv3CPUState *c;
> +    uint32_t val;
> +    int i, j;
> +
> +    pause_all_vcpus();
> +
> +    kvm_dist_getbmp(s, GICD_ISENABLER, s->enabled);
> +    kvm_dist_get_priority(s, GICD_IPRIORITYR, s->gicd_ipriority);
> +    for (i = 0; i < s->num_cpu; i++) {
> +        c = &s->cpu[i];
> +
> +        kvm_gicr_access(s, GICR_ISENABLER0, i, &val, false);
> +        c->gicr_ienabler0 = val;
> +
> +        for (j = 0; j < GIC_INTERNAL; j += 4) {
> +            kvm_gicr_access(s, GICR_IPRIORITYR + j, i, &val, false);
> +            c->gicr_ipriorityr[j] = extract32(val, 0, 8);
> +            c->gicr_ipriorityr[j + 1] = extract32(val, 8, 8);
> +            c->gicr_ipriorityr[j + 2] = extract32(val, 16, 8);
> +            c->gicr_ipriorityr[j + 3] = extract32(val, 24, 8);
> +        }
> +    }
> +
> +    resume_all_vcpus();
> +}
> +
> +static void kvm_arm_gicv3_inject_nmi(DeviceState *dev,
> +                                     int cpu_index, Error **errp)
> +{
> +    GICv3State *s = KVM_ARM_GICV3(dev);
> +    int irq_count = (s->num_irq + (GIC_INTERNAL * (s->num_cpu - 1)));
> +    bool injected;
> +
> +    kvm_arm_gicv3_snapshot(s);
> +
> +    injected = kvm_arm_gicv3_inject_nmi_once(s, s->last_nmi_index, irq_count);
> +    if (!injected) {
> +        injected = kvm_arm_gicv3_inject_nmi_once(s, 0, s->last_nmi_index);
> +    }
> +
> +    if (!injected) {
> +        error_setg(errp, "No NMI found");
> +    }
> +}
> +
>   static void kvm_arm_gicv3_get(GICv3State *s)
>   {
>       uint32_t regl, regh, reg;
> @@ -882,6 +973,7 @@ static void kvm_arm_gicv3_class_init(ObjectClass *klass, 
> void *data)
>       ARMGICv3CommonClass *agcc = ARM_GICV3_COMMON_CLASS(klass);
>       KVMARMGICv3Class *kgc = KVM_ARM_GICV3_CLASS(klass);
>   
> +    agcc->inject_nmi = kvm_arm_gicv3_inject_nmi;
>       agcc->pre_save = kvm_arm_gicv3_get;
>       agcc->post_load = kvm_arm_gicv3_put;
>       device_class_set_parent_realize(dc, kvm_arm_gicv3_realize,
> diff --git a/include/hw/intc/arm_gicv3_common.h 
> b/include/hw/intc/arm_gicv3_common.h
> index 31ec9a1ae4..0ae9c45aa2 100644
> --- a/include/hw/intc/arm_gicv3_common.h
> +++ b/include/hw/intc/arm_gicv3_common.h
> @@ -225,6 +225,7 @@ struct GICv3State {
>   
>       int dev_fd; /* kvm device fd if backed by kvm vgic support */
>       Error *migration_blocker;
> +    int last_nmi_index;
>   
>       /* Distributor */
>   
> @@ -291,6 +292,7 @@ typedef struct ARMGICv3CommonClass {
>       SysBusDeviceClass parent_class;
>       /*< public >*/
>   
> +    void (*inject_nmi)(DeviceState *dev, int cpu_index, Error **errp);
>       void (*pre_save)(GICv3State *s);
>       void (*post_load)(GICv3State *s);
>   } ARMGICv3CommonClass;