Re: [PATCH V8 28/39] vfio-pci: cpr part 2 (msi)

[Steven Sistare]

On 6/29/2022 4:19 PM, Alex Williamson wrote:
> On Wed, 15 Jun 2022 07:52:15 -0700
> Steve Sistare <steven.sistare@oracle.com> wrote:
> 
>> Finish cpr for vfio-pci MSI/MSI-X devices by preserving eventfd's and
>> vector state.
>>
>> Signed-off-by: Steve Sistare <steven.sistare@oracle.com>
>> ---
>>  hw/vfio/pci.c | 122 
>> +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
>>  1 file changed, 121 insertions(+), 1 deletion(-)
>>
>> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
>> index 237231b..2fd7121 100644
>> --- a/hw/vfio/pci.c
>> +++ b/hw/vfio/pci.c
>> @@ -53,17 +53,53 @@ static void vfio_disable_interrupts(VFIOPCIDevice *vdev);
>>  static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled);
>>  static void vfio_msi_disable_common(VFIOPCIDevice *vdev);
>>  
>> +#define EVENT_FD_NAME(vdev, name)   \
>> +    g_strdup_printf("%s_%s", (vdev)->vbasedev.name, (name))
>> +
>> +static int save_event_fd(VFIOPCIDevice *vdev, const char *name, int nr,
>> +                         EventNotifier *ev)
>> +{
>> +    int fd = event_notifier_get_fd(ev);
>> +
>> +    if (fd >= 0) {
>> +        Error *err;
>> +        g_autofree char *fdname = EVENT_FD_NAME(vdev, name);
>> +
>> +        if (cpr_resave_fd(fdname, nr, fd, &err)) {
>> +            error_report_err(err);
>> +            return 1;
> 
> 
> Preferably -1, but the caller doesn't actually test the return value
> anyway :-\

Per my previous email, I suggest that cpr_resave_fd return void, and hence
save_event_fd becomes void as well.

>> +        }
>> +    }
>> +    return 0;
>> +}
>> +
>> +static int load_event_fd(VFIOPCIDevice *vdev, const char *name, int nr)
>> +{
>> +    g_autofree char *fdname = EVENT_FD_NAME(vdev, name);
>> +    int fd = cpr_find_fd(fdname, nr);
>> +    return fd;
> 
> 
>     return cpr_find_fd(EVENT_FD_NAME(vdev, name), nr);

That leaks EVENT_FD_NAME, produced by g_strdup_printf, but I can reduce it to:
    g_autofree char *fdname = EVENT_FD_NAME(vdev, name);
    return cpr_find_fd(fdname, nr);

>> +}
>> +
>> +static void delete_event_fd(VFIOPCIDevice *vdev, const char *name, int nr)
>> +{
>> +    g_autofree char *fdname = EVENT_FD_NAME(vdev, name);
>> +    cpr_delete_fd(fdname, nr);
> 
> 
>     cpr_delete_fd(EVENT_FD_NAME(vdev, name), nr);

Ditto.

>> +}
>> +
>>  /* Create new or reuse existing eventfd */
>>  static int vfio_notifier_init(VFIOPCIDevice *vdev, EventNotifier *e,
>>                                const char *name, int nr)
>>  {
>> -    int fd = -1;   /* placeholder until a subsequent patch */
>>      int ret = 0;
>> +    int fd = load_event_fd(vdev, name, nr);
>>  
>>      if (fd >= 0) {
>>          event_notifier_init_fd(e, fd);
>>      } else {
>>          ret = event_notifier_init(e, 0);
>> +        if (!ret) {
>> +            save_event_fd(vdev, name, nr, e);
> 
> 
> Return value not tested.  The function generates an error report if it
> fails, but it doesn't seem that actually blocks a cpr attempt.  Do we
> just wind up with that error report as a breadcrumb to why cpr breaks
> with a missing fd down the road?

Thanks, that is a bug, it should have been:
    ret = save_event_fd(vdev, name, nr, e)
... but per the previous comment save_event_fd becomes void.

>> +        }
>>      }
>>      return ret;
>>  }
>> @@ -71,6 +107,7 @@ static int vfio_notifier_init(VFIOPCIDevice *vdev, 
>> EventNotifier *e,
>>  static void vfio_notifier_cleanup(VFIOPCIDevice *vdev, EventNotifier *e,
>>                                    const char *name, int nr)
>>  {
>> +    delete_event_fd(vdev, name, nr);
>>      event_notifier_cleanup(e);
>>  }
>>  
>> @@ -511,6 +548,15 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, 
>> unsigned int nr,
>>      VFIOMSIVector *vector;
>>      int ret;
>>  
>> +    /*
>> +     * Ignore the callback from msix_set_vector_notifiers during resume.
>> +     * The necessary subset of these actions is called from 
>> vfio_claim_vectors
>> +     * during post load.
>> +     */
>> +    if (vdev->vbasedev.reused) {
>> +        return 0;
>> +    }
>> +
>>      trace_vfio_msix_vector_do_use(vdev->vbasedev.name, nr);
>>  
>>      vector = &vdev->msi_vectors[nr];
>> @@ -2784,6 +2830,11 @@ static void vfio_register_err_notifier(VFIOPCIDevice 
>> *vdev)
>>      fd = event_notifier_get_fd(&vdev->err_notifier);
>>      qemu_set_fd_handler(fd, vfio_err_notifier_handler, NULL, vdev);
>>  
>> +    /* Do not alter irq_signaling during vfio_realize for cpr */
>> +    if (vdev->vbasedev.reused) {
>> +        return;
>> +    }
>> +
>>      if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_ERR_IRQ_INDEX, 0,
>>                                 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
>>          error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
>> @@ -2849,6 +2900,12 @@ static void vfio_register_req_notifier(VFIOPCIDevice 
>> *vdev)
>>      fd = event_notifier_get_fd(&vdev->req_notifier);
>>      qemu_set_fd_handler(fd, vfio_req_notifier_handler, NULL, vdev);
>>  
>> +    /* Do not alter irq_signaling during vfio_realize for cpr */
>> +    if (vdev->vbasedev.reused) {
>> +        vdev->req_enabled = true;
>> +        return;
>> +    }
> 
> vfio_notifier_init() transparently gets the old fd or creates a new
> one, how do we know which has occurred to know that this eventfd is
> already configured?

The caller can check the reused flag, which is set iff an old fd exists.
I could pass reused to vfio_notifier_init to assert that, but in some cases
I would need to pass a reused flag down through several functions to reach
vfio_notifier_init, which just seems ugly.

> Don't we also have the same issue relative to vdev->pci_aer for the
> error handler?

Same answer:
    vfio_register_err_notifier()
        vfio_notifier_init();
        if (vdev->vbasedev.reused)
            return;
        vfio_set_irq_signaling() ...

>> +
>>      if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_REQ_IRQ_INDEX, 0,
>>                             VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
>>          error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
>> @@ -3357,6 +3414,43 @@ static Property vfio_pci_dev_properties[] = {
>>      DEFINE_PROP_END_OF_LIST(),
>>  };
>>  
>> +static void vfio_claim_vectors(VFIOPCIDevice *vdev, int nr_vectors, bool 
>> msix)
>> +{
>> +    int i, fd;
>> +    bool pending = false;
>> +    PCIDevice *pdev = &vdev->pdev;
>> +
>> +    vdev->nr_vectors = nr_vectors;
>> +    vdev->msi_vectors = g_new0(VFIOMSIVector, nr_vectors);
>> +    vdev->interrupt = msix ? VFIO_INT_MSIX : VFIO_INT_MSI;
>> +
>> +    for (i = 0; i < nr_vectors; i++) {
>> +        VFIOMSIVector *vector = &vdev->msi_vectors[i];
>> +
>> +        fd = load_event_fd(vdev, "interrupt", i);
>> +        if (fd >= 0) {
>> +            vfio_vector_init(vdev, i);
>> +            qemu_set_fd_handler(fd, vfio_msi_interrupt, NULL, vector);
>> +        }
>> +
>> +        if (load_event_fd(vdev, "kvm_interrupt", i) >= 0) {
>> +            vfio_route_change = kvm_irqchip_begin_route_changes(kvm_state);
>> +            vfio_add_kvm_msi_virq(vdev, vector, i, msix);
>> +            kvm_irqchip_commit_route_changes(&vfio_route_change);
>> +            vfio_connect_kvm_msi_virq(vector, i);
> 
> 
> Shouldn't we take advantage of the batching support here?

OK, will do.

>> +        }
> 
> How do we debug if one of the above fails that shouldn't have failed?
> Should we have an assert or change this to a non-void return if we
> cannot setup an interrupt that we think is configured?

The path above ending with qemu_set_fd_handler always succeeds, because:

    fd = load_event_fd(vdev, "interrupt", i);
    if (fd >= 0) {
        vfio_vector_init(vdev, i)
            vfio_notifier_init(..., "interrupt", i)
                int fd = load_event_fd(vdev, name, i);
                if (fd >= 0) {
                    event_notifier_init_fd(e, fd);      <-- void, never fails

In the kvm_interrupt clause, only vfio_connect_kvm_msi_virq() can fail.  But, it
returns void, and other callers also assume it succeeds.  Good enough, or do
you want to do better here?

>> +
>> +        if (msix && msix_is_pending(pdev, i) && msix_is_masked(pdev, i)) {
>> +            set_bit(i, vdev->msix->pending);
>> +            pending = true;
>> +        }
>> +    }
>> +
>> +    if (msix) {
>> +        memory_region_set_enabled(&pdev->msix_pba_mmio, pending);
>> +    }
>> +}
>> +
>>  /*
>>   * The kernel may change non-emulated config bits.  Exclude them from the
>>   * changed-bits check in get_pci_config_device.
>> @@ -3375,6 +3469,29 @@ static int vfio_pci_pre_load(void *opaque)
>>      return 0;
>>  }
>>  
>> +static int vfio_pci_post_load(void *opaque, int version_id)
>> +{
>> +    VFIOPCIDevice *vdev = opaque;
>> +    PCIDevice *pdev = &vdev->pdev;
>> +    int nr_vectors;
>> +
>> +    if (msix_enabled(pdev)) {
>> +        msix_set_vector_notifiers(pdev, vfio_msix_vector_use,
>> +                                   vfio_msix_vector_release, NULL);
>> +        nr_vectors = vdev->msix->entries;
> 
> Maybe this is why we're not generating an error above, we don't know
> which vectors are configured other than if they have a saved eventfd,
> where we don't test whether we were able to actually save the fd.
> Thanks,
> 
> Alex
> 
> 
>> +        vfio_claim_vectors(vdev, nr_vectors, true);
>> +
>> +    } else if (msi_enabled(pdev)) {
>> +        nr_vectors = msi_nr_vectors_allocated(pdev);
>> +        vfio_claim_vectors(vdev, nr_vectors, false);
>> +
>> +    } else if (vfio_pci_read_config(pdev, PCI_INTERRUPT_PIN, 1)) {
>> +        assert(0);      /* completed in a subsequent patch */
>> +    }
>> +
>> +    return 0;
>> +}
>> +
>>  static bool vfio_pci_needed(void *opaque)
>>  {
>>      return cpr_get_mode() == CPR_MODE_RESTART;
>> @@ -3387,8 +3504,11 @@ static const VMStateDescription vfio_pci_vmstate = {
>>      .minimum_version_id = 0,
>>      .priority = MIG_PRI_VFIO_PCI,       /* must load before container */
>>      .pre_load = vfio_pci_pre_load,
>> +    .post_load = vfio_pci_post_load,
>>      .needed = vfio_pci_needed,
>>      .fields = (VMStateField[]) {
>> +        VMSTATE_PCI_DEVICE(pdev, VFIOPCIDevice),
>> +        VMSTATE_MSIX_TEST(pdev, VFIOPCIDevice, vfio_msix_present),
>>          VMSTATE_END_OF_LIST()
>>      }
>>  };
>