Re: [Qemu-devel] [PATCH qemu v10 13/14] vfio: spapr: Add SPAPR IOMMU v2 support (DMA memory preregistering)

[Alexey Kardashevskiy]

On 07/07/2015 05:23 PM, Thomas Huth wrote:
> On Mon,  6 Jul 2015 12:11:09 +1000
> Alexey Kardashevskiy <address@hidden> wrote:
>
> > This makes use of the new "memory registering" feature. The idea is
> > to provide the userspace ability to notify the host kernel about pages
> > which are going to be used for DMA. Having this information, the host
> > kernel can pin them all once per user process, do locked pages
> > accounting (once) and not spent time on doing that in real time with
> > possible failures which cannot be handled nicely in some cases.
> >
> > This adds a guest RAM memory listener which notifies a VFIO container
> > about memory which needs to be pinned/unpinned. VFIO MMIO regions
> > (i.e. "skip dump" regions) are skipped.
> >
> > The feature is only enabled for SPAPR IOMMU v2. The host kernel changes
> > are required. Since v2 does not need/support VFIO_IOMMU_ENABLE, this does
> > not call it when v2 is detected and enabled.
> >
> > This does not change the guest visible interface.
> >
> > Signed-off-by: Alexey Kardashevskiy <address@hidden>
> > Reviewed-by: David Gibson <address@hidden>
> > ---
> > Changes:
> > v9:
> > * since there is no more SPAPR-specific data in container::iommu_data,
> > the memory preregistration fields are common and potentially can be used
> > by other architectures
> >
> > v7:
> > * in vfio_spapr_ram_listener_region_del(), do unref() after ioctl()
> > * s'ramlistener'register_listener'
> >
> > v6:
> > * fixed commit log (s/guest/userspace/), added note about no guest visible
> > change
> > * fixed error checking if ram registration failed
> > * added alignment check for section->offset_within_region
> >
> > v5:
> > * simplified the patch
> > * added trace points
> > * added round_up() for the size
> > * SPAPR IOMMU v2 used
> > ---
> >   hw/vfio/common.c              | 109 ++++++++++++++++++++++++++++++++++++++----
> >   include/hw/vfio/vfio-common.h |   3 ++
> >   trace-events                  |   1 +
> >   3 files changed, 104 insertions(+), 9 deletions(-)
> >
> > diff --git a/hw/vfio/common.c b/hw/vfio/common.c
> > index 8eacfd7..0c7ba8c 100644
> > --- a/hw/vfio/common.c
> > +++ b/hw/vfio/common.c
> > @@ -488,6 +488,76 @@ static void vfio_listener_release(VFIOContainer *container)
> >       memory_listener_unregister(&container->iommu_data.type1.listener);
> >   }
> >
> > +static void vfio_ram_do_region(VFIOContainer *container,
> > +                              MemoryRegionSection *section, unsigned long req)
> > +{
> > +    int ret;
> > +    struct vfio_iommu_spapr_register_memory reg = { .argsz = sizeof(reg) };
> > +
> > +    if (!memory_region_is_ram(section->mr) ||
> > +        memory_region_is_skip_dump(section->mr)) {
> > +        return;
> > +    }
> > +
> > +    if (unlikely((section->offset_within_region & (getpagesize() - 1)))) {
> > +        error_report("%s received unaligned region", __func__);
> > +        return;
> > +    }
> > +
> > +    reg.vaddr = (__u64) memory_region_get_ram_ptr(section->mr) +
>
> We're in usespace here ... I think it would be better to use uint64_t
> instead of the kernel-type __u64.


We are calling a kernel here - @reg is a kernel-defined struct.


> > +        section->offset_within_region;
> > +    reg.size = ROUND_UP(int128_get64(section->size), TARGET_PAGE_SIZE);
> > +
> > +    ret = ioctl(container->fd, req, &reg);
> > +    trace_vfio_ram_register(_IOC_NR(req) - VFIO_BASE, reg.vaddr, reg.size,
> > +            ret ? -errno : 0);
> > +    if (!ret) {
> > +        return;
> > +    }
> > +
> > +    /*
> > +     * On the initfn path, store the first error in the container so we
> > +     * can gracefully fail.  Runtime, there's not much we can do other
> > +     * than throw a hardware error.
> > +     */
> > +    if (!container->iommu_data.ram_reg_initialized) {
> > +        if (!container->iommu_data.ram_reg_error) {
> > +            container->iommu_data.ram_reg_error = -errno;
> > +        }
> > +    } else {
> > +        hw_error("vfio: RAM registering failed, unable to continue");
> > +    }
> > +}
> > +
> > +static void vfio_ram_listener_region_add(MemoryListener *listener,
> > +                                         MemoryRegionSection *section)
> > +{
> > +    VFIOContainer *container = container_of(listener, VFIOContainer,
> > +                                            iommu_data.register_listener);
> > +    memory_region_ref(section->mr);
> > +    vfio_ram_do_region(container, section, VFIO_IOMMU_SPAPR_REGISTER_MEMORY);
> > +}
> > +
> > +static void vfio_ram_listener_region_del(MemoryListener *listener,
> > +                                         MemoryRegionSection *section)
> > +{
> > +    VFIOContainer *container = container_of(listener, VFIOContainer,
> > +                                            iommu_data.register_listener);
> > +    vfio_ram_do_region(container, section, VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY);
> > +    memory_region_unref(section->mr);
> > +}
> > +
> > +static const MemoryListener vfio_ram_memory_listener = {
> > +    .region_add = vfio_ram_listener_region_add,
> > +    .region_del = vfio_ram_listener_region_del,
> > +};
> > +
> > +static void vfio_spapr_listener_release_v2(VFIOContainer *container)
> > +{
> > +    memory_listener_unregister(&container->iommu_data.register_listener);
> > +    vfio_listener_release(container);
> > +}
> > +
> >   int vfio_mmap_region(Object *obj, VFIORegion *region,
> >                        MemoryRegion *mem, MemoryRegion *submem,
> >                        void **map, size_t size, off_t offset,
> > @@ -698,14 +768,18 @@ static int vfio_connect_container(VFIOGroup *group, 
> > AddressSpace *as)
> >
> >           container->iommu_data.type1.initialized = true;
> >
> > -    } else if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_IOMMU)) {
> > +    } else if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_IOMMU) ||
> > +               ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_v2_IOMMU)) {
> > +        bool v2 = !!ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_v2_IOMMU);
>
> That "!!" sounds somewhat wrong here. I think you either want to check
> for "ioctl() == 1" (because only in this case you can be sure that v2
> is supported), or you can simply omit the "!!" because you're 100% sure
> that the ioctl only returns 0 or 1 (and never a negative error code).


The host kernel does not return an error on these ioctls, it returns 0 or 
1. And "!!" is shorter than "(bool)". VFIO_CHECK_EXTENSION for Type1 does 
exactly the same already.



> >           ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd);
> >           if (ret) {
> >               error_report("vfio: failed to set group container: %m");
> >               ret = -errno;
> >               goto free_container_exit;
> >           }
> > -        ret = ioctl(fd, VFIO_SET_IOMMU, VFIO_SPAPR_TCE_IOMMU);
> > +        ret = ioctl(fd, VFIO_SET_IOMMU,
> > +                v2 ? VFIO_SPAPR_TCE_v2_IOMMU : VFIO_SPAPR_TCE_IOMMU);
> >           if (ret) {
> >               error_report("vfio: failed to set iommu for container: %m");
> >               ret = -errno;
> > @@ -717,19 +791,36 @@ static int vfio_connect_container(VFIOGroup *group, 
> > AddressSpace *as)
> >            * when container fd is closed so we do not call it explicitly
> >            * in this file.
> >            */
> > -        ret = ioctl(fd, VFIO_IOMMU_ENABLE);
> > -        if (ret) {
> > -            error_report("vfio: failed to enable container: %m");
> > -            ret = -errno;
> > -            goto free_container_exit;
> > +        if (!v2) {
> > +            ret = ioctl(fd, VFIO_IOMMU_ENABLE);
> > +            if (ret) {
> > +                error_report("vfio: failed to enable container: %m");
> > +                ret = -errno;
> > +                goto free_container_exit;
> > +            }
> >           }
> >
> >           container->iommu_data.type1.listener = vfio_memory_listener;
> > -        container->iommu_data.release = vfio_listener_release;
> > -
> >           memory_listener_register(&container->iommu_data.type1.listener,
> >                                    container->space->as);
> >
> > +        if (!v2) {
> > +            container->iommu_data.release = vfio_listener_release;
> > +        } else {
> > +            container->iommu_data.release = vfio_spapr_listener_release_v2;
> > +            container->iommu_data.register_listener =
> > +                    vfio_ram_memory_listener;
> > +            memory_listener_register(&container->iommu_data.register_listener,
> > +                                     &address_space_memory);
> > +
> > +            if (container->iommu_data.ram_reg_error) {
> > +                error_report("vfio: RAM memory listener initialization failed for 
> > container");
>
> Line > 80 columns?

afaik user visible strings are an exception in QEMU and kernel.


> > +                goto listener_release_exit;
> > +            }
> > +
> > +            container->iommu_data.ram_reg_initialized = true;
> > +        }
> > +
>
>   Thomas


--
Alexey