+ }
+
+ ret = ioctl(container->fd, VFIO_IOMMU_CACHE_INVALIDATE, &ustruct);
+ if (ret) {
+ error_report("%p: failed to invalidate CACHE (%d)",
container, ret);
+ }
+}
+
static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry
*iotlb)
{
VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n);
@@ -776,6 +843,35 @@ static void
vfio_dma_unmap_ram_section(VFIOContainer *container,
}
}
+static void vfio_prereg_listener_region_add(MemoryListener
*listener,
+ MemoryRegionSection
*section)
+{
+ VFIOContainer *container =
+ container_of(listener, VFIOContainer, prereg_listener);
+ Error *err = NULL;
+
+ if (!memory_region_is_ram(section->mr)) {
+ return;
+ }
+
+ vfio_dma_map_ram_section(container, section, &err);
+ if (err) {
+ error_report_err(err);
+ }
+}
+static void vfio_prereg_listener_region_del(MemoryListener *listener,
+ MemoryRegionSection *section)
+{
+ VFIOContainer *container =
+ container_of(listener, VFIOContainer, prereg_listener);
+
+ if (!memory_region_is_ram(section->mr)) {
+ return;
+ }
+
+ vfio_dma_unmap_ram_section(container, section);
+}
+
static void vfio_listener_region_add(MemoryListener *listener,
MemoryRegionSection *section)
{
@@ -879,9 +975,10 @@ static void
vfio_listener_region_add(MemoryListener *listener,
memory_region_ref(section->mr);
if (memory_region_is_iommu(section->mr)) {
+ IOMMUNotify notify;
VFIOGuestIOMMU *giommu;
IOMMUMemoryRegion *iommu_mr =
IOMMU_MEMORY_REGION(section->mr);
- int iommu_idx;
+ int iommu_idx, flags;
trace_vfio_listener_region_add_iommu(iova, end);
/*
@@ -900,8 +997,18 @@ static void
vfio_listener_region_add(MemoryListener *listener,
llend = int128_sub(llend, int128_one());
iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr,
MEMTXATTRS_UNSPECIFIED);
- iommu_notifier_init(&giommu->n, vfio_iommu_map_notify,
- IOMMU_NOTIFIER_IOTLB_EVENTS,
+
+ if (container->iommu_type == VFIO_TYPE1_NESTING_IOMMU) {
+ /* IOTLB unmap notifier to propagate guest IOTLB
invalidations */
+ flags = IOMMU_NOTIFIER_UNMAP;
+ notify = vfio_iommu_unmap_notify;
+ } else {
+ /* MAP/UNMAP IOTLB notifier */
+ flags = IOMMU_NOTIFIER_IOTLB_EVENTS;
+ notify = vfio_iommu_map_notify;
+ }
+
+ iommu_notifier_init(&giommu->n, notify, flags,
section->offset_within_region,
int128_get64(llend),
iommu_idx);
@@ -921,7 +1028,9 @@ static void
vfio_listener_region_add(MemoryListener *listener,
goto fail;
}
QLIST_INSERT_HEAD(&container->giommu_list, giommu,
giommu_next);
- memory_region_iommu_replay(giommu->iommu, &giommu->n);
+ if (flags & IOMMU_NOTIFIER_MAP) {
+ memory_region_iommu_replay(giommu->iommu, &giommu->n);
+ }
return;
}
@@ -1205,10 +1314,16 @@ static const MemoryListener
vfio_memory_listener = {
.log_sync = vfio_listener_log_sync,
};
+static MemoryListener vfio_memory_prereg_listener = {
+ .region_add = vfio_prereg_listener_region_add,
+ .region_del = vfio_prereg_listener_region_del,
+};
+
static void vfio_listener_release(VFIOContainer *container)
{
memory_listener_unregister(&container->listener);
- if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) {
+ if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU ||
+ container->iommu_type == VFIO_TYPE1_NESTING_IOMMU) {
memory_listener_unregister(&container->prereg_listener);
}
}
@@ -1858,6 +1973,20 @@ static int vfio_connect_container(VFIOGroup
*group, AddressSpace *as,
vfio_get_iommu_info_migration(container, info);
}
g_free(info);
+
+ if (container->iommu_type == VFIO_TYPE1_NESTING_IOMMU) {
+ container->prereg_listener = vfio_memory_prereg_listener;
+ memory_listener_register(&container->prereg_listener,
+ &address_space_memory);
+ if (container->error) {
+
memory_listener_unregister(&container->prereg_listener);
+ ret = -1;
+ error_propagate_prepend(errp, container->error,
+ "RAM memory listener
initialization failed "
+ "for container");
+ goto free_container_exit;
+ }
+ }
break;
}
case VFIO_SPAPR_TCE_v2_IOMMU:
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 5c65aa0a98..cad7deec71 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -2773,6 +2773,25 @@ static void
vfio_unregister_req_notifier(VFIOPCIDevice *vdev)
vdev->req_enabled = false;
}
+static int vfio_iommu_set_pasid_table(PCIBus *bus, int32_t devfn,
+ IOMMUConfig *config)
+{
+ PCIDevice *pdev = bus->devices[devfn];
+ VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
+ VFIOContainer *container = vdev->vbasedev.group->container;
+ struct vfio_iommu_type1_set_pasid_table info;
+
+ info.argsz = sizeof(info);
+ info.flags = VFIO_PASID_TABLE_FLAG_SET;
+ memcpy(&info.config, &config->pasid_cfg,
sizeof(config->pasid_cfg));
+
+ return ioctl(container->fd, VFIO_IOMMU_SET_PASID_TABLE, &info);
+}
+
+static PCIPASIDOps vfio_pci_pasid_ops = {
+ .set_pasid_table = vfio_iommu_set_pasid_table,
+};
+
static void vfio_realize(PCIDevice *pdev, Error **errp)
{
VFIOPCIDevice *vdev = VFIO_PCI(pdev);
@@ -3084,6 +3103,8 @@ static void vfio_realize(PCIDevice *pdev, Error
**errp)
vfio_register_req_notifier(vdev);
vfio_setup_resetfn_quirk(vdev);
+ pci_setup_pasid_ops(pdev, &vfio_pci_pasid_ops);
+
return;
out_deregister:
diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events
index 936d29d150..43696afc15 100644
--- a/hw/vfio/trace-events
+++ b/hw/vfio/trace-events
@@ -120,6 +120,8 @@ vfio_region_sparse_mmap_header(const char *name,
int index, int nr_areas) "Devic
vfio_region_sparse_mmap_entry(int i, unsigned long start, unsigned
long end) "sparse entry %d [0x%lx - 0x%lx]"
vfio_get_dev_region(const char *name, int index, uint32_t type,
uint32_t subtype) "%s index %d, %08x/%0x8"
vfio_dma_unmap_overflow_workaround(void) ""
+vfio_iommu_addr_inv_iotlb(int asid, uint64_t addr, uint64_t size,
uint64_t nb_granules, bool leaf) "nested IOTLB invalidate asid=%d,
addr=0x%"PRIx64" granule_size=0x%"PRIx64" nb_granules=0x%"PRIx64"
leaf=%d"
+vfio_iommu_asid_inv_iotlb(int asid) "nested IOTLB invalidate asid=%d"
# platform.c
vfio_platform_base_device_init(char *name, int groupid) "%s belongs
to group #%d"