[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[PATCH v3 kvm/queue 10/16] KVM: Implement fd-based memory invalidation
From: |
Chao Peng |
Subject: |
[PATCH v3 kvm/queue 10/16] KVM: Implement fd-based memory invalidation |
Date: |
Thu, 23 Dec 2021 20:30:05 +0800 |
KVM gets notified when userspace punches a hole in a fd which is used
for guest memory. KVM should invalidate the mapping in the secondary
MMU page tables. This is the same logic as MMU notifier invalidation
except the fd related information is carried around to indicate the
memory range. KVM hence can reuse most of existing MMU notifier
invalidation code including looping through the memslots and then
calling into kvm_unmap_gfn_range() which should do whatever needed for
fd-based memory unmapping (e.g. for private memory managed by TDX it
may need call into SEAM-MODULE).
Signed-off-by: Yu Zhang <yu.c.zhang@linux.intel.com>
Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
---
include/linux/kvm_host.h | 8 ++++-
virt/kvm/kvm_main.c | 69 +++++++++++++++++++++++++++++++---------
virt/kvm/memfd.c | 2 ++
3 files changed, 63 insertions(+), 16 deletions(-)
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 07863ff855cd..be567925831b 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -233,7 +233,7 @@ bool kvm_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t
cr2_or_gpa,
int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu);
#endif
-#ifdef KVM_ARCH_WANT_MMU_NOTIFIER
+#if defined(KVM_ARCH_WANT_MMU_NOTIFIER) || defined(CONFIG_MEMFD_OPS)
struct kvm_gfn_range {
struct kvm_memory_slot *slot;
gfn_t start;
@@ -2012,4 +2012,10 @@ static inline void kvm_handle_signal_exit(struct
kvm_vcpu *vcpu)
/* Max number of entries allowed for each kvm dirty ring */
#define KVM_DIRTY_RING_MAX_ENTRIES 65536
+#ifdef CONFIG_MEMFD_OPS
+int kvm_memfd_invalidate_range(struct kvm *kvm, struct inode *inode,
+ unsigned long start, unsigned long end);
+#endif /* CONFIG_MEMFD_OPS */
+
+
#endif
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 7b7530b1ea1e..f495c1a313bd 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -468,7 +468,8 @@ void kvm_destroy_vcpus(struct kvm *kvm)
}
EXPORT_SYMBOL_GPL(kvm_destroy_vcpus);
-#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
+#if defined(CONFIG_MEMFD_OPS) ||\
+ (defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER))
typedef bool (*gfn_handler_t)(struct kvm *kvm, struct kvm_gfn_range *range);
@@ -595,6 +596,30 @@ static __always_inline int
__kvm_handle_useraddr_range(struct kvm *kvm,
/* The notifiers are averse to booleans. :-( */
return (int)ret;
}
+
+static void mn_active_invalidate_count_inc(struct kvm *kvm)
+{
+ spin_lock(&kvm->mn_invalidate_lock);
+ kvm->mn_active_invalidate_count++;
+ spin_unlock(&kvm->mn_invalidate_lock);
+
+}
+
+static void mn_active_invalidate_count_dec(struct kvm *kvm)
+{
+ bool wake;
+
+ spin_lock(&kvm->mn_invalidate_lock);
+ wake = (--kvm->mn_active_invalidate_count == 0);
+ spin_unlock(&kvm->mn_invalidate_lock);
+
+ /*
+ * There can only be one waiter, since the wait happens under
+ * slots_lock.
+ */
+ if (wake)
+ rcuwait_wake_up(&kvm->mn_memslots_update_rcuwait);
+}
#endif
#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
@@ -732,9 +757,7 @@ static int kvm_mmu_notifier_invalidate_range_start(struct
mmu_notifier *mn,
*
* Pairs with the decrement in range_end().
*/
- spin_lock(&kvm->mn_invalidate_lock);
- kvm->mn_active_invalidate_count++;
- spin_unlock(&kvm->mn_invalidate_lock);
+ mn_active_invalidate_count_inc(kvm);
__kvm_handle_useraddr_range(kvm, &useraddr_range);
@@ -773,21 +796,11 @@ static void kvm_mmu_notifier_invalidate_range_end(struct
mmu_notifier *mn,
.may_block = mmu_notifier_range_blockable(range),
.inode = NULL,
};
- bool wake;
__kvm_handle_useraddr_range(kvm, &useraddr_range);
/* Pairs with the increment in range_start(). */
- spin_lock(&kvm->mn_invalidate_lock);
- wake = (--kvm->mn_active_invalidate_count == 0);
- spin_unlock(&kvm->mn_invalidate_lock);
-
- /*
- * There can only be one waiter, since the wait happens under
- * slots_lock.
- */
- if (wake)
- rcuwait_wake_up(&kvm->mn_memslots_update_rcuwait);
+ mn_active_invalidate_count_dec(kvm);
BUG_ON(kvm->mmu_notifier_count < 0);
}
@@ -872,6 +885,32 @@ static int kvm_init_mmu_notifier(struct kvm *kvm)
#endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
+#ifdef CONFIG_MEMFD_OPS
+int kvm_memfd_invalidate_range(struct kvm *kvm, struct inode *inode,
+ unsigned long start, unsigned long end)
+{
+ int ret;
+ const struct kvm_useraddr_range useraddr_range = {
+ .start = start,
+ .end = end,
+ .pte = __pte(0),
+ .handler = kvm_unmap_gfn_range,
+ .on_lock = (void *)kvm_null_fn,
+ .flush_on_ret = true,
+ .may_block = false,
+ .inode = inode,
+ };
+
+
+ /* Prevent memslot modification */
+ mn_active_invalidate_count_inc(kvm);
+ ret = __kvm_handle_useraddr_range(kvm, &useraddr_range);
+ mn_active_invalidate_count_dec(kvm);
+
+ return ret;
+}
+#endif /* CONFIG_MEMFD_OPS */
+
#ifdef CONFIG_HAVE_KVM_PM_NOTIFIER
static int kvm_pm_notifier_call(struct notifier_block *bl,
unsigned long state,
diff --git a/virt/kvm/memfd.c b/virt/kvm/memfd.c
index 662393a76782..547f65f5a187 100644
--- a/virt/kvm/memfd.c
+++ b/virt/kvm/memfd.c
@@ -16,6 +16,8 @@ static const struct memfd_pfn_ops *memfd_ops;
static void memfd_invalidate_page_range(struct inode *inode, void *owner,
pgoff_t start, pgoff_t end)
{
+ kvm_memfd_invalidate_range(owner, inode, start >> PAGE_SHIFT,
+ end >> PAGE_SHIFT);
}
static void memfd_fallocate(struct inode *inode, void *owner,
--
2.17.1
- [PATCH v3 kvm/queue 07/16] KVM: Refactor hva based memory invalidation code, (continued)
- [PATCH v3 kvm/queue 07/16] KVM: Refactor hva based memory invalidation code, Chao Peng, 2021/12/23
- [PATCH v3 kvm/queue 08/16] KVM: Special handling for fd-based memory invalidation, Chao Peng, 2021/12/23
- [PATCH v3 kvm/queue 06/16] KVM: Implement fd-based memory using MEMFD_OPS interfaces, Chao Peng, 2021/12/23
- Re: [PATCH v3 kvm/queue 06/16] KVM: Implement fd-based memory using MEMFD_OPS interfaces, Chao Peng, 2021/12/23
- Re: [PATCH v3 kvm/queue 06/16] KVM: Implement fd-based memory using MEMFD_OPS interfaces, Chao Peng, 2021/12/23
[PATCH v3 kvm/queue 09/16] KVM: Split out common memory invalidation code, Chao Peng, 2021/12/23
[PATCH v3 kvm/queue 10/16] KVM: Implement fd-based memory invalidation,
Chao Peng <=
[PATCH v3 kvm/queue 11/16] KVM: Add kvm_map_gfn_range, Chao Peng, 2021/12/23
[PATCH v3 kvm/queue 12/16] KVM: Implement fd-based memory fallocation, Chao Peng, 2021/12/23
[PATCH v3 kvm/queue 13/16] KVM: Add KVM_EXIT_MEMORY_ERROR exit, Chao Peng, 2021/12/23
[PATCH v3 kvm/queue 14/16] KVM: Handle page fault for private memory, Chao Peng, 2021/12/23
[PATCH v3 kvm/queue 15/16] KVM: Use kvm_userspace_memory_region_ext, Chao Peng, 2021/12/23
[PATCH v3 kvm/queue 16/16] KVM: Register/unregister private memory slot to memfd, Chao Peng, 2021/12/23