[Qemu-devel] [RFC][PATCH 38/45] msi: Implement config notifiers for legacy MSI

[Jan Kiszka]

Realize support for MSI config notifiers analogously to MSI-X. The logic
is slightly more complex for legacy MSI as per-vector masking is option
here. Device assignment will be the first user.

Note that this change does not introduce per-vector masking support.
This can to be added at some later point, using the notifications the
MSI layer provides now.

Signed-off-by: Jan Kiszka <address@hidden>
---
 hw/msi.c |  171 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++------
 hw/msi.h |    7 ++-
 hw/pci.c |    2 +-
 hw/pci.h |    3 +
 4 files changed, 166 insertions(+), 17 deletions(-)

diff --git a/hw/msi.c b/hw/msi.c
index 23d79dd..2380ee3 100644
--- a/hw/msi.c
+++ b/hw/msi.c
@@ -241,15 +241,15 @@ void msi_uninit(struct PCIDevice *dev)
 
 void msi_reset(PCIDevice *dev)
 {
-    uint16_t flags;
+    uint16_t flags, old_flags;
     bool msi64bit;
 
     if (!msi_present(dev)) {
         return;
     }
 
-    flags = pci_get_word(dev->config + msi_flags_off(dev));
-    flags &= ~(PCI_MSI_FLAGS_QSIZE | PCI_MSI_FLAGS_ENABLE);
+    old_flags = pci_get_word(dev->config + msi_flags_off(dev));
+    flags = old_flags & ~(PCI_MSI_FLAGS_QSIZE | PCI_MSI_FLAGS_ENABLE);
     msi64bit = flags & PCI_MSI_FLAGS_64BIT;
 
     pci_set_word(dev->config + msi_flags_off(dev), flags);
@@ -262,6 +262,8 @@ void msi_reset(PCIDevice *dev)
         pci_set_long(dev->config + msi_mask_off(dev, msi64bit), 0);
         pci_set_long(dev->config + msi_pending_off(dev, msi64bit), 0);
     }
+    /* trigger notifier on potential changes */
+    msi_write_config(dev, msi_flags_off(dev), old_flags, 2);
     MSI_DEV_PRINTF(dev, "reset\n");
 }
 
@@ -306,16 +308,20 @@ void msi_notify(PCIDevice *dev, unsigned int vector)
 }
 
 /* Normally called by pci_default_write_config(). */
-void msi_write_config(PCIDevice *dev, uint32_t addr, uint32_t val, int len)
+void msi_write_config(PCIDevice *dev, uint32_t addr, uint32_t old_val, int len)
 {
     uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
     bool msi64bit = flags & PCI_MSI_FLAGS_64BIT;
     bool msi_per_vector_mask = flags & PCI_MSI_FLAGS_MASKBIT;
+    bool fire_vector_notifier = false;
     unsigned int nr_vectors;
     uint8_t log_num_vecs;
     uint8_t log_max_vecs;
     unsigned int vector;
     uint32_t pending;
+    MSIMessage msg;
+    bool enabled;
+    int ret;
 
     if (!msi_present(dev) ||
         !ranges_overlap(addr, len, dev->msi_cap, msi_cap_sizeof(flags))) {
@@ -342,7 +348,35 @@ void msi_write_config(PCIDevice *dev, uint32_t addr, 
uint32_t val, int len)
     fprintf(stderr, "\n");
 #endif
 
-    if (!(flags & PCI_MSI_FLAGS_ENABLE)) {
+    enabled = flags & PCI_MSI_FLAGS_ENABLE;
+    nr_vectors = msi_nr_vectors(flags);
+
+    if (dev->msi_enable_notifier &&
+        range_covers_byte(addr, len, msi_flags_off(dev))) {
+        old_val >>= (msi_flags_off(dev) - addr) * 8;
+        if ((old_val & PCI_MSI_FLAGS_ENABLE) != enabled) {
+            dev->msi_enable_notifier(dev, enabled);
+            if (enabled && dev->msi_vector_config_notifier) {
+                fire_vector_notifier = true;
+            }
+        }
+    }
+    if (dev->msi_vector_config_notifier) {
+        if (ranges_overlap(addr, len, msi_address_lo_off(dev),
+                   msi64bit ? 10 : 6)) {
+            fire_vector_notifier = true;
+        }
+    }
+    if (fire_vector_notifier) {
+        for (vector = 0; vector < nr_vectors; ++vector) {
+            msi_message_from_vector(dev, flags, vector, &msg);
+            ret = dev->msi_vector_config_notifier(dev, vector, &msg,
+                                                  msi_is_masked(dev, vector));
+            assert(ret >= 0);
+        }
+    }
+
+    if (!enabled) {
         kvm_msi_free(dev);
         return;
     }
@@ -375,13 +409,12 @@ void msi_write_config(PCIDevice *dev, uint32_t addr, 
uint32_t val, int len)
         pci_set_word(dev->config + msi_flags_off(dev), flags);
     }
 
-    if (!msi_per_vector_mask) {
-        /* if per vector masking isn't supported,
-           there is no pending interrupt. */
+    if (!msi_per_vector_mask ||
+        !ranges_overlap(addr, len, msi_mask_off(dev, msi64bit), 4)) {
         return;
     }
 
-    nr_vectors = msi_nr_vectors(flags);
+    old_val >>= (msi_mask_off(dev, msi64bit) - addr) * 8;
 
     /* This will discard pending interrupts, if any. */
     pending = pci_get_long(dev->config + msi_pending_off(dev, msi64bit));
@@ -390,13 +423,22 @@ void msi_write_config(PCIDevice *dev, uint32_t addr, 
uint32_t val, int len)
 
     /* deliver pending interrupts which are unmasked */
     for (vector = 0; vector < nr_vectors; ++vector) {
-        if (msi_is_masked(dev, vector) || !(pending & (1U << vector))) {
-            continue;
+        bool is_masked = msi_is_masked(dev, vector);
+        unsigned int vector_mask = 1U << vector;
+
+        if (!fire_vector_notifier && dev->msi_vector_config_notifier &&
+            (bool)(old_val & vector_mask) != is_masked) {
+            msi_message_from_vector(dev, flags, vector, &msg);
+            ret = dev->msi_vector_config_notifier(dev, vector, &msg,
+                                                  is_masked);
+            assert(ret >= 0);
+        }
+        if (!is_masked && pending & vector_mask) {
+            pci_long_test_and_clear_mask(dev->config +
+                                         msi_pending_off(dev, msi64bit),
+                                         vector_mask);
+            msi_notify(dev, vector);
         }
-
-        pci_long_test_and_clear_mask(
-            dev->config + msi_pending_off(dev, msi64bit), 1U << vector);
-        msi_notify(dev, vector);
     }
 }
 
@@ -405,3 +447,102 @@ unsigned int msi_nr_vectors_allocated(const PCIDevice 
*dev)
     uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
     return msi_nr_vectors(flags);
 }
+
+/* Invoke the notifier if vector entry is unmasked. */
+static int
+msi_notify_if_unmasked(PCIDevice *dev, unsigned int vector, int masked)
+{
+    uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
+    MSIMessage msg;
+
+    assert(dev->msi_vector_config_notifier);
+
+    if (msi_is_masked(dev, vector)) {
+        return 0;
+    }
+    msi_message_from_vector(dev, flags, vector, &msg);
+    return dev->msi_vector_config_notifier(dev, vector, &msg, masked);
+}
+
+static int
+msi_set_config_notifier_for_vector(PCIDevice *dev, unsigned int vector)
+{
+    /* Notifier has been set. Invoke it on unmasked vectors. */
+    return msi_notify_if_unmasked(dev, vector, 0);
+}
+
+static int
+msi_unset_config_notifier_for_vector(PCIDevice *dev, unsigned int vector)
+{
+    /* Notifier will be unset. Invoke it to mask unmasked entries. */
+    return msi_notify_if_unmasked(dev, vector, 1);
+}
+
+int msi_set_config_notifiers(PCIDevice *dev, MSIEnableNotifier enable_notifier,
+                             MSIVectorConfigNotifier vector_config_notifier)
+{
+    unsigned int nr_vectors;
+    int r, vector;
+
+    assert(!dev->msi_vector_config_notifier);
+
+    dev->msi_enable_notifier = enable_notifier;
+    dev->msi_vector_config_notifier = vector_config_notifier;
+
+    if (enable_notifier && msi_enabled(dev)) {
+        enable_notifier(dev, true);
+    }
+    if (msi_enabled(dev)) {
+        nr_vectors =
+            msi_nr_vectors(pci_get_word(dev->config + msi_flags_off(dev)));
+        for (vector = 0; vector < nr_vectors; ++vector) {
+            r = msi_set_config_notifier_for_vector(dev, vector);
+            if (r < 0) {
+                goto undo;
+            }
+        }
+    }
+    return 0;
+
+undo:
+    while (--vector >= 0) {
+        msi_unset_config_notifier_for_vector(dev, vector);
+    }
+    if (enable_notifier && msi_enabled(dev)) {
+        enable_notifier(dev, false);
+    }
+    dev->msi_enable_notifier = NULL;
+    dev->msi_vector_config_notifier = NULL;
+    return r;
+}
+
+int msi_unset_config_notifiers(PCIDevice *dev)
+{
+    unsigned int nr_vectors;
+    int r, vector;
+
+    assert(dev->msi_vector_config_notifier);
+
+    if (msi_enabled(dev)) {
+        nr_vectors =
+            msi_nr_vectors(pci_get_word(dev->config + msi_flags_off(dev)));
+        for (vector = 0; vector < nr_vectors; ++vector) {
+            r = msi_unset_config_notifier_for_vector(dev, vector);
+            if (r < 0) {
+                goto undo;
+            }
+        }
+    }
+    if (dev->msi_enable_notifier && msi_enabled(dev)) {
+        dev->msi_enable_notifier(dev, false);
+    }
+    dev->msi_enable_notifier = NULL;
+    dev->msi_vector_config_notifier = NULL;
+    return 0;
+
+undo:
+    while (--vector >= 0) {
+        msi_set_config_notifier_for_vector(dev, vector);
+    }
+    return r;
+}
diff --git a/hw/msi.h b/hw/msi.h
index 74f6d52..c28665b 100644
--- a/hw/msi.h
+++ b/hw/msi.h
@@ -50,9 +50,14 @@ int msi_init(struct PCIDevice *dev, uint8_t offset,
 void msi_uninit(struct PCIDevice *dev);
 void msi_reset(PCIDevice *dev);
 void msi_notify(PCIDevice *dev, unsigned int vector);
-void msi_write_config(PCIDevice *dev, uint32_t addr, uint32_t val, int len);
+void msi_write_config(PCIDevice *dev, uint32_t addr, uint32_t old_val,
+                      int len);
 unsigned int msi_nr_vectors_allocated(const PCIDevice *dev);
 
+int msi_set_config_notifiers(PCIDevice *dev, MSIEnableNotifier enable_notifier,
+                             MSIVectorConfigNotifier vector_config_notifier);
+int msi_unset_config_notifiers(PCIDevice *dev);
+
 static inline bool msi_present(const PCIDevice *dev)
 {
     return dev->cap_present & QEMU_PCI_CAP_MSI;
diff --git a/hw/pci.c b/hw/pci.c
index 4f0d7e1..96cd334 100644
--- a/hw/pci.c
+++ b/hw/pci.c
@@ -1155,7 +1155,7 @@ void pci_default_write_config(PCIDevice *d, uint32_t 
addr, uint32_t val, int l)
     if (range_covers_byte(addr, l, PCI_COMMAND))
         pci_update_irq_disabled(d, was_irq_disabled);
 
-    msi_write_config(d, addr, val, l);
+    msi_write_config(d, addr, old_val, l);
     msix_write_config(d, addr, old_val, l);
 }
 
diff --git a/hw/pci.h b/hw/pci.h
index 5cf9a16..266fe34 100644
--- a/hw/pci.h
+++ b/hw/pci.h
@@ -206,6 +206,9 @@ struct PCIDevice {
      * on the rest of the region. */
     target_phys_addr_t msix_page_size;
 
+    MSIEnableNotifier msi_enable_notifier;
+    MSIVectorConfigNotifier msi_vector_config_notifier;
+
     MSIEnableNotifier msix_enable_notifier;
     MSIVectorConfigNotifier msix_vector_config_notifier;
 };
-- 
1.7.3.4