[Qemu-devel] [RFC v5 3/6] acpi-build: allocate mcfg for pxb-pcie host bridges

[Zihan Yang]

Allocate new segment for pxb-pcie host bridges in MCFG table, and reserve
corresponding MCFG space for them. This allows user-defined pxb-pcie
host bridges to be placed in different pci domain than q35 host.

The pci_host_bridges list is changed to be tail list to ensure the q35 host
is always the first element when traversing the list, because q35 host is
inserted beofre pxb-pcie hosts

A few new callbacks are added to PCIBusClass to get domain number and
max_bus property of a given PCIBus. Only pxb-pcie with a non-zero domain
number will have an item in MCFG table, others will still reside in pci
domain 0 under q35 host

Signed-off-by: Zihan Yang <address@hidden>
---
 hw/i386/acpi-build.c                        | 109 +++++++++++++++++++++-------
 hw/pci-bridge/pci_expander_bridge.c         |  51 +++++++++----
 hw/pci/pci.c                                |  30 +++++++-
 include/hw/pci-bridge/pci_expander_bridge.h |  16 ++++
 include/hw/pci/pci.h                        |   2 +
 include/hw/pci/pci_bus.h                    |   2 +
 include/hw/pci/pci_host.h                   |   2 +-
 7 files changed, 167 insertions(+), 45 deletions(-)
 create mode 100644 include/hw/pci-bridge/pci_expander_bridge.h

diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c
index e1ee8ae..9b49b0e 100644
--- a/hw/i386/acpi-build.c
+++ b/hw/i386/acpi-build.c
@@ -55,6 +55,7 @@
 #include "hw/i386/ich9.h"
 #include "hw/pci/pci_bus.h"
 #include "hw/pci-host/q35.h"
+#include "hw/pci-bridge/pci_expander_bridge.h"
 #include "hw/i386/x86-iommu.h"
 
 #include "hw/acpi/aml-build.h"
@@ -89,6 +90,10 @@
 typedef struct AcpiMcfgInfo {
     uint64_t mcfg_base;
     uint32_t mcfg_size;
+    uint32_t domain_nr;
+    uint8_t start_bus;
+    uint8_t end_bus;
+    QTAILQ_ENTRY(AcpiMcfgInfo) next;
 } AcpiMcfgInfo;
 
 typedef struct AcpiPmInfo {
@@ -119,6 +124,9 @@ typedef struct AcpiBuildPciBusHotplugState {
     bool pcihp_bridge_en;
 } AcpiBuildPciBusHotplugState;
 
+static QTAILQ_HEAD(, AcpiMcfgInfo) mcfg =
+         QTAILQ_HEAD_INITIALIZER(mcfg);
+
 static void init_common_fadt_data(Object *o, AcpiFadtData *data)
 {
     uint32_t io = object_property_get_uint(o, ACPI_PM_PROP_PM_IO_BASE, NULL);
@@ -2427,18 +2435,28 @@ build_srat(GArray *table_data, BIOSLinker *linker, 
MachineState *machine)
 }
 
 static void
-build_mcfg_q35(GArray *table_data, BIOSLinker *linker, AcpiMcfgInfo *info)
+build_mcfg_q35(GArray *table_data, BIOSLinker *linker)
 {
-    AcpiTableMcfg *mcfg;
+    AcpiTableMcfg *mcfg_tbl;
     const char *sig;
-    int len = sizeof(*mcfg) + 1 * sizeof(mcfg->allocation[0]);
-
-    mcfg = acpi_data_push(table_data, len);
-    mcfg->allocation[0].address = cpu_to_le64(info->mcfg_base);
-    /* Only a single allocation so no need to play with segments */
-    mcfg->allocation[0].pci_segment = cpu_to_le16(0);
-    mcfg->allocation[0].start_bus_number = 0;
-    mcfg->allocation[0].end_bus_number = PCIE_MMCFG_BUS(info->mcfg_size - 1);
+    int len, count = 0;
+    AcpiMcfgInfo *info;
+
+    QTAILQ_FOREACH(info, &mcfg, next) {
+        count++;
+    }
+
+    len = sizeof(*mcfg_tbl) + count * sizeof(mcfg_tbl->allocation[0]);
+
+    mcfg_tbl = acpi_data_push(table_data, len);
+
+    count = 0;
+    QTAILQ_FOREACH(info, &mcfg, next) {
+        mcfg_tbl->allocation[count].address = cpu_to_le64(info->mcfg_base);
+        mcfg_tbl->allocation[count].pci_segment = cpu_to_le16(info->domain_nr);
+        mcfg_tbl->allocation[count].start_bus_number = info->start_bus;
+        mcfg_tbl->allocation[count++].end_bus_number = info->end_bus;
+    }
 
     /* MCFG is used for ECAM which can be enabled or disabled by guest.
      * To avoid table size changes (which create migration issues),
@@ -2446,13 +2464,13 @@ build_mcfg_q35(GArray *table_data, BIOSLinker *linker, 
AcpiMcfgInfo *info)
      * but set the signature to a reserved value in this case.
      * ACPI spec requires OSPMs to ignore such tables.
      */
-    if (info->mcfg_base == PCIE_BASE_ADDR_UNMAPPED) {
+    if (QTAILQ_FIRST(&mcfg)->mcfg_base == PCIE_BASE_ADDR_UNMAPPED) {
         /* Reserved signature: ignored by OSPM */
         sig = "QEMU";
     } else {
         sig = "MCFG";
     }
-    build_header(linker, table_data, (void *)mcfg, sig, len, 1, NULL, NULL);
+    build_header(linker, table_data, (void *)mcfg_tbl, sig, len, 1, NULL, 
NULL);
 }
 
 /*
@@ -2606,25 +2624,66 @@ struct AcpiBuildState {
     MemoryRegion *linker_mr;
 } AcpiBuildState;
 
-static bool acpi_get_mcfg(AcpiMcfgInfo *mcfg)
+static inline void cleanup_mcfg(void)
+{
+    AcpiMcfgInfo *cfg, *tmp;
+
+    QTAILQ_FOREACH_SAFE (cfg, &mcfg, next, tmp) {
+        QTAILQ_REMOVE(&mcfg, cfg, next);
+        free(cfg);
+    }
+}
+
+static bool acpi_get_mcfg(void)
 {
     Object *pci_host;
     QObject *o;
+    AcpiMcfgInfo *info;
+    uint8_t bus_nr = 0, end_bus = 255;
+    uint32_t domain_nr = 0, mcfg_size = MCH_HOST_BRIDGE_PCIEXBAR_MAX;
+    uint64_t mcfg_base = MCH_HOST_BRIDGE_PCIEXBAR_DEFAULT;
+    PCIBus *bus;
 
     pci_host = acpi_get_i386_pci_host();
     g_assert(pci_host);
 
-    o = object_property_get_qobject(pci_host, PCIE_HOST_MCFG_BASE, NULL);
-    if (!o) {
-        return false;
+    while (pci_host) {
+        if (object_dynamic_cast(pci_host, TYPE_PXB_PCIE_HOST)) {
+            /* we are in pxb-pcie, overwrite default value */
+            bus = PCI_HOST_BRIDGE(pci_host)->bus;
+            domain_nr = pci_bus_domain_num(bus);
+            if (domain_nr == 0) {
+                pci_host = OBJECT(QTAILQ_NEXT(PCI_HOST_BRIDGE(pci_host), 
next));
+                continue;
+            }
+
+            o = object_property_get_qobject(pci_host, PCIE_HOST_MCFG_BASE, 
NULL);
+            assert(o);
+            mcfg_base = qnum_get_uint(qobject_to(QNum, o));
+            qobject_unref(o);
+
+            o = object_property_get_qobject(pci_host, PCIE_HOST_MCFG_SIZE, 
NULL);
+            assert(o);
+            mcfg_size = qnum_get_uint(qobject_to(QNum, o));
+            qobject_unref(o);
+
+            bus_nr = pci_bus_num(bus);
+            domain_nr = pci_bus_domain_num(bus);
+            end_bus = pci_bus_max_bus(bus);
+        }
+
+        info = g_new0(AcpiMcfgInfo, 1);
+        g_assert(info);
+        info->domain_nr = domain_nr;
+        info->start_bus = bus_nr;
+        info->end_bus = end_bus;
+        info->mcfg_base = mcfg_base;
+        info->mcfg_size = mcfg_size;
+
+        QTAILQ_INSERT_TAIL(&mcfg, info, next);
+        pci_host = OBJECT(QTAILQ_NEXT(PCI_HOST_BRIDGE(pci_host), next));
     }
-    mcfg->mcfg_base = qnum_get_uint(qobject_to(QNum, o));
-    qobject_unref(o);
 
-    o = object_property_get_qobject(pci_host, PCIE_HOST_MCFG_SIZE, NULL);
-    assert(o);
-    mcfg->mcfg_size = qnum_get_uint(qobject_to(QNum, o));
-    qobject_unref(o);
     return true;
 }
 
@@ -2637,7 +2696,6 @@ void acpi_build(AcpiBuildTables *tables, MachineState 
*machine)
     unsigned facs, dsdt, rsdt, fadt;
     AcpiPmInfo pm;
     AcpiMiscInfo misc;
-    AcpiMcfgInfo mcfg;
     Range pci_hole, pci_hole64;
     uint8_t *u;
     size_t aml_len = 0;
@@ -2718,10 +2776,11 @@ void acpi_build(AcpiBuildTables *tables, MachineState 
*machine)
             build_slit(tables_blob, tables->linker);
         }
     }
-    if (acpi_get_mcfg(&mcfg)) {
+    if (acpi_get_mcfg()) {
         acpi_add_table(table_offsets, tables_blob);
-        build_mcfg_q35(tables_blob, tables->linker, &mcfg);
+        build_mcfg_q35(tables_blob, tables->linker);
     }
+    cleanup_mcfg();
     if (x86_iommu_get_default()) {
         IommuType IOMMUType = x86_iommu_get_type();
         if (IOMMUType == TYPE_AMD) {
diff --git a/hw/pci-bridge/pci_expander_bridge.c 
b/hw/pci-bridge/pci_expander_bridge.c
index 20fec50..1e1999d 100644
--- a/hw/pci-bridge/pci_expander_bridge.c
+++ b/hw/pci-bridge/pci_expander_bridge.c
@@ -17,6 +17,7 @@
 #include "hw/pci/pci_host.h"
 #include "hw/pci/pcie_host.h"
 #include "hw/pci/pci_bridge.h"
+#include "hw/pci-bridge/pci_expander_bridge.h"
 #include "qemu/range.h"
 #include "qemu/error-report.h"
 #include "sysemu/numa.h"
@@ -58,16 +59,6 @@ typedef struct PXBDev {
     uint8_t max_bus;    /* max bus number to use(including this one) */
 } PXBDev;
 
-#define TYPE_PXB_PCIE_HOST "pxb-pcie-host"
-#define PXB_PCIE_HOST_DEVICE(obj) \
-     OBJECT_CHECK(PXBPCIEHost, (obj), TYPE_PXB_PCIE_HOST)
-
-typedef struct PXBPCIEHost {
-    /*< private >*/
-    PCIExpressHost parent_obj;
-    /*< public >*/
-} PXBPCIEHost;
-
 static PXBDev *convert_to_pxb(PCIDevice *dev)
 {
     return pci_bus_is_express(pci_get_bus(dev))
@@ -85,11 +76,17 @@ static int pxb_bus_num(PCIBus *bus)
     return pxb->bus_nr;
 }
 
-static int pxb_domain_num(PCIBus *bus)
+static int pxb_max_bus(PCIBus *bus)
+{
+    PXBDev *pxb = convert_to_pxb(bus->parent_dev);
+
+    return pxb->max_bus;
+}
+
+static uint32_t pxb_domain_num(PCIBus *bus)
 {
     PXBDev *pxb = convert_to_pxb(bus->parent_dev);
 
-    /* for pxb, this should always be zero */
     return pxb->domain_nr;
 }
 
@@ -110,8 +107,10 @@ static void pxb_bus_class_init(ObjectClass *class, void 
*data)
     PCIBusClass *pbc = PCI_BUS_CLASS(class);
 
     pbc->bus_num = pxb_bus_num;
+    pbc->max_bus = pxb_max_bus;
     pbc->is_root = pxb_is_root;
     pbc->numa_node = pxb_bus_numa_node;
+    pbc->domain_num = pxb_domain_num;
 }
 
 static const TypeInfo pxb_bus_info = {
@@ -174,7 +173,17 @@ static void pxb_pcie_host_get_mmcfg_base(Object *obj, 
Visitor *v, const char *na
 {
     PCIExpressHost *e = PCIE_HOST_BRIDGE(obj);
 
-    visit_type_uint64(v, name, &e->size, errp);
+    visit_type_uint64(v, name, &e->base_addr, errp);
+}
+
+static void pxb_pcie_host_set_mmcfg_base(Object *obj, Visitor *v, const char 
*name,
+                       void *opaque, Error **errp)
+{
+    PXBPCIEHost *host = PXB_PCIE_HOST_DEVICE(obj);
+    uint64_t value;
+
+    visit_type_uint64(v, name, &value, errp);
+    host->parent_obj.base_addr = value;
 }
 
 static void pxb_pcie_host_get_mmcfg_size(Object *obj, Visitor *v, const char 
*name,
@@ -185,6 +194,16 @@ static void pxb_pcie_host_get_mmcfg_size(Object *obj, 
Visitor *v, const char *na
     visit_type_uint64(v, name, &e->size, errp);
 }
 
+static void pxb_pcie_host_set_mmcfg_size(Object *obj, Visitor *v, const char 
*name,
+                       void *opaque, Error **errp)
+{
+    PXBPCIEHost *host = PXB_PCIE_HOST_DEVICE(obj);
+    uint32_t value;
+
+    visit_type_uint32(v, name, &value, errp);
+    host->parent_obj.size = value;
+}
+
 static void pxb_pcie_host_initfn(Object *obj)
 {
     PCIHostState *phb = PCI_HOST_BRIDGE(obj);
@@ -196,11 +215,13 @@ static void pxb_pcie_host_initfn(Object *obj)
 
     object_property_add(obj, PCIE_HOST_MCFG_BASE, "uint64",
                          pxb_pcie_host_get_mmcfg_base,
-                         NULL, NULL, NULL, NULL);
+                         pxb_pcie_host_set_mmcfg_base,
+                         NULL, NULL, NULL);
 
     object_property_add(obj, PCIE_HOST_MCFG_SIZE, "uint64",
                          pxb_pcie_host_get_mmcfg_size,
-                         NULL, NULL, NULL, NULL);
+                         pxb_pcie_host_set_mmcfg_size,
+                         NULL, NULL, NULL);
 }
 
 static void pxb_host_class_init(ObjectClass *class, void *data)
diff --git a/hw/pci/pci.c b/hw/pci/pci.c
index 80bc459..fe69672 100644
--- a/hw/pci/pci.c
+++ b/hw/pci/pci.c
@@ -142,6 +142,15 @@ static int pcibus_num(PCIBus *bus)
     return bus->parent_dev->config[PCI_SECONDARY_BUS];
 }
 
+/* return 0 unless user overwrite this callback */
+static uint32_t pcibus_domain_num(PCIBus *bus) {
+    return 0;
+}
+
+static int pcibus_max_bus(PCIBus *bus) {
+    return 255;
+}
+
 static uint16_t pcibus_numa_node(PCIBus *bus)
 {
     return NUMA_NODE_UNASSIGNED;
@@ -162,6 +171,8 @@ static void pci_bus_class_init(ObjectClass *klass, void 
*data)
     pbc->is_root = pcibus_is_root;
     pbc->bus_num = pcibus_num;
     pbc->numa_node = pcibus_numa_node;
+    pbc->domain_num = pcibus_domain_num;
+    pbc->max_bus = pcibus_max_bus;
 }
 
 static const TypeInfo pci_bus_info = {
@@ -196,7 +207,8 @@ static void pci_del_option_rom(PCIDevice *pdev);
 static uint16_t pci_default_sub_vendor_id = PCI_SUBVENDOR_ID_REDHAT_QUMRANET;
 static uint16_t pci_default_sub_device_id = PCI_SUBDEVICE_ID_QEMU;
 
-static QLIST_HEAD(, PCIHostState) pci_host_bridges;
+static QTAILQ_HEAD(, PCIHostState) pci_host_bridges =
+    QTAILQ_HEAD_INITIALIZER(pci_host_bridges);
 
 int pci_bar(PCIDevice *d, int reg)
 {
@@ -330,7 +342,7 @@ static void pci_host_bus_register(DeviceState *host)
 {
     PCIHostState *host_bridge = PCI_HOST_BRIDGE(host);
 
-    QLIST_INSERT_HEAD(&pci_host_bridges, host_bridge, next);
+    QTAILQ_INSERT_TAIL(&pci_host_bridges, host_bridge, next);
 }
 
 PCIBus *pci_device_root_bus(const PCIDevice *d)
@@ -444,6 +456,16 @@ int pci_bus_num(PCIBus *s)
     return PCI_BUS_GET_CLASS(s)->bus_num(s);
 }
 
+uint32_t pci_bus_domain_num(PCIBus *s)
+{
+    return PCI_BUS_GET_CLASS(s)->domain_num(s);
+}
+
+int pci_bus_max_bus(PCIBus *s)
+{
+    return PCI_BUS_GET_CLASS(s)->max_bus(s);
+}
+
 int pci_bus_numa_node(PCIBus *bus)
 {
     return PCI_BUS_GET_CLASS(bus)->numa_node(bus);
@@ -1798,7 +1820,7 @@ PciInfoList *qmp_query_pci(Error **errp)
     PciInfoList *info, *head = NULL, *cur_item = NULL;
     PCIHostState *host_bridge;
 
-    QLIST_FOREACH(host_bridge, &pci_host_bridges, next) {
+    QTAILQ_FOREACH(host_bridge, &pci_host_bridges, next) {
         info = g_malloc0(sizeof(*info));
         info->value = qmp_query_pci_bus(host_bridge->bus,
                                         pci_bus_num(host_bridge->bus));
@@ -2493,7 +2515,7 @@ int pci_qdev_find_device(const char *id, PCIDevice **pdev)
     PCIHostState *host_bridge;
     int rc = -ENODEV;
 
-    QLIST_FOREACH(host_bridge, &pci_host_bridges, next) {
+    QTAILQ_FOREACH(host_bridge, &pci_host_bridges, next) {
         int tmp = pci_qdev_find_recursive(host_bridge->bus, id, pdev);
         if (!tmp) {
             rc = 0;
diff --git a/include/hw/pci-bridge/pci_expander_bridge.h 
b/include/hw/pci-bridge/pci_expander_bridge.h
new file mode 100644
index 0000000..e017c62
--- /dev/null
+++ b/include/hw/pci-bridge/pci_expander_bridge.h
@@ -0,0 +1,16 @@
+#ifndef HW_PCI_EXPANDER_H
+#define HW_PCI_EXPANDER_H
+
+#include "hw/pci/pcie_host.h"
+
+#define TYPE_PXB_PCIE_HOST "pxb-pcie-host"
+#define PXB_PCIE_HOST_DEVICE(obj) \
+     OBJECT_CHECK(PXBPCIEHost, (obj), TYPE_PXB_PCIE_HOST)
+
+typedef struct PXBPCIEHost {
+    /*< private >*/
+    PCIExpressHost parent_obj;
+    /*< public >*/
+} PXBPCIEHost;
+
+#endif
diff --git a/include/hw/pci/pci.h b/include/hw/pci/pci.h
index 990d6fc..b127fcf 100644
--- a/include/hw/pci/pci.h
+++ b/include/hw/pci/pci.h
@@ -436,6 +436,8 @@ static inline PCIBus *pci_get_bus(const PCIDevice *dev)
     return PCI_BUS(qdev_get_parent_bus(DEVICE(dev)));
 }
 int pci_bus_num(PCIBus *s);
+uint32_t pci_bus_domain_num(PCIBus *s);
+int pci_bus_max_bus(PCIBus *s);
 static inline int pci_dev_bus_num(const PCIDevice *dev)
 {
     return pci_bus_num(pci_get_bus(dev));
diff --git a/include/hw/pci/pci_bus.h b/include/hw/pci/pci_bus.h
index b7da8f5..faad155 100644
--- a/include/hw/pci/pci_bus.h
+++ b/include/hw/pci/pci_bus.h
@@ -15,7 +15,9 @@ typedef struct PCIBusClass {
 
     bool (*is_root)(PCIBus *bus);
     int (*bus_num)(PCIBus *bus);
+    int (*max_bus)(PCIBus *bus);
     uint16_t (*numa_node)(PCIBus *bus);
+    uint32_t (*domain_num)(PCIBus *bus);
 } PCIBusClass;
 
 struct PCIBus {
diff --git a/include/hw/pci/pci_host.h b/include/hw/pci/pci_host.h
index ba31595..a5617cf 100644
--- a/include/hw/pci/pci_host.h
+++ b/include/hw/pci/pci_host.h
@@ -47,7 +47,7 @@ struct PCIHostState {
     uint32_t config_reg;
     PCIBus *bus;
 
-    QLIST_ENTRY(PCIHostState) next;
+    QTAILQ_ENTRY(PCIHostState) next;
 };
 
 typedef struct PCIHostBridgeClass {
-- 
2.7.4