feat: DWC MSI and SR-IOV support

Cargo.toml

@@ -51,6 +51,7 @@ uefi-raw = "=0.9.0"
 ############# general ##############
 iommu = [] # supported by: aarch64, riscv64
 pci = [] # supported by: aarch64, riscv64,loongarch64
+sriov = ["pci"] # SR-IOV PF/VF enumeration and passthrough
 print_timestamp = [] # print timestamp when logging
 
 ############# IOMMU ##############
@@ -65,6 +66,8 @@ ecam_pcie = [] # Standard ECAM mechanism (default for most platforms)
 dwc_pcie = [] # DesignWare PCIe Core mechanism (CFG0/CFG1, used by RK3568)
 loongarch64_pcie = [] # LoongArch PCIe mechanism (used by LoongArch platforms)
 no_pcie_bar_realloc = []
+dwc_msi = ["dwc_pcie"] # DesignWare Native MSI (used when arch doesn't provide MSI, e.g., ARM without ITS)
+pci_init_delay = ["dwc_pcie"] # Delay hvisor PCI init until guest DBI offset 0 access
 virtio_pci = [] # Virtual virtio pci support
 
 ############# aarch64 ##############

platform/aarch64/imx8mp/board.rs

@@ -21,6 +21,9 @@ use crate::{
     config::*,
 };
 
+use crate::pci::vpci_dev::VpciDevType;
+use crate::pci_dev;
+
 pub const BOARD_NAME: &str = "imx8mp";
 
 pub const BOARD_NCPUS: usize = 4;
@@ -107,14 +110,23 @@ pub const ROOT_ZONE_MEMORY_REGIONS: [HvConfigMemoryRegion; 8] = [
        //     virtual_start: 0x30890000,
        //     size: 0x1000,
        // }, // serial
+       // 0x32f00000
+       // HvConfigMemoryRegion {
+       //     mem_type: MEM_TYPE_IO,
+       //     physical_start: 0x32f00000,
+       //     virtual_start: 0x32f00000,
+       //     size: 0x10000,
+       // }, // pcie-phy
 ];
 
 pub const IRQ_WAKEUP_VIRTIO_DEVICE: usize = 32 + 0x20;
 pub const ROOT_ZONE_IRQS_BITMAP: &[BitmapWord] = &get_irqs_bitmap(&[
     35, 36, 37, 38, 45, 52, 55, 56, 57, 59, 64, 67, 75, 96, 97, 98, 99, 100, 101, 102, 103, 104,
-    105, 135, 150, 151, 152, 162,
+    105, 135, 150, 151, 152, 162, 172, 159,
 ]);
 
+pub const ROOT_ZONE_IVC_CONFIG: [HvIvcConfig; 0] = [];
+
 pub const ROOT_ARCH_ZONE_CONFIG: HvArchZoneConfig = HvArchZoneConfig {
     is_aarch32: 0,
     gic_config: GicConfig::Gicv3(Gicv3Config {
@@ -127,4 +139,39 @@ pub const ROOT_ARCH_ZONE_CONFIG: HvArchZoneConfig = HvArchZoneConfig {
     }),
 };
 
-pub const ROOT_ZONE_IVC_CONFIG: [HvIvcConfig; 0] = [];
+pub const ROOT_PCI_CONFIG: &[HvPciConfig] = &[HvPciConfig {
+    ecam_base: 0x33800000,
+    ecam_size: 0x400000,
+    io_base: 0x1ff80000,
+    io_size: 0x10000,
+    pci_io_base: 0x0,
+    mem32_base: 0x1800_0000,
+    mem32_size: 0x7f0_0000,
+    pci_mem32_base: 0x1800_0000,
+    mem64_base: 0x0,
+    mem64_size: 0x0,
+    pci_mem64_base: 0x0,
+    bus_range_begin: 0x0,
+    bus_range_end: 0x1f,
+    domain: 0x0,
+}];
+
+pub const ROOT_DWC_ATU_CONFIG: &[HvDwcAtuConfig] = &[HvDwcAtuConfig {
+    ecam_base: 0x33800000,
+    dbi_base: 0x33800000,
+    dbi_size: 0x400000,
+    apb_base: 0x0,
+    apb_size: 0x0,
+    cfg_base: 0x1ff00000,
+    cfg_size: 0x80000,
+    io_cfg_atu_shared: 1,
+    io_atu_index: 1,
+    dw_msi_irq: 172,
+}];
+
+pub const ROOT_PCI_DEVS: [HvPciDevConfig; 2] = [
+    pci_dev!(0x0, 0x00, 0x0, 0x0 => 0x0, 0x0, 0x0, VpciDevType::Physical),
+    pci_dev!(0x0, 0x01, 0x0, 0x0 => 0x1, 0x0, 0x0, VpciDevType::Physical),
+    // pci_dev!(0x0, 0x02, 0x10, 0x0 => 0x02, 0x10, 0x0, VpciDevType::Physical), // SRIOV VF
+    // pci_dev!(0x0, 0x02, 0x10, 0x4 => 0x02, 0x10, 0x4, VpciDevType::Physical), // SRIOV VF
+];

platform/aarch64/imx8mp/cargo/features

@@ -1,2 +1,8 @@
 gicv3
 imx_uart
+pci
+dwc_pcie
+no_pcie_bar_realloc
+dwc_msi
+pci_init_delay
+sriov

platform/aarch64/imx8mp/configs/zone1-linux.json

@@ -2,80 +2,154 @@
     "arch": "arm64",
     "name": "linux2",
     "zone_id": 1,
-    "cpus": [2, 3],
+    "cpus": [
+        2,
+        3
+    ],
     "memory_regions": [
         {
             "type": "ram",
             "physical_start": "0x50000000",
-            "virtual_start":  "0x50000000",
+            "virtual_start": "0x50000000",
             "size": "0x30000000"
         },
         {
             "type": "io",
             "physical_start": "0x30a60000",
-            "virtual_start":  "0x30a60000",
+            "virtual_start": "0x30a60000",
             "size": "0x10000"
         },
         {
             "type": "virtio",
             "physical_start": "0xa003c00",
-            "virtual_start":  "0xa003c00",
+            "virtual_start": "0xa003c00",
             "size": "0x200"
         },
         {
             "type": "virtio",
             "physical_start": "0xa003800",
-            "virtual_start":  "0xa003800",
+            "virtual_start": "0xa003800",
             "size": "0x200"
         },
         {
             "type": "virtio",
             "physical_start": "0xa003600",
-            "virtual_start":  "0xa003600",
+            "virtual_start": "0xa003600",
             "size": "0x200"
         },
         {
             "type": "io",
             "physical_start": "0x38000000",
-            "virtual_start":  "0x38000000",
+            "virtual_start": "0x38000000",
             "size": "0x10000"
         },
         {
             "type": "io",
             "physical_start": "0x38500000",
-            "virtual_start":  "0x38500000",
+            "virtual_start": "0x38500000",
             "size": "0x20000"
         },
         {
             "type": "io",
             "physical_start": "0x32e80000",
-            "virtual_start":  "0x32e80000",
+            "virtual_start": "0x32e80000",
             "size": "0x20000"
         },
         {
             "type": "io",
             "physical_start": "0x30c00000",
-            "virtual_start":  "0x30c00000",
+            "virtual_start": "0x30c00000",
             "size": "0x400000"
         },
         {
             "type": "io",
             "physical_start": "0x32fc0000",
-            "virtual_start":  "0x32fc0000",
+            "virtual_start": "0x32fc0000",
             "size": "0x20000"
         }
     ],
-    "interrupts": [35, 37, 38, 45, 56, 57, 61, 75, 76, 78, 135, 162],
+    "interrupts": [
+        35,
+        37,
+        38,
+        45,
+        56,
+        57,
+        61,
+        75,
+        76,
+        78,
+        135,
+        162,
+        172
+    ],
     "ivc_configs": [],
     "kernel_filepath": "./Image",
     "dtb_filepath": "./linux2.dtb",
     "kernel_load_paddr": "0x50400000",
-    "dtb_load_paddr":   "0x50000000",
-    "entry_point":      "0x50400000",
+    "dtb_load_paddr": "0x50000000",
+    "entry_point": "0x50400000",
     "arch_config": {
+        "gic_version": "v3",
         "gicd_base": "0x38800000",
         "gicd_size": "0x10000",
         "gicr_base": "0x38880000",
-        "gicr_size": "0xc0000"
-    }
-}
+        "gicr_size": "0xc0000",
+        "is_aarch32": false
+    },
+    "pci_config": [
+        {
+            "ecam_base": "0x33800000",
+            "ecam_size": "0x400000",
+            "io_base": "0x1ff80000",
+            "io_size": "0x10000",
+            "pci_io_base": "0x0",
+            "mem32_base": "0x18000000",
+            "mem32_size": "0x7f00000",
+            "pci_mem32_base": "0x18000000",
+            "mem64_base": "0x0",
+            "mem64_size": "0x0",
+            "pci_mem64_base": "0x0",
+            "bus_range_begin": "0x0",
+            "bus_range_end": "0x1f",
+            "domain": "0x0"
+        }
+    ],
+    "dwc_atu_config": [
+        {
+            "ecam_base": "0x33800000",
+            "dbi_base": "0x33800000",
+            "dbi_size": "0x400000",
+            "apb_base": "0x0",
+            "apb_size": "0x0",
+            "cfg_base": "0x1ff00000",
+            "cfg_size": "0x10000",
+            "io_cfg_atu_shared": 1,
+            "io_atu_index": 1,
+            "dw_msi_irq": 172
+        }
+    ],
+    "num_pci_devs": 2,
+    "alloc_pci_devs": [
+        {
+            "domain": "0x0",
+            "bus": "0x0",
+            "device": "0x0",
+            "function": "0x0",
+            "v_bus": "0x0",
+            "v_device": "0x0",
+            "v_function": "0x0",
+            "dev_type": "0"
+        },
+        {
+            "domain": "0x0",
+            "bus": "0x1",
+            "device": "0x0",
+            "function": "0x1",
+            "v_bus": "0x1",
+            "v_device": "0x0",
+            "v_function": "0x0",
+            "dev_type": "0"
+        }
+    ]
+}

platform/aarch64/qemu-gicv2/board.rs

@@ -119,7 +119,7 @@ pub const ROOT_PCI_CONFIG: [HvPciConfig; 1] = [
 pub const ROOT_ZONE_IVC_CONFIG: [HvIvcConfig; 0] = [];
 
 pub const ROOT_PCI_DEVS: [HvPciDevConfig; 3] = [
-    pci_dev!(0x0, 0x0, 0x0, 0x0, VpciDevType::Physical),
-    pci_dev!(0x0, 0x0, 0x1, 0x0, VpciDevType::Physical),
-    pci_dev!(0x0, 0x0, 0x2, 0x0, VpciDevType::Physical),
+    pci_dev!(0x0, 0x0, 0x0, 0x0 => 0x0, 0x0, 0x0, VpciDevType::Physical),
+    pci_dev!(0x0, 0x0, 0x1, 0x0 => 0x0, 0x1, 0x0, VpciDevType::Physical),
+    pci_dev!(0x0, 0x0, 0x2, 0x0 => 0x0, 0x2, 0x0, VpciDevType::Physical),
 ];

platform/aarch64/qemu-gicv3/board.rs

@@ -117,8 +117,8 @@ pub const ROOT_PCI_CONFIG: [HvPciConfig; 1] = [HvPciConfig {
 pub const ROOT_ZONE_IVC_CONFIG: [HvIvcConfig; 0] = [];
 
 pub const ROOT_PCI_DEVS: &[HvPciDevConfig] = &[
-    pci_dev!(0x0, 0x0, 0x0, 0x0, VpciDevType::Physical),
-    pci_dev!(0x0, 0x0, 0x1, 0x0, VpciDevType::Physical),
-    // pci_dev!(0x0, 0x0, 0x3, 0x0, VpciDevType::Physical),
-    pci_dev!(0x0, 0x0, 0x5, 0x0, VpciDevType::StandardVdev),
+    pci_dev!(0x0, 0x0, 0x0, 0x0 => 0x0, 0x0, 0x0, VpciDevType::Physical),
+    pci_dev!(0x0, 0x0, 0x1, 0x0 => 0x0, 0x3, 0x0, VpciDevType::Physical),
+    // pci_dev!(0x0, 0x0, 0x3, 0x0 => 0x0, 0x3, 0x0, VpciDevType::Physical),
+    // pci_dev!(0x0, 0x0, 0x5, 0x0 => 0x0, 0x5, 0x0, VpciDevType::StandardVdev),
 ];

platform/aarch64/qemu-gicv3/configs/zone1-linux.json

@@ -1,12 +1,15 @@
 {
     "name": "linux2",
     "zone_id": 1,
-    "cpus": [2, 3],
+    "cpus": [
+        2,
+        3
+    ],
     "memory_regions": [
         {
             "type": "ram",
             "physical_start": "0x50000000",
-            "virtual_start":  "0x50000000",
+            "virtual_start": "0x50000000",
             "size": "0x30000000"
         },
         {
@@ -37,8 +40,8 @@
     "kernel_filepath": "./Image",
     "dtb_filepath": "./zone1-linux.dtb",
     "kernel_load_paddr": "0x50400000",
-    "dtb_load_paddr":   "0x50000000",
-    "entry_point":      "0x50400000",
+    "dtb_load_paddr": "0x50000000",
+    "entry_point": "0x50400000",
     "arch_config": {
         "gic_version": "v3",
         "gicd_base": "0x8000000",
@@ -49,36 +52,44 @@
         "gits_size": "0x20000",
         "is_aarch32": false
     },
-    "pci_config": [{
-        "ecam_base": "0x4010000000",
-        "ecam_size": "0x10000000",
-        "io_base": "0x3eff0000",
-        "io_size": "0x10000",
-        "pci_io_base": "0x0",
-        "mem32_base": "0x10000000",
-        "mem32_size": "0x2eff0000",
-        "pci_mem32_base": "0x10000000",
-        "mem64_base": "0x8000000000",
-        "mem64_size": "0x8000000000",
-        "pci_mem64_base": "0x8000000000",
-        "bus_range_begin": "0x0",
-        "bus_range_end": "0x1f",
-        "domain": "0x0"
-    }],
+    "pci_config": [
+        {
+            "ecam_base": "0x4010000000",
+            "ecam_size": "0x10000000",
+            "io_base": "0x3eff0000",
+            "io_size": "0x10000",
+            "pci_io_base": "0x0",
+            "mem32_base": "0x10000000",
+            "mem32_size": "0x2eff0000",
+            "pci_mem32_base": "0x10000000",
+            "mem64_base": "0x8000000000",
+            "mem64_size": "0x8000000000",
+            "pci_mem64_base": "0x8000000000",
+            "bus_range_begin": "0x0",
+            "bus_range_end": "0x1f",
+            "domain": "0x0"
+        }
+    ],
     "num_pci_devs": 2,
     "alloc_pci_devs": [
         {
             "domain": "0x0",
             "bus": "0x0",
             "device": "0x0",
             "function": "0x0",
+            "v_bus": "0x0",
+            "v_device": "0x0",
+            "v_function": "0x0",
             "dev_type": "0"
         },
         {
             "domain": "0x0",
             "bus": "0x0",
-            "device": "0x1",
+            "device": "0x2",
             "function": "0x0",
+            "v_bus": "0x0",
+            "v_device": "0x1",
+            "v_function": "0x0",
             "dev_type": "0"
         }
     ]