From f82f74cfc809ef77e9195beeee4e780914f68661 Mon Sep 17 00:00:00 2001 From: Dan Zheng Date: Thu, 5 Mar 2026 16:31:23 +0800 Subject: [PATCH 1/2] Add case for mutiple device passthrough Signed-off-by: Dan Zheng --- .../cmdqv_various_hugepages_membackends.cfg | 59 +++++++ .../cfg/gpu/multiple_devices_passthrough.cfg | 80 +++++++++ .../src/gpu/multiple_devices_passthrough.py | 156 ++++++++++++++++++ 3 files changed, 295 insertions(+) create mode 100644 libvirt/tests/cfg/gpu/cmdqv_various_hugepages_membackends.cfg create mode 100644 libvirt/tests/cfg/gpu/multiple_devices_passthrough.cfg create mode 100644 libvirt/tests/src/gpu/multiple_devices_passthrough.py diff --git a/libvirt/tests/cfg/gpu/cmdqv_various_hugepages_membackends.cfg b/libvirt/tests/cfg/gpu/cmdqv_various_hugepages_membackends.cfg new file mode 100644 index 00000000000..ff3fa040f84 --- /dev/null +++ b/libvirt/tests/cfg/gpu/cmdqv_various_hugepages_membackends.cfg @@ -0,0 +1,59 @@ +- gpu.cmdqv_various_hugepages_membackends: + type = cmdqv_various_hugepages_membackends + start_vm = "no" + mem_value = 32 + vcpu = 32 + numa_attrs = "'cpu': {'numa_cell':[{'id':'0','cpus':'0-31','memory':'32','unit':'GiB'},{'id':'1','memory':'0'},{'id':'2','memory':'0'},{'id':'3','memory':'0'},{'id':'4','memory':'0'},{'id':'5','memory':'0'},{'id':'6','memory':'0'},{'id':'7','memory':'0'},{'id':'8','memory':'0'}]}" + kernel_hp_file = '/sys/devices/system/node/node%s/hugepages/hugepages-%skB/nr_hugepages' + acpi_nodeset = {'nodeset':'1-8'} + test_devices = '["3D", "Network Connection", "Ethernet Controller Virtual Function", "Mellanox Technologies"]' + pci_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}" + cmdqv_on_num = 2 + cmdqv_dict = {'cmdqv':'on'} + cmdqv_dict_2 = {'cmdqv':'on'} + iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on','ats':'on','ril':'off','ssidSize':'20','oas':'48'}} + # For another VFIO device + iommu_dict_2 = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on','ats':'on','ril':'off','ssidSize':'20','oas':'48'}} + # gpu smmuv3 goes first + smmuv3_desc_defined = no + nic_driver = "'driver':{'driver_attr':{'iommufd':'yes'}}" + only aarch64 + variants: + - 512M: + hugepage_size = 512 + - 2M: + hugepage_size = 2 + - 16384M: + hugepage_size = 16384 + variants source_type: + - memfd: + no 512M + - file: + no 16384M + variants: + - pxb_pcie: + pxb_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':0x00,'slot':'%s','function':'0x0'}}" + pxb_ctrl_attrs = {'type':'pci','model': 'pcie-expander-bus','index': '%s',${pxb_addr}} + pcie_root_port_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}" + pcie_root_port_attrs = {'type':'pci','model':'pcie-root-port','index':'%s',${pcie_root_port_addr}} + pxb_ctrl_attrs_2 = {'type':'pci','model': 'pcie-expander-bus','index': '%s',${pxb_addr}} + pcie_root_port_attrs_2 = {'type':'pci','model':'pcie-root-port','index':'%s',${pcie_root_port_addr}} + variants dev_type: + - nic_interface: + variants dev_source: + - vf: + only 2M 16384M + iface_dict = {'managed':'yes','type_name':'hostdev','hostdev_address':{'type_name':'pci','attrs':vf_pci_addr},${nic_driver},'alias':{'name':'ua-89cbe690-6c6c-4f2f-adac-5826fe52ea74'},'mac_address':mac_addr,${pci_addr}} + check_ping_time = "yes" + - nic_hostdev: + variants dev_source: + - vf: + no 16384M + hostdev_dict = {${nic_driver},'alias':{'name':'ua-1bcbabff-f022-4d4f-ae8c-13f2d3a07906'},'mode':'subsystem','type':'pci','source':{'untyped_address':vf_pci_addr},'managed':'yes',${pci_addr}} + - pf: + no 512M + allow_unlink_pf = "yes" + hostdev_dict = {${nic_driver},'mode': 'subsystem','type':'pci','source':{'untyped_address':pf_pci_addr},'managed':'yes',${pci_addr}} + vm_attrs = {'memory_unit':'GiB','memory':${mem_value},'vcpu':${vcpu},${numa_attrs}} + gpu_hostdev_dict = {'driver':{'driver_attr':{'iommufd':'yes'}},'acpi':${acpi_nodeset},'mode':'subsystem','type':'pci','source':{'untyped_address':gpu_pci_addr},'managed':'yes',${pci_addr}} + memory_backing = {'hugepages': {'pages': [{'size':'${hugepage_size}','unit':'MiB'}]},'source_type':'${source_type}','access_mode':'shared','allocation':{'mode':'immediate'}} diff --git a/libvirt/tests/cfg/gpu/multiple_devices_passthrough.cfg b/libvirt/tests/cfg/gpu/multiple_devices_passthrough.cfg new file mode 100644 index 00000000000..b10c367e6bb --- /dev/null +++ b/libvirt/tests/cfg/gpu/multiple_devices_passthrough.cfg @@ -0,0 +1,80 @@ +- gpu.multiple_devices_passthrough: + type = multiple_devices_passthrough + start_vm = "no" + mem_value = 32 + vcpu = 32 + numa_attrs = "'cpu': {'numa_cell':[{'id':'0','cpus':'0-31','memory':'32','unit':'GiB'},{'id':'1','memory':'0'},{'id':'2','memory':'0'},{'id':'3','memory':'0'},{'id':'4','memory':'0'},{'id':'5','memory':'0'},{'id':'6','memory':'0'},{'id':'7','memory':'0'},{'id':'8','memory':'0'}]}" + memory_backing = {'hugepages': {'pages': [{'size': '512', 'unit': 'MiB'}]},'source_type':'memfd','access_mode':'shared','allocation':{'mode':'immediate'}} + kernel_hp_file = '/sys/devices/system/node/node%s/hugepages/hugepages-%skB/nr_hugepages' + acpi_nodeset = {'nodeset':'1-8'} + test_devices = '["3D", "Network Connection", "Ethernet Controller Virtual Function", "Mellanox Technologies"]' + pci_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}" + only aarch64 + variants: + - both_cmdqv_on: + cmdqv_on_num = 2 + cmdqv_dict = {'cmdqv':'on'} + cmdqv_dict_2 = {'cmdqv':'on'} + - both_cmdqv_off: + cmdqv_on_num = 0 + cmdqv_dict = {'cmdqv':'off'} + cmdqv_dict_2 = {'cmdqv':'off'} + memory_backing = {} + - gpu_cmdqv_off_nic_cmdqv_on: + cmdqv_on_num = 1 + cmdqv_dict = {'cmdqv':'off'} + cmdqv_dict_2 = {'cmdqv':'on'} + - gpu_cmdqv_on_nic_cmdqv_off: + cmdqv_on_num = 1 + cmdqv_dict = {'cmdqv':'on'} + cmdqv_dict_2 = {'cmdqv':'off'} + - gpu_cmdqv_on: + cmdqv_on_num = 1 + cmdqv_dict = {'cmdqv':'on'} + - gpu_cmdqv_off: + cmdqv_on_num = 0 + cmdqv_dict = {'cmdqv':'off'} + variants: + - only_gpu_smmuv3: + only gpu_cmdqv_on gpu_cmdqv_off + iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on','ats':'on','ril':'off','ssidSize':'20','oas':'48'}} + - both_smmuv3: + no gpu_cmdqv_on gpu_cmdqv_off + # For GPU as default + iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on','ats':'on','ril':'off','ssidSize':'20','oas':'48'}} + # For another VFIO device + iommu_dict_2 = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on','ats':'on','ril':'off','ssidSize':'20','oas':'48'}} + variants: + - nic_smmuv3_first: + smmuv3_desc_defined = yes + - gpu_smmuv3_first: + smmuv3_desc_defined = no + variants: + - pxb_pcie: + pxb_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':0x00,'slot':'%s','function':'0x0'}}" + pxb_ctrl_attrs = {'type':'pci','model': 'pcie-expander-bus','index': '%s',${pxb_addr}} + pcie_root_port_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}" + pcie_root_port_attrs = {'type':'pci','model':'pcie-root-port','index':'%s',${pcie_root_port_addr}} + pxb_ctrl_attrs_2 = {'type':'pci','model': 'pcie-expander-bus','index': '%s',${pxb_addr}} + pcie_root_port_attrs_2 = {'type':'pci','model':'pcie-root-port','index':'%s',${pcie_root_port_addr}} + variants: + - nic_iommufd_yes: + nic_driver = "'driver':{'driver_attr':{'iommufd':'yes'}}" + - nic_iommufd_no: + only only_gpu_smmuv3 + nic_driver = "'driver':{'driver_attr':{'iommufd':'no'}}" + variants dev_type: + - nic_interface: + variants dev_source: + - vf: + iface_dict = {'managed':'yes','type_name':'hostdev','hostdev_address':{'type_name':'pci','attrs':vf_pci_addr},${nic_driver},'alias':{'name':'ua-89cbe690-6c6c-4f2f-adac-5826fe52ea74'},'mac_address':mac_addr,${pci_addr}} + check_ping_time = "yes" + - nic_hostdev: + variants dev_source: + - vf: + hostdev_dict = {${nic_driver},'alias':{'name':'ua-1bcbabff-f022-4d4f-ae8c-13f2d3a07906'},'mode':'subsystem','type':'pci','source':{'untyped_address':vf_pci_addr},'managed':'yes',${pci_addr}} + - pf: + allow_unlink_pf = "yes" + hostdev_dict = {${nic_driver},'mode': 'subsystem','type':'pci','source':{'untyped_address':pf_pci_addr},'managed':'yes',${pci_addr}} + vm_attrs = {'memory_unit':'GiB','memory':${mem_value},'vcpu':${vcpu},${numa_attrs}} + gpu_hostdev_dict = {'driver':{'driver_attr':{'iommufd':'yes'}},'acpi':${acpi_nodeset},'mode':'subsystem','type':'pci','source':{'untyped_address':gpu_pci_addr},'managed':'yes',${pci_addr}} diff --git a/libvirt/tests/src/gpu/multiple_devices_passthrough.py b/libvirt/tests/src/gpu/multiple_devices_passthrough.py new file mode 100644 index 00000000000..5c3f3fb8ad1 --- /dev/null +++ b/libvirt/tests/src/gpu/multiple_devices_passthrough.py @@ -0,0 +1,156 @@ +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# Copyright Red Hat +# +# SPDX-License-Identifier: GPL-2.0 +# +# Author: dzheng +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +from virttest import utils_misc +from virttest import virsh + +from virttest.libvirt_xml import vm_xml +from virttest.utils_libvirt import libvirt_vmxml +from virttest.utils_libvirt import libvirt_vfio +from virttest.utils_test import libvirt + +from provider.gpu import gpu_base +from provider.gpu import check_points +from provider.sriov import sriov_base +from provider.sriov import check_points as sriov_check_points + + +def run(test, params, env): + """ + Start vm with GPU device and NIC device (VF/PF) passthrough + + Steps: + 1. Setup GPU hostdev device + 2. Setup NIC hostdev device or interface (VF/PF) + 3. Start the VM + 4. Check devices using unified check_lspci() entry + 5. Check NIC-specific features (mac address, network accessibility) + 6. Destroy VM + 7. Check environment recovery (driver, mac address) + """ + + def check_nic_specific(): + """ + Check NIC-specific features + """ + test.log.info("TEST_STEP: Check NIC-specific features") + nic_iface_dict_updated = nic_iface_dict.copy() + if nic_dev_type == "network_interface": + nic_iface_dict_updated.update({'type_name': 'hostdev'}) + + sriov_check_points.comp_hostdev_xml( + vm, nic_device_type, nic_iface_dict_updated) + sriov_check_points.check_mac_addr( + vm_session, vm.name, nic_device_type, nic_iface_dict) + + check_ping_time = params.get("check_ping_time") + # TODO:check the Linked status and decide if network connectivity is checked + #if check_ping_time == "yes": + # sriov_check_points.check_vm_network_accessed(vm_session) + # test_pf = params.get("test_pf") + # if test_pf and sriov_vfio.is_linked(pf_name=test_pf): + # migration_obj.migration_test.ping_vm(vm, params) + # migration_obj.check_vm_cont_ping(False) + # else: + test.log.debug(f"Skip ping test due to no link to PF ") + + def check_recovery(): + """ + Check environment recovery after VM destroy + """ + test.log.info("TEST_STEP: Check GPU driver recovery") + if not utils_misc.wait_for( + lambda: libvirt_vfio.check_vfio_pci( + gpu_dev_pci, not gpu_managed_disabled, True, exp_driver="nvgrace_gpu_vfio_pci"), 10, 5): + test.fail("GPU driver recovery failed!") + if gpu_managed_disabled: + virsh.nodedev_reattach( + gpu_dev_name, debug=True, ignore_status=False) + libvirt_vfio.check_vfio_pci( + gpu_dev_pci, True, exp_driver="nvgrace_gpu_vfio_pci") + + test.log.info("TEST_STEP: Check NIC driver and mac recovery") + + if not utils_misc.wait_for( + lambda: libvirt_vfio.check_vfio_pci( + nic_dev_pci, not nic_managed_disabled, True), 10, 5): + test.fail("NIC driver recovery failed!") + if nic_managed_disabled: + virsh.nodedev_reattach( + nic_dev_name, debug=True, ignore_status=False) + libvirt_vfio.check_vfio_pci(nic_dev_pci, True) + sriov_check_points.check_mac_addr_recovery( + sriov_test_obj.pf_name, nic_device_type, nic_iface_dict) + + vm_name = params.get("main_vm", "avocado-vt-vm1") + vm = env.get_vm(vm_name) + nic_dev_type = params.get("dev_type", "") + nic_dev_source = params.get("dev_source", "") + + sriov_test_obj = sriov_base.SRIOVTest(vm, test, params) + if nic_dev_type == "nic_hostdev" and nic_dev_source.startswith("pf"): + nic_dev_name = sriov_test_obj.pf_dev_name + nic_dev_pci = sriov_test_obj.pf_pci + else: + nic_dev_name = sriov_test_obj.vf_dev_name + nic_dev_pci = sriov_test_obj.vf_pci + + nic_iface_dict = sriov_test_obj.parse_iface_dict() + nic_managed_disabled = nic_iface_dict.get('managed') != "yes" + nic_device_type = "hostdev" if nic_dev_type == "nic_hostdev" else "interface" + + gpu_test = gpu_base.GPUTest(vm, test, params, sriov_helper=sriov_test_obj) + gpu_dev_name = gpu_test.gpu_dev_name + gpu_dev_pci = gpu_test.gpu_pci + gpu_hostdev_dict = gpu_test.parse_hostdev_dict() + gpu_managed_disabled = gpu_hostdev_dict.get('managed') != "yes" + vm_session = None + + try: + test.log.info("TEST_SETUP: Setup GPU and NIC devices") + gpu_test.setup_default(dev_name=gpu_dev_name, test_hopper_gpu="yes") + + sriov_test_obj.setup_default( + dev_name=nic_dev_name, + managed_disabled=nic_managed_disabled, + cleanup_ifaces="no") + + test.log.info("TEST_STEP: Start the VM") + vm.start() + vm_session = vm.wait_for_login(timeout=240) + test.log.debug( + f'VMXML of {vm_name}:\n{virsh.dumpxml(vm_name).stdout_text}') + check_points.check_lspci( + test, + vm_session, + eval(params.get("test_devices", "3D")), + dev_iommu=True if params.get("iommu_dict_2") else False, + ) + check_points.check_nvidia_smi(test, vm_session) + cmdqv_on_num = int(params.get("cmdqv_on_num", "1")) + check_points.check_guest_cmdqv_dmesg(test, vm_session, expect_num=cmdqv_on_num) + libvirt_vfio.check_vfio_pci(gpu_dev_pci, exp_driver="nvgrace_gpu_vfio_pci") + test.log.info("Verify: GPU driver is nvgrace_gpu_vfio_pci - PASS") + if vm_session: + vm_session.close() + test.log.info("TEST_STEP: Destroy VM") + vm.destroy() + check_recovery() + #check_points.check_qemu_log(test, vm) + + finally: + test.log.info("TEST_TEARDOWN: Cleanup test environment") + gpu_test.teardown_default( + managed_disabled=gpu_managed_disabled, + dev_name=gpu_dev_name) + sriov_test_obj.teardown_default( + managed_disabled=nic_managed_disabled, + dev_name=nic_dev_name) From 5aedb84a687b5071343601c4edfc5410214dabc1 Mon Sep 17 00:00:00 2001 From: Dan Zheng Date: Fri, 3 Apr 2026 07:11:27 +0800 Subject: [PATCH 2/2] libvirt/gpu: update test type to multiple_devices_passthrough Signed-off-by: Dan Zheng --- libvirt/tests/cfg/gpu/cmdqv_various_hugepages_membackends.cfg | 4 ++-- libvirt/tests/cfg/gpu/multiple_devices_passthrough.cfg | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libvirt/tests/cfg/gpu/cmdqv_various_hugepages_membackends.cfg b/libvirt/tests/cfg/gpu/cmdqv_various_hugepages_membackends.cfg index ff3fa040f84..cd8edb82d46 100644 --- a/libvirt/tests/cfg/gpu/cmdqv_various_hugepages_membackends.cfg +++ b/libvirt/tests/cfg/gpu/cmdqv_various_hugepages_membackends.cfg @@ -1,10 +1,10 @@ - gpu.cmdqv_various_hugepages_membackends: - type = cmdqv_various_hugepages_membackends + type = multiple_devices_passthrough start_vm = "no" mem_value = 32 vcpu = 32 numa_attrs = "'cpu': {'numa_cell':[{'id':'0','cpus':'0-31','memory':'32','unit':'GiB'},{'id':'1','memory':'0'},{'id':'2','memory':'0'},{'id':'3','memory':'0'},{'id':'4','memory':'0'},{'id':'5','memory':'0'},{'id':'6','memory':'0'},{'id':'7','memory':'0'},{'id':'8','memory':'0'}]}" - kernel_hp_file = '/sys/devices/system/node/node%s/hugepages/hugepages-%skB/nr_hugepages' + kernel_hp_file = '/sys/kernel/mm/hugepages/hugepages-%skB/nr_hugepages' acpi_nodeset = {'nodeset':'1-8'} test_devices = '["3D", "Network Connection", "Ethernet Controller Virtual Function", "Mellanox Technologies"]' pci_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}" diff --git a/libvirt/tests/cfg/gpu/multiple_devices_passthrough.cfg b/libvirt/tests/cfg/gpu/multiple_devices_passthrough.cfg index b10c367e6bb..60962ba72e4 100644 --- a/libvirt/tests/cfg/gpu/multiple_devices_passthrough.cfg +++ b/libvirt/tests/cfg/gpu/multiple_devices_passthrough.cfg @@ -5,7 +5,7 @@ vcpu = 32 numa_attrs = "'cpu': {'numa_cell':[{'id':'0','cpus':'0-31','memory':'32','unit':'GiB'},{'id':'1','memory':'0'},{'id':'2','memory':'0'},{'id':'3','memory':'0'},{'id':'4','memory':'0'},{'id':'5','memory':'0'},{'id':'6','memory':'0'},{'id':'7','memory':'0'},{'id':'8','memory':'0'}]}" memory_backing = {'hugepages': {'pages': [{'size': '512', 'unit': 'MiB'}]},'source_type':'memfd','access_mode':'shared','allocation':{'mode':'immediate'}} - kernel_hp_file = '/sys/devices/system/node/node%s/hugepages/hugepages-%skB/nr_hugepages' + kernel_hp_file = '/sys/kernel/mm/hugepages/hugepages-%skB/nr_hugepages' acpi_nodeset = {'nodeset':'1-8'} test_devices = '["3D", "Network Connection", "Ethernet Controller Virtual Function", "Mellanox Technologies"]' pci_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"