Add case for mutiple device passthrough

libvirt/tests/cfg/gpu/cmdqv_various_hugepages_membackends.cfg

@@ -0,0 +1,59 @@
+- gpu.cmdqv_various_hugepages_membackends:
+    type = multiple_devices_passthrough
+    start_vm = "no"
+    mem_value = 32
+    vcpu = 32 
+    numa_attrs = "'cpu': {'numa_cell':[{'id':'0','cpus':'0-31','memory':'32','unit':'GiB'},{'id':'1','memory':'0'},{'id':'2','memory':'0'},{'id':'3','memory':'0'},{'id':'4','memory':'0'},{'id':'5','memory':'0'},{'id':'6','memory':'0'},{'id':'7','memory':'0'},{'id':'8','memory':'0'}]}"    
+    kernel_hp_file = '/sys/kernel/mm/hugepages/hugepages-%skB/nr_hugepages'
+    acpi_nodeset = {'nodeset':'1-8'}
+    test_devices = '["3D", "Network Connection", "Ethernet Controller Virtual Function", "Mellanox Technologies"]'
+    pci_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"
+    cmdqv_on_num = 2
+    cmdqv_dict = {'cmdqv':'on'}
+    cmdqv_dict_2 = {'cmdqv':'on'}
+    iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on','ats':'on','ril':'off','ssidSize':'20','oas':'48'}}
+    # For another VFIO device
+    iommu_dict_2 = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on','ats':'on','ril':'off','ssidSize':'20','oas':'48'}}
+    # gpu smmuv3 goes first
+    smmuv3_desc_defined = no
+    nic_driver = "'driver':{'driver_attr':{'iommufd':'yes'}}"
+    only aarch64
+    variants:
+        - 512M:
+            hugepage_size = 512
+        - 2M:
+            hugepage_size = 2
+        - 16384M:
+            hugepage_size = 16384
+    variants source_type:
+        - memfd:
+            no 512M
+        - file:
+            no 16384M
+    variants:
+        - pxb_pcie:
+            pxb_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':0x00,'slot':'%s','function':'0x0'}}"
+            pxb_ctrl_attrs = {'type':'pci','model': 'pcie-expander-bus','index': '%s',${pxb_addr}}
+            pcie_root_port_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"
+            pcie_root_port_attrs = {'type':'pci','model':'pcie-root-port','index':'%s',${pcie_root_port_addr}}
+            pxb_ctrl_attrs_2 = {'type':'pci','model': 'pcie-expander-bus','index': '%s',${pxb_addr}}
+            pcie_root_port_attrs_2 = {'type':'pci','model':'pcie-root-port','index':'%s',${pcie_root_port_addr}}
+    variants dev_type:
+        - nic_interface:
+            variants dev_source:
+                - vf:
+                    only 2M 16384M
+                    iface_dict = {'managed':'yes','type_name':'hostdev','hostdev_address':{'type_name':'pci','attrs':vf_pci_addr},${nic_driver},'alias':{'name':'ua-89cbe690-6c6c-4f2f-adac-5826fe52ea74'},'mac_address':mac_addr,${pci_addr}}
+                    check_ping_time = "yes"
+        - nic_hostdev:
+            variants dev_source:
+                - vf:
+                    no 16384M
+                    hostdev_dict = {${nic_driver},'alias':{'name':'ua-1bcbabff-f022-4d4f-ae8c-13f2d3a07906'},'mode':'subsystem','type':'pci','source':{'untyped_address':vf_pci_addr},'managed':'yes',${pci_addr}}
+                - pf:
+                    no 512M
+                    allow_unlink_pf = "yes"
+                    hostdev_dict = {${nic_driver},'mode': 'subsystem','type':'pci','source':{'untyped_address':pf_pci_addr},'managed':'yes',${pci_addr}}
+    vm_attrs = {'memory_unit':'GiB','memory':${mem_value},'vcpu':${vcpu},${numa_attrs}}
+    gpu_hostdev_dict = {'driver':{'driver_attr':{'iommufd':'yes'}},'acpi':${acpi_nodeset},'mode':'subsystem','type':'pci','source':{'untyped_address':gpu_pci_addr},'managed':'yes',${pci_addr}}
+    memory_backing = {'hugepages': {'pages': [{'size':'${hugepage_size}','unit':'MiB'}]},'source_type':'${source_type}','access_mode':'shared','allocation':{'mode':'immediate'}}

libvirt/tests/cfg/gpu/multiple_devices_passthrough.cfg

@@ -0,0 +1,80 @@
+- gpu.multiple_devices_passthrough:
+    type = multiple_devices_passthrough
+    start_vm = "no"
+    mem_value = 32
+    vcpu = 32 
+    numa_attrs = "'cpu': {'numa_cell':[{'id':'0','cpus':'0-31','memory':'32','unit':'GiB'},{'id':'1','memory':'0'},{'id':'2','memory':'0'},{'id':'3','memory':'0'},{'id':'4','memory':'0'},{'id':'5','memory':'0'},{'id':'6','memory':'0'},{'id':'7','memory':'0'},{'id':'8','memory':'0'}]}"
+    memory_backing = {'hugepages': {'pages': [{'size': '512', 'unit': 'MiB'}]},'source_type':'memfd','access_mode':'shared','allocation':{'mode':'immediate'}}
+    kernel_hp_file = '/sys/kernel/mm/hugepages/hugepages-%skB/nr_hugepages'
+    acpi_nodeset = {'nodeset':'1-8'}
+    test_devices = '["3D", "Network Connection", "Ethernet Controller Virtual Function", "Mellanox Technologies"]'
+    pci_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"
+    only aarch64
+    variants:
+        - both_cmdqv_on:
+            cmdqv_on_num = 2
+            cmdqv_dict = {'cmdqv':'on'}
+            cmdqv_dict_2 = {'cmdqv':'on'}            
+        - both_cmdqv_off:
+            cmdqv_on_num = 0
+            cmdqv_dict = {'cmdqv':'off'}
+            cmdqv_dict_2 = {'cmdqv':'off'}
+            memory_backing = {}
+        - gpu_cmdqv_off_nic_cmdqv_on:
+            cmdqv_on_num = 1
+            cmdqv_dict = {'cmdqv':'off'}
+            cmdqv_dict_2 = {'cmdqv':'on'}
+        - gpu_cmdqv_on_nic_cmdqv_off:
+            cmdqv_on_num = 1
+            cmdqv_dict = {'cmdqv':'on'}
+            cmdqv_dict_2 = {'cmdqv':'off'}
+        - gpu_cmdqv_on:
+            cmdqv_on_num = 1
+            cmdqv_dict = {'cmdqv':'on'}
+        - gpu_cmdqv_off:
+            cmdqv_on_num = 0
+            cmdqv_dict = {'cmdqv':'off'}
+    variants:
+        - only_gpu_smmuv3:
+            only gpu_cmdqv_on gpu_cmdqv_off
+            iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on','ats':'on','ril':'off','ssidSize':'20','oas':'48'}}            
+        - both_smmuv3:
+            no gpu_cmdqv_on gpu_cmdqv_off
+            # For GPU as default
+            iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on','ats':'on','ril':'off','ssidSize':'20','oas':'48'}}
+            # For another VFIO device
+            iommu_dict_2 = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on','ats':'on','ril':'off','ssidSize':'20','oas':'48'}}
+            variants:
+                - nic_smmuv3_first:
+                    smmuv3_desc_defined = yes
+                - gpu_smmuv3_first:
+                    smmuv3_desc_defined = no
+    variants:
+        - pxb_pcie:
+            pxb_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':0x00,'slot':'%s','function':'0x0'}}"
+            pxb_ctrl_attrs = {'type':'pci','model': 'pcie-expander-bus','index': '%s',${pxb_addr}}
+            pcie_root_port_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"
+            pcie_root_port_attrs = {'type':'pci','model':'pcie-root-port','index':'%s',${pcie_root_port_addr}}
+            pxb_ctrl_attrs_2 = {'type':'pci','model': 'pcie-expander-bus','index': '%s',${pxb_addr}}
+            pcie_root_port_attrs_2 = {'type':'pci','model':'pcie-root-port','index':'%s',${pcie_root_port_addr}}
+    variants:
+        - nic_iommufd_yes:
+            nic_driver = "'driver':{'driver_attr':{'iommufd':'yes'}}"
+        - nic_iommufd_no:
+            only only_gpu_smmuv3
+            nic_driver = "'driver':{'driver_attr':{'iommufd':'no'}}"
+    variants dev_type:
+        - nic_interface:
+            variants dev_source:
+                - vf:
+                    iface_dict = {'managed':'yes','type_name':'hostdev','hostdev_address':{'type_name':'pci','attrs':vf_pci_addr},${nic_driver},'alias':{'name':'ua-89cbe690-6c6c-4f2f-adac-5826fe52ea74'},'mac_address':mac_addr,${pci_addr}}
+                    check_ping_time = "yes"
+        - nic_hostdev:
+            variants dev_source:
+                - vf:
+                    hostdev_dict = {${nic_driver},'alias':{'name':'ua-1bcbabff-f022-4d4f-ae8c-13f2d3a07906'},'mode':'subsystem','type':'pci','source':{'untyped_address':vf_pci_addr},'managed':'yes',${pci_addr}}
+                - pf:
+                    allow_unlink_pf = "yes"
+                    hostdev_dict = {${nic_driver},'mode': 'subsystem','type':'pci','source':{'untyped_address':pf_pci_addr},'managed':'yes',${pci_addr}}
+    vm_attrs = {'memory_unit':'GiB','memory':${mem_value},'vcpu':${vcpu},${numa_attrs}}
+    gpu_hostdev_dict = {'driver':{'driver_attr':{'iommufd':'yes'}},'acpi':${acpi_nodeset},'mode':'subsystem','type':'pci','source':{'untyped_address':gpu_pci_addr},'managed':'yes',${pci_addr}}

libvirt/tests/src/gpu/multiple_devices_passthrough.py

@@ -0,0 +1,156 @@
+#
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+#
+#   Copyright Red Hat
+#
+#   SPDX-License-Identifier: GPL-2.0
+#
+#   Author: dzheng <dzheng@redhat.com>
+#
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+from virttest import utils_misc
+from virttest import virsh
+
+from virttest.libvirt_xml import vm_xml
+from virttest.utils_libvirt import libvirt_vmxml
+from virttest.utils_libvirt import libvirt_vfio
+from virttest.utils_test import libvirt
+
+from provider.gpu import gpu_base
+from provider.gpu import check_points
+from provider.sriov import sriov_base
+from provider.sriov import check_points as sriov_check_points
+
+
+def run(test, params, env):
+    """
+    Start vm with GPU device and NIC device (VF/PF) passthrough
+
+    Steps:
+    1. Setup GPU hostdev device
+    2. Setup NIC hostdev device or interface (VF/PF)
+    3. Start the VM
+    4. Check devices using unified check_lspci() entry
+    5. Check NIC-specific features (mac address, network accessibility)
+    6. Destroy VM
+    7. Check environment recovery (driver, mac address)
+    """
+
+    def check_nic_specific():
+        """
+        Check NIC-specific features
+        """
+        test.log.info("TEST_STEP: Check NIC-specific features")
+        nic_iface_dict_updated = nic_iface_dict.copy()
+        if nic_dev_type == "network_interface":
+            nic_iface_dict_updated.update({'type_name': 'hostdev'})
+
+        sriov_check_points.comp_hostdev_xml(
+            vm, nic_device_type, nic_iface_dict_updated)
+        sriov_check_points.check_mac_addr(
+            vm_session, vm.name, nic_device_type, nic_iface_dict)
+
+        check_ping_time = params.get("check_ping_time")
+        # TODO:check the Linked status and decide if network connectivity is checked
+        #if check_ping_time == "yes":
+        #    sriov_check_points.check_vm_network_accessed(vm_session)
+        # test_pf = params.get("test_pf")
+        # if test_pf and sriov_vfio.is_linked(pf_name=test_pf):
+        #     migration_obj.migration_test.ping_vm(vm, params)
+        #     migration_obj.check_vm_cont_ping(False)
+        # else:
+        test.log.debug(f"Skip ping test due to no link to PF ")
+
+    def check_recovery():
+        """
+        Check environment recovery after VM destroy
+        """
+        test.log.info("TEST_STEP: Check GPU driver recovery")
+        if not utils_misc.wait_for(
+            lambda: libvirt_vfio.check_vfio_pci(
+                gpu_dev_pci, not gpu_managed_disabled, True, exp_driver="nvgrace_gpu_vfio_pci"), 10, 5):
+            test.fail("GPU driver recovery failed!")
+        if gpu_managed_disabled:
+            virsh.nodedev_reattach(
+                gpu_dev_name, debug=True, ignore_status=False)
+            libvirt_vfio.check_vfio_pci(
+                gpu_dev_pci, True, exp_driver="nvgrace_gpu_vfio_pci")
+
+        test.log.info("TEST_STEP: Check NIC driver and mac recovery")
+
+        if not utils_misc.wait_for(
+            lambda: libvirt_vfio.check_vfio_pci(
+                nic_dev_pci, not nic_managed_disabled, True), 10, 5):
+            test.fail("NIC driver recovery failed!")
+        if nic_managed_disabled:
+            virsh.nodedev_reattach(
+                nic_dev_name, debug=True, ignore_status=False)
+            libvirt_vfio.check_vfio_pci(nic_dev_pci, True)
+        sriov_check_points.check_mac_addr_recovery(
+            sriov_test_obj.pf_name, nic_device_type, nic_iface_dict)
+
+    vm_name = params.get("main_vm", "avocado-vt-vm1")
+    vm = env.get_vm(vm_name)
+    nic_dev_type = params.get("dev_type", "")
+    nic_dev_source = params.get("dev_source", "")
+
+    sriov_test_obj = sriov_base.SRIOVTest(vm, test, params)
+    if nic_dev_type == "nic_hostdev" and nic_dev_source.startswith("pf"):
+        nic_dev_name = sriov_test_obj.pf_dev_name
+        nic_dev_pci = sriov_test_obj.pf_pci
+    else:
+        nic_dev_name = sriov_test_obj.vf_dev_name
+        nic_dev_pci = sriov_test_obj.vf_pci
+
+    nic_iface_dict = sriov_test_obj.parse_iface_dict()
+    nic_managed_disabled = nic_iface_dict.get('managed') != "yes"
+    nic_device_type = "hostdev" if nic_dev_type == "nic_hostdev" else "interface"
+
+    gpu_test = gpu_base.GPUTest(vm, test, params, sriov_helper=sriov_test_obj)
+    gpu_dev_name = gpu_test.gpu_dev_name
+    gpu_dev_pci = gpu_test.gpu_pci
+    gpu_hostdev_dict = gpu_test.parse_hostdev_dict()
+    gpu_managed_disabled = gpu_hostdev_dict.get('managed') != "yes"
+    vm_session = None
+
+    try:
+        test.log.info("TEST_SETUP: Setup GPU and NIC devices")
+        gpu_test.setup_default(dev_name=gpu_dev_name, test_hopper_gpu="yes")
+
+        sriov_test_obj.setup_default(
+            dev_name=nic_dev_name,
+            managed_disabled=nic_managed_disabled,
+            cleanup_ifaces="no")
+
+        test.log.info("TEST_STEP: Start the VM")
+        vm.start()
+        vm_session = vm.wait_for_login(timeout=240)
+        test.log.debug(
+            f'VMXML of {vm_name}:\n{virsh.dumpxml(vm_name).stdout_text}')
+        check_points.check_lspci(
+            test,
+            vm_session,
+            eval(params.get("test_devices", "3D")),
+            dev_iommu=True if params.get("iommu_dict_2") else False,
+        )
+        check_points.check_nvidia_smi(test, vm_session)
+        cmdqv_on_num = int(params.get("cmdqv_on_num", "1"))
+        check_points.check_guest_cmdqv_dmesg(test, vm_session, expect_num=cmdqv_on_num)
+        libvirt_vfio.check_vfio_pci(gpu_dev_pci, exp_driver="nvgrace_gpu_vfio_pci")
+        test.log.info("Verify: GPU driver is nvgrace_gpu_vfio_pci - PASS")
+        if vm_session:
+            vm_session.close()
+        test.log.info("TEST_STEP: Destroy VM")
+        vm.destroy()
+        check_recovery()
+        #check_points.check_qemu_log(test, vm)
+
+    finally:
+        test.log.info("TEST_TEARDOWN: Cleanup test environment")
+        gpu_test.teardown_default(
+            managed_disabled=gpu_managed_disabled,
+            dev_name=gpu_dev_name)
+        sriov_test_obj.teardown_default(
+            managed_disabled=nic_managed_disabled,
+            dev_name=nic_dev_name)