From 5f97b031d08db66096d40ec5840587d8135569a6 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Srivastava Date: Thu, 17 Apr 2025 20:38:43 +0000 Subject: [PATCH] Add guest boot sanity tests with passthrough device support Add test and configurations to validate the following scenarios: 1. Launch guests in different guest interrupt modes: a. AVIC b. APIC c. x2APIC d. x2AVIC 2. Boot the above guests with vCPU counts: 512, 288, 256, 254, 128, 64 3. Boot guests with PCI passthrough devices. 4. Boot guests with emulated AMD IOMMU and emulated Intel IOMMU. Notes: 1. AVIC and APIC support fewer than 255 vCPUs. 2. x2APIC and x2AVIC modes require extended-apicid=on, or the presence of emulated AMD IOMMU / emulated Intel IOMMU, to support booting with >255 vCPUs. Signed-off-by: Dheeraj Kumar Srivastava --- qemu/tests/cfg/emulated_amd_iommu.cfg | 57 ++++++ .../cfg/iommu_accelerated_guest_mode.cfg | 65 +++++++ qemu/tests/cfg/iommu_guest_mode.cfg | 69 +++++++ qemu/tests/cfg/vfio_pci_passthrough.cfg | 19 ++ qemu/tests/qemu_pci_passthrough.py | 169 ++++++++++++++++++ 5 files changed, 379 insertions(+) create mode 100644 qemu/tests/cfg/emulated_amd_iommu.cfg create mode 100644 qemu/tests/cfg/iommu_accelerated_guest_mode.cfg create mode 100644 qemu/tests/cfg/iommu_guest_mode.cfg create mode 100644 qemu/tests/cfg/vfio_pci_passthrough.cfg create mode 100755 qemu/tests/qemu_pci_passthrough.py diff --git a/qemu/tests/cfg/emulated_amd_iommu.cfg b/qemu/tests/cfg/emulated_amd_iommu.cfg new file mode 100644 index 0000000000..bf4d618f47 --- /dev/null +++ b/qemu/tests/cfg/emulated_amd_iommu.cfg @@ -0,0 +1,57 @@ +- emulated_amd_iommu: + type = qemu_pci_passthrough + start_vm = no + only Linux + only x86_64 + only HostCpuVendor.amd + pcie_extra_root_port = 0 + vt_ulimit_nofile = 65536 + usb_devices = "" + auto_cpu_model = "no" + cpu_model = "host" + machine_type = "q35" + machine_type_extra_params = "kernel-irqchip=split" + extra_params = " -global kvm-pit.lost_tick_policy=discard" + vcpu_sockets = 1 + vcpu_dies = 1 + vcpu_threads = 1 + mem = 4G + mode = "x2apic" + variants: + - no_passthrough: + required_qemu = [8.2.90,) + # TODO Add passthrough variant + variants: + - vcpu_512: + smp_fixed = 512 + vcpu_maxcpus = 512 + vcpu_cores = 512 + - vcpu_288: + smp_fixed = 288 + vcpu_maxcpus = 288 + vcpu_cores = 288 + - vcpu_256: + smp_fixed = 256 + vcpu_maxcpus = 256 + vcpu_cores = 256 + - vcpu_254: + smp_fixed = 254 + vcpu_maxcpus = 254 + vcpu_cores = 254 + - vcpu_128: + smp_fixed = 128 + vcpu_maxcpus = 128 + vcpu_cores = 128 + - vcpu_64: + smp_fixed = 64 + vcpu_maxcpus = 64 + vcpu_cores = 64 + variants: + - x2apic_emul_amd_iommu: + cpu_model_flags += ",+x2apic" + extra_params += " -device amd-iommu,intremap=on,xtsup=on" + kvm_probe_module_parameters = "avic=0" + - x2avic_emul_amd_iommu: + cpu_model_flags += ",+x2apic" + extra_params += " -device amd-iommu,intremap=on,xtsup=on" + kvm_probe_module_parameters = "avic=1" diff --git a/qemu/tests/cfg/iommu_accelerated_guest_mode.cfg b/qemu/tests/cfg/iommu_accelerated_guest_mode.cfg new file mode 100644 index 0000000000..a3ef94de91 --- /dev/null +++ b/qemu/tests/cfg/iommu_accelerated_guest_mode.cfg @@ -0,0 +1,65 @@ +- iommu_accelerated_guest_mode: + type = qemu_pci_passthrough + start_vm = no + only Linux + only x86_64 + only HostCpuVendor.amd + pcie_extra_root_port = 0 + vt_ulimit_nofile = 65536 + usb_devices = "" + auto_cpu_model = "no" + cpu_model = "host" + machine_type = "q35" + machine_type_extra_params = "kernel-irqchip=split" + extra_params = " -global kvm-pit.lost_tick_policy=discard" + vcpu_sockets = 1 + vcpu_dies = 1 + vcpu_threads = 1 + mem = 4G + mode = "x2apic" + kvm_probe_module_parameters = "avic=1" + # Update pci_device with host PCI device/s for passthrough to the guest + # enabling IOMMU functionality validation. eg. pci_device = "0000:01:00.0" + # Note: Without a PCI device input, the guest will still boot, but the + # test won't exercise the host IOMMU. + #pci_device = "" + variants: + - vcpu_512: + smp_fixed = 512 + vcpu_maxcpus = 512 + vcpu_cores = 512 + - vcpu_288: + smp_fixed = 288 + vcpu_maxcpus = 288 + vcpu_cores = 288 + - vcpu_256: + smp_fixed = 256 + vcpu_maxcpus = 256 + vcpu_cores = 256 + - vcpu_254: + smp_fixed = 254 + vcpu_maxcpus = 254 + vcpu_cores = 254 + - vcpu_128: + smp_fixed = 128 + vcpu_maxcpus = 128 + vcpu_cores = 128 + - vcpu_64: + smp_fixed = 64 + vcpu_maxcpus = 64 + vcpu_cores = 64 + variants: + - avic: + mode = "apic" + cpu_model_flags += ",-x2apic" + no vcpu_512, vcpu_288, vcpu_256 + - x2avic_ext_apic_id: + cpu_model_flags += ",+x2apic" + cpu_model_flags += ",kvm-msi-ext-dest-id=on" + - x2avic_emul_intel_iommu: + intel_iommu = yes + iommu_eim = on + iommu_intremap = on + guest_iommu_option = pt + iommu_caching_mode = on + cpu_model_flags += ",+x2apic" diff --git a/qemu/tests/cfg/iommu_guest_mode.cfg b/qemu/tests/cfg/iommu_guest_mode.cfg new file mode 100644 index 0000000000..626f838034 --- /dev/null +++ b/qemu/tests/cfg/iommu_guest_mode.cfg @@ -0,0 +1,69 @@ +- iommu_guest_mode: + type = qemu_pci_passthrough + start_vm = no + only Linux + only x86_64 + only HostCpuVendor.amd + pcie_extra_root_port = 0 + vt_ulimit_nofile = 65536 + usb_devices = "" + auto_cpu_model = "no" + cpu_model = "host" + machine_type = "q35" + machine_type_extra_params = "kernel-irqchip=split" + extra_params = "" + vcpu_sockets = 1 + vcpu_dies = 1 + vcpu_threads = 1 + mem = 4G + mode = "x2apic" + kvm_probe_module_parameters = "avic=0" + # Update pci_device with host PCI device/s for passthrough to the guest + # enabling IOMMU functionality validation. eg. pci_device = "0000:01:00.0" + # Note: Without a PCI device input, the guest will still boot, but the + # test won't exercise the host IOMMU. + #pci_device = "" + variants: + - vcpu_1024: + smp_fixed = 1024 + vcpu_maxcpus = 1024 + vcpu_cores = 1024 + - vcpu_512: + smp_fixed = 512 + vcpu_maxcpus = 512 + vcpu_cores = 512 + - vcpu_288: + smp_fixed = 288 + vcpu_maxcpus = 288 + vcpu_cores = 288 + - vcpu_256: + smp_fixed = 256 + vcpu_maxcpus = 256 + vcpu_cores = 256 + - vcpu_254: + smp_fixed = 254 + vcpu_maxcpus = 254 + vcpu_cores = 254 + - vcpu_128: + smp_fixed = 128 + vcpu_maxcpus = 128 + vcpu_cores = 128 + - vcpu_64: + smp_fixed = 64 + vcpu_maxcpus = 64 + vcpu_cores = 64 + variants: + - apic: + mode = "apic" + cpu_model_flags += ",-x2apic" + no vcpu_1024, vcpu_512, vcpu_288, vcpu_256 + - x2apic_ext_apic_id: + cpu_model_flags += ",+x2apic" + cpu_model_flags += ",kvm-msi-ext-dest-id=on" + - x2apic_emul_intel_iommu: + intel_iommu = yes + iommu_eim = on + iommu_intremap = on + guest_iommu_option = pt + iommu_caching_mode = on + cpu_model_flags += ",+x2apic" diff --git a/qemu/tests/cfg/vfio_pci_passthrough.cfg b/qemu/tests/cfg/vfio_pci_passthrough.cfg new file mode 100644 index 0000000000..5451bd7ad6 --- /dev/null +++ b/qemu/tests/cfg/vfio_pci_passthrough.cfg @@ -0,0 +1,19 @@ +- vfio_pci_passthrough: + type = qemu_pci_passthrough + start_vm = no + only Linux + only x86_64 + only HostCpuVendor.amd + pcie_extra_root_port = 0 + vt_ulimit_nofile = 65536 + usb_devices = "" + auto_cpu_model = "no" + cpu_model = "host" + machine_type = "q35" + extra_params = "" + mem = 4G + # Update pci_device with host PCI device/s for passthrough to the guest + # enabling IOMMU functionality validation. eg. pci_device = "0000:01:00.0" + # Note: Without a PCI device input, the guest will still boot, but the + # test won't exercise the host IOMMU. + #pci_device = "" diff --git a/qemu/tests/qemu_pci_passthrough.py b/qemu/tests/qemu_pci_passthrough.py new file mode 100755 index 0000000000..f46eb96307 --- /dev/null +++ b/qemu/tests/qemu_pci_passthrough.py @@ -0,0 +1,169 @@ +""" +Guest boot sanity test with passthrough device in different mode +- Apic, X2apic, Avic, X2avic +""" + +from avocado.utils import dmesg, linux_modules, pci, process +from virttest import env_process + + +def run(test, params, env): # pylint: disable=R0915 + """ + Guest boot sanity test with passthrough device + Steps: + 1. Launch a guest with a vfio pci passthrough device + 2. Verify if vfio pci passthrough of device to guest is successful + and guest boots in expected mode - apic, avic, x2apic and x2avic. + + :param test: QEMU test object. + :param params: Dictionary with test parameters: + - pci_device: Pci device to passthrough. Default: "" + - mode: Defines guest modes - [apic, x2apic]. Default: x2apic. + - kvm_probe_module_parameters: To enable/disable avic on host. + Possible values ["avic=1", "avic=0"] + - login_timeout: VM login timeout in seconds. Default: 240). + :param env: Dictionary with test environment. + :raises: cancel if + 1. At start of test, KVM module is not loaded. + 2. KVM, msr, vfio_pci is not built or loadable. + 3. pci_device != "" + a. vfio_pci is not built or loadable. + b. Interrupt remapping is not enabled on host. Required for + passthrough. + c. Pci device/s not found on host system. + 4. kvm_probe_module_parameters = "avic=1" + a. Host doesnot support expected mode - avic or x2avic. + b. Host doesnot have expected mode enabled for guest - avic + or x2avic. + fails if + 1. Pci device cannot be bind to vfio_pci module for passthrough. + (pci_device != "") + 2. Unable to login to guest within login timeout. + """ + kvm_probe_module_parameters = params.get("kvm_probe_module_parameters", "") + login_timeout = int(params.get("login_timeout", 240)) + pci_device = params.get("pci_device", "") + mode = params.get("mode", "x2apic") + driver_list = [] + session = None + vm = None + + try: + + def check_avic_support(): + """ + Check if system supports avic. + """ + cmd = "rdmsr -p 0 0xc00110dd --bitfield 13:13" + out = process.run(cmd, sudo=True, shell=True).stdout_text.strip() + if out == "0": + test.cancel("System doesnot support avic") + + def check_x2avic_support(): + """ + check if system supports x2avic. + """ + cmd = "rdmsr -p 0 0xc00110dd --bitfield 18:18" + out = process.run(cmd, sudo=True, shell=True).stdout_text.strip() + if out == "0": + test.cancel("System doesnot support x2avic") + + def verify_avic_enablement(mode): + """ + Check AVIC and x2AVIC status in dmesg logs diff. + + :param mode: Whether apic or x2apic. + """ + + # Check for the "avic enabled" in dmesg + if not dmesg.check_kernel_logs("AVIC enabled"): + test.cancel("AVIC not enabled after loading kvm_amd with avic=1") + + # Check for the "x2avic enabled" only if the test is "x2apic" + if (not dmesg.check_kernel_logs("x2AVIC enabled")) and (mode == "x2apic"): + test.cancel("x2AVIC not enabled after loading kvm_amd with avic=1.") + + def prepare_pci_passthrough(): + """ + Validate IOMMU, Interrupt Remapping, and vfio-pci module availability + """ + + # Check if interrupt remapping is enabled on system + if not dmesg.check_kernel_logs("AMD-Vi: Interrupt remapping enabled"): + test.cancel("IOMMU interrupt remaping is not enabled") + + # Check and load vfio-pci module + linux_modules.configure_module("vfio-pci", "CONFIG_VFIO_PCI") + + def guest_system_details(session): + """ + Collect guest system details + + :param session: active guest login session + """ + test.log.debug("Debug: %s", session.cmd_output("cat /etc/os-release")) + test.log.debug("Debug: %s", session.cmd_output("uname -a")) + test.log.debug("Debug: %s", session.cmd_output("ls /boot/")) + test.log.debug("Debug: %s", session.cmd_output("lspci -k")) + test.log.debug("Debug: %s", session.cmd_output("lscpu")) + test.log.debug("Debug: %s", session.cmd_output("lsblk")) + test.log.debug("Debug: %s", session.cmd_output("df -h")) + test.log.debug("Debug: %s", session.cmd_output("dmesg")) + + # Check system support for avic or x2avic + if kvm_probe_module_parameters == "avic=1": + linux_modules.configure_module("msr", "CONFIG_X86_MSR") + if mode == "apic": + check_avic_support() + if mode == "x2apic": + check_x2avic_support() + # Validate dmesg for avic and x2avic enablement + verify_avic_enablement(mode) + + # Passthrough device/s and validate if passthrough is successful + if pci_device != "": + # Perform pre-checks and prereq enablements before pci passthrough + prepare_pci_passthrough() + + # Prepare for pci passthrough + for i in range(len(pci_device.split(" "))): + # Check if device input is valid + if pci_device.split(" ")[i] not in pci.get_pci_addresses(): + test.cancel("Please provide valid pci device input.") + + driver_list.append(pci.get_driver(pci_device.split(" ")[i])) + pci.attach_driver(pci_device.split(" ")[i], "vfio-pci") + params["extra_params"] += ( + f" -device vfio-pci,host={pci_device.split(' ')[i]}" + ) + + params["start_vm"] = "yes" + vm = env.get_vm(params["main_vm"]) + try: + env_process.preprocess_vm(test, params, env, params.get("main_vm")) + vm.verify_alive() + except Exception as e: + test.fail(f"Failed to create VM: {str(e)}") + try: + session = vm.wait_for_login(timeout=login_timeout) + except Exception as e: + test.fail(f"Failed to login VM: {str(e)}") + vm.verify_kernel_crash() + + # Collect guest system details + guest_system_details(session) + except ValueError as e: + test.fail(f"{e}") + finally: + if session: + session.close() + if vm: + vm.destroy() + try: + if pci_device != "": + for i in range(len(pci_device.split(" "))): + if pci_device.split(" ")[i] not in pci.get_pci_addresses(): + break + pci.attach_driver(pci_device.split(" ")[i], driver_list[i]) + except ValueError as e: + test.fail(f"Failed to reset devices after test: Reason {e}")