From bb5243bcadf3841dbfb55fd48a2fa15e34d480b6 Mon Sep 17 00:00:00 2001 From: Stefan Kober Date: Mon, 30 Mar 2026 11:43:50 +0200 Subject: [PATCH 1/7] test_helper: add nested guest helper In order to test nested virtualization in different contexts (live migration and with CPU profiles) we add some helper methods to easily spawn a nested CHV Cirros guest. Signed-off-by: Stefan Kober On-behalf-of: SAP stefan.kober@sap.com --- test_helper/test_helper/__init__.py | 4 + test_helper/test_helper/test_helper.py | 104 +++++++++++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/test_helper/test_helper/__init__.py b/test_helper/test_helper/__init__.py index 45ac00b8..5792ad35 100644 --- a/test_helper/test_helper/__init__.py +++ b/test_helper/test_helper/__init__.py @@ -6,6 +6,7 @@ VIRTIO_ENTROPY_SOURCE, VIRTIO_NETWORK_DEVICE, allocate_hugepages, + assert_nested_cirros_connectivity, hotplug, hotplug_fail, initialComputeVMSetup, @@ -18,6 +19,7 @@ parse_devices_from_dom_def, pci_devices_by_bdf, reset_system_image, + setup_nested_cirros, setupTestComputeVM, setupTestControllerVM, ssh, @@ -50,6 +52,7 @@ "FORBIDDEN_NON_ARCHITECTURAL_MSRS", "BENIGN_FORBIDDEN_MSRS", "allocate_hugepages", + "assert_nested_cirros_connectivity", "hotplug", "hotplug_fail", "initialComputeVMSetup", @@ -62,6 +65,7 @@ "parse_devices_from_dom_def", "pci_devices_by_bdf", "reset_system_image", + "setup_nested_cirros", "setupTestComputeVM", "setupTestControllerVM", "ssh", diff --git a/test_helper/test_helper/test_helper.py b/test_helper/test_helper/test_helper.py index a2b2c1dd..beea28ec 100644 --- a/test_helper/test_helper/test_helper.py +++ b/test_helper/test_helper/test_helper.py @@ -984,3 +984,107 @@ def vcpu_affinity_checks(testcase: TestCase, machine: Machine, context: str = "" testcase.assertEqual( int(taskset_vcpu2_controller, 16), 0xC, "vCPU taskset should match" ) + + +NESTED_CH_API_SOCKET = "/run/nested-chv.sock" +NESTED_CIRROS_DISK = "/root/nested-cirros.img" +NESTED_CIRROS_IP = "192.168.30.42" +NESTED_CIRROS_HOST_IP = "192.168.30.1" +NESTED_CIRROS_NETMASK = "255.255.255.0" +NESTED_CIRROS_TAP = "nestedtap0" +NESTED_CIRROS_MAC = "52:54:00:e5:b9:01" +NESTED_CIRROS_DNSMASQ_LEASEFILE = "/run/nested-cirros-dnsmasq.leases" + + +def setup_nested_cirros(machine: Machine) -> None: + """ + Setup a nested cirros VM inside of a already active guest VM running on the + given 'machine'. + Thus, we have following chain of virtual machines: machine -> guest -> + cirros, where guest is our nixos image in the default scenario. + We use Cirros as the image to boot because it is very small in size and we + do not need to provide any additional disk space. + """ + ssh(machine, f"cp /etc/cirros.img {NESTED_CIRROS_DISK}") + ssh(machine, f"chmod +w {NESTED_CIRROS_DISK}") + + ssh(machine, f"tunctl -t {NESTED_CIRROS_TAP}") + ssh(machine, f"ip link show dev {NESTED_CIRROS_TAP}") + ssh(machine, f"ip addr add {NESTED_CIRROS_HOST_IP}/24 dev {NESTED_CIRROS_TAP}") + ssh(machine, f"ip link set dev {NESTED_CIRROS_TAP} up") + ssh(machine, f"ip link show dev {NESTED_CIRROS_TAP}") + ssh( + machine, + "dnsmasq " + f"--interface={NESTED_CIRROS_TAP} " + "--bind-dynamic " + "--port=0 " + "--dhcp-authoritative " + f"--dhcp-host={NESTED_CIRROS_MAC},{NESTED_CIRROS_IP} " + f"--dhcp-range={NESTED_CIRROS_IP},{NESTED_CIRROS_IP},{NESTED_CIRROS_NETMASK},12h " + f"--listen-address={NESTED_CIRROS_HOST_IP} " + f"--dhcp-leasefile={NESTED_CIRROS_DNSMASQ_LEASEFILE} " + ">/tmp/nested-cirros-dnsmasq.log 2>&1 &", + ) + + ssh( + machine, + "screen -dmS nested-ch cloud-hypervisor " + f"--api-socket {NESTED_CH_API_SOCKET} " + "--log-file /tmp/nested-cirros.log " + "--memory size=256M " + "--cpus boot=1 " + "--kernel /etc/CLOUDHV.fd " + f"--disk path={NESTED_CIRROS_DISK} " + f"--net tap={NESTED_CIRROS_TAP},mac={NESTED_CIRROS_MAC} " + "--console off " + "--serial file=/tmp/nested-serial.log", + ) + + def nested_guest_available(): + try: + ssh(machine, f"test -S {NESTED_CH_API_SOCKET}") + ssh( + machine, + f"ch-remote --api-socket {NESTED_CH_API_SOCKET} ping", + ) + return True + except RuntimeError: + pass + return False + + try: + wait_until_succeed( + nested_guest_available, + retries=100, + ) + except RuntimeError as e: + ch_log = ssh(machine, "cat /tmp/nested-cirros.log || true") + dnsmasq_log = ssh(machine, "cat /tmp/nested-cirros-dnsmasq.log || true") + raise RuntimeError( + "nested cloud-hypervisor did not start successfully\n" + f"cloud-hypervisor log:\n{ch_log}\n" + f"dnsmasq log:\n{dnsmasq_log}\n" + ) from e + + +def assert_nested_cirros_connectivity(machine: Machine) -> None: + """ + Test if the nested guest is alive and reachable over its DHCP-backed + network. As the Cirros image takes very long to reach a state where SSH + login is possible, we only use a ping to check the liveliness of the VM. + """ + ssh( + machine, + f"ch-remote --api-socket {NESTED_CH_API_SOCKET} info >/dev/null", + ) + + def login(): + try: + ssh(machine, f"ping -c 1 -W 1 {NESTED_CIRROS_IP}") + return True + except RuntimeError: + pass + return False + + wait_until_succeed(login) From c9f1ab9ccd134e1dc2274a477c83f4400682c7fc Mon Sep 17 00:00:00 2001 From: Stefan Kober Date: Mon, 30 Mar 2026 11:45:31 +0200 Subject: [PATCH 2/7] nix: put firmware and cirros image into nixos image In order to boot nested guests we need a disk image and a firmware blob inside of our nixos image. Signed-off-by: Stefan Kober On-behalf-of: SAP stefan.kober@sap.com --- flake.nix | 3 ++- images/nixos-image.nix | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index c6ccef2d..3b771895 100644 --- a/flake.nix +++ b/flake.nix @@ -105,7 +105,8 @@ }); nixos-image' = - (pkgs.callPackage ./images/nixos-image.nix { inherit nixpkgs; }).config.system.build.isoImage; + (pkgs.callPackage ./images/nixos-image.nix { inherit nixpkgs chv-ovmf; }) + .config.system.build.isoImage; nixos-image = pkgs.runCommand "nixos.iso" diff --git a/images/nixos-image.nix b/images/nixos-image.nix index 0e8c41a6..3576d685 100644 --- a/images/nixos-image.nix +++ b/images/nixos-image.nix @@ -2,6 +2,7 @@ { nixpkgs, + chv-ovmf, }: nixpkgs.lib.nixosSystem { @@ -15,6 +16,12 @@ nixpkgs.lib.nixosSystem { lib, ... }: + let + cirros_qcow = pkgs.fetchurl { + url = "https://download.cirros-cloud.net/0.6.2/cirros-0.6.2-x86_64-disk.img"; + hash = "sha256-B+RKc+VMlNmIAoUVQDwe12IFXgG4OnZ+3zwrOH94zgA="; + }; + in { imports = [ # The minimal ch installer module has given us the smallest size for @@ -153,6 +160,20 @@ nixpkgs.lib.nixosSystem { systemd.services.resolvconf.enable = false; # We use a dummy key for the test VM to shortcut the boot time. systemd.services.sshd-keygen.enable = false; + systemd.tmpfiles.settings = { + "10-chv" = { + "/etc/CLOUDHV.fd" = { + "C+" = { + argument = "${chv-ovmf.fd}/FV/CLOUDHV.fd"; + }; + }; + "/etc/cirros.img" = { + "L+" = { + argument = "${cirros_qcow}"; + }; + }; + }; + }; # pw: root users.mutableUsers = false; From 6ae8e98078815d132d25af1254c534cd1faf5e21 Mon Sep 17 00:00:00 2001 From: Stefan Kober Date: Mon, 13 Apr 2026 15:28:48 +0200 Subject: [PATCH 3/7] nix: add packages for nesting in nixos image In order to setup and use networking for the nested guest (TUN, DHCP, SSH) we need to add some packages to the nixos image. Signed-off-by: Stefan Kober On-behalf-of: SAP stefan.kober@sap.com --- images/nixos-image.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/images/nixos-image.nix b/images/nixos-image.nix index 3576d685..d23015f4 100644 --- a/images/nixos-image.nix +++ b/images/nixos-image.nix @@ -80,10 +80,15 @@ nixpkgs.lib.nixosSystem { }; environment.stub-ld.enable = false; environment.systemPackages = with pkgs; [ + cloud-hypervisor dmidecode + dnsmasq + iproute2 + msr screen + sshpass stress - msr + tunctl ]; isoImage.makeUsbBootable = true; From 639349d1e48119c098169a15576c077b1f554e6a Mon Sep 17 00:00:00 2001 From: Stefan Kober Date: Mon, 30 Mar 2026 11:46:55 +0200 Subject: [PATCH 4/7] tests: add nested guest boot test Utilize the helper functions for nested guests to create a simple live boot test that spawns a nested guest, boots and checks for the nested guest being alive via network. Signed-off-by: Stefan Kober On-behalf-of: SAP stefan.kober@sap.com --- tests/testsuite_default.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/testsuite_default.py b/tests/testsuite_default.py index 7bcc81bf..70e0170f 100644 --- a/tests/testsuite_default.py +++ b/tests/testsuite_default.py @@ -9,8 +9,9 @@ # additional IDE configuration. try: from ..test_helper.test_helper import ( # type: ignore - assert_domain_domstate, LibvirtTestsBase, + assert_domain_domstate, + assert_nested_cirros_connectivity, hotplug, hotplug_fail, initialComputeVMSetup, @@ -19,6 +20,7 @@ number_of_network_devices, parse_devices_from_dom_def, pci_devices_by_bdf, + setup_nested_cirros, ssh, vcpu_affinity_checks, vm_unresponsive, @@ -28,8 +30,9 @@ ) except Exception: from test_helper import ( - assert_domain_domstate, LibvirtTestsBase, + assert_domain_domstate, + assert_nested_cirros_connectivity, hotplug, hotplug_fail, initialComputeVMSetup, @@ -38,6 +41,7 @@ number_of_network_devices, parse_devices_from_dom_def, pci_devices_by_bdf, + setup_nested_cirros, ssh, vcpu_affinity_checks, vm_unresponsive, @@ -1360,6 +1364,18 @@ def test_raw_image_is_properly_attached(self): controllerVM.succeed(f"rm {image}") + def test_nested_chv_guest(self): + """ + Test that we are able to boot a nested CHV VM using a Cirros image. + """ + + controllerVM.succeed("virsh define /etc/domain-chv.xml") + controllerVM.succeed("virsh start testvm") + + wait_for_ssh(controllerVM) + setup_nested_cirros(controllerVM) + assert_nested_cirros_connectivity(controllerVM) + def suite(): # Test cases sorted in alphabetical order. @@ -1383,6 +1399,7 @@ def suite(): LibvirtTests.test_list_smbios_oem_strings, LibvirtTests.test_list_smbios_sysinfo, LibvirtTests.test_managedsave, + LibvirtTests.test_nested_chv_guest, LibvirtTests.test_network_hotplug_attach_detach_persistent, LibvirtTests.test_network_hotplug_attach_detach_transient, LibvirtTests.test_network_hotplug_persistent_transient_detach_vm_restart, From f1568cf17c423a1b7766b28013b16b77f545b79f Mon Sep 17 00:00:00 2001 From: Stefan Kober Date: Mon, 30 Mar 2026 12:42:28 +0200 Subject: [PATCH 5/7] common.nix: add sapphire-rapids profile config Signed-off-by: Stefan Kober On-behalf-of: SAP stefan.kober@sap.com --- tests/common.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/common.nix b/tests/common.nix index 45f8fbd7..3bb9af8c 100644 --- a/tests/common.nix +++ b/tests/common.nix @@ -794,6 +794,13 @@ in })}"; }; }; + "/etc/domain-chv-cpu-sapphire-rapids.xml" = { + "C+" = { + argument = "${pkgs.writeText "cirros-sapphire-rapids.xml" (virsh_ch_xml { + cpuModel = "sapphire-rapids"; + })}"; + }; + }; "/etc/new_interface.xml" = { "C+" = { argument = "${pkgs.writeText "new_interface.xml" (new_interface { })}"; From 12a0cc6850fdfdf2a10c3bbfaef2c95f4662d145 Mon Sep 17 00:00:00 2001 From: Stefan Kober Date: Mon, 30 Mar 2026 12:42:52 +0200 Subject: [PATCH 6/7] tests: add nesting with CPU profile test We test that running a nested guest works when having some active CPU profile. Signed-off-by: Stefan Kober On-behalf-of: SAP stefan.kober@sap.com --- tests/testsuite_cpu_profiles_host.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/testsuite_cpu_profiles_host.py b/tests/testsuite_cpu_profiles_host.py index 01f87cf1..dfbc29ff 100644 --- a/tests/testsuite_cpu_profiles_host.py +++ b/tests/testsuite_cpu_profiles_host.py @@ -8,15 +8,19 @@ try: from ..test_helper.test_helper import ( # type: ignore LibvirtTestsBase, + assert_nested_cirros_connectivity, initialComputeVMSetup, initialControllerVMSetup, + setup_nested_cirros, wait_for_ssh, ) except Exception: from test_helper import ( LibvirtTestsBase, + assert_nested_cirros_connectivity, initialComputeVMSetup, initialControllerVMSetup, + setup_nested_cirros, wait_for_ssh, ) @@ -82,11 +86,25 @@ def test_ubuntu_with_cpu_profiles(self): retries=350, ) + def test_nested_chv_guest(self): + """ + Test that we are able to boot a nested CHV VM using a Cirros image when + a CPU profile is in use. + """ + + controllerVM.succeed("virsh define /etc/domain-chv-cpu-sapphire-rapids.xml") + controllerVM.succeed("virsh start testvm") + + wait_for_ssh(controllerVM) + setup_nested_cirros(controllerVM) + assert_nested_cirros_connectivity(controllerVM) + def suite(): # Test cases involving live migration sorted in alphabetical order. testcases = [ LibvirtTests.test_cirros_with_cpu_profiles, + LibvirtTests.test_nested_chv_guest, LibvirtTests.test_ubuntu_with_cpu_profiles, ] From c6880754b1dffffda9201ea2ad5fae1012e58418 Mon Sep 17 00:00:00 2001 From: Stefan Kober Date: Tue, 14 Apr 2026 09:51:16 +0200 Subject: [PATCH 7/7] doc: add nested tap to networks.md Signed-off-by: Stefan Kober On-behalf-of: SAP stefan.kober@sap.com --- docs/networks.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/networks.md b/docs/networks.md index 187bc788..f766f446 100644 --- a/docs/networks.md +++ b/docs/networks.md @@ -1,9 +1,10 @@ # Host VM and Test VM Networks Overview -| IP net | Host Interface (MAC, Name) | Host Bridge | Guest Interface (Name) | Comment | -|--------------------|------------------------------|-------------|------------------------|-------------------------------------| -| `192.168.1.0/24` | `52:54:00:e5:b8:01`, `tap1` | - | `eth1337` | Main network into test VM | -| `192.168.2.0/24` | `52:54:00:e5:b8:02`, `tap2` | - | `eth1338` | Hotplugged device (type `ethernet`) | -| `192.168.3.0/24` | `52:54:00:e5:b8:03`, `tap3` | `br3` | `eth1339` | Hotplugged device (type `network`) | -| `192.168.4.0/24` | `52:54:00:e5:b8:04`, `tap4` | `br4` | `eth1340` | Hotplugged device (type `bridge`) | -| `192.168.100.0/24` | , `eth1` | - | - | Network between host VMs | +| IP net | Host Interface (MAC, Name) | Host Bridge | Guest Interface (Name) | Comment | +|--------------------|-----------------------------------|-------------|------------------------|-------------------------------------| +| `192.168.1.0/24` | `52:54:00:e5:b8:01`, `tap1` | - | `eth1337` | Main network into test VM | +| `192.168.2.0/24` | `52:54:00:e5:b8:02`, `tap2` | - | `eth1338` | Hotplugged device (type `ethernet`) | +| `192.168.3.0/24` | `52:54:00:e5:b8:03`, `tap3` | `br3` | `eth1339` | Hotplugged device (type `network`) | +| `192.168.4.0/24` | `52:54:00:e5:b8:04`, `tap4` | `br4` | `eth1340` | Hotplugged device (type `bridge`) | +| `192.168.30.0/24` | `52:54:00:e5:b9:01`, `nestedtap0` | - | - | Tap for nested VM tests | +| `192.168.100.0/24` | , `eth1` | - | - | Network between host VMs |