Test case for nested virtualization

docs/networks.md

@@ -1,9 +1,10 @@
 # Host VM and Test VM Networks Overview
 
-| IP net             | Host Interface (MAC, Name)   | Host Bridge | Guest Interface (Name) | Comment                             |
-|--------------------|------------------------------|-------------|------------------------|-------------------------------------|
-| `192.168.1.0/24`   | `52:54:00:e5:b8:01`, `tap1`  | -           | `eth1337`              | Main network into test VM           |
-| `192.168.2.0/24`   | `52:54:00:e5:b8:02`, `tap2`  | -           | `eth1338`              | Hotplugged device (type `ethernet`) |
-| `192.168.3.0/24`   | `52:54:00:e5:b8:03`, `tap3`  | `br3`       | `eth1339`              | Hotplugged device (type `network`)  |
-| `192.168.4.0/24`   | `52:54:00:e5:b8:04`, `tap4`  | `br4`       | `eth1340`              | Hotplugged device (type `bridge`)   |
-| `192.168.100.0/24` | <dynamic>          , `eth1`  | -           | -                      | Network between host VMs            |
+| IP net             | Host Interface (MAC, Name)        | Host Bridge | Guest Interface (Name) | Comment                             |
+|--------------------|-----------------------------------|-------------|------------------------|-------------------------------------|
+| `192.168.1.0/24`   | `52:54:00:e5:b8:01`, `tap1`       | -           | `eth1337`              | Main network into test VM           |
+| `192.168.2.0/24`   | `52:54:00:e5:b8:02`, `tap2`       | -           | `eth1338`              | Hotplugged device (type `ethernet`) |
+| `192.168.3.0/24`   | `52:54:00:e5:b8:03`, `tap3`       | `br3`       | `eth1339`              | Hotplugged device (type `network`)  |
+| `192.168.4.0/24`   | `52:54:00:e5:b8:04`, `tap4`       | `br4`       | `eth1340`              | Hotplugged device (type `bridge`)   |
+| `192.168.30.0/24`  | `52:54:00:e5:b9:01`, `nestedtap0` | -           | -                      | Tap for nested VM tests             |
+| `192.168.100.0/24` | <dynamic>          , `eth1`       | -           | -                      | Network between host VMs            |

flake.nix

@@ -105,7 +105,8 @@
         });
 
         nixos-image' =
-          (pkgs.callPackage ./images/nixos-image.nix { inherit nixpkgs; }).config.system.build.isoImage;
+          (pkgs.callPackage ./images/nixos-image.nix { inherit nixpkgs chv-ovmf; })
+          .config.system.build.isoImage;
 
         nixos-image =
           pkgs.runCommand "nixos.iso"

images/nixos-image.nix

@@ -2,6 +2,7 @@
 
 {
   nixpkgs,
+  chv-ovmf,
 }:
 
 nixpkgs.lib.nixosSystem {
@@ -15,6 +16,12 @@ nixpkgs.lib.nixosSystem {
         lib,
         ...
       }:
+      let
+        cirros_qcow = pkgs.fetchurl {
+          url = "https://download.cirros-cloud.net/0.6.2/cirros-0.6.2-x86_64-disk.img";
+          hash = "sha256-B+RKc+VMlNmIAoUVQDwe12IFXgG4OnZ+3zwrOH94zgA=";
+        };
+      in
       {
         imports = [
           # The minimal ch installer module has given us the smallest size for
@@ -73,10 +80,15 @@ nixpkgs.lib.nixosSystem {
         };
         environment.stub-ld.enable = false;
         environment.systemPackages = with pkgs; [
+          cloud-hypervisor
           dmidecode
+          dnsmasq
+          iproute2
+          msr
           screen
+          sshpass
           stress
-          msr
+          tunctl
         ];
 
         isoImage.makeUsbBootable = true;
@@ -153,6 +165,20 @@ nixpkgs.lib.nixosSystem {
         systemd.services.resolvconf.enable = false;
         # We use a dummy key for the test VM to shortcut the boot time.
         systemd.services.sshd-keygen.enable = false;
+        systemd.tmpfiles.settings = {
+          "10-chv" = {
+            "/etc/CLOUDHV.fd" = {
+              "C+" = {
+                argument = "${chv-ovmf.fd}/FV/CLOUDHV.fd";
+              };
+            };
+            "/etc/cirros.img" = {
+              "L+" = {
+                argument = "${cirros_qcow}";
+              };
+            };
+          };
+        };
 
         # pw: root
         users.mutableUsers = false;

test_helper/test_helper/__init__.py

@@ -6,6 +6,7 @@
     VIRTIO_ENTROPY_SOURCE,
     VIRTIO_NETWORK_DEVICE,
     allocate_hugepages,
+    assert_nested_cirros_connectivity,
     hotplug,
     hotplug_fail,
     initialComputeVMSetup,
@@ -18,6 +19,7 @@
     parse_devices_from_dom_def,
     pci_devices_by_bdf,
     reset_system_image,
+    setup_nested_cirros,
     setupTestComputeVM,
     setupTestControllerVM,
     ssh,
@@ -50,6 +52,7 @@
     "FORBIDDEN_NON_ARCHITECTURAL_MSRS",
     "BENIGN_FORBIDDEN_MSRS",
     "allocate_hugepages",
+    "assert_nested_cirros_connectivity",
     "hotplug",
     "hotplug_fail",
     "initialComputeVMSetup",
@@ -62,6 +65,7 @@
     "parse_devices_from_dom_def",
     "pci_devices_by_bdf",
     "reset_system_image",
+    "setup_nested_cirros",
     "setupTestComputeVM",
     "setupTestControllerVM",
     "ssh",

test_helper/test_helper/test_helper.py

@@ -984,3 +984,107 @@ def vcpu_affinity_checks(testcase: TestCase, machine: Machine, context: str = ""
     testcase.assertEqual(
         int(taskset_vcpu2_controller, 16), 0xC, "vCPU taskset should match"
     )
+
+
+NESTED_CH_API_SOCKET = "/run/nested-chv.sock"
+NESTED_CIRROS_DISK = "/root/nested-cirros.img"
+NESTED_CIRROS_IP = "192.168.30.42"
+NESTED_CIRROS_HOST_IP = "192.168.30.1"
+NESTED_CIRROS_NETMASK = "255.255.255.0"
+NESTED_CIRROS_TAP = "nestedtap0"
+NESTED_CIRROS_MAC = "52:54:00:e5:b9:01"
+NESTED_CIRROS_DNSMASQ_LEASEFILE = "/run/nested-cirros-dnsmasq.leases"
+
+
+def setup_nested_cirros(machine: Machine) -> None:
+    """
+    Setup a nested cirros VM inside of a already active guest VM running on the
+    given 'machine'.
+    Thus, we have following chain of virtual machines: machine -> guest ->
+    cirros, where guest is our nixos image in the default scenario.
+    We use Cirros as the image to boot because it is very small in size and we
+    do not need to provide any additional disk space.
+    """
+    ssh(machine, f"cp /etc/cirros.img {NESTED_CIRROS_DISK}")
+    ssh(machine, f"chmod +w {NESTED_CIRROS_DISK}")
+
+    ssh(machine, f"tunctl -t {NESTED_CIRROS_TAP}")
+    ssh(machine, f"ip link show dev {NESTED_CIRROS_TAP}")
+    ssh(machine, f"ip addr add {NESTED_CIRROS_HOST_IP}/24 dev {NESTED_CIRROS_TAP}")
+    ssh(machine, f"ip link set dev {NESTED_CIRROS_TAP} up")
+    ssh(machine, f"ip link show dev {NESTED_CIRROS_TAP}")
+    ssh(
+        machine,
+        "dnsmasq "
+        f"--interface={NESTED_CIRROS_TAP} "
+        "--bind-dynamic "
+        "--port=0 "
+        "--dhcp-authoritative "
+        f"--dhcp-host={NESTED_CIRROS_MAC},{NESTED_CIRROS_IP} "
+        f"--dhcp-range={NESTED_CIRROS_IP},{NESTED_CIRROS_IP},{NESTED_CIRROS_NETMASK},12h "
+        f"--listen-address={NESTED_CIRROS_HOST_IP} "
+        f"--dhcp-leasefile={NESTED_CIRROS_DNSMASQ_LEASEFILE} "
+        ">/tmp/nested-cirros-dnsmasq.log 2>&1 &",
+    )
+
+    ssh(
+        machine,
+        "screen -dmS nested-ch cloud-hypervisor "
+        f"--api-socket {NESTED_CH_API_SOCKET} "
+        "--log-file /tmp/nested-cirros.log "
+        "--memory size=256M "
+        "--cpus boot=1 "
+        "--kernel /etc/CLOUDHV.fd "
+        f"--disk path={NESTED_CIRROS_DISK} "
+        f"--net tap={NESTED_CIRROS_TAP},mac={NESTED_CIRROS_MAC} "
+        "--console off "
+        "--serial file=/tmp/nested-serial.log",
+    )
+
+    def nested_guest_available():
+        try:
+            ssh(machine, f"test -S {NESTED_CH_API_SOCKET}")
+            ssh(
+                machine,
+                f"ch-remote --api-socket {NESTED_CH_API_SOCKET} ping",
+            )
+            return True
+        except RuntimeError:
+            pass
+        return False
+
+    try:
+        wait_until_succeed(
+            nested_guest_available,
+            retries=100,
+        )
+    except RuntimeError as e:
+        ch_log = ssh(machine, "cat /tmp/nested-cirros.log || true")
+        dnsmasq_log = ssh(machine, "cat /tmp/nested-cirros-dnsmasq.log || true")
+        raise RuntimeError(
+            "nested cloud-hypervisor did not start successfully\n"
+            f"cloud-hypervisor log:\n{ch_log}\n"
+            f"dnsmasq log:\n{dnsmasq_log}\n"
+        ) from e
+
+
+def assert_nested_cirros_connectivity(machine: Machine) -> None:
+    """
+    Test if the nested guest is alive and reachable over its DHCP-backed
+    network. As the Cirros image takes very long to reach a state where SSH
+    login is possible, we only use a ping to check the liveliness of the VM.
+    """
+    ssh(
+        machine,
+        f"ch-remote --api-socket {NESTED_CH_API_SOCKET} info >/dev/null",
+    )
+
+    def login():
+        try:
+            ssh(machine, f"ping -c 1 -W 1 {NESTED_CIRROS_IP}")
+            return True
+        except RuntimeError:
+            pass
+        return False
+
+    wait_until_succeed(login)

tests/common.nix

@@ -794,6 +794,13 @@ in
             })}";
           };
         };
+        "/etc/domain-chv-cpu-sapphire-rapids.xml" = {
+          "C+" = {
+            argument = "${pkgs.writeText "cirros-sapphire-rapids.xml" (virsh_ch_xml {
+              cpuModel = "sapphire-rapids";
+            })}";
+          };
+        };
         "/etc/new_interface.xml" = {
           "C+" = {
             argument = "${pkgs.writeText "new_interface.xml" (new_interface { })}";

tests/testsuite_cpu_profiles_host.py

@@ -8,15 +8,19 @@
 try:
     from ..test_helper.test_helper import (  # type: ignore
         LibvirtTestsBase,
+        assert_nested_cirros_connectivity,
         initialComputeVMSetup,
         initialControllerVMSetup,
+        setup_nested_cirros,
         wait_for_ssh,
     )
 except Exception:
     from test_helper import (
         LibvirtTestsBase,
+        assert_nested_cirros_connectivity,
         initialComputeVMSetup,
         initialControllerVMSetup,
+        setup_nested_cirros,
         wait_for_ssh,
     )
 
@@ -82,11 +86,25 @@ def test_ubuntu_with_cpu_profiles(self):
             retries=350,
         )
 
+    def test_nested_chv_guest(self):
+        """
+        Test that we are able to boot a nested CHV VM using a Cirros image when
+        a CPU profile is in use.
+        """
+
+        controllerVM.succeed("virsh define /etc/domain-chv-cpu-sapphire-rapids.xml")
+        controllerVM.succeed("virsh start testvm")
+
+        wait_for_ssh(controllerVM)
+        setup_nested_cirros(controllerVM)
+        assert_nested_cirros_connectivity(controllerVM)
+
 
 def suite():
     # Test cases involving live migration sorted in alphabetical order.
     testcases = [
         LibvirtTests.test_cirros_with_cpu_profiles,
+        LibvirtTests.test_nested_chv_guest,
         LibvirtTests.test_ubuntu_with_cpu_profiles,
     ]
 

tests/testsuite_default.py

@@ -9,8 +9,9 @@
 # additional IDE configuration.
 try:
     from ..test_helper.test_helper import (  # type: ignore
-        assert_domain_domstate,
         LibvirtTestsBase,
+        assert_domain_domstate,
+        assert_nested_cirros_connectivity,
         hotplug,
         hotplug_fail,
         initialComputeVMSetup,
@@ -19,6 +20,7 @@
         number_of_network_devices,
         parse_devices_from_dom_def,
         pci_devices_by_bdf,
+        setup_nested_cirros,
         ssh,
         vcpu_affinity_checks,
         vm_unresponsive,
@@ -28,8 +30,9 @@
     )
 except Exception:
     from test_helper import (
-        assert_domain_domstate,
         LibvirtTestsBase,
+        assert_domain_domstate,
+        assert_nested_cirros_connectivity,
         hotplug,
         hotplug_fail,
         initialComputeVMSetup,
@@ -38,6 +41,7 @@
         number_of_network_devices,
         parse_devices_from_dom_def,
         pci_devices_by_bdf,
+        setup_nested_cirros,
         ssh,
         vcpu_affinity_checks,
         vm_unresponsive,
@@ -1360,6 +1364,18 @@ def test_raw_image_is_properly_attached(self):
 
         controllerVM.succeed(f"rm {image}")
 
+    def test_nested_chv_guest(self):
+        """
+        Test that we are able to boot a nested CHV VM using a Cirros image.
+        """
+
+        controllerVM.succeed("virsh define /etc/domain-chv.xml")
+        controllerVM.succeed("virsh start testvm")
+
+        wait_for_ssh(controllerVM)
+        setup_nested_cirros(controllerVM)
+        assert_nested_cirros_connectivity(controllerVM)
+
 
 def suite():
     # Test cases sorted in alphabetical order.
@@ -1383,6 +1399,7 @@ def suite():
         LibvirtTests.test_list_smbios_oem_strings,
         LibvirtTests.test_list_smbios_sysinfo,
         LibvirtTests.test_managedsave,
+        LibvirtTests.test_nested_chv_guest,
         LibvirtTests.test_network_hotplug_attach_detach_persistent,
         LibvirtTests.test_network_hotplug_attach_detach_transient,
         LibvirtTests.test_network_hotplug_persistent_transient_detach_vm_restart,