diff --git a/docs/networks.md b/docs/networks.md index 187bc788..f766f446 100644 --- a/docs/networks.md +++ b/docs/networks.md @@ -1,9 +1,10 @@ # Host VM and Test VM Networks Overview -| IP net | Host Interface (MAC, Name) | Host Bridge | Guest Interface (Name) | Comment | -|--------------------|------------------------------|-------------|------------------------|-------------------------------------| -| `192.168.1.0/24` | `52:54:00:e5:b8:01`, `tap1` | - | `eth1337` | Main network into test VM | -| `192.168.2.0/24` | `52:54:00:e5:b8:02`, `tap2` | - | `eth1338` | Hotplugged device (type `ethernet`) | -| `192.168.3.0/24` | `52:54:00:e5:b8:03`, `tap3` | `br3` | `eth1339` | Hotplugged device (type `network`) | -| `192.168.4.0/24` | `52:54:00:e5:b8:04`, `tap4` | `br4` | `eth1340` | Hotplugged device (type `bridge`) | -| `192.168.100.0/24` | , `eth1` | - | - | Network between host VMs | +| IP net | Host Interface (MAC, Name) | Host Bridge | Guest Interface (Name) | Comment | +|--------------------|-----------------------------------|-------------|------------------------|-------------------------------------| +| `192.168.1.0/24` | `52:54:00:e5:b8:01`, `tap1` | - | `eth1337` | Main network into test VM | +| `192.168.2.0/24` | `52:54:00:e5:b8:02`, `tap2` | - | `eth1338` | Hotplugged device (type `ethernet`) | +| `192.168.3.0/24` | `52:54:00:e5:b8:03`, `tap3` | `br3` | `eth1339` | Hotplugged device (type `network`) | +| `192.168.4.0/24` | `52:54:00:e5:b8:04`, `tap4` | `br4` | `eth1340` | Hotplugged device (type `bridge`) | +| `192.168.30.0/24` | `52:54:00:e5:b9:01`, `nestedtap0` | - | - | Tap for nested VM tests | +| `192.168.100.0/24` | , `eth1` | - | - | Network between host VMs | diff --git a/flake.nix b/flake.nix index c6ccef2d..3b771895 100644 --- a/flake.nix +++ b/flake.nix @@ -105,7 +105,8 @@ }); nixos-image' = - (pkgs.callPackage ./images/nixos-image.nix { inherit nixpkgs; }).config.system.build.isoImage; + (pkgs.callPackage ./images/nixos-image.nix { inherit nixpkgs chv-ovmf; }) + .config.system.build.isoImage; nixos-image = pkgs.runCommand "nixos.iso" diff --git a/images/nixos-image.nix b/images/nixos-image.nix index 0e8c41a6..d23015f4 100644 --- a/images/nixos-image.nix +++ b/images/nixos-image.nix @@ -2,6 +2,7 @@ { nixpkgs, + chv-ovmf, }: nixpkgs.lib.nixosSystem { @@ -15,6 +16,12 @@ nixpkgs.lib.nixosSystem { lib, ... }: + let + cirros_qcow = pkgs.fetchurl { + url = "https://download.cirros-cloud.net/0.6.2/cirros-0.6.2-x86_64-disk.img"; + hash = "sha256-B+RKc+VMlNmIAoUVQDwe12IFXgG4OnZ+3zwrOH94zgA="; + }; + in { imports = [ # The minimal ch installer module has given us the smallest size for @@ -73,10 +80,15 @@ nixpkgs.lib.nixosSystem { }; environment.stub-ld.enable = false; environment.systemPackages = with pkgs; [ + cloud-hypervisor dmidecode + dnsmasq + iproute2 + msr screen + sshpass stress - msr + tunctl ]; isoImage.makeUsbBootable = true; @@ -153,6 +165,20 @@ nixpkgs.lib.nixosSystem { systemd.services.resolvconf.enable = false; # We use a dummy key for the test VM to shortcut the boot time. systemd.services.sshd-keygen.enable = false; + systemd.tmpfiles.settings = { + "10-chv" = { + "/etc/CLOUDHV.fd" = { + "C+" = { + argument = "${chv-ovmf.fd}/FV/CLOUDHV.fd"; + }; + }; + "/etc/cirros.img" = { + "L+" = { + argument = "${cirros_qcow}"; + }; + }; + }; + }; # pw: root users.mutableUsers = false; diff --git a/test_helper/test_helper/__init__.py b/test_helper/test_helper/__init__.py index 45ac00b8..5792ad35 100644 --- a/test_helper/test_helper/__init__.py +++ b/test_helper/test_helper/__init__.py @@ -6,6 +6,7 @@ VIRTIO_ENTROPY_SOURCE, VIRTIO_NETWORK_DEVICE, allocate_hugepages, + assert_nested_cirros_connectivity, hotplug, hotplug_fail, initialComputeVMSetup, @@ -18,6 +19,7 @@ parse_devices_from_dom_def, pci_devices_by_bdf, reset_system_image, + setup_nested_cirros, setupTestComputeVM, setupTestControllerVM, ssh, @@ -50,6 +52,7 @@ "FORBIDDEN_NON_ARCHITECTURAL_MSRS", "BENIGN_FORBIDDEN_MSRS", "allocate_hugepages", + "assert_nested_cirros_connectivity", "hotplug", "hotplug_fail", "initialComputeVMSetup", @@ -62,6 +65,7 @@ "parse_devices_from_dom_def", "pci_devices_by_bdf", "reset_system_image", + "setup_nested_cirros", "setupTestComputeVM", "setupTestControllerVM", "ssh", diff --git a/test_helper/test_helper/test_helper.py b/test_helper/test_helper/test_helper.py index a2b2c1dd..beea28ec 100644 --- a/test_helper/test_helper/test_helper.py +++ b/test_helper/test_helper/test_helper.py @@ -984,3 +984,107 @@ def vcpu_affinity_checks(testcase: TestCase, machine: Machine, context: str = "" testcase.assertEqual( int(taskset_vcpu2_controller, 16), 0xC, "vCPU taskset should match" ) + + +NESTED_CH_API_SOCKET = "/run/nested-chv.sock" +NESTED_CIRROS_DISK = "/root/nested-cirros.img" +NESTED_CIRROS_IP = "192.168.30.42" +NESTED_CIRROS_HOST_IP = "192.168.30.1" +NESTED_CIRROS_NETMASK = "255.255.255.0" +NESTED_CIRROS_TAP = "nestedtap0" +NESTED_CIRROS_MAC = "52:54:00:e5:b9:01" +NESTED_CIRROS_DNSMASQ_LEASEFILE = "/run/nested-cirros-dnsmasq.leases" + + +def setup_nested_cirros(machine: Machine) -> None: + """ + Setup a nested cirros VM inside of a already active guest VM running on the + given 'machine'. + Thus, we have following chain of virtual machines: machine -> guest -> + cirros, where guest is our nixos image in the default scenario. + We use Cirros as the image to boot because it is very small in size and we + do not need to provide any additional disk space. + """ + ssh(machine, f"cp /etc/cirros.img {NESTED_CIRROS_DISK}") + ssh(machine, f"chmod +w {NESTED_CIRROS_DISK}") + + ssh(machine, f"tunctl -t {NESTED_CIRROS_TAP}") + ssh(machine, f"ip link show dev {NESTED_CIRROS_TAP}") + ssh(machine, f"ip addr add {NESTED_CIRROS_HOST_IP}/24 dev {NESTED_CIRROS_TAP}") + ssh(machine, f"ip link set dev {NESTED_CIRROS_TAP} up") + ssh(machine, f"ip link show dev {NESTED_CIRROS_TAP}") + ssh( + machine, + "dnsmasq " + f"--interface={NESTED_CIRROS_TAP} " + "--bind-dynamic " + "--port=0 " + "--dhcp-authoritative " + f"--dhcp-host={NESTED_CIRROS_MAC},{NESTED_CIRROS_IP} " + f"--dhcp-range={NESTED_CIRROS_IP},{NESTED_CIRROS_IP},{NESTED_CIRROS_NETMASK},12h " + f"--listen-address={NESTED_CIRROS_HOST_IP} " + f"--dhcp-leasefile={NESTED_CIRROS_DNSMASQ_LEASEFILE} " + ">/tmp/nested-cirros-dnsmasq.log 2>&1 &", + ) + + ssh( + machine, + "screen -dmS nested-ch cloud-hypervisor " + f"--api-socket {NESTED_CH_API_SOCKET} " + "--log-file /tmp/nested-cirros.log " + "--memory size=256M " + "--cpus boot=1 " + "--kernel /etc/CLOUDHV.fd " + f"--disk path={NESTED_CIRROS_DISK} " + f"--net tap={NESTED_CIRROS_TAP},mac={NESTED_CIRROS_MAC} " + "--console off " + "--serial file=/tmp/nested-serial.log", + ) + + def nested_guest_available(): + try: + ssh(machine, f"test -S {NESTED_CH_API_SOCKET}") + ssh( + machine, + f"ch-remote --api-socket {NESTED_CH_API_SOCKET} ping", + ) + return True + except RuntimeError: + pass + return False + + try: + wait_until_succeed( + nested_guest_available, + retries=100, + ) + except RuntimeError as e: + ch_log = ssh(machine, "cat /tmp/nested-cirros.log || true") + dnsmasq_log = ssh(machine, "cat /tmp/nested-cirros-dnsmasq.log || true") + raise RuntimeError( + "nested cloud-hypervisor did not start successfully\n" + f"cloud-hypervisor log:\n{ch_log}\n" + f"dnsmasq log:\n{dnsmasq_log}\n" + ) from e + + +def assert_nested_cirros_connectivity(machine: Machine) -> None: + """ + Test if the nested guest is alive and reachable over its DHCP-backed + network. As the Cirros image takes very long to reach a state where SSH + login is possible, we only use a ping to check the liveliness of the VM. + """ + ssh( + machine, + f"ch-remote --api-socket {NESTED_CH_API_SOCKET} info >/dev/null", + ) + + def login(): + try: + ssh(machine, f"ping -c 1 -W 1 {NESTED_CIRROS_IP}") + return True + except RuntimeError: + pass + return False + + wait_until_succeed(login) diff --git a/tests/common.nix b/tests/common.nix index 45f8fbd7..3bb9af8c 100644 --- a/tests/common.nix +++ b/tests/common.nix @@ -794,6 +794,13 @@ in })}"; }; }; + "/etc/domain-chv-cpu-sapphire-rapids.xml" = { + "C+" = { + argument = "${pkgs.writeText "cirros-sapphire-rapids.xml" (virsh_ch_xml { + cpuModel = "sapphire-rapids"; + })}"; + }; + }; "/etc/new_interface.xml" = { "C+" = { argument = "${pkgs.writeText "new_interface.xml" (new_interface { })}"; diff --git a/tests/testsuite_cpu_profiles_host.py b/tests/testsuite_cpu_profiles_host.py index 01f87cf1..dfbc29ff 100644 --- a/tests/testsuite_cpu_profiles_host.py +++ b/tests/testsuite_cpu_profiles_host.py @@ -8,15 +8,19 @@ try: from ..test_helper.test_helper import ( # type: ignore LibvirtTestsBase, + assert_nested_cirros_connectivity, initialComputeVMSetup, initialControllerVMSetup, + setup_nested_cirros, wait_for_ssh, ) except Exception: from test_helper import ( LibvirtTestsBase, + assert_nested_cirros_connectivity, initialComputeVMSetup, initialControllerVMSetup, + setup_nested_cirros, wait_for_ssh, ) @@ -82,11 +86,25 @@ def test_ubuntu_with_cpu_profiles(self): retries=350, ) + def test_nested_chv_guest(self): + """ + Test that we are able to boot a nested CHV VM using a Cirros image when + a CPU profile is in use. + """ + + controllerVM.succeed("virsh define /etc/domain-chv-cpu-sapphire-rapids.xml") + controllerVM.succeed("virsh start testvm") + + wait_for_ssh(controllerVM) + setup_nested_cirros(controllerVM) + assert_nested_cirros_connectivity(controllerVM) + def suite(): # Test cases involving live migration sorted in alphabetical order. testcases = [ LibvirtTests.test_cirros_with_cpu_profiles, + LibvirtTests.test_nested_chv_guest, LibvirtTests.test_ubuntu_with_cpu_profiles, ] diff --git a/tests/testsuite_default.py b/tests/testsuite_default.py index 7bcc81bf..70e0170f 100644 --- a/tests/testsuite_default.py +++ b/tests/testsuite_default.py @@ -9,8 +9,9 @@ # additional IDE configuration. try: from ..test_helper.test_helper import ( # type: ignore - assert_domain_domstate, LibvirtTestsBase, + assert_domain_domstate, + assert_nested_cirros_connectivity, hotplug, hotplug_fail, initialComputeVMSetup, @@ -19,6 +20,7 @@ number_of_network_devices, parse_devices_from_dom_def, pci_devices_by_bdf, + setup_nested_cirros, ssh, vcpu_affinity_checks, vm_unresponsive, @@ -28,8 +30,9 @@ ) except Exception: from test_helper import ( - assert_domain_domstate, LibvirtTestsBase, + assert_domain_domstate, + assert_nested_cirros_connectivity, hotplug, hotplug_fail, initialComputeVMSetup, @@ -38,6 +41,7 @@ number_of_network_devices, parse_devices_from_dom_def, pci_devices_by_bdf, + setup_nested_cirros, ssh, vcpu_affinity_checks, vm_unresponsive, @@ -1360,6 +1364,18 @@ def test_raw_image_is_properly_attached(self): controllerVM.succeed(f"rm {image}") + def test_nested_chv_guest(self): + """ + Test that we are able to boot a nested CHV VM using a Cirros image. + """ + + controllerVM.succeed("virsh define /etc/domain-chv.xml") + controllerVM.succeed("virsh start testvm") + + wait_for_ssh(controllerVM) + setup_nested_cirros(controllerVM) + assert_nested_cirros_connectivity(controllerVM) + def suite(): # Test cases sorted in alphabetical order. @@ -1383,6 +1399,7 @@ def suite(): LibvirtTests.test_list_smbios_oem_strings, LibvirtTests.test_list_smbios_sysinfo, LibvirtTests.test_managedsave, + LibvirtTests.test_nested_chv_guest, LibvirtTests.test_network_hotplug_attach_detach_persistent, LibvirtTests.test_network_hotplug_attach_detach_transient, LibvirtTests.test_network_hotplug_persistent_transient_detach_vm_restart,