From 63c820bbce47d12d9e7e55cf6927bd688995c016 Mon Sep 17 00:00:00 2001 From: Harshit Gupta Date: Mon, 9 Jun 2025 18:47:43 -0700 Subject: [PATCH 01/13] node-labeller.go able to query virsh --- BUILD.bazel | 10 +++ cmd/node-labeller/BUILD.bazel | 14 ++++ cmd/node-labeller/node-labeller.go | 113 +++++++++++++++++++++++++++++ hack/bazel-build.sh | 5 ++ 4 files changed, 142 insertions(+) create mode 100644 cmd/node-labeller/BUILD.bazel create mode 100644 cmd/node-labeller/node-labeller.go diff --git a/BUILD.bazel b/BUILD.bazel index cf8237f75aaf..9daf341c2a11 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -523,6 +523,16 @@ genrule( executable = 1, ) +genrule( + name = "build-node-labeller", + srcs = [ + "//cmd/node-labeller", + ], + outs = ["node-labeller"], + cmd = "echo '#!/bin/sh\n\ncp -f $(SRCS) $$1' > \"$@\"", + executable = 1, +) + genrule( name = "build-virtctl-amd64", srcs = [ diff --git a/cmd/node-labeller/BUILD.bazel b/cmd/node-labeller/BUILD.bazel new file mode 100644 index 000000000000..266b927a0f8b --- /dev/null +++ b/cmd/node-labeller/BUILD.bazel @@ -0,0 +1,14 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = ["node-labeller.go"], + importpath = "kubevirt.io/kubevirt/cmd/node-labeller", + visibility = ["//visibility:private"], +) + +go_binary( + name = "node-labeller", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/cmd/node-labeller/node-labeller.go b/cmd/node-labeller/node-labeller.go new file mode 100644 index 000000000000..aa8a2d2c494f --- /dev/null +++ b/cmd/node-labeller/node-labeller.go @@ -0,0 +1,113 @@ +package main + +import ( + "fmt" + "os/exec" + "strings" +) + +func executeCommand(command string) (string, error) { + cmd := exec.Command("bash", "-c", command) + output, err := cmd.Output() + outputStr := strings.TrimSpace(string(output)) + if err != nil { + return "", fmt.Errorf("failed to execute command %s: %w", command, err) + } + return outputStr, nil +} + +func queryMachineArchitecture() (string, error) { + return executeCommand("uname -m") +} + +func queryKvmMinor() (string, error) { + return executeCommand("grep -w 'kvm' /proc/misc | cut -f 1 -d' '") +} + +func main() { + + machine := "q35" + + arch, err := queryMachineArchitecture() + if err != nil { + fmt.Printf("Error querying machine architecture: %v\n", err) + return + } + + fmt.Printf("Detected arch=\"%s\"\n", arch) + + if arch == "aarch64" { + machine = "virt" + } else if arch == "s390x" { + machine = "390-ccw-virtio" + } else if arch != "x86_64" { + // Node labeling cannot proceed for this architecture + fmt.Printf("Node labeling cannot proceed for architecture: %s\n", arch) + return + } + + kvmMinor, err := queryKvmMinor() + if err != nil { + fmt.Printf("Error querying KVM minor: %v\n", err) + return + } + + virttype := "qemu" + + _, err = exec.Command("ls", "/dev/kvm").Output() + kvmExists := err == nil + + if !kvmExists && kvmMinor != "" { + executeCommand("mknod /dev/kvm c 10 " + kvmMinor) + } + + _, err = exec.Command("ls", "/dev/kvm").Output() + kvmExists = err == nil + + if kvmExists { + executeCommand("chmod o+rw /dev/kvm") + virttype = "kvm" + } + + _, err = exec.Command("ls", "/dev/sev").Output() + sevExists := err == nil + + if sevExists { + // QEMU requires RW access to query SEV capabilities + executeCommand("chmod o+rw /dev/kvm") + } + + cmd := exec.Command("virtqemud", "-d") + err = cmd.Start() + if err != nil { + fmt.Printf("Failed to start virtqemud: %v\n", err) + return + } + fmt.Println("virtqemud started in daemon mode") + + domainCapabilitiesOut, err := executeCommand(fmt.Sprintf("virsh domcapabilities --machine %s --arch %s --virttype %s", machine, arch, virttype)) + + if err != nil { + fmt.Printf("Failed to get domain capabilities: %v\n", err) + return + } + fmt.Println("Domain capabilities retrieved successfully. Length = ", len(domainCapabilitiesOut)) + + if arch == "x86_64" || arch == "s390x" { + cmd := fmt.Sprintf("virsh domcapabilities --machine %s --arch %s --virttype %s | virsh hypervisor-cpu-baseline --features /dev/stdin --machine %s --arch %s --virttype %s", machine, arch, virttype, machine, arch, virttype) + supportedFeatures, err := executeCommand(cmd) + if err != nil { + fmt.Printf("Failed to get supported features: %v\n", err) + return + } + fmt.Println("Supported features retrieved successfully. Length = ", len(supportedFeatures)) + } + + nodeCapabilities, err := executeCommand("virsh capabilities") + if err != nil { + fmt.Printf("Failed to get node capabilities: %v\n", err) + return + } + fmt.Println("Node capabilities retrieved successfully. Length = ", len(nodeCapabilities)) + +} diff --git a/hack/bazel-build.sh b/hack/bazel-build.sh index b1baa39eda65..89409b2022fd 100755 --- a/hack/bazel-build.sh +++ b/hack/bazel-build.sh @@ -25,6 +25,7 @@ source hack/config.sh rm -rf ${CMD_OUT_DIR} mkdir -p ${CMD_OUT_DIR}/virtctl +mkdir -p ${CMD_OUT_DIR}/node-labeller mkdir -p ${CMD_OUT_DIR}/dump mkdir -p ${CMD_OUT_DIR}/perfscale-audit mkdir -p ${CMD_OUT_DIR}/perfscale-load-generator @@ -66,6 +67,10 @@ bazel run \ --config="$(uname -m)" \ :build-virtctl -- ${CMD_OUT_DIR}/virtctl/virtctl +bazel run \ + --config="$(uname -m)" \ + :build-node-labeller -- ${CMD_OUT_DIR}/node-labeller/node-labeller + # Copy kubevirt-passt-binding binary to a reachable place outside of the build container bazel run \ --config=${ARCHITECTURE} \ From 2ffdd9573bd24afd40de634f5e03e28e2bc5b1b9 Mon Sep 17 00:00:00 2001 From: Harshit Gupta Date: Mon, 9 Jun 2025 19:19:31 -0700 Subject: [PATCH 02/13] Make node-labeller-go part of virt-launcher --- BUILD.bazel | 10 ---------- cmd/virt-launcher/BUILD.bazel | 1 + .../node-labeller-go}/BUILD.bazel | 4 ++-- .../node-labeller-go}/node-labeller.go | 0 hack/bazel-build.sh | 5 ----- 5 files changed, 3 insertions(+), 17 deletions(-) rename cmd/{node-labeller => virt-launcher/node-labeller-go}/BUILD.bazel (72%) rename cmd/{node-labeller => virt-launcher/node-labeller-go}/node-labeller.go (100%) diff --git a/BUILD.bazel b/BUILD.bazel index 9daf341c2a11..cf8237f75aaf 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -523,16 +523,6 @@ genrule( executable = 1, ) -genrule( - name = "build-node-labeller", - srcs = [ - "//cmd/node-labeller", - ], - outs = ["node-labeller"], - cmd = "echo '#!/bin/sh\n\ncp -f $(SRCS) $$1' > \"$@\"", - executable = 1, -) - genrule( name = "build-virtctl-amd64", srcs = [ diff --git a/cmd/virt-launcher/BUILD.bazel b/cmd/virt-launcher/BUILD.bazel index 47456736ae59..c2d2a4863344 100644 --- a/cmd/virt-launcher/BUILD.bazel +++ b/cmd/virt-launcher/BUILD.bazel @@ -62,6 +62,7 @@ pkg_tar( "//cmd/container-disk-v2alpha:container-disk", "//cmd/virt-freezer", "//cmd/virt-launcher-monitor", + "//cmd/virt-launcher/node-labeller-go", "//cmd/virt-probe", "//cmd/virt-tail", ], diff --git a/cmd/node-labeller/BUILD.bazel b/cmd/virt-launcher/node-labeller-go/BUILD.bazel similarity index 72% rename from cmd/node-labeller/BUILD.bazel rename to cmd/virt-launcher/node-labeller-go/BUILD.bazel index 266b927a0f8b..23e092e93d82 100644 --- a/cmd/node-labeller/BUILD.bazel +++ b/cmd/virt-launcher/node-labeller-go/BUILD.bazel @@ -3,12 +3,12 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") go_library( name = "go_default_library", srcs = ["node-labeller.go"], - importpath = "kubevirt.io/kubevirt/cmd/node-labeller", + importpath = "kubevirt.io/kubevirt/cmd/virt-launcher/node-labeller-go", visibility = ["//visibility:private"], ) go_binary( - name = "node-labeller", + name = "node-labeller-go", embed = [":go_default_library"], visibility = ["//visibility:public"], ) diff --git a/cmd/node-labeller/node-labeller.go b/cmd/virt-launcher/node-labeller-go/node-labeller.go similarity index 100% rename from cmd/node-labeller/node-labeller.go rename to cmd/virt-launcher/node-labeller-go/node-labeller.go diff --git a/hack/bazel-build.sh b/hack/bazel-build.sh index 89409b2022fd..b1baa39eda65 100755 --- a/hack/bazel-build.sh +++ b/hack/bazel-build.sh @@ -25,7 +25,6 @@ source hack/config.sh rm -rf ${CMD_OUT_DIR} mkdir -p ${CMD_OUT_DIR}/virtctl -mkdir -p ${CMD_OUT_DIR}/node-labeller mkdir -p ${CMD_OUT_DIR}/dump mkdir -p ${CMD_OUT_DIR}/perfscale-audit mkdir -p ${CMD_OUT_DIR}/perfscale-load-generator @@ -67,10 +66,6 @@ bazel run \ --config="$(uname -m)" \ :build-virtctl -- ${CMD_OUT_DIR}/virtctl/virtctl -bazel run \ - --config="$(uname -m)" \ - :build-node-labeller -- ${CMD_OUT_DIR}/node-labeller/node-labeller - # Copy kubevirt-passt-binding binary to a reachable place outside of the build container bazel run \ --config=${ARCHITECTURE} \ From b23dbc78604b842dcb6bb14d156020ced66df530 Mon Sep 17 00:00:00 2001 From: Harshit Gupta Date: Mon, 9 Jun 2025 20:44:39 -0700 Subject: [PATCH 03/13] More code --- BUILD.bazel | 10 + .../node-labeller-go/BUILD.bazel | 12 +- cmd/virt-launcher/node-labeller-go/model.go | 76 +++++++ .../node-labeller-go/node-labeller.go | 20 +- cmd/virt-launcher/node-labeller-go/util.go | 11 + .../virtualization-capabilities-interface.go | 31 +++ ...irtualization-capabilities-libvirt-qemu.go | 190 ++++++++++++++++++ hack/bazel-build.sh | 5 + 8 files changed, 345 insertions(+), 10 deletions(-) create mode 100644 cmd/virt-launcher/node-labeller-go/model.go create mode 100644 cmd/virt-launcher/node-labeller-go/util.go create mode 100644 cmd/virt-launcher/node-labeller-go/virtualization-capabilities-interface.go create mode 100644 cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go diff --git a/BUILD.bazel b/BUILD.bazel index cf8237f75aaf..d8dc8a94383e 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -513,6 +513,16 @@ container_push( tag = "$(container_tag)", ) +genrule( + name = "build-node-labeller-go", + srcs = [ + "//cmd/virt-launcher/node-labeller-go", + ], + outs = ["node-labeller-go"], + cmd = "echo '#!/bin/sh\n\ncp -f $(SRCS) $$1' > \"$@\"", + executable = 1, +) + genrule( name = "build-virtctl", srcs = [ diff --git a/cmd/virt-launcher/node-labeller-go/BUILD.bazel b/cmd/virt-launcher/node-labeller-go/BUILD.bazel index 23e092e93d82..165506871944 100644 --- a/cmd/virt-launcher/node-labeller-go/BUILD.bazel +++ b/cmd/virt-launcher/node-labeller-go/BUILD.bazel @@ -2,9 +2,19 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") go_library( name = "go_default_library", - srcs = ["node-labeller.go"], + srcs = [ + "model.go", + "node-labeller.go", + "util.go", + "virtualization-capabilities-interface.go", + "virtualization-capabilities-libvirt-qemu.go", + ], importpath = "kubevirt.io/kubevirt/cmd/virt-launcher/node-labeller-go", visibility = ["//visibility:private"], + deps = [ + "//staging/src/kubevirt.io/api/core/v1:go_default_library", + "//vendor/libvirt.org/go/libvirtxml:go_default_library", + ], ) go_binary( diff --git a/cmd/virt-launcher/node-labeller-go/model.go b/cmd/virt-launcher/node-labeller-go/model.go new file mode 100644 index 000000000000..b3f3eb7b744a --- /dev/null +++ b/cmd/virt-launcher/node-labeller-go/model.go @@ -0,0 +1,76 @@ +package main + +type cpuFeatures map[string]bool + +type supportedFeatures struct { + items []string +} + +type hostCPUModel struct { + Name string + fallback string + requiredFeatures cpuFeatures +} + +// HostDomCapabilities represents structure for parsing output of virsh capabilities +type HostDomCapabilities struct { + CPU CPU `xml:"cpu"` + SEV SEVConfiguration `xml:"features>sev"` +} + +// CPU represents slice of cpu modes +type CPU struct { + Mode []Mode `xml:"mode"` +} + +// Mode represents slice of cpu models +type Mode struct { + Name string `xml:"name,attr"` + Supported string `xml:"supported,attr"` + Vendor Vendor `xml:"vendor"` + Feature []HostFeature `xml:"feature"` + Model []Model `xml:"model"` +} + +type SupportedHostFeature struct { + Feature []HostFeature `xml:"feature"` +} + +type HostFeature struct { + Policy string `xml:"policy,attr"` + Name string `xml:"name,attr"` +} + +// Vendor represents vendor of host CPU +type Vendor struct { + Name string `xml:",chardata"` +} + +// Model represents cpu model +type Model struct { + Name string `xml:",chardata"` + Usable string `xml:"usable,attr"` + Fallback string `xml:"fallback,attr"` +} + +// Structures needed to parse cpu features +type FeatureModel struct { + Model Features `xml:"model"` +} + +type Features struct { + Features []Feature `xml:"feature"` +} + +type Feature struct { + Name string `xml:"name,attr"` +} + +type SEVConfiguration struct { + Supported string `xml:"supported,attr"` + CBitPos uint `xml:"cbitpos"` + ReducedPhysBits uint `xml:"reducedPhysBits"` + MaxGuests uint `xml:"maxGuests"` + MaxESGuests uint `xml:"maxESGuests"` + SupportedES string `xml:"-"` +} diff --git a/cmd/virt-launcher/node-labeller-go/node-labeller.go b/cmd/virt-launcher/node-labeller-go/node-labeller.go index aa8a2d2c494f..f765e3354073 100644 --- a/cmd/virt-launcher/node-labeller-go/node-labeller.go +++ b/cmd/virt-launcher/node-labeller-go/node-labeller.go @@ -6,6 +6,10 @@ import ( "strings" ) +const ( + XmlBasePath = "/var/lib/kubevirt-node-labeller/" +) + func executeCommand(command string) (string, error) { cmd := exec.Command("bash", "-c", command) output, err := cmd.Output() @@ -77,7 +81,7 @@ func main() { executeCommand("chmod o+rw /dev/kvm") } - cmd := exec.Command("virtqemud", "-d") + cmd := exec.Command("libvirtd", "-d") err = cmd.Start() if err != nil { fmt.Printf("Failed to start virtqemud: %v\n", err) @@ -85,29 +89,27 @@ func main() { } fmt.Println("virtqemud started in daemon mode") - domainCapabilitiesOut, err := executeCommand(fmt.Sprintf("virsh domcapabilities --machine %s --arch %s --virttype %s", machine, arch, virttype)) + executeCommand(fmt.Sprintf("mkdir -p %s", XmlBasePath)) // TODO Remove this later, this is just for testing + + _, err = executeCommand(fmt.Sprintf("virsh domcapabilities --machine %s --arch %s --virttype %s > %s/virsh_domcapabilities.xml", machine, arch, virttype, XmlBasePath)) if err != nil { fmt.Printf("Failed to get domain capabilities: %v\n", err) return } - fmt.Println("Domain capabilities retrieved successfully. Length = ", len(domainCapabilitiesOut)) if arch == "x86_64" || arch == "s390x" { - cmd := fmt.Sprintf("virsh domcapabilities --machine %s --arch %s --virttype %s | virsh hypervisor-cpu-baseline --features /dev/stdin --machine %s --arch %s --virttype %s", machine, arch, virttype, machine, arch, virttype) - supportedFeatures, err := executeCommand(cmd) + cmd := fmt.Sprintf("virsh domcapabilities --machine %s --arch %s --virttype %s | virsh hypervisor-cpu-baseline --features /dev/stdin --machine %s --arch %s --virttype %s > %s/supported_features.xml", machine, arch, virttype, machine, arch, virttype, XmlBasePath) + _, err := executeCommand(cmd) if err != nil { fmt.Printf("Failed to get supported features: %v\n", err) return } - fmt.Println("Supported features retrieved successfully. Length = ", len(supportedFeatures)) } - nodeCapabilities, err := executeCommand("virsh capabilities") + _, err = executeCommand(fmt.Sprintf("virsh capabilities > %s/capabilities.xml", XmlBasePath)) if err != nil { fmt.Printf("Failed to get node capabilities: %v\n", err) return } - fmt.Println("Node capabilities retrieved successfully. Length = ", len(nodeCapabilities)) - } diff --git a/cmd/virt-launcher/node-labeller-go/util.go b/cmd/virt-launcher/node-labeller-go/util.go new file mode 100644 index 000000000000..e96de4966724 --- /dev/null +++ b/cmd/virt-launcher/node-labeller-go/util.go @@ -0,0 +1,11 @@ +package main + +const ( + DefaultMinCPUModel = "Penryn" + RequirePolicy = "require" + KVMPath = "/dev/kvm" + VmxFeature = "vmx" + isSupported string = "yes" + isUnusable string = "no" + isRequired string = "require" +) diff --git a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-interface.go b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-interface.go new file mode 100644 index 000000000000..952172a1976c --- /dev/null +++ b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-interface.go @@ -0,0 +1,31 @@ +package main + +// VirtualizationCapabilitiesInterface defines methods for querying virtualization capabilities. +type VirtualizationCapabilitiesInterface interface { + // GetHypervFeatures returns a list of features required for Windows guests. + GetHypervFeatures() ([]string, error) + + // GetNodeTopology returns the node topology. + GetNodeTopology() (interface{}, error) + + // GetSupportedMachineTypes returns supported machine types. + GetSupportedMachineTypes() ([]string, error) + + // GetSupportedCpuModels returns supported CPU models. + GetSupportedCpuModels() ([]string, error) + + // GetHostCpuModelInfo returns host CPU model information. + GetHostCpuModelInfo() (interface{}, error) + + // GetSupportedCpuFeatures returns supported CPU features. + GetSupportedCpuFeatures() ([]string, error) + + // GetNodeTscInfo returns node TSC (Time Stamp Counter) information. + GetNodeTscInfo() (interface{}, error) + + // NodeSupportsRealTime returns true if the node supports real-time capabilities. + NodeSupportsRealTime() (bool, error) + + // GetNodeSevFeatures returns SEV (Secure Encrypted Virtualization) features of the node. + GetNodeSevFeatures() ([]string, error) +} diff --git a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go new file mode 100644 index 000000000000..a2389baacd85 --- /dev/null +++ b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go @@ -0,0 +1,190 @@ +package main + +import ( + "encoding/xml" + "fmt" + "os" + + v1 "kubevirt.io/api/core/v1" + "libvirt.org/go/libvirtxml" +) + +// VirtualizationCapabilitiesLibvirtQemu is a dummy implementation of VirtualizationCapabilitiesInterface. +type VirtualizationCapabilitiesLibvirtQemu struct { + // supportedFeatures.xml path + SupportedFeaturesPath string + // domainCapabilities.xml path + DomainCapabilitiesPath string + // capabilities.xml path + CapabilitiesPath string + + HostDomCapabilities HostDomCapabilities + SupportedHostFeatures []string + NodeCapabilities libvirtxml.Caps + + cpuModelVendor string + + hostCPUModel hostCPUModel +} + +func NewVirtualizationCapabilitiesLibvirtQemu(supportedFeaturesPath string, domainCapabilitiesPath string, capabilitiesPath string) *VirtualizationCapabilitiesLibvirtQemu { + cap := VirtualizationCapabilitiesLibvirtQemu{supportedFeaturesPath, domainCapabilitiesPath, capabilitiesPath, HostDomCapabilities{}, []string{}, libvirtxml.Caps{}, "", hostCPUModel{requiredFeatures: make(map[string]bool)}} + cap.loadAll() + return &cap +} + +func (v *VirtualizationCapabilitiesLibvirtQemu) loadAll() { + v.loadSupportedFeatures() + v.loadDomainCapabilities() + v.loadCapabilities() +} + +func (v *VirtualizationCapabilitiesLibvirtQemu) loadSupportedFeatures() { + hostFeatures := SupportedHostFeature{} + err := v.getStructureFromXMLFile(v.SupportedFeaturesPath, &hostFeatures) + if err != nil { + fmt.Printf("Error loading supported features: %v\n", err) + panic(err) + } + + usableFeatures := make([]string, 0) + for _, f := range hostFeatures.Feature { + if f.Policy == RequirePolicy { + usableFeatures = append(usableFeatures, f.Name) + } + } + + v.SupportedHostFeatures = usableFeatures +} + +func (v *VirtualizationCapabilitiesLibvirtQemu) loadDomainCapabilities() { + hostDomCapabilities := HostDomCapabilities{} + err := v.getStructureFromXMLFile(v.DomainCapabilitiesPath, &hostDomCapabilities) + if err != nil { + fmt.Printf("Error loading domain capabilities: %v\n", err) + panic(err) + } + + if hostDomCapabilities.SEV.Supported == "yes" && hostDomCapabilities.SEV.MaxESGuests > 0 { + hostDomCapabilities.SEV.SupportedES = "yes" + } else { + hostDomCapabilities.SEV.SupportedES = "no" + } + + v.HostDomCapabilities = hostDomCapabilities +} + +func (v *VirtualizationCapabilitiesLibvirtQemu) loadCapabilities() { + var capabilities libvirtxml.Caps + if err := capabilities.Unmarshal(v.CapabilitiesPath); err != nil { + panic(err) + } + v.NodeCapabilities = capabilities +} + +// GetHypervFeatures returns a dummy list of Hyper-V features. +func (v *VirtualizationCapabilitiesLibvirtQemu) GetHypervFeatures() ([]string, error) { + // TODO Query actual Hyper-V features from /dev/kvm + return []string{"hv_relaxed", "hv_vapic"}, nil +} + +// GetNodeTopology returns a dummy node topology. +func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeTopology() (interface{}, error) { + // TODO Get actual node topology + return map[string]interface{}{"sockets": 1, "cores": 2, "threads": 2}, nil +} + +// GetSupportedMachineTypes returns a dummy list of supported machine types. +func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedMachineTypes() ([]string, error) { + var supportedMachines []string + for _, guest := range v.NodeCapabilities.Guests { + for _, machine := range guest.Arch.Machines { + supportedMachines = append(supportedMachines, machine.Name) + } + } + return supportedMachines, nil +} + +// GetSupportedCpuModels returns a dummy list of supported CPU models. +func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedCpuModels() ([]string, error) { + // TODO Incorporate obsolete CPU models logic + + usableModels := make([]string, 0) + for _, mode := range v.HostDomCapabilities.CPU.Mode { + if mode.Name == v1.CPUModeHostModel { + if !true { // TODO This needs to be factored on!n.arch.supportsHostModel() { + fmt.Printf("host-model cpu mode is not supported for %s architecture", "TODO") + continue + } + + v.cpuModelVendor = mode.Vendor.Name + if v.cpuModelVendor == "" { + v.cpuModelVendor = "Intel" // TODO n.arch.defaultVendor() + } + + if len(mode.Model) < 1 { + panic("host model mode is expected to contain a model") + } + if len(mode.Model) > 1 { + panic("host model mode is expected to contain only one model") + } + + hostCpuModel := mode.Model[0] + v.hostCPUModel.Name = hostCpuModel.Name + v.hostCPUModel.fallback = hostCpuModel.Fallback + + for _, feature := range mode.Feature { + if feature.Policy == isRequired { + v.hostCPUModel.requiredFeatures[feature.Name] = true + } + } + } + + for _, model := range mode.Model { + if model.Usable == isUnusable || model.Usable == "" { + continue + } + usableModels = append(usableModels, model.Name) + } + } + + return usableModels, nil +} + +// GetHostCpuModelInfo returns dummy host CPU model information. +func (v *VirtualizationCapabilitiesLibvirtQemu) GetHostCpuModelInfo() (interface{}, error) { + return map[string]interface{}{"model": "Intel", "vendor": "GenuineIntel"}, nil +} + +// GetSupportedCpuFeatures returns a dummy list of supported CPU features. +func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedCpuFeatures() ([]string, error) { + return []string{"vmx", "aes", "fma"}, nil +} + +// GetNodeTscInfo returns dummy node TSC information. +func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeTscInfo() (interface{}, error) { + return map[string]interface{}{"frequency": 2500000000}, nil +} + +// NodeSupportsRealTime returns a dummy value indicating real-time support. +func (v *VirtualizationCapabilitiesLibvirtQemu) NodeSupportsRealTime() (bool, error) { + return true, nil +} + +// GetNodeSevFeatures returns a dummy list of SEV features. +func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeSevFeatures() ([]string, error) { + return []string{"sev", "sev-es"}, nil +} + +// GetStructureFromXMLFile load data from xml file and unmarshals them into given structure +// Given structure has to be pointer +func (v *VirtualizationCapabilitiesLibvirtQemu) getStructureFromXMLFile(path string, structure interface{}) error { + rawFile, err := os.ReadFile(path) + if err != nil { + return err + } + + fmt.Printf("node-labeller - loading data from xml file: %#v\n", string(rawFile)) + + return xml.Unmarshal(rawFile, structure) +} diff --git a/hack/bazel-build.sh b/hack/bazel-build.sh index b1baa39eda65..2a7e29b1e07d 100755 --- a/hack/bazel-build.sh +++ b/hack/bazel-build.sh @@ -25,6 +25,7 @@ source hack/config.sh rm -rf ${CMD_OUT_DIR} mkdir -p ${CMD_OUT_DIR}/virtctl +mkdir -p ${CMD_OUT_DIR}/node-labeller-go mkdir -p ${CMD_OUT_DIR}/dump mkdir -p ${CMD_OUT_DIR}/perfscale-audit mkdir -p ${CMD_OUT_DIR}/perfscale-load-generator @@ -66,6 +67,10 @@ bazel run \ --config="$(uname -m)" \ :build-virtctl -- ${CMD_OUT_DIR}/virtctl/virtctl +bazel run \ + --config="$(uname -m)" \ + :build-node-labeller-go -- ${CMD_OUT_DIR}/node-labeller-go/node-labeller-go + # Copy kubevirt-passt-binding binary to a reachable place outside of the build container bazel run \ --config=${ARCHITECTURE} \ From 57101dac27319595792f7682e25db15572305439 Mon Sep 17 00:00:00 2001 From: Harshit Gupta Date: Mon, 9 Jun 2025 21:14:20 -0700 Subject: [PATCH 04/13] Add basic exporter --- .../node-labeller-go/BUILD.bazel | 1 + .../node-labeller-go/exporter.go | 29 +++++++++++++++++++ .../node-labeller-go/node-labeller.go | 6 +++- ...irtualization-capabilities-libvirt-qemu.go | 11 ++++--- 4 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 cmd/virt-launcher/node-labeller-go/exporter.go diff --git a/cmd/virt-launcher/node-labeller-go/BUILD.bazel b/cmd/virt-launcher/node-labeller-go/BUILD.bazel index 165506871944..f0fee028d5ca 100644 --- a/cmd/virt-launcher/node-labeller-go/BUILD.bazel +++ b/cmd/virt-launcher/node-labeller-go/BUILD.bazel @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") go_library( name = "go_default_library", srcs = [ + "exporter.go", "model.go", "node-labeller.go", "util.go", diff --git a/cmd/virt-launcher/node-labeller-go/exporter.go b/cmd/virt-launcher/node-labeller-go/exporter.go new file mode 100644 index 000000000000..5233b3aaa0bf --- /dev/null +++ b/cmd/virt-launcher/node-labeller-go/exporter.go @@ -0,0 +1,29 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" +) + +func exportVirtualizationCapabilities(v VirtualizationCapabilitiesInterface, filename string) { + + machines, _ := v.GetSupportedMachineTypes() + fmt.Println("Supported machine types:", machines) + data, err := json.MarshalIndent(machines, "", " ") + if err != nil { + panic(err) + } + + file, err := os.Create(filename) + if err != nil { + panic(err) + } + defer file.Close() + + _, err = file.Write(data) + if err != nil { + panic(err) + } + +} diff --git a/cmd/virt-launcher/node-labeller-go/node-labeller.go b/cmd/virt-launcher/node-labeller-go/node-labeller.go index f765e3354073..4564a74148de 100644 --- a/cmd/virt-launcher/node-labeller-go/node-labeller.go +++ b/cmd/virt-launcher/node-labeller-go/node-labeller.go @@ -81,7 +81,7 @@ func main() { executeCommand("chmod o+rw /dev/kvm") } - cmd := exec.Command("libvirtd", "-d") + cmd := exec.Command("libvirtd", "-d") // TODO Revert to virtqemud err = cmd.Start() if err != nil { fmt.Printf("Failed to start virtqemud: %v\n", err) @@ -112,4 +112,8 @@ func main() { fmt.Printf("Failed to get node capabilities: %v\n", err) return } + + capabilityExtractor := NewVirtualizationCapabilitiesLibvirtQemu(fmt.Sprintf("%s/supported_features.xml", XmlBasePath), fmt.Sprintf("%s/virsh_domcapabilities.xml", XmlBasePath), fmt.Sprintf("%s/capabilities.xml", XmlBasePath)) + + exportVirtualizationCapabilities(capabilityExtractor, fmt.Sprintf("%s/virtualization_capabilities.json", XmlBasePath)) } diff --git a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go index a2389baacd85..129c2452241f 100644 --- a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go +++ b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go @@ -75,9 +75,10 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) loadDomainCapabilities() { } func (v *VirtualizationCapabilitiesLibvirtQemu) loadCapabilities() { - var capabilities libvirtxml.Caps - if err := capabilities.Unmarshal(v.CapabilitiesPath); err != nil { - panic(err) + capabilities := libvirtxml.Caps{} + err := v.getStructureFromXMLFile(v.CapabilitiesPath, &capabilities) + if err != nil { + panic(fmt.Sprintf("Error loading capabilities: %v\n", err)) } v.NodeCapabilities = capabilities } @@ -98,8 +99,10 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeTopology() (interface{}, func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedMachineTypes() ([]string, error) { var supportedMachines []string for _, guest := range v.NodeCapabilities.Guests { + fmt.Println("Guest architecture: ", guest.Arch.Name) for _, machine := range guest.Arch.Machines { supportedMachines = append(supportedMachines, machine.Name) + fmt.Println("Adding machine type: ", machine.Name) } } return supportedMachines, nil @@ -184,7 +187,7 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) getStructureFromXMLFile(path str return err } - fmt.Printf("node-labeller - loading data from xml file: %#v\n", string(rawFile)) + //fmt.Printf("node-labeller - loading data from xml file: %#v\n", string(rawFile)) return xml.Unmarshal(rawFile, structure) } From c3885e72a8f756cc9e4a343535d219a3e47eddf5 Mon Sep 17 00:00:00 2001 From: Harshit Gupta Date: Tue, 10 Jun 2025 05:30:24 -0700 Subject: [PATCH 05/13] Update initializer --- .../virtualization-capabilities-libvirt-qemu.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go index 129c2452241f..fbe3cce8dbf2 100644 --- a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go +++ b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go @@ -28,7 +28,11 @@ type VirtualizationCapabilitiesLibvirtQemu struct { } func NewVirtualizationCapabilitiesLibvirtQemu(supportedFeaturesPath string, domainCapabilitiesPath string, capabilitiesPath string) *VirtualizationCapabilitiesLibvirtQemu { - cap := VirtualizationCapabilitiesLibvirtQemu{supportedFeaturesPath, domainCapabilitiesPath, capabilitiesPath, HostDomCapabilities{}, []string{}, libvirtxml.Caps{}, "", hostCPUModel{requiredFeatures: make(map[string]bool)}} + cap := VirtualizationCapabilitiesLibvirtQemu{ + SupportedFeaturesPath: supportedFeaturesPath, + DomainCapabilitiesPath: domainCapabilitiesPath, + CapabilitiesPath: capabilitiesPath, + hostCPUModel: hostCPUModel{requiredFeatures: make(map[string]bool)}} cap.loadAll() return &cap } From a8e79f2a4804424246e848b19db9cc43370178e3 Mon Sep 17 00:00:00 2001 From: Harshit Gupta Date: Tue, 10 Jun 2025 06:29:32 -0700 Subject: [PATCH 06/13] Able to export the capabilities JSON --- .../node-labeller-go/exporter.go | 14 +++-- cmd/virt-launcher/node-labeller-go/model.go | 16 +++--- .../virtualization-capabilities-interface.go | 22 +++++--- ...irtualization-capabilities-libvirt-qemu.go | 54 ++++++++++++++----- 4 files changed, 75 insertions(+), 31 deletions(-) diff --git a/cmd/virt-launcher/node-labeller-go/exporter.go b/cmd/virt-launcher/node-labeller-go/exporter.go index 5233b3aaa0bf..6691544e8e73 100644 --- a/cmd/virt-launcher/node-labeller-go/exporter.go +++ b/cmd/virt-launcher/node-labeller-go/exporter.go @@ -2,15 +2,21 @@ package main import ( "encoding/json" - "fmt" "os" ) func exportVirtualizationCapabilities(v VirtualizationCapabilitiesInterface, filename string) { + virtCaps := make(map[string]interface{}) + virtCaps[HypervFeaturesKey], _ = v.GetHypervFeatures() + virtCaps[SupportedMachineTypeKeys], _ = v.GetSupportedMachineTypes() + virtCaps[SupportedCpuModelsKey], _ = v.GetSupportedCpuModels() + virtCaps[HostCpuModelInfoKey], _ = v.GetHostCpuModelInfo() + virtCaps[SupportedCpuFeaturesKey], _ = v.GetSupportedCpuFeatures() + virtCaps[NodeTscInfoKey], _ = v.GetNodeTscInfo() + virtCaps[NodeSupportsRealTimeKey], _ = v.NodeSupportsRealTime() + virtCaps[NodeSevFeaturesKey], _ = v.GetNodeSevFeatures() - machines, _ := v.GetSupportedMachineTypes() - fmt.Println("Supported machine types:", machines) - data, err := json.MarshalIndent(machines, "", " ") + data, err := json.MarshalIndent(virtCaps, "", " ") if err != nil { panic(err) } diff --git a/cmd/virt-launcher/node-labeller-go/model.go b/cmd/virt-launcher/node-labeller-go/model.go index b3f3eb7b744a..105566f4af8a 100644 --- a/cmd/virt-launcher/node-labeller-go/model.go +++ b/cmd/virt-launcher/node-labeller-go/model.go @@ -1,15 +1,15 @@ package main -type cpuFeatures map[string]bool - -type supportedFeatures struct { - items []string -} - type hostCPUModel struct { Name string - fallback string - requiredFeatures cpuFeatures + Fallback string + RequiredFeatures []string +} + +type TscConfig struct { + HasTscCounter bool + Frequency string + Scalable string } // HostDomCapabilities represents structure for parsing output of virsh capabilities diff --git a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-interface.go b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-interface.go index 952172a1976c..11d8edce782e 100644 --- a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-interface.go +++ b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-interface.go @@ -1,13 +1,23 @@ package main +const kernelSchedRealtimeRuntimeInMicrosecods = "kernel.sched_rt_runtime_us" + +const ( + HypervFeaturesKey = "hyperv-features" + SupportedMachineTypeKeys = "supported-machine-types" + SupportedCpuModelsKey = "supported-cpu-models" + HostCpuModelInfoKey = "host-cpu-model-info" + SupportedCpuFeaturesKey = "supported-cpu-features" + NodeTscInfoKey = "node-tsc-info" + NodeSupportsRealTimeKey = "node-supports-real-time" + NodeSevFeaturesKey = "node-sev-features" +) + // VirtualizationCapabilitiesInterface defines methods for querying virtualization capabilities. type VirtualizationCapabilitiesInterface interface { // GetHypervFeatures returns a list of features required for Windows guests. GetHypervFeatures() ([]string, error) - // GetNodeTopology returns the node topology. - GetNodeTopology() (interface{}, error) - // GetSupportedMachineTypes returns supported machine types. GetSupportedMachineTypes() ([]string, error) @@ -15,17 +25,17 @@ type VirtualizationCapabilitiesInterface interface { GetSupportedCpuModels() ([]string, error) // GetHostCpuModelInfo returns host CPU model information. - GetHostCpuModelInfo() (interface{}, error) + GetHostCpuModelInfo() (hostCPUModel, error) // GetSupportedCpuFeatures returns supported CPU features. GetSupportedCpuFeatures() ([]string, error) // GetNodeTscInfo returns node TSC (Time Stamp Counter) information. - GetNodeTscInfo() (interface{}, error) + GetNodeTscInfo() (TscConfig, error) // NodeSupportsRealTime returns true if the node supports real-time capabilities. NodeSupportsRealTime() (bool, error) // GetNodeSevFeatures returns SEV (Secure Encrypted Virtualization) features of the node. - GetNodeSevFeatures() ([]string, error) + GetNodeSevFeatures() (SEVConfiguration, error) } diff --git a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go index fbe3cce8dbf2..b657feb3a091 100644 --- a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go +++ b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go @@ -4,6 +4,8 @@ import ( "encoding/xml" "fmt" "os" + "os/exec" + "strings" v1 "kubevirt.io/api/core/v1" "libvirt.org/go/libvirtxml" @@ -31,8 +33,7 @@ func NewVirtualizationCapabilitiesLibvirtQemu(supportedFeaturesPath string, doma cap := VirtualizationCapabilitiesLibvirtQemu{ SupportedFeaturesPath: supportedFeaturesPath, DomainCapabilitiesPath: domainCapabilitiesPath, - CapabilitiesPath: capabilitiesPath, - hostCPUModel: hostCPUModel{requiredFeatures: make(map[string]bool)}} + CapabilitiesPath: capabilitiesPath} cap.loadAll() return &cap } @@ -114,7 +115,8 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedMachineTypes() ([]st // GetSupportedCpuModels returns a dummy list of supported CPU models. func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedCpuModels() ([]string, error) { - // TODO Incorporate obsolete CPU models logic + // TODO Incorporate obsolete CPU models logic. + // TODO This can also be done in the virt-handler itself. usableModels := make([]string, 0) for _, mode := range v.HostDomCapabilities.CPU.Mode { @@ -138,13 +140,16 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedCpuModels() ([]strin hostCpuModel := mode.Model[0] v.hostCPUModel.Name = hostCpuModel.Name - v.hostCPUModel.fallback = hostCpuModel.Fallback + v.hostCPUModel.Fallback = hostCpuModel.Fallback for _, feature := range mode.Feature { if feature.Policy == isRequired { - v.hostCPUModel.requiredFeatures[feature.Name] = true + v.hostCPUModel.RequiredFeatures = append(v.hostCPUModel.RequiredFeatures, feature.Name) } } + + fmt.Println("Host CPU Model : ", v.hostCPUModel) + } for _, model := range mode.Model { @@ -159,28 +164,37 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedCpuModels() ([]strin } // GetHostCpuModelInfo returns dummy host CPU model information. -func (v *VirtualizationCapabilitiesLibvirtQemu) GetHostCpuModelInfo() (interface{}, error) { - return map[string]interface{}{"model": "Intel", "vendor": "GenuineIntel"}, nil +func (v *VirtualizationCapabilitiesLibvirtQemu) GetHostCpuModelInfo() (hostCPUModel, error) { + return v.hostCPUModel, nil } // GetSupportedCpuFeatures returns a dummy list of supported CPU features. func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedCpuFeatures() ([]string, error) { - return []string{"vmx", "aes", "fma"}, nil + return v.SupportedHostFeatures, nil } // GetNodeTscInfo returns dummy node TSC information. -func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeTscInfo() (interface{}, error) { - return map[string]interface{}{"frequency": 2500000000}, nil +func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeTscInfo() (TscConfig, error) { + counter := v.NodeCapabilities.Host.CPU.Counter + if counter != nil && counter.Name == "tsc" { + return TscConfig{ + HasTscCounter: true, + Frequency: fmt.Sprintf("%d", counter.Frequency), + Scalable: fmt.Sprintf("%t", counter.Scaling == "yes"), + }, nil + } + return TscConfig{ + HasTscCounter: false}, nil } // NodeSupportsRealTime returns a dummy value indicating real-time support. func (v *VirtualizationCapabilitiesLibvirtQemu) NodeSupportsRealTime() (bool, error) { - return true, nil + return isNodeRealtimeCapable() } // GetNodeSevFeatures returns a dummy list of SEV features. -func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeSevFeatures() ([]string, error) { - return []string{"sev", "sev-es"}, nil +func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeSevFeatures() (SEVConfiguration, error) { + return v.HostDomCapabilities.SEV, nil } // GetStructureFromXMLFile load data from xml file and unmarshals them into given structure @@ -195,3 +209,17 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) getStructureFromXMLFile(path str return xml.Unmarshal(rawFile, structure) } + +// isNodeRealtimeCapable Checks if a node is capable of running realtime workloads. Currently by validating if the kernel system setting value +// for `kernel.sched_rt_runtime_us` is set to allow running realtime scheduling with unlimited time (==-1) +// TODO: This part should be improved to validate against key attributes that determine best if a host is able to run realtime +// workloads at peak performance. + +func isNodeRealtimeCapable() (bool, error) { + ret, err := exec.Command("sysctl", kernelSchedRealtimeRuntimeInMicrosecods).CombinedOutput() + if err != nil { + return false, err + } + st := strings.Trim(string(ret), "\n") + return fmt.Sprintf("%s = -1", kernelSchedRealtimeRuntimeInMicrosecods) == st, nil +} From e296ecc98bd0d8618711bb68eaa5023a9852ab94 Mon Sep 17 00:00:00 2001 From: Harshit Gupta Date: Tue, 10 Jun 2025 06:48:27 -0700 Subject: [PATCH 07/13] Moved common parts of node-labeller to virt-launcher-common --- .../node-labeller-go/BUILD.bazel | 5 +-- .../node-labeller-go/node-labeller.go | 4 ++- cmd/virt-launcher/node-labeller-go/util.go | 11 ------- ...irtualization-capabilities-libvirt-qemu.go | 32 ++++++++++++------- .../virt-capabilities/BUILD.bazel | 13 ++++++++ .../virt-capabilities}/exporter.go | 4 +-- .../virt-capabilities}/model.go | 4 +-- .../virt-capabilities/util.go | 10 ++++++ .../virtualization-capabilities-interface.go | 6 ++-- 9 files changed, 54 insertions(+), 35 deletions(-) delete mode 100644 cmd/virt-launcher/node-labeller-go/util.go create mode 100644 pkg/virt-launcher-common/virt-capabilities/BUILD.bazel rename {cmd/virt-launcher/node-labeller-go => pkg/virt-launcher-common/virt-capabilities}/exporter.go (90%) rename {cmd/virt-launcher/node-labeller-go => pkg/virt-launcher-common/virt-capabilities}/model.go (96%) create mode 100644 pkg/virt-launcher-common/virt-capabilities/util.go rename {cmd/virt-launcher/node-labeller-go => pkg/virt-launcher-common/virt-capabilities}/virtualization-capabilities-interface.go (90%) diff --git a/cmd/virt-launcher/node-labeller-go/BUILD.bazel b/cmd/virt-launcher/node-labeller-go/BUILD.bazel index f0fee028d5ca..46a66b7cc7d3 100644 --- a/cmd/virt-launcher/node-labeller-go/BUILD.bazel +++ b/cmd/virt-launcher/node-labeller-go/BUILD.bazel @@ -3,16 +3,13 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") go_library( name = "go_default_library", srcs = [ - "exporter.go", - "model.go", "node-labeller.go", - "util.go", - "virtualization-capabilities-interface.go", "virtualization-capabilities-libvirt-qemu.go", ], importpath = "kubevirt.io/kubevirt/cmd/virt-launcher/node-labeller-go", visibility = ["//visibility:private"], deps = [ + "//pkg/virt-launcher-common/virt-capabilities:go_default_library", "//staging/src/kubevirt.io/api/core/v1:go_default_library", "//vendor/libvirt.org/go/libvirtxml:go_default_library", ], diff --git a/cmd/virt-launcher/node-labeller-go/node-labeller.go b/cmd/virt-launcher/node-labeller-go/node-labeller.go index 4564a74148de..3744c47886be 100644 --- a/cmd/virt-launcher/node-labeller-go/node-labeller.go +++ b/cmd/virt-launcher/node-labeller-go/node-labeller.go @@ -4,6 +4,8 @@ import ( "fmt" "os/exec" "strings" + + virt_capabilities "kubevirt.io/kubevirt/pkg/virt-launcher-common/virt-capabilities" ) const ( @@ -115,5 +117,5 @@ func main() { capabilityExtractor := NewVirtualizationCapabilitiesLibvirtQemu(fmt.Sprintf("%s/supported_features.xml", XmlBasePath), fmt.Sprintf("%s/virsh_domcapabilities.xml", XmlBasePath), fmt.Sprintf("%s/capabilities.xml", XmlBasePath)) - exportVirtualizationCapabilities(capabilityExtractor, fmt.Sprintf("%s/virtualization_capabilities.json", XmlBasePath)) + virt_capabilities.ExportVirtualizationCapabilities(capabilityExtractor, fmt.Sprintf("%s/virtualization_capabilities.json", XmlBasePath)) } diff --git a/cmd/virt-launcher/node-labeller-go/util.go b/cmd/virt-launcher/node-labeller-go/util.go deleted file mode 100644 index e96de4966724..000000000000 --- a/cmd/virt-launcher/node-labeller-go/util.go +++ /dev/null @@ -1,11 +0,0 @@ -package main - -const ( - DefaultMinCPUModel = "Penryn" - RequirePolicy = "require" - KVMPath = "/dev/kvm" - VmxFeature = "vmx" - isSupported string = "yes" - isUnusable string = "no" - isRequired string = "require" -) diff --git a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go index b657feb3a091..e7dec61bc168 100644 --- a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go +++ b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go @@ -9,6 +9,14 @@ import ( v1 "kubevirt.io/api/core/v1" "libvirt.org/go/libvirtxml" + + virt_capabilities "kubevirt.io/kubevirt/pkg/virt-launcher-common/virt-capabilities" +) + +const ( + isSupported string = "yes" + isUnusable string = "no" + isRequired string = "require" ) // VirtualizationCapabilitiesLibvirtQemu is a dummy implementation of VirtualizationCapabilitiesInterface. @@ -20,13 +28,13 @@ type VirtualizationCapabilitiesLibvirtQemu struct { // capabilities.xml path CapabilitiesPath string - HostDomCapabilities HostDomCapabilities + HostDomCapabilities virt_capabilities.HostDomCapabilities SupportedHostFeatures []string NodeCapabilities libvirtxml.Caps cpuModelVendor string - hostCPUModel hostCPUModel + hostCPUModel virt_capabilities.HostCPUModel } func NewVirtualizationCapabilitiesLibvirtQemu(supportedFeaturesPath string, domainCapabilitiesPath string, capabilitiesPath string) *VirtualizationCapabilitiesLibvirtQemu { @@ -45,7 +53,7 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) loadAll() { } func (v *VirtualizationCapabilitiesLibvirtQemu) loadSupportedFeatures() { - hostFeatures := SupportedHostFeature{} + hostFeatures := virt_capabilities.SupportedHostFeature{} err := v.getStructureFromXMLFile(v.SupportedFeaturesPath, &hostFeatures) if err != nil { fmt.Printf("Error loading supported features: %v\n", err) @@ -54,7 +62,7 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) loadSupportedFeatures() { usableFeatures := make([]string, 0) for _, f := range hostFeatures.Feature { - if f.Policy == RequirePolicy { + if f.Policy == isRequired { usableFeatures = append(usableFeatures, f.Name) } } @@ -63,7 +71,7 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) loadSupportedFeatures() { } func (v *VirtualizationCapabilitiesLibvirtQemu) loadDomainCapabilities() { - hostDomCapabilities := HostDomCapabilities{} + hostDomCapabilities := virt_capabilities.HostDomCapabilities{} err := v.getStructureFromXMLFile(v.DomainCapabilitiesPath, &hostDomCapabilities) if err != nil { fmt.Printf("Error loading domain capabilities: %v\n", err) @@ -164,7 +172,7 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedCpuModels() ([]strin } // GetHostCpuModelInfo returns dummy host CPU model information. -func (v *VirtualizationCapabilitiesLibvirtQemu) GetHostCpuModelInfo() (hostCPUModel, error) { +func (v *VirtualizationCapabilitiesLibvirtQemu) GetHostCpuModelInfo() (virt_capabilities.HostCPUModel, error) { return v.hostCPUModel, nil } @@ -174,16 +182,16 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedCpuFeatures() ([]str } // GetNodeTscInfo returns dummy node TSC information. -func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeTscInfo() (TscConfig, error) { +func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeTscInfo() (virt_capabilities.TscConfig, error) { counter := v.NodeCapabilities.Host.CPU.Counter if counter != nil && counter.Name == "tsc" { - return TscConfig{ + return virt_capabilities.TscConfig{ HasTscCounter: true, Frequency: fmt.Sprintf("%d", counter.Frequency), Scalable: fmt.Sprintf("%t", counter.Scaling == "yes"), }, nil } - return TscConfig{ + return virt_capabilities.TscConfig{ HasTscCounter: false}, nil } @@ -193,7 +201,7 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) NodeSupportsRealTime() (bool, er } // GetNodeSevFeatures returns a dummy list of SEV features. -func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeSevFeatures() (SEVConfiguration, error) { +func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeSevFeatures() (virt_capabilities.SEVConfiguration, error) { return v.HostDomCapabilities.SEV, nil } @@ -216,10 +224,10 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) getStructureFromXMLFile(path str // workloads at peak performance. func isNodeRealtimeCapable() (bool, error) { - ret, err := exec.Command("sysctl", kernelSchedRealtimeRuntimeInMicrosecods).CombinedOutput() + ret, err := exec.Command("sysctl", virt_capabilities.KernelSchedRealtimeRuntimeInMicroseconds).CombinedOutput() if err != nil { return false, err } st := strings.Trim(string(ret), "\n") - return fmt.Sprintf("%s = -1", kernelSchedRealtimeRuntimeInMicrosecods) == st, nil + return fmt.Sprintf("%s = -1", virt_capabilities.KernelSchedRealtimeRuntimeInMicroseconds) == st, nil } diff --git a/pkg/virt-launcher-common/virt-capabilities/BUILD.bazel b/pkg/virt-launcher-common/virt-capabilities/BUILD.bazel new file mode 100644 index 000000000000..097f463f78de --- /dev/null +++ b/pkg/virt-launcher-common/virt-capabilities/BUILD.bazel @@ -0,0 +1,13 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "exporter.go", + "model.go", + "util.go", + "virtualization-capabilities-interface.go", + ], + importpath = "kubevirt.io/kubevirt/pkg/virt-launcher-common/virt-capabilities", + visibility = ["//visibility:public"], +) diff --git a/cmd/virt-launcher/node-labeller-go/exporter.go b/pkg/virt-launcher-common/virt-capabilities/exporter.go similarity index 90% rename from cmd/virt-launcher/node-labeller-go/exporter.go rename to pkg/virt-launcher-common/virt-capabilities/exporter.go index 6691544e8e73..ecc2b83f48a9 100644 --- a/cmd/virt-launcher/node-labeller-go/exporter.go +++ b/pkg/virt-launcher-common/virt-capabilities/exporter.go @@ -1,11 +1,11 @@ -package main +package virt_capabilities import ( "encoding/json" "os" ) -func exportVirtualizationCapabilities(v VirtualizationCapabilitiesInterface, filename string) { +func ExportVirtualizationCapabilities(v VirtualizationCapabilitiesInterface, filename string) { virtCaps := make(map[string]interface{}) virtCaps[HypervFeaturesKey], _ = v.GetHypervFeatures() virtCaps[SupportedMachineTypeKeys], _ = v.GetSupportedMachineTypes() diff --git a/cmd/virt-launcher/node-labeller-go/model.go b/pkg/virt-launcher-common/virt-capabilities/model.go similarity index 96% rename from cmd/virt-launcher/node-labeller-go/model.go rename to pkg/virt-launcher-common/virt-capabilities/model.go index 105566f4af8a..a82a6a368df0 100644 --- a/cmd/virt-launcher/node-labeller-go/model.go +++ b/pkg/virt-launcher-common/virt-capabilities/model.go @@ -1,6 +1,6 @@ -package main +package virt_capabilities -type hostCPUModel struct { +type HostCPUModel struct { Name string Fallback string RequiredFeatures []string diff --git a/pkg/virt-launcher-common/virt-capabilities/util.go b/pkg/virt-launcher-common/virt-capabilities/util.go new file mode 100644 index 000000000000..522f87687312 --- /dev/null +++ b/pkg/virt-launcher-common/virt-capabilities/util.go @@ -0,0 +1,10 @@ +package virt_capabilities + +const ( + DefaultMinCPUModel = "Penryn" + RequirePolicy = "require" + KVMPath = "/dev/kvm" + VmxFeature = "vmx" +) + +// TODO Check which ones of these are required here diff --git a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-interface.go b/pkg/virt-launcher-common/virt-capabilities/virtualization-capabilities-interface.go similarity index 90% rename from cmd/virt-launcher/node-labeller-go/virtualization-capabilities-interface.go rename to pkg/virt-launcher-common/virt-capabilities/virtualization-capabilities-interface.go index 11d8edce782e..adc909836389 100644 --- a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-interface.go +++ b/pkg/virt-launcher-common/virt-capabilities/virtualization-capabilities-interface.go @@ -1,6 +1,6 @@ -package main +package virt_capabilities -const kernelSchedRealtimeRuntimeInMicrosecods = "kernel.sched_rt_runtime_us" +const KernelSchedRealtimeRuntimeInMicroseconds = "kernel.sched_rt_runtime_us" const ( HypervFeaturesKey = "hyperv-features" @@ -25,7 +25,7 @@ type VirtualizationCapabilitiesInterface interface { GetSupportedCpuModels() ([]string, error) // GetHostCpuModelInfo returns host CPU model information. - GetHostCpuModelInfo() (hostCPUModel, error) + GetHostCpuModelInfo() (HostCPUModel, error) // GetSupportedCpuFeatures returns supported CPU features. GetSupportedCpuFeatures() ([]string, error) From 8662cdd7ffad1b66d6d14e15e9f09ee762b56d2c Mon Sep 17 00:00:00 2001 From: Harshit Gupta Date: Tue, 10 Jun 2025 06:58:20 -0700 Subject: [PATCH 08/13] Move libvirt-specific types to cmd --- cmd/virt-launcher/node-labeller-go/BUILD.bazel | 1 + .../virt-launcher/node-labeller-go/types.go | 14 +------------- .../virtualization-capabilities-libvirt-qemu.go | 11 +++++++---- .../virt-capabilities/BUILD.bazel | 2 +- .../virt-capabilities/types.go | 17 +++++++++++++++++ 5 files changed, 27 insertions(+), 18 deletions(-) rename pkg/virt-launcher-common/virt-capabilities/model.go => cmd/virt-launcher/node-labeller-go/types.go (86%) create mode 100644 pkg/virt-launcher-common/virt-capabilities/types.go diff --git a/cmd/virt-launcher/node-labeller-go/BUILD.bazel b/cmd/virt-launcher/node-labeller-go/BUILD.bazel index 46a66b7cc7d3..0fa9010cb641 100644 --- a/cmd/virt-launcher/node-labeller-go/BUILD.bazel +++ b/cmd/virt-launcher/node-labeller-go/BUILD.bazel @@ -4,6 +4,7 @@ go_library( name = "go_default_library", srcs = [ "node-labeller.go", + "types.go", "virtualization-capabilities-libvirt-qemu.go", ], importpath = "kubevirt.io/kubevirt/cmd/virt-launcher/node-labeller-go", diff --git a/pkg/virt-launcher-common/virt-capabilities/model.go b/cmd/virt-launcher/node-labeller-go/types.go similarity index 86% rename from pkg/virt-launcher-common/virt-capabilities/model.go rename to cmd/virt-launcher/node-labeller-go/types.go index a82a6a368df0..9566ef8dc6e9 100644 --- a/pkg/virt-launcher-common/virt-capabilities/model.go +++ b/cmd/virt-launcher/node-labeller-go/types.go @@ -1,16 +1,4 @@ -package virt_capabilities - -type HostCPUModel struct { - Name string - Fallback string - RequiredFeatures []string -} - -type TscConfig struct { - HasTscCounter bool - Frequency string - Scalable string -} +package main // HostDomCapabilities represents structure for parsing output of virsh capabilities type HostDomCapabilities struct { diff --git a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go index e7dec61bc168..a3b03198a5ea 100644 --- a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go +++ b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go @@ -28,7 +28,7 @@ type VirtualizationCapabilitiesLibvirtQemu struct { // capabilities.xml path CapabilitiesPath string - HostDomCapabilities virt_capabilities.HostDomCapabilities + HostDomCapabilities HostDomCapabilities SupportedHostFeatures []string NodeCapabilities libvirtxml.Caps @@ -53,7 +53,7 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) loadAll() { } func (v *VirtualizationCapabilitiesLibvirtQemu) loadSupportedFeatures() { - hostFeatures := virt_capabilities.SupportedHostFeature{} + hostFeatures := SupportedHostFeature{} err := v.getStructureFromXMLFile(v.SupportedFeaturesPath, &hostFeatures) if err != nil { fmt.Printf("Error loading supported features: %v\n", err) @@ -71,7 +71,7 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) loadSupportedFeatures() { } func (v *VirtualizationCapabilitiesLibvirtQemu) loadDomainCapabilities() { - hostDomCapabilities := virt_capabilities.HostDomCapabilities{} + hostDomCapabilities := HostDomCapabilities{} err := v.getStructureFromXMLFile(v.DomainCapabilitiesPath, &hostDomCapabilities) if err != nil { fmt.Printf("Error loading domain capabilities: %v\n", err) @@ -202,7 +202,10 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) NodeSupportsRealTime() (bool, er // GetNodeSevFeatures returns a dummy list of SEV features. func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeSevFeatures() (virt_capabilities.SEVConfiguration, error) { - return v.HostDomCapabilities.SEV, nil + sevCfg := virt_capabilities.SEVConfiguration{ + SupportedES: v.HostDomCapabilities.SEV.SupportedES, + } + return sevCfg, nil } // GetStructureFromXMLFile load data from xml file and unmarshals them into given structure diff --git a/pkg/virt-launcher-common/virt-capabilities/BUILD.bazel b/pkg/virt-launcher-common/virt-capabilities/BUILD.bazel index 097f463f78de..b904ab012a3d 100644 --- a/pkg/virt-launcher-common/virt-capabilities/BUILD.bazel +++ b/pkg/virt-launcher-common/virt-capabilities/BUILD.bazel @@ -4,7 +4,7 @@ go_library( name = "go_default_library", srcs = [ "exporter.go", - "model.go", + "types.go", "util.go", "virtualization-capabilities-interface.go", ], diff --git a/pkg/virt-launcher-common/virt-capabilities/types.go b/pkg/virt-launcher-common/virt-capabilities/types.go new file mode 100644 index 000000000000..215dcbb214f7 --- /dev/null +++ b/pkg/virt-launcher-common/virt-capabilities/types.go @@ -0,0 +1,17 @@ +package virt_capabilities + +type HostCPUModel struct { + Name string + Fallback string + RequiredFeatures []string +} + +type TscConfig struct { + HasTscCounter bool + Frequency string + Scalable string +} + +type SEVConfiguration struct { + SupportedES string +} From bf97d7e5caed5f6500696f480fb570979ecea64c Mon Sep 17 00:00:00 2001 From: Harshit Gupta Date: Tue, 10 Jun 2025 07:13:01 -0700 Subject: [PATCH 09/13] Create struct for JSON (un)marshalling --- ...irtualization-capabilities-libvirt-qemu.go | 39 ++++++++++--------- .../virt-capabilities/exporter.go | 19 ++++----- .../virt-capabilities/types.go | 23 ++++++++--- .../virtualization-capabilities-interface.go | 16 ++++---- 4 files changed, 55 insertions(+), 42 deletions(-) diff --git a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go index a3b03198a5ea..e71d6ce0bc7f 100644 --- a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go +++ b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go @@ -97,19 +97,19 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) loadCapabilities() { } // GetHypervFeatures returns a dummy list of Hyper-V features. -func (v *VirtualizationCapabilitiesLibvirtQemu) GetHypervFeatures() ([]string, error) { +func (v *VirtualizationCapabilitiesLibvirtQemu) GetHypervFeatures() []string { // TODO Query actual Hyper-V features from /dev/kvm - return []string{"hv_relaxed", "hv_vapic"}, nil + return []string{"hv_relaxed", "hv_vapic"} } // GetNodeTopology returns a dummy node topology. -func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeTopology() (interface{}, error) { +func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeTopology() interface{} { // TODO Get actual node topology - return map[string]interface{}{"sockets": 1, "cores": 2, "threads": 2}, nil + return map[string]interface{}{"sockets": 1, "cores": 2, "threads": 2} } // GetSupportedMachineTypes returns a dummy list of supported machine types. -func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedMachineTypes() ([]string, error) { +func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedMachineTypes() []string { var supportedMachines []string for _, guest := range v.NodeCapabilities.Guests { fmt.Println("Guest architecture: ", guest.Arch.Name) @@ -118,11 +118,11 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedMachineTypes() ([]st fmt.Println("Adding machine type: ", machine.Name) } } - return supportedMachines, nil + return supportedMachines } // GetSupportedCpuModels returns a dummy list of supported CPU models. -func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedCpuModels() ([]string, error) { +func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedCpuModels() []string { // TODO Incorporate obsolete CPU models logic. // TODO This can also be done in the virt-handler itself. @@ -168,44 +168,45 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedCpuModels() ([]strin } } - return usableModels, nil + return usableModels } // GetHostCpuModelInfo returns dummy host CPU model information. -func (v *VirtualizationCapabilitiesLibvirtQemu) GetHostCpuModelInfo() (virt_capabilities.HostCPUModel, error) { - return v.hostCPUModel, nil +func (v *VirtualizationCapabilitiesLibvirtQemu) GetHostCpuModelInfo() virt_capabilities.HostCPUModel { + return v.hostCPUModel } // GetSupportedCpuFeatures returns a dummy list of supported CPU features. -func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedCpuFeatures() ([]string, error) { - return v.SupportedHostFeatures, nil +func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedCpuFeatures() []string { + return v.SupportedHostFeatures } // GetNodeTscInfo returns dummy node TSC information. -func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeTscInfo() (virt_capabilities.TscConfig, error) { +func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeTscInfo() virt_capabilities.TscConfig { counter := v.NodeCapabilities.Host.CPU.Counter if counter != nil && counter.Name == "tsc" { return virt_capabilities.TscConfig{ HasTscCounter: true, Frequency: fmt.Sprintf("%d", counter.Frequency), Scalable: fmt.Sprintf("%t", counter.Scaling == "yes"), - }, nil + } } return virt_capabilities.TscConfig{ - HasTscCounter: false}, nil + HasTscCounter: false} } // NodeSupportsRealTime returns a dummy value indicating real-time support. -func (v *VirtualizationCapabilitiesLibvirtQemu) NodeSupportsRealTime() (bool, error) { - return isNodeRealtimeCapable() +func (v *VirtualizationCapabilitiesLibvirtQemu) NodeSupportsRealTime() bool { + isNodeRealtimeCapable, _ := isNodeRealtimeCapable() + return isNodeRealtimeCapable } // GetNodeSevFeatures returns a dummy list of SEV features. -func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeSevFeatures() (virt_capabilities.SEVConfiguration, error) { +func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeSevFeatures() virt_capabilities.SEVConfiguration { sevCfg := virt_capabilities.SEVConfiguration{ SupportedES: v.HostDomCapabilities.SEV.SupportedES, } - return sevCfg, nil + return sevCfg } // GetStructureFromXMLFile load data from xml file and unmarshals them into given structure diff --git a/pkg/virt-launcher-common/virt-capabilities/exporter.go b/pkg/virt-launcher-common/virt-capabilities/exporter.go index ecc2b83f48a9..d86e65a69daa 100644 --- a/pkg/virt-launcher-common/virt-capabilities/exporter.go +++ b/pkg/virt-launcher-common/virt-capabilities/exporter.go @@ -6,15 +6,16 @@ import ( ) func ExportVirtualizationCapabilities(v VirtualizationCapabilitiesInterface, filename string) { - virtCaps := make(map[string]interface{}) - virtCaps[HypervFeaturesKey], _ = v.GetHypervFeatures() - virtCaps[SupportedMachineTypeKeys], _ = v.GetSupportedMachineTypes() - virtCaps[SupportedCpuModelsKey], _ = v.GetSupportedCpuModels() - virtCaps[HostCpuModelInfoKey], _ = v.GetHostCpuModelInfo() - virtCaps[SupportedCpuFeaturesKey], _ = v.GetSupportedCpuFeatures() - virtCaps[NodeTscInfoKey], _ = v.GetNodeTscInfo() - virtCaps[NodeSupportsRealTimeKey], _ = v.NodeSupportsRealTime() - virtCaps[NodeSevFeaturesKey], _ = v.GetNodeSevFeatures() + virtCaps := VirtualizationCapabilities{ + SupportedCPUModels: v.GetSupportedCpuModels(), + SupportedMachineTypes: v.GetSupportedMachineTypes(), + HypervFeatures: v.GetHypervFeatures(), + HostCpuModelInfo: v.GetHostCpuModelInfo(), + SupportedCpuFeatures: v.GetSupportedCpuFeatures(), + NodeTscInfo: v.GetNodeTscInfo(), + NodeSupportsRealTime: v.NodeSupportsRealTime(), + NodeSevFeatures: v.GetNodeSevFeatures(), + } data, err := json.MarshalIndent(virtCaps, "", " ") if err != nil { diff --git a/pkg/virt-launcher-common/virt-capabilities/types.go b/pkg/virt-launcher-common/virt-capabilities/types.go index 215dcbb214f7..6d21fb4bcfbf 100644 --- a/pkg/virt-launcher-common/virt-capabilities/types.go +++ b/pkg/virt-launcher-common/virt-capabilities/types.go @@ -1,17 +1,28 @@ package virt_capabilities type HostCPUModel struct { - Name string - Fallback string - RequiredFeatures []string + Name string `json:"name"` + Fallback string `json:"fallback"` + RequiredFeatures []string `json:"requiredFeatures"` } type TscConfig struct { - HasTscCounter bool - Frequency string - Scalable string + HasTscCounter bool `json:"hasTscCounter"` + Frequency string `json:"frequency"` + Scalable string `json:"scalable"` } type SEVConfiguration struct { SupportedES string } + +type VirtualizationCapabilities struct { + SupportedCPUModels []string `json:"supportedCpuModels"` + SupportedMachineTypes []string `json:"supportedMachineTypes"` + HypervFeatures []string `json:"hypervFeatures"` + HostCpuModelInfo HostCPUModel `json:"hostCpuModelInfo"` + SupportedCpuFeatures []string `json:"supportedCpuFeatures"` + NodeTscInfo TscConfig `json:"nodeTscInfo"` + NodeSupportsRealTime bool `json:"nodeSupportsRealTime"` + NodeSevFeatures SEVConfiguration `json:"nodeSevFeatures"` +} diff --git a/pkg/virt-launcher-common/virt-capabilities/virtualization-capabilities-interface.go b/pkg/virt-launcher-common/virt-capabilities/virtualization-capabilities-interface.go index adc909836389..f15707c7127b 100644 --- a/pkg/virt-launcher-common/virt-capabilities/virtualization-capabilities-interface.go +++ b/pkg/virt-launcher-common/virt-capabilities/virtualization-capabilities-interface.go @@ -16,26 +16,26 @@ const ( // VirtualizationCapabilitiesInterface defines methods for querying virtualization capabilities. type VirtualizationCapabilitiesInterface interface { // GetHypervFeatures returns a list of features required for Windows guests. - GetHypervFeatures() ([]string, error) + GetHypervFeatures() []string // GetSupportedMachineTypes returns supported machine types. - GetSupportedMachineTypes() ([]string, error) + GetSupportedMachineTypes() []string // GetSupportedCpuModels returns supported CPU models. - GetSupportedCpuModels() ([]string, error) + GetSupportedCpuModels() []string // GetHostCpuModelInfo returns host CPU model information. - GetHostCpuModelInfo() (HostCPUModel, error) + GetHostCpuModelInfo() HostCPUModel // GetSupportedCpuFeatures returns supported CPU features. - GetSupportedCpuFeatures() ([]string, error) + GetSupportedCpuFeatures() []string // GetNodeTscInfo returns node TSC (Time Stamp Counter) information. - GetNodeTscInfo() (TscConfig, error) + GetNodeTscInfo() TscConfig // NodeSupportsRealTime returns true if the node supports real-time capabilities. - NodeSupportsRealTime() (bool, error) + NodeSupportsRealTime() bool // GetNodeSevFeatures returns SEV (Secure Encrypted Virtualization) features of the node. - GetNodeSevFeatures() (SEVConfiguration, error) + GetNodeSevFeatures() SEVConfiguration } From f8710ea538f639b406eff1ceb30ad19693327d7b Mon Sep 17 00:00:00 2001 From: Harshit Gupta Date: Tue, 10 Jun 2025 08:34:18 -0700 Subject: [PATCH 10/13] Run node-labeller-go container in virt-handler pod --- .../node-labeller-go/node-labeller.go | 8 +++++-- .../generate/components/daemonsets.go | 21 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/cmd/virt-launcher/node-labeller-go/node-labeller.go b/cmd/virt-launcher/node-labeller-go/node-labeller.go index 3744c47886be..902ab2df563c 100644 --- a/cmd/virt-launcher/node-labeller-go/node-labeller.go +++ b/cmd/virt-launcher/node-labeller-go/node-labeller.go @@ -4,12 +4,13 @@ import ( "fmt" "os/exec" "strings" + "time" virt_capabilities "kubevirt.io/kubevirt/pkg/virt-launcher-common/virt-capabilities" ) const ( - XmlBasePath = "/var/lib/kubevirt-node-labeller/" + XmlBasePath = "/var/lib/kubevirt-node-labeller" ) func executeCommand(command string) (string, error) { @@ -83,7 +84,7 @@ func main() { executeCommand("chmod o+rw /dev/kvm") } - cmd := exec.Command("libvirtd", "-d") // TODO Revert to virtqemud + cmd := exec.Command("virtqemud", "-d") err = cmd.Start() if err != nil { fmt.Printf("Failed to start virtqemud: %v\n", err) @@ -93,6 +94,9 @@ func main() { executeCommand(fmt.Sprintf("mkdir -p %s", XmlBasePath)) // TODO Remove this later, this is just for testing + fmt.Println("Waiting for virtqemud to start...") + time.Sleep(5 * time.Second) // Wait for virtqemud to start + _, err = executeCommand(fmt.Sprintf("virsh domcapabilities --machine %s --arch %s --virttype %s > %s/virsh_domcapabilities.xml", machine, arch, virttype, XmlBasePath)) if err != nil { diff --git a/pkg/virt-operator/resource/generate/components/daemonsets.go b/pkg/virt-operator/resource/generate/components/daemonsets.go index 878a1b3e76c1..05015058c5f8 100644 --- a/pkg/virt-operator/resource/generate/components/daemonsets.go +++ b/pkg/virt-operator/resource/generate/components/daemonsets.go @@ -152,6 +152,27 @@ func NewHandlerDaemonSet(namespace, repository, imagePrefix, version, launcherVe }, TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError, }, + { + Command: []string{ + "/bin/sh", + "-c", + }, + Image: launcherImage, + Name: "virt-capability-extractor", + Args: []string{ + "node-labeller-go", + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: pointer.P(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "node-labeller", + MountPath: nodeLabellerVolumePath, + }, + }, + TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError, + }, } // If there is any image pull secret added to the `virt-handler` deployment From 86738194d5e0ae3fb126f895bbe3d5466294b5cf Mon Sep 17 00:00:00 2001 From: Harshit Gupta Date: Tue, 10 Jun 2025 11:01:57 -0700 Subject: [PATCH 11/13] Labeling working E2E --- cmd/virt-handler/BUILD.bazel | 1 + cmd/virt-handler/virt-handler.go | 15 ++- ...irtualization-capabilities-libvirt-qemu.go | 6 + pkg/virt-handler/node-labeller/BUILD.bazel | 2 +- pkg/virt-handler/node-labeller/cpu_plugin.go | 122 +----------------- .../node-labeller/node_labeller.go | 103 ++++----------- .../virt-capabilities/types.go | 2 + 7 files changed, 56 insertions(+), 195 deletions(-) diff --git a/cmd/virt-handler/BUILD.bazel b/cmd/virt-handler/BUILD.bazel index 43883d670fd2..75af5bbcc621 100644 --- a/cmd/virt-handler/BUILD.bazel +++ b/cmd/virt-handler/BUILD.bazel @@ -43,6 +43,7 @@ go_library( "//pkg/virt-handler/seccomp:go_default_library", "//pkg/virt-handler/selinux:go_default_library", "//pkg/virt-handler/vsock:go_default_library", + "//pkg/virt-launcher-common/virt-capabilities:go_default_library", "//staging/src/kubevirt.io/api/core/v1:go_default_library", "//staging/src/kubevirt.io/client-go/kubecli:go_default_library", "//staging/src/kubevirt.io/client-go/log:go_default_library", diff --git a/cmd/virt-handler/virt-handler.go b/cmd/virt-handler/virt-handler.go index 7d92fc5ce77e..97c74fdcc642 100644 --- a/cmd/virt-handler/virt-handler.go +++ b/cmd/virt-handler/virt-handler.go @@ -22,6 +22,7 @@ package main import ( "context" "crypto/tls" + "encoding/json" "fmt" "net/http" "os" @@ -83,6 +84,7 @@ import ( nodelabeller "kubevirt.io/kubevirt/pkg/virt-handler/node-labeller" "kubevirt.io/kubevirt/pkg/virt-handler/rest" "kubevirt.io/kubevirt/pkg/virt-handler/selinux" + virt_capabilities "kubevirt.io/kubevirt/pkg/virt-launcher-common/virt-capabilities" ) const ( @@ -290,6 +292,16 @@ func (app *virtHandlerApp) Run() { stop := make(chan struct{}) defer close(stop) + + var virtCaps virt_capabilities.VirtualizationCapabilities + virtCapsFile, err := os.ReadFile(filepath.Join(nodelabeller.NodeLabellerVolumePath, "virtualization_capabilities.json")) + if err != nil { + panic(err) + } + if err := json.Unmarshal(virtCapsFile, &virtCaps); err != nil { + panic(err) + } + var capabilities libvirtxml.Caps var hostCpuModel string @@ -307,8 +319,7 @@ func (app *virtHandlerApp) Run() { app.virtCli.CoreV1().Nodes(), app.HostOverride, nodeLabellerrecorder, - capabilities.Host.CPU.Counter, - capabilities.Guests, + virtCaps, ) if err != nil { panic(err) diff --git a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go index e71d6ce0bc7f..ac32ed38c54f 100644 --- a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go +++ b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go @@ -149,6 +149,7 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedCpuModels() []string hostCpuModel := mode.Model[0] v.hostCPUModel.Name = hostCpuModel.Name v.hostCPUModel.Fallback = hostCpuModel.Fallback + v.hostCPUModel.Vendor = v.cpuModelVendor for _, feature := range mode.Feature { if feature.Policy == isRequired { @@ -178,6 +179,10 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) GetHostCpuModelInfo() virt_capab // GetSupportedCpuFeatures returns a dummy list of supported CPU features. func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedCpuFeatures() []string { + // TODO The below condition was in the virt-handlr code. Implementation should check which architecture this is and based on that expose SupportedHostFeatures. + // host supported features is only available on AMD64 and S390X nodes. + // This is because hypervisor-cpu-baseline virsh command doesnt work for ARM64 architecture. + // if n.arch.hasHostSupportedFeatures() { return v.SupportedHostFeatures } @@ -204,6 +209,7 @@ func (v *VirtualizationCapabilitiesLibvirtQemu) NodeSupportsRealTime() bool { // GetNodeSevFeatures returns a dummy list of SEV features. func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeSevFeatures() virt_capabilities.SEVConfiguration { sevCfg := virt_capabilities.SEVConfiguration{ + Supported: v.HostDomCapabilities.SEV.Supported, SupportedES: v.HostDomCapabilities.SEV.SupportedES, } return sevCfg diff --git a/pkg/virt-handler/node-labeller/BUILD.bazel b/pkg/virt-handler/node-labeller/BUILD.bazel index de1b24306422..2fb5ae6c5d4d 100644 --- a/pkg/virt-handler/node-labeller/BUILD.bazel +++ b/pkg/virt-handler/node-labeller/BUILD.bazel @@ -21,6 +21,7 @@ go_library( "//pkg/apimachinery/patch:go_default_library", "//pkg/virt-config:go_default_library", "//pkg/virt-handler/node-labeller/util:go_default_library", + "//pkg/virt-launcher-common/virt-capabilities:go_default_library", "//staging/src/kubevirt.io/api/core/v1:go_default_library", "//staging/src/kubevirt.io/client-go/log:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", @@ -31,7 +32,6 @@ go_library( "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//vendor/k8s.io/client-go/tools/record:go_default_library", "//vendor/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/libvirt.org/go/libvirtxml:go_default_library", ], ) diff --git a/pkg/virt-handler/node-labeller/cpu_plugin.go b/pkg/virt-handler/node-labeller/cpu_plugin.go index 185fb193172e..195d3370232c 100644 --- a/pkg/virt-handler/node-labeller/cpu_plugin.go +++ b/pkg/virt-handler/node-labeller/cpu_plugin.go @@ -20,15 +20,8 @@ package nodelabeller import ( - "encoding/xml" - "fmt" - "os" - "path/filepath" - - v1 "kubevirt.io/api/core/v1" - "kubevirt.io/client-go/log" - "kubevirt.io/kubevirt/pkg/virt-handler/node-labeller/util" + virt_capabilities "kubevirt.io/kubevirt/pkg/virt-launcher-common/virt-capabilities" ) const ( @@ -36,8 +29,6 @@ const ( isUnusable string = "no" isRequired string = "require" NodeLabellerVolumePath = "/var/lib/kubevirt-node-labeller/" - - supportedFeaturesXml = "supported_features.xml" ) func (n *NodeLabeller) getSupportedCpuModels(obsoleteCPUsx86 map[string]bool) []string { @@ -47,7 +38,7 @@ func (n *NodeLabeller) getSupportedCpuModels(obsoleteCPUsx86 map[string]bool) [] obsoleteCPUsx86 = util.DefaultObsoleteCPUModels } - for _, model := range n.hostCapabilities.items { + for _, model := range n.virtCaps.SupportedCPUModels { if _, ok := obsoleteCPUsx86[model]; ok { continue } @@ -60,116 +51,13 @@ func (n *NodeLabeller) getSupportedCpuModels(obsoleteCPUsx86 map[string]bool) [] func (n *NodeLabeller) getSupportedCpuFeatures() cpuFeatures { supportedCpuFeatures := make(cpuFeatures) - for _, feature := range n.supportedFeatures { + for _, feature := range n.virtCaps.SupportedCpuFeatures { supportedCpuFeatures[feature] = true } return supportedCpuFeatures } -func (n *NodeLabeller) GetHostCpuModel() hostCPUModel { - return n.hostCPUModel -} - -// loadDomCapabilities loads info about cpu models, which can host emulate -func (n *NodeLabeller) loadDomCapabilities() error { - hostDomCapabilities, err := n.getDomCapabilities() - if err != nil { - return err - } - - usableModels := make([]string, 0) - for _, mode := range hostDomCapabilities.CPU.Mode { - if mode.Name == v1.CPUModeHostModel { - if !n.arch.supportsHostModel() { - log.Log.Warningf("host-model cpu mode is not supported for %s architecture", n.arch.arch()) - continue - } - - n.cpuModelVendor = mode.Vendor.Name - if n.cpuModelVendor == "" { - n.cpuModelVendor = n.arch.defaultVendor() - } - - if len(mode.Model) < 1 { - return fmt.Errorf("host model mode is expected to contain a model") - } - if len(mode.Model) > 1 { - log.Log.Warning("host model mode is expected to contain only one model") - } - - hostCpuModel := mode.Model[0] - n.hostCPUModel.Name = hostCpuModel.Name - n.hostCPUModel.fallback = hostCpuModel.Fallback - - for _, feature := range mode.Feature { - if feature.Policy == isRequired { - n.hostCPUModel.requiredFeatures[feature.Name] = true - } - } - } - - for _, model := range mode.Model { - if model.Usable == isUnusable || model.Usable == "" { - continue - } - usableModels = append(usableModels, model.Name) - } - } - - n.hostCapabilities.items = usableModels - n.SEV = hostDomCapabilities.SEV - - return nil -} - -// loadHostSupportedFeatures loads supported features -func (n *NodeLabeller) loadHostSupportedFeatures() error { - featuresFile := filepath.Join(n.volumePath, supportedFeaturesXml) - - hostFeatures := SupportedHostFeature{} - err := n.getStructureFromXMLFile(featuresFile, &hostFeatures) - if err != nil { - return err - } - - usableFeatures := make([]string, 0) - for _, f := range hostFeatures.Feature { - if n.arch.requirePolicy(f.Policy) { - usableFeatures = append(usableFeatures, f.Name) - } - } - - n.supportedFeatures = usableFeatures - return nil -} - -func (n *NodeLabeller) getDomCapabilities() (HostDomCapabilities, error) { - domCapabilitiesFile := filepath.Join(n.volumePath, n.domCapabilitiesFileName) - hostDomCapabilities := HostDomCapabilities{} - err := n.getStructureFromXMLFile(domCapabilitiesFile, &hostDomCapabilities) - if err != nil { - return hostDomCapabilities, err - } - - if hostDomCapabilities.SEV.Supported == "yes" && hostDomCapabilities.SEV.MaxESGuests > 0 { - hostDomCapabilities.SEV.SupportedES = "yes" - } else { - hostDomCapabilities.SEV.SupportedES = "no" - } - - return hostDomCapabilities, err -} - -// GetStructureFromXMLFile load data from xml file and unmarshals them into given structure -// Given structure has to be pointer -func (n *NodeLabeller) getStructureFromXMLFile(path string, structure interface{}) error { - rawFile, err := os.ReadFile(path) - if err != nil { - return err - } - - n.logger.V(4).Infof("node-labeller - loading data from xml file: %#v", string(rawFile)) - - return xml.Unmarshal(rawFile, structure) +func (n *NodeLabeller) GetHostCpuModel() virt_capabilities.HostCPUModel { + return n.virtCaps.HostCpuModelInfo } diff --git a/pkg/virt-handler/node-labeller/node_labeller.go b/pkg/virt-handler/node-labeller/node_labeller.go index a7442c0a7b93..bb98ff2cccb7 100644 --- a/pkg/virt-handler/node-labeller/node_labeller.go +++ b/pkg/virt-handler/node-labeller/node_labeller.go @@ -28,7 +28,6 @@ import ( "time" "k8s.io/client-go/tools/record" - "libvirt.org/go/libvirtxml" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -43,6 +42,7 @@ import ( "kubevirt.io/kubevirt/pkg/apimachinery/patch" virtconfig "kubevirt.io/kubevirt/pkg/virt-config" + virt_capabilities "kubevirt.io/kubevirt/pkg/virt-launcher-common/virt-capabilities" ) var nodeLabellerLabels = []string{ @@ -62,30 +62,23 @@ var nodeLabellerLabels = []string{ // NodeLabeller struct holds information needed to run node-labeller type NodeLabeller struct { - recorder record.EventRecorder - nodeClient k8scli.NodeInterface - host string - logger *log.FilteredLogger - clusterConfig *virtconfig.ClusterConfig - hypervFeatures supportedFeatures - hostCapabilities supportedFeatures - queue workqueue.TypedRateLimitingInterface[string] - supportedFeatures []string - cpuModelVendor string - volumePath string - domCapabilitiesFileName string - cpuCounter *libvirtxml.CapsHostCPUCounter - guestCaps []libvirtxml.CapsGuest - hostCPUModel hostCPUModel - SEV SEVConfiguration - arch archLabeller + recorder record.EventRecorder + nodeClient k8scli.NodeInterface + host string + logger *log.FilteredLogger + clusterConfig *virtconfig.ClusterConfig + hypervFeatures supportedFeatures + queue workqueue.TypedRateLimitingInterface[string] + volumePath string + virtCaps virt_capabilities.VirtualizationCapabilities + arch archLabeller } -func NewNodeLabeller(clusterConfig *virtconfig.ClusterConfig, nodeClient k8scli.NodeInterface, host string, recorder record.EventRecorder, cpuCounter *libvirtxml.CapsHostCPUCounter, guestCaps []libvirtxml.CapsGuest) (*NodeLabeller, error) { - return newNodeLabeller(clusterConfig, nodeClient, host, NodeLabellerVolumePath, recorder, cpuCounter, guestCaps) +func NewNodeLabeller(clusterConfig *virtconfig.ClusterConfig, nodeClient k8scli.NodeInterface, host string, recorder record.EventRecorder, virtCaps virt_capabilities.VirtualizationCapabilities) (*NodeLabeller, error) { + return newNodeLabeller(clusterConfig, nodeClient, host, NodeLabellerVolumePath, recorder, virtCaps) } -func newNodeLabeller(clusterConfig *virtconfig.ClusterConfig, nodeClient k8scli.NodeInterface, host, volumePath string, recorder record.EventRecorder, cpuCounter *libvirtxml.CapsHostCPUCounter, guestCaps []libvirtxml.CapsGuest) (*NodeLabeller, error) { +func newNodeLabeller(clusterConfig *virtconfig.ClusterConfig, nodeClient k8scli.NodeInterface, host, volumePath string, recorder record.EventRecorder, virtCaps virt_capabilities.VirtualizationCapabilities) (*NodeLabeller, error) { n := &NodeLabeller{ recorder: recorder, nodeClient: nodeClient, @@ -96,18 +89,11 @@ func newNodeLabeller(clusterConfig *virtconfig.ClusterConfig, nodeClient k8scli. workqueue.DefaultTypedControllerRateLimiter[string](), workqueue.TypedRateLimitingQueueConfig[string]{Name: "virt-handler-node-labeller"}, ), - volumePath: volumePath, - domCapabilitiesFileName: "virsh_domcapabilities.xml", - cpuCounter: cpuCounter, - guestCaps: guestCaps, - hostCPUModel: hostCPUModel{requiredFeatures: make(map[string]bool)}, - arch: newArchLabeller(runtime.GOARCH), + volumePath: volumePath, + virtCaps: virtCaps, + arch: newArchLabeller(runtime.GOARCH), } - err := n.loadAll() - if err != nil { - return n, err - } return n, nil } @@ -157,28 +143,6 @@ func (n *NodeLabeller) execute() bool { return true } -func (n *NodeLabeller) loadAll() error { - // host supported features is only available on AMD64 and S390X nodes. - // This is because hypervisor-cpu-baseline virsh command doesnt work for ARM64 architecture. - if n.arch.hasHostSupportedFeatures() { - err := n.loadHostSupportedFeatures() - if err != nil { - n.logger.Errorf("node-labeller could not load supported features: " + err.Error()) - return err - } - } - - err := n.loadDomCapabilities() - if err != nil { - n.logger.Errorf("node-labeller could not load host dom capabilities: " + err.Error()) - return err - } - - n.loadHypervFeatures() - - return nil -} - func (n *NodeLabeller) run() error { originalNode, err := n.nodeClient.Get(context.Background(), n.host, metav1.GetOptions{}) if err != nil { @@ -224,6 +188,7 @@ func (n *NodeLabeller) patchNode(originalNode, node *v1.Node) error { return err } +// TODO: Need to implement this in the Libvirt-QEMU-KVM virtualization capabilities exporter func (n *NodeLabeller) loadHypervFeatures() { n.hypervFeatures.items = getCapLabels() } @@ -248,8 +213,8 @@ func (n *NodeLabeller) prepareLabels(node *v1.Node) map[string]string { } } - for _, machine := range n.getSupportedMachines() { - labelKey := kubevirtv1.SupportedMachineTypeLabel + machine.Name + for _, machine := range n.virtCaps.SupportedMachineTypes { + labelKey := kubevirtv1.SupportedMachineTypeLabel + machine newLabels[labelKey] = "true" } @@ -258,8 +223,8 @@ func (n *NodeLabeller) prepareLabels(node *v1.Node) map[string]string { } if n.hasTSCCounter() { - newLabels[kubevirtv1.CPUTimerLabel+"tsc-frequency"] = fmt.Sprintf("%d", n.cpuCounter.Frequency) - newLabels[kubevirtv1.CPUTimerLabel+"tsc-scalable"] = fmt.Sprintf("%t", n.cpuCounter.Scaling == "yes") + newLabels[kubevirtv1.CPUTimerLabel+"tsc-frequency"] = n.virtCaps.NodeTscInfo.Frequency + newLabels[kubevirtv1.CPUTimerLabel+"tsc-scalable"] = n.virtCaps.NodeTscInfo.Scalable } if n.arch.supportsHostModel() { @@ -273,27 +238,23 @@ func (n *NodeLabeller) prepareLabels(node *v1.Node) map[string]string { } } - for feature := range hostCpuModel.requiredFeatures { + for _, feature := range hostCpuModel.RequiredFeatures { newLabels[kubevirtv1.HostModelRequiredFeaturesLabel+feature] = "true" } - newLabels[kubevirtv1.CPUModelVendorLabel+n.cpuModelVendor] = "true" + newLabels[kubevirtv1.CPUModelVendorLabel+n.virtCaps.HostCpuModelInfo.Vendor] = "true" newLabels[kubevirtv1.HostModelCPULabel+hostCpuModel.Name] = "true" } - capable, err := isNodeRealtimeCapable() - if err != nil { - n.logger.Reason(err).Error("failed to identify if a node is capable of running realtime workloads") - } - if capable { + if n.virtCaps.NodeSupportsRealTime { newLabels[kubevirtv1.RealtimeLabel] = "" } - if n.SEV.Supported == "yes" { + if n.virtCaps.NodeSevFeatures.Supported == "yes" { newLabels[kubevirtv1.SEVLabel] = "" } - if n.SEV.SupportedES == "yes" { + if n.virtCaps.NodeSevFeatures.SupportedES == "yes" { newLabels[kubevirtv1.SEVESLabel] = "" } @@ -349,13 +310,5 @@ func (n *NodeLabeller) alertIfHostModelIsObsolete(originalNode *v1.Node, hostMod } func (n *NodeLabeller) hasTSCCounter() bool { - return n.cpuCounter != nil && n.cpuCounter.Name == "tsc" -} - -func (n *NodeLabeller) getSupportedMachines() []libvirtxml.CapsGuestMachine { - var supportedMachines []libvirtxml.CapsGuestMachine - for _, guest := range n.guestCaps { - supportedMachines = append(supportedMachines, guest.Arch.Machines...) - } - return supportedMachines + return n.virtCaps.NodeTscInfo.HasTscCounter } diff --git a/pkg/virt-launcher-common/virt-capabilities/types.go b/pkg/virt-launcher-common/virt-capabilities/types.go index 6d21fb4bcfbf..f3075fd2f40d 100644 --- a/pkg/virt-launcher-common/virt-capabilities/types.go +++ b/pkg/virt-launcher-common/virt-capabilities/types.go @@ -4,6 +4,7 @@ type HostCPUModel struct { Name string `json:"name"` Fallback string `json:"fallback"` RequiredFeatures []string `json:"requiredFeatures"` + Vendor string `json:"vendor"` } type TscConfig struct { @@ -13,6 +14,7 @@ type TscConfig struct { } type SEVConfiguration struct { + Supported string SupportedES string } From 521f00b10dc424f857f9d4fd867bbf842bb1ce48 Mon Sep 17 00:00:00 2001 From: Harshit Gupta Date: Tue, 10 Jun 2025 11:09:44 -0700 Subject: [PATCH 12/13] Remove old node-labeller ctr from handler pod --- .../generate/components/daemonsets.go | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/pkg/virt-operator/resource/generate/components/daemonsets.go b/pkg/virt-operator/resource/generate/components/daemonsets.go index 05015058c5f8..1af121339a85 100644 --- a/pkg/virt-operator/resource/generate/components/daemonsets.go +++ b/pkg/virt-operator/resource/generate/components/daemonsets.go @@ -131,27 +131,6 @@ func NewHandlerDaemonSet(namespace, repository, imagePrefix, version, launcherVe // nodelabeller currently only support x86. The arch check will be done in node-labller.sh pod.InitContainers = []corev1.Container{ - { - Command: []string{ - "/bin/sh", - "-c", - }, - Image: launcherImage, - Name: "virt-launcher", - Args: []string{ - "node-labeller.sh", - }, - SecurityContext: &corev1.SecurityContext{ - Privileged: pointer.P(true), - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "node-labeller", - MountPath: nodeLabellerVolumePath, - }, - }, - TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError, - }, { Command: []string{ "/bin/sh", From f04c3da26eae00817cd43d60ecedf99f8736b8d0 Mon Sep 17 00:00:00 2001 From: Harshit Gupta Date: Wed, 11 Jun 2025 07:58:47 -0700 Subject: [PATCH 13/13] Remove unnecessary Capabilities map keys --- .../virtualization-capabilities-interface.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/pkg/virt-launcher-common/virt-capabilities/virtualization-capabilities-interface.go b/pkg/virt-launcher-common/virt-capabilities/virtualization-capabilities-interface.go index f15707c7127b..c20bfa6cf66d 100644 --- a/pkg/virt-launcher-common/virt-capabilities/virtualization-capabilities-interface.go +++ b/pkg/virt-launcher-common/virt-capabilities/virtualization-capabilities-interface.go @@ -2,17 +2,6 @@ package virt_capabilities const KernelSchedRealtimeRuntimeInMicroseconds = "kernel.sched_rt_runtime_us" -const ( - HypervFeaturesKey = "hyperv-features" - SupportedMachineTypeKeys = "supported-machine-types" - SupportedCpuModelsKey = "supported-cpu-models" - HostCpuModelInfoKey = "host-cpu-model-info" - SupportedCpuFeaturesKey = "supported-cpu-features" - NodeTscInfoKey = "node-tsc-info" - NodeSupportsRealTimeKey = "node-supports-real-time" - NodeSevFeaturesKey = "node-sev-features" -) - // VirtualizationCapabilitiesInterface defines methods for querying virtualization capabilities. type VirtualizationCapabilitiesInterface interface { // GetHypervFeatures returns a list of features required for Windows guests.