diff --git a/BUILD.bazel b/BUILD.bazel index cf8237f75aaf..d8dc8a94383e 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -513,6 +513,16 @@ container_push( tag = "$(container_tag)", ) +genrule( + name = "build-node-labeller-go", + srcs = [ + "//cmd/virt-launcher/node-labeller-go", + ], + outs = ["node-labeller-go"], + cmd = "echo '#!/bin/sh\n\ncp -f $(SRCS) $$1' > \"$@\"", + executable = 1, +) + genrule( name = "build-virtctl", srcs = [ diff --git a/cmd/virt-handler/BUILD.bazel b/cmd/virt-handler/BUILD.bazel index 43883d670fd2..75af5bbcc621 100644 --- a/cmd/virt-handler/BUILD.bazel +++ b/cmd/virt-handler/BUILD.bazel @@ -43,6 +43,7 @@ go_library( "//pkg/virt-handler/seccomp:go_default_library", "//pkg/virt-handler/selinux:go_default_library", "//pkg/virt-handler/vsock:go_default_library", + "//pkg/virt-launcher-common/virt-capabilities:go_default_library", "//staging/src/kubevirt.io/api/core/v1:go_default_library", "//staging/src/kubevirt.io/client-go/kubecli:go_default_library", "//staging/src/kubevirt.io/client-go/log:go_default_library", diff --git a/cmd/virt-handler/virt-handler.go b/cmd/virt-handler/virt-handler.go index 7d92fc5ce77e..97c74fdcc642 100644 --- a/cmd/virt-handler/virt-handler.go +++ b/cmd/virt-handler/virt-handler.go @@ -22,6 +22,7 @@ package main import ( "context" "crypto/tls" + "encoding/json" "fmt" "net/http" "os" @@ -83,6 +84,7 @@ import ( nodelabeller "kubevirt.io/kubevirt/pkg/virt-handler/node-labeller" "kubevirt.io/kubevirt/pkg/virt-handler/rest" "kubevirt.io/kubevirt/pkg/virt-handler/selinux" + virt_capabilities "kubevirt.io/kubevirt/pkg/virt-launcher-common/virt-capabilities" ) const ( @@ -290,6 +292,16 @@ func (app *virtHandlerApp) Run() { stop := make(chan struct{}) defer close(stop) + + var virtCaps virt_capabilities.VirtualizationCapabilities + virtCapsFile, err := os.ReadFile(filepath.Join(nodelabeller.NodeLabellerVolumePath, "virtualization_capabilities.json")) + if err != nil { + panic(err) + } + if err := json.Unmarshal(virtCapsFile, &virtCaps); err != nil { + panic(err) + } + var capabilities libvirtxml.Caps var hostCpuModel string @@ -307,8 +319,7 @@ func (app *virtHandlerApp) Run() { app.virtCli.CoreV1().Nodes(), app.HostOverride, nodeLabellerrecorder, - capabilities.Host.CPU.Counter, - capabilities.Guests, + virtCaps, ) if err != nil { panic(err) diff --git a/cmd/virt-launcher/BUILD.bazel b/cmd/virt-launcher/BUILD.bazel index 47456736ae59..c2d2a4863344 100644 --- a/cmd/virt-launcher/BUILD.bazel +++ b/cmd/virt-launcher/BUILD.bazel @@ -62,6 +62,7 @@ pkg_tar( "//cmd/container-disk-v2alpha:container-disk", "//cmd/virt-freezer", "//cmd/virt-launcher-monitor", + "//cmd/virt-launcher/node-labeller-go", "//cmd/virt-probe", "//cmd/virt-tail", ], diff --git a/cmd/virt-launcher/node-labeller-go/BUILD.bazel b/cmd/virt-launcher/node-labeller-go/BUILD.bazel new file mode 100644 index 000000000000..0fa9010cb641 --- /dev/null +++ b/cmd/virt-launcher/node-labeller-go/BUILD.bazel @@ -0,0 +1,23 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "node-labeller.go", + "types.go", + "virtualization-capabilities-libvirt-qemu.go", + ], + importpath = "kubevirt.io/kubevirt/cmd/virt-launcher/node-labeller-go", + visibility = ["//visibility:private"], + deps = [ + "//pkg/virt-launcher-common/virt-capabilities:go_default_library", + "//staging/src/kubevirt.io/api/core/v1:go_default_library", + "//vendor/libvirt.org/go/libvirtxml:go_default_library", + ], +) + +go_binary( + name = "node-labeller-go", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/cmd/virt-launcher/node-labeller-go/node-labeller.go b/cmd/virt-launcher/node-labeller-go/node-labeller.go new file mode 100644 index 000000000000..902ab2df563c --- /dev/null +++ b/cmd/virt-launcher/node-labeller-go/node-labeller.go @@ -0,0 +1,125 @@ +package main + +import ( + "fmt" + "os/exec" + "strings" + "time" + + virt_capabilities "kubevirt.io/kubevirt/pkg/virt-launcher-common/virt-capabilities" +) + +const ( + XmlBasePath = "/var/lib/kubevirt-node-labeller" +) + +func executeCommand(command string) (string, error) { + cmd := exec.Command("bash", "-c", command) + output, err := cmd.Output() + outputStr := strings.TrimSpace(string(output)) + if err != nil { + return "", fmt.Errorf("failed to execute command %s: %w", command, err) + } + return outputStr, nil +} + +func queryMachineArchitecture() (string, error) { + return executeCommand("uname -m") +} + +func queryKvmMinor() (string, error) { + return executeCommand("grep -w 'kvm' /proc/misc | cut -f 1 -d' '") +} + +func main() { + + machine := "q35" + + arch, err := queryMachineArchitecture() + if err != nil { + fmt.Printf("Error querying machine architecture: %v\n", err) + return + } + + fmt.Printf("Detected arch=\"%s\"\n", arch) + + if arch == "aarch64" { + machine = "virt" + } else if arch == "s390x" { + machine = "390-ccw-virtio" + } else if arch != "x86_64" { + // Node labeling cannot proceed for this architecture + fmt.Printf("Node labeling cannot proceed for architecture: %s\n", arch) + return + } + + kvmMinor, err := queryKvmMinor() + if err != nil { + fmt.Printf("Error querying KVM minor: %v\n", err) + return + } + + virttype := "qemu" + + _, err = exec.Command("ls", "/dev/kvm").Output() + kvmExists := err == nil + + if !kvmExists && kvmMinor != "" { + executeCommand("mknod /dev/kvm c 10 " + kvmMinor) + } + + _, err = exec.Command("ls", "/dev/kvm").Output() + kvmExists = err == nil + + if kvmExists { + executeCommand("chmod o+rw /dev/kvm") + virttype = "kvm" + } + + _, err = exec.Command("ls", "/dev/sev").Output() + sevExists := err == nil + + if sevExists { + // QEMU requires RW access to query SEV capabilities + executeCommand("chmod o+rw /dev/kvm") + } + + cmd := exec.Command("virtqemud", "-d") + err = cmd.Start() + if err != nil { + fmt.Printf("Failed to start virtqemud: %v\n", err) + return + } + fmt.Println("virtqemud started in daemon mode") + + executeCommand(fmt.Sprintf("mkdir -p %s", XmlBasePath)) // TODO Remove this later, this is just for testing + + fmt.Println("Waiting for virtqemud to start...") + time.Sleep(5 * time.Second) // Wait for virtqemud to start + + _, err = executeCommand(fmt.Sprintf("virsh domcapabilities --machine %s --arch %s --virttype %s > %s/virsh_domcapabilities.xml", machine, arch, virttype, XmlBasePath)) + + if err != nil { + fmt.Printf("Failed to get domain capabilities: %v\n", err) + return + } + + if arch == "x86_64" || arch == "s390x" { + cmd := fmt.Sprintf("virsh domcapabilities --machine %s --arch %s --virttype %s | virsh hypervisor-cpu-baseline --features /dev/stdin --machine %s --arch %s --virttype %s > %s/supported_features.xml", machine, arch, virttype, machine, arch, virttype, XmlBasePath) + _, err := executeCommand(cmd) + if err != nil { + fmt.Printf("Failed to get supported features: %v\n", err) + return + } + } + + _, err = executeCommand(fmt.Sprintf("virsh capabilities > %s/capabilities.xml", XmlBasePath)) + if err != nil { + fmt.Printf("Failed to get node capabilities: %v\n", err) + return + } + + capabilityExtractor := NewVirtualizationCapabilitiesLibvirtQemu(fmt.Sprintf("%s/supported_features.xml", XmlBasePath), fmt.Sprintf("%s/virsh_domcapabilities.xml", XmlBasePath), fmt.Sprintf("%s/capabilities.xml", XmlBasePath)) + + virt_capabilities.ExportVirtualizationCapabilities(capabilityExtractor, fmt.Sprintf("%s/virtualization_capabilities.json", XmlBasePath)) +} diff --git a/cmd/virt-launcher/node-labeller-go/types.go b/cmd/virt-launcher/node-labeller-go/types.go new file mode 100644 index 000000000000..9566ef8dc6e9 --- /dev/null +++ b/cmd/virt-launcher/node-labeller-go/types.go @@ -0,0 +1,64 @@ +package main + +// HostDomCapabilities represents structure for parsing output of virsh capabilities +type HostDomCapabilities struct { + CPU CPU `xml:"cpu"` + SEV SEVConfiguration `xml:"features>sev"` +} + +// CPU represents slice of cpu modes +type CPU struct { + Mode []Mode `xml:"mode"` +} + +// Mode represents slice of cpu models +type Mode struct { + Name string `xml:"name,attr"` + Supported string `xml:"supported,attr"` + Vendor Vendor `xml:"vendor"` + Feature []HostFeature `xml:"feature"` + Model []Model `xml:"model"` +} + +type SupportedHostFeature struct { + Feature []HostFeature `xml:"feature"` +} + +type HostFeature struct { + Policy string `xml:"policy,attr"` + Name string `xml:"name,attr"` +} + +// Vendor represents vendor of host CPU +type Vendor struct { + Name string `xml:",chardata"` +} + +// Model represents cpu model +type Model struct { + Name string `xml:",chardata"` + Usable string `xml:"usable,attr"` + Fallback string `xml:"fallback,attr"` +} + +// Structures needed to parse cpu features +type FeatureModel struct { + Model Features `xml:"model"` +} + +type Features struct { + Features []Feature `xml:"feature"` +} + +type Feature struct { + Name string `xml:"name,attr"` +} + +type SEVConfiguration struct { + Supported string `xml:"supported,attr"` + CBitPos uint `xml:"cbitpos"` + ReducedPhysBits uint `xml:"reducedPhysBits"` + MaxGuests uint `xml:"maxGuests"` + MaxESGuests uint `xml:"maxESGuests"` + SupportedES string `xml:"-"` +} diff --git a/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go new file mode 100644 index 000000000000..ac32ed38c54f --- /dev/null +++ b/cmd/virt-launcher/node-labeller-go/virtualization-capabilities-libvirt-qemu.go @@ -0,0 +1,243 @@ +package main + +import ( + "encoding/xml" + "fmt" + "os" + "os/exec" + "strings" + + v1 "kubevirt.io/api/core/v1" + "libvirt.org/go/libvirtxml" + + virt_capabilities "kubevirt.io/kubevirt/pkg/virt-launcher-common/virt-capabilities" +) + +const ( + isSupported string = "yes" + isUnusable string = "no" + isRequired string = "require" +) + +// VirtualizationCapabilitiesLibvirtQemu is a dummy implementation of VirtualizationCapabilitiesInterface. +type VirtualizationCapabilitiesLibvirtQemu struct { + // supportedFeatures.xml path + SupportedFeaturesPath string + // domainCapabilities.xml path + DomainCapabilitiesPath string + // capabilities.xml path + CapabilitiesPath string + + HostDomCapabilities HostDomCapabilities + SupportedHostFeatures []string + NodeCapabilities libvirtxml.Caps + + cpuModelVendor string + + hostCPUModel virt_capabilities.HostCPUModel +} + +func NewVirtualizationCapabilitiesLibvirtQemu(supportedFeaturesPath string, domainCapabilitiesPath string, capabilitiesPath string) *VirtualizationCapabilitiesLibvirtQemu { + cap := VirtualizationCapabilitiesLibvirtQemu{ + SupportedFeaturesPath: supportedFeaturesPath, + DomainCapabilitiesPath: domainCapabilitiesPath, + CapabilitiesPath: capabilitiesPath} + cap.loadAll() + return &cap +} + +func (v *VirtualizationCapabilitiesLibvirtQemu) loadAll() { + v.loadSupportedFeatures() + v.loadDomainCapabilities() + v.loadCapabilities() +} + +func (v *VirtualizationCapabilitiesLibvirtQemu) loadSupportedFeatures() { + hostFeatures := SupportedHostFeature{} + err := v.getStructureFromXMLFile(v.SupportedFeaturesPath, &hostFeatures) + if err != nil { + fmt.Printf("Error loading supported features: %v\n", err) + panic(err) + } + + usableFeatures := make([]string, 0) + for _, f := range hostFeatures.Feature { + if f.Policy == isRequired { + usableFeatures = append(usableFeatures, f.Name) + } + } + + v.SupportedHostFeatures = usableFeatures +} + +func (v *VirtualizationCapabilitiesLibvirtQemu) loadDomainCapabilities() { + hostDomCapabilities := HostDomCapabilities{} + err := v.getStructureFromXMLFile(v.DomainCapabilitiesPath, &hostDomCapabilities) + if err != nil { + fmt.Printf("Error loading domain capabilities: %v\n", err) + panic(err) + } + + if hostDomCapabilities.SEV.Supported == "yes" && hostDomCapabilities.SEV.MaxESGuests > 0 { + hostDomCapabilities.SEV.SupportedES = "yes" + } else { + hostDomCapabilities.SEV.SupportedES = "no" + } + + v.HostDomCapabilities = hostDomCapabilities +} + +func (v *VirtualizationCapabilitiesLibvirtQemu) loadCapabilities() { + capabilities := libvirtxml.Caps{} + err := v.getStructureFromXMLFile(v.CapabilitiesPath, &capabilities) + if err != nil { + panic(fmt.Sprintf("Error loading capabilities: %v\n", err)) + } + v.NodeCapabilities = capabilities +} + +// GetHypervFeatures returns a dummy list of Hyper-V features. +func (v *VirtualizationCapabilitiesLibvirtQemu) GetHypervFeatures() []string { + // TODO Query actual Hyper-V features from /dev/kvm + return []string{"hv_relaxed", "hv_vapic"} +} + +// GetNodeTopology returns a dummy node topology. +func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeTopology() interface{} { + // TODO Get actual node topology + return map[string]interface{}{"sockets": 1, "cores": 2, "threads": 2} +} + +// GetSupportedMachineTypes returns a dummy list of supported machine types. +func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedMachineTypes() []string { + var supportedMachines []string + for _, guest := range v.NodeCapabilities.Guests { + fmt.Println("Guest architecture: ", guest.Arch.Name) + for _, machine := range guest.Arch.Machines { + supportedMachines = append(supportedMachines, machine.Name) + fmt.Println("Adding machine type: ", machine.Name) + } + } + return supportedMachines +} + +// GetSupportedCpuModels returns a dummy list of supported CPU models. +func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedCpuModels() []string { + // TODO Incorporate obsolete CPU models logic. + // TODO This can also be done in the virt-handler itself. + + usableModels := make([]string, 0) + for _, mode := range v.HostDomCapabilities.CPU.Mode { + if mode.Name == v1.CPUModeHostModel { + if !true { // TODO This needs to be factored on!n.arch.supportsHostModel() { + fmt.Printf("host-model cpu mode is not supported for %s architecture", "TODO") + continue + } + + v.cpuModelVendor = mode.Vendor.Name + if v.cpuModelVendor == "" { + v.cpuModelVendor = "Intel" // TODO n.arch.defaultVendor() + } + + if len(mode.Model) < 1 { + panic("host model mode is expected to contain a model") + } + if len(mode.Model) > 1 { + panic("host model mode is expected to contain only one model") + } + + hostCpuModel := mode.Model[0] + v.hostCPUModel.Name = hostCpuModel.Name + v.hostCPUModel.Fallback = hostCpuModel.Fallback + v.hostCPUModel.Vendor = v.cpuModelVendor + + for _, feature := range mode.Feature { + if feature.Policy == isRequired { + v.hostCPUModel.RequiredFeatures = append(v.hostCPUModel.RequiredFeatures, feature.Name) + } + } + + fmt.Println("Host CPU Model : ", v.hostCPUModel) + + } + + for _, model := range mode.Model { + if model.Usable == isUnusable || model.Usable == "" { + continue + } + usableModels = append(usableModels, model.Name) + } + } + + return usableModels +} + +// GetHostCpuModelInfo returns dummy host CPU model information. +func (v *VirtualizationCapabilitiesLibvirtQemu) GetHostCpuModelInfo() virt_capabilities.HostCPUModel { + return v.hostCPUModel +} + +// GetSupportedCpuFeatures returns a dummy list of supported CPU features. +func (v *VirtualizationCapabilitiesLibvirtQemu) GetSupportedCpuFeatures() []string { + // TODO The below condition was in the virt-handlr code. Implementation should check which architecture this is and based on that expose SupportedHostFeatures. + // host supported features is only available on AMD64 and S390X nodes. + // This is because hypervisor-cpu-baseline virsh command doesnt work for ARM64 architecture. + // if n.arch.hasHostSupportedFeatures() { + return v.SupportedHostFeatures +} + +// GetNodeTscInfo returns dummy node TSC information. +func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeTscInfo() virt_capabilities.TscConfig { + counter := v.NodeCapabilities.Host.CPU.Counter + if counter != nil && counter.Name == "tsc" { + return virt_capabilities.TscConfig{ + HasTscCounter: true, + Frequency: fmt.Sprintf("%d", counter.Frequency), + Scalable: fmt.Sprintf("%t", counter.Scaling == "yes"), + } + } + return virt_capabilities.TscConfig{ + HasTscCounter: false} +} + +// NodeSupportsRealTime returns a dummy value indicating real-time support. +func (v *VirtualizationCapabilitiesLibvirtQemu) NodeSupportsRealTime() bool { + isNodeRealtimeCapable, _ := isNodeRealtimeCapable() + return isNodeRealtimeCapable +} + +// GetNodeSevFeatures returns a dummy list of SEV features. +func (v *VirtualizationCapabilitiesLibvirtQemu) GetNodeSevFeatures() virt_capabilities.SEVConfiguration { + sevCfg := virt_capabilities.SEVConfiguration{ + Supported: v.HostDomCapabilities.SEV.Supported, + SupportedES: v.HostDomCapabilities.SEV.SupportedES, + } + return sevCfg +} + +// GetStructureFromXMLFile load data from xml file and unmarshals them into given structure +// Given structure has to be pointer +func (v *VirtualizationCapabilitiesLibvirtQemu) getStructureFromXMLFile(path string, structure interface{}) error { + rawFile, err := os.ReadFile(path) + if err != nil { + return err + } + + //fmt.Printf("node-labeller - loading data from xml file: %#v\n", string(rawFile)) + + return xml.Unmarshal(rawFile, structure) +} + +// isNodeRealtimeCapable Checks if a node is capable of running realtime workloads. Currently by validating if the kernel system setting value +// for `kernel.sched_rt_runtime_us` is set to allow running realtime scheduling with unlimited time (==-1) +// TODO: This part should be improved to validate against key attributes that determine best if a host is able to run realtime +// workloads at peak performance. + +func isNodeRealtimeCapable() (bool, error) { + ret, err := exec.Command("sysctl", virt_capabilities.KernelSchedRealtimeRuntimeInMicroseconds).CombinedOutput() + if err != nil { + return false, err + } + st := strings.Trim(string(ret), "\n") + return fmt.Sprintf("%s = -1", virt_capabilities.KernelSchedRealtimeRuntimeInMicroseconds) == st, nil +} diff --git a/hack/bazel-build.sh b/hack/bazel-build.sh index b1baa39eda65..2a7e29b1e07d 100755 --- a/hack/bazel-build.sh +++ b/hack/bazel-build.sh @@ -25,6 +25,7 @@ source hack/config.sh rm -rf ${CMD_OUT_DIR} mkdir -p ${CMD_OUT_DIR}/virtctl +mkdir -p ${CMD_OUT_DIR}/node-labeller-go mkdir -p ${CMD_OUT_DIR}/dump mkdir -p ${CMD_OUT_DIR}/perfscale-audit mkdir -p ${CMD_OUT_DIR}/perfscale-load-generator @@ -66,6 +67,10 @@ bazel run \ --config="$(uname -m)" \ :build-virtctl -- ${CMD_OUT_DIR}/virtctl/virtctl +bazel run \ + --config="$(uname -m)" \ + :build-node-labeller-go -- ${CMD_OUT_DIR}/node-labeller-go/node-labeller-go + # Copy kubevirt-passt-binding binary to a reachable place outside of the build container bazel run \ --config=${ARCHITECTURE} \ diff --git a/pkg/virt-handler/node-labeller/BUILD.bazel b/pkg/virt-handler/node-labeller/BUILD.bazel index de1b24306422..2fb5ae6c5d4d 100644 --- a/pkg/virt-handler/node-labeller/BUILD.bazel +++ b/pkg/virt-handler/node-labeller/BUILD.bazel @@ -21,6 +21,7 @@ go_library( "//pkg/apimachinery/patch:go_default_library", "//pkg/virt-config:go_default_library", "//pkg/virt-handler/node-labeller/util:go_default_library", + "//pkg/virt-launcher-common/virt-capabilities:go_default_library", "//staging/src/kubevirt.io/api/core/v1:go_default_library", "//staging/src/kubevirt.io/client-go/log:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", @@ -31,7 +32,6 @@ go_library( "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//vendor/k8s.io/client-go/tools/record:go_default_library", "//vendor/k8s.io/client-go/util/workqueue:go_default_library", - "//vendor/libvirt.org/go/libvirtxml:go_default_library", ], ) diff --git a/pkg/virt-handler/node-labeller/cpu_plugin.go b/pkg/virt-handler/node-labeller/cpu_plugin.go index 185fb193172e..195d3370232c 100644 --- a/pkg/virt-handler/node-labeller/cpu_plugin.go +++ b/pkg/virt-handler/node-labeller/cpu_plugin.go @@ -20,15 +20,8 @@ package nodelabeller import ( - "encoding/xml" - "fmt" - "os" - "path/filepath" - - v1 "kubevirt.io/api/core/v1" - "kubevirt.io/client-go/log" - "kubevirt.io/kubevirt/pkg/virt-handler/node-labeller/util" + virt_capabilities "kubevirt.io/kubevirt/pkg/virt-launcher-common/virt-capabilities" ) const ( @@ -36,8 +29,6 @@ const ( isUnusable string = "no" isRequired string = "require" NodeLabellerVolumePath = "/var/lib/kubevirt-node-labeller/" - - supportedFeaturesXml = "supported_features.xml" ) func (n *NodeLabeller) getSupportedCpuModels(obsoleteCPUsx86 map[string]bool) []string { @@ -47,7 +38,7 @@ func (n *NodeLabeller) getSupportedCpuModels(obsoleteCPUsx86 map[string]bool) [] obsoleteCPUsx86 = util.DefaultObsoleteCPUModels } - for _, model := range n.hostCapabilities.items { + for _, model := range n.virtCaps.SupportedCPUModels { if _, ok := obsoleteCPUsx86[model]; ok { continue } @@ -60,116 +51,13 @@ func (n *NodeLabeller) getSupportedCpuModels(obsoleteCPUsx86 map[string]bool) [] func (n *NodeLabeller) getSupportedCpuFeatures() cpuFeatures { supportedCpuFeatures := make(cpuFeatures) - for _, feature := range n.supportedFeatures { + for _, feature := range n.virtCaps.SupportedCpuFeatures { supportedCpuFeatures[feature] = true } return supportedCpuFeatures } -func (n *NodeLabeller) GetHostCpuModel() hostCPUModel { - return n.hostCPUModel -} - -// loadDomCapabilities loads info about cpu models, which can host emulate -func (n *NodeLabeller) loadDomCapabilities() error { - hostDomCapabilities, err := n.getDomCapabilities() - if err != nil { - return err - } - - usableModels := make([]string, 0) - for _, mode := range hostDomCapabilities.CPU.Mode { - if mode.Name == v1.CPUModeHostModel { - if !n.arch.supportsHostModel() { - log.Log.Warningf("host-model cpu mode is not supported for %s architecture", n.arch.arch()) - continue - } - - n.cpuModelVendor = mode.Vendor.Name - if n.cpuModelVendor == "" { - n.cpuModelVendor = n.arch.defaultVendor() - } - - if len(mode.Model) < 1 { - return fmt.Errorf("host model mode is expected to contain a model") - } - if len(mode.Model) > 1 { - log.Log.Warning("host model mode is expected to contain only one model") - } - - hostCpuModel := mode.Model[0] - n.hostCPUModel.Name = hostCpuModel.Name - n.hostCPUModel.fallback = hostCpuModel.Fallback - - for _, feature := range mode.Feature { - if feature.Policy == isRequired { - n.hostCPUModel.requiredFeatures[feature.Name] = true - } - } - } - - for _, model := range mode.Model { - if model.Usable == isUnusable || model.Usable == "" { - continue - } - usableModels = append(usableModels, model.Name) - } - } - - n.hostCapabilities.items = usableModels - n.SEV = hostDomCapabilities.SEV - - return nil -} - -// loadHostSupportedFeatures loads supported features -func (n *NodeLabeller) loadHostSupportedFeatures() error { - featuresFile := filepath.Join(n.volumePath, supportedFeaturesXml) - - hostFeatures := SupportedHostFeature{} - err := n.getStructureFromXMLFile(featuresFile, &hostFeatures) - if err != nil { - return err - } - - usableFeatures := make([]string, 0) - for _, f := range hostFeatures.Feature { - if n.arch.requirePolicy(f.Policy) { - usableFeatures = append(usableFeatures, f.Name) - } - } - - n.supportedFeatures = usableFeatures - return nil -} - -func (n *NodeLabeller) getDomCapabilities() (HostDomCapabilities, error) { - domCapabilitiesFile := filepath.Join(n.volumePath, n.domCapabilitiesFileName) - hostDomCapabilities := HostDomCapabilities{} - err := n.getStructureFromXMLFile(domCapabilitiesFile, &hostDomCapabilities) - if err != nil { - return hostDomCapabilities, err - } - - if hostDomCapabilities.SEV.Supported == "yes" && hostDomCapabilities.SEV.MaxESGuests > 0 { - hostDomCapabilities.SEV.SupportedES = "yes" - } else { - hostDomCapabilities.SEV.SupportedES = "no" - } - - return hostDomCapabilities, err -} - -// GetStructureFromXMLFile load data from xml file and unmarshals them into given structure -// Given structure has to be pointer -func (n *NodeLabeller) getStructureFromXMLFile(path string, structure interface{}) error { - rawFile, err := os.ReadFile(path) - if err != nil { - return err - } - - n.logger.V(4).Infof("node-labeller - loading data from xml file: %#v", string(rawFile)) - - return xml.Unmarshal(rawFile, structure) +func (n *NodeLabeller) GetHostCpuModel() virt_capabilities.HostCPUModel { + return n.virtCaps.HostCpuModelInfo } diff --git a/pkg/virt-handler/node-labeller/node_labeller.go b/pkg/virt-handler/node-labeller/node_labeller.go index a7442c0a7b93..bb98ff2cccb7 100644 --- a/pkg/virt-handler/node-labeller/node_labeller.go +++ b/pkg/virt-handler/node-labeller/node_labeller.go @@ -28,7 +28,6 @@ import ( "time" "k8s.io/client-go/tools/record" - "libvirt.org/go/libvirtxml" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -43,6 +42,7 @@ import ( "kubevirt.io/kubevirt/pkg/apimachinery/patch" virtconfig "kubevirt.io/kubevirt/pkg/virt-config" + virt_capabilities "kubevirt.io/kubevirt/pkg/virt-launcher-common/virt-capabilities" ) var nodeLabellerLabels = []string{ @@ -62,30 +62,23 @@ var nodeLabellerLabels = []string{ // NodeLabeller struct holds information needed to run node-labeller type NodeLabeller struct { - recorder record.EventRecorder - nodeClient k8scli.NodeInterface - host string - logger *log.FilteredLogger - clusterConfig *virtconfig.ClusterConfig - hypervFeatures supportedFeatures - hostCapabilities supportedFeatures - queue workqueue.TypedRateLimitingInterface[string] - supportedFeatures []string - cpuModelVendor string - volumePath string - domCapabilitiesFileName string - cpuCounter *libvirtxml.CapsHostCPUCounter - guestCaps []libvirtxml.CapsGuest - hostCPUModel hostCPUModel - SEV SEVConfiguration - arch archLabeller + recorder record.EventRecorder + nodeClient k8scli.NodeInterface + host string + logger *log.FilteredLogger + clusterConfig *virtconfig.ClusterConfig + hypervFeatures supportedFeatures + queue workqueue.TypedRateLimitingInterface[string] + volumePath string + virtCaps virt_capabilities.VirtualizationCapabilities + arch archLabeller } -func NewNodeLabeller(clusterConfig *virtconfig.ClusterConfig, nodeClient k8scli.NodeInterface, host string, recorder record.EventRecorder, cpuCounter *libvirtxml.CapsHostCPUCounter, guestCaps []libvirtxml.CapsGuest) (*NodeLabeller, error) { - return newNodeLabeller(clusterConfig, nodeClient, host, NodeLabellerVolumePath, recorder, cpuCounter, guestCaps) +func NewNodeLabeller(clusterConfig *virtconfig.ClusterConfig, nodeClient k8scli.NodeInterface, host string, recorder record.EventRecorder, virtCaps virt_capabilities.VirtualizationCapabilities) (*NodeLabeller, error) { + return newNodeLabeller(clusterConfig, nodeClient, host, NodeLabellerVolumePath, recorder, virtCaps) } -func newNodeLabeller(clusterConfig *virtconfig.ClusterConfig, nodeClient k8scli.NodeInterface, host, volumePath string, recorder record.EventRecorder, cpuCounter *libvirtxml.CapsHostCPUCounter, guestCaps []libvirtxml.CapsGuest) (*NodeLabeller, error) { +func newNodeLabeller(clusterConfig *virtconfig.ClusterConfig, nodeClient k8scli.NodeInterface, host, volumePath string, recorder record.EventRecorder, virtCaps virt_capabilities.VirtualizationCapabilities) (*NodeLabeller, error) { n := &NodeLabeller{ recorder: recorder, nodeClient: nodeClient, @@ -96,18 +89,11 @@ func newNodeLabeller(clusterConfig *virtconfig.ClusterConfig, nodeClient k8scli. workqueue.DefaultTypedControllerRateLimiter[string](), workqueue.TypedRateLimitingQueueConfig[string]{Name: "virt-handler-node-labeller"}, ), - volumePath: volumePath, - domCapabilitiesFileName: "virsh_domcapabilities.xml", - cpuCounter: cpuCounter, - guestCaps: guestCaps, - hostCPUModel: hostCPUModel{requiredFeatures: make(map[string]bool)}, - arch: newArchLabeller(runtime.GOARCH), + volumePath: volumePath, + virtCaps: virtCaps, + arch: newArchLabeller(runtime.GOARCH), } - err := n.loadAll() - if err != nil { - return n, err - } return n, nil } @@ -157,28 +143,6 @@ func (n *NodeLabeller) execute() bool { return true } -func (n *NodeLabeller) loadAll() error { - // host supported features is only available on AMD64 and S390X nodes. - // This is because hypervisor-cpu-baseline virsh command doesnt work for ARM64 architecture. - if n.arch.hasHostSupportedFeatures() { - err := n.loadHostSupportedFeatures() - if err != nil { - n.logger.Errorf("node-labeller could not load supported features: " + err.Error()) - return err - } - } - - err := n.loadDomCapabilities() - if err != nil { - n.logger.Errorf("node-labeller could not load host dom capabilities: " + err.Error()) - return err - } - - n.loadHypervFeatures() - - return nil -} - func (n *NodeLabeller) run() error { originalNode, err := n.nodeClient.Get(context.Background(), n.host, metav1.GetOptions{}) if err != nil { @@ -224,6 +188,7 @@ func (n *NodeLabeller) patchNode(originalNode, node *v1.Node) error { return err } +// TODO: Need to implement this in the Libvirt-QEMU-KVM virtualization capabilities exporter func (n *NodeLabeller) loadHypervFeatures() { n.hypervFeatures.items = getCapLabels() } @@ -248,8 +213,8 @@ func (n *NodeLabeller) prepareLabels(node *v1.Node) map[string]string { } } - for _, machine := range n.getSupportedMachines() { - labelKey := kubevirtv1.SupportedMachineTypeLabel + machine.Name + for _, machine := range n.virtCaps.SupportedMachineTypes { + labelKey := kubevirtv1.SupportedMachineTypeLabel + machine newLabels[labelKey] = "true" } @@ -258,8 +223,8 @@ func (n *NodeLabeller) prepareLabels(node *v1.Node) map[string]string { } if n.hasTSCCounter() { - newLabels[kubevirtv1.CPUTimerLabel+"tsc-frequency"] = fmt.Sprintf("%d", n.cpuCounter.Frequency) - newLabels[kubevirtv1.CPUTimerLabel+"tsc-scalable"] = fmt.Sprintf("%t", n.cpuCounter.Scaling == "yes") + newLabels[kubevirtv1.CPUTimerLabel+"tsc-frequency"] = n.virtCaps.NodeTscInfo.Frequency + newLabels[kubevirtv1.CPUTimerLabel+"tsc-scalable"] = n.virtCaps.NodeTscInfo.Scalable } if n.arch.supportsHostModel() { @@ -273,27 +238,23 @@ func (n *NodeLabeller) prepareLabels(node *v1.Node) map[string]string { } } - for feature := range hostCpuModel.requiredFeatures { + for _, feature := range hostCpuModel.RequiredFeatures { newLabels[kubevirtv1.HostModelRequiredFeaturesLabel+feature] = "true" } - newLabels[kubevirtv1.CPUModelVendorLabel+n.cpuModelVendor] = "true" + newLabels[kubevirtv1.CPUModelVendorLabel+n.virtCaps.HostCpuModelInfo.Vendor] = "true" newLabels[kubevirtv1.HostModelCPULabel+hostCpuModel.Name] = "true" } - capable, err := isNodeRealtimeCapable() - if err != nil { - n.logger.Reason(err).Error("failed to identify if a node is capable of running realtime workloads") - } - if capable { + if n.virtCaps.NodeSupportsRealTime { newLabels[kubevirtv1.RealtimeLabel] = "" } - if n.SEV.Supported == "yes" { + if n.virtCaps.NodeSevFeatures.Supported == "yes" { newLabels[kubevirtv1.SEVLabel] = "" } - if n.SEV.SupportedES == "yes" { + if n.virtCaps.NodeSevFeatures.SupportedES == "yes" { newLabels[kubevirtv1.SEVESLabel] = "" } @@ -349,13 +310,5 @@ func (n *NodeLabeller) alertIfHostModelIsObsolete(originalNode *v1.Node, hostMod } func (n *NodeLabeller) hasTSCCounter() bool { - return n.cpuCounter != nil && n.cpuCounter.Name == "tsc" -} - -func (n *NodeLabeller) getSupportedMachines() []libvirtxml.CapsGuestMachine { - var supportedMachines []libvirtxml.CapsGuestMachine - for _, guest := range n.guestCaps { - supportedMachines = append(supportedMachines, guest.Arch.Machines...) - } - return supportedMachines + return n.virtCaps.NodeTscInfo.HasTscCounter } diff --git a/pkg/virt-launcher-common/virt-capabilities/BUILD.bazel b/pkg/virt-launcher-common/virt-capabilities/BUILD.bazel new file mode 100644 index 000000000000..b904ab012a3d --- /dev/null +++ b/pkg/virt-launcher-common/virt-capabilities/BUILD.bazel @@ -0,0 +1,13 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "exporter.go", + "types.go", + "util.go", + "virtualization-capabilities-interface.go", + ], + importpath = "kubevirt.io/kubevirt/pkg/virt-launcher-common/virt-capabilities", + visibility = ["//visibility:public"], +) diff --git a/pkg/virt-launcher-common/virt-capabilities/exporter.go b/pkg/virt-launcher-common/virt-capabilities/exporter.go new file mode 100644 index 000000000000..d86e65a69daa --- /dev/null +++ b/pkg/virt-launcher-common/virt-capabilities/exporter.go @@ -0,0 +1,36 @@ +package virt_capabilities + +import ( + "encoding/json" + "os" +) + +func ExportVirtualizationCapabilities(v VirtualizationCapabilitiesInterface, filename string) { + virtCaps := VirtualizationCapabilities{ + SupportedCPUModels: v.GetSupportedCpuModels(), + SupportedMachineTypes: v.GetSupportedMachineTypes(), + HypervFeatures: v.GetHypervFeatures(), + HostCpuModelInfo: v.GetHostCpuModelInfo(), + SupportedCpuFeatures: v.GetSupportedCpuFeatures(), + NodeTscInfo: v.GetNodeTscInfo(), + NodeSupportsRealTime: v.NodeSupportsRealTime(), + NodeSevFeatures: v.GetNodeSevFeatures(), + } + + data, err := json.MarshalIndent(virtCaps, "", " ") + if err != nil { + panic(err) + } + + file, err := os.Create(filename) + if err != nil { + panic(err) + } + defer file.Close() + + _, err = file.Write(data) + if err != nil { + panic(err) + } + +} diff --git a/pkg/virt-launcher-common/virt-capabilities/types.go b/pkg/virt-launcher-common/virt-capabilities/types.go new file mode 100644 index 000000000000..f3075fd2f40d --- /dev/null +++ b/pkg/virt-launcher-common/virt-capabilities/types.go @@ -0,0 +1,30 @@ +package virt_capabilities + +type HostCPUModel struct { + Name string `json:"name"` + Fallback string `json:"fallback"` + RequiredFeatures []string `json:"requiredFeatures"` + Vendor string `json:"vendor"` +} + +type TscConfig struct { + HasTscCounter bool `json:"hasTscCounter"` + Frequency string `json:"frequency"` + Scalable string `json:"scalable"` +} + +type SEVConfiguration struct { + Supported string + SupportedES string +} + +type VirtualizationCapabilities struct { + SupportedCPUModels []string `json:"supportedCpuModels"` + SupportedMachineTypes []string `json:"supportedMachineTypes"` + HypervFeatures []string `json:"hypervFeatures"` + HostCpuModelInfo HostCPUModel `json:"hostCpuModelInfo"` + SupportedCpuFeatures []string `json:"supportedCpuFeatures"` + NodeTscInfo TscConfig `json:"nodeTscInfo"` + NodeSupportsRealTime bool `json:"nodeSupportsRealTime"` + NodeSevFeatures SEVConfiguration `json:"nodeSevFeatures"` +} diff --git a/pkg/virt-launcher-common/virt-capabilities/util.go b/pkg/virt-launcher-common/virt-capabilities/util.go new file mode 100644 index 000000000000..522f87687312 --- /dev/null +++ b/pkg/virt-launcher-common/virt-capabilities/util.go @@ -0,0 +1,10 @@ +package virt_capabilities + +const ( + DefaultMinCPUModel = "Penryn" + RequirePolicy = "require" + KVMPath = "/dev/kvm" + VmxFeature = "vmx" +) + +// TODO Check which ones of these are required here diff --git a/pkg/virt-launcher-common/virt-capabilities/virtualization-capabilities-interface.go b/pkg/virt-launcher-common/virt-capabilities/virtualization-capabilities-interface.go new file mode 100644 index 000000000000..c20bfa6cf66d --- /dev/null +++ b/pkg/virt-launcher-common/virt-capabilities/virtualization-capabilities-interface.go @@ -0,0 +1,30 @@ +package virt_capabilities + +const KernelSchedRealtimeRuntimeInMicroseconds = "kernel.sched_rt_runtime_us" + +// VirtualizationCapabilitiesInterface defines methods for querying virtualization capabilities. +type VirtualizationCapabilitiesInterface interface { + // GetHypervFeatures returns a list of features required for Windows guests. + GetHypervFeatures() []string + + // GetSupportedMachineTypes returns supported machine types. + GetSupportedMachineTypes() []string + + // GetSupportedCpuModels returns supported CPU models. + GetSupportedCpuModels() []string + + // GetHostCpuModelInfo returns host CPU model information. + GetHostCpuModelInfo() HostCPUModel + + // GetSupportedCpuFeatures returns supported CPU features. + GetSupportedCpuFeatures() []string + + // GetNodeTscInfo returns node TSC (Time Stamp Counter) information. + GetNodeTscInfo() TscConfig + + // NodeSupportsRealTime returns true if the node supports real-time capabilities. + NodeSupportsRealTime() bool + + // GetNodeSevFeatures returns SEV (Secure Encrypted Virtualization) features of the node. + GetNodeSevFeatures() SEVConfiguration +} diff --git a/pkg/virt-operator/resource/generate/components/daemonsets.go b/pkg/virt-operator/resource/generate/components/daemonsets.go index 878a1b3e76c1..1af121339a85 100644 --- a/pkg/virt-operator/resource/generate/components/daemonsets.go +++ b/pkg/virt-operator/resource/generate/components/daemonsets.go @@ -137,9 +137,9 @@ func NewHandlerDaemonSet(namespace, repository, imagePrefix, version, launcherVe "-c", }, Image: launcherImage, - Name: "virt-launcher", + Name: "virt-capability-extractor", Args: []string{ - "node-labeller.sh", + "node-labeller-go", }, SecurityContext: &corev1.SecurityContext{ Privileged: pointer.P(true),