Pluggable node labeler

BUILD.bazel

@@ -513,6 +513,16 @@ container_push(
     tag = "$(container_tag)",
 )
 
+genrule(
+    name = "build-node-labeller-go",
+    srcs = [
+        "//cmd/virt-launcher/node-labeller-go",
+    ],
+    outs = ["node-labeller-go"],
+    cmd = "echo '#!/bin/sh\n\ncp -f $(SRCS) $$1' > \"$@\"",
+    executable = 1,
+)
+
 genrule(
     name = "build-virtctl",
     srcs = [

cmd/virt-handler/BUILD.bazel

@@ -43,6 +43,7 @@ go_library(
         "//pkg/virt-handler/seccomp:go_default_library",
         "//pkg/virt-handler/selinux:go_default_library",
         "//pkg/virt-handler/vsock:go_default_library",
+        "//pkg/virt-launcher-common/virt-capabilities:go_default_library",
         "//staging/src/kubevirt.io/api/core/v1:go_default_library",
         "//staging/src/kubevirt.io/client-go/kubecli:go_default_library",
         "//staging/src/kubevirt.io/client-go/log:go_default_library",

cmd/virt-handler/virt-handler.go

@@ -22,6 +22,7 @@ package main
 import (
 	"context"
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"os"
@@ -83,6 +84,7 @@ import (
 	nodelabeller "kubevirt.io/kubevirt/pkg/virt-handler/node-labeller"
 	"kubevirt.io/kubevirt/pkg/virt-handler/rest"
 	"kubevirt.io/kubevirt/pkg/virt-handler/selinux"
+	virt_capabilities "kubevirt.io/kubevirt/pkg/virt-launcher-common/virt-capabilities"
 )
 
 const (
@@ -290,6 +292,16 @@ func (app *virtHandlerApp) Run() {
 
 	stop := make(chan struct{})
 	defer close(stop)
+
+	var virtCaps virt_capabilities.VirtualizationCapabilities
+	virtCapsFile, err := os.ReadFile(filepath.Join(nodelabeller.NodeLabellerVolumePath, "virtualization_capabilities.json"))
+	if err != nil {
+		panic(err)
+	}
+	if err := json.Unmarshal(virtCapsFile, &virtCaps); err != nil {
+		panic(err)
+	}
+
 	var capabilities libvirtxml.Caps
 	var hostCpuModel string
 
@@ -307,8 +319,7 @@ func (app *virtHandlerApp) Run() {
 		app.virtCli.CoreV1().Nodes(),
 		app.HostOverride,
 		nodeLabellerrecorder,
-		capabilities.Host.CPU.Counter,
-		capabilities.Guests,
+		virtCaps,
 	)
 	if err != nil {
 		panic(err)

cmd/virt-launcher/BUILD.bazel

@@ -62,6 +62,7 @@ pkg_tar(
         "//cmd/container-disk-v2alpha:container-disk",
         "//cmd/virt-freezer",
         "//cmd/virt-launcher-monitor",
+        "//cmd/virt-launcher/node-labeller-go",
         "//cmd/virt-probe",
         "//cmd/virt-tail",
     ],

cmd/virt-launcher/node-labeller-go/BUILD.bazel

@@ -0,0 +1,23 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "node-labeller.go",
+        "types.go",
+        "virtualization-capabilities-libvirt-qemu.go",
+    ],
+    importpath = "kubevirt.io/kubevirt/cmd/virt-launcher/node-labeller-go",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//pkg/virt-launcher-common/virt-capabilities:go_default_library",
+        "//staging/src/kubevirt.io/api/core/v1:go_default_library",
+        "//vendor/libvirt.org/go/libvirtxml:go_default_library",
+    ],
+)
+
+go_binary(
+    name = "node-labeller-go",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)

cmd/virt-launcher/node-labeller-go/node-labeller.go

@@ -0,0 +1,125 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+	"time"
+
+	virt_capabilities "kubevirt.io/kubevirt/pkg/virt-launcher-common/virt-capabilities"
+)
+
+const (
+	XmlBasePath = "/var/lib/kubevirt-node-labeller"
+)
+
+func executeCommand(command string) (string, error) {
+	cmd := exec.Command("bash", "-c", command)
+	output, err := cmd.Output()
+	outputStr := strings.TrimSpace(string(output))
+	if err != nil {
+		return "", fmt.Errorf("failed to execute command %s: %w", command, err)
+	}
+	return outputStr, nil
+}
+
+func queryMachineArchitecture() (string, error) {
+	return executeCommand("uname -m")
+}
+
+func queryKvmMinor() (string, error) {
+	return executeCommand("grep -w 'kvm' /proc/misc | cut -f 1 -d' '")
+}
+
+func main() {
+
+	machine := "q35"
+
+	arch, err := queryMachineArchitecture()
+	if err != nil {
+		fmt.Printf("Error querying machine architecture: %v\n", err)
+		return
+	}
+
+	fmt.Printf("Detected arch=\"%s\"\n", arch)
+
+	if arch == "aarch64" {
+		machine = "virt"
+	} else if arch == "s390x" {
+		machine = "390-ccw-virtio"
+	} else if arch != "x86_64" {
+		// Node labeling cannot proceed for this architecture
+		fmt.Printf("Node labeling cannot proceed for architecture: %s\n", arch)
+		return
+	}
+
+	kvmMinor, err := queryKvmMinor()
+	if err != nil {
+		fmt.Printf("Error querying KVM minor: %v\n", err)
+		return
+	}
+
+	virttype := "qemu"
+
+	_, err = exec.Command("ls", "/dev/kvm").Output()
+	kvmExists := err == nil
+
+	if !kvmExists && kvmMinor != "" {
+		executeCommand("mknod /dev/kvm c 10 " + kvmMinor)
+	}
+
+	_, err = exec.Command("ls", "/dev/kvm").Output()
+	kvmExists = err == nil
+
+	if kvmExists {
+		executeCommand("chmod o+rw /dev/kvm")
+		virttype = "kvm"
+	}
+
+	_, err = exec.Command("ls", "/dev/sev").Output()
+	sevExists := err == nil
+
+	if sevExists {
+		// QEMU requires RW access to query SEV capabilities
+		executeCommand("chmod o+rw /dev/kvm")
+	}
+
+	cmd := exec.Command("virtqemud", "-d")
+	err = cmd.Start()
+	if err != nil {
+		fmt.Printf("Failed to start virtqemud: %v\n", err)
+		return
+	}
+	fmt.Println("virtqemud started in daemon mode")
+
+	executeCommand(fmt.Sprintf("mkdir -p %s", XmlBasePath)) // TODO Remove this later, this is just for testing
+
+	fmt.Println("Waiting for virtqemud to start...")
+	time.Sleep(5 * time.Second) // Wait for virtqemud to start
+
+	_, err = executeCommand(fmt.Sprintf("virsh domcapabilities --machine %s --arch %s --virttype %s > %s/virsh_domcapabilities.xml", machine, arch, virttype, XmlBasePath))
+
+	if err != nil {
+		fmt.Printf("Failed to get domain capabilities: %v\n", err)
+		return
+	}
+
+	if arch == "x86_64" || arch == "s390x" {
+		cmd := fmt.Sprintf("virsh domcapabilities --machine %s --arch %s --virttype %s | virsh hypervisor-cpu-baseline --features /dev/stdin --machine %s --arch %s --virttype %s > %s/supported_features.xml", machine, arch, virttype, machine, arch, virttype, XmlBasePath)
+		_, err := executeCommand(cmd)
+		if err != nil {
+			fmt.Printf("Failed to get supported features: %v\n", err)
+			return
+		}
+	}
+
+	_, err = executeCommand(fmt.Sprintf("virsh capabilities > %s/capabilities.xml", XmlBasePath))
+	if err != nil {
+		fmt.Printf("Failed to get node capabilities: %v\n", err)
+		return
+	}
+
+	capabilityExtractor := NewVirtualizationCapabilitiesLibvirtQemu(fmt.Sprintf("%s/supported_features.xml", XmlBasePath), fmt.Sprintf("%s/virsh_domcapabilities.xml", XmlBasePath), fmt.Sprintf("%s/capabilities.xml", XmlBasePath))
+
+	virt_capabilities.ExportVirtualizationCapabilities(capabilityExtractor, fmt.Sprintf("%s/virtualization_capabilities.json", XmlBasePath))
+}

cmd/virt-launcher/node-labeller-go/types.go

@@ -0,0 +1,64 @@
+package main
+
+// HostDomCapabilities represents structure for parsing output of virsh capabilities
+type HostDomCapabilities struct {
+	CPU CPU              `xml:"cpu"`
+	SEV SEVConfiguration `xml:"features>sev"`
+}
+
+// CPU represents slice of cpu modes
+type CPU struct {
+	Mode []Mode `xml:"mode"`
+}
+
+// Mode represents slice of cpu models
+type Mode struct {
+	Name      string        `xml:"name,attr"`
+	Supported string        `xml:"supported,attr"`
+	Vendor    Vendor        `xml:"vendor"`
+	Feature   []HostFeature `xml:"feature"`
+	Model     []Model       `xml:"model"`
+}
+
+type SupportedHostFeature struct {
+	Feature []HostFeature `xml:"feature"`
+}
+
+type HostFeature struct {
+	Policy string `xml:"policy,attr"`
+	Name   string `xml:"name,attr"`
+}
+
+// Vendor represents vendor of host CPU
+type Vendor struct {
+	Name string `xml:",chardata"`
+}
+
+// Model represents cpu model
+type Model struct {
+	Name     string `xml:",chardata"`
+	Usable   string `xml:"usable,attr"`
+	Fallback string `xml:"fallback,attr"`
+}
+
+// Structures needed to parse cpu features
+type FeatureModel struct {
+	Model Features `xml:"model"`
+}
+
+type Features struct {
+	Features []Feature `xml:"feature"`
+}
+
+type Feature struct {
+	Name string `xml:"name,attr"`
+}
+
+type SEVConfiguration struct {
+	Supported       string `xml:"supported,attr"`
+	CBitPos         uint   `xml:"cbitpos"`
+	ReducedPhysBits uint   `xml:"reducedPhysBits"`
+	MaxGuests       uint   `xml:"maxGuests"`
+	MaxESGuests     uint   `xml:"maxESGuests"`
+	SupportedES     string `xml:"-"`
+}