From 6204d87db4a798cb3ea0ba5fcda8e8d505be153c Mon Sep 17 00:00:00 2001 From: Melvin Hillsman Date: Wed, 20 May 2026 21:22:45 -0500 Subject: [PATCH] feat: add chaos-process workload Implement chaos-process workload that randomly kills non-essential processes to test system resilience. The workload: - Randomly selects and terminates processes using configurable signals - Excludes critical system processes (systemd, sshd, dbus, etc.) - Configurable via environment variables: * CHAOS_SIGNAL (default: SIGTERM) * CHAOS_INTERVAL (default: 30s) * CHAOS_MIN_PID (default: 1000) - Runs as a systemd service with auto-restart - Logs all actions to systemd journal The implementation follows the single-VM workload pattern (like cpu, memory, disk) and includes comprehensive unit tests. Resolves #18 Signed-off-by: Melvin Hillsman --- cmd/virtwork/main_test.go | 4 +- internal/workloads/chaos_process.go | 174 +++++++++++++++++++++++ internal/workloads/chaos_process_test.go | 156 ++++++++++++++++++++ internal/workloads/registry.go | 3 + internal/workloads/registry_test.go | 18 ++- 5 files changed, 350 insertions(+), 5 deletions(-) create mode 100644 internal/workloads/chaos_process.go create mode 100644 internal/workloads/chaos_process_test.go diff --git a/cmd/virtwork/main_test.go b/cmd/virtwork/main_test.go index 8d786bc..9bf8e89 100644 --- a/cmd/virtwork/main_test.go +++ b/cmd/virtwork/main_test.go @@ -628,7 +628,7 @@ var _ = Describe("CLI end-to-end scenarios", func() { Context("when running with default arguments", func() { It("should create VMs for all workloads", func() { - // Default run creates 8 VMs: cpu=1 + memory=1 + disk=1 + database=1 + network=2 + tps=2 + // Default run creates 9 VMs: chaos-process=1 + cpu=1 + memory=1 + disk=1 + database=1 + network=2 + tps=2 registry := workloads.DefaultRegistry() totalVMs := 0 for _, name := range workloads.AllWorkloadNames() { @@ -647,7 +647,7 @@ var _ = Describe("CLI end-to-end scenarios", func() { totalVMs += w.VMCount() } // cpu=1 + database=1 + disk=1 + memory=1 + network=2 + tps=2 = 8 - Expect(totalVMs).To(Equal(8)) + Expect(totalVMs).To(Equal(9)) }) }) diff --git a/internal/workloads/chaos_process.go b/internal/workloads/chaos_process.go new file mode 100644 index 0000000..80c310a --- /dev/null +++ b/internal/workloads/chaos_process.go @@ -0,0 +1,174 @@ +// Copyright 2026 Red Hat +// SPDX-License-Identifier: Apache-2.0 + +package workloads + +import ( + "github.com/opdev/virtwork/internal/config" +) + +const chaosProcessScript = `#!/bin/bash +# Chaos process workload - randomly kills non-essential processes +set -euo pipefail + +# Configuration via environment variables with defaults +CHAOS_SIGNAL="${CHAOS_SIGNAL:-SIGTERM}" +CHAOS_INTERVAL="${CHAOS_INTERVAL:-30}" +CHAOS_MIN_PID="${CHAOS_MIN_PID:-1000}" + +# List of essential processes to exclude from chaos +# These are critical system processes that should never be killed +EXCLUDED_PATTERNS=( + "systemd" + "sshd" + "dbus" + "agetty" + "auditd" + "rsyslogd" + "chronyd" + "NetworkManager" + "bash" + "sh" + "cloud-init" + "virtwork-" +) + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" +} + +build_exclusion_filter() { + local filter="" + for pattern in "${EXCLUDED_PATTERNS[@]}"; do + filter="${filter}${pattern}|" + done + # Remove trailing pipe and return + echo "${filter%|}" +} + +select_random_process() { + local exclusion_filter + exclusion_filter=$(build_exclusion_filter) + + # Get list of PIDs for non-essential processes + # - Exclude PID 1 (init/systemd) + # - Exclude PIDs below CHAOS_MIN_PID (kernel threads and core system processes) + # - Exclude processes matching EXCLUDED_PATTERNS + # - Exclude this script itself + local pids + pids=$(ps -eo pid,comm --no-headers | \ + awk -v min_pid="$CHAOS_MIN_PID" '$1 >= min_pid && $1 != '"$$"' {print $1, $2}' | \ + grep -Ev "($exclusion_filter)" | \ + awk '{print $1}' || true) + + if [ -z "$pids" ]; then + log "No eligible processes found for chaos action" + return 1 + fi + + # Convert to array and select random PID + local pid_array=($pids) + local count=${#pid_array[@]} + local random_index=$((RANDOM % count)) + echo "${pid_array[$random_index]}" +} + +kill_random_process() { + local target_pid + target_pid=$(select_random_process) || return 0 + + # Get process info before killing + local process_info + process_info=$(ps -p "$target_pid" -o pid,comm,args --no-headers 2>/dev/null || echo "unknown") + + log "Sending $CHAOS_SIGNAL to PID $target_pid: $process_info" + + if kill -"$CHAOS_SIGNAL" "$target_pid" 2>/dev/null; then + log "Successfully sent $CHAOS_SIGNAL to PID $target_pid" + else + log "Failed to send signal to PID $target_pid (process may have already exited)" + fi +} + +main() { + log "Starting chaos-process workload" + log "Configuration: SIGNAL=$CHAOS_SIGNAL INTERVAL=${CHAOS_INTERVAL}s MIN_PID=$CHAOS_MIN_PID" + + while true; do + kill_random_process + sleep "$CHAOS_INTERVAL" + done +} + +main "$@" +` + +const chaosProcessSystemdUnit = `[Unit] +Description=Virtwork chaos-process workload +After=network.target + +[Service] +Type=simple +Environment="CHAOS_SIGNAL=SIGTERM" +Environment="CHAOS_INTERVAL=30" +Environment="CHAOS_MIN_PID=1000" +ExecStart=/usr/local/bin/chaos-process.sh +Restart=always +RestartSec=10 +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target +` + +// ChaosProcessWorkload generates cloud-init userdata for a process chaos workload +// that randomly kills non-essential processes to test resilience. +type ChaosProcessWorkload struct { + BaseWorkload +} + +// NewChaosProcessWorkload creates a ChaosProcessWorkload with the given configuration and SSH credentials. +func NewChaosProcessWorkload( + cfg config.WorkloadConfig, + sshUser, sshPassword string, + sshKeys []string, +) *ChaosProcessWorkload { + return &ChaosProcessWorkload{ + BaseWorkload: BaseWorkload{ + Config: cfg, + SSHUser: sshUser, + SSHPassword: sshPassword, + SSHAuthorizedKeys: sshKeys, + }, + } +} + +// Name returns "chaos-process". +func (w *ChaosProcessWorkload) Name() string { + return "chaos-process" +} + +// CloudInitUserdata returns cloud-init YAML that installs the chaos-process script +// and runs it as a systemd service. +func (w *ChaosProcessWorkload) CloudInitUserdata() (string, error) { + return w.BuildCloudConfig(CloudConfigOpts{ + Packages: []string{"procps-ng"}, + WriteFiles: []WriteFile{ + { + Path: "/usr/local/bin/chaos-process.sh", + Content: chaosProcessScript, + Permissions: "0755", + }, + { + Path: "/etc/systemd/system/virtwork-chaos-process.service", + Content: chaosProcessSystemdUnit, + Permissions: "0644", + }, + }, + RunCmd: [][]string{ + {"systemctl", "daemon-reload"}, + {"systemctl", "enable", "--now", "virtwork-chaos-process.service"}, + }, + }) +} diff --git a/internal/workloads/chaos_process_test.go b/internal/workloads/chaos_process_test.go new file mode 100644 index 0000000..a0f4e38 --- /dev/null +++ b/internal/workloads/chaos_process_test.go @@ -0,0 +1,156 @@ +// Copyright 2026 Red Hat +// SPDX-License-Identifier: Apache-2.0 + +package workloads_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/opdev/virtwork/internal/config" + "github.com/opdev/virtwork/internal/workloads" +) + +var _ = Describe("ChaosProcessWorkload", func() { + var w *workloads.ChaosProcessWorkload + + BeforeEach(func() { + w = workloads.NewChaosProcessWorkload(config.WorkloadConfig{ + Enabled: true, + VMCount: 1, + CPUCores: 2, + Memory: "2Gi", + }, "virtwork", "", nil) + }) + + It("should return 'chaos-process' for Name", func() { + Expect(w.Name()).To(Equal("chaos-process")) + }) + + It("should include procps-ng in packages", func() { + result, err := w.CloudInitUserdata() + Expect(err).NotTo(HaveOccurred()) + + parsed := parseYAML(result) + pkgs, ok := parsed["packages"].([]interface{}) + Expect(ok).To(BeTrue()) + Expect(pkgs).To(ContainElement("procps-ng")) + }) + + It("should include chaos script in cloud-init", func() { + result, err := w.CloudInitUserdata() + Expect(err).NotTo(HaveOccurred()) + + parsed := parseYAML(result) + Expect(parsed).To(HaveKey("write_files")) + files := parsed["write_files"].([]interface{}) + Expect(files).To(HaveLen(2)) + + var scriptFile map[string]interface{} + for _, f := range files { + file := f.(map[string]interface{}) + if file["path"] == "/usr/local/bin/chaos-process.sh" { + scriptFile = file + break + } + } + Expect(scriptFile).NotTo(BeNil()) + + content := scriptFile["content"].(string) + Expect(content).To(ContainSubstring("#!/bin/bash")) + Expect(content).To(ContainSubstring("CHAOS_SIGNAL")) + Expect(content).To(ContainSubstring("CHAOS_INTERVAL")) + Expect(content).To(ContainSubstring("kill_random_process")) + Expect(scriptFile["permissions"]).To(Equal("0755")) + }) + + It("should include systemd service in cloud-init", func() { + result, err := w.CloudInitUserdata() + Expect(err).NotTo(HaveOccurred()) + + parsed := parseYAML(result) + Expect(parsed).To(HaveKey("write_files")) + files := parsed["write_files"].([]interface{}) + Expect(files).To(HaveLen(2)) + + var serviceFile map[string]interface{} + for _, f := range files { + file := f.(map[string]interface{}) + if file["path"] == "/etc/systemd/system/virtwork-chaos-process.service" { + serviceFile = file + break + } + } + Expect(serviceFile).NotTo(BeNil()) + + content := serviceFile["content"].(string) + Expect(content).To(ContainSubstring("Virtwork chaos-process workload")) + Expect(content).To(ContainSubstring("ExecStart=/usr/local/bin/chaos-process.sh")) + Expect(content).To(ContainSubstring("Restart=always")) + Expect(content).To(ContainSubstring("CHAOS_SIGNAL")) + Expect(content).To(ContainSubstring("CHAOS_INTERVAL")) + Expect(serviceFile["permissions"]).To(Equal("0644")) + }) + + It("should enable and start the systemd service", func() { + result, err := w.CloudInitUserdata() + Expect(err).NotTo(HaveOccurred()) + + parsed := parseYAML(result) + Expect(parsed).To(HaveKey("runcmd")) + runcmd := parsed["runcmd"].([]interface{}) + + // Check for systemctl commands + hasReload := false + hasEnable := false + for _, cmd := range runcmd { + cmdSlice := cmd.([]interface{}) + if len(cmdSlice) >= 2 && cmdSlice[0] == "systemctl" && cmdSlice[1] == "daemon-reload" { + hasReload = true + } + if len(cmdSlice) >= 4 && cmdSlice[0] == "systemctl" && + cmdSlice[1] == "enable" && cmdSlice[2] == "--now" && + cmdSlice[3] == "virtwork-chaos-process.service" { + hasEnable = true + } + } + Expect(hasReload).To(BeTrue()) + Expect(hasEnable).To(BeTrue()) + }) + + It("should produce valid YAML", func() { + result, err := w.CloudInitUserdata() + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(HavePrefix("#cloud-config\n")) + + parsed := parseYAML(result) + Expect(parsed).NotTo(BeNil()) + }) + + It("should have no extra disks", func() { + Expect(w.ExtraDisks()).To(BeNil()) + }) + + It("should have no extra volumes", func() { + Expect(w.ExtraVolumes()).To(BeNil()) + }) + + It("should have no data volume templates", func() { + Expect(w.DataVolumeTemplates()).To(BeNil()) + }) + + It("should have no service", func() { + Expect(w.RequiresService()).To(BeFalse()) + Expect(w.ServiceSpec()).To(BeNil()) + }) + + It("should reflect config in VMResources", func() { + res := w.VMResources() + Expect(res.CPUCores).To(Equal(2)) + Expect(res.Memory).To(Equal("2Gi")) + }) + + It("should default to 1 VM", func() { + Expect(w.VMCount()).To(Equal(1)) + }) +}) diff --git a/internal/workloads/registry.go b/internal/workloads/registry.go index a6a2cf8..e0c4e69 100644 --- a/internal/workloads/registry.go +++ b/internal/workloads/registry.go @@ -62,6 +62,9 @@ func AllWorkloadNames() []string { // DefaultRegistry returns a Registry pre-populated with all built-in workloads. func DefaultRegistry() Registry { return Registry{ + "chaos-process": func(cfg config.WorkloadConfig, opts *RegistryOpts) Workload { + return NewChaosProcessWorkload(cfg, opts.SSHUser, opts.SSHPassword, opts.SSHAuthorizedKeys) + }, "cpu": func(cfg config.WorkloadConfig, opts *RegistryOpts) Workload { return NewCPUWorkload(cfg, opts.SSHUser, opts.SSHPassword, opts.SSHAuthorizedKeys) }, diff --git a/internal/workloads/registry_test.go b/internal/workloads/registry_test.go index 6eedda8..a909a29 100644 --- a/internal/workloads/registry_test.go +++ b/internal/workloads/registry_test.go @@ -18,8 +18,8 @@ var _ = Describe("Registry", func() { reg = workloads.DefaultRegistry() }) - It("should have 6 entries registered", func() { - Expect(reg.List()).To(HaveLen(6)) + It("should have 7 entries registered", func() { + Expect(reg.List()).To(HaveLen(7)) }) It("should return CPU workload by name", func() { @@ -77,10 +77,22 @@ var _ = Describe("Registry", func() { Expect(w.Name()).To(Equal("disk")) }) + It("should return chaos-process workload by name", func() { + w, err := reg.Get("chaos-process", config.WorkloadConfig{ + Enabled: true, + VMCount: 1, + CPUCores: 2, + Memory: "2Gi", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(w.Name()).To(Equal("chaos-process")) + }) + It("should return error for unknown name with available names", func() { _, err := reg.Get("unknown", config.WorkloadConfig{}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("unknown")) + Expect(err.Error()).To(ContainSubstring("chaos-process")) Expect(err.Error()).To(ContainSubstring("cpu")) Expect(err.Error()).To(ContainSubstring("database")) Expect(err.Error()).To(ContainSubstring("disk")) @@ -91,7 +103,7 @@ var _ = Describe("Registry", func() { It("should list all names sorted alphabetically", func() { names := reg.List() - Expect(names).To(Equal([]string{"cpu", "database", "disk", "memory", "network", "tps"})) + Expect(names).To(Equal([]string{"chaos-process", "cpu", "database", "disk", "memory", "network", "tps"})) }) It("should return tps workload by name", func() {