From 0362926d571eccb10e79d16ce95cad93bdd28f15 Mon Sep 17 00:00:00 2001 From: gziv Date: Sun, 31 May 2026 12:17:28 +0300 Subject: [PATCH 1/2] feat: add vm-lifecycle-manager ASE evaluation submission --- .../vm-lifecycle-manager/evals/evals.json | 17 + .../vm-lifecycle-manager/metadata.yaml | 5 + .../vm-lifecycle-manager/skills/SKILL.md | 314 +++++ .../skills/references/INDEX.md | 332 ++++++ .../skills/references/lifecycle-errors.md | 869 ++++++++++++++ .../skills/references/network-errors.md | 429 +++++++ .../skills/references/runtime-errors.md | 616 ++++++++++ .../skills/references/scheduling-errors.md | 417 +++++++ .../skills/references/storage-errors.md | 1011 +++++++++++++++++ 9 files changed, 4010 insertions(+) create mode 100644 submissions/vm-lifecycle-manager/evals/evals.json create mode 100644 submissions/vm-lifecycle-manager/metadata.yaml create mode 100644 submissions/vm-lifecycle-manager/skills/SKILL.md create mode 100644 submissions/vm-lifecycle-manager/skills/references/INDEX.md create mode 100644 submissions/vm-lifecycle-manager/skills/references/lifecycle-errors.md create mode 100644 submissions/vm-lifecycle-manager/skills/references/network-errors.md create mode 100644 submissions/vm-lifecycle-manager/skills/references/runtime-errors.md create mode 100644 submissions/vm-lifecycle-manager/skills/references/scheduling-errors.md create mode 100644 submissions/vm-lifecycle-manager/skills/references/storage-errors.md diff --git a/submissions/vm-lifecycle-manager/evals/evals.json b/submissions/vm-lifecycle-manager/evals/evals.json new file mode 100644 index 0000000..7cb9970 --- /dev/null +++ b/submissions/vm-lifecycle-manager/evals/evals.json @@ -0,0 +1,17 @@ +{ + "skill_name": "vm-lifecycle-manager", + "evals": [ + { + "id": "vm-lifecycle-manager-eval", + "name": "VM Lifecycle Management", + "prompt": "You are an OpenShift Virtualization administrator. A developer asks you to restart VM `api-server` in namespace `prod`.\n\nDescribe your methodology for handling VM lifecycle operations (start, stop, restart). What confirmation steps do you require? How do you implement restart specifically, and why? What error scenarios do you handle?", + "expected_output": "A lifecycle management approach that requires explicit user confirmation before any action, implements restart as two separate stop-then-start operations to avoid resourceVersion conflicts, and handles edge cases like VMs already in the desired state.", + "assertions": [ + "The output requires explicit user confirmation (yes/no) before executing any lifecycle operation, presenting the operation details and impact first.", + "The output implements restart as two separate sequential operations (stop then start) rather than a single restart command, and explains this avoids resourceVersion conflicts.", + "The output describes verifying the VM has fully stopped before issuing the start command during a restart sequence.", + "The output describes how to handle the case where a VM is already in the desired state (e.g., already running when start is requested)." + ] + } + ] +} diff --git a/submissions/vm-lifecycle-manager/metadata.yaml b/submissions/vm-lifecycle-manager/metadata.yaml new file mode 100644 index 0000000..2af8e86 --- /dev/null +++ b/submissions/vm-lifecycle-manager/metadata.yaml @@ -0,0 +1,5 @@ +name: vm-lifecycle-manager +description: "Manage virtual machine lifecycle operations including start, stop, and" +persona: rh-virt +version: "1.0.0" +eval_engine: ase diff --git a/submissions/vm-lifecycle-manager/skills/SKILL.md b/submissions/vm-lifecycle-manager/skills/SKILL.md new file mode 100644 index 0000000..57b5f6a --- /dev/null +++ b/submissions/vm-lifecycle-manager/skills/SKILL.md @@ -0,0 +1,314 @@ +--- +name: vm-lifecycle-manager +description: 'Manage virtual machine lifecycle operations including start, stop, and + restart. + + + Use when: + + - "Start VM [name]" + + - "Stop the virtual machine [name]" + + - "Restart VM [name]" + + - "Power on/off VM [name]" + + + This skill handles VM state transitions safely with user confirmation for each action. + + + NOT for creating VMs (use vm-create) or deleting VMs (use vm-delete). + + ' +license: Apache-2.0 +--- + +# /vm-lifecycle-manager Skill + +Control virtual machine power state in OpenShift Virtualization using the `vm_lifecycle` tool. + +## Prerequisites + +**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools**: +- `vm_lifecycle` (from openshift-virtualization) - Manage VM power state + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster (>= 4.19) +- OpenShift Virtualization operator installed +- ServiceAccount with RBAC permissions to update VirtualMachine resources + +### Prerequisite Verification + +**Before executing:** +1. Check `openshift-virtualization` exists in `mcps.json` → If missing, report setup +2. Verify `KUBECONFIG` is set (presence only, never expose value) → If missing, report + +**Human Notification Protocol:** `❌ Cannot execute vm-lifecycle-manager: MCP server not available. Setup: Add to mcps.json, set KUBECONFIG, restart Claude Code. Docs: https://github.com/openshift/openshift-mcp-server` + +⚠️ **SECURITY**: Never display KUBECONFIG path or credential values. + +## When to Use This Skill + +**Trigger when:** +- User explicitly invokes `/vm-lifecycle-manager` command +- User requests starting/stopping/restarting a VM +- User wants to change VM power state + +**User phrases:** +- "Start VM web-server in namespace vms" +- "Stop the database VM" +- "Restart test-vm" +- "Power on the VM called api-server" +- "/vm-lifecycle-manager" (explicit command) + +**Do NOT use when:** +- Create VM → `/vm-create` +- List VMs → `/vm-inventory` +- Delete VM → `/vm-delete` + +## Workflow + +### Step 1: Gather Parameters and Confirm + +**Required from user:** VM Name, Namespace, Action (start|stop|restart) + +**Present for confirmation:** + +```markdown +## VM Lifecycle Operation + +| Parameter | Value | Impact | +|-----------|-------|--------| +| VM Name | `` | from user | +| Namespace | `` | from user | +| Action | `` | start: consumes resources / stop: graceful shutdown / restart: brief interruption (~1-2min) | + +Confirm: yes/no +``` + +**WAIT** for explicit "yes" before proceeding to Step 2. + +### Step 2: Execute Lifecycle Operation + +**ONLY AFTER user confirmation in Step 1.** + +**For start or stop actions:** + +**MCP Tool**: `vm_lifecycle` (namespace=``, name=``, action=``) + +**For restart action (composite operation):** + +**CRITICAL**: Implement restart as two separate operations to avoid resourceVersion conflicts: + +1. **Stop VM**: `vm_lifecycle` (namespace=``, name=``, action="stop") +2. **Verify stopped**: `resources_get` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", namespace=``, name=``) → Check `status.printableStatus` == "Stopped" +3. **Wait**: 5 seconds for VM to fully stop +4. **Start VM**: `vm_lifecycle` (namespace=``, name=``, action="start") +5. **Verify started**: `resources_get` → Check `status.printableStatus` == "Running" + +**Errors:** +- VM not found → Report, suggest vm-inventory +- Permission denied → Report RBAC error +- Already in desired state → Inform user +- Stop fails during restart → Report, do not proceed to start +- Start fails during restart → Report, VM is stopped +- Transition fails → Report details + +### Step 3: Report Operation Status + +**On Success:** + +```markdown +## ✓ VM Successful + +**VM**: `` | **Namespace**: `` | **Action**: | **RunStrategy**: + +**Impact**: +- **start**: Running, consuming resources (CPU/memory). Access: virtctl console or SSH. RunStrategy: Always (auto-restart on crash) +- **stop**: Stopped, resources freed. State preserved. Start: "Start VM ". RunStrategy: Halted (stays off) +- **restart**: Running after stop+start. Brief interruption (~1-2min). Monitor app logs. RunStrategy: Always + +**Next**: "Show status of VM " or "List VMs in namespace " +``` + +**On Failure:** + +**OPTIONAL**: Read [lifecycle-errors.md](../../docs/troubleshooting/lifecycle-errors.md) for start/stop failures or [scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) for ErrorUnschedulable. Output: "Consulted lifecycle-errors.md for failure." + +**When to consult**: Start/stop failures, stuck transitions, unexpected errors. **NOT**: Already in state, not found, RBAC errors. + +```markdown +## ❌ Lifecycle Operation Failed + +**Error**: + +**Causes**: VM not found | RBAC denied | Already in desired state | VM in transition (wait 30-60s) | Resource constraints (start) + +**Troubleshoot**: +1. vm-inventory to verify VM exists +2. Check RBAC: `oc auth can-i update virtualmachines -n ` +3. View VM status and events +4. Check node capacity (for start operations) +``` + +## Common Issues + +### Issue 1: VM Not Found +**Error**: "VirtualMachine 'xyz' not found in namespace 'abc'" +**Solution**: Verify spelling, check namespace, use vm-inventory, VM may be deleted + +### Issue 2: VM Already in Desired State +**Warning**: "VM is already running" (when attempting start) +**Solution**: Not an error - VM already in desired state. Use `restart` if intended to restart + +### Issue 3: Permission Denied +**Error**: "Forbidden: User cannot update VirtualMachines" +**Solution**: Verify RBAC permissions (update VirtualMachine resources), contact admin + +### Issue 4: VM Stuck in Transitioning State +**Error**: "VM stuck in 'Terminating' or 'Starting'" +**Solution**: Wait 30-60s, check events (`oc describe vm`), use vm-troubleshooter, check virt-launcher pod + +### Issue 5: Insufficient Resources (Start) +**Error**: "Insufficient CPU/memory to start VM" +**Solution**: Check cluster availability, stop other VMs, scale nodes, resize VM to smaller instance type + +### Issue 6: Restart Implementation +**Note**: Restart is implemented as two separate operations (stop → verify → start → verify) +**Reason**: Avoids Kubernetes resourceVersion conflicts when using single restart action +**Behavior**: If stop succeeds but start fails, VM remains stopped. Check VM status with vm-inventory + +## Understanding RunStrategy + +| Action | RunStrategy | Behavior | +|--------|------------|----------| +| start | Always | Runs, auto-restarts on crash | +| stop | Halted | Stops, stays off | +| restart | Always | Stops, starts, auto-restarts | + +## Dependencies + +### Required MCP Servers +- `openshift-virtualization` - OpenShift MCP server with KubeVirt toolset + +### Required MCP Tools +- `vm_lifecycle` - Manage VM power state (start/stop/restart) + +### Related Skills +- `vm-create` - Create VMs +- `vm-inventory` - Check VM status +- `vm-troubleshooter` (planned) - Diagnose startup/shutdown issues + +### Reference Documentation +- [lifecycle-errors.md](../../docs/troubleshooting/lifecycle-errors.md) - Start/stop failures, stuck transitions (consulted on failures) +- [scheduling-errors.md](../../docs/troubleshooting/scheduling-errors.md) - ErrorUnschedulable, resource constraints (consulted when VM won't start) +- [Troubleshooting INDEX](../../docs/troubleshooting/INDEX.md) - Navigation hub for error categories +- [OpenShift Virt Docs](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index#virt/about_virt/about-virt.html) +- [KubeVirt Lifecycle](https://kubevirt.io/user-guide/virtual_machines/lifecycle/) +- [RunStrategy Docs](https://kubevirt.io/user-guide/virtual_machines/run_strategies/) +- [OpenShift MCP](https://github.com/openshift/openshift-mcp-server) + +## Critical: Human-in-the-Loop Requirements + +**IMPORTANT:** This skill requires explicit user confirmation before executing. You MUST: + +1. **Wait for user confirmation** on specific action (start/stop/restart) before executing `vm_lifecycle` +2. **Do NOT proceed** until user explicitly approves +3. **Present action clearly** with VM name, namespace, intended state change +4. **Never auto-execute** - these operations impact running services +5. **Warn about consequences** - stopping interrupts services, starting consumes resources, restarting causes downtime + +If user says "no" or wants to reconsider, do not proceed. + +**Why**: start (consumes resources), stop (interrupts services), restart (brief downtime). User should verify correct VM and understand impact. + +## Security Considerations + +- Requires RBAC permissions (update VirtualMachine resources) +- Stop operations attempt graceful shutdown; guest OS/app data consistency depends on workload +- Start operations consume resources (quota limits apply) +- All operations audited in Kubernetes API logs +- KUBECONFIG credentials never exposed +- ServiceAccount permissions limit controllable VMs + +## Example Usage + +### Example 1: Start Stopped VM + +``` +User: "Start VM web-server in namespace vms" +Agent: [vm-lifecycle-manager skill] +## VM Lifecycle Operation +| VM Name | `web-server` | Namespace | `vms` | Action | `start` | consumes resources | +Confirm: yes/no +User: "yes" +Agent: [vm_lifecycle(namespace="vms", name="web-server", action="start")] +## ✓ VM Start Successful +VM: `web-server` | Namespace: `vms` | RunStrategy: Always +Impact: Running, consuming resources. Access: virtctl console or SSH +``` + +### Example 2: Stop with User Reconsidering + +``` +User: "Stop the database VM in production namespace" +Agent: [vm-lifecycle-manager skill] +| VM | `database` | Namespace | `production` | Action | `stop` | graceful shutdown | +Confirm: yes/no +User: "wait, no, don't stop it" +Agent: Operation cancelled. VM 'database' in namespace 'production' remains running. +``` + +### Example 3: Restart VM + +``` +User: "Restart test-vm in namespace dev" +Agent: [vm-lifecycle-manager skill] +| VM | `test-vm` | Namespace | `dev` | Action | `restart` | brief interruption ~1-2min | +User: "yes" +Agent: [Composite restart operation] + 1. vm_lifecycle(namespace="dev", name="test-vm", action="stop") + 2. Verify stopped: resources_get → status.printableStatus="Stopped" + 3. Wait 5s + 4. vm_lifecycle(namespace="dev", name="test-vm", action="start") + 5. Verify started: resources_get → status.printableStatus="Running" +## ✓ VM Restart Successful +VM: `test-vm` | Namespace: `dev` | RunStrategy: Always +Impact: Running after stop+start. Brief interruption. Monitor app logs. +``` + +### Example 4: VM Already in Desired State + +``` +User: "Start web-server in namespace vms" +Agent: [vm-lifecycle-manager skill] + [vm_lifecycle(action="start")] +## ℹ️ VM Already Running +VM: `web-server` | Namespace: `vms` | Status: Running +Result: No action taken - VM already in desired state. +To restart: "Restart VM web-server in namespace vms" +``` + +### Example 5: Batch Operations + +``` +User: "Stop VMs web-01, web-02, web-03 in namespace production" +Agent: [vm-lifecycle-manager skill - batch mode] +## Batch Lifecycle Operation +Stopping 3 VMs in 'production': web-01, web-02, web-03 +Impact: All 3 VMs will shut down, services interrupted. +Confirm: yes/no +User: "yes" +Agent: [Executes vm_lifecycle for each VM sequentially] +## ✓ Batch Stop Successful +- web-01: Stopped +- web-02: Stopped +- web-03: Stopped +All VMs stopped. Resources freed. +``` diff --git a/submissions/vm-lifecycle-manager/skills/references/INDEX.md b/submissions/vm-lifecycle-manager/skills/references/INDEX.md new file mode 100644 index 0000000..a2e7696 --- /dev/null +++ b/submissions/vm-lifecycle-manager/skills/references/INDEX.md @@ -0,0 +1,332 @@ +--- +title: VM Troubleshooting Guide - Index +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 + - title: OpenShift Virtualization - Virtual Machine Status + url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html + date_accessed: 2026-02-06 + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] +semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] +use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] +last_updated: 2026-02-17 +--- + +# VM Troubleshooting Guide - Index + +## Overview + +This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: + +- **Creation**: Scheduling failures, storage provisioning issues +- **Lifecycle**: Start/stop failures, stuck states +- **Deletion**: Resources stuck in Terminating, storage cleanup failures +- **Cloning**: DataVolume cloning errors, cross-namespace issues +- **Networking**: Secondary network attachment failures +- **Runtime**: CrashLoopBackOff, guest OS boot failures + +This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. + +--- + +## 🗂️ Troubleshooting Categories + +The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: + +### 1. [Scheduling Errors](scheduling-errors.md) +**When to use**: VM fails to schedule on any node + +**Errors covered**: +- ErrorUnschedulable - Node Taints +- ErrorUnschedulable - Insufficient Resources +- ErrorUnschedulable - Node Selector Mismatch + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 2. [Storage Errors](storage-errors.md) +**When to use**: VM has storage provisioning, deletion, or cloning issues + +**Errors covered**: +- ErrorDataVolumeNotReady (all 3 subsections) +- ErrorPvcNotFound +- Storage Deletion Failures +- DataVolume Cloning Failures + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### 3. [Lifecycle Errors](lifecycle-errors.md) +**When to use**: VM has start/stop/termination issues + +**Errors covered**: +- VM Stuck in Terminating State +- VM Won't Start (Non-Scheduling Issues) +- VM Won't Stop + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### 4. [Runtime Errors](runtime-errors.md) +**When to use**: VM repeatedly crashes or fails at runtime + +**Errors covered**: +- CrashLoopBackOff + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 5. [Network Errors](network-errors.md) +**When to use**: VM has secondary network attachment failures + +**Errors covered**: +- Network Attachment Failures (Multus, SR-IOV) + +**Skills that use this**: vm-create + +--- + +## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last + +### Critical Principle: MCP-First Approach + +**All diagnostic procedures in this guide follow the MCP-first pattern**: + +``` +1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) +2. ⚠️ IF FAILS: Ask user permission to use CLI command +3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval +``` + +### Why MCP Tools First? + +- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources +- **Consistency**: Same interface across all operations +- **Better Error Handling**: MCP tools return structured errors +- **Audit Trail**: MCP tool usage is logged and trackable + +### Command Pattern Examples + +Throughout this guide, you'll see diagnostic steps formatted like this: + +**✅ CORRECT Pattern (MCP First)**: + +```markdown +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Extract**: `.metadata.finalizers` from returned JSON + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` +``` + +### Available MCP Tools + +The `openshift-virtualization` MCP server provides these tools: + +**Resource Operations**: +- `resources_get` - Get specific resource (replaces `oc get `) +- `resources_list` - List resources (replaces `oc get `) +- `resources_delete` - Delete resource (replaces `oc delete`) +- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) + +**Pod Operations**: +- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) +- `pods_get` - Get pod details (replaces `oc get pod`) +- `pods_log` - Get pod logs (replaces `oc logs`) +- `pods_exec` - Execute in pod (replaces `oc exec`) +- `pods_delete` - Delete pod (replaces `oc delete pod`) +- `pods_top` - Pod resource usage (replaces `oc top pods`) + +**Events & Monitoring**: +- `events_list` - List events (replaces `oc get events`) +- `nodes_top` - Node resource usage (replaces `oc top nodes`) +- `nodes_log` - Node logs (replaces `oc adm node-logs`) +- `nodes_stats_summary` - Detailed node stats + +**VM Operations** (KubeVirt toolset): +- `vm_create` - Create VMs +- `vm_lifecycle` - Start/stop/restart VMs + +**Namespaces**: +- `namespaces_list` - List namespaces (replaces `oc get namespaces`) +- `projects_list` - List OpenShift projects (replaces `oc get projects`) + +### When CLI Commands Are Required + +Some operations have **NO MCP equivalent** and require CLI: + +- `virtctl` commands (console, VNC access) +- `oc debug node` (node debugging) +- `oc auth can-i` (permission checks) +- `oc adm taint` (node taint management) +- Complex JSON patch operations + +For these, the guide will note: **"CLI Only - No MCP equivalent"** + +### Quick Reference: CLI → MCP Mapping + +| CLI Command | MCP Tool Equivalent | +|-------------|---------------------| +| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | +| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | +| `oc logs -n ` | `pods_log` with name="", namespace="" | +| `oc get events -n ` | `events_list` with namespace="" | +| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | + +**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. + +### How to Read Diagnostic Sections + +Each error section includes: +1. **Symptom** - What you observe +2. **Description** - What's happening +3. **Possible Causes** - Why it's happening +4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback +5. **Solutions** - **MCP tools first**, then CLI fallback +6. **Verification** - **MCP tools first**, then CLI fallback + +**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. + +--- + +## 🔍 Quick Navigation by Skill + +**vm-create**: +- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics +- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning +- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff +- [Network Errors](network-errors.md) - Network attachment failures + +**vm-delete**: +- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state +- [Storage Errors](storage-errors.md) - Storage deletion failures + +**vm-clone**: +- [Storage Errors](storage-errors.md) - DataVolume cloning failures + +**vm-lifecycle-manager**: +- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop +- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling + +**vm-inventory**: +- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed + +--- + +## 📊 Documentation Coverage & Maintenance + +### Current Coverage + +This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: + +- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) +- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures +- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop +- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) +- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems + +**Total errors documented**: 12 error types across 6 categories + +--- + +### Encountering Undocumented Errors + +#### For AI Agents (Claude Code) + +If you encounter an error **not documented** in the categories above: + +1. **Report to user** with all available details (error message, affected resources, namespace) +2. **Provide best-effort diagnostics** using MCP tools: + - `resources_get` to inspect resource status + - `pods_log` to check virt-launcher or compute container logs + - `events_list` to view Kubernetes events +3. **Suggest documentation update**: + ``` + ⚠️ This error is not yet documented in the troubleshooting guide. + + **Error**: + **Affected resource**: / + + I recommend adding this error to the troubleshooting guide: + - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file + - If it's a new error class → Create new category file in docs/troubleshooting/ + + Would you like me to help document this error for future reference? + ``` + +4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs + +--- + +#### For Human Contributors + +To document a new error: + +1. **Determine the category**: + - Scheduling issue → `scheduling-errors.md` + - Storage problem → `storage-errors.md` + - Lifecycle/start/stop → `lifecycle-errors.md` + - Runtime crash → `runtime-errors.md` + - Network issue → `network-errors.md` + - New error class → Create new category file (e.g., `security-errors.md`) + +2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): + - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback + - **Solutions**: MCP Tool approach → Implementation → CLI Fallback + - **Verification**: MCP Tool checks → Expected results → CLI Fallback + +3. **Update semantic index**: + - Edit `.ai-index/semantic-index.json` + - Add error to `error_to_docs_mapping` + - Update relevant skill's `primary_docs` or `secondary_docs` + - Increment token estimates if needed + +4. **Reference in skill files**: + - Update skill's Reference Documentation section + - Ensure skill's Document Consultation steps point to the right category + +5. **Update this INDEX.md**: + - Add error to the appropriate category section (lines 50-120) + - Update "Current Coverage" list above + +--- + +## 📚 Additional Resources + +- [rh-virt README](../../README.md) - MCP server setup and skill documentation +- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources +- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/submissions/vm-lifecycle-manager/skills/references/lifecycle-errors.md b/submissions/vm-lifecycle-manager/skills/references/lifecycle-errors.md new file mode 100644 index 0000000..6167fcc --- /dev/null +++ b/submissions/vm-lifecycle-manager/skills/references/lifecycle-errors.md @@ -0,0 +1,869 @@ +--- +title: VM Lifecycle Errors +category: kubevirt +sources: + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 +tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] +semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] +use_cases: [vm-deletion, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Lifecycle Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM lifecycle issues including start/stop failures and stuck termination states. + +**When to use this document**: +- VM stuck in `Terminating` state +- VM won't start (runStrategy is Always but VM never reaches Running) +- VM won't stop (runStrategy is Halted but VM never reaches Stopped) + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### VM Stuck in Terminating State + +**Symptom**: VM shows status `Terminating` but deletion never completes + +**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). + +**Possible Causes**: +- Finalizers blocking deletion +- PVC/DataVolume still attached and preventing cleanup +- VirtualMachineInstance (VMI) not terminating properly +- Custom controllers or operators blocking deletion +- Stuck virt-launcher pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check finalizers on the VM**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` + +**2. Check if VMI still exists**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name containing "virt-launcher-". + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check events for deletion issues**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results where `involvedObject.name` == "" and sort by timestamp. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' +``` + +**5. Check VM deletion timestamp**: + +Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. + +If present, VM is in deletion process. If null, VM is not being deleted. + +**Common Finalizer Patterns**: +- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer +- `foregroundDeletion` - Waits for dependent objects to be deleted +- Custom finalizers from operators + +**Solutions** (Use MCP Tools First): + +1. **Wait for dependent resources** (safest approach): + - Use `resources_list` to check if PVCs, DataVolumes are still deleting + - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) + +2. **Force delete VMI** (if VMI is stuck): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable - requires explicit user permission): + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +3. **Force delete virt-launcher pod** (if pod is stuck): + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + # Find the virt-launcher pod name first + oc get pods -n | grep virt-launcher- + + # Then delete it + oc delete pod virt-launcher--xxx -n --force --grace-period=0 + ``` + +4. **Remove finalizers** (⚠️ dangerous - use only as last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` + 2. Remove items from `.metadata.finalizers` array + 3. Update VM using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch not easily done via MCP): + ```bash + # This operation is complex for MCP - may need CLI + oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Check for protection labels** (vm-delete skill specific): + + Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. + + If value is "true", the vm-delete skill refuses deletion (this is expected behavior). + +**Verification**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error if deletion successful. + +**CLI Fallback**: +```bash +oc get vm -n +# Should return: Error from server (NotFound) +``` + +--- + + +--- + +### VM Won't Start (Non-Scheduling Issues) + +**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state + +**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. + +**Possible Causes**: +- Guest OS kernel panic or boot failure +- Cloud-init configuration errors +- Missing or corrupted disk image +- Insufficient memory for guest OS to boot +- QEMU/KVM virtualization errors +- VirtualMachineInstance (VMI) creation failures +- virt-launcher pod crashes + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VMI (VirtualMachineInstance) status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and overall VMI state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**2. Check VMI conditions for errors**: + +Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. View virt-launcher pod logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +First, get pod name from step 3, then: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +Look for QEMU errors, memory allocation failures, device errors. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**5. Check virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. + +Alternatively, use `pods_get` to get full pod details: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**6. Access VM console to see guest OS boot messages**: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +Look for kernel panic, initramfs errors, filesystem errors. + +**7. Check VM events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep -A 20 "Events:" +``` + +**Common Error Patterns**: + +1. **Guest OS Boot Failure**: + - Console shows kernel panic + - Guest hangs at GRUB or boot loader + - Cloud-init errors during first boot + +2. **Insufficient Memory**: + - Guest OS kills processes due to OOM + - VMI logs show memory allocation errors + +3. **QEMU Crashes**: + - virt-launcher logs show QEMU segfaults + - VMI repeatedly restarts + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for kernel panic, initramfs errors, filesystem errors. + +2. **Check virt-launcher pod logs for QEMU errors**: + + Use `pods_log` from diagnostic step 4 to view logs. + + Look for: + - "qemu-system-x86_64: ..." errors + - Memory allocation failures + - Device errors + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx + ``` + +3. **Increase memory if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check VMI memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +4. **Verify disk image integrity**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check PVC is bound: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + If using container disk, verify image exists and is accessible (check VMI spec). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get pvc -n + ``` + +5. **Check cloud-init configuration** (if applicable): + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. + + Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq + ``` + +6. **Restart VMI** (soft reset): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete VMI (VM controller will recreate it): + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for new VMI to start (use `resources_get` to check status). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n -w + ``` + +7. **Check virtualization extensions** (KVM): + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + + # In debug shell: + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should eventually return `Running`). + +Check VMI is running: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should show `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should eventually return: Running + +oc get vmi -n +# Should show: Running +``` + +--- + +### VM Won't Stop + +**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state + +**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. + +**Possible Causes**: +- Guest OS not responding to ACPI shutdown signal +- virt-launcher pod stuck and not terminating +- VirtualMachineInstance (VMI) deletion blocked by finalizers +- Guest shutdown scripts hanging +- Filesystem sync issues in guest OS + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (might show `Stopping`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**2. Check VMI status and deletion timestamp**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check VMI events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vmi -n | grep -A 10 "Events:" +``` + +**5. Check VMI finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.metadata.finalizers}' +``` + +**6. Check if guest is responsive** (if VMI still exists): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Patterns**: +- VMI shows `deletionTimestamp` but never actually deletes +- virt-launcher pod in `Terminating` state +- VM runStrategy is `Halted` but printableStatus shows `Stopping` + +**Solutions** (Use MCP Tools First): + +1. **Wait for graceful shutdown** (default: 30 seconds): + + Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. + +2. **Force stop by deleting VMI**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + This is the standard way to force-stop a VM. + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n + # Should return: Error from server (NotFound) + ``` + +3. **Force delete VMI with grace period** (if VMI won't delete): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +4. **Force delete virt-launcher pod**: + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find the pod using `pods_list_in_namespace` from diagnostic step 3. + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + ⚠️ **Note**: For force deletion with grace period, use CLI fallback. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') + oc delete pod $POD_NAME -n --force --grace-period=0 + ``` + +5. **Remove VMI finalizers** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VMI using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update VMI using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Patch VM runStrategy directly** (ensure consistency): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` (diagnostic step 1) + 2. Set `.spec.runStrategy` to `"Halted"` + 3. Update VM using `resources_create_or_update` with modified JSON + + **CLI Fallback** (merge patch easier via CLI): + Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Stopped` or `Halted`). + +Verify VMI is gone: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Verify virt-launcher pod is gone: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for pods matching `virt-launcher-`. Should return no results. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Stopped or Halted + +oc get vmi -n +# Should return: Error from server (NotFound) + +oc get pods -n | grep virt-launcher- +# Should return: No resources found +``` + +**Prevention**: +- Ensure guest OS has ACPI support enabled +- Use proper shutdown commands in guest OS +- Avoid forceful stops unless necessary (can corrupt guest filesystem) + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/submissions/vm-lifecycle-manager/skills/references/network-errors.md b/submissions/vm-lifecycle-manager/skills/references/network-errors.md new file mode 100644 index 0000000..82fa313 --- /dev/null +++ b/submissions/vm-lifecycle-manager/skills/references/network-errors.md @@ -0,0 +1,429 @@ +--- +title: VM Network Errors +category: kubevirt +sources: + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] +semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] +use_cases: [vm-creation, network-troubleshooting] +related_docs: [INDEX.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Network Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. + +**When to use this document**: +- VM created successfully but secondary networks not attached +- NetworkAttachmentDefinition not found errors +- Multus CNI failures +- SR-IOV device attachment issues + +**Skills that use this**: vm-create + +--- + +### Network Attachment Failures + +**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working + +**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). + +**Possible Causes**: +- NetworkAttachmentDefinition doesn't exist in the namespace +- Multus CNI not installed or not configured on cluster +- Namespace mismatch (NAD in different namespace than VM) +- Interface name conflicts in VM spec +- Bridge/network configuration errors in NAD +- SR-IOV device not available (if using SR-IOV) + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List NetworkAttachmentDefinitions in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "" +} +``` + +Review `.items[].metadata.name` for available NADs. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -n +``` + +**2. List NetworkAttachmentDefinitions in all namespaces**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition" +} +``` + +Omit `namespace` parameter to list across all namespaces. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -A +``` + +**3. Check specific NetworkAttachmentDefinition**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "", + "name": "" +} +``` + +Review `.spec.config` for CNI configuration. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definition -n -o yaml +``` + +**4. Check VM network configuration**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.networks` to see network references. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq +``` + +**5. Check VM domain interfaces**: + +Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq +``` + +**6. Check virt-launcher pod network annotations** (shows actual attachments): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**7. Check Multus is installed**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "openshift-multus" +} +``` + +Should show Multus CNI pods running. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n openshift-multus +``` + +**8. Check for errors in virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n +``` + +**Common Error Messages**: +- `"network-attachment-definition not found"` - NAD doesn't exist in namespace +- `"multus CNI not configured"` - Multus not installed or misconfigured +- `"interface name conflict"` - Duplicate interface names in VM spec +- `"failed to add network"` - CNI plugin error (check NAD config) +- `"no available devices"` - SR-IOV device not available (if using SR-IOV) + +**Solutions** (Use MCP Tools First): + +1. **Verify NetworkAttachmentDefinition exists in correct namespace**: + + Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. + + If NAD is in different namespace, copy it to VM namespace: + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get NAD from source namespace using `resources_get` + 2. Modify `.metadata.namespace` to target namespace + 3. Create NAD in target namespace using `resources_create_or_update` + + **CLI Fallback** (stream processing easier via CLI): + Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" + ```bash + oc get network-attachment-definition -n -o yaml | \ + sed "s/namespace: /namespace: /" | \ + oc apply -f - + ``` + +2. **Create missing NetworkAttachmentDefinition**: + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + Example: Linux bridge network + + **Parameters**: + ```json + { + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": { + "name": "vlan100", + "namespace": "" + }, + "spec": { + "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" + ```bash + cat < + spec: + config: '{ + "cniVersion": "0.3.1", + "type": "bridge", + "bridge": "br1", + "vlan": 100, + "ipam": { + "type": "host-local", + "subnet": "192.168.100.0/24" + } + }' + EOF + ``` + +3. **Check Multus CNI installation**: + + Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. + + To check cluster network operator: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterOperator" + } + ``` + + Filter for `network` operator. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n openshift-multus + oc get clusteroperators network + ``` + +4. **Fix interface name conflicts** (if VM has duplicate names): + + Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. + + Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. + + **CLI Fallback** (interactive edit easier via CLI): + Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' + oc edit vm -n + ``` + +5. **Validate NAD configuration syntax**: + + Use `resources_get` from diagnostic step 3, extract `.spec.config`. + + Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get network-attachment-definition -n -o jsonpath='{.spec.config}' + ``` + +6. **Check SR-IOV device availability** (if using SR-IOV networks): + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + List SR-IOV network node policies: + + **Parameters**: + ```json + { + "apiVersion": "sriovnetwork.openshift.io/v1", + "kind": "SriovNetworkNodePolicy", + "namespace": "openshift-sriov-network-operator" + } + ``` + + Check SR-IOV device plugin pods: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "openshift-sriov-network-operator" + } + ``` + + Filter for pods with "device-plugin" in name. + + Check available SR-IOV devices on node: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "Node", + "name": "" + } + ``` + + Review `.status.allocatable` for SR-IOV resources. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get sriovnetworknodepolicy -n openshift-sriov-network-operator + oc get pods -n openshift-sriov-network-operator | grep device-plugin + oc describe node | grep -A 10 "Allocatable:" + ``` + +7. **Recreate VM with corrected network configuration** (if needed): + + If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +After remediation, check virt-launcher pod network status: + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +Should show all attached networks with status. Example output: +```json +[ + { + "name": "openshift-sdn", + "interface": "eth0", + "ips": ["10.128.2.10"], + "default": true + }, + { + "name": "vlan100", + "interface": "net1", + "ips": ["192.168.100.5"] + } +] +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**Check from inside VM** (via console): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# In guest OS: +ip addr show +# Should show all network interfaces (eth0, net1, etc.) +``` + +**Common Network Types**: +- **Linux Bridge**: Layer 2 bridge for VLAN networks +- **SR-IOV**: High-performance direct device assignment +- **macvlan**: MAC-based VLAN for container networks +- **OVN-Kubernetes**: OpenShift native overlay network + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/submissions/vm-lifecycle-manager/skills/references/runtime-errors.md b/submissions/vm-lifecycle-manager/skills/references/runtime-errors.md new file mode 100644 index 0000000..804d3cb --- /dev/null +++ b/submissions/vm-lifecycle-manager/skills/references/runtime-errors.md @@ -0,0 +1,616 @@ +--- +title: VM Runtime Errors +category: kubevirt +sources: + - title: Kubernetes Pod Lifecycle + url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ + date_accessed: 2026-02-17 +tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] +semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] +use_cases: [vm-creation, vm-lifecycle, diagnostics] +related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Runtime Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. + +**When to use this document**: +- VM or virt-launcher pod shows `CrashLoopBackOff` status +- virt-launcher pod repeatedly restarting +- Guest OS kernel panics on boot + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### CrashLoopBackOff + +**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting + +**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. + +**Possible Causes**: +- Guest OS kernel panic on boot +- Insufficient resources (memory/CPU) for guest OS +- Corrupted disk image or filesystem +- QEMU/libvirt crashes due to configuration errors +- Missing or incompatible device drivers in guest +- Resource limits too low for virt-launcher pod +- Virtualization features (KVM) not available on node + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check virt-launcher pod restart count**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# Look at RESTARTS column +``` + +**2. View recent crash logs** (previous container instance): + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "previous": true, + "tail": 100 +} +``` + +Look for QEMU errors, kernel panics, or segfaults. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n virt-launcher--xxx --previous +``` + +**3. Check current virt-launcher logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. + +**CLI Fallback** (if MCP unavailable or all containers needed): +```bash +oc logs -n virt-launcher--xxx --all-containers +``` + +**4. Check VMI conditions for crash details**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Extract `.status.conditions` for crash details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**5. Check pod events for crash reasons**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" +``` + +**6. Check pod resource limits**: + +**MCP Tool**: `pods_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +Extract `.spec.containers[0].resources` for resource limits. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq +``` + +**7. Check node kubelet logs for OOM kills**: + +⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +oc adm node-logs -u kubelet | grep -i oom +``` + +**8. Access guest console** (if VM briefly starts): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Crash Patterns**: + +1. **Guest Kernel Panic**: + - Console logs show kernel panic messages + - Guest crashes immediately after boot + - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" + +2. **OOM (Out of Memory)**: + - Pod killed with reason: `OOMKilled` + - Guest runs out of memory during boot or operation + - virt-launcher logs show memory allocation failures + +3. **QEMU Crash**: + - virt-launcher logs show QEMU segmentation fault + - Symptoms: "qemu-system-x86_64: terminated by signal" + - Configuration incompatibility or QEMU bug + +4. **Disk Image Corruption**: + - Guest cannot boot from disk + - Filesystem errors in guest console + - DataVolume import failed + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for kernel panic or boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for: + - Kernel panic messages + - Initramfs errors + - Filesystem mounting failures + - Missing device errors + +2. **Review virt-launcher crash logs**: + + Use `pods_log` with `previous: true` from diagnostic step 2. + + Look for: + - QEMU command line errors + - Device initialization failures + - Memory allocation errors + - Signal termination (SIGSEGV, SIGABRT) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx --previous + ``` + +3. **Check for OOM (Out of Memory) kills**: + + **MCP Tool**: `pods_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + Extract `.status.containerStatuses[0].lastState.terminated.reason`. + + If returns `"OOMKilled"`: + - Option 1: Increase virt-launcher memory limits + - Option 2: Decrease guest memory allocation + - Option 3: Use smaller instance type + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' + ``` + +4. **Increase resources if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check current memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +5. **Verify disk image integrity**: + + **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters for list**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" + } + ``` + + **Parameters for specific DV**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check `.status.phase` (should be `Succeeded`). + + If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get datavolume -n -o jsonpath='{.status.phase}' + ``` + +6. **Check virtualization (KVM) availability**: + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +7. **Simplify VM configuration** (eliminate variables): + + Try creating minimal VM using vm-create skill with: + - Small instance type + - No secondary networks + - Simple container disk (e.g., Fedora) + - No cloud-init + + If minimal VM works, add features back one by one. + +8. **Recreate VM with different workload** (test disk image): + + If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) + +After remediation, check pod restart count stops increasing: + +**Parameters for pods**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). + +Check VM reaches Running state: + +**Parameters for VM**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# RESTARTS should stabilize + +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Running +``` + +Verify guest is responsive: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# Should show login prompt or OS console +``` + +**Advanced Debugging**: + +**MCP Tool**: `pods_exec` (from openshift-virtualization) + +Check libvirt domain XML: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["virsh", "dumpxml", "1"] +} +``` + +Check QEMU process: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["ps", "aux"] +} +``` + +Filter output for "qemu" process. + +**CLI Fallback** (if MCP unavailable): +```bash +oc exec -n virt-launcher--xxx -- virsh dumpxml 1 +oc exec -n virt-launcher--xxx -- ps aux | grep qemu +``` + +**Prevention**: +- Start with minimal VM configuration and add complexity gradually +- Use recommended instance types for your workload +- Test disk images before deploying to production +- Ensure nodes have adequate resources and KVM support +- Monitor resource usage and set appropriate limits + +--- + +## Workaround Patterns for MCP Tool Limitations + +### General Pattern: Diagnose → Propose → Confirm → Execute + +When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): + +**1. Diagnose** the root cause using Kubernetes commands +```bash +oc describe vm -n +oc get events -n --field-selector involvedObject.name= +``` + +**2. Propose** a manual workaround to the user with clear explanation +- Explain why the issue occurred +- Show what will be changed +- List alternative options + +**3. Confirm** with user before executing (Human-in-the-Loop) +- Wait for explicit "yes" or "apply workaround" +- Never auto-execute modifications + +**4. Execute** the workaround using oc/kubectl +```bash +oc patch vm -n --type=merge -p '...' +``` + +**5. Verify** the fix was successful +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**6. Document** the limitation and suggest filing enhancement request +- Note this is temporary until MCP tool is enhanced +- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues + +### Example: Adding Tolerations Workaround + +**Diagnostic Output**: +```markdown +## ⚠️ VM Scheduling Issue Detected + +**Root Cause**: Node taints prevent VM scheduling + +**Details**: +- Found 3 nodes with taint: `virtualization=true:NoSchedule` +- VM spec does not include matching tolerations +- This prevents VM scheduling on virtualization-dedicated nodes +``` + +**Proposed Workaround**: +```bash +oc patch vm web-server -n vms --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' +``` + +**User Confirmation Required**: +``` +How would you like to proceed? +- "apply workaround" - I'll patch the VM with tolerations +- "manual" - I'll provide instructions for you to apply manually +- "cancel" - Delete the VM and abort creation +``` + +**After Execution**: +```markdown +## ✓ Workaround Applied Successfully + +**Action**: Added tolerations for taint `virtualization=true:NoSchedule` +**New Status**: Stopped (VM can now be scheduled) + +**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. +Future VMs in this cluster will need the same fix until the tool is enhanced. +``` + +--- + +## VM Status Reference + +### Status Values + +| Status | Meaning | Action Required | +|--------|---------|-----------------| +| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | +| `Running` | VM is running | Normal | +| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | +| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | +| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | +| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | +| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | +| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | +| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | +| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | + +### Checking VM Status + +```bash +# Get printable status +oc get vm -n -o jsonpath='{.status.printableStatus}' + +# Get detailed status and conditions +oc get vm -n -o jsonpath='{.status}' | jq + +# Watch status changes in real-time +oc get vm -n -w +``` + +--- + +## Best Practices for Agents + +When implementing diagnostic workflows: + +1. **Always verify VM status** after creation (wait 5-10 seconds first) +2. **Consult this document** when encountering error status values +3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) +4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) +5. **Respect human-in-the-loop** for all VM modifications +6. **Document temporary workarounds** and their limitations clearly +7. **Suggest filing issues** for missing MCP tool features + +### Document Consultation Pattern + +```markdown +**Document Consultation** (REQUIRED): +1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes +2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." +``` + +--- + +## Known MCP Tool Limitations + +### vm_create tool + +**Currently Supported**: +- ✓ Namespace, name (required) +- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) +- ✓ Size hints (small, medium, large) +- ✓ Storage size +- ✓ Autostart flag +- ✓ Networks (Multus NetworkAttachmentDefinitions) +- ✓ Performance family (u1, o1, c1, m1) +- ✓ Instance type, preference + +**Not Currently Supported** (requires workarounds): +- ✗ Tolerations (for node taints) +- ✗ Node selectors +- ✗ Affinity/anti-affinity rules +- ✗ Resource requests/limits (beyond instance type) +- ✗ Custom labels/annotations +- ✗ SSH keys injection +- ✗ Cloud-init user data + +**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. + +**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues + +--- + +## Additional Resources + +- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) +- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) +- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) +- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) + +--- + +[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/submissions/vm-lifecycle-manager/skills/references/scheduling-errors.md b/submissions/vm-lifecycle-manager/skills/references/scheduling-errors.md new file mode 100644 index 0000000..e47c8c4 --- /dev/null +++ b/submissions/vm-lifecycle-manager/skills/references/scheduling-errors.md @@ -0,0 +1,417 @@ +--- +title: VM Scheduling Errors +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 +tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] +semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] +use_cases: [vm-creation, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Scheduling Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**When to use this document**: +- VM shows status `ErrorUnschedulable` after creation or start attempt +- VM events mention scheduling failures, taints, resources, or node selectors + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +## ErrorUnschedulable + +**Symptom**: VM shows status `ErrorUnschedulable` after creation + +**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**Possible Causes**: + +### 1. Node Taints (Most Common) + +Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM events for scheduling failures**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `involvedObject.name` == "" and look for messages like: +- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= +``` + +**2. Check node taints in the cluster**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' +``` + +**Common Taint Patterns**: +- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule +- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes +- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads + +**Solution - Add Tolerations to VM**: + +The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. + +**Workaround (post-creation using MCP Tools)**: + +**Step 1**: Get current VM spec + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Step 2**: Modify the returned JSON to add tolerations + +Add to `.spec.template.spec.tolerations`: +```json +{ + "tolerations": [ + { + "key": "virtualization", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule" + } + ] +} +``` + +**Step 3**: Update VM with modified spec + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +```json +{ + "resource": "" +} +``` + +Pass the complete modified VM resource as YAML or JSON string. + +**Step 4**: Verify tolerations were added + +Use `resources_get` again and check `.spec.template.spec.tolerations` in response. + +**Step 5**: Check if VM status improved + +Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. + +**CLI Fallback** (if MCP patch is too complex): +```bash +# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' + +# Verify tolerations +oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq + +# Check status +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**Example - Multiple Tolerations**: +```bash +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" + - key: "dedicated" + operator: "Equal" + value: "virt-workloads" + effect: "NoSchedule" +' +``` + +**Toleration Operators**: +- `Equal` - Key and value must match exactly +- `Exists` - Only key must exist (ignores value) + +**Toleration Effects**: +- `NoSchedule` - Don't schedule new pods (existing pods continue) +- `PreferNoSchedule` - Avoid scheduling if possible +- `NoExecute` - Don't schedule AND evict existing pods + +**Alternative Solutions**: +1. **Remove node taints** (if you have cluster-admin access): + ```bash + oc adm taint nodes virtualization=true:NoSchedule- + ``` + +2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits + +3. **File enhancement request** - Request tolerations support in openshift-mcp-server: + https://github.com/openshift/openshift-mcp-server/issues + +--- + +### 2. Insufficient Resources + +Not enough CPU, memory, or storage available on any node. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM resource requests**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' +``` + +**2. Check node resource availability**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. + +Alternatively, use `nodes_top` MCP tool for current resource usage. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe nodes | grep -A 5 "Allocated resources" +``` + +**3. Look for VM events mentioning "Insufficient"**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep "Insufficient" +``` + +**Example Event**: +``` +0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. +``` + +**Solutions** (Use MCP Tools First): + +1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) +2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill +3. **Delete unused VMs** - Use vm-delete skill to free up resources +4. **Check resource quotas**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters for quota**: + ```json + { + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" + } + ``` + + **Parameters for limit range**: + ```json + { + "apiVersion": "v1", + "kind": "LimitRange", + "namespace": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + oc describe limitrange -n + ``` + +--- + +### 3. Node Selector Mismatch + +VM requires specific node labels that don't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM node selector requirements**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.nodeSelector` to see required node labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' +``` + +**2. List available node labels**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.metadata.labels` for available labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes --show-labels +``` + +**3. Check if any nodes match the selector**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node", + "labelSelector": "=" +} +``` + +Should return at least one node with matching labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -l = +``` + +**Solutions** (Use MCP Tools First): + +**Option 1: Remove node selector from VM** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get current VM using `resources_get` (diagnostic step 1) +2. Remove `.spec.template.spec.nodeSelector` field +3. Update VM using `resources_create_or_update` with modified JSON + +**CLI Fallback** (JSON patch easier via CLI): +Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" +```bash +oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' +``` + +**Option 2: Add label to nodes** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get node using `resources_get` +2. Add label to `.metadata.labels` +3. Update node using `resources_create_or_update` + +⚠️ **Note**: Node labeling typically requires cluster admin privileges. + +**CLI Fallback** (simpler via CLI): +Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" +```bash +oc label node = +``` + +--- + +[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/submissions/vm-lifecycle-manager/skills/references/storage-errors.md b/submissions/vm-lifecycle-manager/skills/references/storage-errors.md new file mode 100644 index 0000000..16ba584 --- /dev/null +++ b/submissions/vm-lifecycle-manager/skills/references/storage-errors.md @@ -0,0 +1,1011 @@ +--- +title: VM Storage Errors +category: kubevirt +sources: + - title: KubeVirt CDI - DataVolumes + url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ + date_accessed: 2026-02-17 + - title: Kubernetes Persistent Volumes + url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ + date_accessed: 2026-02-17 +tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] +semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] +use_cases: [vm-creation, vm-deletion, vm-cloning] +related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] +last_updated: 2026-02-17 +--- + +# VM Storage Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. + +**When to use this document**: +- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` +- Storage deletion fails after VM deletion +- DataVolume cloning operations fail +- PVC provisioning issues + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### ErrorDataVolumeNotReady + +**Symptom**: VM shows status `ErrorDataVolumeNotReady` + +**Description**: The DataVolume (persistent storage) backing the VM is not ready. + +**Possible Causes**: + +#### 1. DataVolume Still Provisioning + +Storage provisioning takes time, especially for large disks or when importing images. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Get detailed DataVolume information**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` and `.status.conditions` for provisioning details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o json +``` + +**3. Check PVC (PersistentVolumeClaim) bound status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Check `.status.phase` for each PVC (should be `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. + +#### 2. Storage Class Not Found + +The requested storage class doesn't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List available storage classes**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" +} +``` + +Review the list of available storage classes (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass +``` + +**2. Check DataVolume's requested storage class**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.spec.pvc.storageClassName` in the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' +``` + +**Solution**: +1. Use a valid storage class from the cluster +2. Recreate VM with correct storage class parameter + +#### 3. Insufficient Storage Quota + +Namespace has insufficient storage quota to provision the PVC. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check resource quotas**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**2. Check storage usage**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase +``` + +**Solution**: +1. Request quota increase from cluster admin +2. Delete unused PVCs to free quota +3. Reduce VM storage size + +--- + +### ErrorPvcNotFound + +**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List PVCs in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review the list of available PVCs (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check VM's PVC references**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' +``` + +**Solution**: +- Wait for DataVolume to create the PVC +- Manually create missing PVC +- Fix VM spec to reference correct PVC name + +--- + + +--- + +### Storage Deletion Failures + +**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace + +**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. + +**Possible Causes**: +- PVC still bound to active PersistentVolume with `Retain` policy +- DataVolume still being referenced by another resource +- CDI (Containerized Data Importer) controller issues +- Storage class retention policy preventing deletion +- Finalizers on PVC/DataVolume blocking cleanup +- PVC still mounted by a pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check PVC status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check specific PVC phase**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Released` or `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o jsonpath='{.status.phase}' +``` + +**3. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**4. Check what's using the PVC**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' +``` + +**5. Check PVC finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. + +**6. Check DataVolume finalizers**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.metadata.finalizers}' +``` + +**7. Check PV reclaim policy**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume" +} +``` + +Filter results for PV where `.spec.claimRef.name` matches ``. + +To get specific PV policy: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume", + "name": "" +} +``` + +Check `.spec.persistentVolumeReclaimPolicy`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pv | grep +oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' +``` + +**Common Finalizer Patterns**: +- `kubernetes.io/pvc-protection` - Protects PVC while in use +- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer + +**Solutions** (Use MCP Tools First): + +1. **Delete DataVolume first, then PVC**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete DataVolume first (often blocks PVC deletion): + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Wait a few seconds, then delete PVC: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete datavolume -n + oc delete pvc -n + ``` + +2. **Check for pods still using PVC**: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "" + } + ``` + + Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. + + Then delete the pods using `pods_delete`: + + **Parameters**: + ```json + { + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' + oc delete pod -n + ``` + +3. **Force delete PVC** (if safe to do so): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete pvc -n --grace-period=0 --force + ``` + +4. **Remove finalizers from PVC** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PVC using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update PVC using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Remove finalizers from DataVolume** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current DataVolume using `resources_get` (diagnostic step 6) + 2. Remove items from `.metadata.finalizers` array + 3. Update DataVolume using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Change PV reclaim policy** (if PV has Retain policy): + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PV policy using `resources_get` (diagnostic step 7) + 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` + 3. Update PV using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" + ```bash + oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' + oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' + ``` + +**Storage Quota Check** (Use MCP Tools First): + +After deletion, verify storage quota is freed: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters for quota check**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.used` to verify storage quota is freed. + +**Parameters for PVC verification**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +oc get pvc -n +``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Confirm PVC is deleted: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Confirm DataVolume is deleted: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +# Should return: Error from server (NotFound) + +oc get datavolume -n +# Should return: Error from server (NotFound) +``` + +--- + +### DataVolume Cloning Failures + +**Symptom**: VM clone created successfully but DataVolume clone operation fails + +**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. + +**Possible Causes**: +- CSI driver doesn't support volume cloning +- Source PVC storage class incompatible with cloning +- Cross-namespace cloning not permitted by storage backend +- Insufficient storage quota in target namespace +- Source PVC not in `Bound` state +- Storage class doesn't have volume cloning enabled +- CDI (Containerized Data Importer) controller issues + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume clone status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and `.metadata.name`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Check DataVolume events for errors**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe datavolume -n +``` + +**3. Check DataVolume phase**: + +Use `resources_get` from step 1, extract `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +``` + +**4. Check if storage class supports cloning**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "name": "" +} +``` + +Review the full YAML output for cloning-related configurations. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass -o yaml | grep -A 5 -i clone +``` + +**5. Check CSI driver capabilities**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "CSIDriver" +} +``` + +Review `.items[].metadata.name` for available CSI drivers. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get csidriver +``` + +**6. Check source PVC status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Bound` for cloning to work). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**7. Check target namespace storage quota**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**8. Check CDI controller logs**: + +**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) + +First, list pods in openshift-cnv namespace: + +**Parameters for pods_list_in_namespace**: +```json +{ + "namespace": "openshift-cnv", + "labelSelector": "app.kubernetes.io/component=cdi-deployment" +} +``` + +Then get logs using `pods_log`: + +**Parameters**: +```json +{ + "namespace": "openshift-cnv", + "name": "", + "tail": 100 +} +``` + +**CLI Fallback** (if MCP unavailable or easier via CLI): +```bash +oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') +``` + +**Common Error Messages**: +- `"volume cloning is not supported"` - CSI driver lacks clone capability +- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage +- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace +- `"insufficient quota"` - Target namespace lacks storage quota +- `"source PVC not bound"` - Source PVC must be in Bound state for cloning +- `"StorageClass does not support cloning"` - Storage class configuration issue + +**Solutions** (Use MCP Tools First): + +1. **Check storage class clone support**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" + } + ``` + + For each storage class in `.items[]`, check: + - `.metadata.name` (storage class name) + - `.provisioner` (CSI driver) + + Storage classes using CSI drivers typically support cloning. Look for provisioners like: + - `csi.ovirt.org` (oVirt CSI) + - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) + - `ebs.csi.aws.com` (AWS EBS CSI) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner + ``` + +2. **Verify source PVC is bound**: + + Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pvc -n -o jsonpath='{.status.phase}' + ``` + +3. **Check target namespace quota**: + + Use `resources_list` from diagnostic step 7 to check quota. + + If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + ``` + +4. **Use snapshot-based cloning** (alternative method): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Step 1**: Create VolumeSnapshot of source PVC + + **Parameters**: + ```json + { + "apiVersion": "snapshot.storage.k8s.io/v1", + "kind": "VolumeSnapshot", + "metadata": { + "name": "-snapshot", + "namespace": "" + }, + "spec": { + "source": { + "persistentVolumeClaimName": "" + } + } + } + ``` + + **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) + + **Step 3**: Create new DataVolume from snapshot + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": "-rootdisk", + "namespace": "" + }, + "spec": { + "source": { + "snapshot": { + "name": "-snapshot", + "namespace": "" + } + }, + "storage": { + "resources": { + "requests": { + "storage": "50Gi" + } + }, + "storageClassName": "" + } + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" + ```bash + cat <-snapshot + namespace: + spec: + source: + persistentVolumeClaimName: + EOF + + oc get volumesnapshot -snapshot -n + + cat <-rootdisk + namespace: + spec: + source: + snapshot: + name: -snapshot + namespace: + storage: + resources: + requests: + storage: 50Gi + storageClassName: + EOF + ``` + +5. **Use "new empty storage" option** (vm-clone skill): + - If cloning isn't supported, create VM with empty storage + - Manually copy data if needed + +6. **Cross-namespace cloning workaround**: + - Some storage backends require snapshot for cross-namespace cloning + - Create snapshot in source namespace, restore in target namespace (see solution 4 above) + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Check DataVolume reached Succeeded phase: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should return `Succeeded`). + +Check PVC was created and bound: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "-rootdisk" +} +``` + +Check `.status.phase` (should return `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +# Should return: Succeeded + +oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' +# Should return: Bound +``` + +**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) From 1a714e144242e78fc3f02ea6d342d60b6ad291f8 Mon Sep 17 00:00:00 2001 From: gziv Date: Sun, 31 May 2026 15:19:57 +0300 Subject: [PATCH 2/2] retrigger: queue runner parallel