From 1072b2417091b1c7725daab00c1d008741fc08c0 Mon Sep 17 00:00:00 2001 From: gziv Date: Sun, 31 May 2026 12:17:21 +0300 Subject: [PATCH 1/2] feat: add vm-delete ASE evaluation submission --- submissions/vm-delete/evals/evals.json | 19 + submissions/vm-delete/metadata.yaml | 5 + submissions/vm-delete/skills/SKILL.md | 410 +++++++ .../vm-delete/skills/references/INDEX.md | 332 ++++++ .../skills/references/lifecycle-errors.md | 869 ++++++++++++++ .../skills/references/network-errors.md | 429 +++++++ .../skills/references/runtime-errors.md | 616 ++++++++++ .../skills/references/scheduling-errors.md | 417 +++++++ .../skills/references/storage-errors.md | 1011 +++++++++++++++++ 9 files changed, 4108 insertions(+) create mode 100644 submissions/vm-delete/evals/evals.json create mode 100644 submissions/vm-delete/metadata.yaml create mode 100644 submissions/vm-delete/skills/SKILL.md create mode 100644 submissions/vm-delete/skills/references/INDEX.md create mode 100644 submissions/vm-delete/skills/references/lifecycle-errors.md create mode 100644 submissions/vm-delete/skills/references/network-errors.md create mode 100644 submissions/vm-delete/skills/references/runtime-errors.md create mode 100644 submissions/vm-delete/skills/references/scheduling-errors.md create mode 100644 submissions/vm-delete/skills/references/storage-errors.md diff --git a/submissions/vm-delete/evals/evals.json b/submissions/vm-delete/evals/evals.json new file mode 100644 index 0000000..1d3aaac --- /dev/null +++ b/submissions/vm-delete/evals/evals.json @@ -0,0 +1,19 @@ +{ + "skill_name": "vm-delete", + "evals": [ + { + "id": "vm-delete-eval", + "name": "VM Deletion Safety Methodology", + "prompt": "You are an OpenShift Virtualization administrator. A team lead asks you to delete VM `legacy-app` in namespace `decommission`.\n\nDescribe your complete deletion methodology: what safety checks you would perform, how you handle different deletion scopes, what safeguards prevent accidental deletion, and how you handle edge cases like stuck VMs or protected resources.", + "expected_output": "A deletion methodology with strict safety protocols: typed name confirmation, protected label checks, stop-before-delete, deletion scope options (VM-only vs VM+storage), finalizer handling for stuck VMs, and PV reclaim policy considerations.", + "assertions": [ + "The output describes stopping a running VM before deletion as a required pre-deletion step.", + "The output presents deletion scope options distinguishing VM-only deletion from VM+storage deletion, and warns about orphaned PVCs or DataVolumes.", + "The output requires typed VM name confirmation (exact match) as a safeguard before proceeding with deletion.", + "The output describes checking for a protected: true label that blocks deletion and explains how to handle it.", + "The output describes a finalizer removal procedure for VMs stuck in Terminating state.", + "The output addresses PV reclaim policy Retain potentially blocking PVC cleanup after VM deletion." + ] + } + ] +} diff --git a/submissions/vm-delete/metadata.yaml b/submissions/vm-delete/metadata.yaml new file mode 100644 index 0000000..a94b072 --- /dev/null +++ b/submissions/vm-delete/metadata.yaml @@ -0,0 +1,5 @@ +name: vm-delete +description: "Permanently delete virtual machines and their associated resources from" +persona: rh-virt +version: "1.0.0" +eval_engine: ase diff --git a/submissions/vm-delete/skills/SKILL.md b/submissions/vm-delete/skills/SKILL.md new file mode 100644 index 0000000..b8eb570 --- /dev/null +++ b/submissions/vm-delete/skills/SKILL.md @@ -0,0 +1,410 @@ +--- +name: vm-delete +description: 'Permanently delete virtual machines and their associated resources from + OpenShift Virtualization. + + + Use when: + + - "Delete VM [name]" + + - "Remove virtual machine [name]" + + - "Destroy VM [name]" + + - "Clean up VM [name]" + + + This skill handles permanent VM deletion with strict safety confirmations and typed + verification. + + + NOT for power management (use vm-lifecycle-manager to stop VMs). + + ' +license: Apache-2.0 +--- + +# /vm-delete Skill + +Permanently delete virtual machines and their associated resources (storage, DataVolumes) from OpenShift Virtualization clusters. This skill enforces strict safety protocols including typed confirmation and pre-deletion validation. + +## Prerequisites + +**Required MCP Server**: `openshift-virtualization` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools**: +- `resources_get` (from openshift-virtualization) - Verify VM exists and get details +- `resources_delete` (from openshift-virtualization) - Delete Kubernetes resources +- `resources_list` (from openshift-virtualization) - List dependent resources (PVCs, DataVolumes) +- `resources_create_or_update` (from openshift-virtualization) - Update resources (e.g., remove finalizers) +- `vm_lifecycle` (from openshift-virtualization) - Stop running VMs before deletion +- `pods_list_in_namespace` (from openshift-virtualization) - List pods for diagnostics + +**Required Environment Variables**: +- `KUBECONFIG` - Path to Kubernetes configuration file with cluster access + +**Required Cluster Setup**: +- OpenShift cluster (>= 4.19) +- OpenShift Virtualization operator installed +- ServiceAccount with RBAC permissions to delete VirtualMachine and PVC resources + +### Prerequisite Verification + +**Before executing:** +1. Check `openshift-virtualization` exists in `mcps.json` → If missing, report setup +2. Verify `KUBECONFIG` is set (presence only, never expose value) → If missing, report +3. Check RBAC permissions (optional) → Verify delete permissions for VirtualMachine and PVC + +**Human Notification Protocol:** `❌ Cannot execute vm-delete: MCP server not available. Setup: Add to mcps.json, set KUBECONFIG, restart Claude Code. Docs: https://github.com/openshift/openshift-mcp-server` + +⚠️ **SECURITY**: Never display KUBECONFIG path or credential values. + +## When to Use This Skill + +**Trigger when:** +- User explicitly invokes `/vm-delete` command +- User requests permanent VM deletion +- User wants to clean up test/development VMs +- User needs to free cluster resources +- User wants to decommission VMs + +**User phrases:** +- "Delete VM test-vm in namespace dev" +- "Remove virtual machine web-server" +- "Destroy VM old-database" +- "/vm-delete" + +**Do NOT use when:** +- Stop VM temporarily → `/vm-lifecycle-manager` +- Create VM → `/vm-create` +- View VMs → `/vm-inventory` + +## Workflow + +### Step 1: Gather and Validate + +**CRITICAL**: Complete ALL validation BEFORE user confirmation. + +**Required from user:** VM Name, Namespace + +**1.1: Verify VM Exists** + +**MCP Tool**: `resources_get` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", namespace=``, name=``) + +**Errors:** +- Not found → Report error, suggest vm-inventory +- Permission denied → Report RBAC error + +**1.2: Check Protection Label** + +Check `metadata.labels` for `protected: "true"`. + +**If protected:** Report: `❌ Cannot Delete Protected VM. VM has protected label. Remove: oc label vm -n protected-. Operation cancelled.` **STOP workflow.** + +**1.3: Check Running State** + +Check `status.printableStatus` (Running/Starting/Migrating = running, Stopped/Halted = stopped). + +**If running:** Report: `⚠️ VM Running. Must stop before deletion. Options: stop-and-delete / cancel` **Wait for response.** + +**1.4: Stop VM (if applicable)** + +**ONLY if user chose "stop-and-delete".** + +**MCP Tool**: `vm_lifecycle` (namespace=``, name=``, action="stop") + +Report: `⏸️ Stopping VM... Wait 10-30s.` **Wait 10s**, verify stopped. + +**1.5: Discover Storage** + +**MCP Tool**: `resources_list` + +**DataVolumes**: apiVersion="cdi.kubevirt.io/v1beta1", kind="DataVolume", namespace=``, labelSelector="vm.kubevirt.io/name=``" + +**PVCs** (if no DVs): apiVersion="v1", kind="PersistentVolumeClaim", namespace=``, labelSelector="vm.kubevirt.io/name=``" + +Parse: Extract names, calculate total storage size. + +### Step 2: Present Scope and Get Options + +Display deletion scope in this format: + +```markdown +## ⚠️ VM Deletion - Review Scope + +**VM**: `` | **Namespace**: `` | **Status**: + +**Resources**: VM `` (Age: , vCPU: , Memory: ) +**Storage**: DataVolume `` (30Gi), PVC `` (30Gi) - Total: 30Gi +OR **Storage**: None (ephemeral) + +### Deletion Options +**1: VM Only** - Preserves storage for reuse +**2: VM + Storage** ← Recommended (test/dev) - Frees storage +**3: Cancel** + +Select (1, 2, or 3): +``` + +**Wait for selection.** Handle: 3→Cancel, 1→delete_storage=false, 2→delete_storage=true + +### Step 3: Typed Confirmation (MANDATORY) + +**CRITICAL**: User MUST type exact VM name. + +Display typed confirmation prompt (adjust based on delete_storage flag): + +```markdown +## 🔴 PERMANENT DELETION - Typed Confirmation Required + +**CANNOT BE UNDONE** + +**Will delete**: +✗ VirtualMachine: `` (namespace: ``) +[If delete_storage=true, show:] +✗ DataVolume: `` | ✗ PVC: `` | ✗ All data lost +[If delete_storage=false, show:] +✓ Storage PRESERVED + +Type `` to confirm: _____ +``` + +**Validation:** +- Match → Continue to Step 4 (Execute Deletion) +- Mismatch → Report: `❌ Confirmation Failed. You typed: . Expected: . Cancelled.` **STOP.** + +### Step 4: Execute Deletion + +**ONLY AFTER**: ✓ Validation ✓ Option selected ✓ Typed name confirmed + +**4.1: Delete VM** + +**MCP Tool**: `resources_delete` (apiVersion="kubevirt.io/v1", kind="VirtualMachine", namespace=``, name=``) + +**Errors:** Fails → Report, don't delete storage; Not found → Continue + +Report: `🗑️ Deleting VM... ✓ Deleted` + +**4.2: Delete Storage (if delete_storage=true)** + +**For each DataVolume:** +**MCP Tool**: `resources_delete` (apiVersion="cdi.kubevirt.io/v1beta1", kind="DataVolume", namespace=``, name=``) + +**For each PVC:** +**MCP Tool**: `resources_delete` (apiVersion="v1", kind="PersistentVolumeClaim", namespace=``, name=``) + +**Errors:** Report which failed, continue with others + +Report: `🗑️ Deleting storage... ✓ DV deleted (storage freed) ✓ PVC deleted` + +### Step 5: Report Results + +**Success (with storage):** + +```markdown +## ✓ VM Deleted (Complete Cleanup) +**Deleted**: VM + DataVolume + PVC | **Freed**: +**Impact**: Permanent removal. Cannot recover. +**Verify**: "List VMs in namespace " - VM should not appear +``` + +**Success (storage preserved):** + +```markdown +## ✓ VM Deleted (Storage Preserved) +**Deleted**: VM | **Preserved**: DataVolume + PVC () +**Reuse**: Create new VM with existing DV/PVC +**Delete later**: `oc delete datavolume -n ` +``` + +**Partial failure (storage failed):** + +**OPTIONAL**: Read [storage-errors.md](../../docs/troubleshooting/storage-errors.md) for PVC cleanup. Output: "Consulted storage-errors.md for failure." + +```markdown +## ⚠️ Partial Deletion +**Deleted**: VM | **Failed**: DV/PVC (error: ) +**Action**: Manual cleanup: `oc delete datavolume -n ` +``` + +**Complete failure:** + +**OPTIONAL**: Read [lifecycle-errors.md](../../docs/troubleshooting/lifecycle-errors.md) for deletion failures. Output: "Consulted lifecycle-errors.md for failure causes." + +```markdown +## ❌ VM Deletion Failed +**Error**: +**Troubleshooting**: Check permissions, verify VM exists, check finalizers (see lifecycle-errors.md) +``` + +## Common Issues + +### Issue 1: VM Not Found +**Error**: "VirtualMachine not found" +**Solution**: Verify name/namespace with vm-inventory. Check spelling. + +### Issue 2: RBAC Permissions +**Error**: "Forbidden: Cannot delete VirtualMachines" +**Solution**: Verify delete permissions for VirtualMachine and PVC. Contact admin. Check: `oc auth can-i delete virtualmachines -n ` + +### Issue 3: VM Has Finalizers +**Error**: "VM deletion blocked by finalizers" +**Solution**: Consult [lifecycle-errors.md](../../docs/troubleshooting/lifecycle-errors.md) "VM Stuck in Terminating" for MCP-first approach using `resources_get` to check finalizers, `resources_create_or_update` to remove if needed. + +### Issue 4: Storage Deletion Failure +**Error**: "PVC deletion failed: resource in use" +**Solution**: Verify VM deleted first. Consult [storage-errors.md](../../docs/troubleshooting/storage-errors.md) for MCP-first diagnostics using `pods_list_in_namespace` to check mounts, `resources_get` for PVC status. + +### Issue 5: Confirmation Mismatch +**Error**: "Names do not match" +**Solution**: Type exact VM name (case-sensitive). Copy-paste from deletion scope. Retry. + +### Issue 6: Protected VM +**Error**: "VM has protected label" +**Solution**: Remove: `oc label vm -n protected-`. Retry deletion. + +## Dependencies + +### Required MCP Servers +- `openshift-virtualization` - OpenShift MCP with KubeVirt toolset (https://github.com/openshift/openshift-mcp-server) + +### Required MCP Tools +- `resources_get` - Get VM (apiVersion, kind, namespace, name) +- `resources_delete` - Delete resources (apiVersion, kind, namespace, name) +- `resources_list` - List resources (apiVersion, kind, namespace, labelSelector) +- `resources_create_or_update` - Update resources (resource JSON) - for finalizer removal +- `vm_lifecycle` - VM lifecycle (namespace, name, action: stop) +- `pods_list_in_namespace` - List pods (namespace) - for PVC mount diagnostics + +### Related Skills +- `vm-lifecycle-manager` - Stop VMs | `vm-inventory` - List VMs | `vm-create` - Create VMs | `vm-clone` - Clone VMs + +### Reference Documentation +- [lifecycle-errors.md](../../docs/troubleshooting/lifecycle-errors.md) - Deletion failures, finalizers, stuck Terminating (consulted on deletion failure) +- [storage-errors.md](../../docs/troubleshooting/storage-errors.md) - Storage deletion, PVC cleanup (consulted on storage failure) +- [Troubleshooting INDEX](../../docs/troubleshooting/INDEX.md) - Full error index +- [OpenShift Virt Docs](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html-single/virtualization/index) +- [KubeVirt API](https://kubevirt.io/api-reference/) +- [K8s Finalizers](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/) + +## Critical: Human-in-the-Loop Requirements + +**CRITICAL: IRREVERSIBLE operations.** You MUST: + +1. **Pre-Deletion Validation** (Execute FIRST) + - Verify VM exists (`resources_get`) + - Check VM running state + - Discover dependent resources + - Check protection labels (`protected: "true"`) + +2. **Safety Checks** + - **REFUSE deletion** if protected label exists + - **REQUIRE VM stopped** if running + - **List all resources** to be deleted + +3. **Typed Confirmation (MANDATORY)** + - Display complete deletion scope + - **Require exact VM name** to confirm + - Accept only exact match (case-sensitive) + - Mismatch → Cancel, don't proceed + - Match → Proceed directly to deletion + +4. **Deletion Options** + - Ask: VM only / VM+Storage / Cancel + - **NEVER assume** which option + +5. **Never Auto-Execute** + - NEVER delete without typed confirmation + - NEVER proceed if typed name mismatches + - NEVER skip typed verification + +**Why**: Permanent data loss, service impact, accidental deletion prevention + +## Security Considerations + +- **RBAC**: Requires delete VirtualMachine, PVC, DataVolume +- **Typed Verification**: Prevents accidental deletion +- **Protection Labels**: `protected: "true"` blocks deletion +- **Audit**: Kubernetes audit logs +- **Graceful Shutdown**: Stops VMs before deletion +- **KUBECONFIG**: Never exposed +- **Namespace Isolation**: RBAC-enforced +- **No Force Delete**: Never uses `--force` or `--grace-period=0` + +## Example Usage + +### Example 1: Complete Deletion (VM + Storage) + +``` +User: "Delete test-vm in dev" +Agent: [Validates ✓, discovers 30Gi storage] + [Shows deletion scope - see Step 2 format] +User: "2" [VM+Storage] +Agent: [Shows typed confirmation - see Step 3 format] +User: "test-vm" +Agent: [Executes: VM ✓, DV ✓, PVC ✓] + [Shows success report - see Step 5 format] +## ✓ VM Deleted (Complete Cleanup) | Freed: 30Gi +``` + +### Example 2: VM Only (Preserve Storage) + +``` +User: "Remove web-server but keep storage" +Agent: [Validates, shows scope] +User: "1" [VM only] +Agent: [Typed confirmation] +User: "web-server" +Agent: [Executes deletion] +## ✓ VM Deleted (Storage Preserved) | Preserved: 50Gi +``` + +### Example 3: Protected VM (Blocked) + +``` +User: "Delete production-db" +Agent: [Validates: protected label found] +❌ Cannot Delete Protected VM. Remove label: oc label vm production-db protected- +``` + +### Example 4: Running VM (Stop First) + +``` +User: "Delete api-server" +Agent: [Validates: VM running] +⚠️ VM Running. Options: stop-and-delete / cancel +User: "stop-and-delete" +Agent: [Stops VM, waits, continues with deletion] +``` + +### Example 5: User Cancels + +``` +User: "Delete test-vm" +Agent: [Shows typed confirmation] +Type `test-vm` to confirm: +User: "wait, cancel" +❌ Confirmation Failed. You typed: wait, cancel. Expected: test-vm. Cancelled. +``` + +### Example 6: Dry-Run Preview + +``` +User: "What would be deleted if I delete test-vm?" +Agent: [Execute Step 1-2 only, stop before confirmation] +## 🔍 Deletion Preview +**Would delete (Option 2)**: VM + DV + PVC (30Gi freed) +This is preview only. No resources deleted. +``` + +## Advanced Features + +### Batch Deletion +Delete multiple VMs: `"Delete VMs test-01, test-02, test-03 in dev"` → Process each VM individually through the full workflow (Steps 1-5), requiring typed confirmation of each VM name separately. Never use a generic batch confirmation — each VM must be confirmed by its exact name. + +### Dry-Run Mode +Show deletion scope without executing: Execute Step 1-2, skip Steps 3-4. User request: "Show what would be deleted if I delete VM xyz" + +### Protected VM Label +Automatic enforcement: If VM has `protected: "true"` label, refuse deletion in Step 1.2. Example YAML: `metadata.labels.protected: "true"` diff --git a/submissions/vm-delete/skills/references/INDEX.md b/submissions/vm-delete/skills/references/INDEX.md new file mode 100644 index 0000000..a2e7696 --- /dev/null +++ b/submissions/vm-delete/skills/references/INDEX.md @@ -0,0 +1,332 @@ +--- +title: VM Troubleshooting Guide - Index +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 + - title: OpenShift Virtualization - Virtual Machine Status + url: https://docs.openshift.com/container-platform/latest/virt/virtual_machines/virt-managing-vms.html + date_accessed: 2026-02-06 + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, scheduling, taints, tolerations, errors, deletion, cloning, lifecycle, networking, crashloop, index, navigation] +semantic_keywords: [troubleshooting index, error categories, VM diagnostics, navigation hub] +use_cases: [vm-creation, vm-deletion, vm-cloning, vm-lifecycle, diagnostics, error-handling, network-troubleshooting] +last_updated: 2026-02-17 +--- + +# VM Troubleshooting Guide - Index + +## Overview + +This guide provides comprehensive diagnostic procedures and workarounds for VirtualMachine errors and issues in OpenShift Virtualization. Use this document when VMs encounter problems during: + +- **Creation**: Scheduling failures, storage provisioning issues +- **Lifecycle**: Start/stop failures, stuck states +- **Deletion**: Resources stuck in Terminating, storage cleanup failures +- **Cloning**: DataVolume cloning errors, cross-namespace issues +- **Networking**: Secondary network attachment failures +- **Runtime**: CrashLoopBackOff, guest OS boot failures + +This guide is consulted by all rh-virt skills (vm-create, vm-inventory, vm-lifecycle-manager, vm-delete, vm-clone) when diagnosing and remediating VM issues. + +--- + +## 🗂️ Troubleshooting Categories + +The troubleshooting documentation is organized by error category for easier navigation and token optimization. Each category file contains MCP-first diagnostic procedures: + +### 1. [Scheduling Errors](scheduling-errors.md) +**When to use**: VM fails to schedule on any node + +**Errors covered**: +- ErrorUnschedulable - Node Taints +- ErrorUnschedulable - Insufficient Resources +- ErrorUnschedulable - Node Selector Mismatch + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 2. [Storage Errors](storage-errors.md) +**When to use**: VM has storage provisioning, deletion, or cloning issues + +**Errors covered**: +- ErrorDataVolumeNotReady (all 3 subsections) +- ErrorPvcNotFound +- Storage Deletion Failures +- DataVolume Cloning Failures + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### 3. [Lifecycle Errors](lifecycle-errors.md) +**When to use**: VM has start/stop/termination issues + +**Errors covered**: +- VM Stuck in Terminating State +- VM Won't Start (Non-Scheduling Issues) +- VM Won't Stop + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### 4. [Runtime Errors](runtime-errors.md) +**When to use**: VM repeatedly crashes or fails at runtime + +**Errors covered**: +- CrashLoopBackOff + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### 5. [Network Errors](network-errors.md) +**When to use**: VM has secondary network attachment failures + +**Errors covered**: +- Network Attachment Failures (Multus, SR-IOV) + +**Skills that use this**: vm-create + +--- + +## 🔧 How to Use This Guide: MCP Tools First, CLI Commands Last + +### Critical Principle: MCP-First Approach + +**All diagnostic procedures in this guide follow the MCP-first pattern**: + +``` +1. ✅ TRY: MCP Tool (resources_get, resources_list, etc.) +2. ⚠️ IF FAILS: Ask user permission to use CLI command +3. ❌ LAST RESORT: Execute CLI command (oc/kubectl) with explicit user approval +``` + +### Why MCP Tools First? + +- **Structured Access**: MCP tools provide programmatic, type-safe access to cluster resources +- **Consistency**: Same interface across all operations +- **Better Error Handling**: MCP tools return structured errors +- **Audit Trail**: MCP tool usage is logged and trackable + +### Command Pattern Examples + +Throughout this guide, you'll see diagnostic steps formatted like this: + +**✅ CORRECT Pattern (MCP First)**: + +```markdown +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Extract**: `.metadata.finalizers` from returned JSON + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` +``` + +### Available MCP Tools + +The `openshift-virtualization` MCP server provides these tools: + +**Resource Operations**: +- `resources_get` - Get specific resource (replaces `oc get `) +- `resources_list` - List resources (replaces `oc get `) +- `resources_delete` - Delete resource (replaces `oc delete`) +- `resources_create_or_update` - Create/update resource (replaces `oc apply` / `oc patch`) + +**Pod Operations**: +- `pods_list_in_namespace` - List pods in namespace (replaces `oc get pods -n`) +- `pods_get` - Get pod details (replaces `oc get pod`) +- `pods_log` - Get pod logs (replaces `oc logs`) +- `pods_exec` - Execute in pod (replaces `oc exec`) +- `pods_delete` - Delete pod (replaces `oc delete pod`) +- `pods_top` - Pod resource usage (replaces `oc top pods`) + +**Events & Monitoring**: +- `events_list` - List events (replaces `oc get events`) +- `nodes_top` - Node resource usage (replaces `oc top nodes`) +- `nodes_log` - Node logs (replaces `oc adm node-logs`) +- `nodes_stats_summary` - Detailed node stats + +**VM Operations** (KubeVirt toolset): +- `vm_create` - Create VMs +- `vm_lifecycle` - Start/stop/restart VMs + +**Namespaces**: +- `namespaces_list` - List namespaces (replaces `oc get namespaces`) +- `projects_list` - List OpenShift projects (replaces `oc get projects`) + +### When CLI Commands Are Required + +Some operations have **NO MCP equivalent** and require CLI: + +- `virtctl` commands (console, VNC access) +- `oc debug node` (node debugging) +- `oc auth can-i` (permission checks) +- `oc adm taint` (node taint management) +- Complex JSON patch operations + +For these, the guide will note: **"CLI Only - No MCP equivalent"** + +### Quick Reference: CLI → MCP Mapping + +| CLI Command | MCP Tool Equivalent | +|-------------|---------------------| +| `oc get vm -n ` | `resources_get` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc get vms -n ` | `resources_list` with apiVersion="kubevirt.io/v1", kind="VirtualMachine" | +| `oc delete vmi -n ` | `resources_delete` with kind="VirtualMachineInstance" | +| `oc get pods -n ` | `pods_list_in_namespace` with namespace="" | +| `oc logs -n ` | `pods_log` with name="", namespace="" | +| `oc get events -n ` | `events_list` with namespace="" | +| `oc get nodes` | `resources_list` with apiVersion="v1", kind="Node" | + +**Note**: The table above covers the most common patterns. For MCP tools specific to VM operations, see the "Available MCP Tools" section above, and consult the README for complete MCP server configuration. + +### How to Read Diagnostic Sections + +Each error section includes: +1. **Symptom** - What you observe +2. **Description** - What's happening +3. **Possible Causes** - Why it's happening +4. **Diagnostic Steps** - **MCP tools first**, then CLI fallback +5. **Solutions** - **MCP tools first**, then CLI fallback +6. **Verification** - **MCP tools first**, then CLI fallback + +**Note**: Where CLI commands appear without MCP tool alternatives in older sections, they should be treated as **fallback only**. Skills should attempt MCP tools first, then request user permission before using CLI. + +--- + +## 🔍 Quick Navigation by Skill + +**vm-create**: +- [Scheduling Errors](scheduling-errors.md) - ErrorUnschedulable diagnostics +- [Storage Errors](storage-errors.md) - ErrorDataVolumeNotReady, storage provisioning +- [Runtime Errors](runtime-errors.md) - CrashLoopBackOff +- [Network Errors](network-errors.md) - Network attachment failures + +**vm-delete**: +- [Lifecycle Errors](lifecycle-errors.md) - VM stuck in Terminating state +- [Storage Errors](storage-errors.md) - Storage deletion failures + +**vm-clone**: +- [Storage Errors](storage-errors.md) - DataVolume cloning failures + +**vm-lifecycle-manager**: +- [Lifecycle Errors](lifecycle-errors.md) - VM won't start/stop +- [Scheduling Errors](scheduling-errors.md) - VM won't start due to scheduling + +**vm-inventory**: +- [INDEX.md](INDEX.md) - General guidance, consult specific categories as needed + +--- + +## 📊 Documentation Coverage & Maintenance + +### Current Coverage + +This troubleshooting guide covers the most common VM errors encountered in OpenShift Virtualization: + +- ✅ **Scheduling failures** - ErrorUnschedulable (3 root causes: node taints, insufficient resources, node selector mismatch) +- ✅ **Storage issues** - ErrorDataVolumeNotReady (3 scenarios), ErrorPvcNotFound, storage deletion failures, DataVolume cloning failures +- ✅ **Lifecycle problems** - VM stuck in Terminating state, VM won't start (non-scheduling), VM won't stop +- ✅ **Runtime crashes** - CrashLoopBackOff (kernel panic, QEMU crashes, OOM, guest OS failures) +- ✅ **Network attachment failures** - Multus NetworkAttachmentDefinition issues, SR-IOV problems + +**Total errors documented**: 12 error types across 6 categories + +--- + +### Encountering Undocumented Errors + +#### For AI Agents (Claude Code) + +If you encounter an error **not documented** in the categories above: + +1. **Report to user** with all available details (error message, affected resources, namespace) +2. **Provide best-effort diagnostics** using MCP tools: + - `resources_get` to inspect resource status + - `pods_log` to check virt-launcher or compute container logs + - `events_list` to view Kubernetes events +3. **Suggest documentation update**: + ``` + ⚠️ This error is not yet documented in the troubleshooting guide. + + **Error**: + **Affected resource**: / + + I recommend adding this error to the troubleshooting guide: + - If it's a [scheduling/storage/lifecycle/runtime/network] issue → Add to existing category file + - If it's a new error class → Create new category file in docs/troubleshooting/ + + Would you like me to help document this error for future reference? + ``` + +4. **Do NOT make up solutions** - only provide factual diagnostics from MCP tool outputs + +--- + +#### For Human Contributors + +To document a new error: + +1. **Determine the category**: + - Scheduling issue → `scheduling-errors.md` + - Storage problem → `storage-errors.md` + - Lifecycle/start/stop → `lifecycle-errors.md` + - Runtime crash → `runtime-errors.md` + - Network issue → `network-errors.md` + - New error class → Create new category file (e.g., `security-errors.md`) + +2. **Follow the MCP-first pattern** (see "How to Use This Guide" above): + - **Diagnostic Steps**: MCP Tool → Parameters → Extract → CLI Fallback + - **Solutions**: MCP Tool approach → Implementation → CLI Fallback + - **Verification**: MCP Tool checks → Expected results → CLI Fallback + +3. **Update semantic index**: + - Edit `.ai-index/semantic-index.json` + - Add error to `error_to_docs_mapping` + - Update relevant skill's `primary_docs` or `secondary_docs` + - Increment token estimates if needed + +4. **Reference in skill files**: + - Update skill's Reference Documentation section + - Ensure skill's Document Consultation steps point to the right category + +5. **Update this INDEX.md**: + - Add error to the appropriate category section (lines 50-120) + - Update "Current Coverage" list above + +--- + +## 📚 Additional Resources + +- [rh-virt README](../../README.md) - MCP server setup and skill documentation +- [SOURCES.md](../SOURCES.md) - Official Red Hat documentation sources +- [Semantic Index](.ai-index/semantic-index.json) - AI-optimized doc discovery diff --git a/submissions/vm-delete/skills/references/lifecycle-errors.md b/submissions/vm-delete/skills/references/lifecycle-errors.md new file mode 100644 index 0000000..6167fcc --- /dev/null +++ b/submissions/vm-delete/skills/references/lifecycle-errors.md @@ -0,0 +1,869 @@ +--- +title: VM Lifecycle Errors +category: kubevirt +sources: + - title: Kubernetes Finalizers + url: https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/ + date_accessed: 2026-02-17 + - title: KubeVirt Virtual Machine Status Conditions + url: https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/ + date_accessed: 2026-02-17 +tags: [troubleshooting, lifecycle, terminating, start, stop, VMI, virt-launcher] +semantic_keywords: [VM stuck terminating, finalizers, VM won't start, VM won't stop, lifecycle management, runStrategy] +use_cases: [vm-deletion, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Lifecycle Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM lifecycle issues including start/stop failures and stuck termination states. + +**When to use this document**: +- VM stuck in `Terminating` state +- VM won't start (runStrategy is Always but VM never reaches Running) +- VM won't stop (runStrategy is Halted but VM never reaches Stopped) + +**Skills that use this**: vm-delete, vm-lifecycle-manager + +--- + +### VM Stuck in Terminating State + +**Symptom**: VM shows status `Terminating` but deletion never completes + +**Description**: The VM deletion process is blocked, usually by finalizers, attached resources, or stuck VirtualMachineInstance (VMI). + +**Possible Causes**: +- Finalizers blocking deletion +- PVC/DataVolume still attached and preventing cleanup +- VirtualMachineInstance (VMI) not terminating properly +- Custom controllers or operators blocking deletion +- Stuck virt-launcher pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check finalizers on the VM**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +Ask user: "MCP tool unavailable. May I use `oc get vm` to check finalizers?" +```bash +oc get vm -n -o jsonpath='{.metadata.finalizers}' +``` + +**2. Check if VMI still exists**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +If returns "Not Found", VMI is deleted. If returns resource, VMI is stuck. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name containing "virt-launcher-". + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check events for deletion issues**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results where `involvedObject.name` == "" and sort by timestamp. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= --sort-by='.lastTimestamp' +``` + +**5. Check VM deletion timestamp**: + +Use the same `resources_get` call from step 1, extract `.metadata.deletionTimestamp`. + +If present, VM is in deletion process. If null, VM is not being deleted. + +**Common Finalizer Patterns**: +- `kubevirt.io/virtualMachineControllerFinalize` - Standard KubeVirt finalizer +- `foregroundDeletion` - Waits for dependent objects to be deleted +- Custom finalizers from operators + +**Solutions** (Use MCP Tools First): + +1. **Wait for dependent resources** (safest approach): + - Use `resources_list` to check if PVCs, DataVolumes are still deleting + - Let Kubernetes complete cascade deletion naturally (wait 2-5 minutes) + +2. **Force delete VMI** (if VMI is stuck): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable - requires explicit user permission): + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +3. **Force delete virt-launcher pod** (if pod is stuck): + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find pod name using `pods_list_in_namespace` (see diagnostic step 3 above). + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + # Find the virt-launcher pod name first + oc get pods -n | grep virt-launcher- + + # Then delete it + oc delete pod virt-launcher--xxx -n --force --grace-period=0 + ``` + +4. **Remove finalizers** (⚠️ dangerous - use only as last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` + 2. Remove items from `.metadata.finalizers` array + 3. Update VM using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: This can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch not easily done via MCP): + ```bash + # This operation is complex for MCP - may need CLI + oc patch vm -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Check for protection labels** (vm-delete skill specific): + + Use `resources_get` from diagnostic step 1, extract `.metadata.labels.protected`. + + If value is "true", the vm-delete skill refuses deletion (this is expected behavior). + +**Verification**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error if deletion successful. + +**CLI Fallback**: +```bash +oc get vm -n +# Should return: Error from server (NotFound) +``` + +--- + + +--- + +### VM Won't Start (Non-Scheduling Issues) + +**Symptom**: VM start command succeeds (runStrategy changed to Always) but VM never reaches Running state + +**Description**: The VM fails to start for reasons other than scheduling problems (ErrorUnschedulable). This typically involves guest OS boot issues, resource problems, or virtualization errors. + +**Possible Causes**: +- Guest OS kernel panic or boot failure +- Cloud-init configuration errors +- Missing or corrupted disk image +- Insufficient memory for guest OS to boot +- QEMU/KVM virtualization errors +- VirtualMachineInstance (VMI) creation failures +- virt-launcher pod crashes + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VMI (VirtualMachineInstance) status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and overall VMI state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +``` + +**2. Check VMI conditions for errors**: + +Use `resources_get` from step 1, extract `.status.conditions` for detailed error information. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. View virt-launcher pod logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +First, get pod name from step 3, then: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +Look for QEMU errors, memory allocation failures, device errors. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**5. Check virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.name` matches the virt-launcher pod name from step 3. + +Alternatively, use `pods_get` to get full pod details: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod $(oc get pods -n | grep virt-launcher- | awk '{print $1}') +``` + +**6. Access VM console to see guest OS boot messages**: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +Look for kernel panic, initramfs errors, filesystem errors. + +**7. Check VM events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachine` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep -A 20 "Events:" +``` + +**Common Error Patterns**: + +1. **Guest OS Boot Failure**: + - Console shows kernel panic + - Guest hangs at GRUB or boot loader + - Cloud-init errors during first boot + +2. **Insufficient Memory**: + - Guest OS kills processes due to OOM + - VMI logs show memory allocation errors + +3. **QEMU Crashes**: + - virt-launcher logs show QEMU segfaults + - VMI repeatedly restarts + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for kernel panic, initramfs errors, filesystem errors. + +2. **Check virt-launcher pod logs for QEMU errors**: + + Use `pods_log` from diagnostic step 4 to view logs. + + Look for: + - "qemu-system-x86_64: ..." errors + - Memory allocation failures + - Device errors + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx + ``` + +3. **Increase memory if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check VMI memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too low, delete VM and recreate with larger instance type (e.g., change from "small" to "medium" in vm-create). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +4. **Verify disk image integrity**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check PVC is bound: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + If using container disk, verify image exists and is accessible (check VMI spec). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get pvc -n + ``` + +5. **Check cloud-init configuration** (if applicable): + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.template.spec.volumes[]` and look for `cloudInitNoCloud` or `cloudInitConfigDrive` entries. + + Cloud-init syntax errors prevent boot. Check virt-launcher logs (diagnostic step 4) for cloud-init errors. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.volumes[?(@.cloudInitNoCloud)]}' | jq + ``` + +6. **Restart VMI** (soft reset): + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete VMI (VM controller will recreate it): + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for new VMI to start (use `resources_get` to check status). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n -w + ``` + +7. **Check virtualization extensions** (KVM): + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + + # In debug shell: + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should eventually return `Running`). + +Check VMI is running: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should show `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should eventually return: Running + +oc get vmi -n +# Should show: Running +``` + +--- + +### VM Won't Stop + +**Symptom**: VM runStrategy changed to Halted but VM never reaches Stopped state + +**Description**: The VM stop/shutdown process fails to complete, leaving VM in Stopping state indefinitely. + +**Possible Causes**: +- Guest OS not responding to ACPI shutdown signal +- virt-launcher pod stuck and not terminating +- VirtualMachineInstance (VMI) deletion blocked by finalizers +- Guest shutdown scripts hanging +- Filesystem sync issues in guest OS + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (might show `Stopping`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**2. Check VMI status and deletion timestamp**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Check `.metadata.deletionTimestamp` (if set, VMI is being deleted). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n +oc get vmi -n -o jsonpath='{.metadata.deletionTimestamp}' +``` + +**3. Check virt-launcher pod status**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for pods with name matching `virt-launcher-`. Check if pod is in `Terminating` state. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +``` + +**4. Check VMI events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `.involvedObject.kind` is `VirtualMachineInstance` and `.involvedObject.name` matches ``. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vmi -n | grep -A 10 "Events:" +``` + +**5. Check VMI finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.metadata.finalizers}' +``` + +**6. Check if guest is responsive** (if VMI still exists): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Patterns**: +- VMI shows `deletionTimestamp` but never actually deletes +- virt-launcher pod in `Terminating` state +- VM runStrategy is `Halted` but printableStatus shows `Stopping` + +**Solutions** (Use MCP Tools First): + +1. **Wait for graceful shutdown** (default: 30 seconds): + + Wait 1-2 minutes for guest OS to complete shutdown. Check status periodically using `resources_get` from diagnostic step 1. + +2. **Force stop by deleting VMI**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + This is the standard way to force-stop a VM. + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Wait for VMI deletion, then verify using `resources_get` (should return "Not Found" error). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete vmi -n + oc get vmi -n + # Should return: Error from server (NotFound) + ``` + +3. **Force delete VMI with grace period** (if VMI won't delete): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete vmi -n --grace-period=0 --force + ``` + +4. **Force delete virt-launcher pod**: + + **MCP Tool**: `pods_delete` (from openshift-virtualization) + + First, find the pod using `pods_list_in_namespace` from diagnostic step 3. + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + ⚠️ **Note**: For force deletion with grace period, use CLI fallback. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + POD_NAME=$(oc get pods -n | grep virt-launcher- | awk '{print $1}') + oc delete pod $POD_NAME -n --force --grace-period=0 + ``` + +5. **Remove VMI finalizers** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VMI using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update VMI using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned resources. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vmi -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Patch VM runStrategy directly** (ensure consistency): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current VM using `resources_get` (diagnostic step 1) + 2. Set `.spec.runStrategy` to `"Halted"` + 3. Update VM using `resources_create_or_update` with modified JSON + + **CLI Fallback** (merge patch easier via CLI): + Ask user: "Patching runStrategy is easier via CLI. May I use `oc patch`?" + ```bash + oc patch vm -n --type=merge -p '{"spec":{"runStrategy":"Halted"}}' + ``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +After remediation, check VM status: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Stopped` or `Halted`). + +Verify VMI is gone: + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Verify virt-launcher pod is gone: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for pods matching `virt-launcher-`. Should return no results. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Stopped or Halted + +oc get vmi -n +# Should return: Error from server (NotFound) + +oc get pods -n | grep virt-launcher- +# Should return: No resources found +``` + +**Prevention**: +- Ensure guest OS has ACPI support enabled +- Use proper shutdown commands in guest OS +- Avoid forceful stops unless necessary (can corrupt guest filesystem) + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Storage Errors](storage-errors.md) | [Runtime Errors →](runtime-errors.md) diff --git a/submissions/vm-delete/skills/references/network-errors.md b/submissions/vm-delete/skills/references/network-errors.md new file mode 100644 index 0000000..82fa313 --- /dev/null +++ b/submissions/vm-delete/skills/references/network-errors.md @@ -0,0 +1,429 @@ +--- +title: VM Network Errors +category: kubevirt +sources: + - title: Multus CNI - Network Attachment Definitions + url: https://github.com/k8snetworkplumbingwg/multus-cni + date_accessed: 2026-02-17 +tags: [troubleshooting, networking, Multus, NAD, SR-IOV, secondary networks] +semantic_keywords: [network attachment failures, Multus, NetworkAttachmentDefinition, SR-IOV, secondary networks] +use_cases: [vm-creation, network-troubleshooting] +related_docs: [INDEX.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Network Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM secondary network attachment failures using Multus CNI and NetworkAttachmentDefinitions. + +**When to use this document**: +- VM created successfully but secondary networks not attached +- NetworkAttachmentDefinition not found errors +- Multus CNI failures +- SR-IOV device attachment issues + +**Skills that use this**: vm-create + +--- + +### Network Attachment Failures + +**Symptom**: VM created successfully but secondary networks (Multus) not attached or not working + +**Description**: The VM fails to attach to secondary networks defined via NetworkAttachmentDefinitions (Multus CNI). + +**Possible Causes**: +- NetworkAttachmentDefinition doesn't exist in the namespace +- Multus CNI not installed or not configured on cluster +- Namespace mismatch (NAD in different namespace than VM) +- Interface name conflicts in VM spec +- Bridge/network configuration errors in NAD +- SR-IOV device not available (if using SR-IOV) + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List NetworkAttachmentDefinitions in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "" +} +``` + +Review `.items[].metadata.name` for available NADs. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -n +``` + +**2. List NetworkAttachmentDefinitions in all namespaces**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition" +} +``` + +Omit `namespace` parameter to list across all namespaces. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definitions -A +``` + +**3. Check specific NetworkAttachmentDefinition**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "namespace": "", + "name": "" +} +``` + +Review `.spec.config` for CNI configuration. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get network-attachment-definition -n -o yaml +``` + +**4. Check VM network configuration**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.networks` to see network references. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.networks}' | jq +``` + +**5. Check VM domain interfaces**: + +Use `resources_get` from step 4, extract `.spec.template.spec.domain.devices.interfaces`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces}' | jq +``` + +**6. Check virt-launcher pod network annotations** (shows actual attachments): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**7. Check Multus is installed**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "openshift-multus" +} +``` + +Should show Multus CNI pods running. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n openshift-multus +``` + +**8. Check for errors in virt-launcher pod events**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n +``` + +**Common Error Messages**: +- `"network-attachment-definition not found"` - NAD doesn't exist in namespace +- `"multus CNI not configured"` - Multus not installed or misconfigured +- `"interface name conflict"` - Duplicate interface names in VM spec +- `"failed to add network"` - CNI plugin error (check NAD config) +- `"no available devices"` - SR-IOV device not available (if using SR-IOV) + +**Solutions** (Use MCP Tools First): + +1. **Verify NetworkAttachmentDefinition exists in correct namespace**: + + Use `resources_list` from diagnostic step 1 to check if NAD exists in VM's namespace. + + If NAD is in different namespace, copy it to VM namespace: + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get NAD from source namespace using `resources_get` + 2. Modify `.metadata.namespace` to target namespace + 3. Create NAD in target namespace using `resources_create_or_update` + + **CLI Fallback** (stream processing easier via CLI): + Ask user: "Copying NAD across namespaces is easier via CLI. May I use `oc` with sed?" + ```bash + oc get network-attachment-definition -n -o yaml | \ + sed "s/namespace: /namespace: /" | \ + oc apply -f - + ``` + +2. **Create missing NetworkAttachmentDefinition**: + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + Example: Linux bridge network + + **Parameters**: + ```json + { + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": { + "name": "vlan100", + "namespace": "" + }, + "spec": { + "config": "{\"cniVersion\":\"0.3.1\",\"type\":\"bridge\",\"bridge\":\"br1\",\"vlan\":100,\"ipam\":{\"type\":\"host-local\",\"subnet\":\"192.168.100.0/24\"}}" + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Creating NAD with complex config is easier via CLI. May I use `oc apply -f`?" + ```bash + cat < + spec: + config: '{ + "cniVersion": "0.3.1", + "type": "bridge", + "bridge": "br1", + "vlan": 100, + "ipam": { + "type": "host-local", + "subnet": "192.168.100.0/24" + } + }' + EOF + ``` + +3. **Check Multus CNI installation**: + + Use `pods_list_in_namespace` from diagnostic step 7 to verify Multus pods are running. + + To check cluster network operator: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "config.openshift.io/v1", + "kind": "ClusterOperator" + } + ``` + + Filter for `network` operator. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n openshift-multus + oc get clusteroperators network + ``` + +4. **Fix interface name conflicts** (if VM has duplicate names): + + Use `resources_get` from diagnostic step 4, extract `.spec.template.spec.domain.devices.interfaces[*].name`. + + Each interface must have unique name. If duplicates found, edit VM using `resources_create_or_update`. + + **CLI Fallback** (interactive edit easier via CLI): + Ask user: "Editing VM is easier via CLI. May I use `oc edit`?" + ```bash + oc get vm -n -o jsonpath='{.spec.template.spec.domain.devices.interfaces[*].name}' + oc edit vm -n + ``` + +5. **Validate NAD configuration syntax**: + + Use `resources_get` from diagnostic step 3, extract `.spec.config`. + + Ensure valid JSON. Common issues: missing quotes, wrong CNI type, invalid IPAM config. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get network-attachment-definition -n -o jsonpath='{.spec.config}' + ``` + +6. **Check SR-IOV device availability** (if using SR-IOV networks): + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + List SR-IOV network node policies: + + **Parameters**: + ```json + { + "apiVersion": "sriovnetwork.openshift.io/v1", + "kind": "SriovNetworkNodePolicy", + "namespace": "openshift-sriov-network-operator" + } + ``` + + Check SR-IOV device plugin pods: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "openshift-sriov-network-operator" + } + ``` + + Filter for pods with "device-plugin" in name. + + Check available SR-IOV devices on node: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "Node", + "name": "" + } + ``` + + Review `.status.allocatable` for SR-IOV resources. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get sriovnetworknodepolicy -n openshift-sriov-network-operator + oc get pods -n openshift-sriov-network-operator | grep device-plugin + oc describe node | grep -A 10 "Allocatable:" + ``` + +7. **Recreate VM with corrected network configuration** (if needed): + + If network attachment is fundamentally broken, delete and recreate VM with correct NAD references using vm-create skill. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +After remediation, check virt-launcher pod network status: + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod, then extract `.metadata.annotations["k8s.v1.cni.cncf.io/network-status"]`. + +Should show all attached networks with status. Example output: +```json +[ + { + "name": "openshift-sdn", + "interface": "eth0", + "ips": ["10.128.2.10"], + "default": true + }, + { + "name": "vlan100", + "interface": "net1", + "ips": ["192.168.100.5"] + } +] +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq +``` + +**Check from inside VM** (via console): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# In guest OS: +ip addr show +# Should show all network interfaces (eth0, net1, etc.) +``` + +**Common Network Types**: +- **Linux Bridge**: Layer 2 bridge for VLAN networks +- **SR-IOV**: High-performance direct device assignment +- **macvlan**: MAC-based VLAN for container networks +- **OVN-Kubernetes**: OpenShift native overlay network + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Runtime Errors](runtime-errors.md) diff --git a/submissions/vm-delete/skills/references/runtime-errors.md b/submissions/vm-delete/skills/references/runtime-errors.md new file mode 100644 index 0000000..804d3cb --- /dev/null +++ b/submissions/vm-delete/skills/references/runtime-errors.md @@ -0,0 +1,616 @@ +--- +title: VM Runtime Errors +category: kubevirt +sources: + - title: Kubernetes Pod Lifecycle + url: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ + date_accessed: 2026-02-17 +tags: [troubleshooting, runtime, CrashLoopBackOff, guest OS, QEMU, crashes] +semantic_keywords: [CrashLoopBackOff, pod crashes, guest kernel panic, QEMU crash, OOM, virt-launcher restart] +use_cases: [vm-creation, vm-lifecycle, diagnostics] +related_docs: [INDEX.md, lifecycle-errors.md, scheduling-errors.md] +last_updated: 2026-02-17 +--- + +# VM Runtime Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM runtime failures where the virt-launcher pod or guest OS repeatedly crashes. + +**When to use this document**: +- VM or virt-launcher pod shows `CrashLoopBackOff` status +- virt-launcher pod repeatedly restarting +- Guest OS kernel panics on boot + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +### CrashLoopBackOff + +**Symptom**: VM status shows `CrashLoopBackOff` or virt-launcher pod repeatedly restarting + +**Description**: The virt-launcher pod or guest OS is repeatedly crashing and restarting, indicating a critical failure in the virtualization stack or guest OS. + +**Possible Causes**: +- Guest OS kernel panic on boot +- Insufficient resources (memory/CPU) for guest OS +- Corrupted disk image or filesystem +- QEMU/libvirt crashes due to configuration errors +- Missing or incompatible device drivers in guest +- Resource limits too low for virt-launcher pod +- Virtualization features (KVM) not available on node + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check virt-launcher pod restart count**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` (>0 indicates crashes). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# Look at RESTARTS column +``` + +**2. View recent crash logs** (previous container instance): + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "previous": true, + "tail": 100 +} +``` + +Look for QEMU errors, kernel panics, or segfaults. + +**CLI Fallback** (if MCP unavailable): +```bash +oc logs -n virt-launcher--xxx --previous +``` + +**3. Check current virt-launcher logs**: + +**MCP Tool**: `pods_log` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "tail": 100 +} +``` + +⚠️ **Note**: MCP `pods_log` doesn't support `--all-containers` flag. Call `pods_log` separately for each container if needed. + +**CLI Fallback** (if MCP unavailable or all containers needed): +```bash +oc logs -n virt-launcher--xxx --all-containers +``` + +**4. Check VMI conditions for crash details**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" +} +``` + +Extract `.status.conditions` for crash details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vmi -n -o jsonpath='{.status.conditions}' | jq +``` + +**5. Check pod events for crash reasons**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches the virt-launcher pod name. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe pod virt-launcher--xxx -n | grep -A 20 "Events:" +``` + +**6. Check pod resource limits**: + +**MCP Tool**: `pods_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx" +} +``` + +Extract `.spec.containers[0].resources` for resource limits. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pod virt-launcher--xxx -n -o jsonpath='{.spec.containers[0].resources}' | jq +``` + +**7. Check node kubelet logs for OOM kills**: + +⚠️ **Note**: Node log access requires `oc adm node-logs` CLI command (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +oc adm node-logs -u kubelet | grep -i oom +``` + +**8. Access guest console** (if VM briefly starts): + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +``` + +**Common Crash Patterns**: + +1. **Guest Kernel Panic**: + - Console logs show kernel panic messages + - Guest crashes immediately after boot + - Symptoms: "Kernel panic - not syncing: VFS: Unable to mount root fs" + +2. **OOM (Out of Memory)**: + - Pod killed with reason: `OOMKilled` + - Guest runs out of memory during boot or operation + - virt-launcher logs show memory allocation failures + +3. **QEMU Crash**: + - virt-launcher logs show QEMU segmentation fault + - Symptoms: "qemu-system-x86_64: terminated by signal" + - Configuration incompatibility or QEMU bug + +4. **Disk Image Corruption**: + - Guest cannot boot from disk + - Filesystem errors in guest console + - DataVolume import failed + +**Solutions** (Use MCP Tools First): + +1. **Check guest console for kernel panic or boot errors**: + + ⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + virtctl console -n + ``` + + Look for: + - Kernel panic messages + - Initramfs errors + - Filesystem mounting failures + - Missing device errors + +2. **Review virt-launcher crash logs**: + + Use `pods_log` with `previous: true` from diagnostic step 2. + + Look for: + - QEMU command line errors + - Device initialization failures + - Memory allocation errors + - Signal termination (SIGSEGV, SIGABRT) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc logs -n virt-launcher--xxx --previous + ``` + +3. **Check for OOM (Out of Memory) kills**: + + **MCP Tool**: `pods_get` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "", + "name": "virt-launcher--xxx" + } + ``` + + Extract `.status.containerStatuses[0].lastState.terminated.reason`. + + If returns `"OOMKilled"`: + - Option 1: Increase virt-launcher memory limits + - Option 2: Decrease guest memory allocation + - Option 3: Use smaller instance type + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pod virt-launcher--xxx -n -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}' + ``` + +4. **Increase resources if OOM detected**: + + **MCP Tool**: `resources_get` (from openshift-virtualization) + + Check current memory allocation: + + **Parameters**: + ```json + { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "namespace": "", + "name": "" + } + ``` + + Extract `.spec.domain.resources.requests.memory`. + + If too high for node, delete and recreate with smaller instance type using vm-create skill (change from "large" to "medium" or "small"). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get vmi -n -o jsonpath='{.spec.domain.resources.requests.memory}' + ``` + +5. **Verify disk image integrity**: + + **MCP Tool**: `resources_list` + `resources_get` (from openshift-virtualization) + + Check DataVolume status: + + **Parameters for list**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" + } + ``` + + **Parameters for specific DV**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Check `.status.phase` (should be `Succeeded`). + + If using container disk, verify image pullable by checking virt-launcher events using diagnostic step 5. + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get datavolume -n + oc get datavolume -n -o jsonpath='{.status.phase}' + ``` + +6. **Check virtualization (KVM) availability**: + + ⚠️ **Note**: Node debugging requires `oc debug` CLI command (no MCP equivalent). + + **CLI Required** (no MCP alternative): + ```bash + oc debug node/ + chroot /host + lsmod | grep kvm + # Should show kvm_intel or kvm_amd + ``` + +7. **Simplify VM configuration** (eliminate variables): + + Try creating minimal VM using vm-create skill with: + - Small instance type + - No secondary networks + - Simple container disk (e.g., Fedora) + - No cloud-init + + If minimal VM works, add features back one by one. + +8. **Recreate VM with different workload** (test disk image): + + If guest OS consistently crashes, use vm-create skill to try different OS image (e.g., switch from Ubuntu to Fedora). This tests if issue is workload-specific. + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `pods_list_in_namespace` + `resources_get` (from openshift-virtualization) + +After remediation, check pod restart count stops increasing: + +**Parameters for pods**: +```json +{ + "namespace": "" +} +``` + +Filter for virt-launcher pod. Check `.status.containerStatuses[0].restartCount` - should stabilize (not keep increasing). + +Check VM reaches Running state: + +**Parameters for VM**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Check `.status.printableStatus` (should return `Running`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n | grep virt-launcher- +# RESTARTS should stabilize + +oc get vm -n -o jsonpath='{.status.printableStatus}' +# Should return: Running +``` + +Verify guest is responsive: + +⚠️ **Note**: Console access requires `virtctl` CLI tool (no MCP equivalent). + +**CLI Required** (no MCP alternative): +```bash +virtctl console -n +# Should show login prompt or OS console +``` + +**Advanced Debugging**: + +**MCP Tool**: `pods_exec` (from openshift-virtualization) + +Check libvirt domain XML: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["virsh", "dumpxml", "1"] +} +``` + +Check QEMU process: + +**Parameters**: +```json +{ + "namespace": "", + "name": "virt-launcher--xxx", + "command": ["ps", "aux"] +} +``` + +Filter output for "qemu" process. + +**CLI Fallback** (if MCP unavailable): +```bash +oc exec -n virt-launcher--xxx -- virsh dumpxml 1 +oc exec -n virt-launcher--xxx -- ps aux | grep qemu +``` + +**Prevention**: +- Start with minimal VM configuration and add complexity gradually +- Use recommended instance types for your workload +- Test disk images before deploying to production +- Ensure nodes have adequate resources and KVM support +- Monitor resource usage and set appropriate limits + +--- + +## Workaround Patterns for MCP Tool Limitations + +### General Pattern: Diagnose → Propose → Confirm → Execute + +When the MCP tool lacks feature support (e.g., tolerations, node selectors, advanced networking): + +**1. Diagnose** the root cause using Kubernetes commands +```bash +oc describe vm -n +oc get events -n --field-selector involvedObject.name= +``` + +**2. Propose** a manual workaround to the user with clear explanation +- Explain why the issue occurred +- Show what will be changed +- List alternative options + +**3. Confirm** with user before executing (Human-in-the-Loop) +- Wait for explicit "yes" or "apply workaround" +- Never auto-execute modifications + +**4. Execute** the workaround using oc/kubectl +```bash +oc patch vm -n --type=merge -p '...' +``` + +**5. Verify** the fix was successful +```bash +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**6. Document** the limitation and suggest filing enhancement request +- Note this is temporary until MCP tool is enhanced +- Provide link to file issue: https://github.com/openshift/openshift-mcp-server/issues + +### Example: Adding Tolerations Workaround + +**Diagnostic Output**: +```markdown +## ⚠️ VM Scheduling Issue Detected + +**Root Cause**: Node taints prevent VM scheduling + +**Details**: +- Found 3 nodes with taint: `virtualization=true:NoSchedule` +- VM spec does not include matching tolerations +- This prevents VM scheduling on virtualization-dedicated nodes +``` + +**Proposed Workaround**: +```bash +oc patch vm web-server -n vms --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' +``` + +**User Confirmation Required**: +``` +How would you like to proceed? +- "apply workaround" - I'll patch the VM with tolerations +- "manual" - I'll provide instructions for you to apply manually +- "cancel" - Delete the VM and abort creation +``` + +**After Execution**: +```markdown +## ✓ Workaround Applied Successfully + +**Action**: Added tolerations for taint `virtualization=true:NoSchedule` +**New Status**: Stopped (VM can now be scheduled) + +**Note**: This workaround was needed because the MCP tool doesn't yet support tolerations. +Future VMs in this cluster will need the same fix until the tool is enhanced. +``` + +--- + +## VM Status Reference + +### Status Values + +| Status | Meaning | Action Required | +|--------|---------|-----------------| +| `Stopped` / `Halted` | VM created but not running | Normal - use vm-lifecycle-manager to start | +| `Running` | VM is running | Normal | +| `Provisioning` | VM resources being prepared | Wait 5-10 seconds, check again | +| `Starting` | VM is booting | Wait for Running status or see "VM Won't Start" section if stuck | +| `Stopping` | VM is shutting down | Wait for Stopped status or see "VM Won't Stop" section if stuck | +| `Terminating` | VM is being deleted | Wait for deletion to complete or see "VM Stuck in Terminating State" section if stuck | +| `ErrorUnschedulable` | Cannot find node to run VM | **Action needed** - see ErrorUnschedulable section | +| `ErrorDataVolumeNotReady` | Storage not ready | **Action needed** - see ErrorDataVolumeNotReady section | +| `ErrorPvcNotFound` | PVC missing | **Action needed** - see ErrorPvcNotFound section | +| `CrashLoopBackOff` | VM repeatedly crashing | **Action needed** - see CrashLoopBackOff section | + +### Checking VM Status + +```bash +# Get printable status +oc get vm -n -o jsonpath='{.status.printableStatus}' + +# Get detailed status and conditions +oc get vm -n -o jsonpath='{.status}' | jq + +# Watch status changes in real-time +oc get vm -n -w +``` + +--- + +## Best Practices for Agents + +When implementing diagnostic workflows: + +1. **Always verify VM status** after creation (wait 5-10 seconds first) +2. **Consult this document** when encountering error status values +3. **Provide clear diagnosis** with evidence (show events, node taints, resource availability) +4. **Offer multiple solutions** (automated workaround vs manual steps vs alternative approaches) +5. **Respect human-in-the-loop** for all VM modifications +6. **Document temporary workarounds** and their limitations clearly +7. **Suggest filing issues** for missing MCP tool features + +### Document Consultation Pattern + +```markdown +**Document Consultation** (REQUIRED): +1. **Action**: Read [runtime-errors.md](../../docs/troubleshooting/runtime-errors.md) to understand CrashLoopBackOff causes +2. **Output to user**: "I consulted runtime-errors.md to diagnose the CrashLoopBackOff issue." +``` + +--- + +## Known MCP Tool Limitations + +### vm_create tool + +**Currently Supported**: +- ✓ Namespace, name (required) +- ✓ Workload/OS selection (fedora, ubuntu, rhel, etc.) +- ✓ Size hints (small, medium, large) +- ✓ Storage size +- ✓ Autostart flag +- ✓ Networks (Multus NetworkAttachmentDefinitions) +- ✓ Performance family (u1, o1, c1, m1) +- ✓ Instance type, preference + +**Not Currently Supported** (requires workarounds): +- ✗ Tolerations (for node taints) +- ✗ Node selectors +- ✗ Affinity/anti-affinity rules +- ✗ Resource requests/limits (beyond instance type) +- ✗ Custom labels/annotations +- ✗ SSH keys injection +- ✗ Cloud-init user data + +**Workaround Strategy**: Use `oc patch` after VM creation to add missing fields. + +**Enhancement Requests**: File issues at https://github.com/openshift/openshift-mcp-server/issues + +--- + +## Additional Resources + +- [KubeVirt Virtual Machine Status Conditions](https://kubevirt.io/user-guide/virtual_machines/vm_status_conditions/) +- [OpenShift Virtualization Troubleshooting](https://docs.openshift.com/container-platform/latest/virt/support/virt-troubleshooting.html) +- [Kubernetes Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) +- [OpenShift MCP Server Issues](https://github.com/openshift/openshift-mcp-server/issues) + +--- + +[← Back to Index](INDEX.md) | [← Lifecycle Errors](lifecycle-errors.md) | [Network Errors →](network-errors.md) diff --git a/submissions/vm-delete/skills/references/scheduling-errors.md b/submissions/vm-delete/skills/references/scheduling-errors.md new file mode 100644 index 0000000..e47c8c4 --- /dev/null +++ b/submissions/vm-delete/skills/references/scheduling-errors.md @@ -0,0 +1,417 @@ +--- +title: VM Scheduling Errors +category: kubevirt +sources: + - title: KubeVirt User Guide - Node Placement + url: https://kubevirt.io/user-guide/virtual_machines/node_placement/ + date_accessed: 2026-02-06 + - title: Kubernetes Taints and Tolerations + url: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + date_accessed: 2026-02-06 +tags: [troubleshooting, scheduling, taints, tolerations, ErrorUnschedulable, node selector, resources] +semantic_keywords: [ErrorUnschedulable, scheduling failure, node taints, insufficient resources, node selector mismatch, tolerations] +use_cases: [vm-creation, vm-lifecycle] +related_docs: [INDEX.md, storage-errors.md, runtime-errors.md] +last_updated: 2026-02-17 +--- + +# VM Scheduling Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM scheduling failures where the Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**When to use this document**: +- VM shows status `ErrorUnschedulable` after creation or start attempt +- VM events mention scheduling failures, taints, resources, or node selectors + +**Skills that use this**: vm-create, vm-lifecycle-manager + +--- + +## ErrorUnschedulable + +**Symptom**: VM shows status `ErrorUnschedulable` after creation + +**Description**: The Kubernetes scheduler cannot find a suitable node to run the VM's underlying virt-launcher pod. + +**Possible Causes**: + +### 1. Node Taints (Most Common) + +Nodes have taints that the VM doesn't tolerate. Common in environments with dedicated virtualization infrastructure. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM events for scheduling failures**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events where `involvedObject.name` == "" and look for messages like: +- "0/X nodes are available: X node(s) had taints that the pod didn't tolerate" + +**CLI Fallback** (if MCP unavailable): +```bash +oc get events -n --field-selector involvedObject.name= +``` + +**2. Check node taints in the cluster**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +Extract `.spec.taints` from each node in the returned list. Filter for nodes with non-null taints. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -o json | jq '.items[] | select(.spec.taints != null) | {name: .metadata.name, taints: .spec.taints}' +``` + +**Common Taint Patterns**: +- `virtualization=true:NoSchedule` - Only VMs with matching toleration can schedule +- `node-role.kubernetes.io/infra:NoSchedule` - Infrastructure-only nodes +- `node.kubernetes.io/not-ready:NoSchedule` - Node not ready for workloads + +**Solution - Add Tolerations to VM**: + +The openshift-virtualization MCP server's `vm_create` tool does NOT currently support the `tolerations` parameter. This requires a post-creation workaround using MCP tools. + +**Workaround (post-creation using MCP Tools)**: + +**Step 1**: Get current VM spec + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +**Step 2**: Modify the returned JSON to add tolerations + +Add to `.spec.template.spec.tolerations`: +```json +{ + "tolerations": [ + { + "key": "virtualization", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule" + } + ] +} +``` + +**Step 3**: Update VM with modified spec + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Parameters**: +```json +{ + "resource": "" +} +``` + +Pass the complete modified VM resource as YAML or JSON string. + +**Step 4**: Verify tolerations were added + +Use `resources_get` again and check `.spec.template.spec.tolerations` in response. + +**Step 5**: Check if VM status improved + +Wait 5-10 seconds, then use `resources_get` and check `.status.printableStatus`. + +**CLI Fallback** (if MCP patch is too complex): +```bash +# Ask user permission first: "MCP patch is complex. May I use oc patch instead?" +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" +' + +# Verify tolerations +oc get vm -n -o jsonpath='{.spec.template.spec.tolerations}' | jq + +# Check status +oc get vm -n -o jsonpath='{.status.printableStatus}' +``` + +**Example - Multiple Tolerations**: +```bash +oc patch vm -n --type=merge -p ' +spec: + template: + spec: + tolerations: + - key: "virtualization" + operator: "Equal" + value: "true" + effect: "NoSchedule" + - key: "dedicated" + operator: "Equal" + value: "virt-workloads" + effect: "NoSchedule" +' +``` + +**Toleration Operators**: +- `Equal` - Key and value must match exactly +- `Exists` - Only key must exist (ignores value) + +**Toleration Effects**: +- `NoSchedule` - Don't schedule new pods (existing pods continue) +- `PreferNoSchedule` - Avoid scheduling if possible +- `NoExecute` - Don't schedule AND evict existing pods + +**Alternative Solutions**: +1. **Remove node taints** (if you have cluster-admin access): + ```bash + oc adm taint nodes virtualization=true:NoSchedule- + ``` + +2. **Use different nodes** - If non-tainted nodes exist, ensure VM fits + +3. **File enhancement request** - Request tolerations support in openshift-mcp-server: + https://github.com/openshift/openshift-mcp-server/issues + +--- + +### 2. Insufficient Resources + +Not enough CPU, memory, or storage available on any node. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM resource requests**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.domain.resources` to see CPU/memory requests. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.domain.resources}' +``` + +**2. Check node resource availability**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.status.allocatable` and `.status.capacity` for available resources. + +Alternatively, use `nodes_top` MCP tool for current resource usage. + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe nodes | grep -A 5 "Allocated resources" +``` + +**3. Look for VM events mentioning "Insufficient"**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter for events where `.involvedObject.name` matches `` and `.message` contains "Insufficient". + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe vm -n | grep "Insufficient" +``` + +**Example Event**: +``` +0/5 nodes are available: 2 Insufficient cpu, 3 Insufficient memory. +``` + +**Solutions** (Use MCP Tools First): + +1. **Scale cluster** - Add more worker nodes (cluster admin task, no MCP tool) +2. **Reduce VM resources** - Delete and recreate with smaller instance type using vm-create skill +3. **Delete unused VMs** - Use vm-delete skill to free up resources +4. **Check resource quotas**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters for quota**: + ```json + { + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" + } + ``` + + **Parameters for limit range**: + ```json + { + "apiVersion": "v1", + "kind": "LimitRange", + "namespace": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + oc describe limitrange -n + ``` + +--- + +### 3. Node Selector Mismatch + +VM requires specific node labels that don't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check VM node selector requirements**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.nodeSelector` to see required node labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.nodeSelector}' +``` + +**2. List available node labels**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node" +} +``` + +For each node in `.items[]`, review `.metadata.labels` for available labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes --show-labels +``` + +**3. Check if any nodes match the selector**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "Node", + "labelSelector": "=" +} +``` + +Should return at least one node with matching labels. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get nodes -l = +``` + +**Solutions** (Use MCP Tools First): + +**Option 1: Remove node selector from VM** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get current VM using `resources_get` (diagnostic step 1) +2. Remove `.spec.template.spec.nodeSelector` field +3. Update VM using `resources_create_or_update` with modified JSON + +**CLI Fallback** (JSON patch easier via CLI): +Ask user: "Patching node selector is easier via CLI. May I use `oc patch`?" +```bash +oc patch vm -n --type=json -p '[{"op": "remove", "path": "/spec/template/spec/nodeSelector"}]' +``` + +**Option 2: Add label to nodes** + +**MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + +**Process**: +1. Get node using `resources_get` +2. Add label to `.metadata.labels` +3. Update node using `resources_create_or_update` + +⚠️ **Note**: Node labeling typically requires cluster admin privileges. + +**CLI Fallback** (simpler via CLI): +Ask user: "Adding node labels is easier via CLI. May I use `oc label`?" +```bash +oc label node = +``` + +--- + +[← Back to Index](INDEX.md) | [Storage Errors →](storage-errors.md) diff --git a/submissions/vm-delete/skills/references/storage-errors.md b/submissions/vm-delete/skills/references/storage-errors.md new file mode 100644 index 0000000..16ba584 --- /dev/null +++ b/submissions/vm-delete/skills/references/storage-errors.md @@ -0,0 +1,1011 @@ +--- +title: VM Storage Errors +category: kubevirt +sources: + - title: KubeVirt CDI - DataVolumes + url: https://kubevirt.io/user-guide/storage/containerized_data_importer/ + date_accessed: 2026-02-17 + - title: Kubernetes Persistent Volumes + url: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ + date_accessed: 2026-02-17 +tags: [troubleshooting, storage, DataVolume, PVC, ErrorDataVolumeNotReady, ErrorPvcNotFound, cloning, CDI] +semantic_keywords: [ErrorDataVolumeNotReady, ErrorPvcNotFound, storage deletion, PVC, DataVolume cloning, storage provisioning, storage class] +use_cases: [vm-creation, vm-deletion, vm-cloning] +related_docs: [INDEX.md, scheduling-errors.md, lifecycle-errors.md] +last_updated: 2026-02-17 +--- + +# VM Storage Errors + +[← Back to Index](INDEX.md) + +## Overview + +This document covers VM storage-related failures including storage provisioning, deletion, and cloning issues. + +**When to use this document**: +- VM shows status `ErrorDataVolumeNotReady` or `ErrorPvcNotFound` +- Storage deletion fails after VM deletion +- DataVolume cloning operations fail +- PVC provisioning issues + +**Skills that use this**: vm-create, vm-delete, vm-clone + +--- + +### ErrorDataVolumeNotReady + +**Symptom**: VM shows status `ErrorDataVolumeNotReady` + +**Description**: The DataVolume (persistent storage) backing the VM is not ready. + +**Possible Causes**: + +#### 1. DataVolume Still Provisioning + +Storage provisioning takes time, especially for large disks or when importing images. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Look for status in response: `Pending`, `ImportScheduled`, `ImportInProgress`, or `Succeeded`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Get detailed DataVolume information**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` and `.status.conditions` for provisioning details. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o json +``` + +**3. Check PVC (PersistentVolumeClaim) bound status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Check `.status.phase` for each PVC (should be `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**Solution**: Wait for DataVolume provisioning to complete (can take 1-5 minutes). Check status periodically using `resources_get`. + +#### 2. Storage Class Not Found + +The requested storage class doesn't exist in the cluster. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List available storage classes**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" +} +``` + +Review the list of available storage classes (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass +``` + +**2. Check DataVolume's requested storage class**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.spec.pvc.storageClassName` in the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.spec.pvc.storageClassName}' +``` + +**Solution**: +1. Use a valid storage class from the cluster +2. Recreate VM with correct storage class parameter + +#### 3. Insufficient Storage Quota + +Namespace has insufficient storage quota to provision the PVC. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check resource quotas**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (quota limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**2. Check storage usage**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +For each PVC, check `.metadata.name`, `.spec.resources.requests.storage`, and `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o custom-columns=NAME:.metadata.name,STORAGE:.spec.resources.requests.storage,STATUS:.status.phase +``` + +**Solution**: +1. Request quota increase from cluster admin +2. Delete unused PVCs to free quota +3. Reduce VM storage size + +--- + +### ErrorPvcNotFound + +**Symptom**: VM references a PersistentVolumeClaim that doesn't exist. + +**Diagnostic Steps** (Use MCP Tools First): + +**1. List PVCs in namespace**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review the list of available PVCs (check `.items[].metadata.name`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check VM's PVC references**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "namespace": "", + "name": "" +} +``` + +Extract `.spec.template.spec.volumes[*].persistentVolumeClaim.claimName` from the returned JSON to see which PVCs the VM is referencing. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get vm -n -o jsonpath='{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' +``` + +**Solution**: +- Wait for DataVolume to create the PVC +- Manually create missing PVC +- Fix VM spec to reference correct PVC name + +--- + + +--- + +### Storage Deletion Failures + +**Symptom**: VM deleted successfully but PVC or DataVolume remains in namespace + +**Description**: Storage resources (PersistentVolumeClaims, DataVolumes) fail to delete after VM removal. + +**Possible Causes**: +- PVC still bound to active PersistentVolume with `Retain` policy +- DataVolume still being referenced by another resource +- CDI (Containerized Data Importer) controller issues +- Storage class retention policy preventing deletion +- Finalizers on PVC/DataVolume blocking cleanup +- PVC still mounted by a pod + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check PVC status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**2. Check specific PVC phase**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Released` or `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n -o jsonpath='{.status.phase}' +``` + +**3. Check DataVolume status**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "" +} +``` + +Review `.items[].metadata.name` and `.items[].status.phase` for each DataVolume. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**4. Check what's using the PVC**: + +**MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +For each pod in `.items[]`, check `.spec.volumes[].persistentVolumeClaim.claimName` to find pods using the PVC. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pods -n -o json | jq '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' +``` + +**5. Check PVC finalizers**: + +Use `resources_get` from step 2, extract `.metadata.finalizers` from the returned JSON. + +**6. Check DataVolume finalizers**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Extract `.metadata.finalizers` from the returned JSON. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.metadata.finalizers}' +``` + +**7. Check PV reclaim policy**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume" +} +``` + +Filter results for PV where `.spec.claimRef.name` matches ``. + +To get specific PV policy: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolume", + "name": "" +} +``` + +Check `.spec.persistentVolumeReclaimPolicy`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pv | grep +oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' +``` + +**Common Finalizer Patterns**: +- `kubernetes.io/pvc-protection` - Protects PVC while in use +- `cdi.kubevirt.io/dataVolumeFinalizer` - CDI cleanup finalizer + +**Solutions** (Use MCP Tools First): + +1. **Delete DataVolume first, then PVC**: + + **MCP Tool**: `resources_delete` (from openshift-virtualization) + + Delete DataVolume first (often blocks PVC deletion): + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" + } + ``` + + Wait a few seconds, then delete PVC: + + **Parameters**: + ```json + { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc delete datavolume -n + oc delete pvc -n + ``` + +2. **Check for pods still using PVC**: + + **MCP Tool**: `pods_list_in_namespace` (from openshift-virtualization) + + **Parameters**: + ```json + { + "namespace": "" + } + ``` + + Filter results for pods where `.spec.volumes[].persistentVolumeClaim.claimName` equals ``. + + Then delete the pods using `pods_delete`: + + **Parameters**: + ```json + { + "namespace": "", + "name": "" + } + ``` + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pods -n -o json | jq -r '.items[] | select(.spec.volumes[]?.persistentVolumeClaim.claimName=="") | .metadata.name' + oc delete pod -n + ``` + +3. **Force delete PVC** (if safe to do so): + + ⚠️ **Note**: MCP `resources_delete` does not support `--grace-period` or `--force` flags. Use CLI for force deletion. + + **CLI Fallback** (required for force delete): + Ask user: "Force deletion requires CLI. May I use `oc delete --force`?" + ```bash + oc delete pvc -n --grace-period=0 --force + ``` + +4. **Remove finalizers from PVC** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PVC using `resources_get` (diagnostic step 2) + 2. Remove items from `.metadata.finalizers` array + 3. Update PVC using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch pvc -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +5. **Remove finalizers from DataVolume** (⚠️ last resort): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current DataVolume using `resources_get` (diagnostic step 6) + 2. Remove items from `.metadata.finalizers` array + 3. Update DataVolume using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Can leave orphaned storage. Only use if you understand the implications. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching finalizers is easier via CLI. May I use `oc patch`?" + ```bash + oc patch datavolume -n --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]' + ``` + +6. **Change PV reclaim policy** (if PV has Retain policy): + + **MCP Tool**: `resources_get` + `resources_create_or_update` (from openshift-virtualization) + + **Process**: + 1. Get current PV policy using `resources_get` (diagnostic step 7) + 2. Modify `.spec.persistentVolumeReclaimPolicy` to `"Delete"` + 3. Update PV using `resources_create_or_update` with modified JSON + + ⚠️ **WARNING**: Setting to `Delete` will delete underlying storage. + + **CLI Fallback** (JSON patch easier via CLI): + Ask user: "Patching PV reclaim policy is easier via CLI. May I use `oc patch`?" + ```bash + oc get pv -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' + oc patch pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' + ``` + +**Storage Quota Check** (Use MCP Tools First): + +After deletion, verify storage quota is freed: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters for quota check**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.used` to verify storage quota is freed. + +**Parameters for PVC verification**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "" +} +``` + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +oc get pvc -n +``` + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Confirm PVC is deleted: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +Confirm DataVolume is deleted: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Should return "Not Found" error. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +# Should return: Error from server (NotFound) + +oc get datavolume -n +# Should return: Error from server (NotFound) +``` + +--- + +### DataVolume Cloning Failures + +**Symptom**: VM clone created successfully but DataVolume clone operation fails + +**Description**: The DataVolume cloning process (used by vm-clone skill) fails to create a copy of the source storage. + +**Possible Causes**: +- CSI driver doesn't support volume cloning +- Source PVC storage class incompatible with cloning +- Cross-namespace cloning not permitted by storage backend +- Insufficient storage quota in target namespace +- Source PVC not in `Bound` state +- Storage class doesn't have volume cloning enabled +- CDI (Containerized Data Importer) controller issues + +**Diagnostic Steps** (Use MCP Tools First): + +**1. Check DataVolume clone status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Review `.status.phase`, `.status.conditions`, and `.metadata.name`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n +``` + +**2. Check DataVolume events for errors**: + +**MCP Tool**: `events_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "namespace": "" +} +``` + +Filter results for events related to the DataVolume (check `.involvedObject.name` equals ``). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe datavolume -n +``` + +**3. Check DataVolume phase**: + +Use `resources_get` from step 1, extract `.status.phase`. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +``` + +**4. Check if storage class supports cloning**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "name": "" +} +``` + +Review the full YAML output for cloning-related configurations. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get storageclass -o yaml | grep -A 5 -i clone +``` + +**5. Check CSI driver capabilities**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "storage.k8s.io/v1", + "kind": "CSIDriver" +} +``` + +Review `.items[].metadata.name` for available CSI drivers. + +**CLI Fallback** (if MCP unavailable): +```bash +oc get csidriver +``` + +**6. Check source PVC status**: + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should be `Bound` for cloning to work). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get pvc -n +``` + +**7. Check target namespace storage quota**: + +**MCP Tool**: `resources_list` (from openshift-virtualization) + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "ResourceQuota", + "namespace": "" +} +``` + +Review `.items[].status.hard` (limits) and `.items[].status.used` (current usage). + +**CLI Fallback** (if MCP unavailable): +```bash +oc describe quota -n +``` + +**8. Check CDI controller logs**: + +**MCP Tool**: `pods_list_in_namespace` + `pods_log` (from openshift-virtualization) + +First, list pods in openshift-cnv namespace: + +**Parameters for pods_list_in_namespace**: +```json +{ + "namespace": "openshift-cnv", + "labelSelector": "app.kubernetes.io/component=cdi-deployment" +} +``` + +Then get logs using `pods_log`: + +**Parameters**: +```json +{ + "namespace": "openshift-cnv", + "name": "", + "tail": 100 +} +``` + +**CLI Fallback** (if MCP unavailable or easier via CLI): +```bash +oc logs -n openshift-cnv $(oc get pods -n openshift-cnv | grep cdi-deployment | awk '{print $1}') +``` + +**Common Error Messages**: +- `"volume cloning is not supported"` - CSI driver lacks clone capability +- `"cross namespace clone is not supported"` - Cloning between namespaces forbidden by storage +- `"source PVC not found"` - Source PVC doesn't exist or wrong namespace +- `"insufficient quota"` - Target namespace lacks storage quota +- `"source PVC not bound"` - Source PVC must be in Bound state for cloning +- `"StorageClass does not support cloning"` - Storage class configuration issue + +**Solutions** (Use MCP Tools First): + +1. **Check storage class clone support**: + + **MCP Tool**: `resources_list` (from openshift-virtualization) + + **Parameters**: + ```json + { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass" + } + ``` + + For each storage class in `.items[]`, check: + - `.metadata.name` (storage class name) + - `.provisioner` (CSI driver) + + Storage classes using CSI drivers typically support cloning. Look for provisioners like: + - `csi.ovirt.org` (oVirt CSI) + - `openshift-storage.rbd.csi.ceph.com` (Ceph RBD) + - `ebs.csi.aws.com` (AWS EBS CSI) + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get storageclass -o custom-columns=NAME:.metadata.name,PROVISIONER:.provisioner + ``` + +2. **Verify source PVC is bound**: + + Use `resources_get` from diagnostic step 6, check `.status.phase` (should be `Bound`). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc get pvc -n -o jsonpath='{.status.phase}' + ``` + +3. **Check target namespace quota**: + + Use `resources_list` from diagnostic step 7 to check quota. + + If quota increase needed, this requires cluster admin privileges (cannot be done via MCP). + + **CLI Fallback** (if MCP unavailable): + ```bash + oc describe quota -n + ``` + +4. **Use snapshot-based cloning** (alternative method): + + **MCP Tool**: `resources_create_or_update` (from openshift-virtualization) + + **Step 1**: Create VolumeSnapshot of source PVC + + **Parameters**: + ```json + { + "apiVersion": "snapshot.storage.k8s.io/v1", + "kind": "VolumeSnapshot", + "metadata": { + "name": "-snapshot", + "namespace": "" + }, + "spec": { + "source": { + "persistentVolumeClaimName": "" + } + } + } + ``` + + **Step 2**: Wait for snapshot to be ready (use `resources_get` to check `.status.readyToUse`) + + **Step 3**: Create new DataVolume from snapshot + + **Parameters**: + ```json + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "name": "-rootdisk", + "namespace": "" + }, + "spec": { + "source": { + "snapshot": { + "name": "-snapshot", + "namespace": "" + } + }, + "storage": { + "resources": { + "requests": { + "storage": "50Gi" + } + }, + "storageClassName": "" + } + } + } + ``` + + **CLI Fallback** (YAML easier via CLI): + Ask user: "Snapshot-based cloning involves complex YAML. May I use `oc apply -f` instead?" + ```bash + cat <-snapshot + namespace: + spec: + source: + persistentVolumeClaimName: + EOF + + oc get volumesnapshot -snapshot -n + + cat <-rootdisk + namespace: + spec: + source: + snapshot: + name: -snapshot + namespace: + storage: + resources: + requests: + storage: 50Gi + storageClassName: + EOF + ``` + +5. **Use "new empty storage" option** (vm-clone skill): + - If cloning isn't supported, create VM with empty storage + - Manually copy data if needed + +6. **Cross-namespace cloning workaround**: + - Some storage backends require snapshot for cross-namespace cloning + - Create snapshot in source namespace, restore in target namespace (see solution 4 above) + +**Verification** (Use MCP Tools First): + +**MCP Tool**: `resources_get` (from openshift-virtualization) + +Check DataVolume reached Succeeded phase: + +**Parameters**: +```json +{ + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "namespace": "", + "name": "" +} +``` + +Check `.status.phase` (should return `Succeeded`). + +Check PVC was created and bound: + +**Parameters**: +```json +{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "namespace": "", + "name": "-rootdisk" +} +``` + +Check `.status.phase` (should return `Bound`). + +**CLI Fallback** (if MCP unavailable): +```bash +oc get datavolume -n -o jsonpath='{.status.phase}' +# Should return: Succeeded + +oc get pvc -rootdisk -n -o jsonpath='{.status.phase}' +# Should return: Bound +``` + +**Alternative**: If cloning continuously fails, use vm-create skill to create new VM with container disk or DataSource instead. + +--- + + +--- + +[← Back to Index](INDEX.md) | [← Scheduling Errors](scheduling-errors.md) | [Lifecycle Errors →](lifecycle-errors.md) From f357d4ea6aadefc0fb8abda43312f2180a90ef4d Mon Sep 17 00:00:00 2001 From: gziv Date: Sun, 31 May 2026 15:14:41 +0300 Subject: [PATCH 2/2] retrigger: queue runner parallel