From 8a66d3666a6413758c5760a5da0698b7ec86ae93 Mon Sep 17 00:00:00 2001 From: gziv Date: Sun, 31 May 2026 12:16:46 +0300 Subject: [PATCH 1/2] feat: add model-deploy ASE evaluation submission --- submissions/model-deploy/evals/evals.json | 17 + submissions/model-deploy/metadata.yaml | 5 + submissions/model-deploy/skills/SKILL.md | 471 ++++++++++++++++++ .../skills/references/known-model-profiles.md | 83 +++ .../skills/references/model-deploy.md | 160 ++++++ .../skills/references/nim-setup.md | 115 +++++ .../skills/references/supported-runtimes.md | 104 ++++ 7 files changed, 955 insertions(+) create mode 100644 submissions/model-deploy/evals/evals.json create mode 100644 submissions/model-deploy/metadata.yaml create mode 100644 submissions/model-deploy/skills/SKILL.md create mode 100644 submissions/model-deploy/skills/references/known-model-profiles.md create mode 100644 submissions/model-deploy/skills/references/model-deploy.md create mode 100644 submissions/model-deploy/skills/references/nim-setup.md create mode 100644 submissions/model-deploy/skills/references/supported-runtimes.md diff --git a/submissions/model-deploy/evals/evals.json b/submissions/model-deploy/evals/evals.json new file mode 100644 index 0000000..559ea56 --- /dev/null +++ b/submissions/model-deploy/evals/evals.json @@ -0,0 +1,17 @@ +{ + "skill_name": "model-deploy", + "evals": [ + { + "id": "model-deploy-eval", + "name": "AI Model Deployment Methodology", + "prompt": "You need to deploy a Llama 3 model on OpenShift AI. The cluster has NVIDIA GPUs available.\n\nDescribe your model deployment methodology: what serving runtimes are available, how you select the right one, how you validate GPU availability, how you create the InferenceService, and how you monitor the deployment rollout.", + "expected_output": "A deployment methodology using KServe InferenceService with runtime selection (vLLM, NVIDIA NIM, Caikit+TGIS), GPU pre-flight checks (node availability, taints, tolerations), hardware profile lookup, and rollout monitoring with pod status checks.", + "assertions": [ + "The output describes selecting between serving runtimes: vLLM, NVIDIA NIM, and Caikit+TGIS, explaining when each is appropriate.", + "The output describes GPU pre-flight validation: checking node availability, GPU taints, and configuring tolerations on the InferenceService.", + "The output describes creating a KServe InferenceService custom resource as the deployment mechanism.", + "The output describes monitoring the deployment rollout by checking predictor pod status and InferenceService readiness conditions." + ] + } + ] +} diff --git a/submissions/model-deploy/metadata.yaml b/submissions/model-deploy/metadata.yaml new file mode 100644 index 0000000..bdbc988 --- /dev/null +++ b/submissions/model-deploy/metadata.yaml @@ -0,0 +1,5 @@ +name: model-deploy +description: "Deploy AI/ML models on OpenShift AI using KServe with vLLM, NVIDIA NIM," +persona: rh-ai-engineer +version: "1.0.0" +eval_engine: ase diff --git a/submissions/model-deploy/skills/SKILL.md b/submissions/model-deploy/skills/SKILL.md new file mode 100644 index 0000000..699b774 --- /dev/null +++ b/submissions/model-deploy/skills/SKILL.md @@ -0,0 +1,471 @@ +--- +name: model-deploy +description: 'Deploy AI/ML models on OpenShift AI using KServe with vLLM, NVIDIA NIM, + or Caikit+TGIS runtimes. + + + Use when: + + - "Deploy Llama 3 on my cluster" + + - "Set up a vLLM inference endpoint" + + - "Deploy a model with NIM" + + - "Create an InferenceService for Granite" + + - "I need to serve a model on OpenShift AI" + + + Handles runtime selection, GPU validation, InferenceService CR creation, and rollout + monitoring. + + + NOT for NIM platform setup (use /nim-setup first). + + NOT for custom runtime creation (use /serving-runtime-config). + + ' +license: Apache-2.0 +--- + +# /model-deploy Skill + +Deploy AI/ML models on Red Hat OpenShift AI using KServe. Supports vLLM, NVIDIA NIM, and Caikit+TGIS serving runtimes. Handles runtime selection, hardware profile lookup (with live doc fallback), GPU pre-flight checks, InferenceService CR creation, rollout monitoring, and post-deployment validation. + +## Prerequisites + +**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools** (from openshift): +- `resources_get` - Check NIM Account CR, LimitRange, GPU node taints, InferenceService status +- `resources_list` - Check Knative availability, GPU nodes, existing deployments, ServingRuntimes +- `resources_create_or_update` - Create/patch InferenceService, add tolerations (OpenShift fallback) +- `pods_list` - Check predictor pod status during rollout +- `pods_log` - Retrieve pod logs for debugging +- `events_list` - Check events for errors + +**Preferred MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) — used when available, automatic OpenShift fallback on failure + +**Preferred MCP Tools** (from rhoai): +- `deploy_model` - Create InferenceService with high-level parameters (no YAML construction needed). **Known limitation**: does not support tolerations or NIM-specific env vars — see fallback patterns below. +- `list_inference_services` - List deployed models with structured status data +- `get_inference_service` - Get detailed model deployment status (conditions, endpoint, ready state) +- `get_model_endpoint` - Get inference endpoint URL directly +- `list_serving_runtimes` - List available runtimes including platform templates with supported model formats +- `list_data_science_projects` - Discover RHOAI projects for namespace validation +- `list_data_connections` - Verify model storage access (S3 data connections) + +**Optional MCP Server**: `ai-observability` ([AI Observability MCP](https://github.com/rh-ai-quickstart/ai-observability-summarizer)) + +**Optional MCP Tools** (from ai-observability): +- `get_gpu_info` - Pre-flight GPU inventory check +- `get_deployment_info` - Post-deployment validation +- `analyze_vllm` - Verify metrics are flowing after deployment + +**Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, KServe, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). + +**Fallback templates**: See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md) for OpenShift YAML templates used when RHOAI tools are unavailable. + +**Additional cluster requirements**: +- For NIM runtime: NIM platform set up via `/nim-setup` +- For vLLM/NIM: NVIDIA GPU nodes available in the cluster + +## When to Use This Skill + +**Use this skill when you need to:** +- Deploy an AI/ML model on OpenShift AI (KServe InferenceService) +- Set up vLLM, NIM, or Caikit+TGIS inference endpoints +- Look up hardware profiles and GPU requirements for a model +- Perform pre-flight validation before model deployment (GPU availability, namespace readiness, LimitRange conflicts) + +**Do NOT use this skill when:** +- You need to set up the NIM platform first (use `/nim-setup`) +- You need to create or customize a ServingRuntime (use `/serving-runtime-config`) +- You need to troubleshoot a failed or slow deployment (use `/debug-inference`) +- You need to analyze model performance or GPU metrics (use `/ai-observability`) + +## Workflow + +### Step 1: Gather Target and Validate Environment + +Collect the deployment target from the user, then immediately validate the environment before gathering remaining details. + +**Ask the user for:** +- **Model name**: Which model to deploy (e.g., "Llama 3.1 8B", "Granite 3.1 8B") +- **Namespace**: Target namespace (must have model serving enabled) +- **Model source**: Where the model weights are stored (S3, OCI registry, PVC, NGC for NIM, or artifact URI from `/model-registry`) +- **Deployment mode**: Serverless (Knative, default) or RawDeployment + +**Pre-flight Environment Validation**: + +**CRITICAL**: Run these checks BEFORE gathering remaining deployment details to avoid wasted configuration effort. + +Read [model-deploy-preflight-checklist.md](references/model-deploy-preflight-checklist.md) for the full pre-flight protocol. The checklist validates: +- Namespace is an RHOAI Data Science Project +- Model storage access (S3 data connections) +- Deployment mode support (Knative availability) +- Namespace resource constraints (LimitRange conflicts with KServe sidecars) +- GPU node taints (auto-generate tolerations) +- Existing deployments (reference configuration) +- Model source accessibility (OCI registry entitlements) + +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: v1`, `kind: Namespace`, `labelSelector: opendatahub.io/dashboard=true` to find RHOAI projects. + +**Present pre-flight results** in a summary table and note any adjustments made. **WAIT for user confirmation if significant changes were needed** (e.g., deployment mode switch, resource adjustments, tolerations added). + +### Step 2: Gather Deployment Details + +After the environment is validated, collect remaining deployment configuration. Use pre-flight findings to inform defaults (e.g., if Knative is unavailable, default to RawDeployment). + +**Ask the user for:** +- **Runtime preference**: vLLM (default), NIM, or Caikit+TGIS (auto-detect if not specified) +- **Model source**: Where the model weights are stored (S3, OCI registry, PVC, or NGC for NIM) +- **Deployment mode**: Serverless (Knative, default) or RawDeployment + +**Present configuration table for review:** + +| Setting | Value | Source | +|---------|-------|--------| +| Model | [model-name] | user input (Step 1) | +| Runtime | [to be determined in Step 3] | auto-detect / user input | +| Namespace | [namespace] | user input (Step 1) | +| Model Source | [source-uri] | user input | +| Deployment Mode | [Serverless/RawDeployment] | user input / default (informed by pre-flight) | + +**WAIT for user to confirm or modify these settings.** + +### Step 3: Determine Runtime + +**Document Consultation** (read before selecting runtime): +1. **Action**: Read [supported-runtimes.md](../../docs/references/supported-runtimes.md) using the Read tool to understand runtime capabilities and selection criteria +2. **Output to user**: "I consulted [supported-runtimes.md](../../docs/references/supported-runtimes.md) to understand runtime capabilities." + +**Runtime Selection Logic:** + +- User explicitly requested a runtime -> Use that runtime +- Model available in NGC NIM catalog -> Suggest NIM (with vLLM as fallback) +- Model is a standard open-source LLM (HuggingFace-compatible) -> Default to vLLM +- Model is in Caikit format -> Caikit+TGIS +- None of the above -> Suggest custom runtime via `/serving-runtime-config` + +**Present recommendation** with rationale. **WAIT for user confirmation.** + +### Step 4: Look Up Model Hardware Profile + +**Document Consultation** (read before determining hardware requirements): +1. **Action**: Read [known-model-profiles.md](../../docs/references/known-model-profiles.md) using the Read tool to find hardware profile for the requested model +2. **Output to user**: "I consulted [known-model-profiles.md](../../docs/references/known-model-profiles.md) to find hardware requirements for [model-name]." + +**If model IS in known-model-profiles.md:** +- Extract: GPU count, GPU type, VRAM, key vLLM args +- Present to user + +**If model is NOT in known-model-profiles.md -> Trigger live doc lookup:** +1. **Action**: Read [live-doc-lookup.md](../references/live-doc-lookup.md) using the Read tool for the lookup protocol +2. **Output to user**: "Model [model-name] is not in my cached profiles. I'll look up its hardware requirements." +3. Use **WebFetch** tool to retrieve specs from: + - For NIM models: `https://build.nvidia.com/models` or `https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html` + - For other models: `https://docs.redhat.com/en/documentation/red_hat_openshift_ai_cloud_service/1` +4. Extract: GPU requirements, model-specific args, known issues +5. **Output to user**: "I looked up [model-name] on [source] to confirm its hardware requirements: [summary]" + +**Present hardware requirements** in a table (GPUs, VRAM, Key Args). + +### Step 5: Pre-flight GPU Check (Optional) + +**Condition**: Only if `ai-observability` MCP server is available. + +**MCP Tool**: `get_gpu_info` (from ai-observability) + +Compare available GPUs against model requirements from Step 4: +- If sufficient GPUs available -> Report match and proceed +- If insufficient -> Warn user with options: smaller model, quantized version, different cluster, or proceed at user's risk + +**If ai-observability not available**: Skip with note: "GPU pre-flight check skipped (ai-observability MCP not configured)." + +### Step 6: Verify NIM Platform (NIM Runtime Only) + +**Condition**: Only when the selected runtime is NIM. + +**MCP Tool**: `resources_get` (from openshift) + +**Parameters**: +- `apiVersion`: `"nim.opendatahub.io/v1alpha1"` - REQUIRED +- `kind`: `"Account"` - REQUIRED +- `namespace`: target namespace - REQUIRED +- `name`: `"nim-account"` - REQUIRED + +**If Account CR not found or not ready:** +Offer options: (1) Run `/nim-setup` now, (2) Switch to vLLM, (3) Abort. **WAIT for user decision.** + +### Step 7: Select ServingRuntime and Prepare Deployment Parameters + +**Verify available ServingRuntimes:** + +**MCP Tool**: `list_serving_runtimes` (from rhoai) + +**Parameters**: +- `namespace`: target namespace - REQUIRED +- `include_templates`: `true` - REQUIRED (shows both existing runtimes and platform templates) + +The response shows existing runtimes and available templates with their supported model formats and `requires_instantiation` flag. + +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: serving.kserve.io/v1alpha1`, `kind: ServingRuntime`, `namespace: [target]` to list namespace runtimes, and `kind: ClusterServingRuntime` for platform templates. Filter by label `opendatahub.io/dashboard=true`. + +If the needed runtime shows `requires_instantiation: true`, it must first be instantiated via `/serving-runtime-config` or the rhoai `create_serving_runtime` tool. + +Use the runtime list to select the correct `runtime` name for the deployment. + +**Prepare deployment parameters** from Steps 1-4 and environment data from Step 1: + +| Parameter | Value | Source | +|-----------|-------|--------| +| `name` | [model-deployment-name] | user input (DNS-compatible) | +| `namespace` | [namespace] | user input | +| `runtime` | [serving-runtime-name] | selected from `list_serving_runtimes` (Step 7) | +| `model_format` | [vLLM/pytorch/onnx/caikit/etc.] | runtime selection | +| `storage_uri` | [model-source-uri] | user input (prefer `hf://` for public models) | +| `gpu_count` | [gpu-count] | from hardware profile (Step 4) | +| `cpu_request` | [cpu] | from profile, adjusted for LimitRange | +| `memory_request` | [memory] | from profile, adjusted for LimitRange | +| `min_replicas` | [1] | default 1 (0 for scale-to-zero) | +| `max_replicas` | [1] | default 1 | + +**Model sizing guide** for LLMs: +- 7B models: 1x 24GB GPU (e.g., A10G) or 1x 16GB GPU with quantization +- 13B models: 2x 24GB GPUs +- 70B models: 4+ 80GB GPUs (A100/H100) or quantized on fewer GPUs + +**Scale-to-zero note**: Setting `min_replicas=0` saves resources but introduces cold start latency (30s-2min for model loading). + +### Step 8: User Review and Confirmation + +**Display the deployment parameters table** and a configuration summary to the user. + +**Ask**: "Proceed with deploying this model? (yes/no/modify)" + +**WAIT for explicit confirmation.** + +- If **yes** -> Proceed to Step 9 +- If **no** -> Abort +- If **modify** -> Ask what to change, update parameters, return to this step + +### Step 9: Deploy Model + +**MCP Tool**: `deploy_model` (from rhoai) + +**Parameters**: +- `name`: deployment name (DNS-compatible) - REQUIRED +- `namespace`: target namespace - REQUIRED +- `runtime`: serving runtime name from Step 7 - REQUIRED +- `model_format`: model format string (e.g., `"vLLM"`, `"pytorch"`, `"onnx"`) - REQUIRED +- `storage_uri`: model location (e.g., `"hf://ibm-granite/granite-3.1-2b-instruct"`, `"s3://bucket/path"`, `"pvc://pvc-name/path"`) - REQUIRED +- `display_name`: human-readable display name - OPTIONAL +- `min_replicas`: minimum replicas (default: 1, 0 for scale-to-zero) - OPTIONAL +- `max_replicas`: maximum replicas (default: 1) - OPTIONAL +- `cpu_request`: CPU request per replica (default: `"1"`) - OPTIONAL +- `cpu_limit`: CPU limit per replica (default: `"2"`) - OPTIONAL +- `memory_request`: memory request per replica (default: `"4Gi"`) - OPTIONAL +- `memory_limit`: memory limit per replica (default: `"8Gi"`) - OPTIONAL +- `gpu_count`: number of GPUs per replica (default: 0) - OPTIONAL + +**Note**: For NIM deployments, ensure the NGC API key secret is referenced. If `deploy_model` does not support NIM-specific env vars, fall back to `resources_create_or_update` (from openshift) with a NIM InferenceService YAML that includes `spec.predictor.env` referencing the `ngc-api-key` secretKeyRef. + +#### GPU Toleration Handling + +After `deploy_model` succeeds (or after creating InferenceService via OpenShift fallback), check if GPU tolerations are needed: + +**MCP Tool**: `resources_list` (from openshift) +- `apiVersion`: `v1`, `kind`: `Node`, `labelSelector`: `nvidia.com/gpu.present=true` + +If GPU nodes have taints (check `.spec.taints[]`), patch the InferenceService to add matching tolerations: + +**MCP Tool**: `resources_create_or_update` (from openshift) + +Add tolerations to `spec.predictor.tolerations` matching the discovered taints. Common GPU taints include: +- `nvidia.com/gpu` (Exists/NoSchedule) +- `ai-app=true` (Equal/NoSchedule) +- `ai-node=big` (Equal/NoSchedule) + +After patching, delete the stuck Pending pod to force rescheduling with the new tolerations. + +See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md#toleration-post-deploy-patch) for the complete pattern. + +#### NIM Deployment via OpenShift + +When deploying with NIM runtime and `deploy_model` does not support NIM-specific env vars (NGC_API_KEY secretKeyRef, NIM_MAX_MODEL_LEN, image pull secrets): + +Use `resources_create_or_update` (from openshift) with the NIM InferenceService template from [openshift-fallback-templates.md](../references/openshift-fallback-templates.md#inferenceservice-nim). + +**Key NIM-specific fields:** +- `spec.predictor.containers[0].env` with NGC_API_KEY from secretKeyRef +- `spec.predictor.imagePullSecrets` referencing `ngc-image-pull-secret` +- Use a specific image tag (e.g., `1.8.3`) — the `latest` tag may have CUDA driver incompatibility +- Set `NIM_MAX_MODEL_LEN` to prevent KV cache OOM (use `16384` for T4 GPUs) + +**Error Handling**: +- If namespace not found -> Report error, suggest creating namespace or using `/ds-project-setup` +- If ServingRuntime not found -> Report error, verify runtime name, suggest `/serving-runtime-config` +- If quota exceeded -> Report error, suggest reducing resource requests +- If RBAC error -> Report insufficient permissions + +### Step 10: Monitor Rollout + +Poll InferenceService status until ready or timeout (10 minutes). + +**MCP Tool**: `get_inference_service` (from rhoai) +- `name`: deployment name, `namespace`: target namespace, `verbosity`: `"full"` + +**If rhoai unavailable or returns error**: Use `resources_get` (from openshift) with `apiVersion: serving.kserve.io/v1beta1`, `kind: InferenceService`, `name: [model-name]`, `namespace: [namespace]`. Check `.status.conditions` for `Ready=True`. + +Check the Ready condition and status. Repeat every 15-30 seconds until Ready=True or timeout. + +**Check predictor pod status:** + +**MCP Tool**: `pods_list` (from openshift) +- `namespace`: target namespace, `labelSelector`: `"serving.kserve.io/inferenceservice=[model-name]"` + +Show deployment progress tracking: Pod Scheduled, Image Pulled, Container Started, Model Loaded, Ready. Include pod name, status, and restart count. + +**On failure:** Check pod logs (`pods_log`) and events (`events_list`) for diagnostics. Present options: (1) View full pod logs, (2) Check namespace events, (3) Invoke `/debug-inference`, (4) Delete and retry, (5) Continue waiting. **WAIT for user decision. NEVER auto-delete failed deployments.** + +### Step 11: Deployment Complete + +**Get endpoint URL:** + +**MCP Tool**: `get_model_endpoint` (from rhoai) +- `name`: deployment name, `namespace`: target namespace + +**If rhoai unavailable or returns error**: Extract endpoint from `resources_get` (from openshift) on the InferenceService — the URL is in `.status.url`. + +**Report success** showing: model name, runtime, namespace, GPUs, inference endpoint URL, API type (OpenAI-compatible REST), and next steps (`/ai-observability`, `/model-monitor`, `/guardrails-config`). + +**Provide test commands** based on runtime: +- **vLLM (OpenAI-compatible)**: `curl -X POST [endpoint]/v1/completions -H "Content-Type: application/json" -d '{"model":"[model-name]","prompt":"Hello","max_tokens":100}'` +- **KServe v2**: `curl -X POST [endpoint]/v2/models/[model-name]/infer -H "Content-Type: application/json" -d '{"inputs":[...]}'` + +**Post-deployment validation** (if ai-observability MCP available): +- `get_deployment_info` to confirm model appears in monitoring +- `analyze_vllm` with a short time range to verify initial metrics are flowing +- Report findings to user + +## Common Issues + +For common issues (GPU scheduling, OOMKilled, image pull errors, RBAC), see [common-issues.md](../references/common-issues.md). + +### Issue 1: InferenceService Stuck in "Unknown" + +**Error**: InferenceService `status.conditions` shows "Unknown" state + +**Cause**: ServingRuntime not found in the namespace, or model serving platform not enabled. + +**Solution:** +1. Verify ServingRuntime exists: `resources_list` for `servingruntimes` in namespace +2. Ensure model serving is enabled: namespace has label `opendatahub.io/dashboard: "true"` +3. Check the runtime name in the InferenceService matches an available ServingRuntime +4. If no matching runtime, use `/serving-runtime-config` to create one + +### Issue 2: Model Download Timeout + +**Error**: Pod starts but times out while downloading model weights from S3 or OCI registry + +**Cause**: Large model size combined with slow network connection to storage. + +**Solution:** +1. Add `serving.knative.dev/progress-deadline` annotation with a longer timeout (e.g., `"1800s"`) +2. Verify S3/storage credentials are valid +3. Consider using a PVC with pre-downloaded model weights instead +4. Check network connectivity between the pod and storage endpoint + +### Issue 3: LimitRange Conflicts with KServe Sidecars + +**Error**: Pod rejected with `minimum cpu usage per Container is 50m, but request is 10m` or `minimum memory usage per Container is 64Mi, but request is 15Mi` + +**Cause**: The namespace has a LimitRange with minimum resource constraints that exceed the hardcoded resource requests of KServe-injected sidecar containers (oauth-proxy, queue-proxy, or modelcar containers request 10m CPU / 15Mi memory). These sidecar resource values cannot be controlled through the InferenceService spec. + +**Solution:** +1. Check LimitRange: `resources_list` for `LimitRange` in the namespace +2. If LimitRange minimum CPU > 10m or minimum memory > 15Mi, the LimitRange must be adjusted +3. Options: (a) Lower LimitRange minimums to accommodate sidecars (min CPU ≤ 10m, min memory ≤ 15Mi), (b) Remove the LimitRange entirely, (c) Deploy in a different namespace without restrictive LimitRanges +4. **Prevention**: Step 1 pre-flight validation now checks for this conflict before deployment + +### Issue 4: GPU Node Taints Prevent Scheduling + +**Error**: Pod stuck in Pending with events showing `node(s) had untolerated taint {ai-app: true}` or similar custom taint messages, while also showing `Insufficient nvidia.com/gpu` on remaining nodes + +**Cause**: GPU nodes are tainted with custom taints (e.g., `ai-app=true:NoSchedule`) to reserve them for AI workloads. The InferenceService predictor pod does not have matching tolerations, so it cannot be scheduled on GPU nodes. + +**Solution:** +1. Identify GPU node taints: `resources_get` for GPU nodes, check `.spec.taints` +2. Add matching tolerations to the InferenceService predictor spec: + ```yaml + spec: + predictor: + tolerations: + - key: "ai-app" + operator: "Equal" + value: "true" + effect: "NoSchedule" + ``` +3. **Prevention**: Step 1 pre-flight validation now auto-discovers GPU node taints and generates tolerations + +### Issue: Pod Stuck Pending Due to GPU Node Taints + +**Error**: Pod shows "0/N nodes are available: node(s) had untolerated taint" in events + +**Cause**: `deploy_model` does not support tolerations. GPU nodes in production clusters are almost always tainted. + +**Solution**: Patch InferenceService with tolerations matching the GPU node taints, then delete the stuck pod. See [common-issues.md](../references/common-issues.md#deploy-model-missing-gpu-tolerations) for details. + +### Issue: NIM CUDA Driver Incompatibility + +**Error**: NIM container crashes with error code 803 or CUDA-related errors + +**Cause**: The `latest` NIM image tag may bundle a CUDA version incompatible with the GPU node's driver. + +**Solution**: Pin NIM image to a specific tag compatible with the cluster's GPU driver version (e.g., `1.8.3` for T4 nodes with older drivers). Check the NVIDIA NIM release notes for driver compatibility. + +### Issue: Stale ReplicaSets After InferenceService Patch + +**Error**: Multiple ReplicaSets exist after patching the InferenceService (e.g., adding tolerations), causing duplicate Pending pods + +**Cause**: Each InferenceService spec change triggers a new ReplicaSet. Old ReplicaSets are not automatically cleaned up. + +**Solution**: Scale down stale ReplicaSets to 0 replicas via `resources_create_or_update` (from openshift), or delete them. Identify the current ReplicaSet by checking which one has the latest creation timestamp. + +## Dependencies + +### MCP Tools +See [Prerequisites](#prerequisites) for the complete list of required and optional MCP tools. + +### Related Skills +- `/nim-setup` - Prerequisite for NIM runtime deployments +- `/debug-inference` - Troubleshoot InferenceService failures +- `/ai-observability` - Analyze deployed model performance +- `/serving-runtime-config` - Create custom ServingRuntime CRs +- `/ds-project-setup` - Create a namespace with model serving enabled +- `/model-registry` - Get artifact URIs for registered model versions to deploy +- `/model-monitor` - Configure bias and drift monitoring after deployment +- `/guardrails-config` - Add content safety guardrails to LLM deployments + +### Reference Documentation +- [known-model-profiles.md](../../docs/references/known-model-profiles.md) - Hardware profiles for common models +- [supported-runtimes.md](../../docs/references/supported-runtimes.md) - Runtime capabilities and selection criteria +- [live-doc-lookup.md](../references/live-doc-lookup.md) - Protocol for fetching specs for unknown models + +## Critical: Human-in-the-Loop Requirements + +See [skill-conventions.md](../references/skill-conventions.md) for general HITL and security conventions. + +**Skill-specific checkpoints:** +- After pre-flight validation (Step 1): confirm if significant adjustments were needed (deployment mode, tolerations, resource changes) +- After gathering deployment details (Step 2): confirm configuration table +- After runtime selection (Step 3): confirm runtime choice +- User review before deploy_model (Step 8): confirm deployment parameters +- On deployment failure (Step 10): present diagnostic options, wait for user decision +- **NEVER** auto-delete failed deployments or auto-select runtimes without confirmation + +## Example Usage + +See [model-deploy examples](../../docs/examples/model-deploy.md) for complete deployment walkthroughs (vLLM and NIM). diff --git a/submissions/model-deploy/skills/references/known-model-profiles.md b/submissions/model-deploy/skills/references/known-model-profiles.md new file mode 100644 index 0000000..4014898 --- /dev/null +++ b/submissions/model-deploy/skills/references/known-model-profiles.md @@ -0,0 +1,83 @@ +--- +title: Known Model Hardware Profiles +category: references +tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] +semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] +use_cases: [model-deploy, debug-inference, ai-observability] +last_updated: 2026-03-10 +--- + +# Known Model Hardware Profiles + +Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). + +**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. + +## Model Source Conventions + +Each model lists a recommended `storageUri` with its authentication requirements: +- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** +- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). +- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. + +When the user does not specify a model source, use the `hf://` URI listed in the profile below. + +## Llama 3.x (Meta) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | +| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | +| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | +| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | + +- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) +- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly + +## Granite 3.x (IBM/Red Hat) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | +| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | + +- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) +- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser granite --chat-template` +- Red Hat-supported model family on RHOAI + +## Mixtral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | +| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) +- Mixture-of-Experts architecture: only ~13B/45B parameters active per token + +## Mistral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | +| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) + +## When a Model Is Not Listed + +If the requested model is not in this file, the agent MUST use the live doc lookup protocol: + +1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure +2. Fetch hardware specs from the appropriate source +3. Report findings to the user before proceeding with deployment + +Common cases requiring live lookup: +- Newly released models (after this file's last update) +- Domain-specific fine-tuned models +- Models with custom quantization +- NIM-specific optimized profiles diff --git a/submissions/model-deploy/skills/references/model-deploy.md b/submissions/model-deploy/skills/references/model-deploy.md new file mode 100644 index 0000000..dcd79e4 --- /dev/null +++ b/submissions/model-deploy/skills/references/model-deploy.md @@ -0,0 +1,160 @@ +# /model-deploy Examples + +Example interactions demonstrating the model-deploy skill workflow. + +## Example 1: Deploy Llama 3.1 8B with vLLM + +``` +User: "Deploy Llama 3.1 8B on my cluster" + +Agent: [Invokes /model-deploy skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand runtime capabilities. + +Step 1 of 10: Gathering deployment information. + +| Setting | Value | Source | +|---------|-------|--------| +| Model | Llama 3.1 8B Instruct | user input | +| Runtime | TBD | auto-detect | +| Namespace | ? | needs input | +| Model Source | ? | needs input | +| Deployment Mode | Serverless | default | + +What namespace should I deploy to, and where are the model weights stored? +(e.g., S3 bucket, OCI registry, PVC) + +User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" + +Agent: Step 2 of 10: Determining runtime. + +I consulted supported-runtimes.md to understand runtime capabilities. + +Based on Llama 3.1 8B (open-source HuggingFace model): + +Recommended runtime: vLLM +Reason: Default runtime for open-source LLMs. Broad compatibility, + OpenAI-compatible API, no additional setup required. + +Proceed with vLLM? (yes / no / suggest alternative) + +User: "yes" + +Agent: Step 3 of 10: Looking up hardware profile. + +I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. + +| Requirement | Value | +|-------------|-------| +| GPUs | 1x A10G/L4/A100/H100 | +| VRAM | 16GB | +| Key Args | --max-model-len=4096 --tool-call-parser hermes | + +Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). +Ensure your cluster has at least 1 GPU with 16GB VRAM. + +Step 5 of 10: Skipped (NIM platform check not needed for vLLM). + +Step 6-7 of 10: Generating InferenceService YAML. + +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: llama-3-1-8b + namespace: ai-models + annotations: + serving.kserve.io/deploymentMode: Serverless +spec: + predictor: + model: + modelFormat: + name: vLLM + runtime: vllm-runtime + storageUri: s3://my-bucket/llama-3.1-8b + resources: + limits: + nvidia.com/gpu: "1" + requests: + cpu: "4" + memory: "16Gi" + args: + - --max-model-len=4096 + - --tool-call-parser=hermes + +| Setting | Value | +|---------|-------| +| Model | llama-3-1-8b | +| Runtime | vllm-runtime | +| GPUs | 1 | +| Namespace | ai-models | +| Mode | Serverless | + +Proceed with creating this InferenceService? (yes/no/modify) + +User: "yes" + +Agent: Step 8 of 10: Creating InferenceService... + [Creates via resources_create_or_update] + + InferenceService created. Monitoring rollout... + +Step 9 of 10: Monitoring rollout. + +| Stage | Status | +|-------|--------| +| Pod Scheduled | Complete | +| Image Pulled | Complete | +| Container Started | Complete | +| Model Loaded | Complete | +| Ready | Complete | + +Step 10 of 10: Deployment complete! + +Model Deployment Successful + +Model: llama-3-1-8b +Runtime: vLLM +Namespace: ai-models +GPUs: 1 + +Inference Endpoint: + URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com + API: OpenAI-compatible REST + +Quick Test: + curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' + +Next Steps: + Monitor performance: "/ai-observability llama-3-1-8b" +``` + +## Example 2: Deploy a Model with NIM (NIM Not Set Up) + +``` +User: "Deploy Llama 3.1 70B with NIM" + +Agent: [Invokes /model-deploy skill] + ... + Step 2: User requested NIM runtime. + + Step 5: Checking NIM platform... + +NIM platform is not set up in namespace ai-models. + +The NIM runtime requires a one-time platform setup. Would you like to: +1. Run /nim-setup now to configure the NIM platform +2. Switch to vLLM runtime instead (no setup required) +3. Abort deployment + +User: "1" + +Agent: [Invokes /nim-setup skill — walks through NGC credential setup] + ... + NIM Platform Setup Complete. + + Resuming model deployment... + [Continues from Step 6 with NIM runtime] +``` diff --git a/submissions/model-deploy/skills/references/nim-setup.md b/submissions/model-deploy/skills/references/nim-setup.md new file mode 100644 index 0000000..ccfc1a5 --- /dev/null +++ b/submissions/model-deploy/skills/references/nim-setup.md @@ -0,0 +1,115 @@ +# /nim-setup Examples + +Example interactions demonstrating the nim-setup skill workflow. + +## Example 1: First-Time NIM Setup + +``` +User: "Set up NIM on my cluster" + +Agent: [Invokes /nim-setup skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand NIM platform requirements. + +Let me verify the prerequisites for NIM setup. + +Step 1 of 7: Checking GPU Operator and NFD Operator... + +GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) +NFD Operator: nfd.v4.19.0 (Succeeded) + +Both operators are installed and healthy. + +Step 2 of 7: I need your NGC credentials. + +To set up NIM, I need your NVIDIA NGC API key. +You can generate one at: https://ngc.nvidia.com/setup/api-key + +Please provide: +1. Your NGC API key +2. The target namespace for NIM resources + +User: "Here's my key: [user provides NGC API key], namespace: ai-models" + +Agent: Step 3 of 7: Creating NGC image pull secret... + +I'll create a docker-registry secret for pulling NIM images from nvcr.io: + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-image-pull-secret + namespace: ai-models + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] + +Should I create this image pull secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + Image pull secret created successfully. + +Step 4 of 7: Creating NGC API key secret... + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-api-key + namespace: ai-models + type: Opaque + stringData: + NGC_API_KEY: "[REDACTED]" + +Should I create this API key secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + API key secret created successfully. + +Step 5 of 7: Creating NIM Account CR... + + apiVersion: nim.opendatahub.io/v1 + kind: Account + metadata: + name: nim-account + namespace: ai-models + spec: + apiKeySecret: + name: ngc-api-key + imagePullSecret: + name: ngc-image-pull-secret + +Should I create this NIM Account CR in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates Account CR via resources_create_or_update] + Account CR created successfully. + +Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) + +User: "no" + +Agent: Step 7 of 7: Validating NIM platform readiness... + +NIM Platform Setup Complete + +Account CR: `nim-account` (namespace: ai-models) + Status: Ready + +NGC Credentials: + Image Pull Secret: ngc-image-pull-secret [created] + API Key Secret: ngc-api-key [created] + +NIM ServingRuntimes Available: + - nim-llama-3-1-8b-instruct + - nim-llama-3-1-70b-instruct + - nim-mistral-7b-instruct + +Next Steps: + To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" +``` diff --git a/submissions/model-deploy/skills/references/supported-runtimes.md b/submissions/model-deploy/skills/references/supported-runtimes.md new file mode 100644 index 0000000..9d01865 --- /dev/null +++ b/submissions/model-deploy/skills/references/supported-runtimes.md @@ -0,0 +1,104 @@ +--- +title: Supported Serving Runtimes +category: references +tags: [runtimes, vllm, nim, caikit, tgis, serving] +semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] +use_cases: [model-deploy, serving-runtime-config, nim-setup] +last_updated: 2026-02-26 +--- + +# Supported Serving Runtimes + +This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. + +## Runtime Comparison + +| Runtime | API Type | Model Format | GPU Support | License | Setup Required | +|---------|----------|-------------|-------------|---------|----------------| +| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | +| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | +| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | +| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | + +## vLLM + +**Default runtime for most open-source models.** + +- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) +- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized +- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) +- **Model source**: S3-compatible storage, OCI registry, PVC, URI +- **Key features**: + - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) + - PagedAttention for efficient memory management + - Tensor parallelism for multi-GPU inference + - Tool/function calling support (`--tool-call-parser`) + - Continuous batching for high throughput +- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name + +## NVIDIA NIM + +**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** + +- **API**: OpenAI-compatible REST +- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog +- **GPU support**: NVIDIA only (requires specific GPU models per model profile) +- **Model source**: NGC model catalog (pulled automatically via NGC credentials) +- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) +- **Key features**: + - TensorRT-LLM optimization for lower latency + - Pre-compiled model engines (no compilation on first load) + - Optimized and generic profiles per GPU type + - Automatic model download from NGC +- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. +- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM +- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. +- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state + +## Caikit+TGIS + +**Red Hat's Caikit format with Text Generation Inference Server.** + +- **API**: gRPC (not REST) +- **Model formats**: Caikit format (requires conversion from HuggingFace) +- **GPU support**: NVIDIA +- **Model source**: S3-compatible storage +- **Key features**: + - Red Hat-supported runtime + - gRPC API for streaming inference + - Integrated with RHOAI model serving platform +- **When to choose**: When the model is already in Caikit format or when gRPC API is required. +- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name + +## Custom Runtimes + +**User-provided ServingRuntime CRs for unsupported frameworks.** + +- **API**: Defined by the custom runtime +- **Model formats**: Defined by the custom runtime +- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. +- **How to create**: Use `/serving-runtime-config` skill +- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility + +## Runtime Selection Decision Tree + +``` +Is the user's preferred runtime explicitly stated? +├── Yes → Use that runtime +└── No → Continue + +Is the model available in the NGC NIM catalog? +├── Yes → Suggest NIM (with vLLM as fallback) +│ Note: Requires /nim-setup and NVIDIA GPUs +└── No → Continue + +Is the model in Caikit format? +├── Yes → Caikit+TGIS +└── No → Continue + +Is the model a standard open-source LLM (HuggingFace-compatible)? +├── Yes → vLLM (default) +└── No → Custom runtime via /serving-runtime-config +``` From d04ec803240a1fb3af634e8bc766fa8a0c27bfd7 Mon Sep 17 00:00:00 2001 From: gziv Date: Sun, 31 May 2026 14:19:29 +0300 Subject: [PATCH 2/2] retrigger: queue runner parallel