From 267095a8592547e759a4a7bb5f2a8ebeb7bf9280 Mon Sep 17 00:00:00 2001 From: gziv Date: Sun, 31 May 2026 12:16:53 +0300 Subject: [PATCH 1/2] feat: add pipeline-manage ASE evaluation submission --- submissions/pipeline-manage/evals/evals.json | 17 + submissions/pipeline-manage/metadata.yaml | 5 + submissions/pipeline-manage/skills/SKILL.md | 315 ++++++++++++++++++ .../skills/references/known-model-profiles.md | 83 +++++ .../skills/references/model-deploy.md | 160 +++++++++ .../skills/references/nim-setup.md | 115 +++++++ .../skills/references/supported-runtimes.md | 104 ++++++ 7 files changed, 799 insertions(+) create mode 100644 submissions/pipeline-manage/evals/evals.json create mode 100644 submissions/pipeline-manage/metadata.yaml create mode 100644 submissions/pipeline-manage/skills/SKILL.md create mode 100644 submissions/pipeline-manage/skills/references/known-model-profiles.md create mode 100644 submissions/pipeline-manage/skills/references/model-deploy.md create mode 100644 submissions/pipeline-manage/skills/references/nim-setup.md create mode 100644 submissions/pipeline-manage/skills/references/supported-runtimes.md diff --git a/submissions/pipeline-manage/evals/evals.json b/submissions/pipeline-manage/evals/evals.json new file mode 100644 index 0000000..5d83447 --- /dev/null +++ b/submissions/pipeline-manage/evals/evals.json @@ -0,0 +1,17 @@ +{ + "skill_name": "pipeline-manage", + "evals": [ + { + "id": "pipeline-manage-eval", + "name": "Data Science Pipeline Management", + "prompt": "You have a Data Science Project on OpenShift AI and need to run ML pipelines. Describe the full pipeline lifecycle: how you verify or set up the pipeline server, how you submit pipeline runs, how you schedule recurring runs, and how you monitor execution with step-level detail.", + "expected_output": "A pipeline management methodology covering DSPA (pipeline server) verification/setup, submitting PipelineRun CRs from YAML definitions, scheduling recurring runs with cron expressions, monitoring run status with step-level progress and pod logs.", + "assertions": [ + "The output describes verifying or setting up the pipeline server (DSPA - DataSciencePipelinesApplication) as a prerequisite before running pipelines.", + "The output describes submitting pipeline runs from YAML definitions as PipelineRun custom resources.", + "The output describes scheduling recurring pipeline runs using cron expressions.", + "The output describes monitoring pipeline execution with step-level progress and the ability to view step pod logs." + ] + } + ] +} diff --git a/submissions/pipeline-manage/metadata.yaml b/submissions/pipeline-manage/metadata.yaml new file mode 100644 index 0000000..6e0842c --- /dev/null +++ b/submissions/pipeline-manage/metadata.yaml @@ -0,0 +1,5 @@ +name: pipeline-manage +description: "Create, run, schedule, and monitor Data Science Pipelines (Kubeflow" +persona: ocp-admin +version: "1.0.0" +eval_engine: ase diff --git a/submissions/pipeline-manage/skills/SKILL.md b/submissions/pipeline-manage/skills/SKILL.md new file mode 100644 index 0000000..d8210cc --- /dev/null +++ b/submissions/pipeline-manage/skills/SKILL.md @@ -0,0 +1,315 @@ +--- +name: pipeline-manage +description: 'Create, run, schedule, and monitor Data Science Pipelines (Kubeflow + Pipelines 2.0) on OpenShift AI. + + + Use when: + + - "Run a pipeline in my project" + + - "Schedule a recurring pipeline" + + - "Check my pipeline run status" + + - "List pipeline runs and their logs" + + - "Set up the pipeline server" + + - "Delete a pipeline or pipeline run" + + + Handles pipeline server setup, pipeline run submission from YAML, scheduling recurring + runs, monitoring execution, and viewing step logs. + + + NOT for creating data science projects (use /ds-project-setup). + + NOT for deploying models (use /model-deploy). + + NOT for model training jobs (use training skills). + + ' +license: Apache-2.0 +--- + +# /pipeline-manage Skill + +Create, run, schedule, and monitor Data Science Pipelines (Kubeflow Pipelines 2.0) on Red Hat OpenShift AI. Handles the full pipeline lifecycle: verifying or setting up the pipeline server (DSPA), submitting pipeline runs from YAML definitions, scheduling recurring runs with cron expressions, monitoring run status with step-level progress, viewing pipeline step logs, and deleting pipeline resources with proper warnings. + +## Prerequisites + +**Required MCP Server**: `openshift` ([OpenShift MCP Server](https://github.com/openshift/openshift-mcp-server)) + +**Required MCP Tools** (from openshift): +- `resources_create_or_update` - Create DSPA CR (pipeline server), PipelineRun and ScheduledWorkflow CRs +- `resources_list` - List PipelineRun resources, DSPA status +- `resources_get` - Get PipelineRun status, DSPA details +- `resources_delete` - Delete pipeline run resources, DSPA +- `events_list` - Check pipeline pod events for errors +- `pods_list` - List pipeline step pods +- `pods_log` - Retrieve pipeline step container logs + +**Preferred MCP Server**: `rhoai` ([RHOAI MCP Server](https://github.com/opendatahub-io/rhoai-mcp)) — used when available, automatic OpenShift fallback on failure + +**Preferred MCP Tools** (from rhoai): +- `list_data_science_projects` - Validate namespace is an RHOAI Data Science Project +- `get_pipeline_server` - Check pipeline server (DSPA) status in a project +- `delete_pipeline_server` - Delete pipeline server and all pipeline infrastructure +- `list_resources` - List pipeline resources in a namespace (resource_type="pipelines") +- `get_resource` - Get pipeline resource details (resource_type="pipeline") +- `list_resource_names` - Token-efficient pipeline name listing (resource_type="pipelines") +- `resource_status` - Quick pipeline status check (resource_type="pipeline") +- `diagnose_resource` - Full diagnostic for a pipeline (resource_type="pipeline") +- `list_data_connections` - Verify S3 data connections for pipeline artifact storage +- `project_summary` - Project overview including pipeline status + +**Common prerequisites** (KUBECONFIG, OpenShift+RHOAI cluster, verification protocol): See [skill-conventions.md](../references/skill-conventions.md). + +**Fallback templates**: See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md) for OpenShift YAML templates used when RHOAI tools are unavailable. + +**Additional cluster requirements**: +- Target namespace is an RHOAI Data Science Project (label: `opendatahub.io/dashboard: "true"`) +- Pipeline server requires S3-compatible object storage (a data connection must exist in the project) +- Data Science Pipelines operator component enabled in the RHOAI DataScienceCluster + +## When to Use This Skill + +**Use this skill when you need to:** +- Set up or verify the pipeline server in a Data Science Project +- Submit a pipeline run from a YAML or JSON pipeline definition +- Schedule a recurring pipeline run with a cron expression +- Monitor pipeline run status and step-level progress +- View logs from pipeline steps (especially failing ones) +- List existing pipelines and pipeline runs in a project +- Delete a pipeline run or the pipeline server + +**Do NOT use this skill when:** +- You need to create a Data Science Project first (use `/ds-project-setup`) +- You want to deploy a model for inference (use `/model-deploy`) +- You need to manage model training jobs (use training skills) +- You want to register models in the Model Registry (use `/model-registry`) + +## Workflow + +### Step 1: Determine Intent + +Ask the user what they want to do: **Setup** server, **List** pipelines/runs, **Run** pipeline, **Schedule** recurring run, **Monitor** run, **View logs**, **Delete** resources. + +Ask for target namespace. Validate via `list_data_science_projects` (from rhoai). If invalid, suggest `/ds-project-setup`. + +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: v1`, `kind: Namespace`, `labelSelector: opendatahub.io/dashboard=true`. + +Route: Setup -> Step 2, List -> Step 3, Run -> Step 4, Schedule -> Step 5, Monitor -> Step 6, Logs -> Step 7, Delete -> Step 8. + +### Step 2: Verify / Setup Pipeline Server + +Check via `get_pipeline_server` (from rhoai) with `namespace`. If healthy, proceed. If unhealthy, offer diagnostics via `diagnose_resource`. If not exists, offer setup. + +**If rhoai unavailable or returns error**: Use `resources_get` (from openshift) with `apiVersion: datasciencepipelinesapplications.opendatahub.io/v1alpha1`, `kind: DataSciencePipelinesApplication`, `name: dspa`, `namespace: [namespace]`. Check `.status.conditions` for `Ready=True`. + +**For setup**: Check data connections via `list_data_connections` (from rhoai). If none exist, offer to delegate to `/ds-project-setup`. + +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: v1`, `kind: Secret`, `namespace: [namespace]`, `labelSelector: opendatahub.io/dashboard=true`. Filter by annotation `opendatahub.io/connection-type: s3`. + +**Gather:** Select from available data connections (the data connection name is the S3 secret name). The bucket, endpoint, and region can be extracted from the data connection secret. Present configuration for review. **WAIT for confirmation.** + +**Pipeline Server Creation** (OpenShift direct — `create_pipeline_server` from rhoai is not used because it constructs invalid DSPA manifests): + +**MCP Tool**: `resources_create_or_update` (from openshift) + +Create a DataSciencePipelinesApplication CR. See [openshift-fallback-templates.md](../references/openshift-fallback-templates.md#datasciencepipelinesapplication-dspa) for the YAML template. + +**Parameters to fill in the template:** +- `namespace`: target namespace +- `bucket`: S3 bucket name from the data connection +- `host`: S3 endpoint without protocol prefix (e.g., `minio.namespace.svc:9000`) +- `scheme`: `http` or `https` +- `secretName`: name of the S3 data connection secret +- `region`: AWS region or empty string for MinIO + +**Verify DSPA is ready:** + +**MCP Tool**: `resources_get` (from openshift) +- `apiVersion`: `datasciencepipelinesapplications.opendatahub.io/v1alpha1`, `kind`: `DataSciencePipelinesApplication`, `name`: `dspa`, `namespace`: [namespace] + +Check `.status.conditions` for `Ready=True`. Poll every 15 seconds until ready or timeout (5 minutes). + +**Error Handling**: +- If S3 secret not found -> Suggest creating via `/ds-project-setup` +- If S3 endpoint unreachable -> Advise verifying URL and cluster network access + +### Step 3: List Pipelines and Runs + +Use `list_resources` (from rhoai) with `resource_type="pipelines"`, `namespace` for detailed listing, or `list_resource_names` for a quick name-only view. + +For specific run status: `resource_status` (from rhoai) with `resource_type="pipeline"`, `name`, `namespace`. + +For project-wide overview: `project_summary` (from rhoai) with `namespace`. + +**If rhoai unavailable or returns error**: Use `resources_list` (from openshift) with `apiVersion: tekton.dev/v1`, `kind: PipelineRun`, `namespace: [namespace]` to list pipeline runs directly. + +If pipeline server not configured, suggest setup via Step 2. + +### Step 4: Submit a Pipeline Run + +Verify pipeline server is ready via `get_pipeline_server` (from rhoai). If not ready, offer Step 2. + +**Gather from user:** pipeline definition (file path or inline YAML/JSON), run name (DNS-compatible), pipeline parameters (key-value pairs), service account (default: `pipeline-runner-dspa`). + +Read pipeline definition using the Read tool if a file path is provided. Present configuration for review. **WAIT for confirmation.** + +**MCP Tool**: `resources_create_or_update` (from openshift) + +**Parameters**: +- `resource`: PipelineRun CR (apiVersion: `tekton.dev/v1`, kind: `PipelineRun`) with `spec.pipelineSpec` or `spec.pipelineRef`, `spec.params`, `spec.serviceAccountName` - REQUIRED + +If the pipeline uses KFP v2 compiled format (Argo-based), adapt apiVersion/kind accordingly. + +Proceed to Step 6 to monitor the run. + +### Step 5: Schedule a Recurring Pipeline Run + +Verify pipeline server is ready (same as Step 4). + +**Gather from user:** pipeline reference (name or YAML), schedule (cron expression or natural language), pipeline parameters, max concurrent runs (default: 1), optional start/end time. + +Convert natural language to cron if needed. Present schedule configuration for review. **WAIT for confirmation.** + +**MCP Tool**: `resources_create_or_update` (from openshift) + +**Parameters**: +- `resource`: ScheduledWorkflow CR (apiVersion: `scheduledworkflows.kubeflow.org/v1beta1`, kind: `ScheduledWorkflow`) with `spec.enabled`, `spec.maxConcurrency`, `spec.trigger.cronSchedule.cron`, `spec.trigger.cronSchedule.startTime`/`endTime`, `spec.workflow.spec.params` - REQUIRED + +**Error Handling**: +- If ScheduledWorkflow CRD not available -> DSP operator may not support scheduling in this version + +### Step 6: Monitor Pipeline Run + +**Get run status** via `get_resource` (from rhoai) with `resource_type="pipeline"`, `name`, `namespace`, `verbosity="full"`. + +**For deeper diagnostics**: `diagnose_resource` (from rhoai) with `resource_type="pipeline"`, `name`, `namespace`. + +**If rhoai unavailable or returns error**: Use `resources_get` (from openshift) with `apiVersion: tekton.dev/v1`, `kind: PipelineRun`, `name: [run-name]`, `namespace: [namespace]`. Extract task status from `.status.childReferences` or `.status.taskRuns`. + +**Track step-level progress** via `resources_get` (from openshift) with apiVersion `tekton.dev/v1`, kind `PipelineRun`. Extract task statuses from `.status.childReferences` or `.status.taskRuns`. + +**List pipeline pods** via `pods_list` (from openshift) with `namespace` and `labelSelector="tekton.dev/pipelineRun="`. + +**Present step progress table:** + +| Step | Status | Duration | Message | +|------|--------|----------|---------| +| [step-name] | [Running/Succeeded/Failed] | [duration] | [message] | + +**On failure**: Present options: (1) View step logs, (2) Check events, (3) Run diagnostics, (4) Retry. **WAIT for user decision. NEVER auto-retry or auto-delete failed runs.** + +**Error Handling**: +- If run not found -> List available runs and ask user to select +- If run still pending -> Check events for scheduling issues + +### Step 7: View Pipeline Step Logs + +From the PipelineRun status (Step 6), identify the pod name for the failing or target step. If user did not specify a step, list available steps and ask which one to inspect. + +**MCP Tool**: `pods_log` (from openshift) + +**Parameters**: +- `namespace`: target namespace - REQUIRED +- `name`: pod name of the pipeline step - REQUIRED +- `container`: container name - OPTIONAL (specify if multiple containers) + +Also check `events_list` (from openshift) filtered by namespace for scheduling/resource issues. + +**Suggest fixes** based on common log patterns: `OOMKilled` -> increase memory limits; `ImagePullBackOff` -> verify image reference; `AccessDenied` on S3 -> check data connection via `/ds-project-setup`; `Permission denied` -> verify ServiceAccount permissions. + +**Error Handling**: +- If pod not found -> Pod may have been cleaned up; check events for historical data +- If multiple containers -> List container names and ask user to select + +### Step 8: Delete Pipeline Resources + +**Delete a pipeline run:** + +Get run details via `get_resource` (from rhoai) with `resource_type="pipeline"`, `name`, `namespace`. Display run details (name, status, created, step count). + +**Ask**: "Delete pipeline run ``? This will remove the run record and associated pods. (yes/no)" **WAIT for confirmation.** + +**MCP Tool**: `resources_delete` (from openshift) with apiVersion `tekton.dev/v1`, kind `PipelineRun`, `name`, `namespace`. + +--- + +**Delete the pipeline server (DESTRUCTIVE):** + +Display warning: deleting the DSPA removes all pipeline runs, history, API/UI endpoints, and terminates running pipelines. S3 data is preserved. + +**Ask**: "Type the namespace name `` to confirm deletion:" **WAIT for typed confirmation. Verify exact match.** + +**MCP Tool**: `delete_pipeline_server` (from rhoai) with `namespace`, `confirm=true`. + +**Verify** via `get_pipeline_server` (from rhoai). Confirm removal. + +**Error Handling**: +- If typed confirmation mismatch -> Cancel deletion +- If RBAC error -> Report insufficient permissions +- If pipeline server not found -> Report no server in namespace + +## Common Issues + +For common issues (GPU scheduling, OOMKilled, image pull errors, RBAC), see [common-issues.md](../references/common-issues.md). + +### Issue 1: Pipeline Server Not Ready +**Cause**: Invalid S3 credentials, unreachable S3 endpoint, or database connectivity issues. +**Solution**: Use `diagnose_resource` for automated diagnostics. Check data connection credentials via `list_data_connections`. Verify S3 bucket accessibility. Check DSPA pod logs for specific errors. + +### Issue 2: Pipeline Run Stuck in Pending +**Cause**: Insufficient resources, missing ServiceAccount, unbound PVC, or scheduling issues. +**Solution**: Check events for scheduling errors. Verify ServiceAccount `pipeline-runner-dspa` exists. Check ResourceQuota/LimitRange. For GPU pipelines, verify availability via `/ai-observability`. + +### Issue 3: Step OOMKilled +**Cause**: Step container exceeded memory limit. +**Solution**: View step logs before OOM kill. Increase `resources.limits.memory` in the pipeline YAML. Consider streaming data for data-intensive steps. + +### Issue 4: S3 Artifact Upload Fails +**Cause**: Expired credentials, missing bucket, or unreachable endpoint. +**Solution**: Verify data connection via `list_data_connections`. Check S3 bucket exists. Re-create data connection if credentials rotated. + +### Issue 5: Schedule Not Triggering +**Cause**: Invalid cron, ScheduledWorkflow controller not running, or schedule disabled. +**Solution**: Verify ScheduledWorkflow CR `spec.enabled` is `true`. Validate cron format. Check controller pod logs. Verify `startTime`/`endTime` constraints. + +## Dependencies + +### MCP Tools +See [Prerequisites](#prerequisites) for the complete list of required MCP tools. + +### Related Skills +- `/ds-project-setup` - Create a Data Science Project with data connections (prerequisite for pipeline server) +- `/model-deploy` - Deploy a model produced by a pipeline +- `/model-registry` - Register models produced by pipeline runs +- `/ai-observability` - Check GPU/cluster resources before running resource-intensive pipelines +- `/debug-inference` - Debug models deployed from pipeline outputs + +### Reference Documentation +- [skill-conventions.md](../references/skill-conventions.md) - Shared prerequisite, HITL, and security conventions + +## Critical: Human-in-the-Loop Requirements + +See [skill-conventions.md](../references/skill-conventions.md) for general HITL and security conventions. + +**Skill-specific checkpoints:** +- Before creating pipeline server (Step 2): display S3 storage configuration, confirm +- Before submitting pipeline run (Step 4): display run configuration table, confirm +- Before scheduling recurring run (Step 5): display cron schedule and parameters, confirm +- On pipeline failure (Step 6): present diagnostic options, wait for user decision +- Before deleting pipeline run (Step 8): display run details, confirm +- Before deleting pipeline server (Step 8): display destructive warning, require typed namespace confirmation +- **NEVER** auto-submit pipeline runs without confirmation +- **NEVER** auto-delete pipeline servers or auto-retry failed runs +- **NEVER** display S3 credentials or secret values in output + +## Example Usage + +**User**: "Set up the pipeline server in my ml-training project and run a data preprocessing pipeline" + +**Skill response**: Validates project, sets up pipeline server with S3 data connection after confirmation, monitors readiness, then gathers pipeline YAML, presents run config for confirmation, submits PipelineRun, and monitors step-level progress. diff --git a/submissions/pipeline-manage/skills/references/known-model-profiles.md b/submissions/pipeline-manage/skills/references/known-model-profiles.md new file mode 100644 index 0000000..4014898 --- /dev/null +++ b/submissions/pipeline-manage/skills/references/known-model-profiles.md @@ -0,0 +1,83 @@ +--- +title: Known Model Hardware Profiles +category: references +tags: [models, gpu, hardware, profiles, llama, granite, mixtral, mistral] +semantic_keywords: [model GPU requirements, hardware profiles, vLLM configuration, model deployment specs] +use_cases: [model-deploy, debug-inference, ai-observability] +last_updated: 2026-03-10 +--- + +# Known Model Hardware Profiles + +Hardware profiles for commonly deployed models on OpenShift AI. This file serves as a fast-path cache for `/model-deploy` — when a model is listed here, the agent uses these specs directly. When a model is not listed, the agent falls back to the live doc lookup protocol defined in [live-doc-lookup.md](live-doc-lookup.md). + +**Important**: These are recommended minimums. Actual requirements may vary based on quantization, sequence length, and batch size. Validate against live documentation for production deployments. + +## Model Source Conventions + +Each model lists a recommended `storageUri` with its authentication requirements: +- **`hf://`** — HuggingFace Hub. Public models require no authentication. **Preferred default for public open-source models.** +- **`oci://`** — OCI container registry. Requires image pull secrets with appropriate entitlements. `registry.redhat.io/rhelai1/*` images require RHEL AI subscription entitlements (not included in standard OpenShift pull secrets). +- **`s3://`** — S3-compatible storage. Requires storage credentials configured in the namespace. + +When the user does not specify a model source, use the `hf://` URI listed in the profile below. + +## Llama 3.x (Meta) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Llama 3.1 8B | 8B | 1 | A10G/L4/A100/H100 | 16GB | `--max-model-len=4096` | +| Llama 3.1 70B | 70B | 4 | A100 80GB | 320GB | `--max-model-len=4096 --tensor-parallel-size=4` | +| Llama 3.1 70B | 70B | 2 | H100 80GB | 160GB | `--max-model-len=4096 --tensor-parallel-size=2` | +| Llama 3.1 405B | 405B | 8 | A100 80GB / H100 | 640GB | `--max-model-len=4096 --tensor-parallel-size=8` | + +- **Recommended storageUri**: `hf://meta-llama/Llama-3.1-8B-Instruct` (public, no auth — requires HuggingFace license acceptance) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/llama-3-1-8b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser hermes --chat-template` (for Llama 3.1+ instruct variants) +- Quantization: AWQ, GPTQ, FP8 variants reduce GPU requirements significantly + +## Granite 3.x (IBM/Red Hat) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Granite 3.1 2B | 2B | 1 | Any GPU | 8GB | `--max-model-len=4096` | +| Granite 3.1 8B | 8B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=4096` | + +- **Recommended storageUri (2B)**: `hf://ibm-granite/granite-3.1-2b-instruct` (public, no auth) +- **Recommended storageUri (8B)**: `hf://ibm-granite/granite-3.1-8b-instruct` (public, no auth) +- **OCI alternative**: `oci://registry.redhat.io/rhelai1/granite-3-1-2b-instruct` (requires RHEL AI entitlements) +- Tool calling: `--tool-call-parser granite --chat-template` +- Red Hat-supported model family on RHOAI + +## Mixtral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mixtral 8x7B | 46.7B (MoE) | 2 | A100 80GB | 160GB | `--tensor-parallel-size=2` | +| Mixtral 8x22B | 141B (MoE) | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (8x7B)**: `hf://mistralai/Mixtral-8x7B-Instruct-v0.1` (public, no auth) +- Mixture-of-Experts architecture: only ~13B/45B parameters active per token + +## Mistral (Mistral AI) + +| Variant | Parameters | GPUs | GPU Type | VRAM | Key vLLM Args | +|---------|-----------|------|----------|------|---------------| +| Mistral 7B | 7B | 1 | A10G/L4/A100 | 16GB | `--max-model-len=8192` | +| Mistral Large (123B) | 123B | 4 | A100 80GB | 320GB | `--tensor-parallel-size=4` | + +- **Recommended storageUri (7B)**: `hf://mistralai/Mistral-7B-Instruct-v0.3` (public, no auth) + +## When a Model Is Not Listed + +If the requested model is not in this file, the agent MUST use the live doc lookup protocol: + +1. Read [live-doc-lookup.md](live-doc-lookup.md) for the lookup procedure +2. Fetch hardware specs from the appropriate source +3. Report findings to the user before proceeding with deployment + +Common cases requiring live lookup: +- Newly released models (after this file's last update) +- Domain-specific fine-tuned models +- Models with custom quantization +- NIM-specific optimized profiles diff --git a/submissions/pipeline-manage/skills/references/model-deploy.md b/submissions/pipeline-manage/skills/references/model-deploy.md new file mode 100644 index 0000000..dcd79e4 --- /dev/null +++ b/submissions/pipeline-manage/skills/references/model-deploy.md @@ -0,0 +1,160 @@ +# /model-deploy Examples + +Example interactions demonstrating the model-deploy skill workflow. + +## Example 1: Deploy Llama 3.1 8B with vLLM + +``` +User: "Deploy Llama 3.1 8B on my cluster" + +Agent: [Invokes /model-deploy skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand runtime capabilities. + +Step 1 of 10: Gathering deployment information. + +| Setting | Value | Source | +|---------|-------|--------| +| Model | Llama 3.1 8B Instruct | user input | +| Runtime | TBD | auto-detect | +| Namespace | ? | needs input | +| Model Source | ? | needs input | +| Deployment Mode | Serverless | default | + +What namespace should I deploy to, and where are the model weights stored? +(e.g., S3 bucket, OCI registry, PVC) + +User: "namespace: ai-models, model is on S3 at s3://my-bucket/llama-3.1-8b" + +Agent: Step 2 of 10: Determining runtime. + +I consulted supported-runtimes.md to understand runtime capabilities. + +Based on Llama 3.1 8B (open-source HuggingFace model): + +Recommended runtime: vLLM +Reason: Default runtime for open-source LLMs. Broad compatibility, + OpenAI-compatible API, no additional setup required. + +Proceed with vLLM? (yes / no / suggest alternative) + +User: "yes" + +Agent: Step 3 of 10: Looking up hardware profile. + +I consulted known-model-profiles.md to find hardware requirements for Llama 3.1 8B. + +| Requirement | Value | +|-------------|-------| +| GPUs | 1x A10G/L4/A100/H100 | +| VRAM | 16GB | +| Key Args | --max-model-len=4096 --tool-call-parser hermes | + +Step 4 of 10: GPU pre-flight check skipped (ai-observability MCP not configured). +Ensure your cluster has at least 1 GPU with 16GB VRAM. + +Step 5 of 10: Skipped (NIM platform check not needed for vLLM). + +Step 6-7 of 10: Generating InferenceService YAML. + +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: llama-3-1-8b + namespace: ai-models + annotations: + serving.kserve.io/deploymentMode: Serverless +spec: + predictor: + model: + modelFormat: + name: vLLM + runtime: vllm-runtime + storageUri: s3://my-bucket/llama-3.1-8b + resources: + limits: + nvidia.com/gpu: "1" + requests: + cpu: "4" + memory: "16Gi" + args: + - --max-model-len=4096 + - --tool-call-parser=hermes + +| Setting | Value | +|---------|-------| +| Model | llama-3-1-8b | +| Runtime | vllm-runtime | +| GPUs | 1 | +| Namespace | ai-models | +| Mode | Serverless | + +Proceed with creating this InferenceService? (yes/no/modify) + +User: "yes" + +Agent: Step 8 of 10: Creating InferenceService... + [Creates via resources_create_or_update] + + InferenceService created. Monitoring rollout... + +Step 9 of 10: Monitoring rollout. + +| Stage | Status | +|-------|--------| +| Pod Scheduled | Complete | +| Image Pulled | Complete | +| Container Started | Complete | +| Model Loaded | Complete | +| Ready | Complete | + +Step 10 of 10: Deployment complete! + +Model Deployment Successful + +Model: llama-3-1-8b +Runtime: vLLM +Namespace: ai-models +GPUs: 1 + +Inference Endpoint: + URL: https://llama-3-1-8b-ai-models.apps.cluster.example.com + API: OpenAI-compatible REST + +Quick Test: + curl -X POST https://llama-3-1-8b-ai-models.apps.cluster.example.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "llama-3-1-8b", "messages": [{"role": "user", "content": "Hello"}]}' + +Next Steps: + Monitor performance: "/ai-observability llama-3-1-8b" +``` + +## Example 2: Deploy a Model with NIM (NIM Not Set Up) + +``` +User: "Deploy Llama 3.1 70B with NIM" + +Agent: [Invokes /model-deploy skill] + ... + Step 2: User requested NIM runtime. + + Step 5: Checking NIM platform... + +NIM platform is not set up in namespace ai-models. + +The NIM runtime requires a one-time platform setup. Would you like to: +1. Run /nim-setup now to configure the NIM platform +2. Switch to vLLM runtime instead (no setup required) +3. Abort deployment + +User: "1" + +Agent: [Invokes /nim-setup skill — walks through NGC credential setup] + ... + NIM Platform Setup Complete. + + Resuming model deployment... + [Continues from Step 6 with NIM runtime] +``` diff --git a/submissions/pipeline-manage/skills/references/nim-setup.md b/submissions/pipeline-manage/skills/references/nim-setup.md new file mode 100644 index 0000000..ccfc1a5 --- /dev/null +++ b/submissions/pipeline-manage/skills/references/nim-setup.md @@ -0,0 +1,115 @@ +# /nim-setup Examples + +Example interactions demonstrating the nim-setup skill workflow. + +## Example 1: First-Time NIM Setup + +``` +User: "Set up NIM on my cluster" + +Agent: [Invokes /nim-setup skill] + [Reads supported-runtimes.md] + +I consulted supported-runtimes.md to understand NIM platform requirements. + +Let me verify the prerequisites for NIM setup. + +Step 1 of 7: Checking GPU Operator and NFD Operator... + +GPU Operator: gpu-operator-certified.v24.6.2 (Succeeded) +NFD Operator: nfd.v4.19.0 (Succeeded) + +Both operators are installed and healthy. + +Step 2 of 7: I need your NGC credentials. + +To set up NIM, I need your NVIDIA NGC API key. +You can generate one at: https://ngc.nvidia.com/setup/api-key + +Please provide: +1. Your NGC API key +2. The target namespace for NIM resources + +User: "Here's my key: [user provides NGC API key], namespace: ai-models" + +Agent: Step 3 of 7: Creating NGC image pull secret... + +I'll create a docker-registry secret for pulling NIM images from nvcr.io: + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-image-pull-secret + namespace: ai-models + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: [base64-encoded config for nvcr.io - REDACTED] + +Should I create this image pull secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + Image pull secret created successfully. + +Step 4 of 7: Creating NGC API key secret... + + apiVersion: v1 + kind: Secret + metadata: + name: ngc-api-key + namespace: ai-models + type: Opaque + stringData: + NGC_API_KEY: "[REDACTED]" + +Should I create this API key secret in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates secret via resources_create_or_update] + API key secret created successfully. + +Step 5 of 7: Creating NIM Account CR... + + apiVersion: nim.opendatahub.io/v1 + kind: Account + metadata: + name: nim-account + namespace: ai-models + spec: + apiKeySecret: + name: ngc-api-key + imagePullSecret: + name: ngc-image-pull-secret + +Should I create this NIM Account CR in namespace `ai-models`? (yes/no) + +User: "yes" + +Agent: [Creates Account CR via resources_create_or_update] + Account CR created successfully. + +Step 6 of 7: Would you like to customize the NIM model catalog? (yes/no, default: no) + +User: "no" + +Agent: Step 7 of 7: Validating NIM platform readiness... + +NIM Platform Setup Complete + +Account CR: `nim-account` (namespace: ai-models) + Status: Ready + +NGC Credentials: + Image Pull Secret: ngc-image-pull-secret [created] + API Key Secret: ngc-api-key [created] + +NIM ServingRuntimes Available: + - nim-llama-3-1-8b-instruct + - nim-llama-3-1-70b-instruct + - nim-mistral-7b-instruct + +Next Steps: + To deploy a model with NIM: "/model-deploy llama-3.1-8b --runtime nim" +``` diff --git a/submissions/pipeline-manage/skills/references/supported-runtimes.md b/submissions/pipeline-manage/skills/references/supported-runtimes.md new file mode 100644 index 0000000..9d01865 --- /dev/null +++ b/submissions/pipeline-manage/skills/references/supported-runtimes.md @@ -0,0 +1,104 @@ +--- +title: Supported Serving Runtimes +category: references +tags: [runtimes, vllm, nim, caikit, tgis, serving] +semantic_keywords: [serving runtime selection, model format compatibility, inference runtime comparison] +use_cases: [model-deploy, serving-runtime-config, nim-setup] +last_updated: 2026-02-26 +--- + +# Supported Serving Runtimes + +This document maps each serving runtime available on Red Hat OpenShift AI to its capabilities, supported model formats, and selection criteria. Used by `/model-deploy` to determine the appropriate runtime for a given model. + +## Runtime Comparison + +| Runtime | API Type | Model Format | GPU Support | License | Setup Required | +|---------|----------|-------------|-------------|---------|----------------| +| vLLM | OpenAI-compatible REST | vLLM | NVIDIA, Intel Gaudi, CPU | Open-source (Apache 2.0) | None (built-in) | +| NVIDIA NIM | OpenAI-compatible REST | NIM (TensorRT-LLM) | NVIDIA only | NVIDIA AI Enterprise | `/nim-setup` | +| Caikit+TGIS | gRPC | Caikit | NVIDIA | Red Hat (built-in) | Model conversion | +| Custom | Varies | Varies | Varies | Varies | `/serving-runtime-config` | + +## vLLM + +**Default runtime for most open-source models.** + +- **API**: OpenAI-compatible REST (`/v1/completions`, `/v1/chat/completions`) +- **Model formats**: HuggingFace Transformers, AWQ, GPTQ, FP8 quantized +- **GPU support**: NVIDIA (CUDA), Intel Gaudi (HPU), CPU (limited) +- **Model source**: S3-compatible storage, OCI registry, PVC, URI +- **Key features**: + - Broad model compatibility (Llama, Granite, Mixtral, Mistral, Falcon, etc.) + - PagedAttention for efficient memory management + - Tensor parallelism for multi-GPU inference + - Tool/function calling support (`--tool-call-parser`) + - Continuous batching for high throughput +- **When to choose**: Default choice for any open-source LLM. Best balance of compatibility and performance. +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `vLLM` in the name + +## NVIDIA NIM + +**Optimized inference for NVIDIA GPUs with TensorRT-LLM.** + +- **API**: OpenAI-compatible REST +- **Model formats**: Pre-optimized TensorRT-LLM engines from NGC catalog +- **GPU support**: NVIDIA only (requires specific GPU models per model profile) +- **Model source**: NGC model catalog (pulled automatically via NGC credentials) +- **Prerequisites**: `/nim-setup` must be completed first (NGC secrets, Account CR) +- **Key features**: + - TensorRT-LLM optimization for lower latency + - Pre-compiled model engines (no compilation on first load) + - Optimized and generic profiles per GPU type + - Automatic model download from NGC +- **When to choose**: When maximum inference performance on NVIDIA GPUs is required and the model is available in the NGC catalog. +- **Limitations**: NVIDIA GPUs only, requires NVIDIA AI Enterprise license, limited model selection compared to vLLM +- **ServingRuntime CR name**: Created automatically by the NIM Account CR. Check with `resources_list` for `ServingRuntime` resources with `nim` in the name. +- **CRDs**: Account (`nim.opendatahub.io/v1`) manages NIM platform state + +## Caikit+TGIS + +**Red Hat's Caikit format with Text Generation Inference Server.** + +- **API**: gRPC (not REST) +- **Model formats**: Caikit format (requires conversion from HuggingFace) +- **GPU support**: NVIDIA +- **Model source**: S3-compatible storage +- **Key features**: + - Red Hat-supported runtime + - gRPC API for streaming inference + - Integrated with RHOAI model serving platform +- **When to choose**: When the model is already in Caikit format or when gRPC API is required. +- **Limitations**: Requires model conversion to Caikit format, smaller model ecosystem, gRPC-only API +- **ServingRuntime CR name**: Check available runtimes with `resources_list` for `ServingRuntime` resources with `caikit` in the name + +## Custom Runtimes + +**User-provided ServingRuntime CRs for unsupported frameworks.** + +- **API**: Defined by the custom runtime +- **Model formats**: Defined by the custom runtime +- **When to choose**: When none of the built-in runtimes support the model framework or when specific customization is needed. +- **How to create**: Use `/serving-runtime-config` skill +- **Limitations**: Not supported by Red Hat, user responsibility for maintenance and compatibility + +## Runtime Selection Decision Tree + +``` +Is the user's preferred runtime explicitly stated? +├── Yes → Use that runtime +└── No → Continue + +Is the model available in the NGC NIM catalog? +├── Yes → Suggest NIM (with vLLM as fallback) +│ Note: Requires /nim-setup and NVIDIA GPUs +└── No → Continue + +Is the model in Caikit format? +├── Yes → Caikit+TGIS +└── No → Continue + +Is the model a standard open-source LLM (HuggingFace-compatible)? +├── Yes → vLLM (default) +└── No → Custom runtime via /serving-runtime-config +``` From c15a2c6d64eb09bcf9832f31a7b8aa1fb27e5ff2 Mon Sep 17 00:00:00 2001 From: gziv Date: Sun, 31 May 2026 14:22:42 +0300 Subject: [PATCH 2/2] retrigger: queue runner parallel