diff --git a/docs/content/examples/collecting-engine-metrics.md b/docs/content/examples/collecting-engine-metrics.md index 9c83137a..5029767e 100644 --- a/docs/content/examples/collecting-engine-metrics.md +++ b/docs/content/examples/collecting-engine-metrics.md @@ -44,30 +44,14 @@ Modelplane stamps on them, and the `monitoring` namespace Prometheus discovers a `PodMonitor`, so this is the whole config. The engine container port is unnamed, so reference it by number with `targetPort`: -```yaml -apiVersion: monitoring.coreos.com/v1 -kind: PodMonitor -metadata: - name: qwen2-5-0-5b-metrics - namespace: default -spec: - selector: - matchExpressions: - - key: modelplane.ai/serving # carried by every serving pod - operator: Exists - podMetricsEndpoints: - - targetPort: 8000 - path: /metrics - interval: 30s -``` +{{< manifests "examples/collecting-engine-metrics/podmonitor.yaml" >}} The engine pods and the `PodMonitor` CRD live on the workload cluster, not the control plane, so apply it there. Then read the metrics from the in-cluster Prometheus over a `port-forward`: ```bash -kubectl apply -f podmonitor.yaml # workload cluster -kubectl -n monitoring port-forward svc/prometheus-prometheus 9090:9090 +kubectl -n monitoring port-forward svc/prometheus-prometheus 9090:9090 # workload cluster # open http://localhost:9090, Status > Targets to confirm the scrape, then query # e.g. vllm:num_requests_running or vllm:gpu_cache_usage_perc ``` diff --git a/docs/content/models/model-endpoint.md b/docs/content/models/model-endpoint.md index 1ecce28f..568976e1 100644 --- a/docs/content/models/model-endpoint.md +++ b/docs/content/models/model-endpoint.md @@ -17,52 +17,16 @@ the provider when your fleet is busy, or fail over to it as a break-glass option Create a `ModelEndpoint` with three things: -```yaml {nocopy=true} -apiVersion: modelplane.ai/v1alpha1 -kind: ModelEndpoint -metadata: - name: kimi-k2-together - namespace: ml-team - labels: - # 1. A label of your own for a ModelService to select on. Any label - # works; modelplane.ai/external-provider is a readable convention. - modelplane.ai/external-provider: together -spec: - # 2. The provider's base URL. - url: https://api.together.xyz/ - # 3. The path to rewrite requests to. A ModelService receives requests at - # ///v1/... and strips only the /// - # prefix, so an OpenAI-compatible provider that already serves /v1/... - # takes just /. - rewritePath: / -``` +{{< manifests "concepts/model-endpoint.yaml" >}} Then point a [`ModelService`]({{< ref "model-service.md" >}}) at it. Selecting `modelplane.ai/external-provider: together` routes to the provider; adding a second entry for a deployment fronts both behind one URL, so traffic can spill over to the provider alongside your own replicas: -```yaml {nocopy=true} -apiVersion: modelplane.ai/v1alpha1 -kind: ModelService -metadata: - name: kimi-k2 - namespace: ml-team -spec: - endpoints: - - selector: - matchLabels: - modelplane.ai/deployment: kimi-k2 # your own replicas - - selector: - matchLabels: - modelplane.ai/external-provider: together # the endpoint above -``` +{{< manifests "concepts/model-service-external.yaml" >}} The provider must speak the OpenAI API, since that's the contract a `ModelService` exposes. Anything OpenAI-compatible works; `url` and `rewritePath` are all that change between providers. - -## Example - -{{< manifests "concepts/model-endpoint.yaml" >}} diff --git a/docs/content/overview/_index.md b/docs/content/overview/_index.md index f10547f3..f1e3ed89 100644 --- a/docs/content/overview/_index.md +++ b/docs/content/overview/_index.md @@ -39,48 +39,13 @@ Once a platform team has provisioned inference clusters and declared the availab GPUs and networking fabric, an ML development team deploys a model with a declarative manifest: -```yaml {nocopy=true} -apiVersion: modelplane.ai/v1alpha1 -kind: ModelDeployment -metadata: - name: qwen-demo - namespace: ml-team -spec: - replicas: 1 - engines: - - name: qwen - members: - - role: Standalone - nodeSelector: - devices: - - name: gpu - count: 1 - selectors: - - cel: device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("20Gi")) >= 0 - template: - spec: - containers: - - name: engine - image: vllm/vllm-openai:v0.23.0 - args: ["--model=Qwen/Qwen2.5-0.5B-Instruct"] -``` +{{< manifests "overview/model-deployment.yaml" >}} Modelplane schedules a model replica onto an inference cluster with free, compatible GPUs and memory, and deploys the serving engine. Exposing an OpenAI-compatible endpoint can be done by declaring a model service: -```yaml {nocopy=true} -apiVersion: modelplane.ai/v1alpha1 -kind: ModelService -metadata: - name: qwen - namespace: ml-team -spec: - endpoints: - - selector: - matchLabels: - modelplane.ai/deployment: qwen-demo -``` +{{< manifests "overview/model-service.yaml" >}} ## A universal control plane for AI inference diff --git a/docs/manifests/concepts/model-endpoint.yaml b/docs/manifests/concepts/model-endpoint.yaml index 84529aac..8a6b1824 100644 --- a/docs/manifests/concepts/model-endpoint.yaml +++ b/docs/manifests/concepts/model-endpoint.yaml @@ -1,17 +1,20 @@ # Modelplane composes a ModelEndpoint per ModelReplica automatically. Create one # manually only to register an external inference endpoint with a ModelService, # for example a SaaS provider like Together or BaseTen. -# -# Give it a label of your own for a ModelService to select on -# (modelplane.ai/external-provider is a readable convention), and set -# url/rewritePath to the provider's OpenAI-compatible endpoint. apiVersion: modelplane.ai/v1alpha1 kind: ModelEndpoint metadata: - name: qwen3-coder-together + name: kimi-k2-together namespace: ml-team labels: + # 1. A label of your own for a ModelService to select on. Any label + # works; modelplane.ai/external-provider is a readable convention. modelplane.ai/external-provider: together spec: + # 2. The provider's base URL. url: https://api.together.xyz/ + # 3. The path to rewrite requests to. A ModelService receives requests at + # ///v1/... and strips only the /// + # prefix, so an OpenAI-compatible provider that already serves /v1/... + # takes just /. rewritePath: / diff --git a/docs/manifests/concepts/model-service-external.yaml b/docs/manifests/concepts/model-service-external.yaml new file mode 100644 index 00000000..4aeeabef --- /dev/null +++ b/docs/manifests/concepts/model-service-external.yaml @@ -0,0 +1,17 @@ +# A ModelService's endpoints list combines: one entry can select your own +# deployment's replicas while another selects an external ModelEndpoint, so +# overflow or break-glass traffic to a SaaS provider sits behind the same URL +# as your own replicas. +apiVersion: modelplane.ai/v1alpha1 +kind: ModelService +metadata: + name: kimi-k2 + namespace: ml-team +spec: + endpoints: + - selector: + matchLabels: + modelplane.ai/deployment: kimi-k2 # your own replicas + - selector: + matchLabels: + modelplane.ai/external-provider: together # the endpoint above diff --git a/docs/manifests/examples/collecting-engine-metrics/podmonitor.yaml b/docs/manifests/examples/collecting-engine-metrics/podmonitor.yaml new file mode 100644 index 00000000..00db036a --- /dev/null +++ b/docs/manifests/examples/collecting-engine-metrics/podmonitor.yaml @@ -0,0 +1,14 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: qwen2-5-0-5b-metrics + namespace: default +spec: + selector: + matchExpressions: + - key: modelplane.ai/serving # carried by every serving pod + operator: Exists + podMetricsEndpoints: + - targetPort: 8000 + path: /metrics + interval: 30s diff --git a/docs/manifests/overview/model-deployment.yaml b/docs/manifests/overview/model-deployment.yaml new file mode 100644 index 00000000..9a64aed5 --- /dev/null +++ b/docs/manifests/overview/model-deployment.yaml @@ -0,0 +1,23 @@ +apiVersion: modelplane.ai/v1alpha1 +kind: ModelDeployment +metadata: + name: qwen-demo + namespace: ml-team +spec: + replicas: 1 + engines: + - name: qwen + members: + - role: Standalone + nodeSelector: + devices: + - name: gpu + count: 1 + selectors: + - cel: device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("20Gi")) >= 0 + template: + spec: + containers: + - name: engine + image: vllm/vllm-openai:v0.23.0 + args: ["--model=Qwen/Qwen2.5-0.5B-Instruct"] diff --git a/docs/manifests/overview/model-service.yaml b/docs/manifests/overview/model-service.yaml new file mode 100644 index 00000000..e354b2c1 --- /dev/null +++ b/docs/manifests/overview/model-service.yaml @@ -0,0 +1,10 @@ +apiVersion: modelplane.ai/v1alpha1 +kind: ModelService +metadata: + name: qwen + namespace: ml-team +spec: + endpoints: + - selector: + matchLabels: + modelplane.ai/deployment: qwen-demo