From 5b409482c6b063cf7e2b97175407ab2a40d3298d Mon Sep 17 00:00:00 2001 From: WenjiaoYue Date: Wed, 22 Jul 2026 18:24:43 -0700 Subject: [PATCH] feat: add plain service benchmark baseline Signed-off-by: WenjiaoYue --- README.md | 2 +- config/templates/jinja/08_httproute.yaml.j2 | 8 +- .../templates/jinja/10_helmfile-main.yaml.j2 | 11 +- config/templates/jinja/11_infra.yaml.j2 | 5 +- .../13a_modelservice-direct-service.yaml.j2 | 31 +++ config/templates/values/defaults.yaml | 2 + docs/standup.md | 6 +- llmdbenchmark/interface/experiment.py | 3 +- llmdbenchmark/interface/plan.py | 3 +- llmdbenchmark/interface/run.py | 3 +- llmdbenchmark/interface/standup.py | 3 +- llmdbenchmark/parser/README.md | 5 + llmdbenchmark/parser/render_plans.py | 75 +++++++ .../run/steps/step_03_detect_endpoint.py | 18 ++ llmdbenchmark/smoketests/base.py | 11 + .../steps/step_02_admin_prerequisites.py | 46 +++-- .../standup/steps/step_08_deploy_router.py | 13 +- .../steps/step_09_deploy_modelservice.py | 91 ++++---- .../standup/steps/step_10_smoketest.py | 9 + .../standup/steps/step_11_inference_test.py | 8 + .../steps/step_03_delete_resources.py | 2 + llmdbenchmark/utilities/endpoint.py | 48 +++++ tests/test_direct_service_mode.py | 194 ++++++++++++++++++ 23 files changed, 527 insertions(+), 70 deletions(-) create mode 100644 config/templates/jinja/13a_modelservice-direct-service.yaml.j2 create mode 100644 tests/test_direct_service_mode.py diff --git a/README.md b/README.md index 08bb38b9f..0ac8f713f 100644 --- a/README.md +++ b/README.md @@ -414,7 +414,7 @@ llmdbenchmark --version | `-p NS` | `LLMDBENCH_NAMESPACE` | Namespace(s) to render into the plan | | `-m MODELS` | `LLMDBENCH_MODELS` | Model to render the plan for | | `-t METHODS` | `LLMDBENCH_METHODS` | Deployment method (`standalone`, `modelservice`) | -| `--gateway-class CLASS` | `LLMDBENCH_GATEWAY_CLASS` | Override the scenario's `gateway.className`. Accepted on the modelservice path: `epponly`, `istio`, `agentgateway`, `gke`, `data-science-gateway-class`. Ignored (any value accepted, including `none`) when the active deploy method is `kustomize`, `standalone`, or `fma`. | +| `--gateway-class CLASS` | `LLMDBENCH_GATEWAY_CLASS` | Override the scenario's `gateway.className`. Accepted on the modelservice path: `none`, `epponly`, `istio`, `agentgateway`, `gke`, `data-science-gateway-class`. `none` exposes decode vLLM directly through a plain Service with no Gateway, EPP, Envoy, or routing proxy. Ignored when the active deploy method is `kustomize`, `standalone`, or `fma`. | | `-f` / `--monitoring` | | Enable monitoring in rendered templates (PodMonitor, EPP verbosity) | | `-k FILE` | `LLMDBENCH_KUBECONFIG` / `KUBECONFIG` | Kubeconfig path (used for cluster resource auto-detection) | diff --git a/config/templates/jinja/08_httproute.yaml.j2 b/config/templates/jinja/08_httproute.yaml.j2 index 332a909a3..473f6b548 100644 --- a/config/templates/jinja/08_httproute.yaml.j2 +++ b/config/templates/jinja/08_httproute.yaml.j2 @@ -35,13 +35,15 @@ `siblingStacks` and `stackIndex` are injected by llmdbenchmark.parser.render_plans._process_stack. #} -{# Modelservice-only template. Two skip conditions: +{# Modelservice-only template. Skip when: 1. -t kustomize -- upstream guide manifests define their own routing. 2. gateway.className == 'epponly' -- no Gateway exists, so an HTTPRoute - would be unresolvable; clients hit the EPP service directly. #} + would be unresolvable; clients hit the EPP service directly. + 3. gateway.className == 'none' -- clients hit the model server's plain + Service directly; no Gateway, router, or HTTPRoute is deployed. #} {% if standalone.enabled is defined and not standalone.enabled and not (kustomize.enabled | default(false)) - and gateway.className | default('') != 'epponly' %} + and gateway.className | default('') not in ['epponly', 'none'] %} {% set _mode = (httpRoute.mode if httpRoute is defined and httpRoute.mode is defined else 'per-stack') %} {% if _mode == 'shared' %} {% set _owner_index = sharedInfraStackIndex | default(1) | int %} diff --git a/config/templates/jinja/10_helmfile-main.yaml.j2 b/config/templates/jinja/10_helmfile-main.yaml.j2 index 54d176dac..0b7b3a05d 100644 --- a/config/templates/jinja/10_helmfile-main.yaml.j2 +++ b/config/templates/jinja/10_helmfile-main.yaml.j2 @@ -20,13 +20,16 @@ helmDefaults: ghcr.io/llm-d/charts. The scenario layer uses the chart's `router.*` keys directly -- see 12_router-values.yaml.j2 for the renderer. #} {% set _epponly = (gateway.className | default('') == 'epponly') %} +{% set _direct = (gateway.className | default('') == 'none') %} {% set _gaie_chart = 'oci://ghcr.io/llm-d/charts/llm-d-router-standalone' if _epponly else 'oci://ghcr.io/llm-d/charts/llm-d-router-gateway' %} repositories: - name: llm-d-modelservice url: {{ helmRepositories.llmDModelservice.url }} +{% if not _direct %} - name: llm-d-infra url: {{ helmRepositories.llmDInfra.url }} +{% endif %} releases: {# `infra-llmdbench` (llm-d-infra chart -> Gateway resource + CRDs) is scenario- @@ -44,7 +47,7 @@ releases: `epponly` skips this release entirely - there is no Gateway resource. #} {% set _is_shared_infra_owner = (stackIndex is not defined or stackIndex | int == sharedInfraStackIndex | default(1) | int) %} {% set _is_first_stack = _is_shared_infra_owner %} -{% if _is_first_stack and not _epponly %} +{% if _is_first_stack and not _epponly and not _direct %} - name: infra-llmdbench namespace: {{ gateway.namespace }} chart: llm-d-infra/llm-d-infra @@ -62,16 +65,19 @@ releases: chart: llm-d-modelservice/llm-d-modelservice version: {{ chartVersions.llmDModelservice }} installed: true + {% if not _direct %} needs: -{% if _is_first_stack and not _epponly %} + {% if _is_first_stack and not _epponly and not _direct %} - {{ gateway.namespace }}/infra-llmdbench {% endif %} - {{ gateway.namespace }}/{{ model_id_label }}-router + {% endif %} values: - ms-values.yaml labels: kind: inference-stack +{% if not _direct %} - name: {{ model_id_label }}-router namespace: {{ gateway.namespace }} chart: {{ _gaie_chart }} @@ -89,3 +95,4 @@ releases: labels: kind: inference-stack {% endif %} +{% endif %} diff --git a/config/templates/jinja/11_infra.yaml.j2 b/config/templates/jinja/11_infra.yaml.j2 index 4739a6bce..e74b910d8 100644 --- a/config/templates/jinja/11_infra.yaml.j2 +++ b/config/templates/jinja/11_infra.yaml.j2 @@ -2,8 +2,9 @@ {% if standalone.enabled is defined and not standalone.enabled and not (kustomize.enabled | default(false)) %} {% set gw_class = gateway.className %} {# `epponly` renders no infra values: no Gateway is installed, the standalone - router chart provides HTTP service directly via the EPP Envoy sidecar. #} -{% if gw_class == 'epponly' %} + router chart provides HTTP service directly via the EPP Envoy sidecar. + `none` also renders no infra values because clients use a plain Service. #} +{% if gw_class in ['epponly', 'none'] %} {% elif gw_class == 'gke' %} {# ── GKE: managed gateway with destinationRule ── #} gateway: diff --git a/config/templates/jinja/13a_modelservice-direct-service.yaml.j2 b/config/templates/jinja/13a_modelservice-direct-service.yaml.j2 new file mode 100644 index 000000000..8006b0cc3 --- /dev/null +++ b/config/templates/jinja/13a_modelservice-direct-service.yaml.j2 @@ -0,0 +1,31 @@ +{# Modelservice direct baseline: expose decode vLLM without Gateway, EPP, + Envoy, or the routing proxy. RenderPlans forces routing.proxy.enabled=false + for this mode, so vLLM binds directly to routing.servicePort. #} +{% set decode_create = (decode.enabled if decode.enabled is defined else (decode.replicas | default(0) | int > 0)) %} +{% if standalone.enabled is defined and not standalone.enabled + and not (kustomize.enabled | default(false)) + and gateway.className | default('') == 'none' + and decode_create %} +apiVersion: v1 +kind: Service +metadata: + name: {{ model_id_label }}-direct + namespace: {{ namespace.name }} + labels: + app.kubernetes.io/managed-by: llm-d-benchmark + llm-d.ai/model: {{ model_id_label }} + llm-d.ai/role: decode + stood-up-from: llm-d-benchmark + stood-up-via: modelservice +spec: + type: ClusterIP + selector: + llm-d.ai/model: {{ model_id_label }} + llm-d.ai/role: decode + ports: + - name: http + appProtocol: http + port: {{ routing.servicePort }} + targetPort: {{ routing.servicePort }} + protocol: TCP +{% endif %} \ No newline at end of file diff --git a/config/templates/values/defaults.yaml b/config/templates/values/defaults.yaml index 20a00e182..bd99bc855 100644 --- a/config/templates/values/defaults.yaml +++ b/config/templates/values/defaults.yaml @@ -641,6 +641,8 @@ gateway: # Kubernetes Gateway is deployed, the EPP # runs with an Envoy sidecar that serves # HTTP directly (single-stack only). + # none - plain ClusterIP Service to decode vLLM; + # no Gateway, EPP, Envoy, or routing proxy. className: istio providerNamespace: istio-system # Gateway version is derived from chartVersions.istiod at plan time diff --git a/docs/standup.md b/docs/standup.md index f80908cc8..17fb159a2 100644 --- a/docs/standup.md +++ b/docs/standup.md @@ -142,18 +142,22 @@ The scenario parameters can be roughly categorized in four groups: | Variable | Meaning | Note | | -------------------------------------------- | ---------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | -| LLMDBENCH_VLLM_MODELSERVICE_GATEWAY_CLASS_NAME | Gateway implementation used for the inference gateway | Default=`istio`. Supported: `istio`, `agentgateway`, `gke`, `data-science-gateway-class`, `epponly`. | +| LLMDBENCH_VLLM_MODELSERVICE_GATEWAY_CLASS_NAME | Gateway implementation used for the inference gateway | Default=`istio`. Supported: `none`, `istio`, `agentgateway`, `gke`, `data-science-gateway-class`, `epponly`. | Gateway class options (set via `gateway.className` in the scenario YAML): | `className` | What it deploys | Use when | |------------------------------|------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------| +| `none` | ModelService decode pods plus a plain ClusterIP Service; **no** Gateway, HTTPRoute, EPP, Envoy, or routing proxy | Measuring direct model-server performance and routing overhead | | `istio` (default) | istio-base + istiod control plane, a Gateway + HTTPRoute, the `llm-d-router-gateway-dev` chart | Default; most flexible / production deployments | | `agentgateway` | agentgateway-crds + agentgateway controller, a Gateway + HTTPRoute, the `llm-d-router-gateway-dev` chart | Want agentgateway's data plane instead of Envoy/Istio | | `gke` | Uses GKE-managed Gateway controller; same `llm-d-router-gateway-dev` chart | Running on GKE | | `data-science-gateway-class` | OpenDataHub / OpenShift AI managed Gateway | Running on OpenShift AI | | `epponly` | **No** Kubernetes Gateway, **no** HTTPRoute, the `llm-d-router-standalone-dev` chart (EPP with an Envoy sidecar serving HTTP) | You want llm-d's standalone router topology without any gateway | +`none` is a baseline lane, not a routing topology. It requires at least one +decode replica and does not support P/D disaggregation. + ### Overriding `gateway.className` from the CLI Every subcommand that renders templates (`plan`, `standup`, `experiment`, diff --git a/llmdbenchmark/interface/experiment.py b/llmdbenchmark/interface/experiment.py index 121cc3ec7..a749c0e26 100644 --- a/llmdbenchmark/interface/experiment.py +++ b/llmdbenchmark/interface/experiment.py @@ -47,7 +47,8 @@ def add_subcommands( default=env("LLMDBENCH_GATEWAY_CLASS"), help=( "Override the scenario's gateway.className. Supported values: " - "epponly, istio, agentgateway, gke, data-science-gateway-class. " + "none, epponly, istio, agentgateway, gke, " + "data-science-gateway-class. " "Only takes effect on the modelservice deploy path." ), ) diff --git a/llmdbenchmark/interface/plan.py b/llmdbenchmark/interface/plan.py index 76e2ea21c..bade4674a 100644 --- a/llmdbenchmark/interface/plan.py +++ b/llmdbenchmark/interface/plan.py @@ -49,7 +49,8 @@ def add_subcommands( default=env("LLMDBENCH_GATEWAY_CLASS"), help=( "Override the scenario's gateway.className. Supported values: " - "epponly, istio, agentgateway, gke, data-science-gateway-class. " + "none, epponly, istio, agentgateway, gke, " + "data-science-gateway-class. " "Only takes effect on the modelservice deploy path." ), ) diff --git a/llmdbenchmark/interface/run.py b/llmdbenchmark/interface/run.py index f63be0d15..5285c688e 100644 --- a/llmdbenchmark/interface/run.py +++ b/llmdbenchmark/interface/run.py @@ -44,7 +44,8 @@ def add_subcommands( help=( "Override the scenario's gateway.className when the run phase " "re-renders templates for setup overrides. Supported values: " - "epponly, istio, agentgateway, gke, data-science-gateway-class." + "none, epponly, istio, agentgateway, gke, " + "data-science-gateway-class." ), ) run_parser.add_argument( diff --git a/llmdbenchmark/interface/standup.py b/llmdbenchmark/interface/standup.py index 1fb83d525..fda19b1b0 100644 --- a/llmdbenchmark/interface/standup.py +++ b/llmdbenchmark/interface/standup.py @@ -52,7 +52,8 @@ def add_subcommands( default=env("LLMDBENCH_GATEWAY_CLASS"), help=( "Override the scenario's gateway.className. Supported values: " - "epponly, istio, agentgateway, gke, data-science-gateway-class. " + "none, epponly, istio, agentgateway, gke, " + "data-science-gateway-class. " "Only takes effect on the modelservice deploy path -- ignored " "by kustomize/standalone/fma." ), diff --git a/llmdbenchmark/parser/README.md b/llmdbenchmark/parser/README.md index 6720e5fd1..60f05b47c 100644 --- a/llmdbenchmark/parser/README.md +++ b/llmdbenchmark/parser/README.md @@ -95,6 +95,11 @@ chain runs, then pops the nested copy so `config.yaml` has a single home per section. Templates, resolvers and standup steps always read the **top-level** keys, so the two spellings render identically -- pick one per section. +Set `gateway.className: none` to skip Gateway and router resources and expose +decode vLLM directly through a plain Service. The renderer also disables the +modelservice routing proxy so the resulting lane measures the model server +without Gateway, EPP, or Envoy overhead. + The hoist runs **before** setup overrides so the precedence stays `defaults < scenario < treatment/CLI`: DoE experiment treatments and CLI overrides target the **top-level** dotted path (e.g. diff --git a/llmdbenchmark/parser/render_plans.py b/llmdbenchmark/parser/render_plans.py index 3d31f167b..553b77d96 100644 --- a/llmdbenchmark/parser/render_plans.py +++ b/llmdbenchmark/parser/render_plans.py @@ -667,6 +667,7 @@ def _resolve_deploy_method(self, values: dict) -> dict: # at render time so a typo doesn't silently produce a broken Gateway / # InferencePool chart configuration. _SUPPORTED_GATEWAY_CLASSES: tuple[str, ...] = ( + "none", "epponly", "istio", "agentgateway", @@ -791,6 +792,71 @@ def _validate_epponly_constraints( return errors + @staticmethod + def _normalize_direct_service_mode(values: dict) -> dict: + """Make ``gateway.className=none`` a true direct-vLLM baseline. + + The modelservice chart enables its per-pod routing proxy by default. + A plain Service pointing at that port would still put the proxy in the + request path, defeating the baseline. Disable it before templates are + rendered so the chart makes vLLM bind directly to ``servicePort``. + """ + gateway_class = (values.get("gateway") or {}).get("className", "") + modelservice_enabled = (values.get("modelservice") or {}).get("enabled", True) + if gateway_class == "none" and modelservice_enabled: + routing = values.setdefault("routing", {}) + routing.setdefault("proxy", {})["enabled"] = False + + # Accelerator-neutral guides may provide a custom command that + # binds decode vLLM to the proxy backend port. Direct mode has no + # proxy, so make the custom command follow the chart's normal + # proxy-disabled behavior and listen on the Service port instead. + decode_vllm = values.setdefault("decode", {}).get("vllm") or {} + custom_command = decode_vllm.get("customCommand") + if isinstance(custom_command, str): + decode_vllm["customCommand"] = custom_command.replace( + "$VLLM_METRICS_PORT", + "$VLLM_INFERENCE_PORT", + ) + values["decode"]["vllm"] = decode_vllm + return values + + @staticmethod + def _validate_direct_service_constraints( + values: dict, + stack_name: str, + ) -> list[str]: + """Reject configurations that require routing in direct mode.""" + gateway_class = (values.get("gateway") or {}).get("className", "") + modelservice_enabled = (values.get("modelservice") or {}).get("enabled", True) + if gateway_class != "none" or not modelservice_enabled: + return [] + + errors: list[str] = [] + http_route_mode = (values.get("httpRoute") or {}).get("mode") + if http_route_mode == "shared": + errors.append( + f"[{stack_name}] gateway.className=none cannot be used with " + "httpRoute.mode=shared (direct mode deploys no Gateway or " + "HTTPRoute)." + ) + + prefill = values.get("prefill") or {} + if prefill.get("enabled") and int(prefill.get("replicas", 0) or 0) > 0: + errors.append( + f"[{stack_name}] gateway.className=none cannot be used with " + "prefill replicas (direct mode bypasses P/D routing)." + ) + + decode = values.get("decode") or {} + decode_enabled = decode.get("enabled", int(decode.get("replicas", 0) or 0) > 0) + if not decode_enabled or int(decode.get("replicas", 0) or 0) < 1: + errors.append( + f"[{stack_name}] gateway.className=none requires at least one " + "decode replica to back the direct Service." + ) + return errors + def _log_image_overrides(self, values: dict) -> None: """Log images that have been explicitly set (not 'auto'). @@ -1596,6 +1662,7 @@ def _process_stack( merged_values, total_stacks=total_stacks ) merged_values = self._resolve_inference_pool_host(merged_values) + merged_values = self._normalize_direct_service_mode(merged_values) merged_values = self._normalize_router_block(merged_values) merged_values = self._substitute_config_variables(merged_values) @@ -1614,6 +1681,14 @@ def _process_stack( self.logger.log_error(msg) stack_errors.render_errors.append(msg) + direct_service_errors = self._validate_direct_service_constraints( + merged_values, + stack_name=stack_name, + ) + for msg in direct_service_errors: + self.logger.log_error(msg) + stack_errors.render_errors.append(msg) + kustomize_errors = self._validate_kustomize_patches( merged_values, stack_name=stack_name, diff --git a/llmdbenchmark/run/steps/step_03_detect_endpoint.py b/llmdbenchmark/run/steps/step_03_detect_endpoint.py index 62f9d722e..3e5056d55 100644 --- a/llmdbenchmark/run/steps/step_03_detect_endpoint.py +++ b/llmdbenchmark/run/steps/step_03_detect_endpoint.py @@ -9,6 +9,7 @@ find_fma_endpoint, find_gateway_endpoint, find_epponly_endpoint, + find_direct_modelservice_endpoint, find_custom_endpoint, find_kustomize_endpoint, discover_hf_token_secret, @@ -146,6 +147,23 @@ def execute( namespace, model_id_label, ) + elif gateway_class == "none": + model_id_label = plan_config.get("model_id_label", "") + direct_port = str( + self._resolve( + plan_config, + "routing.servicePort", + default="8000", + ) + ) + service_ip, service_name, gateway_port = ( + find_direct_modelservice_endpoint( + cmd, + namespace, + model_id_label, + direct_port, + ) + ) else: service_ip, service_name, gateway_port = find_gateway_endpoint( cmd, namespace, release diff --git a/llmdbenchmark/smoketests/base.py b/llmdbenchmark/smoketests/base.py index 296d7a4fb..aea53c785 100644 --- a/llmdbenchmark/smoketests/base.py +++ b/llmdbenchmark/smoketests/base.py @@ -15,6 +15,7 @@ _rand_suffix, compute_gateway_path_prefix, find_custom_endpoint, + find_direct_modelservice_endpoint, find_epponly_endpoint, find_gateway_endpoint, find_kustomize_endpoint, @@ -144,6 +145,16 @@ def discover_endpoint( namespace, model_id_label, ) + elif gateway_class == "none": + direct_port = str( + _nested_get(plan_config, "routing", "servicePort") or "8000" + ) + service_ip, _, gateway_port = find_direct_modelservice_endpoint( + cmd, + namespace, + model_id_label, + direct_port, + ) else: service_ip, _, gateway_port = find_gateway_endpoint(cmd, namespace, release) diff --git a/llmdbenchmark/standup/steps/step_02_admin_prerequisites.py b/llmdbenchmark/standup/steps/step_02_admin_prerequisites.py index 0203735d4..c77574418 100644 --- a/llmdbenchmark/standup/steps/step_02_admin_prerequisites.py +++ b/llmdbenchmark/standup/steps/step_02_admin_prerequisites.py @@ -201,27 +201,35 @@ def execute( deploy_methods = context.deployed_methods or [] modelservice_active = "modelservice" in deploy_methods + gateway_class = (plan_config.get("gateway") or {}).get("className", "") + direct_service_mode = modelservice_active and gateway_class == "none" if modelservice_active: - self._install_gateway_api_crds( - cmd, - plan_config, - errors, - existing_crds, - ) - self._install_gateway_api_extension_crds( - cmd, - plan_config, - errors, - existing_crds, - ) - self._install_gateway_provider( - cmd, - context, - plan_config, - errors, - existing_crds, - ) + if direct_service_mode: + context.logger.log_info( + "✅ gateway.className=none -- skipping Gateway API, " + "inference extension, and gateway provider prerequisites" + ) + else: + self._install_gateway_api_crds( + cmd, + plan_config, + errors, + existing_crds, + ) + self._install_gateway_api_extension_crds( + cmd, + plan_config, + errors, + existing_crds, + ) + self._install_gateway_provider( + cmd, + context, + plan_config, + errors, + existing_crds, + ) self._install_lws_if_needed( cmd, plan_config, diff --git a/llmdbenchmark/standup/steps/step_08_deploy_router.py b/llmdbenchmark/standup/steps/step_08_deploy_router.py index df424b6ab..7dd4218ac 100644 --- a/llmdbenchmark/standup/steps/step_08_deploy_router.py +++ b/llmdbenchmark/standup/steps/step_08_deploy_router.py @@ -39,6 +39,17 @@ def execute( errors = [] cmd = context.require_cmd() + plan_config = self._load_stack_config(stack_path) + gateway_class = self._require_config(plan_config, "gateway", "className") + if gateway_class == "none": + return StepResult( + step_number=self.number, + step_name=self.name, + success=True, + message="Direct Service mode does not deploy a router", + stack_name=stack_path.name, + ) + router_values = self._find_yaml(stack_path, "12_router-values") if not router_values: @@ -50,7 +61,6 @@ def execute( stack_name=stack_path.name, ) - plan_config = self._load_stack_config(stack_path) release = self._require_config(plan_config, "release") namespace = context.require_namespace() stack_name = stack_path.name @@ -107,7 +117,6 @@ def execute( # Wait for gateway pod only (not EPP -- it stays NOT_SERVING until step 09) if not errors and not context.dry_run: - gateway_class = self._require_config(plan_config, "gateway", "className") if gateway_class == "epponly": # No Gateway resource is deployed in epponly mode; the EPP # pod itself is the data-plane proxy and is waited on by diff --git a/llmdbenchmark/standup/steps/step_09_deploy_modelservice.py b/llmdbenchmark/standup/steps/step_09_deploy_modelservice.py index 703f475ff..5bef7e942 100644 --- a/llmdbenchmark/standup/steps/step_09_deploy_modelservice.py +++ b/llmdbenchmark/standup/steps/step_09_deploy_modelservice.py @@ -112,6 +112,21 @@ def execute( # pylint: disable=too-many-branches,too-many-locals,too-many-state if not result.success: errors.append(f"Failed to deploy modelservice: {result.stderr}") + gateway_class = self._require_config(plan_config, "gateway", "className") + direct_service_mode = gateway_class == "none" + if direct_service_mode: + direct_service_yaml = self._find_yaml( + stack_path, "13a_modelservice-direct-service" + ) + if not direct_service_yaml or not self._has_yaml_content( + direct_service_yaml + ): + errors.append("Direct modelservice Service manifest was not rendered") + else: + result = cmd.kube("apply", "-f", str(direct_service_yaml)) + if not result.success: + errors.append(f"Failed to apply direct Service: {result.stderr}") + httproute_yaml = self._find_yaml(stack_path, "08_httproute") if httproute_yaml and self._has_yaml_content(httproute_yaml): result = cmd.kube("apply", "-f", str(httproute_yaml)) @@ -225,39 +240,43 @@ def execute( # pylint: disable=too-many-branches,too-many-locals,too-many-state # # Probe both candidate labels once each so we discover which # the chart actually applied, then wait on that one. - release_epp = f"{model_id_label}-router-epp" - chosen_label = f"llm-d-router-gateway={release_epp}" # default - for candidate_key in ("llm-d-router-gateway", "llm-d-router-standalone"): - probe_label = f"{candidate_key}={release_epp}" - probe = cmd.kube( - "get", - "pods", - "-l", - probe_label, - "--namespace", - namespace, - "-o", - "jsonpath={.items[*].metadata.name}", - check=False, - ) - if probe.success and probe.stdout.strip(): - chosen_label = probe_label - break - - pool_wait = cmd.wait_for_pods( - label=chosen_label, - namespace=namespace, - timeout=timeout, - poll_interval=10, - description="inference pool", - ) - if not pool_wait.success: - stderr_lower = pool_wait.stderr.lower() - if ( - "no matching resources found" not in stderr_lower - and "no pods found" not in stderr_lower + if not direct_service_mode: + release_epp = f"{model_id_label}-router-epp" + chosen_label = f"llm-d-router-gateway={release_epp}" # default + for candidate_key in ( + "llm-d-router-gateway", + "llm-d-router-standalone", ): - errors.append(f"Inference pool not ready: {pool_wait.stderr}") + probe_label = f"{candidate_key}={release_epp}" + probe = cmd.kube( + "get", + "pods", + "-l", + probe_label, + "--namespace", + namespace, + "-o", + "jsonpath={.items[*].metadata.name}", + check=False, + ) + if probe.success and probe.stdout.strip(): + chosen_label = probe_label + break + + pool_wait = cmd.wait_for_pods( + label=chosen_label, + namespace=namespace, + timeout=timeout, + poll_interval=10, + description="inference pool", + ) + if not pool_wait.success: + stderr_lower = pool_wait.stderr.lower() + if ( + "no matching resources found" not in stderr_lower + and "no pods found" not in stderr_lower + ): + errors.append(f"Inference pool not ready: {pool_wait.stderr}") if not errors and not context.dry_run: self._collect_logs(cmd, context, namespace) @@ -297,9 +316,9 @@ def execute( # pylint: disable=too-many-branches,too-many-locals,too-many-state "PodMonitor skipped (template not rendered for this configuration)" ) - gateway_class = self._require_config(plan_config, "gateway", "className") - - if gateway_class in ("kgateway", "agentgateway"): + if direct_service_mode: + service_name = f"{model_id_label}-direct" + elif gateway_class in ("kgateway", "agentgateway"): service_name = f"infra-{release}-inference-gateway" else: # Covers istio / gke / data-science-gateway-class / epponly. @@ -312,7 +331,7 @@ def execute( # pylint: disable=too-many-branches,too-many-locals,too-many-state # In epponly mode there is no Gateway resource to label, and the # router chart's auto-generated route is to the EPP gRPC port which # we'd otherwise rewrite to point at the Gateway. Skip both. - if gateway_class != "epponly": + if gateway_class not in ("epponly", "none"): username = context.username or "unknown" cmd.kube( "label", diff --git a/llmdbenchmark/standup/steps/step_10_smoketest.py b/llmdbenchmark/standup/steps/step_10_smoketest.py index 3bdab2f2c..44394b85d 100644 --- a/llmdbenchmark/standup/steps/step_10_smoketest.py +++ b/llmdbenchmark/standup/steps/step_10_smoketest.py @@ -14,6 +14,7 @@ find_standalone_endpoint, find_gateway_endpoint, find_epponly_endpoint, + find_direct_modelservice_endpoint, test_model_serving, ) @@ -64,6 +65,7 @@ def execute( # pylint: disable=too-many-branches,too-many-locals,too-many-state gateway_class = plan_config.get("gateway", {}).get("className", "") is_epponly = gateway_class == "epponly" + is_direct = gateway_class == "none" model_id_label = plan_config.get("model_id_label", "") if is_standalone: @@ -76,6 +78,13 @@ def execute( # pylint: disable=too-many-branches,too-many-locals,too-many-state namespace, model_id_label, ) + elif is_direct: + service_ip, _, gateway_port = find_direct_modelservice_endpoint( + cmd, + namespace, + model_id_label, + str(plan_config.get("routing", {}).get("servicePort", "8000")), + ) else: service_ip, _, gateway_port = find_gateway_endpoint(cmd, namespace, release) diff --git a/llmdbenchmark/standup/steps/step_11_inference_test.py b/llmdbenchmark/standup/steps/step_11_inference_test.py index b171db6e7..7c2860b3c 100644 --- a/llmdbenchmark/standup/steps/step_11_inference_test.py +++ b/llmdbenchmark/standup/steps/step_11_inference_test.py @@ -27,6 +27,7 @@ _ephemeral_label_args, find_standalone_endpoint, find_gateway_endpoint, + find_direct_modelservice_endpoint, find_epponly_endpoint, ) @@ -109,6 +110,13 @@ def execute( namespace, model_id_label, ) + elif gateway_class == "none": + service_ip, _, gateway_port = find_direct_modelservice_endpoint( + cmd, + namespace, + model_id_label, + str(plan_config.get("routing", {}).get("servicePort", "8000")), + ) else: service_ip, _, gateway_port = find_gateway_endpoint(cmd, namespace, release) diff --git a/llmdbenchmark/teardown/steps/step_03_delete_resources.py b/llmdbenchmark/teardown/steps/step_03_delete_resources.py index 01f468b23..81eaa949f 100644 --- a/llmdbenchmark/teardown/steps/step_03_delete_resources.py +++ b/llmdbenchmark/teardown/steps/step_03_delete_resources.py @@ -50,6 +50,8 @@ # of Helm release matching -- "endpoint-picker" above only matches the # container image repo, not the Deployment's own object name. "-epp", + # Plain-Service baseline created for gateway.className=none. + "-direct", ] DEEP_RESOURCE_KINDS = [ diff --git a/llmdbenchmark/utilities/endpoint.py b/llmdbenchmark/utilities/endpoint.py index 987d051e8..f3fc5dd26 100644 --- a/llmdbenchmark/utilities/endpoint.py +++ b/llmdbenchmark/utilities/endpoint.py @@ -250,6 +250,54 @@ def find_epponly_endpoint( return ip, svc_name, port +def find_direct_modelservice_endpoint( + cmd: CommandExecutor, + namespace: str, + model_id_label: str, + default_port: str = "8000", +) -> tuple[str | None, str | None, str]: + """Find the plain Service used by ``gateway.className=none``. + + The Service selects modelservice decode pods and targets vLLM directly; + no Gateway, EPP, Envoy, or routing proxy is in the request path. + """ + if not model_id_label: + return None, None, default_port + + svc_name = f"{model_id_label}-direct" + result = cmd.kube( + "get", + "service", + svc_name, + "--namespace", + namespace, + "-o", + "json", + check=False, + ) + if not (result.success and result.stdout.strip()): + return None, svc_name, default_port + + try: + service = json.loads(result.stdout) + except (json.JSONDecodeError, TypeError): + return None, svc_name, default_port + + spec = service.get("spec", {}) or {} + service_ip = spec.get("clusterIP") or None + ports = spec.get("ports") or [] + service_port = default_port + for port in ports: + if port.get("name") == "http": + service_port = str(port.get("port", default_port)) + break + else: + if ports: + service_port = str(ports[0].get("port", default_port)) + + return service_ip, svc_name, service_port + + def find_gateway_endpoint( cmd: CommandExecutor, namespace: str, release: str ) -> tuple[str | None, str | None, str]: diff --git a/tests/test_direct_service_mode.py b/tests/test_direct_service_mode.py new file mode 100644 index 000000000..d4741c349 --- /dev/null +++ b/tests/test_direct_service_mode.py @@ -0,0 +1,194 @@ +"""Tests for the gateway.className=none direct modelservice baseline.""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +import yaml + +from llmdbenchmark.parser.cluster_resource_resolver import ClusterResourceResolver +from llmdbenchmark.parser.render_plans import RenderPlans +from llmdbenchmark.parser.version_resolver import VersionResolver +from llmdbenchmark.standup.steps.step_08_deploy_router import DeployRouterStep +from llmdbenchmark.utilities.endpoint import find_direct_modelservice_endpoint + +_REPO = Path(__file__).resolve().parents[1] + + +def _renderer() -> RenderPlans: + renderer = RenderPlans.__new__(RenderPlans) + renderer.logger = MagicMock() + return renderer + + +def test_direct_mode_disables_per_pod_routing_proxy() -> None: + values = { + "gateway": {"className": "none"}, + "modelservice": {"enabled": True}, + "routing": {"proxy": {"enabled": True}}, + } + + result = RenderPlans._normalize_direct_service_mode(values) + + assert result["routing"]["proxy"]["enabled"] is False + + +def test_direct_mode_retargets_custom_decode_command_to_service_port() -> None: + values = { + "gateway": {"className": "none"}, + "modelservice": {"enabled": True}, + "decode": { + "vllm": {"customCommand": "vllm serve model --port $VLLM_METRICS_PORT"} + }, + } + + result = RenderPlans._normalize_direct_service_mode(values) + + command = result["decode"]["vllm"]["customCommand"] + assert command == "vllm serve model --port $VLLM_INFERENCE_PORT" + + +def test_direct_mode_rejects_prefill_routing() -> None: + values = { + "gateway": {"className": "none"}, + "modelservice": {"enabled": True}, + "decode": {"replicas": 1}, + "prefill": {"enabled": True, "replicas": 1}, + } + + errors = RenderPlans._validate_direct_service_constraints(values, "stack") + + assert any("bypasses P/D routing" in error for error in errors) + + +def test_direct_mode_allows_multiple_independent_stacks() -> None: + values = { + "gateway": {"className": "none"}, + "modelservice": {"enabled": True}, + "decode": {"replicas": 1}, + "prefill": {"enabled": False, "replicas": 0}, + } + + assert RenderPlans._validate_direct_service_constraints(values, "stack") == [] + + +def test_none_is_supported_modelservice_gateway_class() -> None: + renderer = _renderer() + renderer.cli_gateway_class = "none" + values = { + "gateway": {"className": "epponly"}, + "modelservice": {"enabled": True}, + } + + result = renderer._resolve_gateway_class(values) + + assert result["gateway"]["className"] == "none" + + +def test_direct_endpoint_prefers_named_http_port() -> None: + service = { + "spec": { + "clusterIP": "10.0.0.42", + "ports": [ + {"name": "metrics", "port": 9090}, + {"name": "http", "port": 8000}, + ], + } + } + cmd = MagicMock() + cmd.kube.return_value = SimpleNamespace( + success=True, + stdout=json.dumps(service), + ) + + endpoint = find_direct_modelservice_endpoint(cmd, "bench", "model-id") + + assert endpoint == ("10.0.0.42", "model-id-direct", "8000") + cmd.kube.assert_called_once_with( + "get", + "service", + "model-id-direct", + "--namespace", + "bench", + "-o", + "json", + check=False, + ) + + +def test_direct_mode_skips_router_deployment(tmp_path: Path) -> None: + (tmp_path / "config.yaml").write_text( + yaml.safe_dump({"gateway": {"className": "none"}}), + encoding="utf-8", + ) + cmd = MagicMock() + context = MagicMock() + context.require_cmd.return_value = cmd + + result = DeployRouterStep().execute(context, tmp_path) + + assert result.success + assert result.message == "Direct Service mode does not deploy a router" + cmd.helmfile.assert_not_called() + cmd.wait_for_pods.assert_not_called() + + +def test_gpu_example_renders_plain_service_without_router(tmp_path: Path) -> None: + logger = MagicMock() + result = RenderPlans( + template_dir=_REPO / "config/templates/jinja", + defaults_file=_REPO / "config/templates/values/defaults.yaml", + scenarios_file=_REPO / "config/scenarios/examples/gpu.yaml", + output_dir=tmp_path, + logger=logger, + version_resolver=VersionResolver(logger=logger, dry_run=True), + cluster_resource_resolver=ClusterResourceResolver(logger=logger, dry_run=True), + cli_gateway_class="none", + ).eval() + + assert not result.has_errors, result.to_dict() + stack_dirs = [path.parent for path in tmp_path.rglob("config.yaml")] + assert len(stack_dirs) == 1 + stack_dir = stack_dirs[0] + + config = yaml.safe_load((stack_dir / "config.yaml").read_text(encoding="utf-8")) + assert config["gateway"]["className"] == "none" + assert config["routing"]["proxy"]["enabled"] is False + + helmfile = yaml.safe_load_all( + (stack_dir / "10_helmfile-main.yaml").read_text(encoding="utf-8") + ) + releases = [ + release + for document in helmfile + if document + for release in document.get("releases", []) + ] + assert [release["name"] for release in releases] == [ + f"{config['model_id_label']}-ms" + ] + + direct_service = yaml.safe_load( + (stack_dir / "13a_modelservice-direct-service.yaml").read_text(encoding="utf-8") + ) + assert direct_service["kind"] == "Service" + assert direct_service["metadata"]["name"] == f"{config['model_id_label']}-direct" + assert direct_service["spec"]["selector"] == { + "llm-d.ai/model": config["model_id_label"], + "llm-d.ai/role": "decode", + } + assert direct_service["spec"]["ports"][0]["targetPort"] == 8000 + + assert not (stack_dir / "08_httproute.yaml").read_text(encoding="utf-8").strip() + assert not (stack_dir / "11_infra.yaml").read_text(encoding="utf-8").strip() + + modelservice_values = yaml.safe_load( + (stack_dir / "13_ms-values.yaml").read_text(encoding="utf-8") + ) + assert modelservice_values["routing"]["proxy"]["enabled"] is False + command = modelservice_values["decode"]["containers"][0]["args"][0] + assert "--port $VLLM_INFERENCE_PORT" in command + assert "$VLLM_METRICS_PORT" not in command