From 8c1e4fae50722a171603950851a0b21d074b971a Mon Sep 17 00:00:00 2001 From: "Rohan R. Arora" Date: Thu, 20 Jun 2024 14:29:25 -0500 Subject: [PATCH 1/5] bump: adding Prometheus service monitor-based scrapping --- docs/SETUP.md | 24 +++++++++++++++++++ fmperf/Cluster.py | 37 ++++++++++++++++++++--------- fmperf/ModelSpecs.py | 45 ++++++++++++++++++++++++++++++++---- fmperf/utils/Benchmarking.py | 3 ++- 4 files changed, 92 insertions(+), 17 deletions(-) diff --git a/docs/SETUP.md b/docs/SETUP.md index 8aa1d53..a8fa59c 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -162,3 +162,27 @@ nvidia-operator-validator-ztpls 1/1 Runnin ``` The cluster is now ready to run the benchmark. As a first try, run the examples/example_vllm.py script. + + +## Prometheus (Optional) + +```shell +# Run +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +``` + +# Run (to enable collection of TGIS and vLLM metrics) +```shell +helm install prometheus-community/kube-prometheus-stack --create-namespace --namespace monitoring --generate-name +``` + +INSTEAD (/OR) + +```shell +# To also enable collection of DCGM metrics in addition to TGIS and vLLM metrics, follow steps at to amend /tmp/kube-prometheus-stack.values +``` + +```shell +# Run +helm install prometheus-community/kube-prometheus-stack --create-namespace --namespace monitoring --generate-name --values /tmp/kube-prometheus-stack.values +``` diff --git a/fmperf/Cluster.py b/fmperf/Cluster.py index c634058..2cc9784 100644 --- a/fmperf/Cluster.py +++ b/fmperf/Cluster.py @@ -47,6 +47,7 @@ def deploy_model( self, model: ModelSpec, id: str = "", + prometheus_enabled: bool = False, ): creating = Creating(self.apigetter, self.logger, ignore_exists=True) vars = model.get_vars() @@ -110,10 +111,6 @@ def deploy_model( "strategy": {"rollingUpdate": {"maxSurge": 1}}, "template": { "metadata": { - "annotations": { - "prometheus.io/port": "3000", - "prometheus.io/scrape": "true", - }, "labels": { "app": name, }, @@ -167,18 +164,12 @@ def deploy_model( "namespace": self.namespace, }, "spec": { - "ports": [{"name": "grpc", "port": 8033, "targetPort": "grpc"}], + "ports": model.get_service_ports(), "selector": {"app": name}, "type": "ClusterIP", }, } - # Change the service manifest if vllm server is to be deployed - if type(model) is vLLMModelSpec: - manifest["spec"]["ports"] = [ - {"name": "http", "port": 8000, "targetPort": "http"} - ] - # create service creating.create_namespaced_service( name=name, namespace=self.namespace, payload=manifest @@ -194,6 +185,30 @@ def deploy_model( # get server url url = model.get_url() % (out.spec.cluster_ip) + if prometheus_enabled: + manifest = { + "apiVersion": "monitoring.coreos.com/v1", + "kind": "ServiceMonitor", + "metadata": { + "name": name, + "namespace": self.namespace, + }, + "spec": { + "selector": {"matchLabels": {"app": name}}, + "endpoints": self.get_service_monitor_endpoints() + }, + } + + # create service monitor + creating.create_namespaced_custom_object( + name, + "monitoring.coreos.com/v1", + self.namespace, + "servicemonitors", + manifest, + wait_until=None, + ) + return DeployedModel( spec=model, name=name, diff --git a/fmperf/ModelSpecs.py b/fmperf/ModelSpecs.py index 7977828..f286b68 100644 --- a/fmperf/ModelSpecs.py +++ b/fmperf/ModelSpecs.py @@ -215,7 +215,8 @@ def get_vars(self): "name": "MAX_CONCURRENT_REQUESTS", "value": str(self.max_concurrent_requests), }, - {"name": "DTYPE_STR", "value": self.dtype_str}, + { "name": "DTYPE_STR", + "value": self.dtype_str}, { "name": "DEPLOYMENT_FRAMEWORK", "value": self.deployment_framework, @@ -244,7 +245,10 @@ def get_vars(self): "name": "OUTPUT_SPECIAL_TOKENS", "value": str(self.output_special_tokens).lower(), }, - {"name": "PAGED_ATTENTION", "value": str(self.paged_attention).lower()}, + { + "name": "PAGED_ATTENTION", + "value": str(self.paged_attention).lower(), + }, { "name": "PORT", "value": str(self.port), @@ -353,10 +357,27 @@ def get_liveness_probe(self): } def get_ports(self): - # Specifies the ports for service + # Specifies the ports for the container return [ - {"containerPort": 3000, "name": "http"}, {"containerPort": 8033, "name": "grpc"}, + {"containerPort": 3000, "name": "http"}, + ] + + def get_service_ports(self): + # Specifies the ports for service + return [ + {"name": "grpc", "port": 8033, "targetPort": "grpc"}, + {"name": "http", "port": 3000, "targetPort": "http"}, + ] + + def get_service_monitor_endpoints(self): + # Specifies the end-points for the service monitor + return [ + { + "port": "http", + "path": "/metrics", + "interval": "1s" + }, ] def get_readiness_probe(self): @@ -552,9 +573,23 @@ def get_liveness_probe(self): } def get_ports(self): - # Specifies the ports for service + # Specifies the ports for the container return [{"containerPort": 8000, "name": "http"}] + def get_service_ports(self): + # Specifies the ports for service + return [{"name": "http", "port": 8000, "targetPort": "http"}] + + def get_service_monitor_endpoints(self): + # Specifies the ports for the service monitor + return [ + { + "port": "http", + "path": "/metrics", + "interval": "1s" + } + ] + def get_readiness_probe(self): # Runs a healthcheck on a running service return { diff --git a/fmperf/utils/Benchmarking.py b/fmperf/utils/Benchmarking.py index 53ba867..da514b1 100644 --- a/fmperf/utils/Benchmarking.py +++ b/fmperf/utils/Benchmarking.py @@ -14,13 +14,14 @@ def run_benchmark( number_users: int, duration: str, id: str = "", + prometheus_enabled: bool = False, ) -> None: if isinstance(model_spec, ModelSpec): model_spec = [model_spec] for spec in model_spec: # create the inference server - model = cluster.deploy_model(spec, id) + model = cluster.deploy_model(spec, id, prometheus_enabled=prometheus_enabled) # create the jobs for requests workload = cluster.generate_workload(model, workload_spec, id=id) for rep in range(repetition): From 781aa98b914d482d0ea56be3f1ee4dffa0b8fe94 Mon Sep 17 00:00:00 2001 From: "Rohan R. Arora" Date: Mon, 24 Jun 2024 08:30:52 -0500 Subject: [PATCH 2/5] fix: post e2e run fixes --- examples/example_vllm.py | 1 + fmperf/Cluster.py | 2 +- fmperf/ModelSpecs.py | 7 ++++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/example_vllm.py b/examples/example_vllm.py index 0994cd3..a5ad882 100644 --- a/examples/example_vllm.py +++ b/examples/example_vllm.py @@ -93,4 +93,5 @@ def initialize_kubernetes(location): repetition=REPETITION, number_users=NUM_USERS, duration=DURATION, + prometheus_enabled=True ) diff --git a/fmperf/Cluster.py b/fmperf/Cluster.py index 2cc9784..ac71e5d 100644 --- a/fmperf/Cluster.py +++ b/fmperf/Cluster.py @@ -195,7 +195,7 @@ def deploy_model( }, "spec": { "selector": {"matchLabels": {"app": name}}, - "endpoints": self.get_service_monitor_endpoints() + "endpoints": model.get_service_monitor_endpoints() }, } diff --git a/fmperf/ModelSpecs.py b/fmperf/ModelSpecs.py index f286b68..dac7024 100644 --- a/fmperf/ModelSpecs.py +++ b/fmperf/ModelSpecs.py @@ -216,7 +216,8 @@ def get_vars(self): "value": str(self.max_concurrent_requests), }, { "name": "DTYPE_STR", - "value": self.dtype_str}, + "value": self.dtype_str + }, { "name": "DEPLOYMENT_FRAMEWORK", "value": self.deployment_framework, @@ -371,7 +372,7 @@ def get_service_ports(self): ] def get_service_monitor_endpoints(self): - # Specifies the end-points for the service monitor + # Specifies the endpoints for the service monitor return [ { "port": "http", @@ -581,7 +582,7 @@ def get_service_ports(self): return [{"name": "http", "port": 8000, "targetPort": "http"}] def get_service_monitor_endpoints(self): - # Specifies the ports for the service monitor + # Specifies the endpoints for the service monitor return [ { "port": "http", From 7ac5f2e5b07dff71490b76e6350ff377b6f21f4d Mon Sep 17 00:00:00 2001 From: "Rohan R. Arora" Date: Thu, 27 Jun 2024 09:35:04 -0500 Subject: [PATCH 3/5] fix: formatting changes based on black --- fmperf/Cluster.py | 2 +- fmperf/ModelSpecs.py | 18 +++--------------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/fmperf/Cluster.py b/fmperf/Cluster.py index ac71e5d..95302d7 100644 --- a/fmperf/Cluster.py +++ b/fmperf/Cluster.py @@ -195,7 +195,7 @@ def deploy_model( }, "spec": { "selector": {"matchLabels": {"app": name}}, - "endpoints": model.get_service_monitor_endpoints() + "endpoints": model.get_service_monitor_endpoints(), }, } diff --git a/fmperf/ModelSpecs.py b/fmperf/ModelSpecs.py index dac7024..97733f5 100644 --- a/fmperf/ModelSpecs.py +++ b/fmperf/ModelSpecs.py @@ -215,9 +215,7 @@ def get_vars(self): "name": "MAX_CONCURRENT_REQUESTS", "value": str(self.max_concurrent_requests), }, - { "name": "DTYPE_STR", - "value": self.dtype_str - }, + {"name": "DTYPE_STR", "value": self.dtype_str}, { "name": "DEPLOYMENT_FRAMEWORK", "value": self.deployment_framework, @@ -374,11 +372,7 @@ def get_service_ports(self): def get_service_monitor_endpoints(self): # Specifies the endpoints for the service monitor return [ - { - "port": "http", - "path": "/metrics", - "interval": "1s" - }, + {"port": "http", "path": "/metrics", "interval": "1s"}, ] def get_readiness_probe(self): @@ -583,13 +577,7 @@ def get_service_ports(self): def get_service_monitor_endpoints(self): # Specifies the endpoints for the service monitor - return [ - { - "port": "http", - "path": "/metrics", - "interval": "1s" - } - ] + return [{"port": "http", "path": "/metrics", "interval": "1s"}] def get_readiness_probe(self): # Runs a healthcheck on a running service From f66fad5b0ca4662dccb33cb76690fdb430218b67 Mon Sep 17 00:00:00 2001 From: "Rohan R. Arora" Date: Thu, 27 Jun 2024 09:38:32 -0500 Subject: [PATCH 4/5] fix: formatting changes on the examples directory -- based on black --- examples/example_vllm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example_vllm.py b/examples/example_vllm.py index 00958c4..e4609a8 100644 --- a/examples/example_vllm.py +++ b/examples/example_vllm.py @@ -95,5 +95,5 @@ def initialize_kubernetes(location): repetition=REPETITION, number_users=NUM_USERS, duration=DURATION, - prometheus_enabled=True + prometheus_enabled=True, ) From 2a218261742784e49ae4ed62e4e6458296246c63 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Thu, 31 Jul 2025 16:13:09 -0400 Subject: [PATCH 5/5] Update docs/SETUP.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/SETUP.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/SETUP.md b/docs/SETUP.md index a8fa59c..365479e 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -171,8 +171,8 @@ The cluster is now ready to run the benchmark. As a first try, run the examples/ helm repo add prometheus-community https://prometheus-community.github.io/helm-charts ``` -# Run (to enable collection of TGIS and vLLM metrics) ```shell +# Run (to enable collection of TGIS and vLLM metrics) helm install prometheus-community/kube-prometheus-stack --create-namespace --namespace monitoring --generate-name ```