Enabling prometheus support with service monitors

docs/SETUP.md

@@ -162,3 +162,27 @@ nvidia-operator-validator-ztpls                                   1/1     Runnin
 ```
 
 The cluster is now ready to run the benchmark. As a first try, run the examples/example_vllm.py script.
+
+
+## Prometheus (Optional)
+
+```shell
+# Run
+helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+```
+
+```shell
+# Run (to enable collection of TGIS and vLLM metrics)
+helm install prometheus-community/kube-prometheus-stack --create-namespace --namespace monitoring --generate-name
+```
+
+INSTEAD (/OR)
+
+```shell
+# To also enable collection of DCGM metrics in addition to TGIS and vLLM metrics, follow steps at <https://docs.nvidia.com/datacenter/cloud-native/gpu-telemetry/latest/kube-prometheus.html> to amend /tmp/kube-prometheus-stack.values
+```
+
+```shell
+# Run
+helm install prometheus-community/kube-prometheus-stack --create-namespace --namespace monitoring --generate-name --values /tmp/kube-prometheus-stack.values
+```

examples/example_vllm.py

@@ -95,4 +95,5 @@ def initialize_kubernetes(location):
         repetition=REPETITION,
         number_users=NUM_USERS,
         duration=DURATION,
+        prometheus_enabled=True,
     )

fmperf/Cluster.py

@@ -47,6 +47,7 @@ def deploy_model(
         self,
         model: ModelSpec,
         id: str = "",
+        prometheus_enabled: bool = False,
     ):
         creating = Creating(self.apigetter, self.logger, ignore_exists=True)
         vars = model.get_vars()
@@ -110,10 +111,6 @@ def deploy_model(
                 "strategy": {"rollingUpdate": {"maxSurge": 1}},
                 "template": {
                     "metadata": {
-                        "annotations": {
-                            "prometheus.io/port": "3000",
-                            "prometheus.io/scrape": "true",
-                        },
                         "labels": {
                             "app": name,
                         },
@@ -167,18 +164,12 @@ def deploy_model(
                 "namespace": self.namespace,
             },
             "spec": {
-                "ports": [{"name": "grpc", "port": 8033, "targetPort": "grpc"}],
+                "ports": model.get_service_ports(),
                 "selector": {"app": name},
                 "type": "ClusterIP",
             },
         }
 
-        # Change the service manifest if vllm server is to be deployed
-        if type(model) is vLLMModelSpec:
-            manifest["spec"]["ports"] = [
-                {"name": "http", "port": 8000, "targetPort": "http"}
-            ]
-
         # create service
         creating.create_namespaced_service(
             name=name, namespace=self.namespace, payload=manifest
@@ -194,6 +185,30 @@ def deploy_model(
         # get server url
         url = model.get_url() % (out.spec.cluster_ip)
 
+        if prometheus_enabled:
+            manifest = {
+                "apiVersion": "monitoring.coreos.com/v1",
+                "kind": "ServiceMonitor",
+                "metadata": {
+                    "name": name,
+                    "namespace": self.namespace,
+                },
+                "spec": {
+                    "selector": {"matchLabels": {"app": name}},
+                    "endpoints": model.get_service_monitor_endpoints(),
+                },
+            }
+
+            # create service monitor
+            creating.create_namespaced_custom_object(
+                name,
+                "monitoring.coreos.com/v1",
+                self.namespace,
+                "servicemonitors",
+                manifest,
+                wait_until=None,
+            )
+
         return DeployedModel(
             spec=model,
             name=name,

fmperf/ModelSpecs.py

@@ -244,7 +244,10 @@ def get_vars(self):
                 "name": "OUTPUT_SPECIAL_TOKENS",
                 "value": str(self.output_special_tokens).lower(),
             },
-            {"name": "PAGED_ATTENTION", "value": str(self.paged_attention).lower()},
+            {
+                "name": "PAGED_ATTENTION",
+                "value": str(self.paged_attention).lower(),
+            },
             {
                 "name": "PORT",
                 "value": str(self.port),
@@ -353,10 +356,23 @@ def get_liveness_probe(self):
         }
 
     def get_ports(self):
-        # Specifies the ports for service
+        # Specifies the ports for the container
         return [
-            {"containerPort": 3000, "name": "http"},
             {"containerPort": 8033, "name": "grpc"},
+            {"containerPort": 3000, "name": "http"},
+        ]
+
+    def get_service_ports(self):
+        # Specifies the ports for service
+        return [
+            {"name": "grpc", "port": 8033, "targetPort": "grpc"},
+            {"name": "http", "port": 3000, "targetPort": "http"},
+        ]
+
+    def get_service_monitor_endpoints(self):
+        # Specifies the endpoints for the service monitor
+        return [
+            {"port": "http", "path": "/metrics", "interval": "1s"},
         ]
 
     def get_readiness_probe(self):
@@ -552,9 +568,17 @@ def get_liveness_probe(self):
         }
 
     def get_ports(self):
-        # Specifies the ports for service
+        # Specifies the ports for the container
         return [{"containerPort": 8000, "name": "http"}]
 
+    def get_service_ports(self):
+        # Specifies the ports for service
+        return [{"name": "http", "port": 8000, "targetPort": "http"}]
+
+    def get_service_monitor_endpoints(self):
+        # Specifies the endpoints for the service monitor
+        return [{"port": "http", "path": "/metrics", "interval": "1s"}]
+
     def get_readiness_probe(self):
         # Runs a healthcheck on a running service
         return {

fmperf/utils/Benchmarking.py

@@ -14,13 +14,14 @@ def run_benchmark(
     number_users: int,
     duration: str,
     id: str = "",
+    prometheus_enabled: bool = False,
 ) -> None:
     if isinstance(model_spec, ModelSpec):
         model_spec = [model_spec]
 
     for spec in model_spec:
         # create the inference server
-        model = cluster.deploy_model(spec, id)
+        model = cluster.deploy_model(spec, id, prometheus_enabled=prometheus_enabled)
         # create the jobs for requests
         workload = cluster.generate_workload(model, workload_spec, id=id)
         for rep in range(repetition):