Added support to RAG/Apps DS for LMBenchmark

examples/example_llm-d-lmbenchmark-openshift-apps.py

@@ -0,0 +1,91 @@
+"""
+This script runs benchmarking on an existing vllm-d stack deployment using LMBenchmark workload.
+Note: When using LMBenchmarkWorkloadSpec, only the repetition parameter is used.
+The duration and number_users parameters are ignored as the workload specification
+controls these through max_requests and max_seconds.
+"""
+
+import os
+import urllib3
+
+import kubernetes
+from kubernetes import client, config
+
+from fmperf import Cluster
+from fmperf import LMBenchmarkWorkload
+from fmperf.StackSpec import StackSpec
+from fmperf.utils import run_benchmark
+from fmperf.utils.storage import create_local_storage, create_vpc_block_storage
+
+
+# Initialize Kubernetes Configuration
+def initialize_kubernetes(location):
+    if location == "local":
+        kubernetes.config.load_kube_config()
+        apiclient = client.ApiClient()
+        cluster = Cluster(name=location, apiclient=apiclient, namespace="default")
+
+        # Create local storage for workload data
+        _, workload_pvc_name = create_local_storage(
+            apiclient=apiclient,
+            namespace="default",
+            host_path="/results"  # Using /results for LMBenchmark
+        )
+    elif location == "remote":
+        config = client.Configuration()
+        config.host = os.environ.get("OPENSHIFT_HOST")
+        config.api_key_prefix["authorization"] = "Bearer"
+        config.api_key["authorization"] = os.environ.get("OPENSHIFT_TOKEN")
+        config.verify_ssl = False
+        apiclient = client.ApiClient(config)
+        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+        cluster = Cluster(
+            name="llm", apiclient=apiclient, namespace=os.environ.get("OPENSHIFT_NAMESPACE")
+        )
+
+        # Create PVC using VPC Block Storage for remote deployment
+        workload_pvc_name = create_vpc_block_storage(
+            apiclient=apiclient,
+            namespace=os.environ.get("OPENSHIFT_NAMESPACE")
+        )
+    else:
+        raise ValueError("Valid choices for model_mode are local and remote")
+
+    return cluster, workload_pvc_name
+
+
+if __name__ == "__main__":
+    # USER Entry: Specify the deployment location [local or remote]
+    LOCATION: str = "remote"
+
+    ## USER Entry: File Location for model workload parameters
+    WORKLOAD_FILE = os.path.join(os.path.dirname(__file__), "lmbench_vllmd_workload_with_apps_spec.yaml")
+
+    # Initialize Kubernetes
+    cluster, workload_pvc_name = initialize_kubernetes(LOCATION)
+
+    # Create workload object
+    workload_spec = LMBenchmarkWorkload.from_yaml(WORKLOAD_FILE)
+    print("workload_spec: ", workload_spec)
+    workload_spec.pvc_name = workload_pvc_name
+
+    # Create stack spec for the existing vllm-d deployment
+    stack_spec = StackSpec(
+        name="llm-d-KVCacheAwareEPP-appConfig",
+        stack_type="vllm-d",  # This will automatically set endpoint to vllm-router-service
+        refresh_interval=300,  # Refresh model list every 5 minutes
+        endpoint_url="inference-gateway"  # Service name with http:// protocol
+    )
+
+    # USER Entry: Experiment variables
+    # Note: For LMBenchmarkWorkload, only repetition is used
+    # duration and number_users are controlled by the workload spec
+    REPETITION = 1  # Repeat the experiments this many times
+
+    # Run benchmarking experiment against the stack
+    run_benchmark(
+        cluster=cluster,
+        stack_spec=stack_spec,  # Using stack_spec instead of model_spec
+        workload_spec=workload_spec,
+        repetition=REPETITION,
+    ) 

examples/lmbench_vllmd_workload_with_apps_spec.yaml

@@ -0,0 +1,19 @@
+# LMBenchmark Workload Specification Example
+# This specification is used for running benchmarks with the LMBenchmark container
+model_name: "meta-llama/Llama-3.1-70B-Instruct"  # Model identifier
+scenarios: "apps"  # Scenarios to run (all, or sharegpt, long-input, short-input)
+# sharegpt: 0.5 0.67 0.84 1 1.17 1.34
+# long-input: 1.1 0.9 0.7 0.5 0.3 0.1
+# short-input: 0.5 1 2 5 10
+# apps: 0.5 0.8 1 1.4 2
+qps_values: "0.5 0.84 1.17 1.34 5 10 15 20 25 30"  # Space-separated list of QPS values to test
+image: "gilgs/lm-benchmark-apps:appconfig"  # Container image to use
+service_account: "inference-gateway"  # Service account to use for the job 
+num_apps: 8
+users_per_app: 5
+system_prompt_len: 12000
+rag_doc_len: 130
+rag_doc_count: 10
+num_users: 40
+num_rounds: 5
+duration: 50

fmperf/Cluster.py

@@ -437,7 +437,7 @@ def evaluate(
             container_args = [
                 "QPS_VALUES=($(env | grep QPS_VALUES_ | sort -V | cut -d= -f2)); "
                 ". ~/.bashrc && . .venv/bin/activate && "
-                "/app/run_benchmarks.sh \"$MODEL\" \"$BASE_URL\" \"$SAVE_FILE_KEY\" \"$SCENARIOS\" \"${QPS_VALUES[@]}\""
+                "/app/run_benchmarks.sh --model=\"$MODEL\" --base_url=\"$BASE_URL\" --save_file_key=\"$SAVE_FILE_KEY\" --scenarios=\"$SCENARIOS\" --num_apps=\"$NUM_APPS\" --users_per_app=\"$USERS_PER_APP\" --system_prompt_len=\"$SYSTEM_PROMPT_LEN\" --rag_doc_len=\"$RAG_DOC_LEN\" --rag_doc_count=\"$RAG_DOC_COUNT\" --num_users=\"$NUM_USERS\" --num_rounds=\"$NUM_ROUNDS\" --duration=\"$DURATION\" --qps_values=\"${QPS_VALUES[@]}\""
             ]
         else:
             env = [

fmperf/WorkloadSpecs.py

@@ -303,13 +303,29 @@ def __init__(
         overwrite: bool = False,
         service_account: str = None,
         chat_template: str = None,
+        num_apps: int = 1,
+        users_per_app: int = 1,
+        system_prompt_len: int = 100,
+        rag_doc_len: int = 100,
+        rag_doc_count: int = 10,
+        num_users: int = 10,
+        num_rounds: int = 10,
+        duration: int = 10,
     ):
         self.model_name = model_name
         self.base_url = base_url
         self.scenarios = scenarios
         self.qps_values = qps_values
         self.service_account = service_account
         self.chat_template = chat_template
+        self.num_apps = num_apps
+        self.users_per_app = users_per_app
+        self.system_prompt_len = system_prompt_len
+        self.rag_doc_len = rag_doc_len
+        self.rag_doc_count = rag_doc_count
+        self.num_users = num_users
+        self.num_rounds = num_rounds
+        self.duration = duration
         super().__init__(1, image, pvc_name, overwrite)
 
     @classmethod
@@ -342,6 +358,14 @@ def get_env(
             {"name": "BASE_URL", "value": model_url},
             {"name": "SAVE_FILE_KEY", "value": f"/requests/{folder_name}/LMBench"},
             {"name": "SCENARIOS", "value": self.scenarios},
+            {"name": "NUM_APPS", "value": str(self.num_apps)},
+            {"name": "USERS_PER_APP", "value": str(self.users_per_app)},
+            {"name": "SYSTEM_PROMPT_LEN", "value": str(self.system_prompt_len)},
+            {"name": "RAG_DOC_LEN", "value": str(self.rag_doc_len)},
+            {"name": "RAG_DOC_COUNT", "value": str(self.rag_doc_count)},
+            {"name": "NUM_USERS", "value": str(self.num_users)},
+            {"name": "NUM_ROUNDS", "value": str(self.num_rounds)},
+            {"name": "DURATION", "value": str(self.duration)},
         ]
 
         # Split QPS values and add them as individual environment variables