From 394267a3e5625ac48a68c0a336b5840ea9e13fbc Mon Sep 17 00:00:00 2001 From: Gil Sharon Date: Tue, 13 May 2025 21:08:04 +0300 Subject: [PATCH] Added support to RAG/Apps DS for LMBenchmark --- ...xample_llm-d-lmbenchmark-openshift-apps.py | 91 +++++++++++++++++++ ...lmbench_vllmd_workload_with_apps_spec.yaml | 19 ++++ fmperf/Cluster.py | 2 +- fmperf/WorkloadSpecs.py | 24 +++++ 4 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 examples/example_llm-d-lmbenchmark-openshift-apps.py create mode 100644 examples/lmbench_vllmd_workload_with_apps_spec.yaml diff --git a/examples/example_llm-d-lmbenchmark-openshift-apps.py b/examples/example_llm-d-lmbenchmark-openshift-apps.py new file mode 100644 index 0000000..13733de --- /dev/null +++ b/examples/example_llm-d-lmbenchmark-openshift-apps.py @@ -0,0 +1,91 @@ +""" +This script runs benchmarking on an existing vllm-d stack deployment using LMBenchmark workload. +Note: When using LMBenchmarkWorkloadSpec, only the repetition parameter is used. +The duration and number_users parameters are ignored as the workload specification +controls these through max_requests and max_seconds. +""" + +import os +import urllib3 + +import kubernetes +from kubernetes import client, config + +from fmperf import Cluster +from fmperf import LMBenchmarkWorkload +from fmperf.StackSpec import StackSpec +from fmperf.utils import run_benchmark +from fmperf.utils.storage import create_local_storage, create_vpc_block_storage + + +# Initialize Kubernetes Configuration +def initialize_kubernetes(location): + if location == "local": + kubernetes.config.load_kube_config() + apiclient = client.ApiClient() + cluster = Cluster(name=location, apiclient=apiclient, namespace="default") + + # Create local storage for workload data + _, workload_pvc_name = create_local_storage( + apiclient=apiclient, + namespace="default", + host_path="/results" # Using /results for LMBenchmark + ) + elif location == "remote": + config = client.Configuration() + config.host = os.environ.get("OPENSHIFT_HOST") + config.api_key_prefix["authorization"] = "Bearer" + config.api_key["authorization"] = os.environ.get("OPENSHIFT_TOKEN") + config.verify_ssl = False + apiclient = client.ApiClient(config) + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + cluster = Cluster( + name="llm", apiclient=apiclient, namespace=os.environ.get("OPENSHIFT_NAMESPACE") + ) + + # Create PVC using VPC Block Storage for remote deployment + workload_pvc_name = create_vpc_block_storage( + apiclient=apiclient, + namespace=os.environ.get("OPENSHIFT_NAMESPACE") + ) + else: + raise ValueError("Valid choices for model_mode are local and remote") + + return cluster, workload_pvc_name + + +if __name__ == "__main__": + # USER Entry: Specify the deployment location [local or remote] + LOCATION: str = "remote" + + ## USER Entry: File Location for model workload parameters + WORKLOAD_FILE = os.path.join(os.path.dirname(__file__), "lmbench_vllmd_workload_with_apps_spec.yaml") + + # Initialize Kubernetes + cluster, workload_pvc_name = initialize_kubernetes(LOCATION) + + # Create workload object + workload_spec = LMBenchmarkWorkload.from_yaml(WORKLOAD_FILE) + print("workload_spec: ", workload_spec) + workload_spec.pvc_name = workload_pvc_name + + # Create stack spec for the existing vllm-d deployment + stack_spec = StackSpec( + name="llm-d-KVCacheAwareEPP-appConfig", + stack_type="vllm-d", # This will automatically set endpoint to vllm-router-service + refresh_interval=300, # Refresh model list every 5 minutes + endpoint_url="inference-gateway" # Service name with http:// protocol + ) + + # USER Entry: Experiment variables + # Note: For LMBenchmarkWorkload, only repetition is used + # duration and number_users are controlled by the workload spec + REPETITION = 1 # Repeat the experiments this many times + + # Run benchmarking experiment against the stack + run_benchmark( + cluster=cluster, + stack_spec=stack_spec, # Using stack_spec instead of model_spec + workload_spec=workload_spec, + repetition=REPETITION, + ) \ No newline at end of file diff --git a/examples/lmbench_vllmd_workload_with_apps_spec.yaml b/examples/lmbench_vllmd_workload_with_apps_spec.yaml new file mode 100644 index 0000000..014aaca --- /dev/null +++ b/examples/lmbench_vllmd_workload_with_apps_spec.yaml @@ -0,0 +1,19 @@ +# LMBenchmark Workload Specification Example +# This specification is used for running benchmarks with the LMBenchmark container +model_name: "meta-llama/Llama-3.1-70B-Instruct" # Model identifier +scenarios: "apps" # Scenarios to run (all, or sharegpt, long-input, short-input) +# sharegpt: 0.5 0.67 0.84 1 1.17 1.34 +# long-input: 1.1 0.9 0.7 0.5 0.3 0.1 +# short-input: 0.5 1 2 5 10 +# apps: 0.5 0.8 1 1.4 2 +qps_values: "0.5 0.84 1.17 1.34 5 10 15 20 25 30" # Space-separated list of QPS values to test +image: "gilgs/lm-benchmark-apps:appconfig" # Container image to use +service_account: "inference-gateway" # Service account to use for the job +num_apps: 8 +users_per_app: 5 +system_prompt_len: 12000 +rag_doc_len: 130 +rag_doc_count: 10 +num_users: 40 +num_rounds: 5 +duration: 50 \ No newline at end of file diff --git a/fmperf/Cluster.py b/fmperf/Cluster.py index 90cd66e..5263b5c 100644 --- a/fmperf/Cluster.py +++ b/fmperf/Cluster.py @@ -437,7 +437,7 @@ def evaluate( container_args = [ "QPS_VALUES=($(env | grep QPS_VALUES_ | sort -V | cut -d= -f2)); " ". ~/.bashrc && . .venv/bin/activate && " - "/app/run_benchmarks.sh \"$MODEL\" \"$BASE_URL\" \"$SAVE_FILE_KEY\" \"$SCENARIOS\" \"${QPS_VALUES[@]}\"" + "/app/run_benchmarks.sh --model=\"$MODEL\" --base_url=\"$BASE_URL\" --save_file_key=\"$SAVE_FILE_KEY\" --scenarios=\"$SCENARIOS\" --num_apps=\"$NUM_APPS\" --users_per_app=\"$USERS_PER_APP\" --system_prompt_len=\"$SYSTEM_PROMPT_LEN\" --rag_doc_len=\"$RAG_DOC_LEN\" --rag_doc_count=\"$RAG_DOC_COUNT\" --num_users=\"$NUM_USERS\" --num_rounds=\"$NUM_ROUNDS\" --duration=\"$DURATION\" --qps_values=\"${QPS_VALUES[@]}\"" ] else: env = [ diff --git a/fmperf/WorkloadSpecs.py b/fmperf/WorkloadSpecs.py index 8aed48e..7ee6f68 100644 --- a/fmperf/WorkloadSpecs.py +++ b/fmperf/WorkloadSpecs.py @@ -303,6 +303,14 @@ def __init__( overwrite: bool = False, service_account: str = None, chat_template: str = None, + num_apps: int = 1, + users_per_app: int = 1, + system_prompt_len: int = 100, + rag_doc_len: int = 100, + rag_doc_count: int = 10, + num_users: int = 10, + num_rounds: int = 10, + duration: int = 10, ): self.model_name = model_name self.base_url = base_url @@ -310,6 +318,14 @@ def __init__( self.qps_values = qps_values self.service_account = service_account self.chat_template = chat_template + self.num_apps = num_apps + self.users_per_app = users_per_app + self.system_prompt_len = system_prompt_len + self.rag_doc_len = rag_doc_len + self.rag_doc_count = rag_doc_count + self.num_users = num_users + self.num_rounds = num_rounds + self.duration = duration super().__init__(1, image, pvc_name, overwrite) @classmethod @@ -342,6 +358,14 @@ def get_env( {"name": "BASE_URL", "value": model_url}, {"name": "SAVE_FILE_KEY", "value": f"/requests/{folder_name}/LMBench"}, {"name": "SCENARIOS", "value": self.scenarios}, + {"name": "NUM_APPS", "value": str(self.num_apps)}, + {"name": "USERS_PER_APP", "value": str(self.users_per_app)}, + {"name": "SYSTEM_PROMPT_LEN", "value": str(self.system_prompt_len)}, + {"name": "RAG_DOC_LEN", "value": str(self.rag_doc_len)}, + {"name": "RAG_DOC_COUNT", "value": str(self.rag_doc_count)}, + {"name": "NUM_USERS", "value": str(self.num_users)}, + {"name": "NUM_ROUNDS", "value": str(self.num_rounds)}, + {"name": "DURATION", "value": str(self.duration)}, ] # Split QPS values and add them as individual environment variables