From 590cb24db8cf53837c5b30a3dc363c93edacf987 Mon Sep 17 00:00:00 2001 From: xangcastle Date: Thu, 25 Jun 2026 00:22:41 -0600 Subject: [PATCH] ci(benchmark): add benchmark config for nativelink --- benchmark/config/buildbarn-frontend.json | 37 +++-- benchmark/config/buildbarn-runner.json | 9 ++ benchmark/config/buildbarn-scheduler.json | 44 +++--- benchmark/config/buildbarn-storage.json | 100 +++++++------ benchmark/config/buildbarn-worker.json | 58 ++++---- benchmark/config/buildbuddy-config.yaml | 19 ++- benchmark/config/buildfarm-server.config | 60 +++----- benchmark/config/buildfarm-worker.config | 82 +++++------ benchmark/config/nativelink-server.json5 | 164 ++++++++++++++++++++++ benchmark/config/nativelink-worker.json5 | 93 ++++++++++++ 10 files changed, 445 insertions(+), 221 deletions(-) create mode 100644 benchmark/config/buildbarn-runner.json create mode 100644 benchmark/config/nativelink-server.json5 create mode 100644 benchmark/config/nativelink-worker.json5 diff --git a/benchmark/config/buildbarn-frontend.json b/benchmark/config/buildbarn-frontend.json index 8a5adc6..523dff9 100644 --- a/benchmark/config/buildbarn-frontend.json +++ b/benchmark/config/buildbarn-frontend.json @@ -1,31 +1,30 @@ { - "httpListenAddress": ":8980", "grpcServers": [ { "listenAddresses": [":8980"], - "authentication": { - "allowAll": true - } + "authenticationPolicy": {"allow": {}} } ], - " schedulers": { - "default": { - "endpoint": "buildbarn-scheduler:8982" + "schedulers": { + "": { + "endpoint": {"address": "buildbarn-scheduler:8982"} } }, "contentAddressableStorage": { - "shard": { - "shards": [ - { - "backend": { - "grpc": { - "endpoint": "buildbarn-storage:8981" - } - }, - "weight": 1 - } - ] - } + "backend": { + "grpc": {"client": {"address": "buildbarn-storage:8981"}} + }, + "getAuthorizer": {"allow": {}}, + "putAuthorizer": {"allow": {}}, + "findMissingAuthorizer": {"allow": {}} + }, + "actionCache": { + "backend": { + "grpc": {"client": {"address": "buildbarn-storage:8981"}} + }, + "getAuthorizer": {"allow": {}}, + "putAuthorizer": {"allow": {}} }, + "executeAuthorizer": {"allow": {}}, "maximumMessageSizeBytes": 16777216 } diff --git a/benchmark/config/buildbarn-runner.json b/benchmark/config/buildbarn-runner.json new file mode 100644 index 0000000..0c542b3 --- /dev/null +++ b/benchmark/config/buildbarn-runner.json @@ -0,0 +1,9 @@ +{ + "buildDirectoryPath": "/worker/build", + "grpcServers": [ + { + "listenPaths": ["/worker/runner"], + "authenticationPolicy": {"allow": {}} + } + ] +} diff --git a/benchmark/config/buildbarn-scheduler.json b/benchmark/config/buildbarn-scheduler.json index 86a3c4b..f823ab5 100644 --- a/benchmark/config/buildbarn-scheduler.json +++ b/benchmark/config/buildbarn-scheduler.json @@ -1,44 +1,32 @@ { - "httpListenAddress": ":8982", "clientGrpcServers": [ { "listenAddresses": [":8982"], - "authentication": { - "allowAll": true - } + "authenticationPolicy": {"allow": {}} } ], "workerGrpcServers": [ { "listenAddresses": [":8983"], - "authentication": { - "allowAll": true - } + "authenticationPolicy": {"allow": {}} } ], - "scheduler": { - "buildQueue": { - "redis": { - "endpoint": "buildbarn-redis:6379" - } - }, - "actionRouter": { - "simple": {} - } - }, "contentAddressableStorage": { - "shard": { - "shards": [ - { - "backend": { - "grpc": { - "endpoint": "buildbarn-storage:8981" - } - }, - "weight": 1 - } - ] + "grpc": {"client": {"address": "buildbarn-storage:8981"}} + }, + "executeAuthorizer": {"allow": {}}, + "modifyDrainsAuthorizer": {"allow": {}}, + "killOperationsAuthorizer": {"allow": {}}, + "synchronizeAuthorizer": {"allow": {}}, + "actionRouter": { + "simple": { + "platformKeyExtractor": {"action": {}}, + "initialSizeClassAnalyzer": { + "defaultExecutionTimeout": "600s", + "maximumExecutionTimeout": "3600s" + } } }, + "platformQueueWithNoWorkersTimeout": "900s", "maximumMessageSizeBytes": 16777216 } diff --git a/benchmark/config/buildbarn-storage.json b/benchmark/config/buildbarn-storage.json index acd48d0..e138324 100644 --- a/benchmark/config/buildbarn-storage.json +++ b/benchmark/config/buildbarn-storage.json @@ -1,52 +1,66 @@ { - "blobstore": { - "contentAddressableStorage": { - "sharding": { - "hashInitialization": 11946695773637837490, - "shards": [ - { - "backend": { - "local": { - "keyLocationMap": { - "file": { - "path": "/storage/key_location_map", - "sizeBytes": 10000000 - } - }, - "keyLocationMapMaximumGetAttempts": 8, - "keyLocationMapMaximumPutAttempts": 32, - "oldBlockFiles": [ - { - "path": "/storage/old_blocks", - "sizeBytes": 10000000000 - } - ], - "currentBlockFile": { - "path": "/storage/current_blocks", - "sizeBytes": 10000000000 - } - } - }, - "weight": 1 + "grpcServers": [ + { + "listenAddresses": [":8981"], + "authenticationPolicy": {"allow": {}} + } + ], + "contentAddressableStorage": { + "backend": { + "local": { + "keyLocationMapOnBlockDevice": { + "file": { + "path": "/storage-cas/key_location_map", + "sizeBytes": 134217728 } - ] + }, + "keyLocationMapMaximumGetAttempts": 16, + "keyLocationMapMaximumPutAttempts": 64, + "oldBlocks": 8, + "currentBlocks": 24, + "newBlocks": 3, + "blocksOnBlockDevice": { + "source": { + "file": { + "path": "/storage-cas/blocks", + "sizeBytes": 8589934592 + } + }, + "spareBlocks": 3 + } } }, - "actionCache": { + "getAuthorizer": {"allow": {}}, + "putAuthorizer": {"allow": {}}, + "findMissingAuthorizer": {"allow": {}} + }, + "actionCache": { + "backend": { "local": { - "path": "/storage/ac", - "sizeBytes": 1000000000 + "keyLocationMapOnBlockDevice": { + "file": { + "path": "/storage-ac/key_location_map", + "sizeBytes": 1048576 + } + }, + "keyLocationMapMaximumGetAttempts": 16, + "keyLocationMapMaximumPutAttempts": 64, + "oldBlocks": 8, + "currentBlocks": 24, + "newBlocks": 1, + "blocksOnBlockDevice": { + "source": { + "file": { + "path": "/storage-ac/blocks", + "sizeBytes": 67108864 + } + }, + "spareBlocks": 3 + } } - } + }, + "getAuthorizer": {"allow": {}}, + "putAuthorizer": {"allow": {}} }, - "httpListenAddress": ":8981", - "grpcServers": [ - { - "listenAddresses": [":8981"], - "authentication": { - "allowAll": true - } - } - ], "maximumMessageSizeBytes": 16777216 } diff --git a/benchmark/config/buildbarn-worker.json b/benchmark/config/buildbarn-worker.json index 04e92c9..3b10837 100644 --- a/benchmark/config/buildbarn-worker.json +++ b/benchmark/config/buildbarn-worker.json @@ -1,36 +1,38 @@ { - "scheduler": { - "endpoint": "buildbarn-scheduler:8983" + "blobstore": { + "contentAddressableStorage": { + "grpc": {"client": {"address": "buildbarn-storage:8981"}} + }, + "actionCache": { + "grpc": {"client": {"address": "buildbarn-storage:8981"}} + } }, - "contentAddressableStorage": { - "shard": { - "shards": [ + "browserUrl": "http://localhost:7984", + "scheduler": {"address": "buildbarn-scheduler:8983"}, + "maximumMessageSizeBytes": 16777216, + "buildDirectories": [ + { + "native": { + "buildDirectoryPath": "/worker/build", + "cacheDirectoryPath": "/worker/cache", + "maximumCacheFileCount": 10000, + "maximumCacheSizeBytes": 1073741824, + "cacheReplacementPolicy": "LEAST_RECENTLY_USED" + }, + "runners": [ { - "backend": { - "grpc": { - "endpoint": "buildbarn-storage:8981" - } - }, - "weight": 1 + "endpoint": {"address": "unix:///worker/runner"}, + "concurrency": 4, + "platform": {"properties": []} } ] } - }, - "buildDirectoryPath": "/worker/build", - "cacheDirectoryPath": "/worker/cache", - "maximumCacheFileCount": 10000, - "maximumCacheSizeBytes": 1073741824, - "instanceName": "", - "platform": { - "properties": [ - { - "name": "OSFamily", - "value": "linux" - }, - { - "name": "container-image", - "value": "docker://marketplace.gcr.io/google/rbe-ubuntu16-04" - } - ] + ], + "inputDownloadConcurrency": 10, + "outputUploadConcurrency": 10, + "directoryCache": { + "maximumCount": 1000, + "maximumSizeBytes": 1024000, + "cacheReplacementPolicy": "LEAST_RECENTLY_USED" } } diff --git a/benchmark/config/buildbuddy-config.yaml b/benchmark/config/buildbuddy-config.yaml index 9a1ce31..3fd16fb 100644 --- a/benchmark/config/buildbuddy-config.yaml +++ b/benchmark/config/buildbuddy-config.yaml @@ -1,15 +1,18 @@ # BuildBuddy Configuration -# Minimal config for benchmarking +# Minimal config for benchmarking — CAS only, no Redis dependency. +# +# Redis health checks timeout under QEMU (amd64 on arm64) due to emulation +# overhead. Auth works correctly without Redis when remote_execution is disabled. app: build_buddy_url: "http://localhost:8080" events_api_url: "grpc://localhost:1985" cache_api_url: "grpc://localhost:1985" remote_execution_api_url: "grpc://localhost:1985" - no_default_user_group: true + no_default_user_group: false database: - data_source: "postgres://buildbuddy:buildbuddy@buildbuddy-postgres:5432/buildbuddy?sslmode=disable" + data_source: "sqlite3:///data/buildbuddy.db" storage: disk: @@ -24,16 +27,10 @@ cache: max_size_bytes: 10000000000 remote_execution: - enabled: true - default_pool_name: "default" - pools: - - name: "default" - os: "linux" - architecture: "amd64" - redis_target: "buildbuddy-redis:6379" + enabled: false executor: - enabled: false # Run separately + enabled: false auth: enable_anonymous_usage: true diff --git a/benchmark/config/buildfarm-server.config b/benchmark/config/buildfarm-server.config index 6f4205c..41b5a1e 100644 --- a/benchmark/config/buildfarm-server.config +++ b/benchmark/config/buildfarm-server.config @@ -1,45 +1,19 @@ -# Bazel Buildfarm Server Configuration -# Minimal configuration for benchmarking +# Bazel Buildfarm Server Configuration (SHARD mode) +# Minimal configuration for benchmarking. +# +# Schema follows the modern buildfarm YAML config (the old proto-text format +# previously in this file was silently ignored by bazelbuild/buildfarm:latest, +# which then fell back to defaults and tried to reach Redis on localhost). +backplane: + redisUri: "redis://buildfarm-redis:6379" + queues: + - name: "cpu" + allowUnmatched: true + properties: + - name: "*" + value: "*" -instance_name: "benchmark" - -# CAS configuration -content_addressable_storage { - delegate_cas { - grpc { - target: "buildfarm-redis:6379" - } - } -} - -# Action Cache configuration -action_cache { - delegate_ac { - grpc { - target: "buildfarm-redis:6379" - } - } -} - -# Operations memory queue -operations { - memory { - list_operations_default_page_size: 1024 - list_operations_max_page_size: 16384 - } -} - -# Memory backplane for operation queue -memory_backplane { - redis_uri: "redis://buildfarm-redis:6379" - queues { - name: "cpu" - allow_unmatched: true - } -} - -# Server configuration -server { +server: + instanceType: SHARD + name: shard port: 9092 - prometheus_port: 9093 -} diff --git a/benchmark/config/buildfarm-worker.config b/benchmark/config/buildfarm-worker.config index c9022ca..58abede 100644 --- a/benchmark/config/buildfarm-worker.config +++ b/benchmark/config/buildfarm-worker.config @@ -1,50 +1,34 @@ -# Bazel Buildfarm Worker Configuration -# Minimal configuration for benchmarking +# Bazel Buildfarm Worker Configuration (SHARD mode) +# Minimal configuration for benchmarking. +# +# Schema follows the modern buildfarm YAML config; workers register themselves +# in the Redis backplane under publicName, which must be resolvable by the +# server inside the compose network. +backplane: + redisUri: "redis://buildfarm-redis:6379" + queues: + - name: "cpu" + allowUnmatched: true + properties: + - name: "*" + value: "*" -worker { - port: 9092 - - # CAS endpoint - content_addressable_storage { - target: "buildfarm-server:9092" - } - - # Action cache endpoint - action_cache { - target: "buildfarm-server:9092" - } - - # Execution endpoint - execution { - target: "buildfarm-server:9092" - } - - # Worker settings - platform { - properties { - name: "OSFamily" - value: "linux" - } - properties { - name: "container-image" - value: "docker://gcr.io/cloud-marketplace/google/rbe-ubuntu16-04@sha256:..." - } - } - - # Execution settings - execution { - # Run operations locally in the container - local { - # Sandbox settings - spawn { - spawn_timeout_seconds: 3600 - } - } - } -} - -# Root for worker execution -root: "/tmp/worker" - -# Public name (for identifying this worker) -public_name: "buildfarm-worker" +worker: + port: 8981 + publicName: "buildfarm-worker:8981" + root: /tmp/worker + capabilities: + cas: true + execution: true + storages: + - type: FILESYSTEM + path: cache + maxSizeBytes: 2147483648 + executeStageWidth: 4 + inputFetchStageWidth: 2 + dequeueMatchSettings: + allowUnmatched: true + # cgroups are unavailable inside the unprivileged-by-default benchmark + # containers; the default (true) makes every execution fail. + sandboxSettings: + alwaysUseCgroups: false diff --git a/benchmark/config/nativelink-server.json5 b/benchmark/config/nativelink-server.json5 new file mode 100644 index 0000000..f1c31dd --- /dev/null +++ b/benchmark/config/nativelink-server.json5 @@ -0,0 +1,164 @@ +// NativeLink all-in-one server (CAS + AC + scheduler) for benchmarking. +// +// Unlike ferrisrbe/buildbarn/buildfarm, NativeLink serves the CAS in the same +// process as the scheduler, so this single container replaces both the +// "server" and "bazel-remote" roles of the other stacks. Store sizing mirrors +// the ferrisrbe stack: 10GB disk-backed CAS (BAZEL_REMOTE_MAX_SIZE=10) with a +// small in-memory hot tier so the server fits the same 512M container limit. +{ + stores: [ + { + name: "CAS_MAIN_STORE", + existence_cache: { + backend: { + verify: { + verify_size: true, + verify_hash: true, + backend: { + fast_slow: { + fast: { + size_partitioning: { + size: "64kb", + lower_store: { + memory: { + eviction_policy: { + max_bytes: "64mb", + max_count: 100000, + }, + }, + }, + upper_store: { + noop: {}, + }, + }, + }, + slow: { + compression: { + compression_algorithm: { + lz4: {}, + }, + backend: { + filesystem: { + content_path: "/data/content_path-cas", + temp_path: "/data/tmp_path-cas", + eviction_policy: { + max_bytes: "10gb", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "AC_MAIN_STORE", + completeness_checking: { + backend: { + fast_slow: { + fast: { + size_partitioning: { + size: "1kb", + lower_store: { + memory: { + eviction_policy: { + max_bytes: "16mb", + max_count: 150000, + }, + }, + }, + upper_store: { + noop: {}, + }, + }, + }, + slow: { + filesystem: { + content_path: "/data/content_path-ac", + temp_path: "/data/tmp_path-ac", + eviction_policy: { + max_bytes: "1gb", + }, + }, + }, + }, + }, + cas_store: { + ref_store: { + name: "CAS_MAIN_STORE", + }, + }, + }, + }, + ], + schedulers: [ + { + name: "MAIN_SCHEDULER", + simple: { + supported_platform_properties: { + cpu_count: "priority", + memory_kb: "priority", + cpu_arch: "priority", + OSFamily: "priority", + ISA: "exact", + }, + }, + }, + ], + servers: [ + { + listener: { + http: { + socket_address: "0.0.0.0:50051", + }, + }, + services: { + cas: [ + { + cas_store: "CAS_MAIN_STORE", + }, + ], + ac: [ + { + ac_store: "AC_MAIN_STORE", + }, + ], + capabilities: [ + { + remote_execution: { + scheduler: "MAIN_SCHEDULER", + }, + }, + ], + execution: [ + { + cas_store: "CAS_MAIN_STORE", + scheduler: "MAIN_SCHEDULER", + }, + ], + bytestream: { + cas_stores: { + "": "CAS_MAIN_STORE", + }, + }, + }, + }, + { + listener: { + http: { + socket_address: "0.0.0.0:50061", + }, + }, + services: { + // Worker-only backend API; not exposed on the host. + worker_api: { + scheduler: "MAIN_SCHEDULER", + }, + health: {}, + }, + }, + ], +} diff --git a/benchmark/config/nativelink-worker.json5 b/benchmark/config/nativelink-worker.json5 new file mode 100644 index 0000000..04da6b7 --- /dev/null +++ b/benchmark/config/nativelink-worker.json5 @@ -0,0 +1,93 @@ +// NativeLink worker for benchmarking. The worker binary is injected into a +// plain ubuntu:22.04 container by the nativelink-worker-init service (the +// upstream image is a static musl binary with no shell, unusable as an +// execution environment by itself). +// +// ${NATIVELINK_ENDPOINT} is interpolated by nativelink at startup, not by +// docker-compose. +{ + stores: [ + { + name: "GRPC_LOCAL_STORE", + grpc: { + instance_name: "", + endpoints: [ + { + address: "grpc://${NATIVELINK_ENDPOINT:-127.0.0.1}:50051", + }, + ], + store_type: "cas", + }, + }, + { + name: "GRPC_LOCAL_AC_STORE", + grpc: { + instance_name: "", + endpoints: [ + { + address: "grpc://${NATIVELINK_ENDPOINT:-127.0.0.1}:50051", + }, + ], + store_type: "ac", + }, + }, + { + name: "WORKER_FAST_SLOW_STORE", + fast_slow: { + fast: { + filesystem: { + content_path: "/workspace/content_path-cas", + temp_path: "/workspace/tmp_path-cas", + eviction_policy: { + max_bytes: "5gb", + }, + }, + }, + slow: { + ref_store: { + name: "GRPC_LOCAL_STORE", + }, + }, + }, + }, + ], + workers: [ + { + local: { + worker_api_endpoint: { + uri: "grpc://${NATIVELINK_ENDPOINT:-127.0.0.1}:50061", + }, + cas_fast_slow_store: "WORKER_FAST_SLOW_STORE", + upload_action_result: { + ac_store: "GRPC_LOCAL_AC_STORE", + }, + work_directory: "/workspace/work", + platform_properties: { + cpu_count: { + query_cmd: "nproc", + }, + cpu_arch: { + values: [ + "x86_64", + ], + }, + OSFamily: { + values: [ + "linux", + "Linux", + ], + }, + ISA: { + values: [ + "x86-64", + ], + }, + }, + }, + }, + ], + servers: [], + global: { + max_open_files: 524288, + }, +}