From b24a4e6a46385aff87c224bb45585fad75729d8f Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 3 Jun 2026 13:46:43 +0200 Subject: [PATCH 01/23] Builds: isolated builds Small changes to use `readthedocs-builder` repository to isolate builds. --- dockerfiles/docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml index cbd90da..d33d5af 100644 --- a/dockerfiles/docker-compose.yml +++ b/dockerfiles/docker-compose.yml @@ -214,6 +214,9 @@ services: - ${PWD}/common/dockerfiles/nodemon.json:/usr/src/app/checkouts/nodemon.json - ${PWD}/${RTDDEV_PATH_EXT:-../readthedocs-ext}:/usr/src/app/checkouts/readthedocs-ext - ${PWD}/${RTDDEV_PATH_EXT_THEME:-../ext-theme}:/usr/src/app/checkouts/ext-theme + + # Docker in Docker + - /var/run/docker.sock:/var/run/docker.sock links: - storage - database @@ -256,6 +259,7 @@ services: - RTD_S3_STATIC_STORAGE_BUCKET - RTD_AWS_S3_REGION_NAME - RTD_OPENAI_API_KEY + - RTD_BUILDER_TOKEN stdin_open: true tty: true networks: From d93fd14f3bf60c97566d2189d4970f457694bf1e Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 3 Jun 2026 14:09:28 +0200 Subject: [PATCH 02/23] Mount docker on webs (to cancel builds) and add docstrings --- dockerfiles/docker-compose.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml index d33d5af..61ed33b 100644 --- a/dockerfiles/docker-compose.yml +++ b/dockerfiles/docker-compose.yml @@ -128,6 +128,9 @@ services: - ${PWD}/common/dockerfiles/nodemon.json:/usr/src/app/checkouts/nodemon.json - ${PWD}/${RTDDEV_PATH_EXT:-../readthedocs-ext}:/usr/src/app/checkouts/readthedocs-ext - ${PWD}/${RTDDEV_PATH_EXT_THEME:-../ext-theme}:/usr/src/app/checkouts/ext-theme + + # Docker in Docker: cancel running builds. + - /var/run/docker.sock:/var/run/docker.sock links: - storage - database @@ -215,7 +218,7 @@ services: - ${PWD}/${RTDDEV_PATH_EXT:-../readthedocs-ext}:/usr/src/app/checkouts/readthedocs-ext - ${PWD}/${RTDDEV_PATH_EXT_THEME:-../ext-theme}:/usr/src/app/checkouts/ext-theme - # Docker in Docker + # Docker in Docker: Fargate (call `docker run` from inside `web` queue) - /var/run/docker.sock:/var/run/docker.sock links: - storage @@ -331,7 +334,7 @@ services: # Because of this, we need to use a shared volume between them - build-user-builds:/usr/src/app/checkouts/readthedocs.org/user_builds - # Docker in Docker + # Docker in Docker: old implementation (`build-default` queue) - /var/run/docker.sock:/var/run/docker.sock links: - web From 60c9241eec1f0bdbc0a1f53c5dfc8a5b2852df6e Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 4 Jun 2026 17:36:56 +0200 Subject: [PATCH 03/23] Pass environment variable --- dockerfiles/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml index 61ed33b..b6d5652 100644 --- a/dockerfiles/docker-compose.yml +++ b/dockerfiles/docker-compose.yml @@ -263,6 +263,7 @@ services: - RTD_AWS_S3_REGION_NAME - RTD_OPENAI_API_KEY - RTD_BUILDER_TOKEN + - RTD_PATH_BUILDER stdin_open: true tty: true networks: From 6c20922e374a5cc57b7fc4d8bff79390f5817bdb Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 11 Jun 2026 15:19:53 +0200 Subject: [PATCH 04/23] Celery on EC2 for development --- dockerfiles/docker-compose.yml | 50 ++++++++++++++++- dockerfiles/entrypoints/isolated-builder.sh | 61 +++++++++++++++++++++ dockerfiles/tasks.py | 2 +- 3 files changed, 111 insertions(+), 2 deletions(-) create mode 100755 dockerfiles/entrypoints/isolated-builder.sh diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml index b6d5652..1f48894 100644 --- a/dockerfiles/docker-compose.yml +++ b/dockerfiles/docker-compose.yml @@ -263,7 +263,6 @@ services: - RTD_AWS_S3_REGION_NAME - RTD_OPENAI_API_KEY - RTD_BUILDER_TOKEN - - RTD_PATH_BUILDER stdin_open: true tty: true networks: @@ -370,6 +369,55 @@ services: readthedocs: command: ["../../docker/build.sh"] + isolated-builder: + # Dev emulation of the production isolated-builders AMI. Picks up + # tasks from the ``isolated-builds`` queue and spawns build + # containers on the host docker daemon (via the bind-mounted + # socket). The companion entrypoint clones readthedocs-builder (or + # skips if the host's checkout is bind-mounted in), creates the + # worker venv via uv, then runs Celery — no systemd, no New Relic, + # no Sentry. See readthedocs-builder/docs/architecture.md. + stop_grace_period: 1s + build: + context: ${PWD} + dockerfile: ${PWD}/dockerfiles/Dockerfile.isolated-builder + volumes: + # Host's readthedocs-builder checkout. The entrypoint sees the + # mount as populated and skips ``git clone`` so you can iterate + # on the runner / worker code by editing on the host and + # restarting the container. + - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}:/opt/readthedocs-builder + + # The entrypoint script itself, bind-mounted so edits don't + # need a rebuild (matches how celery.sh / build.sh wire up). + - ${PWD}/common/dockerfiles/entrypoints/isolated-builder.sh:/usr/src/app/docker/isolated-builder.sh + + # docker-out-of-docker: the worker calls ``docker run`` against + # the host daemon to spawn build containers. + - /var/run/docker.sock:/var/run/docker.sock + links: + - cache + - storage + - web + environment: + - RTD_BROKER_URL=redis://:redispassword@cache:6379/0 + - RTD_BUILDS_QUEUE=isolated-builds + - RTD_BUILDER_REF=celery-on-ec2 + # Host path the worker passes to ``docker run -v ...`` when + # spawning build containers. Must be a HOST-visible path because + # the daemon resolves it on the host. ``${PWD}`` here is the + # host's CWD (where ``docker compose`` runs), so this resolves + # to the same checkout the bind-mount above points at. + - RTD_HOST_BUILDER_PATH=${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder} + # Have build containers join the compose network so they can + # reach the API at ``http://web`` via service-name DNS. + - RTD_BUILD_NETWORK=community_readthedocs + stdin_open: true + tty: true + networks: + readthedocs: + command: ["/usr/src/app/docker/isolated-builder.sh"] + cache: image: redis:6.2.6 # Match production version command: > diff --git a/dockerfiles/entrypoints/isolated-builder.sh b/dockerfiles/entrypoints/isolated-builder.sh new file mode 100755 index 0000000..a3738df --- /dev/null +++ b/dockerfiles/entrypoints/isolated-builder.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Dev entrypoint for the isolated-builders pattern. +# +# Emulates the production AMI's two systemd units — +# ``readthedocs-builder-setup.service`` (clone + worker venv setup) and +# then ``celery-readthedocs-builder.service`` (start the worker) — as +# one foreground bash script. No systemd, no New Relic, no Sentry: +# dev only. +# +# Compose wiring is in ``common/dockerfiles/docker-compose.yml`` under +# the ``isolated-builder`` service. + +set -euo pipefail + +: "${RTD_BROKER_URL:?RTD_BROKER_URL must be set (e.g. redis://cache:6379/0)}" +: "${RTD_BUILDS_QUEUE:=isolated-builds}" +: "${RTD_BUILDER_REPO:=https://github.com/readthedocs/readthedocs-builder.git}" +: "${RTD_BUILDER_REF:=celery-on-ec2}" +: "${RTD_BUILDER_TOKEN:=}" + +SRC="/opt/readthedocs-builder" +WORKER_VENV="/opt/worker-venv" + +# 1. Clone (or skip if the host's checkout is bind-mounted in). +# A bind-mount means $SRC is already populated; we skip ``git clone`` +# so dev iteration on the runner code (edit on host, restart the +# container) doesn't blow the host checkout away. +if [ -z "$(ls -A "$SRC" 2>/dev/null)" ]; then + echo "[isolated-builder] $SRC empty; cloning $RTD_BUILDER_REPO@$RTD_BUILDER_REF ..." + clone_url="$RTD_BUILDER_REPO" + if [ -n "$RTD_BUILDER_TOKEN" ] && [ "${RTD_BUILDER_REPO#https://}" != "$RTD_BUILDER_REPO" ]; then + clone_url="https://${RTD_BUILDER_TOKEN}@${RTD_BUILDER_REPO#https://}" + fi + git clone --depth=1 --branch "$RTD_BUILDER_REF" "$clone_url" "$SRC" +else + echo "[isolated-builder] $SRC already populated; skipping clone (dev bind-mount)." +fi + +cd "$SRC" + +# 2. Worker venv. Dev omits newrelic/sentry-sdk (no observability in +# dev). The runner's venv is NOT created here — the build container +# spawned by the worker creates its own venv at startup via +# ``uv sync``. +echo "[isolated-builder] Creating worker venv at $WORKER_VENV ..." +uv venv --clear "$WORKER_VENV" +uv pip install --python "$WORKER_VENV/bin/python" -r "$SRC/worker/requirements.txt" + +# 3. Replace this process with the Celery worker. PYTHONPATH so +# ``-A worker.celery`` resolves; --max-tasks-per-child=1 so the +# worker exits after one task (matches prod's ephemeral pattern, +# even though there's no AWS API call to terminate anything in +# dev — the worker just exits and compose can be configured to +# restart or not). +echo "[isolated-builder] Starting Celery worker on queue '$RTD_BUILDS_QUEUE' ..." +export PYTHONPATH="$SRC" +exec "$WORKER_VENV/bin/celery" -A worker.celery worker \ + --loglevel=INFO \ + --concurrency=1 \ + --max-tasks-per-child=1 \ + -Q "$RTD_BUILDS_QUEUE" diff --git a/dockerfiles/tasks.py b/dockerfiles/tasks.py index 02a8c74..a94f93a 100644 --- a/dockerfiles/tasks.py +++ b/dockerfiles/tasks.py @@ -242,7 +242,7 @@ def test(c, arguments="", running=True): help={ "tool": "build.tool to compile (python, nodejs, rust, golang)", "version": "specific version for the tool", - "os": "ubuntu-20.04, ubuntu-22.04, ubuntu-24.04 (default)", + "os": "ubuntu-22.04, ubuntu-24.04 (default), ubuntu-26.04", } ) def compilebuildtool(c, tool, version, os=None): From b120fdde5e9a28d672f47c2bea0b819d91734819 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 11 Jun 2026 15:30:06 +0200 Subject: [PATCH 05/23] Same runner env on development --- dockerfiles/docker-compose.yml | 24 ++++++++++++++++++++- dockerfiles/entrypoints/isolated-builder.sh | 23 +++++++++++++++----- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml index 1f48894..7a04ff2 100644 --- a/dockerfiles/docker-compose.yml +++ b/dockerfiles/docker-compose.yml @@ -4,6 +4,12 @@ volumes: storage_rustfs_data: postgres_data_16: postgres_backups_data: + # Runner venv used by the isolated-builder service. Populated by + # that service's entrypoint via ``uv sync``, then bind-mounted into + # the build containers it spawns (the worker references it by its + # docker-compose-namespaced name ``community_runner-venv`` — see + # the isolated-builder service's RTD_HOST_RUNNER_VENV_PATH env). + runner-venv: services: @@ -388,6 +394,15 @@ services: # restarting the container. - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}:/opt/readthedocs-builder + # Runner venv as a docker NAMED VOLUME. The entrypoint populates + # it via ``uv sync`` exactly like the prod systemd setup unit + # does on the EC2 instance. We use a named volume (instead of a + # bind-mounted host path) so the host docker daemon owns the + # storage — that's what lets the worker pass + # ``community_runner-venv`` to ``docker run -v`` when spawning + # build containers, and have the daemon recognize and mount it. + - runner-venv:/opt/runner-venv + # The entrypoint script itself, bind-mounted so edits don't # need a rebuild (matches how celery.sh / build.sh wire up). - ${PWD}/common/dockerfiles/entrypoints/isolated-builder.sh:/usr/src/app/docker/isolated-builder.sh @@ -403,12 +418,19 @@ services: - RTD_BROKER_URL=redis://:redispassword@cache:6379/0 - RTD_BUILDS_QUEUE=isolated-builds - RTD_BUILDER_REF=celery-on-ec2 - # Host path the worker passes to ``docker run -v ...`` when + # Source path the worker passes to ``docker run -v ...`` when # spawning build containers. Must be a HOST-visible path because # the daemon resolves it on the host. ``${PWD}`` here is the # host's CWD (where ``docker compose`` runs), so this resolves # to the same checkout the bind-mount above points at. - RTD_HOST_BUILDER_PATH=${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder} + # Runner venv: the host-daemon-recognized name of the + # ``runner-venv`` named volume defined above. Compose prefixes + # it with the project name (``community`` for the default + # community-edition project). The worker passes this to + # ``docker run -v :/opt/runner-venv:ro`` and the daemon + # resolves it as a named volume mount. + - RTD_HOST_RUNNER_VENV_PATH=community_runner-venv # Have build containers join the compose network so they can # reach the API at ``http://web`` via service-name DNS. - RTD_BUILD_NETWORK=community_readthedocs diff --git a/dockerfiles/entrypoints/isolated-builder.sh b/dockerfiles/entrypoints/isolated-builder.sh index a3738df..ef5c1bb 100755 --- a/dockerfiles/entrypoints/isolated-builder.sh +++ b/dockerfiles/entrypoints/isolated-builder.sh @@ -19,6 +19,7 @@ set -euo pipefail : "${RTD_BUILDER_TOKEN:=}" SRC="/opt/readthedocs-builder" +RUNNER_VENV="/opt/runner-venv" WORKER_VENV="/opt/worker-venv" # 1. Clone (or skip if the host's checkout is bind-mounted in). @@ -38,15 +39,27 @@ fi cd "$SRC" -# 2. Worker venv. Dev omits newrelic/sentry-sdk (no observability in -# dev). The runner's venv is NOT created here — the build container -# spawned by the worker creates its own venv at startup via -# ``uv sync``. +# 2. Runner venv — mirrors the prod systemd setup unit's +# ``UV_PROJECT_ENVIRONMENT=/opt/runner-venv uv sync --frozen``. +# ``$RUNNER_VENV`` is backed by a docker NAMED VOLUME (not a path +# inside the container), so the venv lives on the host daemon's +# filesystem — that lets the worker bind-mount the same named +# volume into build containers it spawns. +# +# Idempotent: if the venv already matches uv.lock from a previous +# container start, this is fast. +echo "[isolated-builder] Syncing runner venv at $RUNNER_VENV ..." +UV_PROJECT_ENVIRONMENT="$RUNNER_VENV" uv sync --frozen + +# 3. Worker venv. Dev omits newrelic/sentry-sdk (no observability in +# dev). The worker venv stays inside the dev container (no host +# visibility needed — the celery process here is the only one that +# uses it). echo "[isolated-builder] Creating worker venv at $WORKER_VENV ..." uv venv --clear "$WORKER_VENV" uv pip install --python "$WORKER_VENV/bin/python" -r "$SRC/worker/requirements.txt" -# 3. Replace this process with the Celery worker. PYTHONPATH so +# 4. Replace this process with the Celery worker. PYTHONPATH so # ``-A worker.celery`` resolves; --max-tasks-per-child=1 so the # worker exits after one task (matches prod's ephemeral pattern, # even though there's no AWS API call to terminate anything in From 53bd52cb7dd11169d310f8c274f2a0d58fea6aaa Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 11 Jun 2026 16:06:48 +0200 Subject: [PATCH 06/23] Solve problem with Python symlink on /opt/runner-venv --- dockerfiles/docker-compose.yml | 28 +++++---------------- dockerfiles/entrypoints/isolated-builder.sh | 24 +++++------------- 2 files changed, 12 insertions(+), 40 deletions(-) diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml index 7a04ff2..ccb83a9 100644 --- a/dockerfiles/docker-compose.yml +++ b/dockerfiles/docker-compose.yml @@ -4,12 +4,6 @@ volumes: storage_rustfs_data: postgres_data_16: postgres_backups_data: - # Runner venv used by the isolated-builder service. Populated by - # that service's entrypoint via ``uv sync``, then bind-mounted into - # the build containers it spawns (the worker references it by its - # docker-compose-namespaced name ``community_runner-venv`` — see - # the isolated-builder service's RTD_HOST_RUNNER_VENV_PATH env). - runner-venv: services: @@ -394,15 +388,6 @@ services: # restarting the container. - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}:/opt/readthedocs-builder - # Runner venv as a docker NAMED VOLUME. The entrypoint populates - # it via ``uv sync`` exactly like the prod systemd setup unit - # does on the EC2 instance. We use a named volume (instead of a - # bind-mounted host path) so the host docker daemon owns the - # storage — that's what lets the worker pass - # ``community_runner-venv`` to ``docker run -v`` when spawning - # build containers, and have the daemon recognize and mount it. - - runner-venv:/opt/runner-venv - # The entrypoint script itself, bind-mounted so edits don't # need a rebuild (matches how celery.sh / build.sh wire up). - ${PWD}/common/dockerfiles/entrypoints/isolated-builder.sh:/usr/src/app/docker/isolated-builder.sh @@ -424,13 +409,12 @@ services: # host's CWD (where ``docker compose`` runs), so this resolves # to the same checkout the bind-mount above points at. - RTD_HOST_BUILDER_PATH=${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder} - # Runner venv: the host-daemon-recognized name of the - # ``runner-venv`` named volume defined above. Compose prefixes - # it with the project name (``community`` for the default - # community-edition project). The worker passes this to - # ``docker run -v :/opt/runner-venv:ro`` and the daemon - # resolves it as a named volume mount. - - RTD_HOST_RUNNER_VENV_PATH=community_runner-venv + # Host path to ``uv``. The build container's entrypoint runs + # ``uv sync`` to create its runner venv at startup, but the + # upstream readthedocs/build image doesn't have uv — the worker + # bind-mounts the host's binary in. Override per host if your + # uv lives somewhere other than /usr/local/bin/uv. + - RTD_HOST_UV_PATH=${RTDDEV_HOST_UV_PATH:-/usr/local/bin/uv} # Have build containers join the compose network so they can # reach the API at ``http://web`` via service-name DNS. - RTD_BUILD_NETWORK=community_readthedocs diff --git a/dockerfiles/entrypoints/isolated-builder.sh b/dockerfiles/entrypoints/isolated-builder.sh index ef5c1bb..75d0aa0 100755 --- a/dockerfiles/entrypoints/isolated-builder.sh +++ b/dockerfiles/entrypoints/isolated-builder.sh @@ -19,7 +19,6 @@ set -euo pipefail : "${RTD_BUILDER_TOKEN:=}" SRC="/opt/readthedocs-builder" -RUNNER_VENV="/opt/runner-venv" WORKER_VENV="/opt/worker-venv" # 1. Clone (or skip if the host's checkout is bind-mounted in). @@ -39,27 +38,16 @@ fi cd "$SRC" -# 2. Runner venv — mirrors the prod systemd setup unit's -# ``UV_PROJECT_ENVIRONMENT=/opt/runner-venv uv sync --frozen``. -# ``$RUNNER_VENV`` is backed by a docker NAMED VOLUME (not a path -# inside the container), so the venv lives on the host daemon's -# filesystem — that lets the worker bind-mount the same named -# volume into build containers it spawns. -# -# Idempotent: if the venv already matches uv.lock from a previous -# container start, this is fast. -echo "[isolated-builder] Syncing runner venv at $RUNNER_VENV ..." -UV_PROJECT_ENVIRONMENT="$RUNNER_VENV" uv sync --frozen - -# 3. Worker venv. Dev omits newrelic/sentry-sdk (no observability in -# dev). The worker venv stays inside the dev container (no host -# visibility needed — the celery process here is the only one that -# uses it). +# 2. Worker venv. Dev omits newrelic/sentry-sdk (no observability in +# dev). The runner venv is NOT created here — the build container +# spawned by the worker creates its own at startup, because a venv +# built on this host references this host's Python interpreter +# (won't resolve inside the spawned readthedocs/build container). echo "[isolated-builder] Creating worker venv at $WORKER_VENV ..." uv venv --clear "$WORKER_VENV" uv pip install --python "$WORKER_VENV/bin/python" -r "$SRC/worker/requirements.txt" -# 4. Replace this process with the Celery worker. PYTHONPATH so +# 3. Replace this process with the Celery worker. PYTHONPATH so # ``-A worker.celery`` resolves; --max-tasks-per-child=1 so the # worker exits after one task (matches prod's ephemeral pattern, # even though there's no AWS API call to terminate anything in From 2b2b025c824f13c8a5995fc761979af8b2e49a02 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 11 Jun 2026 16:21:24 +0200 Subject: [PATCH 07/23] venv create at AMI boot --- dockerfiles/docker-compose.yml | 33 +++++++++++++++++---- dockerfiles/entrypoints/isolated-builder.sh | 26 ++++++++++++---- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml index ccb83a9..fb57ddd 100644 --- a/dockerfiles/docker-compose.yml +++ b/dockerfiles/docker-compose.yml @@ -4,6 +4,16 @@ volumes: storage_rustfs_data: postgres_data_16: postgres_backups_data: + # Pre-built runner venv + uv-managed Python for the + # isolated-builder service. Populated by that service's entrypoint + # at compose-up time (idempotent — fast on subsequent restarts) and + # bind-mounted into each build container the worker spawns. We use + # named volumes (instead of bind-mounted host paths) so the host + # docker daemon owns the storage, which lets the worker reference + # them by name (``community_runner-venv`` / ``community_uv-python``) + # when it does ``docker run -v ...``. + runner-venv: + uv-python: services: @@ -388,6 +398,15 @@ services: # restarting the container. - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}:/opt/readthedocs-builder + # Runner venv + uv-managed Python (named volumes, see top of + # this file). The entrypoint populates both via ``uv sync + # --python 3.12 --python-preference=only-managed``; the worker + # then bind-mounts them into each build container by the volume + # names ``community_runner-venv`` / ``community_uv-python`` + # (passed through RTD_HOST_*_PATH below). + - runner-venv:/opt/runner-venv + - uv-python:/opt/uv-python + # The entrypoint script itself, bind-mounted so edits don't # need a rebuild (matches how celery.sh / build.sh wire up). - ${PWD}/common/dockerfiles/entrypoints/isolated-builder.sh:/usr/src/app/docker/isolated-builder.sh @@ -409,12 +428,14 @@ services: # host's CWD (where ``docker compose`` runs), so this resolves # to the same checkout the bind-mount above points at. - RTD_HOST_BUILDER_PATH=${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder} - # Host path to ``uv``. The build container's entrypoint runs - # ``uv sync`` to create its runner venv at startup, but the - # upstream readthedocs/build image doesn't have uv — the worker - # bind-mounts the host's binary in. Override per host if your - # uv lives somewhere other than /usr/local/bin/uv. - - RTD_HOST_UV_PATH=${RTDDEV_HOST_UV_PATH:-/usr/local/bin/uv} + # Runner venv + managed Python install: host-daemon-recognized + # names of the named volumes defined at the top of this file. + # Compose prefixes volume names with the project (``community`` + # for the default community-edition project). The worker passes + # these as ``-v ::ro`` and the daemon resolves them + # as named-volume mounts. + - RTD_HOST_RUNNER_VENV_PATH=community_runner-venv + - RTD_HOST_PYTHON_INSTALL_PATH=community_uv-python # Have build containers join the compose network so they can # reach the API at ``http://web`` via service-name DNS. - RTD_BUILD_NETWORK=community_readthedocs diff --git a/dockerfiles/entrypoints/isolated-builder.sh b/dockerfiles/entrypoints/isolated-builder.sh index 75d0aa0..25e9cac 100755 --- a/dockerfiles/entrypoints/isolated-builder.sh +++ b/dockerfiles/entrypoints/isolated-builder.sh @@ -19,6 +19,8 @@ set -euo pipefail : "${RTD_BUILDER_TOKEN:=}" SRC="/opt/readthedocs-builder" +RUNNER_VENV="/opt/runner-venv" +UV_PYTHON_DIR="/opt/uv-python" WORKER_VENV="/opt/worker-venv" # 1. Clone (or skip if the host's checkout is bind-mounted in). @@ -38,16 +40,28 @@ fi cd "$SRC" -# 2. Worker venv. Dev omits newrelic/sentry-sdk (no observability in -# dev). The runner venv is NOT created here — the build container -# spawned by the worker creates its own at startup, because a venv -# built on this host references this host's Python interpreter -# (won't resolve inside the spawned readthedocs/build container). +# 2. Pre-build the runner venv against a uv-managed Python 3.12. +# Same flags as the prod systemd setup unit. Both $RUNNER_VENV and +# $UV_PYTHON_DIR are backed by docker NAMED VOLUMES (see compose), +# so the venv's bin/python symlink into $UV_PYTHON_DIR lives on +# the host daemon's filesystem — that's what lets the worker +# bind-mount the same named volumes into build containers it spawns +# and have the symlink still resolve. +# +# Idempotent: ``uv sync --frozen`` is a no-op when the venv +# already matches uv.lock from a previous run. +echo "[isolated-builder] Syncing runner venv at $RUNNER_VENV (managed Python under $UV_PYTHON_DIR) ..." +UV_PYTHON_INSTALL_DIR="$UV_PYTHON_DIR" \ +UV_PROJECT_ENVIRONMENT="$RUNNER_VENV" \ + uv sync --frozen --python 3.12 --python-preference=only-managed + +# 3. Worker venv. Dev omits newrelic/sentry-sdk (no observability in +# dev). echo "[isolated-builder] Creating worker venv at $WORKER_VENV ..." uv venv --clear "$WORKER_VENV" uv pip install --python "$WORKER_VENV/bin/python" -r "$SRC/worker/requirements.txt" -# 3. Replace this process with the Celery worker. PYTHONPATH so +# 4. Replace this process with the Celery worker. PYTHONPATH so # ``-A worker.celery`` resolves; --max-tasks-per-child=1 so the # worker exits after one task (matches prod's ephemeral pattern, # even though there's no AWS API call to terminate anything in From 05bc7640ce18a8514dc9d2db2a9c31a330d30156 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 15 Jun 2026 12:56:38 +0200 Subject: [PATCH 08/23] Comment about variables --- dockerfiles/entrypoints/isolated-builder.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dockerfiles/entrypoints/isolated-builder.sh b/dockerfiles/entrypoints/isolated-builder.sh index 25e9cac..eefb09d 100755 --- a/dockerfiles/entrypoints/isolated-builder.sh +++ b/dockerfiles/entrypoints/isolated-builder.sh @@ -12,7 +12,10 @@ set -euo pipefail +# Required variable : "${RTD_BROKER_URL:?RTD_BROKER_URL must be set (e.g. redis://cache:6379/0)}" + +# Optional variables with defaults : "${RTD_BUILDS_QUEUE:=isolated-builds}" : "${RTD_BUILDER_REPO:=https://github.com/readthedocs/readthedocs-builder.git}" : "${RTD_BUILDER_REF:=celery-on-ec2}" From b7af14a61340c0201603dbe568f3502867b70260 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 15 Jun 2026 12:56:45 +0200 Subject: [PATCH 09/23] Create the worker venv with Python 3.12 --- dockerfiles/entrypoints/isolated-builder.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/entrypoints/isolated-builder.sh b/dockerfiles/entrypoints/isolated-builder.sh index eefb09d..21c3cb9 100755 --- a/dockerfiles/entrypoints/isolated-builder.sh +++ b/dockerfiles/entrypoints/isolated-builder.sh @@ -61,7 +61,7 @@ UV_PROJECT_ENVIRONMENT="$RUNNER_VENV" \ # 3. Worker venv. Dev omits newrelic/sentry-sdk (no observability in # dev). echo "[isolated-builder] Creating worker venv at $WORKER_VENV ..." -uv venv --clear "$WORKER_VENV" +uv venv --clear --python 3.12 "$WORKER_VENV" uv pip install --python "$WORKER_VENV/bin/python" -r "$SRC/worker/requirements.txt" # 4. Replace this process with the Celery worker. PYTHONPATH so From 6c5a650d3d5d7c6c2554474bced20a649ff6f47d Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 15 Jun 2026 13:01:16 +0200 Subject: [PATCH 10/23] Skip self terminate --- dockerfiles/docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml index fb57ddd..865e1a3 100644 --- a/dockerfiles/docker-compose.yml +++ b/dockerfiles/docker-compose.yml @@ -439,6 +439,10 @@ services: # Have build containers join the compose network so they can # reach the API at ``http://web`` via service-name DNS. - RTD_BUILD_NETWORK=community_readthedocs + # Skip the EC2 IMDS metadata fetch + AWS terminate call on + # task_postrun (we're obviously not on EC2). Without this the + # worker hits a 2 s IMDS timeout after every build. + - RTD_SKIP_SELF_TERMINATE=1 stdin_open: true tty: true networks: From 27fcd1b887b356929ce9a7139709b2a249b0e052 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 15 Jun 2026 13:01:22 +0200 Subject: [PATCH 11/23] Remove build service when consolidated --- dockerfiles/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml index 865e1a3..51a0a33 100644 --- a/dockerfiles/docker-compose.yml +++ b/dockerfiles/docker-compose.yml @@ -328,6 +328,7 @@ services: readthedocs: command: ["../../docker/celery-beat.sh"] + # TODO: remove `build` service once we are consolidated into `isolated-builder`. build: stop_grace_period: 1s volumes: From 5b3f0703ad9c1abdf8138a14e574e2ba267db203 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 15 Jun 2026 13:34:12 +0200 Subject: [PATCH 12/23] Run the isolated builder worker with nodemon reload --- dockerfiles/docker-compose.yml | 5 +++++ dockerfiles/entrypoints/isolated-builder.sh | 14 +++++++++----- dockerfiles/nodemon.json | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml index 51a0a33..e25dc3b 100644 --- a/dockerfiles/docker-compose.yml +++ b/dockerfiles/docker-compose.yml @@ -415,11 +415,16 @@ services: # docker-out-of-docker: the worker calls ``docker run`` against # the host daemon to spawn build containers. - /var/run/docker.sock:/var/run/docker.sock + + # The worker can optionally run with nodemon reload in dev, + # so include the config for that (matches the other services). + - ${PWD}/common/dockerfiles/nodemon.json:/usr/src/app/checkouts/nodemon.json links: - cache - storage - web environment: + - DOCKER_NO_RELOAD - RTD_BROKER_URL=redis://:redispassword@cache:6379/0 - RTD_BUILDS_QUEUE=isolated-builds - RTD_BUILDER_REF=celery-on-ec2 diff --git a/dockerfiles/entrypoints/isolated-builder.sh b/dockerfiles/entrypoints/isolated-builder.sh index 21c3cb9..73a8b57 100755 --- a/dockerfiles/entrypoints/isolated-builder.sh +++ b/dockerfiles/entrypoints/isolated-builder.sh @@ -72,8 +72,12 @@ uv pip install --python "$WORKER_VENV/bin/python" -r "$SRC/worker/requirements.t # restart or not). echo "[isolated-builder] Starting Celery worker on queue '$RTD_BUILDS_QUEUE' ..." export PYTHONPATH="$SRC" -exec "$WORKER_VENV/bin/celery" -A worker.celery worker \ - --loglevel=INFO \ - --concurrency=1 \ - --max-tasks-per-child=1 \ - -Q "$RTD_BUILDS_QUEUE" + +CMD="$WORKER_VENV/bin/celery -A worker.celery worker --loglevel=INFO --concurrency=1 --max-tasks-per-child=1 -Q ${RTD_BUILDS_QUEUE}" +if [ -n "${DOCKER_NO_RELOAD}" ]; then + echo "Running process with no reload" + exec $CMD +else + echo "Running process with reload" + exec nodemon --config /usr/src/app/checkouts/nodemon.json --exec $CMD +fi \ No newline at end of file diff --git a/dockerfiles/nodemon.json b/dockerfiles/nodemon.json index 716227b..5cf1f8d 100644 --- a/dockerfiles/nodemon.json +++ b/dockerfiles/nodemon.json @@ -2,7 +2,7 @@ "verbose": false, "delay": 2000, "ext": "py", - "watch": ["readthedocs", "readthedocsinc", "../readthedocs.org/readthedocs", "../readthedocs-ext/readthedocsext"], + "watch": ["readthedocs", "readthedocsinc", "../readthedocs.org/readthedocs", "../readthedocs-ext/readthedocsext", "/opt/readthedocs-builder"], "ignore": [".tox/*", ".direnv/*", "user_builds/*", "*/management/commands*", "*migrations/*", "*test*", "*.pyc", "*.pyo", "logs/*"], "signal": "SIGTERM" } From 5b5c0078ea2edba2e90bc8fd1561f5a97547b071 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 14 Jul 2026 10:17:26 +0200 Subject: [PATCH 13/23] Use Python 3.14 --- dockerfiles/entrypoints/isolated-builder.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dockerfiles/entrypoints/isolated-builder.sh b/dockerfiles/entrypoints/isolated-builder.sh index 73a8b57..8a6f0a7 100755 --- a/dockerfiles/entrypoints/isolated-builder.sh +++ b/dockerfiles/entrypoints/isolated-builder.sh @@ -43,7 +43,7 @@ fi cd "$SRC" -# 2. Pre-build the runner venv against a uv-managed Python 3.12. +# 2. Pre-build the runner venv against a uv-managed Python 3.14. # Same flags as the prod systemd setup unit. Both $RUNNER_VENV and # $UV_PYTHON_DIR are backed by docker NAMED VOLUMES (see compose), # so the venv's bin/python symlink into $UV_PYTHON_DIR lives on @@ -56,12 +56,12 @@ cd "$SRC" echo "[isolated-builder] Syncing runner venv at $RUNNER_VENV (managed Python under $UV_PYTHON_DIR) ..." UV_PYTHON_INSTALL_DIR="$UV_PYTHON_DIR" \ UV_PROJECT_ENVIRONMENT="$RUNNER_VENV" \ - uv sync --frozen --python 3.12 --python-preference=only-managed + uv sync --frozen --python 3.14 --python-preference=only-managed # 3. Worker venv. Dev omits newrelic/sentry-sdk (no observability in # dev). echo "[isolated-builder] Creating worker venv at $WORKER_VENV ..." -uv venv --clear --python 3.12 "$WORKER_VENV" +uv venv --clear --python 3.14 "$WORKER_VENV" uv pip install --python "$WORKER_VENV/bin/python" -r "$SRC/worker/requirements.txt" # 4. Replace this process with the Celery worker. PYTHONPATH so From 462e8cdd91866ada7ad37a343e45d7feb1b8ffc6 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 14 Jul 2026 10:21:04 +0200 Subject: [PATCH 14/23] Dockerfile for isolated builders --- dockerfiles/Dockerfile.isolated-builder | 68 +++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 dockerfiles/Dockerfile.isolated-builder diff --git a/dockerfiles/Dockerfile.isolated-builder b/dockerfiles/Dockerfile.isolated-builder new file mode 100644 index 0000000..65126c8 --- /dev/null +++ b/dockerfiles/Dockerfile.isolated-builder @@ -0,0 +1,68 @@ +# Dev-only image emulating the production isolated-builders AMI. +# +# Built for: docker-compose service that runs the readthedocs-builder +# worker package against the local broker, picks up tasks from the +# ``isolated-builds`` queue, and spawns build containers on the host +# docker daemon. The companion entrypoint +# (``common/dockerfiles/entrypoints/isolated-builder.sh``) does the +# AMI's bake-time + boot-time work at container start, then ``exec``s +# the Celery worker — no systemd, no New Relic, no Sentry (this is dev). +# +# What this image installs corresponds to the Packer template's +# bake-time steps: +# - git, python3.14, ca-certificates (AL2023 base + dnf step) +# - docker CLI (docker-in-docker for build dispatch) +# - uv at /usr/local/bin/uv (same path as the AMI) +# - empty /opt/{readthedocs-builder,runner-venv,worker-venv} dirs +# (filled by the entrypoint OR a bind-mount from the host; +# matches the AMI's pre-created dirs owned by ``docs``) +# +# The host's readthedocs-builder checkout is meant to be bind-mounted +# to /opt/readthedocs-builder; the runner venv bind-mounts to a host +# path the daemon can also see (so the worker's `docker run -v ...` +# resolves correctly when spawning build containers). See the +# entrypoint header for the full compose wiring. + +FROM ubuntu:26.04 + +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=C.UTF-8 +ENV LC_ALL=C.UTF-8 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + git \ + python3.14 \ + python3.14-venv \ + docker.io \ + nodejs \ + npm \ + && rm -rf /var/lib/apt/lists/* + +# Symlink so ``python`` works without a version suffix (matches the AMI, +# where ``python3.14`` is the default and ``uv`` resolves it via PATH). +RUN ln -sf /usr/bin/python3.14 /usr/local/bin/python + +# Install uv at the same path the AMI uses (/usr/local/bin/uv) so the +# entrypoint script is identical to the production setup unit. +RUN curl -LsSf https://astral.sh/uv/install.sh | UV_INSTALL_DIR=/usr/local/bin sh + +# Install nodemon globally so the entrypoint can optionally run the +# Celery worker with reload (dev-only, not in the AMI). +RUN npm install -g nodemon + +# Pre-create the same directories the AMI's Packer step provisions. +# /opt/readthedocs-builder is replaced by a compose bind-mount of +# the host's checkout. /opt/runner-venv + /opt/uv-python are replaced +# by compose named volumes the entrypoint populates with uv. The +# entrypoint also populates /opt/worker-venv (container-local). +RUN mkdir -p /opt/readthedocs-builder /opt/runner-venv /opt/uv-python /opt/worker-venv + +WORKDIR /opt/readthedocs-builder + +# No ENTRYPOINT or CMD — the compose service supplies +# ``command: ["/usr/src/app/docker/isolated-builder.sh"]`` after +# bind-mounting the script. Keeping it out of the image means +# editing the entrypoint doesn't require a rebuild (matches how +# ``web``, ``celery``, ``build``, etc. wire up their scripts). From 173aad9a456b411452c68c249635bfd602012b81 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 14 Jul 2026 10:50:15 +0200 Subject: [PATCH 15/23] Dockerfile path and Python 3.14 --- dockerfiles/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml index e25dc3b..ec47719 100644 --- a/dockerfiles/docker-compose.yml +++ b/dockerfiles/docker-compose.yml @@ -391,7 +391,7 @@ services: stop_grace_period: 1s build: context: ${PWD} - dockerfile: ${PWD}/dockerfiles/Dockerfile.isolated-builder + dockerfile: ${PWD}/common/dockerfiles/Dockerfile.isolated-builder volumes: # Host's readthedocs-builder checkout. The entrypoint sees the # mount as populated and skips ``git clone`` so you can iterate @@ -401,7 +401,7 @@ services: # Runner venv + uv-managed Python (named volumes, see top of # this file). The entrypoint populates both via ``uv sync - # --python 3.12 --python-preference=only-managed``; the worker + # --python 3.14 --python-preference=only-managed``; the worker # then bind-mounts them into each build container by the volume # names ``community_runner-venv`` / ``community_uv-python`` # (passed through RTD_HOST_*_PATH below). From 37a2fb2fff733e0feaed3f5c7dda421ee20e47d3 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 14 Jul 2026 18:00:35 +0200 Subject: [PATCH 16/23] Use Python packages/projects --- dockerfiles/entrypoints/isolated-builder.sh | 42 ++++++++++----------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/dockerfiles/entrypoints/isolated-builder.sh b/dockerfiles/entrypoints/isolated-builder.sh index 8a6f0a7..790f86a 100755 --- a/dockerfiles/entrypoints/isolated-builder.sh +++ b/dockerfiles/entrypoints/isolated-builder.sh @@ -41,37 +41,37 @@ else echo "[isolated-builder] $SRC already populated; skipping clone (dev bind-mount)." fi -cd "$SRC" - -# 2. Pre-build the runner venv against a uv-managed Python 3.14. -# Same flags as the prod systemd setup unit. Both $RUNNER_VENV and -# $UV_PYTHON_DIR are backed by docker NAMED VOLUMES (see compose), -# so the venv's bin/python symlink into $UV_PYTHON_DIR lives on -# the host daemon's filesystem — that's what lets the worker -# bind-mount the same named volumes into build containers it spawns -# and have the symlink still resolve. +# 2. Pre-build the runner venv against a uv-managed Python 3.14 by +# syncing the builder/ project. Same flags as the prod systemd setup +# unit. Both $RUNNER_VENV and $UV_PYTHON_DIR are backed by docker +# NAMED VOLUMES (see compose), so the venv's bin/python symlink into +# $UV_PYTHON_DIR lives on the host daemon's filesystem — that's what +# lets the worker bind-mount the same named volumes into build +# containers it spawns and have the symlink still resolve. # # Idempotent: ``uv sync --frozen`` is a no-op when the venv # already matches uv.lock from a previous run. echo "[isolated-builder] Syncing runner venv at $RUNNER_VENV (managed Python under $UV_PYTHON_DIR) ..." +cd "$SRC/builder" UV_PYTHON_INSTALL_DIR="$UV_PYTHON_DIR" \ UV_PROJECT_ENVIRONMENT="$RUNNER_VENV" \ uv sync --frozen --python 3.14 --python-preference=only-managed -# 3. Worker venv. Dev omits newrelic/sentry-sdk (no observability in -# dev). -echo "[isolated-builder] Creating worker venv at $WORKER_VENV ..." -uv venv --clear --python 3.14 "$WORKER_VENV" -uv pip install --python "$WORKER_VENV/bin/python" -r "$SRC/worker/requirements.txt" +# 3. Worker venv — sync the worker/ project. Dev omits the +# ``observability`` extra (no New Relic / Sentry in dev). +echo "[isolated-builder] Syncing worker venv at $WORKER_VENV ..." +cd "$SRC/worker" +UV_PROJECT_ENVIRONMENT="$WORKER_VENV" \ + uv sync --frozen --python 3.14 -# 4. Replace this process with the Celery worker. PYTHONPATH so -# ``-A worker.celery`` resolves; --max-tasks-per-child=1 so the -# worker exits after one task (matches prod's ephemeral pattern, -# even though there's no AWS API call to terminate anything in -# dev — the worker just exits and compose can be configured to -# restart or not). +# 4. Replace this process with the Celery worker. PYTHONPATH points at +# the worker/ project dir so ``-A worker.celery`` resolves from the +# live source; --max-tasks-per-child=1 so the worker exits after one +# task (matches prod's ephemeral pattern, even though there's no AWS +# API call to terminate anything in dev — the worker just exits and +# compose can be configured to restart or not). echo "[isolated-builder] Starting Celery worker on queue '$RTD_BUILDS_QUEUE' ..." -export PYTHONPATH="$SRC" +export PYTHONPATH="$SRC/worker" CMD="$WORKER_VENV/bin/celery -A worker.celery worker --loglevel=INFO --concurrency=1 --max-tasks-per-child=1 -Q ${RTD_BUILDS_QUEUE}" if [ -n "${DOCKER_NO_RELOAD}" ]; then From 323e1138b5b64d35ce6bb7d1ceaddcba3834035f Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 15 Jul 2026 10:34:54 +0200 Subject: [PATCH 17/23] Standardize on PATH to mount in both environments (development and production) --- dockerfiles/Dockerfile.isolated-builder | 6 ++- dockerfiles/docker-compose.yml | 46 +++++++++------------ dockerfiles/entrypoints/isolated-builder.sh | 10 ++--- 3 files changed, 29 insertions(+), 33 deletions(-) diff --git a/dockerfiles/Dockerfile.isolated-builder b/dockerfiles/Dockerfile.isolated-builder index 65126c8..856bbb6 100644 --- a/dockerfiles/Dockerfile.isolated-builder +++ b/dockerfiles/Dockerfile.isolated-builder @@ -55,8 +55,10 @@ RUN npm install -g nodemon # Pre-create the same directories the AMI's Packer step provisions. # /opt/readthedocs-builder is replaced by a compose bind-mount of # the host's checkout. /opt/runner-venv + /opt/uv-python are replaced -# by compose named volumes the entrypoint populates with uv. The -# entrypoint also populates /opt/worker-venv (container-local). +# by compose bind-mounts of host paths the entrypoint populates with +# uv (host paths, like production, so the worker can re-mount them +# into build containers). The entrypoint also populates +# /opt/worker-venv (container-local). RUN mkdir -p /opt/readthedocs-builder /opt/runner-venv /opt/uv-python /opt/worker-venv WORKDIR /opt/readthedocs-builder diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml index ec47719..32d5e19 100644 --- a/dockerfiles/docker-compose.yml +++ b/dockerfiles/docker-compose.yml @@ -4,16 +4,6 @@ volumes: storage_rustfs_data: postgres_data_16: postgres_backups_data: - # Pre-built runner venv + uv-managed Python for the - # isolated-builder service. Populated by that service's entrypoint - # at compose-up time (idempotent — fast on subsequent restarts) and - # bind-mounted into each build container the worker spawns. We use - # named volumes (instead of bind-mounted host paths) so the host - # docker daemon owns the storage, which lets the worker reference - # them by name (``community_runner-venv`` / ``community_uv-python``) - # when it does ``docker run -v ...``. - runner-venv: - uv-python: services: @@ -399,14 +389,19 @@ services: # restarting the container. - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}:/opt/readthedocs-builder - # Runner venv + uv-managed Python (named volumes, see top of - # this file). The entrypoint populates both via ``uv sync - # --python 3.14 --python-preference=only-managed``; the worker - # then bind-mounts them into each build container by the volume - # names ``community_runner-venv`` / ``community_uv-python`` - # (passed through RTD_HOST_*_PATH below). - - runner-venv:/opt/runner-venv - - uv-python:/opt/uv-python + # Runner venv + uv-managed Python. Bind-mounted from host paths, + # where these are + # real host directories the worker passes to ``docker run -v``. + # They live inside the host's readthedocs-builder checkout (same + # base path as RTD_HOST_BUILDER_PATH). + # The entrypoint populates both via ``uv sync --python 3.14 + # --python-preference=only-managed``; because they live on the + # host filesystem, the worker can bind-mount the SAME host paths + # into each build container it spawns (passed through + # RTD_HOST_*_PATH below) and the venv's bin/python symlink into + # /opt/uv-python still resolves. + - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}/runner-venv:/opt/runner-venv + - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}/uv-python:/opt/uv-python # The entrypoint script itself, bind-mounted so edits don't # need a rebuild (matches how celery.sh / build.sh wire up). @@ -434,14 +429,13 @@ services: # host's CWD (where ``docker compose`` runs), so this resolves # to the same checkout the bind-mount above points at. - RTD_HOST_BUILDER_PATH=${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder} - # Runner venv + managed Python install: host-daemon-recognized - # names of the named volumes defined at the top of this file. - # Compose prefixes volume names with the project (``community`` - # for the default community-edition project). The worker passes - # these as ``-v ::ro`` and the daemon resolves them - # as named-volume mounts. - - RTD_HOST_RUNNER_VENV_PATH=community_runner-venv - - RTD_HOST_PYTHON_INSTALL_PATH=community_uv-python + # Runner venv + managed Python install: host-visible paths (the + # same directories bind-mounted into this service above). The + # worker passes these as ``-v ::ro`` and the + # daemon resolves them as bind mounts — identical to production, + # where these point at ``/opt/runner-venv`` / ``/opt/uv-python``. + - RTD_HOST_RUNNER_VENV_PATH=${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}/runner-venv + - RTD_HOST_PYTHON_INSTALL_PATH=${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}/uv-python # Have build containers join the compose network so they can # reach the API at ``http://web`` via service-name DNS. - RTD_BUILD_NETWORK=community_readthedocs diff --git a/dockerfiles/entrypoints/isolated-builder.sh b/dockerfiles/entrypoints/isolated-builder.sh index 790f86a..5f22b2e 100755 --- a/dockerfiles/entrypoints/isolated-builder.sh +++ b/dockerfiles/entrypoints/isolated-builder.sh @@ -43,11 +43,11 @@ fi # 2. Pre-build the runner venv against a uv-managed Python 3.14 by # syncing the builder/ project. Same flags as the prod systemd setup -# unit. Both $RUNNER_VENV and $UV_PYTHON_DIR are backed by docker -# NAMED VOLUMES (see compose), so the venv's bin/python symlink into -# $UV_PYTHON_DIR lives on the host daemon's filesystem — that's what -# lets the worker bind-mount the same named volumes into build -# containers it spawns and have the symlink still resolve. +# unit. Both $RUNNER_VENV and $UV_PYTHON_DIR are bind-mounted from +# host paths (see compose), so the venv's bin/python symlink into +# $UV_PYTHON_DIR lives on the host filesystem — that's what lets the +# worker bind-mount the same host paths into build containers it +# spawns and have the symlink still resolve (matches production). # # Idempotent: ``uv sync --frozen`` is a no-op when the venv # already matches uv.lock from a previous run. From 0762a416ca4e0eb0bbba822e9ed105878a22a880 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 15 Jul 2026 10:56:30 +0200 Subject: [PATCH 18/23] Use `/usr/src/builder` structure pattern --- dockerfiles/Dockerfile.isolated-builder | 17 +++++++++-------- dockerfiles/docker-compose.yml | 16 ++++++++-------- dockerfiles/entrypoints/isolated-builder.sh | 10 +++++----- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/dockerfiles/Dockerfile.isolated-builder b/dockerfiles/Dockerfile.isolated-builder index 856bbb6..53f1ca3 100644 --- a/dockerfiles/Dockerfile.isolated-builder +++ b/dockerfiles/Dockerfile.isolated-builder @@ -13,12 +13,13 @@ # - git, python3.14, ca-certificates (AL2023 base + dnf step) # - docker CLI (docker-in-docker for build dispatch) # - uv at /usr/local/bin/uv (same path as the AMI) -# - empty /opt/{readthedocs-builder,runner-venv,worker-venv} dirs +# - empty /usr/src/builder/checkouts/readthedocs-builder + +# /usr/src/builder/{runner-venv,uv-python,worker-venv} dirs # (filled by the entrypoint OR a bind-mount from the host; # matches the AMI's pre-created dirs owned by ``docs``) # # The host's readthedocs-builder checkout is meant to be bind-mounted -# to /opt/readthedocs-builder; the runner venv bind-mounts to a host +# to /usr/src/builder/checkouts/readthedocs-builder; the runner venv bind-mounts to a host # path the daemon can also see (so the worker's `docker run -v ...` # resolves correctly when spawning build containers). See the # entrypoint header for the full compose wiring. @@ -53,18 +54,18 @@ RUN curl -LsSf https://astral.sh/uv/install.sh | UV_INSTALL_DIR=/usr/local/bin s RUN npm install -g nodemon # Pre-create the same directories the AMI's Packer step provisions. -# /opt/readthedocs-builder is replaced by a compose bind-mount of -# the host's checkout. /opt/runner-venv + /opt/uv-python are replaced +# /usr/src/builder/checkouts/readthedocs-builder is replaced by a compose bind-mount of +# the host's checkout. /usr/src/builder/runner-venv + /usr/src/builder/uv-python are replaced # by compose bind-mounts of host paths the entrypoint populates with # uv (host paths, like production, so the worker can re-mount them # into build containers). The entrypoint also populates -# /opt/worker-venv (container-local). -RUN mkdir -p /opt/readthedocs-builder /opt/runner-venv /opt/uv-python /opt/worker-venv +# /usr/src/builder/worker-venv (container-local). +RUN mkdir -p /usr/src/builder/checkouts/readthedocs-builder /usr/src/builder/runner-venv /usr/src/builder/uv-python /usr/src/builder/worker-venv -WORKDIR /opt/readthedocs-builder +WORKDIR /usr/src/builder/checkouts/readthedocs-builder # No ENTRYPOINT or CMD — the compose service supplies -# ``command: ["/usr/src/app/docker/isolated-builder.sh"]`` after +# ``command: ["/usr/src/builder/docker/isolated-builder.sh"]`` after # bind-mounting the script. Keeping it out of the image means # editing the entrypoint doesn't require a rebuild (matches how # ``web``, ``celery``, ``build``, etc. wire up their scripts). diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml index 32d5e19..3a5b3a7 100644 --- a/dockerfiles/docker-compose.yml +++ b/dockerfiles/docker-compose.yml @@ -387,7 +387,7 @@ services: # mount as populated and skips ``git clone`` so you can iterate # on the runner / worker code by editing on the host and # restarting the container. - - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}:/opt/readthedocs-builder + - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}:/usr/src/builder/checkouts/readthedocs-builder # Runner venv + uv-managed Python. Bind-mounted from host paths, # where these are @@ -399,13 +399,13 @@ services: # host filesystem, the worker can bind-mount the SAME host paths # into each build container it spawns (passed through # RTD_HOST_*_PATH below) and the venv's bin/python symlink into - # /opt/uv-python still resolves. - - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}/runner-venv:/opt/runner-venv - - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}/uv-python:/opt/uv-python + # /usr/src/builder/uv-python still resolves. + - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}/runner-venv:/usr/src/builder/runner-venv + - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}/uv-python:/usr/src/builder/uv-python # The entrypoint script itself, bind-mounted so edits don't # need a rebuild (matches how celery.sh / build.sh wire up). - - ${PWD}/common/dockerfiles/entrypoints/isolated-builder.sh:/usr/src/app/docker/isolated-builder.sh + - ${PWD}/common/dockerfiles/entrypoints/isolated-builder.sh:/usr/src/builder/docker/isolated-builder.sh # docker-out-of-docker: the worker calls ``docker run`` against # the host daemon to spawn build containers. @@ -413,7 +413,7 @@ services: # The worker can optionally run with nodemon reload in dev, # so include the config for that (matches the other services). - - ${PWD}/common/dockerfiles/nodemon.json:/usr/src/app/checkouts/nodemon.json + - ${PWD}/common/dockerfiles/nodemon.json:/usr/src/builder/checkouts/nodemon.json links: - cache - storage @@ -433,7 +433,7 @@ services: # same directories bind-mounted into this service above). The # worker passes these as ``-v ::ro`` and the # daemon resolves them as bind mounts — identical to production, - # where these point at ``/opt/runner-venv`` / ``/opt/uv-python``. + # where these point at ``/usr/src/builder/runner-venv`` / ``/usr/src/builder/uv-python``. - RTD_HOST_RUNNER_VENV_PATH=${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}/runner-venv - RTD_HOST_PYTHON_INSTALL_PATH=${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}/uv-python # Have build containers join the compose network so they can @@ -447,7 +447,7 @@ services: tty: true networks: readthedocs: - command: ["/usr/src/app/docker/isolated-builder.sh"] + command: ["/usr/src/builder/docker/isolated-builder.sh"] cache: image: redis:6.2.6 # Match production version diff --git a/dockerfiles/entrypoints/isolated-builder.sh b/dockerfiles/entrypoints/isolated-builder.sh index 5f22b2e..dc2c122 100755 --- a/dockerfiles/entrypoints/isolated-builder.sh +++ b/dockerfiles/entrypoints/isolated-builder.sh @@ -21,10 +21,10 @@ set -euo pipefail : "${RTD_BUILDER_REF:=celery-on-ec2}" : "${RTD_BUILDER_TOKEN:=}" -SRC="/opt/readthedocs-builder" -RUNNER_VENV="/opt/runner-venv" -UV_PYTHON_DIR="/opt/uv-python" -WORKER_VENV="/opt/worker-venv" +SRC="/usr/src/builder/checkouts/readthedocs-builder" +RUNNER_VENV="/usr/src/builder/runner-venv" +UV_PYTHON_DIR="/usr/src/builder/uv-python" +WORKER_VENV="/usr/src/builder/worker-venv" # 1. Clone (or skip if the host's checkout is bind-mounted in). # A bind-mount means $SRC is already populated; we skip ``git clone`` @@ -79,5 +79,5 @@ if [ -n "${DOCKER_NO_RELOAD}" ]; then exec $CMD else echo "Running process with reload" - exec nodemon --config /usr/src/app/checkouts/nodemon.json --exec $CMD + exec nodemon --config /usr/src/builder/checkouts/nodemon.json --exec $CMD fi \ No newline at end of file From 37cb86913e45709db53007c2ef87aa916d8a3911 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 15 Jul 2026 11:13:10 +0200 Subject: [PATCH 19/23] Simplify comments --- dockerfiles/docker-compose.yml | 40 +++------------------------------- 1 file changed, 3 insertions(+), 37 deletions(-) diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml index 3a5b3a7..63f16c5 100644 --- a/dockerfiles/docker-compose.yml +++ b/dockerfiles/docker-compose.yml @@ -371,48 +371,20 @@ services: command: ["../../docker/build.sh"] isolated-builder: - # Dev emulation of the production isolated-builders AMI. Picks up - # tasks from the ``isolated-builds`` queue and spawns build - # containers on the host docker daemon (via the bind-mounted - # socket). The companion entrypoint clones readthedocs-builder (or - # skips if the host's checkout is bind-mounted in), creates the - # worker venv via uv, then runs Celery — no systemd, no New Relic, - # no Sentry. See readthedocs-builder/docs/architecture.md. stop_grace_period: 1s build: context: ${PWD} dockerfile: ${PWD}/common/dockerfiles/Dockerfile.isolated-builder volumes: - # Host's readthedocs-builder checkout. The entrypoint sees the - # mount as populated and skips ``git clone`` so you can iterate - # on the runner / worker code by editing on the host and - # restarting the container. - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}:/usr/src/builder/checkouts/readthedocs-builder - - # Runner venv + uv-managed Python. Bind-mounted from host paths, - # where these are - # real host directories the worker passes to ``docker run -v``. - # They live inside the host's readthedocs-builder checkout (same - # base path as RTD_HOST_BUILDER_PATH). - # The entrypoint populates both via ``uv sync --python 3.14 - # --python-preference=only-managed``; because they live on the - # host filesystem, the worker can bind-mount the SAME host paths - # into each build container it spawns (passed through - # RTD_HOST_*_PATH below) and the venv's bin/python symlink into - # /usr/src/builder/uv-python still resolves. - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}/runner-venv:/usr/src/builder/runner-venv - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}/uv-python:/usr/src/builder/uv-python - - # The entrypoint script itself, bind-mounted so edits don't - # need a rebuild (matches how celery.sh / build.sh wire up). - ${PWD}/common/dockerfiles/entrypoints/isolated-builder.sh:/usr/src/builder/docker/isolated-builder.sh # docker-out-of-docker: the worker calls ``docker run`` against # the host daemon to spawn build containers. - /var/run/docker.sock:/var/run/docker.sock - # The worker can optionally run with nodemon reload in dev, - # so include the config for that (matches the other services). - ${PWD}/common/dockerfiles/nodemon.json:/usr/src/builder/checkouts/nodemon.json links: - cache @@ -425,23 +397,17 @@ services: - RTD_BUILDER_REF=celery-on-ec2 # Source path the worker passes to ``docker run -v ...`` when # spawning build containers. Must be a HOST-visible path because - # the daemon resolves it on the host. ``${PWD}`` here is the - # host's CWD (where ``docker compose`` runs), so this resolves - # to the same checkout the bind-mount above points at. + # the daemon resolves it on the host. - RTD_HOST_BUILDER_PATH=${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder} # Runner venv + managed Python install: host-visible paths (the - # same directories bind-mounted into this service above). The - # worker passes these as ``-v ::ro`` and the - # daemon resolves them as bind mounts — identical to production, - # where these point at ``/usr/src/builder/runner-venv`` / ``/usr/src/builder/uv-python``. + # same directories bind-mounted into this service above). - RTD_HOST_RUNNER_VENV_PATH=${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}/runner-venv - RTD_HOST_PYTHON_INSTALL_PATH=${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}/uv-python # Have build containers join the compose network so they can # reach the API at ``http://web`` via service-name DNS. - RTD_BUILD_NETWORK=community_readthedocs # Skip the EC2 IMDS metadata fetch + AWS terminate call on - # task_postrun (we're obviously not on EC2). Without this the - # worker hits a 2 s IMDS timeout after every build. + # task_postrun (we're obviously not on EC2). - RTD_SKIP_SELF_TERMINATE=1 stdin_open: true tty: true From 12dc573fd91a51abf72a412c344e9a2ecaeb66de Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 15 Jul 2026 11:15:27 +0200 Subject: [PATCH 20/23] Use Python 3.14 for linting --- .github/workflows/pre-commit.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 45e7f24..a8ab69e 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -17,10 +17,10 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: "3.12" + python-version: "3.14" - name: Install uv uses: astral-sh/setup-uv@v7 - name: Run pre-commit - run: uvx --python 3.12 pre-commit run --all-files + run: uvx --python 3.14 pre-commit run --all-files From 9df47aa366c116363ea060fe66eacd21cb84c2cd Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 15 Jul 2026 11:18:32 +0200 Subject: [PATCH 21/23] Remove Docker in Docker where it's not needed --- dockerfiles/docker-compose.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml index 63f16c5..f64982b 100644 --- a/dockerfiles/docker-compose.yml +++ b/dockerfiles/docker-compose.yml @@ -128,9 +128,6 @@ services: - ${PWD}/common/dockerfiles/nodemon.json:/usr/src/app/checkouts/nodemon.json - ${PWD}/${RTDDEV_PATH_EXT:-../readthedocs-ext}:/usr/src/app/checkouts/readthedocs-ext - ${PWD}/${RTDDEV_PATH_EXT_THEME:-../ext-theme}:/usr/src/app/checkouts/ext-theme - - # Docker in Docker: cancel running builds. - - /var/run/docker.sock:/var/run/docker.sock links: - storage - database @@ -217,9 +214,6 @@ services: - ${PWD}/common/dockerfiles/nodemon.json:/usr/src/app/checkouts/nodemon.json - ${PWD}/${RTDDEV_PATH_EXT:-../readthedocs-ext}:/usr/src/app/checkouts/readthedocs-ext - ${PWD}/${RTDDEV_PATH_EXT_THEME:-../ext-theme}:/usr/src/app/checkouts/ext-theme - - # Docker in Docker: Fargate (call `docker run` from inside `web` queue) - - /var/run/docker.sock:/var/run/docker.sock links: - storage - database @@ -380,12 +374,11 @@ services: - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}/runner-venv:/usr/src/builder/runner-venv - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}/uv-python:/usr/src/builder/uv-python - ${PWD}/common/dockerfiles/entrypoints/isolated-builder.sh:/usr/src/builder/docker/isolated-builder.sh + - ${PWD}/common/dockerfiles/nodemon.json:/usr/src/builder/checkouts/nodemon.json # docker-out-of-docker: the worker calls ``docker run`` against # the host daemon to spawn build containers. - /var/run/docker.sock:/var/run/docker.sock - - - ${PWD}/common/dockerfiles/nodemon.json:/usr/src/builder/checkouts/nodemon.json links: - cache - storage From 00d65013a82265b9de3ccf3ca28b924cc4ad2f61 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 15 Jul 2026 11:19:42 +0200 Subject: [PATCH 22/23] pre-commit lint --- dockerfiles/entrypoints/isolated-builder.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/entrypoints/isolated-builder.sh b/dockerfiles/entrypoints/isolated-builder.sh index dc2c122..155cc26 100755 --- a/dockerfiles/entrypoints/isolated-builder.sh +++ b/dockerfiles/entrypoints/isolated-builder.sh @@ -80,4 +80,4 @@ if [ -n "${DOCKER_NO_RELOAD}" ]; then else echo "Running process with reload" exec nodemon --config /usr/src/builder/checkouts/nodemon.json --exec $CMD -fi \ No newline at end of file +fi From 80a3a99d7924dbb119b387ee77b90f74241f6764 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 15 Jul 2026 13:02:42 +0200 Subject: [PATCH 23/23] Rename a file --- dockerfiles/entrypoints/isolated-builder.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/entrypoints/isolated-builder.sh b/dockerfiles/entrypoints/isolated-builder.sh index 155cc26..02173ef 100755 --- a/dockerfiles/entrypoints/isolated-builder.sh +++ b/dockerfiles/entrypoints/isolated-builder.sh @@ -3,7 +3,7 @@ # # Emulates the production AMI's two systemd units — # ``readthedocs-builder-setup.service`` (clone + worker venv setup) and -# then ``celery-readthedocs-builder.service`` (start the worker) — as +# then ``readthedocs-celery-worker.service`` (start the worker) — as # one foreground bash script. No systemd, no New Relic, no Sentry: # dev only. #