Builds: isolated builders (development environment)

.github/workflows/pre-commit.yml

@@ -17,10 +17,10 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
-          python-version: "3.12"
+          python-version: "3.14"
 
       - name: Install uv
         uses: astral-sh/setup-uv@v7
 
       - name: Run pre-commit
-        run: uvx --python 3.12 pre-commit run --all-files
+        run: uvx --python 3.14 pre-commit run --all-files

dockerfiles/Dockerfile.isolated-builder

@@ -0,0 +1,71 @@
+# Dev-only image emulating the production isolated-builders AMI.
+#
+# Built for: docker-compose service that runs the readthedocs-builder
+# worker package against the local broker, picks up tasks from the
+# ``isolated-builds`` queue, and spawns build containers on the host
+# docker daemon. The companion entrypoint
+# (``common/dockerfiles/entrypoints/isolated-builder.sh``) does the
+# AMI's bake-time + boot-time work at container start, then ``exec``s
+# the Celery worker — no systemd, no New Relic, no Sentry (this is dev).
+#
+# What this image installs corresponds to the Packer template's
+# bake-time steps:
+#   - git, python3.14, ca-certificates                (AL2023 base + dnf step)
+#   - docker CLI                                      (docker-in-docker for build dispatch)
+#   - uv at /usr/local/bin/uv                         (same path as the AMI)
+#   - empty /usr/src/builder/checkouts/readthedocs-builder +
+#     /usr/src/builder/{runner-venv,uv-python,worker-venv} dirs
+#     (filled by the entrypoint OR a bind-mount from the host;
+#     matches the AMI's pre-created dirs owned by ``docs``)
+#
+# The host's readthedocs-builder checkout is meant to be bind-mounted
+# to /usr/src/builder/checkouts/readthedocs-builder; the runner venv bind-mounts to a host
+# path the daemon can also see (so the worker's `docker run -v ...`
+# resolves correctly when spawning build containers). See the
+# entrypoint header for the full compose wiring.
+
+FROM ubuntu:26.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV LANG=C.UTF-8
+ENV LC_ALL=C.UTF-8
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        ca-certificates \
+        curl \
+        git \
+        python3.14 \
+        python3.14-venv \
+        docker.io \
+        nodejs \
+        npm \
+    && rm -rf /var/lib/apt/lists/*
+
+# Symlink so ``python`` works without a version suffix (matches the AMI,
+# where ``python3.14`` is the default and ``uv`` resolves it via PATH).
+RUN ln -sf /usr/bin/python3.14 /usr/local/bin/python
+
+# Install uv at the same path the AMI uses (/usr/local/bin/uv) so the
+# entrypoint script is identical to the production setup unit.
+RUN curl -LsSf https://astral.sh/uv/install.sh | UV_INSTALL_DIR=/usr/local/bin sh
+
+# Install nodemon globally so the entrypoint can optionally run the
+# Celery worker with reload (dev-only, not in the AMI).
+RUN npm install -g nodemon
+
+# Pre-create the same directories the AMI's Packer step provisions.
+# /usr/src/builder/checkouts/readthedocs-builder is replaced by a compose bind-mount of
+# the host's checkout. /usr/src/builder/runner-venv + /usr/src/builder/uv-python are replaced
+# by compose bind-mounts of host paths the entrypoint populates with
+# uv (host paths, like production, so the worker can re-mount them
+# into build containers). The entrypoint also populates
+# /usr/src/builder/worker-venv (container-local).
+RUN mkdir -p /usr/src/builder/checkouts/readthedocs-builder /usr/src/builder/runner-venv /usr/src/builder/uv-python /usr/src/builder/worker-venv
+
+WORKDIR /usr/src/builder/checkouts/readthedocs-builder
+
+# No ENTRYPOINT or CMD — the compose service supplies
+# ``command: ["/usr/src/builder/docker/isolated-builder.sh"]`` after
+# bind-mounting the script. Keeping it out of the image means
+# editing the entrypoint doesn't require a rebuild (matches how
+# ``web``, ``celery``, ``build``, etc. wire up their scripts).

dockerfiles/docker-compose.yml

@@ -256,6 +256,7 @@ services:
       - RTD_S3_STATIC_STORAGE_BUCKET
       - RTD_AWS_S3_REGION_NAME
       - RTD_OPENAI_API_KEY
+      - RTD_BUILDER_TOKEN
     stdin_open: true
     tty: true
     networks:
@@ -311,6 +312,7 @@ services:
       readthedocs:
     command: ["../../docker/celery-beat.sh"]
 
+  # TODO: remove `build` service once we are consolidated into `isolated-builder`.
   build:
     stop_grace_period: 1s
     volumes:
@@ -327,7 +329,7 @@ services:
       # Because of this, we need to use a shared volume between them
       - build-user-builds:/usr/src/app/checkouts/readthedocs.org/user_builds
 
-      # Docker in Docker
+      # Docker in Docker: old implementation (`build-default` queue)
       - /var/run/docker.sock:/var/run/docker.sock
     links:
       - web
@@ -362,6 +364,50 @@ services:
       readthedocs:
     command: ["../../docker/build.sh"]
 
+  isolated-builder:
+    stop_grace_period: 1s
+    build:
+      context: ${PWD}
+      dockerfile: ${PWD}/common/dockerfiles/Dockerfile.isolated-builder
+    volumes:
+      - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}:/usr/src/builder/checkouts/readthedocs-builder
+      - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}/runner-venv:/usr/src/builder/runner-venv
+      - ${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}/uv-python:/usr/src/builder/uv-python
+      - ${PWD}/common/dockerfiles/entrypoints/isolated-builder.sh:/usr/src/builder/docker/isolated-builder.sh
+      - ${PWD}/common/dockerfiles/nodemon.json:/usr/src/builder/checkouts/nodemon.json
+
+      # docker-out-of-docker: the worker calls ``docker run`` against
+      # the host daemon to spawn build containers.
+      - /var/run/docker.sock:/var/run/docker.sock
+    links:
+      - cache
+      - storage
+      - web
+    environment:
+      - DOCKER_NO_RELOAD
+      - RTD_BROKER_URL=redis://:redispassword@cache:6379/0
+      - RTD_BUILDS_QUEUE=isolated-builds
+      - RTD_BUILDER_REF=celery-on-ec2
+      # Source path the worker passes to ``docker run -v ...`` when
+      # spawning build containers. Must be a HOST-visible path because
+      # the daemon resolves it on the host.
+      - RTD_HOST_BUILDER_PATH=${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}
+      # Runner venv + managed Python install: host-visible paths (the
+      # same directories bind-mounted into this service above).
+      - RTD_HOST_RUNNER_VENV_PATH=${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}/runner-venv
+      - RTD_HOST_PYTHON_INSTALL_PATH=${PWD}/${RTDDEV_PATH_BUILDER:-../readthedocs-builder}/uv-python
+      # Have build containers join the compose network so they can
+      # reach the API at ``http://web`` via service-name DNS.
+      - RTD_BUILD_NETWORK=community_readthedocs
+      # Skip the EC2 IMDS metadata fetch + AWS terminate call on
+      # task_postrun (we're obviously not on EC2).
+      - RTD_SKIP_SELF_TERMINATE=1
+    stdin_open: true
+    tty: true
+    networks:
+      readthedocs:
+    command: ["/usr/src/builder/docker/isolated-builder.sh"]
+
   cache:
     image: redis:6.2.6  # Match production version
     command: >

dockerfiles/entrypoints/isolated-builder.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# Dev entrypoint for the isolated-builders pattern.
+#
+# Emulates the production AMI's two systemd units —
+# ``readthedocs-builder-setup.service`` (clone + worker venv setup) and
+# then ``celery-readthedocs-builder.service`` (start the worker) — as
+# one foreground bash script. No systemd, no New Relic, no Sentry:
+# dev only.
+#
+# Compose wiring is in ``common/dockerfiles/docker-compose.yml`` under
+# the ``isolated-builder`` service.
+
+set -euo pipefail
+
+# Required variable
+: "${RTD_BROKER_URL:?RTD_BROKER_URL must be set (e.g. redis://cache:6379/0)}"
+
+# Optional variables with defaults
+: "${RTD_BUILDS_QUEUE:=isolated-builds}"
+: "${RTD_BUILDER_REPO:=https://github.com/readthedocs/readthedocs-builder.git}"
+: "${RTD_BUILDER_REF:=celery-on-ec2}"
+: "${RTD_BUILDER_TOKEN:=}"
+
+SRC="/usr/src/builder/checkouts/readthedocs-builder"
+RUNNER_VENV="/usr/src/builder/runner-venv"
+UV_PYTHON_DIR="/usr/src/builder/uv-python"
+WORKER_VENV="/usr/src/builder/worker-venv"
+
+# 1. Clone (or skip if the host's checkout is bind-mounted in).
+#    A bind-mount means $SRC is already populated; we skip ``git clone``
+#    so dev iteration on the runner code (edit on host, restart the
+#    container) doesn't blow the host checkout away.
+if [ -z "$(ls -A "$SRC" 2>/dev/null)" ]; then
+    echo "[isolated-builder] $SRC empty; cloning $RTD_BUILDER_REPO@$RTD_BUILDER_REF ..."
+    clone_url="$RTD_BUILDER_REPO"
+    if [ -n "$RTD_BUILDER_TOKEN" ] && [ "${RTD_BUILDER_REPO#https://}" != "$RTD_BUILDER_REPO" ]; then
+        clone_url="https://${RTD_BUILDER_TOKEN}@${RTD_BUILDER_REPO#https://}"
+    fi
+    git clone --depth=1 --branch "$RTD_BUILDER_REF" "$clone_url" "$SRC"
+else
+    echo "[isolated-builder] $SRC already populated; skipping clone (dev bind-mount)."
+fi
+
+# 2. Pre-build the runner venv against a uv-managed Python 3.14 by
+#    syncing the builder/ project. Same flags as the prod systemd setup
+#    unit. Both $RUNNER_VENV and $UV_PYTHON_DIR are bind-mounted from
+#    host paths (see compose), so the venv's bin/python symlink into
+#    $UV_PYTHON_DIR lives on the host filesystem — that's what lets the
+#    worker bind-mount the same host paths into build containers it
+#    spawns and have the symlink still resolve (matches production).
+#
+#    Idempotent: ``uv sync --frozen`` is a no-op when the venv
+#    already matches uv.lock from a previous run.
+echo "[isolated-builder] Syncing runner venv at $RUNNER_VENV (managed Python under $UV_PYTHON_DIR) ..."
+cd "$SRC/builder"
+UV_PYTHON_INSTALL_DIR="$UV_PYTHON_DIR" \
+UV_PROJECT_ENVIRONMENT="$RUNNER_VENV" \
+    uv sync --frozen --python 3.14 --python-preference=only-managed
+
+# 3. Worker venv — sync the worker/ project. Dev omits the
+#    ``observability`` extra (no New Relic / Sentry in dev).
+echo "[isolated-builder] Syncing worker venv at $WORKER_VENV ..."
+cd "$SRC/worker"
+UV_PROJECT_ENVIRONMENT="$WORKER_VENV" \
+    uv sync --frozen --python 3.14
+
+# 4. Replace this process with the Celery worker. PYTHONPATH points at
+#    the worker/ project dir so ``-A worker.celery`` resolves from the
+#    live source; --max-tasks-per-child=1 so the worker exits after one
+#    task (matches prod's ephemeral pattern, even though there's no AWS
+#    API call to terminate anything in dev — the worker just exits and
+#    compose can be configured to restart or not).
+echo "[isolated-builder] Starting Celery worker on queue '$RTD_BUILDS_QUEUE' ..."
+export PYTHONPATH="$SRC/worker"
+
+CMD="$WORKER_VENV/bin/celery -A worker.celery worker --loglevel=INFO --concurrency=1 --max-tasks-per-child=1 -Q ${RTD_BUILDS_QUEUE}"
+if [ -n "${DOCKER_NO_RELOAD}" ]; then
+  echo "Running process with no reload"
+  exec $CMD
+else
+  echo "Running process with reload"
+  exec nodemon --config /usr/src/builder/checkouts/nodemon.json --exec $CMD
+fi

dockerfiles/nodemon.json

@@ -2,7 +2,7 @@
     "verbose": false,
     "delay": 2000,
     "ext": "py",
-    "watch": ["readthedocs", "readthedocsinc", "../readthedocs.org/readthedocs", "../readthedocs-ext/readthedocsext"],
+    "watch": ["readthedocs", "readthedocsinc", "../readthedocs.org/readthedocs", "../readthedocs-ext/readthedocsext", "/opt/readthedocs-builder"],
     "ignore": [".tox/*", ".direnv/*", "user_builds/*", "*/management/commands*", "*migrations/*", "*test*", "*.pyc", "*.pyo", "logs/*"],
     "signal": "SIGTERM"
 }

dockerfiles/tasks.py

@@ -242,7 +242,7 @@ def test(c, arguments="", running=True):
     help={
         "tool": "build.tool to compile (python, nodejs, rust, golang)",
         "version": "specific version for the tool",
-        "os": "ubuntu-20.04, ubuntu-22.04, ubuntu-24.04 (default)",
+        "os": "ubuntu-22.04, ubuntu-24.04 (default), ubuntu-26.04",
     }
 )
 def compilebuildtool(c, tool, version, os=None):