diff --git a/.github/workflows/casestudy-django-github.yml b/.github/workflows/casestudy-django-github.yml new file mode 100644 index 0000000..ca4ce44 --- /dev/null +++ b/.github/workflows/casestudy-django-github.yml @@ -0,0 +1,21 @@ +name: casestudy-django-github + +# Runs the django case study on a GitHub-hosted runner. Identical to casestudy-django-tempus.yml +# apart from the `runner` input. workflow_dispatch only, to match the tempus caller. +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + casestudy: + uses: ./.github/workflows/casestudy-django.yml + permissions: + contents: read + with: + runner: ubuntu-latest diff --git a/.github/workflows/casestudy-django-tempus.yml b/.github/workflows/casestudy-django-tempus.yml new file mode 100644 index 0000000..c8d8d92 --- /dev/null +++ b/.github/workflows/casestudy-django-tempus.yml @@ -0,0 +1,22 @@ +name: casestudy-django-tempus + +# Runs the django case study on a tempus.build runner. Identical to casestudy-django-github.yml +# apart from the `runner` input. workflow_dispatch only: case studies are run deliberately (>=3x, +# recorded with the upstream ref and date), and a fork PR must never run on a self-hosted runner. +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + casestudy: + uses: ./.github/workflows/casestudy-django.yml + permissions: + contents: read + with: + runner: tempus-ubuntu-24.04-4core diff --git a/.github/workflows/casestudy-django.yml b/.github/workflows/casestudy-django.yml new file mode 100644 index 0000000..876f20c --- /dev/null +++ b/.github/workflows/casestudy-django.yml @@ -0,0 +1,40 @@ +name: casestudy-django + +# Real-OSS case study: django/django's own test command, run on both runners. django is fetched at +# pipeline time at a pinned commit — its code is NOT stored in this repo. The job is reduced to one +# comparable Linux leg (Python 3.14, SQLite, --parallel). The two thin callers +# (casestudy-django-{tempus,github}.yml) differ only in the `runner` input, so the only variable +# is the runner. See case-studies/README.md. +on: + workflow_call: + inputs: + runner: + description: "runs-on label to benchmark" + required: true + type: string + +permissions: + contents: read + +jobs: + django: + name: django tests (${{ inputs.runner }}) + runs-on: ${{ inputs.runner }} + timeout-minutes: 30 + steps: + - name: Check out django at the pinned commit + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: django/django + ref: c14b756185c88f7f2eb745ff061f3c221fea9de7 # v5.2.9 + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.14" + # django's own test command at this pin. Uses pip (not uv) as django ships no uv lockfile. + # SQLite default — no DB service, no secrets. --parallel exercises all 4 vCPU. + # No action-level cache: this is the honest cold path (warm = a primed colocated proxy). + - name: Install django and test dependencies + run: python -m pip install -e . -r tests/requirements/py3.txt + - name: Run django's test suite + run: python tests/runtests.py --parallel diff --git a/.github/workflows/casestudy-flask-github.yml b/.github/workflows/casestudy-flask-github.yml deleted file mode 100644 index cd198e5..0000000 --- a/.github/workflows/casestudy-flask-github.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: casestudy-flask-github - -# Runs the flask case study on a GitHub-hosted runner. Identical to casestudy-flask-tempus.yml apart -# from the `runner` input. workflow_dispatch only, to match the tempus caller. -on: - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - casestudy: - uses: ./.github/workflows/casestudy-flask.yml - permissions: - contents: read - with: - runner: ubuntu-latest diff --git a/.github/workflows/casestudy-flask-tempus.yml b/.github/workflows/casestudy-flask-tempus.yml deleted file mode 100644 index 36c445a..0000000 --- a/.github/workflows/casestudy-flask-tempus.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: casestudy-flask-tempus - -# Runs the flask case study on a tempus.build runner. Identical to casestudy-flask-github.yml apart -# from the `runner` input. workflow_dispatch only: case studies are run deliberately (>=3x, recorded -# with the upstream ref and date), and a fork PR must never run on a self-hosted runner. -on: - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - casestudy: - uses: ./.github/workflows/casestudy-flask.yml - permissions: - contents: read - with: - runner: tempus-ubuntu-24.04-4core diff --git a/.github/workflows/casestudy-flask.yml b/.github/workflows/casestudy-flask.yml deleted file mode 100644 index d5eb8be..0000000 --- a/.github/workflows/casestudy-flask.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: casestudy-flask - -# Real-OSS case study: pallets/flask's own test command, run on both runners. flask is fetched at -# pipeline time at a pinned commit — its code is NOT stored in this repo. The job is reduced to one -# comparable Linux leg (Python 3.14). The two thin callers (casestudy-flask-{tempus,github}.yml) -# differ only in the `runner` input, so the only variable is the runner. See case-studies/README.md. -on: - workflow_call: - inputs: - runner: - description: "runs-on label to benchmark" - required: true - type: string - -permissions: - contents: read - -jobs: - flask: - name: flask tests (${{ inputs.runner }}) - runs-on: ${{ inputs.runner }} - timeout-minutes: 30 - steps: - - name: Check out flask at the pinned commit - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - repository: pallets/flask - ref: 22d924701a6ae2e4cd01e9a15bbaf3946094af65 # v3.1.3 - persist-credentials: false - - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: "3.14" - # flask's own test command at this pin, pinned to the Python 3.14 tox env. No action-level - # cache: this is the honest cold path (warm = a primed colocated proxy, set up by the operator). - - name: Run flask's test suite - run: uv run --locked tox run -e py3.14 diff --git a/README.md b/README.md index 22d9867..3a78e95 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ not the spec. | Upstream | Pinned ref | Test command | | -------- | ---------- | ------------ | -| [pallets/flask](https://github.com/pallets/flask) | v3.1.3 (`22d9247`) | `uv run --locked tox run -e py3.14` | +| [django/django](https://github.com/django/django) | v5.2.9 (`c14b756`) | `python tests/runtests.py --parallel` | Pin records and runner details live in [`case-studies/projects/`](case-studies/projects/). To add a project, see [`case-studies/README.md`](case-studies/README.md). @@ -44,9 +44,10 @@ a project, see [`case-studies/README.md`](case-studies/README.md). # run both caller workflows via workflow_dispatch (>=3x per runner), then: python scripts/case_study_collect.py \ --repo tempusbuild/benchmark \ - --ref 22d924701a6ae2e4cd01e9a15bbaf3946094af65 \ - --tempus-workflow casestudy-flask-tempus.yml \ - --github-workflow casestudy-flask-github.yml + --upstream django/django \ + --ref c14b756185c88f7f2eb745ff061f3c221fea9de7 \ + --tempus-workflow casestudy-django-tempus.yml \ + --github-workflow casestudy-django-github.yml ``` ## Methodology and results diff --git a/case-studies/README.md b/case-studies/README.md index 9cb47a5..ad477b2 100644 --- a/case-studies/README.md +++ b/case-studies/README.md @@ -9,28 +9,30 @@ workflows differ **only** in `runs-on:`. The upstream project's steps are unchan The upstream project's code is **not stored in this repo**. Each case-study run checks the project out at a **pinned commit** at pipeline time and runs its own test command. What we commit is only: -- two thin caller workflows + one reusable workflow (e.g. `casestudy-flask*.yml`), and +- two thin caller workflows + one reusable workflow (e.g. `casestudy-django*.yml`), and - a pin record under [`projects/`](projects/) (upstream repo, commit, licence, command). So there is no mirror repo and no third-party code in our git — only a workflow that says "check -out `pallets/flask@` and run `uv run --locked tox run -e py3.14`". +out `django/django@` and run `python tests/runtests.py --parallel`". -## How it works (flask example) +## How it works (django example) ```text -.github/workflows/casestudy-flask.yml reusable: checkout flask @pinned sha → its test cmd - ├── casestudy-flask-tempus.yml runner: tempus-ubuntu-24.04-4core - └── casestudy-flask-github.yml runner: ubuntu-latest +.github/workflows/casestudy-django.yml reusable: checkout django @pinned sha → its test cmd + ├── casestudy-django-tempus.yml runner: tempus-ubuntu-24.04-4core + └── casestudy-django-github.yml runner: ubuntu-latest run both (workflow_dispatch) >=3x per runner │ - ▼ scripts/case_study_collect.py --repo --ref \ - --tempus-workflow casestudy-flask-tempus.yml --github-workflow casestudy-flask-github.yml + ▼ scripts/case_study_collect.py --repo tempusbuild/benchmark --upstream django/django \ + --ref \ + --tempus-workflow casestudy-django-tempus.yml --github-workflow casestudy-django-github.yml ▼ median wall-clock per runner (from the GitHub API) → comparison row ``` `--repo` is the repo whose Actions runs are read (this benchmark repo, where the case-study -workflows live); `--ref` is the pinned **upstream** flask commit those runs measured. +workflows live); `--upstream` is the real project the case study exercises (shown in the table); +`--ref` is the pinned upstream django commit those runs measured. ## What stays honest @@ -63,5 +65,5 @@ upstream project. Good candidates have a permissive licence, a test command that runs on `ubuntu-latest` without secrets or external services, and a suite that completes in minutes on a single 4 vCPU leg. Copy -the three `casestudy-flask*.yml` workflows and [`projects/flask.yaml`](projects/flask.yaml), set +the three `casestudy-django*.yml` workflows and [`projects/django.yaml`](projects/django.yaml), set the upstream repo, the pinned commit, and the project's own one-leg test command. diff --git a/case-studies/projects/django.yaml b/case-studies/projects/django.yaml new file mode 100644 index 0000000..148d618 --- /dev/null +++ b/case-studies/projects/django.yaml @@ -0,0 +1,19 @@ +# Pin record for the django case study (documentation + attribution; the workflow files are the +# source of truth). The case study fetches django at the pinned commit AT PIPELINE TIME and runs +# its own test command — django's code is never stored in this repo. + +upstream: + repo: django/django + # Pinned commit: the case study is reproducible at this exact upstream state. Bump deliberately; + # every published number records this ref and the date it was run. + ref: c14b756185c88f7f2eb745ff061f3c221fea9de7 # v5.2.9 + license: BSD-3-Clause + +# django's own test command: one comparable Linux leg (Python 3.14, SQLite, --parallel). +# pip is used (django ships no uv lockfile). --parallel exercises all 4 vCPU. +run: python tests/runtests.py --parallel + +workflows: + reusable: .github/workflows/casestudy-django.yml + tempus: .github/workflows/casestudy-django-tempus.yml + github: .github/workflows/casestudy-django-github.yml diff --git a/case-studies/projects/flask.yaml b/case-studies/projects/flask.yaml deleted file mode 100644 index 46a0686..0000000 --- a/case-studies/projects/flask.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Pin record for the flask case study (documentation + attribution; the workflow files are the -# source of truth). The case study fetches flask at the pinned commit AT PIPELINE TIME and runs its -# own test command — flask's code is never stored in this repo. - -upstream: - repo: pallets/flask - # Pinned commit: the case study is reproducible at this exact upstream state. Bump deliberately; - # every published number records this ref and the date it was run. - ref: 22d924701a6ae2e4cd01e9a15bbaf3946094af65 # v3.1.3 - license: BSD-3-Clause - -# flask's own test command, reduced to one comparable Linux leg (Python 3.14). -run: uv run --locked tox run -e py3.14 - -workflows: - reusable: .github/workflows/casestudy-flask.yml - tempus: .github/workflows/casestudy-flask-tempus.yml - github: .github/workflows/casestudy-flask-github.yml diff --git a/docs/results/README.md b/docs/results/README.md index f5e0637..a13622d 100644 --- a/docs/results/README.md +++ b/docs/results/README.md @@ -15,9 +15,10 @@ separately, upstream project pinned to a specific commit). ```bash python scripts/case_study_collect.py \ --repo tempusbuild/benchmark \ + --upstream django/django \ --ref \ - --tempus-workflow casestudy-flask-tempus.yml \ - --github-workflow casestudy-flask-github.yml + --tempus-workflow casestudy-django-tempus.yml \ + --github-workflow casestudy-django-github.yml ``` This prints the Markdown rows that replace the table below. @@ -31,8 +32,8 @@ measured medians, never estimates. There is one row per (upstream project, runne | Upstream | Ref | Runner | Runs | Median wall-clock | Cache | Date | | -------- | --- | ------ | ---- | ----------------- | ----- | ---- | -| `pallets/flask` | — | `ubuntu-latest` | — | — | cold | — | -| `pallets/flask` | — | `tempus-ubuntu-24.04-4core` | — | — | cold | — | +| `django/django` | — | `ubuntu-latest` | — | — | cold | — | +| `django/django` | — | `tempus-ubuntu-24.04-4core` | — | — | cold | — | (Warm-cache rows are added the same way, with `warm` in the Cache column. Cold and warm are never blended in a single row.) diff --git a/justfile b/justfile index c130c49..8c9d25e 100644 --- a/justfile +++ b/justfile @@ -15,12 +15,13 @@ lint: test: uv run pytest scripts/ -# Reduce the flask case study's runs into a comparison row, e.g.: -# just results-flask pallets/flask 22d924701a6ae2e4cd01e9a15bbaf3946094af65 +# Reduce the django case study's runs into a comparison row, e.g.: +# just results-django c14b756185c88f7f2eb745ff061f3c221fea9de7 # One recipe per case study (the workflow filenames are study-specific). -results-flask repo ref: +results-django ref: uv run python scripts/case_study_collect.py \ - --repo {{ repo }} \ + --repo tempusbuild/benchmark \ + --upstream django/django \ --ref {{ ref }} \ - --tempus-workflow casestudy-flask-tempus.yml \ - --github-workflow casestudy-flask-github.yml + --tempus-workflow casestudy-django-tempus.yml \ + --github-workflow casestudy-django-github.yml diff --git a/scripts/case_study_collect.py b/scripts/case_study_collect.py index 25ebc98..bcac01a 100755 --- a/scripts/case_study_collect.py +++ b/scripts/case_study_collect.py @@ -9,15 +9,19 @@ are upstream's — there is no in-workflow timer under our control. Usage: - case_study_collect.py --repo OWNER/REPO --ref UPSTREAM_SHA \ - --tempus-workflow casestudy-flask-tempus.yml \ - --github-workflow casestudy-flask-github.yml [--limit 3] + case_study_collect.py --repo OWNER/REPO --upstream UPSTREAM/REPO --ref UPSTREAM_SHA \ + --tempus-workflow casestudy-django-tempus.yml \ + --github-workflow casestudy-django-github.yml [--limit 3] + +--repo hosts the Actions runs (this benchmark repo); --upstream is the real project the case +study exercises (shown in the table); --ref is the pinned upstream commit those runs measured. """ import argparse import json import statistics import subprocess +import sys from dataclasses import dataclass from datetime import datetime from typing import TypedDict @@ -68,7 +72,7 @@ def _fmt(seconds: float) -> str: def render_comparison( - repo: str, ref: str, github: VariantResult, tempus: VariantResult + upstream: str, ref: str, github: VariantResult, tempus: VariantResult ) -> str: short_ref = ref[:7] header = ( @@ -76,9 +80,9 @@ def render_comparison( "| -------- | --- | ------ | ---- | ----------------- | ---- |" ) rows = [ - f"| `{repo}` | `{short_ref}` | `{github.label}` | {github.runs} | " + f"| `{upstream}` | `{short_ref}` | `{github.label}` | {github.runs} | " f"{_fmt(github.median_seconds)} | {github.date} |", - f"| `{repo}` | `{short_ref}` | `{tempus.label}` | {tempus.runs} | " + f"| `{upstream}` | `{short_ref}` | `{tempus.label}` | {tempus.runs} | " f"{_fmt(tempus.median_seconds)} | {tempus.date} |", ] speedup = ( @@ -92,18 +96,21 @@ def render_comparison( def fetch_runs(repo: str, workflow_file: str, limit: int) -> list[WorkflowRun]: - proc = subprocess.run( - [ - "gh", - "api", - f"repos/{repo}/actions/workflows/{workflow_file}/runs", - "--jq", - f".workflow_runs[:{limit}]", - ], - capture_output=True, - text=True, - check=True, - ) + try: + proc = subprocess.run( + [ + "gh", + "api", + f"repos/{repo}/actions/workflows/{workflow_file}/runs", + "--jq", + f".workflow_runs[:{limit}]", + ], + capture_output=True, + text=True, + check=True, + ) + except subprocess.CalledProcessError as exc: + sys.exit(f"gh api failed for {repo} / {workflow_file}: {exc.stderr.strip()}") return json.loads(proc.stdout) @@ -114,10 +121,15 @@ def main() -> None: required=True, help="OWNER/REPO hosting the workflow runs (this benchmark repo)", ) + parser.add_argument( + "--upstream", + required=True, + help="OWNER/REPO of the real project the case study runs (shown in the table)", + ) parser.add_argument( "--ref", required=True, - help="pinned UPSTREAM commit the runs measured (e.g. the flask sha)", + help="pinned UPSTREAM commit the runs measured (e.g. the django sha)", ) parser.add_argument("--tempus-workflow", required=True) parser.add_argument("--github-workflow", required=True) @@ -134,7 +146,7 @@ def main() -> None: "ubuntu-latest", fetch_runs(args.repo, args.github_workflow, args.limit), ) - print(render_comparison(args.repo, args.ref, github, tempus)) + print(render_comparison(args.upstream, args.ref, github, tempus)) if __name__ == "__main__": diff --git a/scripts/test_case_study_collect.py b/scripts/test_case_study_collect.py index 84cd728..a7336b4 100644 --- a/scripts/test_case_study_collect.py +++ b/scripts/test_case_study_collect.py @@ -55,9 +55,9 @@ def test_render_comparison_has_both_rows_and_speedup() -> None: [_run("2026-06-19T00:00:00Z", "2026-06-19T00:04:00Z")], ) out = render_comparison( - "pallets/flask", "22d924701a6ae2e4cd01e9a15bbaf3946094af65", github, tempus + "django/django", "c14b756185c88f7f2eb745ff061f3c221fea9de7", github, tempus ) - assert "`pallets/flask`" in out - assert "`22d9247`" in out # ref shortened + assert "`django/django`" in out + assert "`c14b756`" in out # ref shortened assert "480s" in out and "240s" in out assert "2.00x" in out