diff --git a/.github/workflows/casestudy-django.yml b/.github/workflows/casestudy-django.yml index 876f20c..ecfae88 100644 --- a/.github/workflows/casestudy-django.yml +++ b/.github/workflows/casestudy-django.yml @@ -26,7 +26,7 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: repository: django/django - ref: c14b756185c88f7f2eb745ff061f3c221fea9de7 # v5.2.9 + ref: ee93f65169c280c9ab3d2ce103dd478c96d05065 # v6.0.6 persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: diff --git a/README.md b/README.md index 3a78e95..bb7fa4b 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ not the spec. | Upstream | Pinned ref | Test command | | -------- | ---------- | ------------ | -| [django/django](https://github.com/django/django) | v5.2.9 (`c14b756`) | `python tests/runtests.py --parallel` | +| [django/django](https://github.com/django/django) | v6.0.6 (`ee93f65`) | `python tests/runtests.py --parallel` | Pin records and runner details live in [`case-studies/projects/`](case-studies/projects/). To add a project, see [`case-studies/README.md`](case-studies/README.md). @@ -45,7 +45,7 @@ a project, see [`case-studies/README.md`](case-studies/README.md). python scripts/case_study_collect.py \ --repo tempusbuild/benchmark \ --upstream django/django \ - --ref c14b756185c88f7f2eb745ff061f3c221fea9de7 \ + --ref ee93f65169c280c9ab3d2ce103dd478c96d05065 \ --tempus-workflow casestudy-django-tempus.yml \ --github-workflow casestudy-django-github.yml ``` diff --git a/case-studies/projects/django.yaml b/case-studies/projects/django.yaml index 148d618..fdc2c2c 100644 --- a/case-studies/projects/django.yaml +++ b/case-studies/projects/django.yaml @@ -6,7 +6,7 @@ upstream: repo: django/django # Pinned commit: the case study is reproducible at this exact upstream state. Bump deliberately; # every published number records this ref and the date it was run. - ref: c14b756185c88f7f2eb745ff061f3c221fea9de7 # v5.2.9 + ref: ee93f65169c280c9ab3d2ce103dd478c96d05065 # v6.0.6 license: BSD-3-Clause # django's own test command: one comparable Linux leg (Python 3.14, SQLite, --parallel). diff --git a/docs/methodology.md b/docs/methodology.md index 60e199b..1d8a2e1 100644 --- a/docs/methodology.md +++ b/docs/methodology.md @@ -34,10 +34,14 @@ Both sides run Ubuntu 24.04 (`ubuntu-latest` currently maps to Ubuntu 24.04). ## What is measured -The upstream project's own test command runs without modification. Timing comes from the workflow -run's own start and end timestamps (`run_started_at` .. `updated_at`) via the GitHub API. -`scripts/case_study_collect.py` reads those timestamps, takes the median, and renders a comparison -row. Only measured runs are ever summarized — never estimates. +The upstream project's own test command runs without modification. Timing is the **job's** +wall-clock on the runner — its `started_at` .. `completed_at` from the GitHub Actions jobs API. +Run-level timing (`run_started_at` .. `updated_at`) is deliberately not used: it also counts run +orchestration (run record → job assignment) and finalization, which is platform overhead that +differs between hosted and self-hosted runners and would bias the comparison. Queue time is +excluded either way (`started_at` is post-assignment). `scripts/case_study_collect.py` reads each +run's job timing, takes the median, and renders a comparison row. Only measured runs are ever +summarized — never estimates. ## Median of at least three runs diff --git a/docs/results/README.md b/docs/results/README.md index a13622d..a71a61a 100644 --- a/docs/results/README.md +++ b/docs/results/README.md @@ -9,7 +9,8 @@ separately, upstream project pinned to a specific commit). 1. Both caller workflows for a case study are run at least three times each, for each cache state (cold and warm), using `workflow_dispatch`. -2. Wall-clock time is read from the GitHub API (`run_started_at` .. `updated_at`). +2. Each run's job wall-clock is read from the GitHub Actions jobs API (`started_at` .. + `completed_at`), excluding run orchestration and queue time. 3. The runs are reduced to a **median** per (case study, runner, cache state): ```bash diff --git a/justfile b/justfile index 8c9d25e..f9fedb3 100644 --- a/justfile +++ b/justfile @@ -16,7 +16,7 @@ test: uv run pytest scripts/ # Reduce the django case study's runs into a comparison row, e.g.: -# just results-django c14b756185c88f7f2eb745ff061f3c221fea9de7 +# just results-django ee93f65169c280c9ab3d2ce103dd478c96d05065 # One recipe per case study (the workflow filenames are study-specific). results-django ref: uv run python scripts/case_study_collect.py \ diff --git a/scripts/case_study_collect.py b/scripts/case_study_collect.py index bcac01a..01e35d6 100755 --- a/scripts/case_study_collect.py +++ b/scripts/case_study_collect.py @@ -2,11 +2,14 @@ """Reduce a case study's workflow runs into a comparison row. A case study runs a real upstream project's own CI on two runners (tempus vs github) via two -workflows that differ only in `runs-on:`. This reads the recent runs of each workflow from the -GitHub API (via `gh`), takes the median wall-clock, and renders a Markdown comparison. +workflows that differ only in `runs-on:`. This reads each workflow's recent runs from the GitHub +API (via `gh`), takes the median job wall-clock, and renders a Markdown comparison. -Timing comes from the run's own start/end (`run_started_at` .. `updated_at`) because the steps -are upstream's — there is no in-workflow timer under our control. +Timing is the JOB's own `started_at` .. `completed_at` (the Actions jobs API) — the wall-clock of +the job on the runner. Run-level `run_started_at` .. `updated_at` is deliberately NOT used: it also +counts run orchestration (run record -> job assignment) and finalization, which is GitHub +infrastructure overhead, differs between hosted and self-hosted runners, and would bias the +comparison. Queue time is excluded either way (`started_at` is post-assignment). Usage: case_study_collect.py --repo OWNER/REPO --upstream UPSTREAM/REPO --ref UPSTREAM_SHA \ @@ -27,23 +30,23 @@ from typing import TypedDict -class WorkflowRun(TypedDict): - """The subset of a GitHub Actions run record this tool reads.""" +class JobTiming(TypedDict): + """One workflow run reduced to its single job's timing (GitHub Actions jobs API).""" - run_started_at: str - updated_at: str - conclusion: str | None # null while a run is still in progress + started_at: str # when the runner began the job (post-queue/assignment) + completed_at: str + conclusion: str | None # null while the run is still in progress -def run_wall_clock_seconds(run: WorkflowRun) -> float: - """Wall-clock of one workflow run: updated_at - run_started_at, in seconds.""" - started = datetime.fromisoformat(run["run_started_at"]) - ended = datetime.fromisoformat(run["updated_at"]) +def wall_clock_seconds(job: JobTiming) -> float: + """Wall-clock of the job on the runner: completed_at - started_at, in seconds.""" + started = datetime.fromisoformat(job["started_at"]) + ended = datetime.fromisoformat(job["completed_at"]) return (ended - started).total_seconds() -def successful(runs: list[WorkflowRun]) -> list[WorkflowRun]: - return [r for r in runs if r["conclusion"] == "success"] +def successful(jobs: list[JobTiming]) -> list[JobTiming]: + return [j for j in jobs if j["conclusion"] == "success"] def median_seconds(values: list[float]) -> float: @@ -60,10 +63,10 @@ class VariantResult: date: str # ISO date of the most recent run -def summarize(runner_label: str, runs: list[WorkflowRun]) -> VariantResult: - ok = successful(runs) - median = median_seconds([run_wall_clock_seconds(r) for r in ok]) # raises if empty - latest_date = max(r["run_started_at"] for r in ok)[:10] # ok is non-empty here +def summarize(runner_label: str, jobs: list[JobTiming]) -> VariantResult: + ok = successful(jobs) + median = median_seconds([wall_clock_seconds(j) for j in ok]) # raises if empty + latest_date = max(j["started_at"] for j in ok)[:10] # ok is non-empty here return VariantResult(runner_label, len(ok), median, latest_date) @@ -95,25 +98,48 @@ def render_comparison( return "\n".join([header, *rows]) + note -def fetch_runs(repo: str, workflow_file: str, limit: int) -> list[WorkflowRun]: +def _gh_json(*api_args: str) -> object: + """Run `gh api ARGS` and parse JSON stdout; exit cleanly (not a traceback) on failure.""" try: proc = subprocess.run( - [ - "gh", - "api", - f"repos/{repo}/actions/workflows/{workflow_file}/runs", - "--jq", - f".workflow_runs[:{limit}]", - ], + ["gh", "api", *api_args], capture_output=True, text=True, check=True, ) except subprocess.CalledProcessError as exc: - sys.exit(f"gh api failed for {repo} / {workflow_file}: {exc.stderr.strip()}") + sys.exit(f"gh api {' '.join(api_args)} failed: {exc.stderr.strip()}") return json.loads(proc.stdout) +def fetch_job_timings(repo: str, workflow_file: str, limit: int) -> list[JobTiming]: + """The `limit` most recent runs of a workflow, each reduced to its single job's timing. + + Two API hops: the runs list gives run ids + conclusion; per run the jobs API gives the job's + started_at/completed_at (the runs list carries no job timing). + """ + runs = _gh_json( + f"repos/{repo}/actions/workflows/{workflow_file}/runs", + "--jq", + f".workflow_runs[:{limit}] | map({{id, conclusion}})", + ) + timings: list[JobTiming] = [] + for run in runs: + job = _gh_json( + f"repos/{repo}/actions/runs/{run['id']}/jobs", + "--jq", + ".jobs[0] | {started_at, completed_at}", + ) + timings.append( + { + "started_at": job["started_at"], + "completed_at": job["completed_at"], + "conclusion": run["conclusion"], + } + ) + return timings + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -140,11 +166,11 @@ def main() -> None: tempus = summarize( "tempus-ubuntu-24.04-4core", - fetch_runs(args.repo, args.tempus_workflow, args.limit), + fetch_job_timings(args.repo, args.tempus_workflow, args.limit), ) github = summarize( "ubuntu-latest", - fetch_runs(args.repo, args.github_workflow, args.limit), + fetch_job_timings(args.repo, args.github_workflow, args.limit), ) print(render_comparison(args.upstream, args.ref, github, tempus)) diff --git a/scripts/test_case_study_collect.py b/scripts/test_case_study_collect.py index a7336b4..d3ca83b 100644 --- a/scripts/test_case_study_collect.py +++ b/scripts/test_case_study_collect.py @@ -3,39 +3,39 @@ import pytest from case_study_collect import ( - WorkflowRun, + JobTiming, median_seconds, render_comparison, - run_wall_clock_seconds, successful, summarize, + wall_clock_seconds, ) -def _run(started: str, ended: str, conclusion: str = "success") -> WorkflowRun: - return {"run_started_at": started, "updated_at": ended, "conclusion": conclusion} +def _job(started: str, ended: str, conclusion: str = "success") -> JobTiming: + return {"started_at": started, "completed_at": ended, "conclusion": conclusion} def test_wall_clock_from_timestamps() -> None: - run = _run("2026-06-19T12:00:00Z", "2026-06-19T12:06:50Z") - assert run_wall_clock_seconds(run) == 410 + job = _job("2026-06-19T12:00:00Z", "2026-06-19T12:06:50Z") + assert wall_clock_seconds(job) == 410 def test_successful_filters_out_failures() -> None: - runs = [ - _run("2026-06-19T12:00:00Z", "2026-06-19T12:05:00Z", "success"), - _run("2026-06-19T12:00:00Z", "2026-06-19T12:05:00Z", "failure"), + jobs = [ + _job("2026-06-19T12:00:00Z", "2026-06-19T12:05:00Z", "success"), + _job("2026-06-19T12:00:00Z", "2026-06-19T12:05:00Z", "failure"), ] - assert len(successful(runs)) == 1 + assert len(successful(jobs)) == 1 def test_summarize_takes_median_and_latest_date() -> None: - runs = [ - _run("2026-06-17T10:00:00Z", "2026-06-17T10:04:00Z"), # 240s - _run("2026-06-19T10:00:00Z", "2026-06-19T10:03:00Z"), # 180s (latest date) - _run("2026-06-18T10:00:00Z", "2026-06-18T10:10:00Z"), # 600s (outlier) + jobs = [ + _job("2026-06-17T10:00:00Z", "2026-06-17T10:04:00Z"), # 240s + _job("2026-06-19T10:00:00Z", "2026-06-19T10:03:00Z"), # 180s (latest date) + _job("2026-06-18T10:00:00Z", "2026-06-18T10:10:00Z"), # 600s (outlier) ] - result = summarize("ubuntu-latest", runs) + result = summarize("ubuntu-latest", jobs) assert result.runs == 3 assert result.median_seconds == 240 assert result.date == "2026-06-19" @@ -48,16 +48,16 @@ def test_median_rejects_empty() -> None: def test_render_comparison_has_both_rows_and_speedup() -> None: github = summarize( - "ubuntu-latest", [_run("2026-06-19T00:00:00Z", "2026-06-19T00:08:00Z")] + "ubuntu-latest", [_job("2026-06-19T00:00:00Z", "2026-06-19T00:08:00Z")] ) tempus = summarize( "tempus-ubuntu-24.04-4core", - [_run("2026-06-19T00:00:00Z", "2026-06-19T00:04:00Z")], + [_job("2026-06-19T00:00:00Z", "2026-06-19T00:04:00Z")], ) out = render_comparison( - "django/django", "c14b756185c88f7f2eb745ff061f3c221fea9de7", github, tempus + "django/django", "ee93f65169c280c9ab3d2ce103dd478c96d05065", github, tempus ) assert "`django/django`" in out - assert "`c14b756`" in out # ref shortened + assert "`ee93f65`" in out # ref shortened assert "480s" in out and "240s" in out assert "2.00x" in out