diff --git a/templates/.pre-commit-config.yaml b/templates/.pre-commit-config.yaml index 080e90e..35f2ab2 100644 --- a/templates/.pre-commit-config.yaml +++ b/templates/.pre-commit-config.yaml @@ -13,6 +13,7 @@ ci: # Alphabetised, for lack of a better order. files: | (?x)( + .github\/workflows\/.*| benchmarks\/.+\.py| docs\/.+\.py| lib\/.+\.py| @@ -120,3 +121,9 @@ repos: hooks: - id: validate-pyproject +- repo: https://github.com/zizmorcore/zizmor-pre-commit + # This template does not keep up-to-date with versions, visit the repo to see the most recent release. + rev: v1.25.2 + hooks: + - id: zizmor + diff --git a/templates/benchmarks/bm_runner.py b/templates/benchmarks/bm_runner.py index bafe294..6d5e4f2 100755 --- a/templates/benchmarks/bm_runner.py +++ b/templates/benchmarks/bm_runner.py @@ -163,6 +163,46 @@ def _asv_compare( raise RuntimeError(message) +def _read_gh_report_command(command_path: Path, commit_dir: Path) -> list[str]: + body_file = commit_dir / "body.txt" + command = command_path.read_text().strip().split("\t") + if len(command) == 3 and command[0] == "pr_comment": + _, pr_number, repo = command + return [ + "gh", + "pr", + "comment", + pr_number, + "--body-file", + str(body_file), + "--repo", + repo, + ] + if len(command) == 4 and command[0] == "issue_create": + _, repo, title, assignee = command + command = [ + "gh", + "issue", + "create", + "--title", + title, + "--body-file", + str(body_file), + "--label", + "Bot", + "--label", + "Type: Performance", + "--repo", + repo, + ] + if assignee: + command.extend(["--assignee", assignee]) + return command + + message = f"Unexpected report command format: {command_path}" + raise ValueError(message) + + def _gh_create_reports(commit_sha: str, results_full: str, results_shifts: str) -> None: """If running under GitHub Actions: record the results in report(s). @@ -220,17 +260,12 @@ def _gh_create_reports(commit_sha: str, results_full: str, results_shifts: str) ) if on_pull_request: - # Command to post the report as a comment on the active PR. + # Strict command format to post report as a comment on the active PR. body_path.write_text(performance_report) - command = ( - f"gh pr comment {pr_number} " - f"--body-file {body_path.absolute()} " - f"--repo {repo}" - ) - command_path.write_text(command) + command_path.write_text(f"pr_comment\t{pr_number}\t{repo}") else: - # Command to post the report as new issue. + # Strict command format to post the report as a new issue. commit_msg = _subprocess_runner_capture( f"git log {commit_sha}^! --oneline".split(" ") ) @@ -281,18 +316,7 @@ def _gh_create_reports(commit_sha: str, results_full: str, results_shifts: str) ) body += performance_report body_path.write_text(body) - - command = ( - "gh issue create " - f'--title "{title}" ' - f"--body-file {body_path.absolute()} " - '--label "Bot" ' - '--label "Type: Performance" ' - f"--repo {repo}" - ) - if assignee: - command += f" --assignee {assignee}" - command_path.write_text(command) + command_path.write_text(f"issue_create\t{repo}\t{title}\t{assignee}") def _gh_post_reports() -> None: @@ -308,12 +332,8 @@ def _gh_post_reports() -> None: commit_dirs = [x for x in GH_REPORT_DIR.iterdir() if x.is_dir()] for commit_dir in commit_dirs: command_path = commit_dir / "command.txt" - command = command_path.read_text() - - # Security: only accept certain commands to run. - assert command.startswith(("gh issue create", "gh pr comment")) - - _subprocess_runner(shlex.split(command)) + command = _read_gh_report_command(command_path, commit_dir) + _subprocess_runner(command) class _SubParserGenerator(ABC): diff --git a/templates/github/workflows/ci-benchmarks-report.yml b/templates/github/workflows/ci-benchmarks-report.yml index 2d37b73..df9fd67 100644 --- a/templates/github/workflows/ci-benchmarks-report.yml +++ b/templates/github/workflows/ci-benchmarks-report.yml @@ -2,17 +2,32 @@ # Separated for security: # https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ +# Reference +# - https://github.com/actions/github-script +# - https://github.com/actions/upload-artifact +# - https://github.com/actions/checkout +# - https://github.com/actions/download-artifact +# - https://github.com/actions/setup-python + name: benchmarks-report run-name: Report benchmark results on: - workflow_run: + # Security: it is impossible to fully avoid this exposure, so long as we want results + # from pull request CI to be posted as a comment. `permissions`, and `bm_runner.py` + # are as locked-down as possible, and maintainers must manually approve workflow + # runs from external authors, to mitigate the risk. The remaining vulnerability + # is spam comments. + workflow_run: # zizmor: ignore[dangerous-triggers] workflows: [benchmarks-run] types: - completed jobs: download: + permissions: + actions: read + contents: read runs-on: ubuntu-latest outputs: reports_exist: ${{ steps.unzip.outputs.reports_exist }} @@ -20,7 +35,7 @@ jobs: - name: Download artifact id: download-artifact # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#using-data-from-the-triggering-workflow - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 with: script: | let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ @@ -54,28 +69,33 @@ jobs: echo "reports_exist=$reports_exist" >> "$GITHUB_OUTPUT" - name: Store artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: benchmark_reports path: benchmark_reports post_reports: + permissions: + issues: write + pull-requests: write runs-on: ubuntu-latest needs: download if: needs.download.outputs.reports_exist == 1 steps: - name: Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + persist-credentials: false - name: Download artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: name: benchmark_reports path: .github/workflows/benchmark_reports - name: Set up Python # benchmarks/bm_runner.py only needs builtins to run. - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 - name: Post reports env: diff --git a/templates/github/workflows/ci-benchmarks-run.yml b/templates/github/workflows/ci-benchmarks-run.yml index d37b9a0..c422fcd 100644 --- a/templates/github/workflows/ci-benchmarks-run.yml +++ b/templates/github/workflows/ci-benchmarks-run.yml @@ -2,6 +2,11 @@ # - In the last 24 hours' commits. # - Introduced by this pull request. +# Reference +# - https://github.com/actions/checkout +# - https://github.com/actions/cache +# - https://github.com/actions/upload-artifact + name: benchmarks-run run-name: Run benchmarks @@ -23,6 +28,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: pre-checks: # This workflow supports two different scenarios (overnight and branch). @@ -33,7 +40,10 @@ jobs: overnight: ${{ steps.overnight.outputs.check }} branch: ${{ steps.branch.outputs.check }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + fetch-depth: 2 + persist-credentials: false # TEMPLATING NOTE: Iris also includes examples of label- and file-triggers. - id: overnight name: Check overnight scenario @@ -59,16 +69,17 @@ jobs: steps: # TEMPLATING NOTE: Iris also includes steps for handling iris-test-data. - name: Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd with: fetch-depth: 0 + persist-credentials: false - name: Install run dependencies run: pip install asv # Some repos also need Nox - name: Cache environment directories id: cache-env-dir - uses: actions/cache@v4 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae with: # TEMPLATING NOTE: consider other repo-specific cache directories. # e.g: .nox @@ -82,10 +93,11 @@ jobs: # the proposed merge with the base branch. if: needs.pre-checks.outputs.branch == 'true' env: + BASE_REF: ${{ github.base_ref }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ github.event.number }} run: | - benchmarks/bm_runner.py branch origin/${{ github.base_ref }} + benchmarks/bm_runner.py branch origin/${BASE_REF} - name: Run overnight benchmarks # If the 'overnight' condition(s) are met: use the bm_runner to compare @@ -93,16 +105,17 @@ jobs: id: overnight if: needs.pre-checks.outputs.overnight == 'true' env: + FIRST_COMMIT: ${{ inputs.first_commit }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # The first_commit argument allows a custom starting point - useful # for manual re-running. run: | - first_commit=${{ inputs.first_commit }} + first_commit=${FIRST_COMMIT} if [ "$first_commit" == "" ] then first_commit=$(git log --after="$(date -d "1 day ago" +"%Y-%m-%d") 23:00:00" --pretty=format:"%h" | tail -n 1) fi - + if [ "$first_commit" != "" ] then benchmarks/bm_runner.py overnight $first_commit @@ -124,7 +137,7 @@ jobs: - name: Upload any benchmark reports # Uploading enables more downstream processing e.g. posting a PR comment. if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: benchmark_reports path: .github/workflows/benchmark_reports @@ -132,7 +145,7 @@ jobs: - name: Archive asv results # Store the raw ASV database(s) to help manual investigations. if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: asv-raw-results path: benchmarks/.asv/results diff --git a/templates/github/workflows/ci-benchmarks-validate.yml b/templates/github/workflows/ci-benchmarks-validate.yml index c7626b3..db77f57 100644 --- a/templates/github/workflows/ci-benchmarks-validate.yml +++ b/templates/github/workflows/ci-benchmarks-validate.yml @@ -1,3 +1,7 @@ +# Reference +# - https://github.com/actions/checkout +# - https://github.com/actions/cache + name: benchmarks-validate run-name: Validate the benchmarking setup @@ -17,6 +21,8 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: validate: runs-on: ubuntu-latest @@ -27,16 +33,17 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd with: fetch-depth: 0 + persist-credentials: false - name: Install run dependencies run: pip install asv # Some repos also need Nox - - name: Cache environment directories - id: cache-env-dir - uses: actions/cache@v4 + - name: Restore environment cache + id: cache-restore + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae with: path: | benchmarks/.asv/env @@ -45,3 +52,14 @@ jobs: - name: Validate setup run: benchmarks/bm_runner.py validate + + - name: Save environment cache + # Security: PRs are potentially malformed/malicious, so only allow runs on trunk + # branches to update the cache, to avoid cache poisoning. + if: github.event_name == 'push' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae + with: + path: | + benchmarks/.asv/env + $CONDA/pkgs + key: ${{ steps.cache-restore.outputs.cache-primary-key }} diff --git a/templates/github/workflows/ci-linkchecks.yml b/templates/github/workflows/ci-linkchecks.yml index 726350e..8a5c9b6 100644 --- a/templates/github/workflows/ci-linkchecks.yml +++ b/templates/github/workflows/ci-linkchecks.yml @@ -1,4 +1,9 @@ -name: Linkcheck +# References: +# - https://github.com/actions/checkout +# - https://github.com/lycheeverse/lychee-action +# - https://github.com/peter-evans/create-issue-from-file + +name: ci-linkchecks on: workflow_dispatch: @@ -16,7 +21,7 @@ jobs: issues: write # required for peter-evans/create-issue-from-file steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd with: fetch-depth: 0 persist-credentials: false