diff --git a/.github/agents/tooling-ci-specialist.md b/.github/agents/tooling-ci-specialist.md index 99515e8510..aecfb7a816 100644 --- a/.github/agents/tooling-ci-specialist.md +++ b/.github/agents/tooling-ci-specialist.md @@ -230,7 +230,7 @@ This file defines standardized environment setup for GitHub Copilot agents. When - Coverage: Python 3.11 only - Caching: pip dependencies -2. **build.yml** +2. **docker-build.yml** - Docker image build validation - PostgreSQL service - Basic functionality tests @@ -239,7 +239,7 @@ This file defines standardized environment setup for GitHub Copilot agents. When - Security analysis - Weekly schedule -4. **release.yml** +4. **pypi-publish.yml** - Package and release automation - Trigger: Push to main diff --git a/.github/workflows/build.yml b/.github/workflows/docker-build.yml similarity index 98% rename from .github/workflows/build.yml rename to .github/workflows/docker-build.yml index 6b97ca4545..10cd23ef76 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/docker-build.yml @@ -49,7 +49,7 @@ jobs: run: docker exec --env-file .env fm-container flexmeasures add toy-account - name: Generate prices dummy data - run: .github/workflows/generate-dummy-price.sh + run: ci/generate-dummy-price.sh - name: Copy prices dummy data run: docker cp prices-tomorrow.csv fm-container:/app/prices-tomorrow.csv - name: Add beliefs diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000000..bd730f9233 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,83 @@ + +# This workflow builds and pushes a Docker image to Docker Hub whenever a +# GitHub Release is published, tagging it with the release version and, +# for stable (non-pre-release) releases, also as `latest`. +# +# Requires the DOCKERHUB_USERNAME and DOCKERHUB_TOKEN repository secrets +# to be configured under Settings -> Secrets and variables -> Actions. + +name: Publish Docker image + +on: + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: "Git tag to build/push (e.g. v1.0.0). Required when run manually." + required: false + +permissions: + contents: read + +jobs: + docker-publish: + runs-on: ubuntu-latest + # Dedicated environments with protections for publishing are strongly recommended. + # For more information, see: https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#deployment-protection-rules + environment: + name: dockerhub + steps: + - name: Determine version and image tags + id: version + run: | + TAG="${{ github.event.release.tag_name || inputs.tag }}" + if [ -z "$TAG" ]; then + echo "No tag available (release.tag_name and inputs.tag are both empty)." >&2 + exit 1 + fi + echo "tag=${TAG}" >> "$GITHUB_OUTPUT" + + # `github.event.release.prerelease` is a boolean field GitHub sets + # on the `release` webhook payload, reflecting whether the "Set as + # a pre-release" checkbox was ticked when the GitHub Release was + # created. We only want stable releases to also claim `:latest`. + IMAGE_TAGS="lfenergy/flexmeasures:${TAG}" + if [ "${{ github.event.release.prerelease }}" != "true" ]; then + IMAGE_TAGS="${IMAGE_TAGS} + lfenergy/flexmeasures:latest" + fi + { + echo "image-tags<> "$GITHUB_OUTPUT" + + - uses: actions/checkout@v4 + with: + ref: ${{ steps.version.outputs.tag }} + fetch-depth: 0 # full history + tags, required for hatch-vcs to resolve the version + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - uses: docker/setup-buildx-action@v3 + + - name: Build and push + # Built once, pushed under both tags: pushing `latest` re-tags the + # exact image just built and pushed under the version tag, rather + # than triggering a second (functionally identical, but distinct) + # build. + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + push: true + tags: ${{ steps.version.outputs.image-tags }} + + - name: Summary + run: | + echo "Pushed: ${{ steps.version.outputs.image-tags }}" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/docker-qa.yml b/.github/workflows/docker-qa.yml new file mode 100644 index 0000000000..426604df30 --- /dev/null +++ b/.github/workflows/docker-qa.yml @@ -0,0 +1,106 @@ + +# Manually-triggered QA workflow for release testing. It spins up the local +# docker compose stack, then runs: +# - the toy tutorials 1-5, using the runner scripts kept in this repo +# (documentation/tut/scripts/run-tutorial*-in-docker.sh) +# - the HEMS walkthrough, using the example kept in the +# FlexMeasures/flexmeasures-client repo (examples/HEMS/HEMS_setup.py), +# including its CLI-based report generation step (see +# flexmeasures-client's FLEXMEASURES_CLI_CMD/FLEXMEASURES_CLI_CONFIG_DIR +# support, used below to run that CLI inside the `server` container) +# +# This does not replace manual QA (e.g. UI login/graph checks, exploratory +# testing) -- see documentation/dev/release_process.rst. + +name: QA (release) + +on: + workflow_dispatch: + inputs: + flexmeasures-client-ref: + description: "Branch/tag/ref to use from FlexMeasures/flexmeasures-client" + required: false + default: "main" + +permissions: + contents: read + +env: + # Picked up by every `docker compose` invocation in this job, so the HEMS + # configs bind mount (added via qa-hems-override.yml, see below) applies + # consistently across the `up`, `exec`, and `down` steps. + COMPOSE_FILE: docker-compose.yml:qa-hems-override.yml + +jobs: + qa: + runs-on: ubuntu-latest + steps: + - name: Checkout flexmeasures + uses: actions/checkout@v4 + + - name: Checkout flexmeasures-client + uses: actions/checkout@v4 + with: + repository: FlexMeasures/flexmeasures-client + ref: ${{ inputs.flexmeasures-client-ref }} + path: flexmeasures-client + + - name: Add compose override to mount the HEMS configs directory + run: | + cat > qa-hems-override.yml <] +set -euo pipefail + +SINCE="${1:-$(git describe --tags --abbrev=0)}" +SINCE_DATE=$(git log -1 --format=%aI "$SINCE") + +REPO_ROOT=$(git rev-parse --show-toplevel) +CHANGELOG_FILES=( + "$REPO_ROOT/documentation/changelog.rst" + "$REPO_ROOT/documentation/cli/change_log.rst" + "$REPO_ROOT/documentation/api/change_log.rst" +) +# PR numbers already referenced by a `PR #NNNN` link in any changelog file. +EXISTING_PRS=$(grep -ohP '(?<=PR #)\d+' "${CHANGELOG_FILES[@]}" 2>/dev/null | sort -un) + +echo "Merged PRs since $SINCE ($SINCE_DATE) not yet mentioned in a changelog file:" +echo + +gh pr list \ + --repo FlexMeasures/flexmeasures \ + --state merged \ + --search "merged:>=$SINCE_DATE" \ + --limit 200 \ + --json number,title,url \ + --jq '.[] | "\(.number)\t* \(.title) [see `PR #\(.number) <\(.url)>`_]"' \ + | while IFS=$'\t' read -r number line; do + if grep -qx "$number" <<< "$EXISTING_PRS"; then + continue + fi + echo "$line" + done + +echo +echo "Review, categorize into New features / Infrastructure / Support / Bugfixes," +echo "and paste into documentation/changelog.rst (and cli/change_log.rst, api/change_log.rst if relevant)." +echo "(PRs already referenced in those files are omitted above.)" diff --git a/documentation/changelog.rst b/documentation/changelog.rst index b920c47173..953e22c2fd 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -27,6 +27,7 @@ Infrastructure / Support * Document suggested cloud architecture [see `PR #2245 `_] * Document the ``TRUSTED_HOSTS`` setting to safeguard against host poisoning from clients [see `PR #2246 `_] * Document the various ways to inspect a (scheduling) job [see `PR #2247 `_] +* Automate Docker Hub image publishing and a PyPI install smoke test on release, add a manually-triggered QA workflow that runs the toy tutorials and HEMS walkthrough against a local Docker Compose stack, and add a helper script to list merged PRs since the last tag [see `PR #2260 `_] Bugfixes ----------- diff --git a/documentation/dev/release_process.rst b/documentation/dev/release_process.rst new file mode 100644 index 0000000000..95429e1a09 --- /dev/null +++ b/documentation/dev/release_process.rst @@ -0,0 +1,70 @@ +Release process +================ + +This page describes how FlexMeasures releases are done, and which parts are automated versus manual. +The canonical, most detailed checklist still lives in `FlexMeasures/tsc RELEASE.md `_. +This page summarizes that flow and points out where CI now does the mechanical work. + +Versioning is derived entirely from git tags via ``hatch-vcs`` (see ``pyproject.toml``'s ``[tool.hatch.version]``), so there is no file where a version string needs to be bumped by hand. + +1. Prepare and test (manual) +----------------------------- + +* Ensure the corresponding GitHub milestone is complete and the version number choice matches the changes made. +* ``uv sync --group dev --group test`` and ``uv run poe test``. +* For MINOR/MAJOR releases, do QA using the Docker Compose stack, and write the accompanying blog post. + + Tutorials 1-5 and the HEMS walkthrough can be run via the manually-triggered ``QA (release)`` GitHub Actions workflow (``.github/workflows/docker-qa.yml``). It spins up the local docker compose stack and runs the tutorial runner scripts from ``documentation/tut/scripts`` and the HEMS example from `FlexMeasures/flexmeasures-client `_ (kept in its own repo, not duplicated here). Trigger it from the Actions tab; it does not run automatically. This does not replace manual UI login/graph checks or exploratory QA. + +2. Assemble the changelog (semi-automated) +------------------------------------------- + +Run: + +.. code-block:: bash + + uv run poe changelog-check + +This lists merged PRs since the last tag that aren't yet mentioned in a changelog file (via ``ci/list-merged-prs-since-tag.sh``, which cross-references PR numbers already linked in ``documentation/changelog.rst`` / ``cli/change_log.rst`` / ``api/change_log.rst``), pre-formatted as changelog bullets. +It is read-only: it does not edit or commit anything. +Review, categorize each entry into *New features* / *Infrastructure / Support* / *Bugfixes*, +and paste them into ``documentation/changelog.rst`` (and ``documentation/cli/change_log.rst`` / ``documentation/api/change_log.rst`` where relevant). + +3. Commit, tag, and release (manual — requires GPG signing) +-------------------------------------------------------------- + +.. code-block:: bash + + git commit -S -sam "changelog updates for v" + git push + git tag -s -a v -m "" + git push --tags + +Then create the GitHub Release from the new tag. This step is intentionally manual: GPG signing requires the maintainer's personal key, which cannot be delegated to CI without exporting private key material into repository secrets. + +4. Automated publishing (CI) +------------------------------- + +Publishing the GitHub Release (step 3) triggers two workflows: + +* ``.github/workflows/pypi-publish.yml`` builds the package and publishes it to PyPI via trusted (OIDC) publishing, then runs a ``pypi-smoke-test`` job that installs the just-published version into a fresh virtual environment and verifies ``flexmeasures.__version__`` matches. +* ``.github/workflows/docker-publish.yml`` builds and pushes the Docker image to Docker Hub, tagged ``lfenergy/flexmeasures:v``, and additionally as ``lfenergy/flexmeasures:latest`` for stable (non-pre-release) releases. + +Check the Actions tab for both workflow runs to confirm they succeeded; also spot-check that the new version shows up on `PyPI `_, `Docker Hub `_, and that the ReadTheDocs build for the new tag completed. + +.. note:: + The ``docker-publish.yml`` workflow can also be re-run manually (``workflow_dispatch``, with a ``tag`` input) if a run needs to be retried. + +5. Announce (manual) +---------------------- + +Publish the blog post (MINOR/MAJOR only), and announce on Mastodon, the mailing list, and the LF Energy Slack. + +6. Post-release (manual, MINOR/MAJOR only) +--------------------------------------------- + +* Start the next development cycle: an empty, signed commit ``Start v`` and tag ``v.dev0``. +* Update the milestone on GitHub, and add changelog placeholders for upcoming releases. +* Open a dependency-upgrade PR tagged ``dependency-hygiene``. + +These are infrequent, low-risk manual steps and involve a judgment call (what the next version number should be), so they are not automated. diff --git a/documentation/index.rst b/documentation/index.rst index 42312e7627..e987794db8 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -271,6 +271,7 @@ In :ref:`getting_started`, we have some helpful tips how to dive into this docum dev/api dev/connection-secrets dev/automated-deploy-via-GHActions + dev/release_process .. autosummary:: :caption: Code Documentation diff --git a/pyproject.toml b/pyproject.toml index dabffc264c..63d917dcf4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -193,6 +193,13 @@ help = "Generate OpenAPI specifications" script = "flexmeasures.api.scripts.generate_open_api_specs" env = {FLEXMEASURES_ENV = "documentation", FLEXMEASURES_PLUGINS = ""} +[tool.poe.tasks.changelog-check] +help = "List merged PRs since the last tag, to help assemble changelog.rst entries (does not write any files)." +shell = "ci/list-merged-prs-since-tag.sh" +args = [ + {name = "since", help = "Git tag to diff from (default: latest tag reachable from HEAD)", default = ""}, +] + [dependency-groups] dev = [ "black==24.8.0",