diff --git a/.ci/scripts/calculate_builds.py b/.ci/scripts/calculate_builds.py index 3ef31fa601..bdfa1522cd 100755 --- a/.ci/scripts/calculate_builds.py +++ b/.ci/scripts/calculate_builds.py @@ -79,11 +79,14 @@ def set_output(key: str, value: str) -> None: }, "mod026": {"sic-version": "0.2.2", "sta-version": "0.13.1", "fcs-version": "0.0.2"}, "mod027": {"sic-version": "0.5.0", "sta-version": "0.13.1", "fcs-version": "0.0.2"}, + # there is no modpack with fcs 0.0.3 as 2 fcs releases have been done since the last synapse release + "mod028": {"sic-version": "0.2.2", "sta-version": "0.13.1", "fcs-version": "0.0.4"}, + "mod029": {"sic-version": "0.5.0", "sta-version": "0.13.1", "fcs-version": "0.0.4"}, } # Adjust this section to decide what gets built and layered on top # THIS IS THE SECTION TO EDIT, after you have added the new versions above -current_mod_packs_to_build = ["mod026", "mod027"] +current_mod_packs_to_build = ["mod028", "mod029"] generated_jobs: list[dict[str, Any]] = [] for mod_pack_job in current_mod_packs_to_build: diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..dc5bcaed14 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,93 @@ +version: 2 +# As dependabot is currently only run on a weekly basis, we raise the +# open-pull-requests-limit to 10 (from the default of 5) to better ensure we +# don't continuously grow a backlog of updates. +updates: + - # "pip" is the correct setting for poetry, per https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem + package-ecosystem: "pip" + directory: "/" + open-pull-requests-limit: 10 + versioning-strategy: "increase-if-necessary" + schedule: + interval: "weekly" + # Group patch updates to packages together into a single PR, as they rarely + # if ever contain breaking changes that need to be reviewed separately. + # + # Less PRs means a streamlined review process. + # + # Python packages follow semantic versioning, and tend to only introduce + # breaking changes in major version bumps. Thus, we'll group minor and patch + # versions together. + groups: + minor-and-patches: + applies-to: version-updates + patterns: + - "*" + update-types: + - "minor" + - "patch" + # Prevent pulling packages that were recently updated to help mitigate + # supply chain attacks. 14 days was taken from the recommendation at + # https://blog.yossarian.net/2025/11/21/We-should-all-be-using-dependency-cooldowns + # where the author noted that 9/10 attacks would have been mitigated by a + # two week cooldown. + # + # The cooldown only applies to general updates; security updates will still + # be pulled in as soon as possible. + cooldown: + default-days: 14 + + - package-ecosystem: "docker" + directory: "/docker" + open-pull-requests-limit: 10 + schedule: + interval: "weekly" + # For container versions, breaking changes are also typically only introduced in major + # package bumps. + groups: + minor-and-patches: + applies-to: version-updates + patterns: + - "*" + update-types: + - "minor" + - "patch" + cooldown: + default-days: 14 + + - package-ecosystem: "github-actions" + directory: "/" + open-pull-requests-limit: 10 + schedule: + interval: "weekly" + # Similarly for GitHub Actions, breaking changes are typically only introduced in major + # package bumps. + groups: + minor-and-patches: + applies-to: version-updates + patterns: + - "*" + update-types: + - "minor" + - "patch" + cooldown: + default-days: 14 + + - package-ecosystem: "cargo" + directory: "/" + open-pull-requests-limit: 10 + versioning-strategy: "lockfile-only" + schedule: + interval: "weekly" + # The Rust ecosystem is special in that breaking changes are often introduced + # in minor version bumps, as packages typically stay pre-1.0 for a long time. + # Thus we specifically keep minor version bumps separate in their own PRs. + groups: + patches: + applies-to: version-updates + patterns: + - "*" + update-types: + - "patch" + cooldown: + default-days: 14 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 051c66ebba..c451e49301 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -41,13 +41,13 @@ jobs: echo "SYNAPSE_VERSION=$(grep "^version" pyproject.toml | sed -E 's/version\s*=\s*["]([^"]*)["]/\1/')" >> $GITHUB_ENV - name: Log in to DockerHub - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -79,7 +79,7 @@ jobs: services/backend-repositories/secret/data/oci.element.io password | OCI_PASSWORD ; - name: Login to Element OCI Registry - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: oci-push.vpn.infra.element.io username: ${{ steps.import-secrets.outputs.OCI_USERNAME }} @@ -87,7 +87,7 @@ jobs: - name: Build and push by digest id: build - uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 with: push: true labels: | @@ -108,7 +108,7 @@ jobs: touch "${{ runner.temp }}/digests/${digest#sha256:}" - name: Upload digest - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: digests-${{ matrix.suffix }} path: ${{ runner.temp }}/digests/* @@ -136,14 +136,14 @@ jobs: merge-multiple: true - name: Log in to DockerHub - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 if: ${{ startsWith(matrix.repository, 'docker.io') }} with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 if: ${{ startsWith(matrix.repository, 'ghcr.io') }} with: registry: ghcr.io @@ -176,7 +176,7 @@ jobs: services/backend-repositories/secret/data/oci.element.io password | OCI_PASSWORD ; - name: Login to Element OCI Registry - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: oci-push.vpn.infra.element.io username: ${{ steps.import-secrets.outputs.OCI_USERNAME }} @@ -186,7 +186,7 @@ jobs: uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Install Cosign - uses: sigstore/cosign-installer@faadad0cce49287aee09b3a48701e75088a2c6ad # v4.0.0 + uses: sigstore/cosign-installer@cad07c2e89fa2edd6e2d7bab4c1aa38e53f76003 # v4.1.1 - name: Calculate docker image tag uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index ba5baf160a..2ff16d8480 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -45,7 +45,7 @@ jobs: cp book/welcome_and_overview.html book/index.html - name: Upload Artifact - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: book path: book diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index d03a929507..fa3f31e2c9 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -172,7 +172,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) diff --git a/.github/workflows/push_complement_image.yml b/.github/workflows/push_complement_image.yml index 3931dbebb3..da45bb2c03 100644 --- a/.github/workflows/push_complement_image.yml +++ b/.github/workflows/push_complement_image.yml @@ -52,7 +52,7 @@ jobs: with: poetry-version: "2.2.1" - name: Login to registry - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 8c625be96e..41a7cb5611 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -64,7 +64,7 @@ jobs: uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Set up docker layer caching - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} @@ -99,7 +99,7 @@ jobs: echo "ARTIFACT_NAME=${DISTRO#*:}" >> "$GITHUB_OUTPUT" - name: Upload debs as artifacts - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: debs-${{ steps.artifact-name.outputs.ARTIFACT_NAME }} path: debs/* @@ -150,7 +150,7 @@ jobs: # musl: (TODO: investigate). CIBW_TEST_SKIP: pp3*-* *musl* - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: Wheel-${{ matrix.os }} path: ./wheelhouse/*.whl @@ -171,7 +171,7 @@ jobs: - name: Build sdist run: python -m build --sdist - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: Sdist path: dist/*.tar.gz diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2d548a3883..c1277e3d5f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -174,7 +174,7 @@ jobs: # Cribbed from # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17 - name: Restore/persist mypy's cache - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: | .mypy_cache @@ -561,7 +561,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }}) @@ -658,7 +658,7 @@ jobs: PGPASSWORD: postgres PGDATABASE: postgres - name: "Upload schema differences" - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: ${{ failure() && !cancelled() && steps.run_tester_script.outcome == 'failure' }} with: name: Schema dumps diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index d9d61152fb..8d26e73e37 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -145,7 +145,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) diff --git a/CHANGES.md b/CHANGES.md index 3861306dfb..5937675dda 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,9 +1,59 @@ +# Synapse 1.153.0 (2026-05-19) + +No significant changes since 1.153.0rc3. + + +# Synapse 1.153.0rc3 (2026-05-15) + +## Bugfixes + +- Revert 'Have [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) return a new response immediately if a room subscription has changed and produced a new response. ([\#19714](https://github.com/element-hq/synapse/issues/19714))' (introduced in 1.153.0rc1) due to performance problems. ([\#19784](https://github.com/element-hq/synapse/issues/19784)) + + +# Synapse 1.153.0rc2 (2026-05-13) + +## Bugfixes + +- Correctly handle arbitrary precision integers in `unsigned` field of events. The bug was introduced in 1.153.0rc1. ([\#19769](https://github.com/element-hq/synapse/issues/19769)) + + +# Synapse 1.153.0rc1 (2026-05-08) + +## Features + +- Make ACLs apply to EDUs per [MSC4163](https://github.com/matrix-org/matrix-spec-proposals/pull/4163). ([\#18475](https://github.com/element-hq/synapse/issues/18475)) +- Stabilize [MSC3266: Room summary API](https://github.com/matrix-org/matrix-spec-proposals/pull/3266), removing the experimental config flag `msc3266_enabled`. Contributed by @dasha-uwu. ([\#19720](https://github.com/element-hq/synapse/issues/19720)) +- Partial [MSC4311](https://github.com/matrix-org/matrix-spec-proposals/pull/4311) implementation: `m.room.create` is now a required part of stripped `invite_state`/`knock_state` . Contributed by @FrenchGithubUser @Famedly. ([\#19722](https://github.com/element-hq/synapse/issues/19722)) +- Expose `tombstoned` and `replacement_room` in room details on admin API endpoint `GET /_synapse/admin/v1/rooms/`. Contributed by Noah Markert. ([\#19737](https://github.com/element-hq/synapse/issues/19737)) + +## Bugfixes + +- Allow self-requested user erasure (upon account deactivation) to succeed even if Synapse has disabled profile changes. Contributed by Famedly. ([\#19398](https://github.com/element-hq/synapse/issues/19398)) +- Fix Synapse not backfilling new history when attempting to use a pagination token near a backward extremity. ([\#19611](https://github.com/element-hq/synapse/issues/19611)) +- Have [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) return a new response immediately if a room subscription has changed and produced a new response. ([\#19714](https://github.com/element-hq/synapse/issues/19714)) +- Fix a bug where when upgrading a room to room version 12, the power level event in the old room got temporarily mutated to remove the user upgrading the room's power. ([\#19727](https://github.com/element-hq/synapse/issues/19727)) +- Fix packaging for Fedora and EPEL caused by unnecessary bumping `authlib` minimum version requirement in `pyproject.toml` file. Contributed by Oleg Girko. ([\#19742](https://github.com/element-hq/synapse/issues/19742)) + +## Improved Documentation + +- Add warning about known problems when configuring `use_frozen_dicts`. ([\#19711](https://github.com/element-hq/synapse/issues/19711)) + +## Internal Changes + +- Port `Event.signatures` field to Rust. ([\#19706](https://github.com/element-hq/synapse/issues/19706)) +- Port `Event.unsigned` field to Rust. ([\#19708](https://github.com/element-hq/synapse/issues/19708)) +- Add a Rust canonical JSON serializer. ([\#19739](https://github.com/element-hq/synapse/issues/19739), [\#19763](https://github.com/element-hq/synapse/issues/19763)) +- Configure Dependabot to only update Python dependencies in the lockfile, unless widening upper bounds. ([\#19743](https://github.com/element-hq/synapse/issues/19743)) +- Reduce `WORKER_LOCK_MAX_RETRY_INTERVAL` to 5 seconds to reduce idle time after lock is released. ([\#19755](https://github.com/element-hq/synapse/issues/19755)) +- Force keyword-only arguments for `Duration` so time units have to be specified. ([\#19756](https://github.com/element-hq/synapse/issues/19756)) + + # Synapse 1.152.1 (2026-05-07) ## Security Fixes -- Prevent CPU starvation (Denial of Service) under worker lock contention, additionally capping the `WorkerLock` time out interval to a maximum of 60 seconds. Contributed by Famedly. ([\#19394](https://github.com/element-hq/synapse/issues/19394), ELEMENTSEC-2026-1706, [GHSA-8q93-326v-3m7g](https://github.com/element-hq/synapse/security/advisories/GHSA-8q93-326v-3m7g), CVE pending) -- Prevent pagination ending when a page is full of rejected events. (ELEMENTSEC-2025-1636, [GHSA-6qf2-7x63-mm6v](https://github.com/element-hq/synapse/security/advisories/GHSA-6qf2-7x63-mm6v), CVE pending) +- Prevent CPU starvation (Denial of Service) under worker lock contention, additionally capping the `WorkerLock` time out interval to a maximum of 60 seconds. Contributed by Famedly. ([\#19394](https://github.com/element-hq/synapse/issues/19394), ELEMENTSEC-2026-1706, [GHSA-8q93-326v-3m7g](https://github.com/element-hq/synapse/security/advisories/GHSA-8q93-326v-3m7g), CVE-2026-45078) +- Prevent pagination ending when a page is full of rejected events. (ELEMENTSEC-2025-1636, [GHSA-6qf2-7x63-mm6v](https://github.com/element-hq/synapse/security/advisories/GHSA-6qf2-7x63-mm6v), CVE-2026-45076) # Synapse 1.152.0 (2026-04-28) diff --git a/Cargo.lock b/Cargo.lock index e1747182e3..8285c7cf38 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,12 +29,6 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" -[[package]] -name = "autocfg" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" - [[package]] name = "base64" version = "0.22.1" @@ -164,6 +158,12 @@ dependencies = [ "syn", ] +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "equivalent" version = "1.0.2" @@ -640,12 +640,6 @@ dependencies = [ "hashbrown", ] -[[package]] -name = "indoc" -version = "2.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" - [[package]] name = "ipnet" version = "2.11.0" @@ -662,6 +656,15 @@ dependencies = [ "serde", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -720,15 +723,6 @@ version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" -[[package]] -name = "memoffset" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" -dependencies = [ - "autocfg", -] - [[package]] name = "mime" version = "0.3.17" @@ -806,37 +800,34 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.27.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab53c047fcd1a1d2a8820fe84f05d6be69e9526be40cb03b73f86b6b03e6d87d" +checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" dependencies = [ "anyhow", "bytes", - "indoc", "libc", - "memoffset", "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", "pyo3-macros", - "unindent", ] [[package]] name = "pyo3-build-config" -version = "0.27.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b455933107de8642b4487ed26d912c2d899dec6114884214a0b3bb3be9261ea6" +checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.27.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c85c9cbfaddf651b1221594209aed57e9e5cff63c4d11d1feead529b872a089" +checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" dependencies = [ "libc", "pyo3-build-config", @@ -855,9 +846,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.27.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a5b10c9bf9888125d917fb4d2ca2d25c8df94c7ab5a52e13313a07e050a3b02" +checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -867,9 +858,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.27.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b51720d314836e53327f5871d4c0cfb4fb37cc2c4a11cc71907a86342c40f9" +checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" dependencies = [ "heck", "proc-macro2", @@ -880,12 +871,13 @@ dependencies = [ [[package]] name = "pythonize" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3a8f29db331e28c332c63496cfcbb822aca3d7320bc08b655d7fd0c29c50ede" +checksum = "0b79f670c9626c8b651c0581011b57b6ba6970bb69faf01a7c4c0cfc81c43f95" dependencies = [ "pyo3", "serde", + "serde_json", ] [[package]] @@ -1125,9 +1117,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.10" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "ring", "rustls-pki-types", @@ -1336,6 +1328,7 @@ dependencies = [ "http", "http-body-util", "icu_segmenter", + "itertools", "lazy_static", "log", "mime", @@ -1375,9 +1368,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.13.2" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] name = "thiserror" @@ -1553,12 +1546,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" -[[package]] -name = "unindent" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" - [[package]] name = "untrusted" version = "0.9.0" diff --git a/complement/go.mod b/complement/go.mod index 716aabdf32..d1ccf6b496 100644 --- a/complement/go.mod +++ b/complement/go.mod @@ -7,6 +7,7 @@ toolchain go1.24.4 require ( github.com/matrix-org/complement v0.0.0-20251120181401-44111a2a8a9d github.com/matrix-org/gomatrixserverlib v0.0.0-20250813150445-9f5070a65744 + github.com/tidwall/gjson v1.18.0 ) require ( @@ -22,6 +23,7 @@ require ( require ( github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -40,16 +42,15 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect - go.opentelemetry.io/otel v1.36.0 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 // indirect - go.opentelemetry.io/otel/metric v1.36.0 // indirect - go.opentelemetry.io/otel/trace v1.36.0 // indirect + go.opentelemetry.io/otel/metric v1.41.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect go.opentelemetry.io/proto/otlp v1.7.0 // indirect golang.org/x/crypto v0.45.0 // indirect golang.org/x/sys v0.38.0 // indirect diff --git a/complement/go.sum b/complement/go.sum index 79a35aa14c..f82487882b 100644 --- a/complement/go.sum +++ b/complement/go.sum @@ -4,6 +4,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= @@ -81,8 +83,8 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -95,24 +97,24 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= -go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9HsPl42NIXFIFSUSSs0fiqra0= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0/go.mod h1:90PoxvaEB5n6AOdZvi+yWJQoE95U8Dhhw2bSyRqnTD0= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.34.0 h1:BEj3SPM81McUZHYjRS5pEgNgnmzGJ5tRpU5krWnV8Bs= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.34.0/go.mod h1:9cKLGBDzI/F3NoHLQGm4ZrYdIHsvGt6ej6hUowxY0J4= -go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= -go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= -go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= -go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/complement/tests/room_messages_test.go b/complement/tests/room_messages_test.go new file mode 100644 index 0000000000..d491d1db85 --- /dev/null +++ b/complement/tests/room_messages_test.go @@ -0,0 +1,394 @@ +// This file is licensed under the Affero General Public License (AGPL) version 3. +// +// Copyright (C) 2026 Element Creations Ltd +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// See the GNU Affero General Public License for more details: +// . + +package synapse_tests + +import ( + "encoding/json" + "fmt" + "net/url" + "slices" + "strings" + "testing" + + "github.com/matrix-org/complement" + "github.com/matrix-org/complement/b" + "github.com/matrix-org/complement/client" + "github.com/matrix-org/complement/helpers" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/tidwall/gjson" +) + +func TestMessagesOverFederation(t *testing.T) { + deployment := complement.Deploy(t, 2) + defer deployment.Destroy(t) + + alice := deployment.Register(t, "hs1", helpers.RegistrationOpts{ + LocalpartSuffix: "alice", + }) + bob := deployment.Register(t, "hs2", helpers.RegistrationOpts{ + LocalpartSuffix: "bob", + }) + + // The typical convention to find backfill points is from the backward extremities in + // the DAG. Backward extremities are the oldest events we know of in the room but we + // only know of them because some other event referenced them by prev_event and aren't + // known to the homeserver yet (meaning we don't know their depth specifically). So we + // can only do approximate depth comparisons (use the depth of the known events + // they're connected to). And we don't know if those backward extremities point to a + // long chain/fork of history that could stretch back far enough to be visible. + // + // This means a naive homeserver implementation that looks for backward extremities <= + // depth of the `/messages?dir=b&from=xxx` token may overlook a backfill point that could + // reveal more history in the window the user is currently paginating in. + // + // This could be a near miss as this test is specifically stressing or a more deep miss + // as the backward extremity could reveal an entire fork of history that stretches + // back far enough to be visible. + // + // In Synapse, we consider "nearby" as anything within range of the `limit` specified + // in `/messages?dir=b&from=xxx&limit=xxx`. + // + // This test lives in our in-repo Complement tests for Synapse because the Matrix spec + // doesn't have any rules for how a homeserver should backfill. Practically speaking, + // homeservers that don't do anything for this problem will just hide messages from + // clients. This underscores the fact why it's necessary for homeservers to indicate that + // there is a gap (using MSC3871) at the very least. + // + // -------------------------------------------------- + // + // Even with MSC3871 gaps, the tested behavior here is necessary as the gap prev/next + // tokens point before/after the event (remember: tokens are positions between + // events), so if you use `/messages?dir=b&from=`, we can't + // rely on naive depth comparison. MSC3871 Complement tests will also exercise this. + // Example: + // + // t0 t1 t2 t3 t4 + // [A] <--- [B] <--- [C] <--- [bob join 4] + // + // When Bob calls `/messages?dir=b&backfill=false`, he sees a gap (`{ event_id: "bob + // join 4", prev_pagination_token: "t3", next_pagination_token: "t4" }`) and tries to + // fill it in with `/messages?dir=b&from=t3&limit=10&backfill=true`. To find backfill + // points, Synapse will compare `t3` with the backward extremity at an approximate + // depth of 4. Which is why we take `t3`, add the `limit=10` and then do the + // comparison (find any backfill points with an approximate depth <= 13). + t.Run("Backfill from nearby backward extremities past token", func(t *testing.T) { + // Alice creates the room + roomID := alice.MustCreateRoom(t, map[string]interface{}{ + // The `public_chat` preset includes `history_visibility: "shared"` ("Previous + // events are always accessible to newly joined members. All events in the + // room are accessible, even those sent when the member was not a part of the + // room."), which is what we want to test. + "preset": "public_chat", + }) + + // Keep track of the order + eventIDs := make([]string, 0) + // Map from event_id to event info + eventMap := make(map[string]EventInfo) + + // Send some message history into the room + numberOfMessagesToSend := 3 + messageDrafts := make([]MessageDraft, 0, numberOfMessagesToSend) + for i := 0; i < numberOfMessagesToSend; i++ { + messageDrafts = append( + messageDrafts, + MessageDraft{alice, fmt.Sprintf("message history %d", i+1)}, + ) + } + sendAndTrackMessages(t, roomID, messageDrafts, &eventIDs, &eventMap) + + // Bob joins the room + bob.MustJoinRoom(t, roomID, []spec.ServerName{ + deployment.GetFullyQualifiedHomeserverName(t, "hs1"), + }) + bobJoinEventID := getStateID(t, bob, roomID, "m.room.member", bob.UserID) + + // Make it easy to cross-reference the events being talked about in the logs + for eventIndex, eventID := range eventIDs { + t.Logf("Message %d -> event_id=%s", eventIndex, eventID) + } + + // Use a `/context` request to get a pagination token just before Bob's join event + // (remember: tokens are positions between events) + // + // Usually a client would just use `/messages?dir=b` to start getting history + // after joining but this is valid as well. To illustrate a more real example of + // this, someone can use `/timestamp_to_event` to jump back in history and + // `/context` to start paginating history. + contextRes := bob.MustDo( + t, + "GET", + []string{"_matrix", "client", "v3", "rooms", roomID, "context", bobJoinEventID}, + client.WithContentType("application/json"), + client.WithQueries(url.Values{ + "limit": []string{"0"}, + }), + ) + contextResResBody := client.ParseJSON(t, contextRes) + // > `start`: A token that can be used to paginate backwards with. + // > - https://spec.matrix.org/v1.17/client-server-api/#get_matrixclientv3roomsroomidcontexteventid + paginationToken := client.GetJSONFieldStr(t, contextResResBody, "start") + + // Paginate backwards from the join event + messagesRes := bob.MustDo( + t, + "GET", + []string{"_matrix", "client", "v3", "rooms", roomID, "messages"}, + client.WithContentType("application/json"), + client.WithQueries(url.Values{ + "dir": []string{"b"}, + "limit": []string{"100"}, + "from": []string{paginationToken}, + }), + ) + messagesResBody := client.ParseJSON(t, messagesRes) + + // Since `dir=b`, these will be in reverse chronological order + actualEventIDsFromRequest := extractEventIDsFromMessagesResponse(t, messagesResBody) + + // Put them in chronological order to match the expected list + chronologicalActualEventIds := slices.Clone(actualEventIDsFromRequest) + slices.Reverse(chronologicalActualEventIds) + + // Assert timeline order + assertEventsInOrder(t, chronologicalActualEventIds, eventIDs) + }) + + // TODO: Backfill test to make sure we backfill from forks when viewing history (see + // docstring above). + // + // 1. Alice (hs1, engineered homeserver) creates a room with events A, B + // 1. Bob (hs2) joins the room + // 1. Bob leaves the room + // 1. Alice creates a fork from A with some history (1, 2, 3) and connects it back with a new event C + // 1. Bob joins back + // 1. Bob paginates `/messages?dir=b&from=` + // 1. Ensure Bob sees events: B, 2, 1, A + // + // 1 <--- 2 <----- 3 + // / \ + // A <------- B ▲ <--- C <-- D + // | + // Paginate backwards from this point + // t.Run("Backfill from nearby backward extremities past token (fork)", func(t *testing.T) { +} + +// These utilities match what we're using in the Complement repo (see +// `matrix-org/complement` -> `tests/csapi/room_messages_test.go`) + +type MessageDraft struct { + Sender *client.CSAPI + Message string +} + +type EventInfo struct { + MessageDraft MessageDraft + EventID string +} + +func sendMessageDrafts( + t *testing.T, + roomID string, + messageDrafts []MessageDraft, +) []string { + t.Helper() + + eventIDs := make([]string, len(messageDrafts)) + for messageDraftIndex, messageDraft := range messageDrafts { + eventID := messageDraft.Sender.SendEventSynced(t, roomID, b.Event{ + Type: "m.room.message", + Content: map[string]interface{}{ + "msgtype": "m.text", + "body": messageDraft.Message, + }, + }) + eventIDs[messageDraftIndex] = eventID + } + + return eventIDs +} + +// sendAndTrackMessages sends the given message drafts to the room, keeping track of the +// new events in the list of `eventIDs` and `eventMap`. Returns the list of new event +// IDs that were sent. +func sendAndTrackMessages( + t *testing.T, + roomID string, + messageDrafts []MessageDraft, + eventIDs *[]string, + eventMap *map[string]EventInfo, +) []string { + t.Helper() + + newEventIDs := sendMessageDrafts(t, roomID, messageDrafts) + + *eventIDs = append(*eventIDs, newEventIDs...) + for i, eventID := range newEventIDs { + (*eventMap)[eventID] = EventInfo{ + MessageDraft: messageDrafts[i], + EventID: eventID, + } + } + + return newEventIDs +} + +// extractEventIDsFromMessagesResponse extracts the event IDs from the given +// `/messages` response body. +func extractEventIDsFromMessagesResponse( + t *testing.T, + messagesResBody json.RawMessage, +) []string { + t.Helper() + + wantKey := "chunk" + keyRes := gjson.GetBytes(messagesResBody, wantKey) + if !keyRes.Exists() { + t.Fatalf("extractEventIDsFromMessagesResponse: missing key '%s'", wantKey) + } + if !keyRes.IsArray() { + t.Fatalf( + "extractEventIDsFromMessagesResponse: key '%s' is not an array (was %s)", + wantKey, + keyRes.Type, + ) + } + + var eventIDs []string + actualEvents := keyRes.Array() + for _, event := range actualEvents { + eventIDs = append(eventIDs, event.Get("event_id").Str) + } + + return eventIDs +} + +func filterEventIDs(t *testing.T, actualEventIDs []string, expectedEventIDs []string) []string { + t.Helper() + + relevantActualEventIDs := make([]string, 0, len(expectedEventIDs)) + for _, eventID := range actualEventIDs { + if slices.Contains(expectedEventIDs, eventID) { + relevantActualEventIDs = append(relevantActualEventIDs, eventID) + } + } + + return relevantActualEventIDs +} + +// assertEventsInOrder asserts all `actualEventIDs` are present and in order according +// to `expectedEventIDs`. Other unrelated events can be in between. +func assertEventsInOrder(t *testing.T, actualEventIDs []string, expectedEventIDs []string) { + t.Helper() + + relevantActualEventIDs := filterEventIDs(t, actualEventIDs, expectedEventIDs) + + if len(relevantActualEventIDs) != len(expectedEventIDs) { + t.Fatalf( + "expected %d events in timeline (got %d relevant events filtered down from %d events)\n%s", + len(expectedEventIDs), + len(relevantActualEventIDs), + len(actualEventIDs), + generateEventOrderDiffString(relevantActualEventIDs, expectedEventIDs), + ) + } + + for i, eventID := range relevantActualEventIDs { + if eventID != expectedEventIDs[i] { + t.Fatalf( + "expected event ID %s (got %s) at index %d\n%s", + expectedEventIDs[i], + eventID, + i, + generateEventOrderDiffString(relevantActualEventIDs, expectedEventIDs), + ) + } + } +} + +func generateEventOrderDiffString(actualEventIDs []string, expectedEventIDs []string) string { + expectedLines := make([]string, len(expectedEventIDs)) + for i, expectedEventID := range expectedEventIDs { + isExpectedInActual := slices.Contains(actualEventIDs, expectedEventID) + isMissingIndicatorString := " " + if !isExpectedInActual { + isMissingIndicatorString = "?" + } + + expectedLines[i] = fmt.Sprintf("%2d: %s %s", i, isMissingIndicatorString, expectedEventID) + } + expectedDiffString := strings.Join(expectedLines, "\n") + + actualLines := make([]string, len(actualEventIDs)) + for actualEventIndex, actualEventID := range actualEventIDs { + isActualInExpected := slices.Contains(expectedEventIDs, actualEventID) + isActualInExpectedIndicatorString := " " + if isActualInExpected { + isActualInExpectedIndicatorString = "+" + } + + expectedIndex := slices.Index(expectedEventIDs, actualEventID) + expectedIndexString := "" + if actualEventIndex != expectedIndex { + expectedDirectionString := "⬆️" + if expectedIndex > actualEventIndex { + expectedDirectionString = "⬇️" + } + + expectedIndexString = fmt.Sprintf( + " (expected index %d %s)", + expectedIndex, + expectedDirectionString, + ) + } + + actualLines[actualEventIndex] = fmt.Sprintf("%2d: %s %s%s", + actualEventIndex, isActualInExpectedIndicatorString, actualEventID, expectedIndexString, + ) + } + actualDiffString := strings.Join(actualLines, "\n") + + return fmt.Sprintf( + "Actual events ('+' = found expected items):\n%s\nExpected events ('?' = missing expected items):\n%s", + actualDiffString, + expectedDiffString, + ) +} + +func getStateID( + t *testing.T, + c *client.CSAPI, + roomID string, + stateType string, + stateKey string, +) string { + t.Helper() + + stateRes := c.MustDo(t, "GET", []string{"_matrix", "client", "v3", "rooms", roomID, "state"}) + stateResBody := client.ParseJSON(t, stateRes) + eventJSON := gjson.ParseBytes(stateResBody) + if !eventJSON.IsArray() { + t.Fatalf("expected array of state events but found %s", eventJSON.Type) + } + + events := eventJSON.Array() + + for _, event := range events { + if event.Get("type").Str == stateType && event.Get("state_key").Str == stateKey { + return event.Get("event_id").Str + } + } + + t.Fatalf("Unable to find state event for (%s, %s). Room state: %s", stateType, stateKey, events) + return "" +} diff --git a/debian/changelog b/debian/changelog index cfefe953e3..b1a4e04bc3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,27 @@ +matrix-synapse-py3 (1.153.0) stable; urgency=medium + + * New Synapse release 1.153.0. + + -- Synapse Packaging team Tue, 19 May 2026 14:12:51 +0100 + +matrix-synapse-py3 (1.153.0~rc3) stable; urgency=medium + + * New Synapse release 1.153.0rc3. + + -- Synapse Packaging team Fri, 15 May 2026 11:42:29 +0100 + +matrix-synapse-py3 (1.153.0~rc2) stable; urgency=medium + + * New Synapse release 1.153.0rc2. + + -- Synapse Packaging team Wed, 13 May 2026 12:00:39 +0100 + +matrix-synapse-py3 (1.153.0~rc1) stable; urgency=medium + + * New Synapse release 1.153.0rc1. + + -- Synapse Packaging team Fri, 08 May 2026 13:05:08 +0100 + matrix-synapse-py3 (1.152.1) stable; urgency=medium * New Synapse release 1.152.1. diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index 9fd7fc954a..e829292aca 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -123,8 +123,6 @@ experimental_features: msc3874_enabled: true # no UIA for x-signing upload for the first time msc3967_enabled: true - # Expose a room summary for public rooms - msc3266_enabled: true # Send to-device messages to application services msc2409_to_device_messages_enabled: true # Allow application services to masquerade devices diff --git a/docs/admin_api/rooms.md b/docs/admin_api/rooms.md index 11e787c236..c7544033e8 100644 --- a/docs/admin_api/rooms.md +++ b/docs/admin_api/rooms.md @@ -308,6 +308,9 @@ The following fields are possible in the JSON response body: If the room does not define a type, the value will be `null`. * `forgotten` - Whether all local users have [forgotten](https://spec.matrix.org/latest/client-server-api/#leaving-rooms) the room. +* `tombstoned` - Whether the room has been tombstoned (permanently closed). +* `replacement_room` - The room ID of the new room that users should join instead, if this room was tombstoned. Will be + `null` if the room has not been tombstoned, or if it was tombstoned without designating a successor room. The API is: @@ -337,7 +340,9 @@ A response body like the following is returned: "history_visibility": "shared", "state_events": 93534, "room_type": "m.space", - "forgotten": false + "forgotten": false, + "tombstoned": false, + "replacement_room": null } ``` diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 963f1631e0..ba398322a7 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -228,7 +228,11 @@ user_agent_suffix: ' (I''m a teapot; Linux x86_64)' --- ### `use_frozen_dicts` -*(boolean)* Determines whether we should freeze the internal dict object in `FrozenEvent`. Freezing prevents bugs where we accidentally share e.g. signature dicts. However, freezing a dict is expensive. Defaults to `false`. +*(boolean)* Determines whether we should freeze the internal dict object in `FrozenEvent`. Freezing prevents bugs where we accidentally share e.g. signature dicts. However, freezing a dict is expensive. + +> ⚠️ **Warning** – This option is known to introduce a new class of [comparison bugs](https://github.com/element-hq/synapse/issues/18117) in Synapse. + +Defaults to `false`. Example configuration: ```yaml diff --git a/poetry.lock b/poetry.lock index b6578ec161..6a8870590d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -14,14 +14,14 @@ files = [ [[package]] name = "attrs" -version = "25.4.0" +version = "26.1.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373"}, - {file = "attrs-25.4.0.tar.gz", hash = "sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11"}, + {file = "attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309"}, + {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, ] [[package]] @@ -582,21 +582,21 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.46" +version = "3.1.47" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "gitpython-3.1.46-py3-none-any.whl", hash = "sha256:79812ed143d9d25b6d176a10bb511de0f9c67b1fa641d82097b0ab90398a2058"}, - {file = "gitpython-3.1.46.tar.gz", hash = "sha256:400124c7d0ef4ea03f7310ac2fbf7151e09ff97f2a3288d64a440c584a29c37f"}, + {file = "gitpython-3.1.47-py3-none-any.whl", hash = "sha256:489f590edfd6d20571b2c0e72c6a6ac6915ee8b8cd04572330e3842207a78905"}, + {file = "gitpython-3.1.47.tar.gz", hash = "sha256:dba27f922bd2b42cb54c87a8ab3cb6beb6bf07f3d564e21ac848913a05a8a3cd"}, ] [package.dependencies] gitdb = ">=4.0.1,<5" [package.extras] -doc = ["sphinx (>=7.1.2,<7.2)", "sphinx-autodoc-typehints", "sphinx_rtd_theme"] +doc = ["sphinx (>=7.4.7,<8)", "sphinx-autodoc-typehints", "sphinx_rtd_theme"] test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3.8\"", "mypy (==1.18.2) ; python_version >= \"3.9\"", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions ; python_version < \"3.11\""] [[package]] @@ -2171,103 +2171,103 @@ files = [ [[package]] name = "pillow" -version = "12.1.1" +version = "12.2.0" description = "Python Imaging Library (fork)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "pillow-12.1.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1f1625b72740fdda5d77b4def688eb8fd6490975d06b909fd19f13f391e077e0"}, - {file = "pillow-12.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:178aa072084bd88ec759052feca8e56cbb14a60b39322b99a049e58090479713"}, - {file = "pillow-12.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b66e95d05ba806247aaa1561f080abc7975daf715c30780ff92a20e4ec546e1b"}, - {file = "pillow-12.1.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:89c7e895002bbe49cdc5426150377cbbc04767d7547ed145473f496dfa40408b"}, - {file = "pillow-12.1.1-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a5cbdcddad0af3da87cb16b60d23648bc3b51967eb07223e9fed77a82b457c4"}, - {file = "pillow-12.1.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9f51079765661884a486727f0729d29054242f74b46186026582b4e4769918e4"}, - {file = "pillow-12.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:99c1506ea77c11531d75e3a412832a13a71c7ebc8192ab9e4b2e355555920e3e"}, - {file = "pillow-12.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:36341d06738a9f66c8287cf8b876d24b18db9bd8740fa0672c74e259ad408cff"}, - {file = "pillow-12.1.1-cp310-cp310-win32.whl", hash = "sha256:6c52f062424c523d6c4db85518774cc3d50f5539dd6eed32b8f6229b26f24d40"}, - {file = "pillow-12.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:c6008de247150668a705a6338156efb92334113421ceecf7438a12c9a12dab23"}, - {file = "pillow-12.1.1-cp310-cp310-win_arm64.whl", hash = "sha256:1a9b0ee305220b392e1124a764ee4265bd063e54a751a6b62eff69992f457fa9"}, - {file = "pillow-12.1.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:e879bb6cd5c73848ef3b2b48b8af9ff08c5b71ecda8048b7dd22d8a33f60be32"}, - {file = "pillow-12.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:365b10bb9417dd4498c0e3b128018c4a624dc11c7b97d8cc54effe3b096f4c38"}, - {file = "pillow-12.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d4ce8e329c93845720cd2014659ca67eac35f6433fd3050393d85f3ecef0dad5"}, - {file = "pillow-12.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc354a04072b765eccf2204f588a7a532c9511e8b9c7f900e1b64e3e33487090"}, - {file = "pillow-12.1.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7e7976bf1910a8116b523b9f9f58bf410f3e8aa330cd9a2bb2953f9266ab49af"}, - {file = "pillow-12.1.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:597bd9c8419bc7c6af5604e55847789b69123bbe25d65cc6ad3012b4f3c98d8b"}, - {file = "pillow-12.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2c1fc0f2ca5f96a3c8407e41cca26a16e46b21060fe6d5b099d2cb01412222f5"}, - {file = "pillow-12.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:578510d88c6229d735855e1f278aa305270438d36a05031dfaae5067cc8eb04d"}, - {file = "pillow-12.1.1-cp311-cp311-win32.whl", hash = "sha256:7311c0a0dcadb89b36b7025dfd8326ecfa36964e29913074d47382706e516a7c"}, - {file = "pillow-12.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:fbfa2a7c10cc2623f412753cddf391c7f971c52ca40a3f65dc5039b2939e8563"}, - {file = "pillow-12.1.1-cp311-cp311-win_arm64.whl", hash = "sha256:b81b5e3511211631b3f672a595e3221252c90af017e399056d0faabb9538aa80"}, - {file = "pillow-12.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ab323b787d6e18b3d91a72fc99b1a2c28651e4358749842b8f8dfacd28ef2052"}, - {file = "pillow-12.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:adebb5bee0f0af4909c30db0d890c773d1a92ffe83da908e2e9e720f8edf3984"}, - {file = "pillow-12.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bb66b7cc26f50977108790e2456b7921e773f23db5630261102233eb355a3b79"}, - {file = "pillow-12.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aee2810642b2898bb187ced9b349e95d2a7272930796e022efaf12e99dccd293"}, - {file = "pillow-12.1.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a0b1cd6232e2b618adcc54d9882e4e662a089d5768cd188f7c245b4c8c44a397"}, - {file = "pillow-12.1.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7aac39bcf8d4770d089588a2e1dd111cbaa42df5a94be3114222057d68336bd0"}, - {file = "pillow-12.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ab174cd7d29a62dd139c44bf74b698039328f45cb03b4596c43473a46656b2f3"}, - {file = "pillow-12.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:339ffdcb7cbeaa08221cd401d517d4b1fe7a9ed5d400e4a8039719238620ca35"}, - {file = "pillow-12.1.1-cp312-cp312-win32.whl", hash = "sha256:5d1f9575a12bed9e9eedd9a4972834b08c97a352bd17955ccdebfeca5913fa0a"}, - {file = "pillow-12.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:21329ec8c96c6e979cd0dfd29406c40c1d52521a90544463057d2aaa937d66a6"}, - {file = "pillow-12.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:af9a332e572978f0218686636610555ae3defd1633597be015ed50289a03c523"}, - {file = "pillow-12.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d242e8ac078781f1de88bf823d70c1a9b3c7950a44cdf4b7c012e22ccbcd8e4e"}, - {file = "pillow-12.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:02f84dfad02693676692746df05b89cf25597560db2857363a208e393429f5e9"}, - {file = "pillow-12.1.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:e65498daf4b583091ccbb2556c7000abf0f3349fcd57ef7adc9a84a394ed29f6"}, - {file = "pillow-12.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6c6db3b84c87d48d0088943bf33440e0c42370b99b1c2a7989216f7b42eede60"}, - {file = "pillow-12.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8b7e5304e34942bf62e15184219a7b5ad4ff7f3bb5cca4d984f37df1a0e1aee2"}, - {file = "pillow-12.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:18e5bddd742a44b7e6b1e773ab5db102bd7a94c32555ba656e76d319d19c3850"}, - {file = "pillow-12.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc44ef1f3de4f45b50ccf9136999d71abb99dca7706bc75d222ed350b9fd2289"}, - {file = "pillow-12.1.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a8eb7ed8d4198bccbd07058416eeec51686b498e784eda166395a23eb99138e"}, - {file = "pillow-12.1.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47b94983da0c642de92ced1702c5b6c292a84bd3a8e1d1702ff923f183594717"}, - {file = "pillow-12.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:518a48c2aab7ce596d3bf79d0e275661b846e86e4d0e7dec34712c30fe07f02a"}, - {file = "pillow-12.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a550ae29b95c6dc13cf69e2c9dc5747f814c54eeb2e32d683e5e93af56caa029"}, - {file = "pillow-12.1.1-cp313-cp313-win32.whl", hash = "sha256:a003d7422449f6d1e3a34e3dd4110c22148336918ddbfc6a32581cd54b2e0b2b"}, - {file = "pillow-12.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:344cf1e3dab3be4b1fa08e449323d98a2a3f819ad20f4b22e77a0ede31f0faa1"}, - {file = "pillow-12.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:5c0dd1636633e7e6a0afe7bf6a51a14992b7f8e60de5789018ebbdfae55b040a"}, - {file = "pillow-12.1.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0330d233c1a0ead844fc097a7d16c0abff4c12e856c0b325f231820fee1f39da"}, - {file = "pillow-12.1.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5dae5f21afb91322f2ff791895ddd8889e5e947ff59f71b46041c8ce6db790bc"}, - {file = "pillow-12.1.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2e0c664be47252947d870ac0d327fea7e63985a08794758aa8af5b6cb6ec0c9c"}, - {file = "pillow-12.1.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:691ab2ac363b8217f7d31b3497108fb1f50faab2f75dfb03284ec2f217e87bf8"}, - {file = "pillow-12.1.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9e8064fb1cc019296958595f6db671fba95209e3ceb0c4734c9baf97de04b20"}, - {file = "pillow-12.1.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:472a8d7ded663e6162dafdf20015c486a7009483ca671cece7a9279b512fcb13"}, - {file = "pillow-12.1.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:89b54027a766529136a06cfebeecb3a04900397a3590fd252160b888479517bf"}, - {file = "pillow-12.1.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:86172b0831b82ce4f7877f280055892b31179e1576aa00d0df3bb1bbf8c3e524"}, - {file = "pillow-12.1.1-cp313-cp313t-win32.whl", hash = "sha256:44ce27545b6efcf0fdbdceb31c9a5bdea9333e664cda58a7e674bb74608b3986"}, - {file = "pillow-12.1.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a285e3eb7a5a45a2ff504e31f4a8d1b12ef62e84e5411c6804a42197c1cf586c"}, - {file = "pillow-12.1.1-cp313-cp313t-win_arm64.whl", hash = "sha256:cc7d296b5ea4d29e6570dabeaed58d31c3fea35a633a69679fb03d7664f43fb3"}, - {file = "pillow-12.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:417423db963cb4be8bac3fc1204fe61610f6abeed1580a7a2cbb2fbda20f12af"}, - {file = "pillow-12.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:b957b71c6b2387610f556a7eb0828afbe40b4a98036fc0d2acfa5a44a0c2036f"}, - {file = "pillow-12.1.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:097690ba1f2efdeb165a20469d59d8bb03c55fb6621eb2041a060ae8ea3e9642"}, - {file = "pillow-12.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2815a87ab27848db0321fb78c7f0b2c8649dee134b7f2b80c6a45c6831d75ccd"}, - {file = "pillow-12.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f7ed2c6543bad5a7d5530eb9e78c53132f93dfa44a28492db88b41cdab885202"}, - {file = "pillow-12.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:652a2c9ccfb556235b2b501a3a7cf3742148cd22e04b5625c5fe057ea3e3191f"}, - {file = "pillow-12.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d6e4571eedf43af33d0fc233a382a76e849badbccdf1ac438841308652a08e1f"}, - {file = "pillow-12.1.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b574c51cf7d5d62e9be37ba446224b59a2da26dc4c1bb2ecbe936a4fb1a7cb7f"}, - {file = "pillow-12.1.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a37691702ed687799de29a518d63d4682d9016932db66d4e90c345831b02fb4e"}, - {file = "pillow-12.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f95c00d5d6700b2b890479664a06e754974848afaae5e21beb4d83c106923fd0"}, - {file = "pillow-12.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:559b38da23606e68681337ad74622c4dbba02254fc9cb4488a305dd5975c7eeb"}, - {file = "pillow-12.1.1-cp314-cp314-win32.whl", hash = "sha256:03edcc34d688572014ff223c125a3f77fb08091e4607e7745002fc214070b35f"}, - {file = "pillow-12.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:50480dcd74fa63b8e78235957d302d98d98d82ccbfac4c7e12108ba9ecbdba15"}, - {file = "pillow-12.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:5cb1785d97b0c3d1d1a16bc1d710c4a0049daefc4935f3a8f31f827f4d3d2e7f"}, - {file = "pillow-12.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:1f90cff8aa76835cba5769f0b3121a22bd4eb9e6884cfe338216e557a9a548b8"}, - {file = "pillow-12.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1f1be78ce9466a7ee64bfda57bdba0f7cc499d9794d518b854816c41bf0aa4e9"}, - {file = "pillow-12.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:42fc1f4677106188ad9a55562bbade416f8b55456f522430fadab3cef7cd4e60"}, - {file = "pillow-12.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:98edb152429ab62a1818039744d8fbb3ccab98a7c29fc3d5fcef158f3f1f68b7"}, - {file = "pillow-12.1.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d470ab1178551dd17fdba0fef463359c41aaa613cdcd7ff8373f54be629f9f8f"}, - {file = "pillow-12.1.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6408a7b064595afcab0a49393a413732a35788f2a5092fdc6266952ed67de586"}, - {file = "pillow-12.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5d8c41325b382c07799a3682c1c258469ea2ff97103c53717b7893862d0c98ce"}, - {file = "pillow-12.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c7697918b5be27424e9ce568193efd13d925c4481dd364e43f5dff72d33e10f8"}, - {file = "pillow-12.1.1-cp314-cp314t-win32.whl", hash = "sha256:d2912fd8114fc5545aa3a4b5576512f64c55a03f3ebcca4c10194d593d43ea36"}, - {file = "pillow-12.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:4ceb838d4bd9dab43e06c363cab2eebf63846d6a4aeaea283bbdfd8f1a8ed58b"}, - {file = "pillow-12.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:7b03048319bfc6170e93bd60728a1af51d3dd7704935feb228c4d4faab35d334"}, - {file = "pillow-12.1.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:600fd103672b925fe62ed08e0d874ea34d692474df6f4bf7ebe148b30f89f39f"}, - {file = "pillow-12.1.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:665e1b916b043cef294bc54d47bf02d87e13f769bc4bc5fa225a24b3a6c5aca9"}, - {file = "pillow-12.1.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:495c302af3aad1ca67420ddd5c7bd480c8867ad173528767d906428057a11f0e"}, - {file = "pillow-12.1.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8fd420ef0c52c88b5a035a0886f367748c72147b2b8f384c9d12656678dfdfa9"}, - {file = "pillow-12.1.1-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f975aa7ef9684ce7e2c18a3aa8f8e2106ce1e46b94ab713d156b2898811651d3"}, - {file = "pillow-12.1.1-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8089c852a56c2966cf18835db62d9b34fef7ba74c726ad943928d494fa7f4735"}, - {file = "pillow-12.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:cb9bb857b2d057c6dfc72ac5f3b44836924ba15721882ef103cecb40d002d80e"}, - {file = "pillow-12.1.1.tar.gz", hash = "sha256:9ad8fa5937ab05218e2b6a4cff30295ad35afd2f83ac592e68c0d871bb0fdbc4"}, + {file = "pillow-12.2.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:a4e8f36e677d3336f35089648c8955c51c6d386a13cf6ee9c189c5f5bd713a9f"}, + {file = "pillow-12.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e589959f10d9824d39b350472b92f0ce3b443c0a3442ebf41c40cb8361c5b97"}, + {file = "pillow-12.2.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a52edc8bfff4429aaabdf4d9ee0daadbbf8562364f940937b941f87a4290f5ff"}, + {file = "pillow-12.2.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:975385f4776fafde056abb318f612ef6285b10a1f12b8570f3647ad0d74b48ec"}, + {file = "pillow-12.2.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd9c0c7a0c681a347b3194c500cb1e6ca9cab053ea4d82a5cf45b6b754560136"}, + {file = "pillow-12.2.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:88d387ff40b3ff7c274947ed3125dedf5262ec6919d83946753b5f3d7c67ea4c"}, + {file = "pillow-12.2.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:51c4167c34b0d8ba05b547a3bb23578d0ba17b80a5593f93bd8ecb123dd336a3"}, + {file = "pillow-12.2.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:34c0d99ecccea270c04882cb3b86e7b57296079c9a4aff88cb3b33563d95afaa"}, + {file = "pillow-12.2.0-cp310-cp310-win32.whl", hash = "sha256:b85f66ae9eb53e860a873b858b789217ba505e5e405a24b85c0464822fe88032"}, + {file = "pillow-12.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:673aa32138f3e7531ccdbca7b3901dba9b70940a19ccecc6a37c77d5fdeb05b5"}, + {file = "pillow-12.2.0-cp310-cp310-win_arm64.whl", hash = "sha256:3e080565d8d7c671db5802eedfb438e5565ffa40115216eabb8cd52d0ecce024"}, + {file = "pillow-12.2.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:8be29e59487a79f173507c30ddf57e733a357f67881430449bb32614075a40ab"}, + {file = "pillow-12.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:71cde9a1e1551df7d34a25462fc60325e8a11a82cc2e2f54578e5e9a1e153d65"}, + {file = "pillow-12.2.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f490f9368b6fc026f021db16d7ec2fbf7d89e2edb42e8ec09d2c60505f5729c7"}, + {file = "pillow-12.2.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8bd7903a5f2a4545f6fd5935c90058b89d30045568985a71c79f5fd6edf9b91e"}, + {file = "pillow-12.2.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3997232e10d2920a68d25191392e3a4487d8183039e1c74c2297f00ed1c50705"}, + {file = "pillow-12.2.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e74473c875d78b8e9d5da2a70f7099549f9eb37ded4e2f6a463e60125bccd176"}, + {file = "pillow-12.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:56a3f9c60a13133a98ecff6197af34d7824de9b7b38c3654861a725c970c197b"}, + {file = "pillow-12.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:90e6f81de50ad6b534cab6e5aef77ff6e37722b2f5d908686f4a5c9eba17a909"}, + {file = "pillow-12.2.0-cp311-cp311-win32.whl", hash = "sha256:8c984051042858021a54926eb597d6ee3012393ce9c181814115df4c60b9a808"}, + {file = "pillow-12.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:6e6b2a0c538fc200b38ff9eb6628228b77908c319a005815f2dde585a0664b60"}, + {file = "pillow-12.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:9a8a34cc89c67a65ea7437ce257cea81a9dad65b29805f3ecee8c8fe8ff25ffe"}, + {file = "pillow-12.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d192a155bbcec180f8564f693e6fd9bccff5a7af9b32e2e4bf8c9c69dbad6b5"}, + {file = "pillow-12.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3f40b3c5a968281fd507d519e444c35f0ff171237f4fdde090dd60699458421"}, + {file = "pillow-12.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:03e7e372d5240cc23e9f07deca4d775c0817bffc641b01e9c3af208dbd300987"}, + {file = "pillow-12.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b86024e52a1b269467a802258c25521e6d742349d760728092e1bc2d135b4d76"}, + {file = "pillow-12.2.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7371b48c4fa448d20d2714c9a1f775a81155050d383333e0a6c15b1123dda005"}, + {file = "pillow-12.2.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62f5409336adb0663b7caa0da5c7d9e7bdbaae9ce761d34669420c2a801b2780"}, + {file = "pillow-12.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:01afa7cf67f74f09523699b4e88c73fb55c13346d212a59a2db1f86b0a63e8c5"}, + {file = "pillow-12.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5"}, + {file = "pillow-12.2.0-cp312-cp312-win32.whl", hash = "sha256:58f62cc0f00fd29e64b29f4fd923ffdb3859c9f9e6105bfc37ba1d08994e8940"}, + {file = "pillow-12.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f84204dee22a783350679a0333981df803dac21a0190d706a50475e361c93f5"}, + {file = "pillow-12.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:af73337013e0b3b46f175e79492d96845b16126ddf79c438d7ea7ff27783a414"}, + {file = "pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:8297651f5b5679c19968abefd6bb84d95fe30ef712eb1b2d9b2d31ca61267f4c"}, + {file = "pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:50d8520da2a6ce0af445fa6d648c4273c3eeefbc32d7ce049f22e8b5c3daecc2"}, + {file = "pillow-12.2.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:766cef22385fa1091258ad7e6216792b156dc16d8d3fa607e7545b2b72061f1c"}, + {file = "pillow-12.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5d2fd0fa6b5d9d1de415060363433f28da8b1526c1c129020435e186794b3795"}, + {file = "pillow-12.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:56b25336f502b6ed02e889f4ece894a72612fe885889a6e8c4c80239ff6e5f5f"}, + {file = "pillow-12.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f1c943e96e85df3d3478f7b691f229887e143f81fedab9b20205349ab04d73ed"}, + {file = "pillow-12.2.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:03f6fab9219220f041c74aeaa2939ff0062bd5c364ba9ce037197f4c6d498cd9"}, + {file = "pillow-12.2.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cdfebd752ec52bf5bb4e35d9c64b40826bc5b40a13df7c3cda20a2c03a0f5ed"}, + {file = "pillow-12.2.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eedf4b74eda2b5a4b2b2fb4c006d6295df3bf29e459e198c90ea48e130dc75c3"}, + {file = "pillow-12.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:00a2865911330191c0b818c59103b58a5e697cae67042366970a6b6f1b20b7f9"}, + {file = "pillow-12.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1e1757442ed87f4912397c6d35a0db6a7b52592156014706f17658ff58bbf795"}, + {file = "pillow-12.2.0-cp313-cp313-win32.whl", hash = "sha256:144748b3af2d1b358d41286056d0003f47cb339b8c43a9ea42f5fea4d8c66b6e"}, + {file = "pillow-12.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:390ede346628ccc626e5730107cde16c42d3836b89662a115a921f28440e6a3b"}, + {file = "pillow-12.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:8023abc91fba39036dbce14a7d6535632f99c0b857807cbbbf21ecc9f4717f06"}, + {file = "pillow-12.2.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:042db20a421b9bafecc4b84a8b6e444686bd9d836c7fd24542db3e7df7baad9b"}, + {file = "pillow-12.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:dd025009355c926a84a612fecf58bb315a3f6814b17ead51a8e48d3823d9087f"}, + {file = "pillow-12.2.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88ddbc66737e277852913bd1e07c150cc7bb124539f94c4e2df5344494e0a612"}, + {file = "pillow-12.2.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d362d1878f00c142b7e1a16e6e5e780f02be8195123f164edf7eddd911eefe7c"}, + {file = "pillow-12.2.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c727a6d53cb0018aadd8018c2b938376af27914a68a492f59dfcaca650d5eea"}, + {file = "pillow-12.2.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd8c21c98c5cc60653bcb311bef2ce0401642b7ce9d09e03a7da87c878289d4"}, + {file = "pillow-12.2.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9f08483a632889536b8139663db60f6724bfcb443c96f1b18855860d7d5c0fd4"}, + {file = "pillow-12.2.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dac8d77255a37e81a2efcbd1fc05f1c15ee82200e6c240d7e127e25e365c39ea"}, + {file = "pillow-12.2.0-cp313-cp313t-win32.whl", hash = "sha256:ee3120ae9dff32f121610bb08e4313be87e03efeadfc6c0d18f89127e24d0c24"}, + {file = "pillow-12.2.0-cp313-cp313t-win_amd64.whl", hash = "sha256:325ca0528c6788d2a6c3d40e3568639398137346c3d6e66bb61db96b96511c98"}, + {file = "pillow-12.2.0-cp313-cp313t-win_arm64.whl", hash = "sha256:2e5a76d03a6c6dcef67edabda7a52494afa4035021a79c8558e14af25313d453"}, + {file = "pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3adc9215e8be0448ed6e814966ecf3d9952f0ea40eb14e89a102b87f450660d8"}, + {file = "pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:6a9adfc6d24b10f89588096364cc726174118c62130c817c2837c60cf08a392b"}, + {file = "pillow-12.2.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6a6e67ea2e6feda684ed370f9a1c52e7a243631c025ba42149a2cc5934dec295"}, + {file = "pillow-12.2.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2bb4a8d594eacdfc59d9e5ad972aa8afdd48d584ffd5f13a937a664c3e7db0ed"}, + {file = "pillow-12.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:80b2da48193b2f33ed0c32c38140f9d3186583ce7d516526d462645fd98660ae"}, + {file = "pillow-12.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22db17c68434de69d8ecfc2fe821569195c0c373b25cccb9cbdacf2c6e53c601"}, + {file = "pillow-12.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7b14cc0106cd9aecda615dd6903840a058b4700fcb817687d0ee4fc8b6e389be"}, + {file = "pillow-12.2.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cbeb542b2ebc6fcdacabf8aca8c1a97c9b3ad3927d46b8723f9d4f033288a0f"}, + {file = "pillow-12.2.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bfd07bc812fbd20395212969e41931001fd59eb55a60658b0e5710872e95286"}, + {file = "pillow-12.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9aba9a17b623ef750a4d11b742cbafffeb48a869821252b30ee21b5e91392c50"}, + {file = "pillow-12.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:deede7c263feb25dba4e82ea23058a235dcc2fe1f6021025dc71f2b618e26104"}, + {file = "pillow-12.2.0-cp314-cp314-win32.whl", hash = "sha256:632ff19b2778e43162304d50da0181ce24ac5bb8180122cbe1bf4673428328c7"}, + {file = "pillow-12.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:4e6c62e9d237e9b65fac06857d511e90d8461a32adcc1b9065ea0c0fa3a28150"}, + {file = "pillow-12.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:b1c1fbd8a5a1af3412a0810d060a78b5136ec0836c8a4ef9aa11807f2a22f4e1"}, + {file = "pillow-12.2.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:57850958fe9c751670e49b2cecf6294acc99e562531f4bd317fa5ddee2068463"}, + {file = "pillow-12.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d5d38f1411c0ed9f97bcb49b7bd59b6b7c314e0e27420e34d99d844b9ce3b6f3"}, + {file = "pillow-12.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c0a9f29ca8e79f09de89293f82fc9b0270bb4af1d58bc98f540cc4aedf03166"}, + {file = "pillow-12.2.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1610dd6c61621ae1cf811bef44d77e149ce3f7b95afe66a4512f8c59f25d9ebe"}, + {file = "pillow-12.2.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a34329707af4f73cf1782a36cd2289c0368880654a2c11f027bcee9052d35dd"}, + {file = "pillow-12.2.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e9c4f5b3c546fa3458a29ab22646c1c6c787ea8f5ef51300e5a60300736905e"}, + {file = "pillow-12.2.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06"}, + {file = "pillow-12.2.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f278f034eb75b4e8a13a54a876cc4a5ab39173d2cdd93a638e1b467fc545ac43"}, + {file = "pillow-12.2.0-cp314-cp314t-win32.whl", hash = "sha256:6bb77b2dcb06b20f9f4b4a8454caa581cd4dd0643a08bacf821216a16d9c8354"}, + {file = "pillow-12.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6562ace0d3fb5f20ed7290f1f929cae41b25ae29528f2af1722966a0a02e2aa1"}, + {file = "pillow-12.2.0-cp314-cp314t-win_arm64.whl", hash = "sha256:aa88ccfe4e32d362816319ed727a004423aab09c5cea43c01a4b435643fa34eb"}, + {file = "pillow-12.2.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0538bd5e05efec03ae613fd89c4ce0368ecd2ba239cc25b9f9be7ed426b0af1f"}, + {file = "pillow-12.2.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:394167b21da716608eac917c60aa9b969421b5dcbbe02ae7f013e7b85811c69d"}, + {file = "pillow-12.2.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5d04bfa02cc2d23b497d1e90a0f927070043f6cbf303e738300532379a4b4e0f"}, + {file = "pillow-12.2.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0c838a5125cee37e68edec915651521191cef1e6aa336b855f495766e77a366e"}, + {file = "pillow-12.2.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a6c9fa44005fa37a91ebfc95d081e8079757d2e904b27103f4f5fa6f0bf78c0"}, + {file = "pillow-12.2.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:25373b66e0dd5905ed63fa3cae13c82fbddf3079f2c8bf15c6fb6a35586324c1"}, + {file = "pillow-12.2.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:bfa9c230d2fe991bed5318a5f119bd6780cda2915cca595393649fc118ab895e"}, + {file = "pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5"}, ] [package.extras] @@ -2808,14 +2808,14 @@ six = ">=1.5" [[package]] name = "python-multipart" -version = "0.0.22" +version = "0.0.26" description = "A streaming multipart parser for Python" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "python_multipart-0.0.22-py3-none-any.whl", hash = "sha256:2b2cd894c83d21bf49d702499531c7bafd057d730c201782048f7945d82de155"}, - {file = "python_multipart-0.0.22.tar.gz", hash = "sha256:7340bef99a7e0032613f56dc36027b959fd3b30a787ed62d310e951f7c3a3a58"}, + {file = "python_multipart-0.0.26-py3-none-any.whl", hash = "sha256:c0b169f8c4484c13b0dcf2ef0ec3a4adb255c4b7d18d8e420477d2b1dd03f185"}, + {file = "python_multipart-0.0.26.tar.gz", hash = "sha256:08fadc45918cd615e26846437f50c5d6d23304da32c341f289a617127b081f17"}, ] [[package]] @@ -4050,4 +4050,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<4.0.0" -content-hash = "6b14e814692dc21ef76e136cceb6b778a18ba9e54c2766a38da71c1cbb87a779" +content-hash = "862c41754b254346f8cba03162cdc392565d69fcd552cb08c1d39f11b860219a" diff --git a/pyproject.toml b/pyproject.toml index 991539bc03..7d3141b2d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.152.1" +version = "1.153.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ @@ -66,7 +66,7 @@ dependencies = [ "prometheus-client>=0.6.0", # we use `order`, which arrived in attrs 19.2.0. # Note: 21.1.0 broke `/sync`, see https://github.com/matrix-org/synapse/issues/9936 - "attrs>=19.2.0,!=21.1.0", + "attrs>=26.1.0,!=21.1.0", "netaddr>=0.7.18", # Jinja 2.x is incompatible with MarkupSafe>=2.1. To ensure that admins do not # end up with a broken installation, with recent MarkupSafe but old Jinja, we @@ -137,7 +137,7 @@ saml2 = [ "defusedxml>=0.7.1", # via pysaml2 "pytz>=2018.3", # via pysaml2 ] -oidc = ["authlib>=1.6.11"] +oidc = ["authlib>=0.15.1"] url-preview = ["lxml>=4.6.3"] sentry = ["sentry-sdk>=0.7.2"] opentracing-jaeger = [ @@ -192,7 +192,7 @@ all = [ # saml2 "pysaml2>=4.5.0", # oidc and jwt - "authlib>=1.6.11", + "authlib>=0.15.1", # url-preview "lxml>=4.6.3", # sentry @@ -310,7 +310,7 @@ dev = [ # The following are used by the release script "click>=8.1.3", # GitPython was == 3.1.14; bumped to 3.1.20, the first release with type hints. - "GitPython>=3.1.20", + "GitPython>=3.1.47", "markdown-it-py>=3.0.0", "pygithub>=1.59", # The following are executed as commands by the release script. diff --git a/rust/Cargo.toml b/rust/Cargo.toml index bca2f6ed70..162bc98182 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -30,7 +30,7 @@ http = "1.1.0" lazy_static = "1.4.0" log = "0.4.17" mime = "0.3.17" -pyo3 = { version = "0.27.2", features = [ +pyo3 = { version = "0.28.3", features = [ "macros", "anyhow", "abi3", @@ -39,12 +39,18 @@ pyo3 = { version = "0.27.2", features = [ # https://docs.rs/pyo3/latest/pyo3/bytes/index.html "bytes", ] } -pyo3-log = "0.13.1" -pythonize = "0.27.0" +pyo3-log = "0.13.3" +pythonize = { version = "0.28.0", features = ["arbitrary_precision"] } regex = "1.6.0" sha2 = "0.10.8" -serde = { version = "1.0.144", features = ["derive"] } -serde_json = "1.0.85" +serde = { version = "1.0.144", features = ["derive", "rc"] } +serde_json = { version = "1.0.85", features = [ + "raw_value", + # We need to be able to parse arbitrary precision numbers, as some numbers + # in the database may be out of range of i64 (as Python uses arbitrary + # precision integers). + "arbitrary_precision", +] } ulid = "1.1.2" icu_segmenter = "2.0.0" reqwest = { version = "0.12.15", default-features = false, features = [ @@ -56,10 +62,7 @@ http-body-util = "0.1.3" futures = "0.3.31" tokio = { version = "1.44.2", features = ["rt", "rt-multi-thread"] } once_cell = "1.18.0" - -[features] -extension-module = ["pyo3/extension-module"] -default = ["extension-module"] +itertools = "0.14.0" [build-dependencies] blake2 = "0.10.4" diff --git a/rust/src/acl/mod.rs b/rust/src/acl/mod.rs index 57b45475fd..ab10b9f037 100644 --- a/rust/src/acl/mod.rs +++ b/rust/src/acl/mod.rs @@ -47,7 +47,7 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> } #[derive(Debug, Clone)] -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] pub struct ServerAclEvaluator { allow_ip_literals: bool, allow: Vec, diff --git a/rust/src/canonical_json.rs b/rust/src/canonical_json.rs new file mode 100644 index 0000000000..94ed6c368d --- /dev/null +++ b/rust/src/canonical_json.rs @@ -0,0 +1,945 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + * Originally licensed under the Apache License, Version 2.0: + * . + * + * [This file includes modifications made by Element Creations Ltd] + */ + +//! Serialize a Rust data structure into canonical JSON data. +//! +//! See the [Canonical +//! JSON](https://matrix.org/docs/spec/appendices#canonical-json) docs for more +//! information. + +use std::{ + collections::BTreeMap, + convert::TryFrom, + io::{self, Write}, +}; + +use serde::{ser::SerializeMap, Serializer as _}; +use serde::{ + ser::{Error as _, SerializeStruct}, + Serialize, +}; +use serde_json::{ + ser::{Formatter, Serializer}, + value::RawValue, + Number, Value, +}; + +/// The minimum integer that can be used in canonical JSON. +pub const MIN_VALID_INTEGER: i64 = -(2i64.pow(53)) + 1; + +/// The maximum integer that can be used in canonical JSON. +pub const MAX_VALID_INTEGER: i64 = (2i64.pow(53)) - 1; + +/// A token used by `serde_json` to identify its internal `Number` type when the +/// `arbitrary_precision` feature is enabled. This is a copy from serde_json's +/// internal `TOKEN` for `Number`, which unfortunately isn't exported by the +/// crate. +const SERDE_JSON_NUMBER_TOKEN: &str = "$serde_json::private::Number"; + +/// Options to control how strict JSON canonicalization is. +#[derive(Clone, Debug)] +pub struct CanonicalizationOptions { + /// Configure the serializer to strictly enforce the canonical JSON allowable number range. + /// Allows JSON for room versions v5 or less when `false`. + enforce_int_range: bool, +} + +impl CanonicalizationOptions { + /// Creates an instance of [CanonicalizationOptions] with permissive JSON enforcement settings. + pub fn relaxed() -> Self { + Self { + enforce_int_range: false, + } + } + + /// Creates an instance of [CanonicalizationOptions] with strict JSON enforcement settings. + pub fn strict() -> Self { + Self { + enforce_int_range: true, + } + } +} + +/// Serialize the given data structure as a canonical JSON byte vector. +/// +/// See the [Canonical +/// JSON](https://matrix.org/docs/spec/appendices#canonical-json) docs for more +/// information. +/// +/// Note: serializing [`RawValue`] is not supported, as it may contain JSON that +/// is not canonical. +/// +/// # Errors +/// +/// Serialization can fail if `T`'s implementation of `Serialize` decides to +/// fail, if `T` contains a map with non-string keys, or if `T` contains numbers +/// that are not integers in the range `[-2**53 + 1, 2**53 - 1]`. +pub fn to_vec_canonical( + value: &T, + options: CanonicalizationOptions, +) -> Result, serde_json::Error> +where + T: Serialize + ?Sized, +{ + let mut vec = Vec::new(); + let mut ser = CanonicalSerializer::new(&mut vec, options); + value.serialize(&mut ser)?; + + Ok(vec) +} + +/// Serialize the given data structure as a canonical JSON string. +/// +/// See the [Canonical +/// JSON](https://matrix.org/docs/spec/appendices#canonical-json) docs for more +/// information. +/// +/// Note: serializing [`RawValue`] is not supported, as it may contain JSON that +/// is not canonical. +/// +/// # Errors +/// +/// Serialization can fail if `T`'s implementation of `Serialize` decides to +/// fail, if `T` contains a map with non-string keys, or if `T` contains numbers +/// that are not integers in the range `[-2**53 + 1, 2**53 - 1]`. +pub fn to_string_canonical( + value: &T, + options: CanonicalizationOptions, +) -> Result +where + T: Serialize + ?Sized, +{ + let vec = to_vec_canonical(value, options)?; + + // We'll always get valid UTF-8 out + let json_string = String::from_utf8(vec).expect("valid utf8"); + + Ok(json_string) +} + +/// A helper function that asserts that an integer is in the valid range. +fn assert_integer_in_range(v: I) -> Result<(), serde_json::Error> +where + i64: TryFrom, +{ + let res = i64::try_from(v); + match res { + Ok(MIN_VALID_INTEGER..=MAX_VALID_INTEGER) => Ok(()), + Ok(_) | Err(_) => Err(serde_json::Error::custom("integer out of range")), + } +} + +/// A JSON formatter that ensures all strings are encoded as per the [Canonical +/// JSON](https://matrix.org/docs/spec/appendices#canonical-json) spec. +pub struct CanonicalFormatter; + +impl Formatter for CanonicalFormatter { + fn write_string_fragment(&mut self, writer: &mut W, fragment: &str) -> io::Result<()> + where + W: ?Sized + io::Write, + { + // `fragment` only contains characters that are not escaped, and don't + // need to be escaped, so they can be written directly to the writer. + writer.write_all(fragment.as_bytes()) + } + + fn write_char_escape( + &mut self, + writer: &mut W, + char_escape: serde_json::ser::CharEscape, + ) -> io::Result<()> + where + W: ?Sized + io::Write, + { + use serde_json::ser::CharEscape::*; + + let s = match char_escape { + Quote => b"\\\"" as &[u8], + ReverseSolidus => b"\\\\", + Solidus => b"/", // Note: this doesn't need to be escaped (and appears unused in serde_json). + Backspace => b"\\b", + FormFeed => b"\\f", + LineFeed => b"\\n", + CarriageReturn => b"\\r", + Tab => b"\\t", + AsciiControl(byte) => { + static HEX_DIGITS: [u8; 16] = *b"0123456789abcdef"; + let bytes = &[ + b'\\', + b'u', + b'0', + b'0', + HEX_DIGITS[(byte >> 4) as usize], + HEX_DIGITS[(byte & 0xF) as usize], + ]; + return writer.write_all(bytes); + } + }; + + writer.write_all(s) + } +} + +/// A JSON serializer that outputs [Canonical +/// JSON](https://matrix.org/docs/spec/appendices#canonical-json). +pub struct CanonicalSerializer { + inner: Serializer, + options: CanonicalizationOptions, +} + +impl CanonicalSerializer +where + W: Write, +{ + /// Create a new serializer that writes the canonical JSON bytes to the + /// given writer. + pub fn new(writer: W, options: CanonicalizationOptions) -> Self { + Self { + inner: Serializer::with_formatter(writer, CanonicalFormatter), + options, + } + } +} + +// We implement the serializer by proxying all calls to the standard +// `serde_json` serializer, except where we a) buffer up maps and structs so that we can +// sort them, and b) ensure that all numbers are integers in the valid range. +impl<'a, W> serde::Serializer for &'a mut CanonicalSerializer +where + W: Write, +{ + type Ok = <&'a mut Serializer as serde::Serializer>::Ok; + + type Error = <&'a mut Serializer as serde::Serializer>::Error; + + type SerializeSeq = + <&'a mut Serializer as serde::Serializer>::SerializeSeq; + + type SerializeTuple = + <&'a mut Serializer as serde::Serializer>::SerializeTuple; + + type SerializeTupleStruct = + <&'a mut Serializer as serde::Serializer>::SerializeTupleStruct; + + type SerializeTupleVariant = + <&'a mut Serializer as serde::Serializer>::SerializeTupleVariant; + + type SerializeMap = CanonicalSerializeMap<'a, W>; + + type SerializeStruct = CanonicalSerializeStruct<'a, W>; + + type SerializeStructVariant = + <&'a mut Serializer as serde::Serializer>::SerializeStructVariant; + + fn serialize_bool(self, v: bool) -> Result { + self.inner.serialize_bool(v) + } + + fn serialize_i8(self, v: i8) -> Result { + assert_integer_in_range(v)?; + + self.inner.serialize_i8(v) + } + + fn serialize_i16(self, v: i16) -> Result { + assert_integer_in_range(v)?; + + self.inner.serialize_i16(v) + } + + fn serialize_i32(self, v: i32) -> Result { + assert_integer_in_range(v)?; + + self.inner.serialize_i32(v) + } + + fn serialize_i64(self, v: i64) -> Result { + if self.options.enforce_int_range { + assert_integer_in_range(v)?; + } + + self.inner.serialize_i64(v) + } + + fn serialize_i128(self, v: i128) -> Result { + if self.options.enforce_int_range { + assert_integer_in_range(v)?; + } + + self.inner.serialize_i128(v) + } + + fn serialize_u8(self, v: u8) -> Result { + assert_integer_in_range(v)?; + + self.inner.serialize_u8(v) + } + + fn serialize_u16(self, v: u16) -> Result { + self.inner.serialize_u16(v) + } + + fn serialize_u32(self, v: u32) -> Result { + assert_integer_in_range(v)?; + + self.inner.serialize_u32(v) + } + + fn serialize_u64(self, v: u64) -> Result { + if self.options.enforce_int_range { + assert_integer_in_range(v)?; + } + + self.inner.serialize_u64(v) + } + + fn serialize_u128(self, v: u128) -> Result { + if self.options.enforce_int_range { + assert_integer_in_range(v)?; + } + + self.inner.serialize_u128(v) + } + + fn serialize_f32(self, _: f32) -> Result { + Err(serde_json::Error::custom( + "non-integer numbers are not allowed", + )) + } + + fn serialize_f64(self, _: f64) -> Result { + Err(serde_json::Error::custom( + "non-integer numbers are not allowed", + )) + } + + fn serialize_char(self, v: char) -> Result { + self.inner.serialize_char(v) + } + + fn serialize_str(self, v: &str) -> Result { + self.inner.serialize_str(v) + } + + fn serialize_bytes(self, v: &[u8]) -> Result { + self.inner.serialize_bytes(v) + } + + fn serialize_none(self) -> Result { + self.inner.serialize_none() + } + + fn serialize_some(self, value: &T) -> Result + where + T: serde::Serialize + ?Sized, + { + self.inner.serialize_some(value) + } + + fn serialize_unit(self) -> Result { + self.inner.serialize_unit() + } + + fn serialize_unit_struct(self, name: &'static str) -> Result { + self.inner.serialize_unit_struct(name) + } + + fn serialize_unit_variant( + self, + name: &'static str, + variant_index: u32, + variant: &'static str, + ) -> Result { + self.inner + .serialize_unit_variant(name, variant_index, variant) + } + + fn serialize_newtype_struct( + self, + name: &'static str, + value: &T, + ) -> Result + where + T: serde::Serialize + ?Sized, + { + self.inner.serialize_newtype_struct(name, value) + } + + fn serialize_newtype_variant( + self, + name: &'static str, + variant_index: u32, + variant: &'static str, + value: &T, + ) -> Result + where + T: serde::Serialize + ?Sized, + { + self.inner + .serialize_newtype_variant(name, variant_index, variant, value) + } + + fn serialize_seq(self, len: Option) -> Result { + self.inner.serialize_seq(len) + } + + fn serialize_tuple(self, len: usize) -> Result { + self.inner.serialize_tuple(len) + } + + fn serialize_tuple_struct( + self, + name: &'static str, + len: usize, + ) -> Result { + self.inner.serialize_tuple_struct(name, len) + } + + fn serialize_tuple_variant( + self, + name: &'static str, + variant_index: u32, + variant: &'static str, + len: usize, + ) -> Result { + self.inner + .serialize_tuple_variant(name, variant_index, variant, len) + } + + fn serialize_map(self, _len: Option) -> Result { + Ok(CanonicalSerializeMap::new( + &mut self.inner, + self.options.clone(), + )) + } + + fn serialize_struct( + self, + name: &'static str, + len: usize, + ) -> Result { + // We want to disallow `RawValue` as we don't know if its contents is + // canonical JSON. + // + // Note: the `name` here comes from `serde_json::raw::TOKEN`, which + // unfortunately isn't exported by the crate. + if name == "$serde_json::private::RawValue" { + return Err(Self::Error::custom("`RawValue` is not supported")); + } + CanonicalSerializeStruct::new(name, len, &mut self.inner, self.options.clone()) + } + + fn serialize_struct_variant( + self, + name: &'static str, + variant_index: u32, + variant: &'static str, + len: usize, + ) -> Result { + self.inner + .serialize_struct_variant(name, variant_index, variant, len) + } + + fn collect_str(self, value: &T) -> Result + where + T: std::fmt::Display + ?Sized, + { + self.inner.collect_str(value) + } +} + +/// A helper type for [`CanonicalSerializer`] that serializes JSON maps in +/// lexicographic order. +#[doc(hidden)] +pub struct CanonicalSerializeMap<'a, W> { + // We buffer up the key and serialized value for each field we see. + // The BTreeMap will then serialize in lexicographic order. + map: BTreeMap>, + // A key which we're still waiting for a value for + last_key: Option, + // The serializer to use to write the sorted map too. + ser: &'a mut Serializer, + options: CanonicalizationOptions, +} + +impl<'a, W> CanonicalSerializeMap<'a, W> { + fn new( + ser: &'a mut Serializer, + options: CanonicalizationOptions, + ) -> Self { + Self { + map: BTreeMap::new(), + last_key: None, + ser, + options, + } + } +} + +impl<'a, W> SerializeMap for CanonicalSerializeMap<'a, W> +where + W: Write, +{ + type Ok = (); + + type Error = serde_json::Error; + + fn serialize_key(&mut self, key: &T) -> Result<(), Self::Error> + where + T: serde::Serialize + ?Sized, + { + if self.last_key.is_some() { + // This can only happen if `serialize_key` is called multiple times + // in a row without a `serialize_value` call in between. This + // violates the contract of `SerializeMap`. + return Err(Self::Error::custom( + "serialize_key called multiple times in a row without serialize_value", + )); + } + + // Parse the `key` into a string. + let key_string = if let Value::String(str) = serde_json::to_value(key)? { + str + } else { + return Err(Self::Error::custom("key must be a string")); + }; + + self.last_key = Some(key_string); + + Ok(()) + } + + fn serialize_value(&mut self, value: &T) -> Result<(), Self::Error> + where + T: serde::Serialize + ?Sized, + { + let key_string = if let Some(key_string) = self.last_key.take() { + key_string + } else { + // `serde` should ensure that for every `serialize_key` there is a + // `serialize_field` call, so `last_key` should never be None here. + unreachable!() + }; + + // We serialize the value canonically, then store it as a `RawValue` in + // the buffer map. + let value_string = to_string_canonical(value, self.options.clone())?; + + self.map + .insert(key_string, RawValue::from_string(value_string)?); + + Ok(()) + } + + fn end(self) -> Result { + // No more entries in the map being serialized, so we can now serialize + // our buffered map (which will be serialized in the correct order as + // its a BTreeMap). + self.map.serialize(self.ser)?; + + Ok(()) + } +} + +/// A helper type for [`CanonicalSerializer`] that serializes structs in +/// lexicographic order. +#[doc(hidden)] +pub struct CanonicalSerializeStruct<'a, W: Write> { + name: &'static str, + // We buffer up the key and serialized value for each field we see. + // The BTreeMap will then serialize in lexicographic order. + map: BTreeMap<&'static str, Box>, + options: CanonicalizationOptions, + // The serializer to use to write the sorted map too. + struct_serializer: + <&'a mut Serializer as serde::Serializer>::SerializeStruct, +} + +impl<'a, W> CanonicalSerializeStruct<'a, W> +where + W: Write, +{ + fn new( + name: &'static str, + len: usize, + ser: &'a mut Serializer, + options: CanonicalizationOptions, + ) -> Result { + let struct_serializer = ser.serialize_struct(name, len)?; + + Ok(Self { + name, + map: BTreeMap::new(), + options, + struct_serializer, + }) + } +} + +impl<'a, W> SerializeStruct for CanonicalSerializeStruct<'a, W> +where + W: Write, +{ + type Ok = (); + + type Error = serde_json::Error; + + fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error> + where + T: Serialize + ?Sized, + { + // Check if this is the special case of `SERDE_JSON_NUMBER_TOKEN`, + // which is used when serializing numbers with the `arbitrary_precision` + // feature. If so, we can just serialize it directly without + // canonicalizing it first, as `serde_json` will have already serialized + // it in a canonical way. + if key == SERDE_JSON_NUMBER_TOKEN && self.name == SERDE_JSON_NUMBER_TOKEN { + if self.options.enforce_int_range { + // We need to check that the number is in the valid range, as + // `serde_json` won't have done this for us as we're using the + // `arbitrary_precision` feature. + + // The value here will be something that serializes to a JSON + // string containing the number, so we first serialize it to a + // Value and pull the string out, then parse it as a `Number`. + + let serde_val = serde_json::to_value(value)?; + let serde_json::Value::String(number_str) = serde_val else { + return Err(serde_json::Error::custom("invalid number")); + }; + + let number: Number = number_str + .parse() + .map_err(|_| serde_json::Error::custom("invalid number"))?; + + // Now check that the number is an integer in the valid range. + if let Some(int) = number.as_i64() { + assert_integer_in_range(int)?; + } else { + // Can't be cast to an i64, so it must be out of range. + return Err(serde_json::Error::custom("integer out of range")); + } + } + + self.struct_serializer.serialize_field(key, value)?; + return Ok(()); + } + + // We serialize the value canonically, then store it as a `RawValue` in + // the buffer map. + let value_string = to_string_canonical(value, self.options.clone())?; + + self.map.insert(key, RawValue::from_string(value_string)?); + + Ok(()) + } + + fn end(mut self) -> Result { + if self.name == SERDE_JSON_NUMBER_TOKEN { + // Map must be empty in this case, as `SERDE_JSON_NUMBER_TOKEN` + // only has one field and we've handled it in `serialize_field`. + if !self.map.is_empty() { + return Err(Self::Error::custom(format!( + "unexpected fields in `{}`", + SERDE_JSON_NUMBER_TOKEN + ))); + } + } + + for (key, value) in self.map { + self.struct_serializer.serialize_field(key, &value)?; + } + + SerializeStruct::end(self.struct_serializer)?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use itertools::Itertools; + use serde::Serializer; + use serde_json::json; + + use super::*; + + #[test] + fn empty() { + let test = json!({}); + + let json_string = to_string_canonical(&test, CanonicalizationOptions::strict()).unwrap(); + + assert_eq!(json_string, r#"{}"#); + } + + #[test] + fn order_struct_fields() { + #[derive(Serialize)] + struct Test { + b: u8, + a: u8, + } + + let test = Test { b: 1, a: 2 }; + + let json_string = to_string_canonical(&test, CanonicalizationOptions::strict()).unwrap(); + + assert_eq!(json_string, r#"{"a":2,"b":1}"#); + } + + #[test] + fn strings() { + let test = json!({ + "a": "\u{1F37B}", + "b": "\n", + "c": "\x01", + }); + + let json_string = to_string_canonical(&test, CanonicalizationOptions::strict()).unwrap(); + + assert_eq!(json_string, r#"{"a":"🍻","b":"\n","c":"\u0001"}"#); + } + + #[test] + fn escapes() { + let mut buffer; + let mut char_buffer = [0u8; 4]; + + // Ensure that we encode every UTF-8 character correctly + for c in '\0'..='\u{10FFFF}' { + // Serialize the character and strip out the quotes to make comparison easier. + let json_string = to_string_canonical(&c, CanonicalizationOptions::strict()).unwrap(); + let unquoted_json_string = &json_string[1..json_string.len() - 1]; + + let expected = match c { + // Some control characters have specific escape codes. + '\x08' => r"\b", + '\x09' => r"\t", + '\x0A' => r"\n", + '\x0C' => r"\f", + '\x0D' => r"\r", + '\x22' => r#"\""#, + '\x5C' => r"\\", + // Otherwise any character less than \x1F gets escaped as + // `\u00xx` + '\0'..='\x1F' => { + buffer = format!(r"\u00{:02x}", c as u32); + &buffer + } + // And everything else doesn't get escaped + _ => c.encode_utf8(&mut char_buffer), + }; + + // The serialized character will be wrapped in quotes. + assert_eq!(unquoted_json_string, expected); + } + } + + #[test] + fn nested_map() { + let test = json!({ + "a": {"b": 1} + }); + + let json_string = to_string_canonical(&test, CanonicalizationOptions::strict()).unwrap(); + + assert_eq!(json_string, r#"{"a":{"b":1}}"#); + } + + #[test] + fn floats() { + assert!(to_string_canonical(&100.0f32, CanonicalizationOptions::strict()).is_err()); + assert!(to_string_canonical(&100.0f64, CanonicalizationOptions::strict()).is_err()); + } + + #[test] + fn integers() { + assert_eq!( + to_string_canonical(&100u8, CanonicalizationOptions::strict()).unwrap(), + "100" + ); + assert_eq!( + to_string_canonical(&100u16, CanonicalizationOptions::strict()).unwrap(), + "100" + ); + assert_eq!( + to_string_canonical(&100u32, CanonicalizationOptions::strict()).unwrap(), + "100" + ); + assert_eq!( + to_string_canonical(&100u64, CanonicalizationOptions::strict()).unwrap(), + "100" + ); + assert_eq!( + to_string_canonical(&100u128, CanonicalizationOptions::strict()).unwrap(), + "100" + ); + + assert_eq!( + to_string_canonical(&100i8, CanonicalizationOptions::strict()).unwrap(), + "100" + ); + assert_eq!( + to_string_canonical(&100i16, CanonicalizationOptions::strict()).unwrap(), + "100" + ); + assert_eq!( + to_string_canonical(&100i32, CanonicalizationOptions::strict()).unwrap(), + "100" + ); + assert_eq!( + to_string_canonical(&100i64, CanonicalizationOptions::strict()).unwrap(), + "100" + ); + assert_eq!( + to_string_canonical(&100i128, CanonicalizationOptions::strict()).unwrap(), + "100" + ); + + assert!(to_string_canonical(&2u64.pow(60), CanonicalizationOptions::strict()).is_err()); + assert!(to_string_canonical(&2u128.pow(60), CanonicalizationOptions::strict()).is_err()); + + assert!(to_string_canonical(&2i64.pow(60), CanonicalizationOptions::strict()).is_err()); + assert!(to_string_canonical(&2i128.pow(60), CanonicalizationOptions::strict()).is_err()); + assert!(to_string_canonical(&-(2i64.pow(60)), CanonicalizationOptions::strict()).is_err()); + assert!(to_string_canonical(&-(2i128.pow(60)), CanonicalizationOptions::strict()).is_err()); + } + + #[test] + fn bigints() { + // Create a `serde_json::Number` that is too big to be represented as an + // i64, but can be represented as a string. + let bigint_string = "10000000000000000000000000000000000000"; + let value: serde_json::Number = bigint_string.parse().unwrap(); + + // This should work with relaxed option. + assert_eq!( + to_string_canonical(&value, CanonicalizationOptions::relaxed()).unwrap(), + bigint_string + ); + + // But should fail with strict option, as it's out of range. + assert!(to_string_canonical(&value, CanonicalizationOptions::strict()).is_err()); + } + + #[test] + fn backwards_compatibility() { + assert_eq!( + to_string_canonical(&u64::MAX, CanonicalizationOptions::relaxed()).unwrap(), + format!("{}", u64::MAX) + ); + assert_eq!( + to_string_canonical(&u128::MAX, CanonicalizationOptions::relaxed()).unwrap(), + format!("{}", u128::MAX) + ); + assert_eq!( + to_string_canonical(&i128::MAX, CanonicalizationOptions::relaxed()).unwrap(), + format!("{}", i128::MAX) + ); + assert_eq!( + to_string_canonical(&-i128::MAX, CanonicalizationOptions::relaxed()).unwrap(), + format!("{}", -i128::MAX) + ); + } + + #[test] + fn hashmap_order() { + let mut test = HashMap::new(); + test.insert("e", 1); + test.insert("d", 1); + test.insert("c", 1); + test.insert("b", 1); + test.insert("a", 1); + test.insert("AA", 1); + + let json_string = to_string_canonical(&test, CanonicalizationOptions::strict()).unwrap(); + + assert_eq!(json_string, r#"{"AA":1,"a":1,"b":1,"c":1,"d":1,"e":1}"#); + } + + #[test] + fn raw_value() { + let raw_value = RawValue::from_string("{}".to_string()).unwrap(); + + assert!(to_string_canonical(&raw_value, CanonicalizationOptions::strict()).is_err()); + } + + #[test] + fn map_with_duplicate_keys() { + let mut output = Vec::new(); + let mut serializer = + CanonicalSerializer::new(&mut output, CanonicalizationOptions::strict()); + let mut map_serializer = serializer.serialize_map(None).unwrap(); + + map_serializer.serialize_entry("a", &1).unwrap(); + map_serializer.serialize_entry("a", &2).unwrap(); + + // Also try with different representations of the same key (e.g. `\t` and `\u{0009}`). + map_serializer.serialize_entry("\t", &2).unwrap(); + map_serializer.serialize_entry("\u{0009}", &2).unwrap(); + + SerializeMap::end(map_serializer).unwrap(); + + assert_eq!(String::from_utf8(output).unwrap(), r#"{"\t":2,"a":2}"#); + } + + #[test] + fn map_with_out_of_order_keys() { + let mut output = Vec::new(); + let mut serializer = + CanonicalSerializer::new(&mut output, CanonicalizationOptions::strict()); + let mut map_serializer = serializer.serialize_map(None).unwrap(); + + // An ordered list of keys to insert, and the expected way they should be serialized. + let ascii_order = [ + ('\0', r"\u0000"), + ('\t', r"\t"), + (' ', r" "), + ('!', r"!"), + ('"', r#"\""#), + ('&', r"&"), + ('A', r"A"), + ('\\', r"\\"), + ('a', r"a"), + ('🍻', r"🍻"), + ]; + + // Double check that the keys are in the expected order. + assert!(ascii_order.is_sorted_by_key(|(c, _)| u32::from(*c))); + + // Serialize the keys in the reverse order. + for (c, _) in ascii_order.iter().rev() { + map_serializer.serialize_entry(c, &1).unwrap(); + } + SerializeMap::end(map_serializer).unwrap(); + + // The expected JSON should have the keys in the correct order, and the + // correct escaping. + let expected_json_inner = ascii_order + .iter() + .map(|(_, escaped)| format!(r#""{escaped}":1"#)) + .join(","); + let expected_json = r"{".to_owned() + &expected_json_inner + r"}"; + + assert_eq!(String::from_utf8(output).unwrap(), expected_json); + } +} diff --git a/rust/src/events/internal_metadata.rs b/rust/src/events/internal_metadata.rs index 6fd3d06b00..4084b8442d 100644 --- a/rust/src/events/internal_metadata.rs +++ b/rust/src/events/internal_metadata.rs @@ -476,7 +476,7 @@ impl EventInternalMetadataInner { } } -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] #[derive(Clone)] pub struct EventInternalMetadata { inner: Arc>, diff --git a/rust/src/events/mod.rs b/rust/src/events/mod.rs index 209efb917b..5f505abb91 100644 --- a/rust/src/events/mod.rs +++ b/rust/src/events/mod.rs @@ -27,11 +27,15 @@ use pyo3::{ pub mod filter; mod internal_metadata; +pub mod signatures; +pub mod unsigned; /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { let child_module = PyModule::new(py, "events")?; child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; child_module.add_function(wrap_pyfunction!(filter::event_visible_to_server_py, m)?)?; m.add_submodule(&child_module)?; diff --git a/rust/src/events/signatures.rs b/rust/src/events/signatures.rs new file mode 100644 index 0000000000..0f2acd5c9b --- /dev/null +++ b/rust/src/events/signatures.rs @@ -0,0 +1,348 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Class for representing event signatures + +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; + +use pyo3::{ + exceptions::{PyKeyError, PyRuntimeError}, + pyclass, pymethods, + types::{PyAnyMethods, PyDict, PyMapping, PyMappingMethods}, + Bound, IntoPyObject, PyAny, PyResult, Python, +}; +use serde::{Deserialize, Serialize}; + +/// A class representing the signatures on an event. +#[pyclass(frozen, skip_from_py_object)] +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(transparent)] +pub struct Signatures { + inner: Arc>>>, +} + +#[pymethods] +impl Signatures { + #[new] + #[pyo3(signature = (signatures = None))] + fn py_new(signatures: Option>>) -> Self { + let mut signatures = signatures.unwrap_or_default(); + + // Prune any entries that have no signatures. + signatures.retain(|_, server_sigs| !server_sigs.is_empty()); + + Self { + inner: Arc::new(RwLock::new(signatures)), + } + } + + /// Check if the signatures contain a signature for the given server name. + fn __contains__(&self, key: Bound<'_, PyAny>) -> PyResult { + let Ok(key) = key.extract::<&str>() else { + return Ok(false); + }; + + let signatures = self + .inner + .read() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + Ok(signatures.contains_key(key)) + } + + /// Get the number of servers that have signatures. + fn __len__(&self) -> PyResult { + let signatures = self + .inner + .read() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + Ok(signatures.len()) + } + + /// Get the signature for the given server name and key ID, if it exists. + fn get_signature(&self, server_name: &str, key_id: &str) -> PyResult> { + let signatures = self + .inner + .read() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + + Ok(signatures + .get(server_name) + .and_then(|server_sigs| server_sigs.get(key_id).cloned())) + } + + /// Get the signatures for the given server name. + fn __getitem__(&self, key: Bound<'_, PyAny>) -> PyResult> { + let Some(server_name) = key.extract::<&str>().ok() else { + return Err(PyKeyError::new_err(key.to_string())); + }; + + let signatures = self + .inner + .read() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + + if let Some(server_sigs) = signatures.get(server_name) { + Ok(server_sigs.clone()) + } else { + Err(PyKeyError::new_err(server_name.to_string())) + } + } + + /// Add a signature for the given server name and key ID. + fn add_signature( + &self, + server_name: String, + key_id: String, + signature: String, + ) -> PyResult<()> { + let mut signatures = self + .inner + .write() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + + signatures + .entry(server_name) + .or_default() + .insert(key_id, signature); + + Ok(()) + } + + /// Update the signatures with the given signatures. + /// + /// Will overwrite all existing signatures for the server names provided. + fn update(&self, other: &Bound<'_, PyMapping>) -> PyResult<()> { + let mut signatures = self + .inner + .write() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + + for list_entry in other.items()? { + let (server_name, server_sigs) = list_entry.extract::<(String, Bound)>()?; + + let mut entry = HashMap::new(); + for list_entry in server_sigs.items()? { + let (key, value) = list_entry.extract::<(String, String)>()?; + entry.insert(key, value); + } + + // Only insert the entry if it has at least one signature. + if !entry.is_empty() { + signatures.insert(server_name, entry); + } else { + signatures.remove(&server_name); + } + } + + Ok(()) + } + + /// Return a copy of the signatures as a dictionary. + fn as_dict<'py>(&self, py: Python<'py>) -> PyResult> { + let signatures = self + .inner + .read() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + + (&*signatures).into_pyobject(py) + } + + fn __repr__(&self) -> PyResult { + let signatures = self + .inner + .read() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + + Ok(format!("Signatures({signatures:?})")) + } +} + +#[cfg(test)] +mod tests { + use pythonize::pythonize; + + use super::*; + + /// Helper that reads the inner map directly. + fn read_inner(sigs: &Signatures) -> HashMap> { + sigs.inner.read().expect("lock poisoned").clone() + } + + /// Helper to create a server signatures map from a list of (key_id, sig) + /// pairs. + fn make_server_sigs(data: &[(&str, &str)]) -> HashMap { + let mut map = HashMap::new(); + for (key_id, sig) in data { + map.insert((*key_id).to_owned(), (*sig).to_owned()); + } + map + } + + /// Helper to create a `Signatures` object from a list of (server_name, + /// key_id, sig) tuples. + fn create_signatures(data: &[(&str, &str, &str)]) -> Signatures { + let mut map: HashMap> = HashMap::new(); + for (server_name, key_id, sig) in data { + map.entry((*server_name).to_owned()) + .or_default() + .insert((*key_id).to_owned(), (*sig).to_owned()); + } + Signatures::py_new(Some(map)) + } + + #[test] + fn test_new_empty() { + let sigs = Signatures::py_new(None); + assert!(read_inner(&sigs).is_empty()); + assert_eq!(sigs.__len__().unwrap(), 0); + } + + #[test] + fn test_new_with_data() { + let sigs = create_signatures(&[("example.com", "ed25519:key1", "sig1")]); + assert_eq!(sigs.__len__().unwrap(), 1); + assert_eq!( + sigs.get_signature("example.com", "ed25519:key1").unwrap(), + Some("sig1".to_string()) + ); + } + + #[test] + fn test_new_prunes_servers_with_no_signatures() { + let mut data = HashMap::new(); + data.insert("empty.example.com".to_string(), HashMap::new()); + data.insert( + "example.com".to_string(), + make_server_sigs(&[("ed25519:key1", "sig1")]), + ); + + let sigs = Signatures::py_new(Some(data)); + + let inner = read_inner(&sigs); + assert_eq!(inner.len(), 1); + assert!(inner.contains_key("example.com")); + assert!(!inner.contains_key("empty.example.com")); + } + + #[test] + fn test_add_signature() { + let sigs = Signatures::py_new(None); + sigs.add_signature( + "example.com".to_string(), + "ed25519:key1".to_string(), + "sig1".to_string(), + ) + .unwrap(); + + let inner = read_inner(&sigs); + assert_eq!(inner.len(), 1); + assert_eq!( + inner.get("example.com").and_then(|m| m.get("ed25519:key1")), + Some(&"sig1".to_string()) + ); + } + + #[test] + fn test_add_signature_to_existing_server() { + let sigs = create_signatures(&[("example.com", "ed25519:key1", "sig1")]); + sigs.add_signature( + "example.com".to_string(), + "ed25519:key2".to_string(), + "sig2".to_string(), + ) + .unwrap(); + + let inner = read_inner(&sigs); + assert_eq!(inner.len(), 1); + assert_eq!( + inner.get("example.com").and_then(|m| m.get("ed25519:key1")), + Some(&"sig1".to_string()) + ); + assert_eq!( + inner.get("example.com").and_then(|m| m.get("ed25519:key2")), + Some(&"sig2".to_string()) + ); + } + + #[test] + fn test_update_signatures_clobbers_existing() { + let sigs = create_signatures(&[("example.com", "ed25519:key1", "sig1")]); + + // Create a new signatures map with a different signature for the same + // server. + let mut other = HashMap::new(); + other.insert( + "example.com".to_string(), + make_server_sigs(&[("ed25519:key2", "sig2")]), + ); + + // Update the signatures with the new map. + Python::initialize(); + Python::attach(|py| { + let value = pythonize(py, &other).unwrap(); + let value = value.cast::().unwrap(); + + sigs.update(value).unwrap(); + }); + + // Check that the old signature has been replaced with the new one. + let inner = read_inner(&sigs); + assert_eq!(inner.len(), 1); + assert_eq!(inner["example.com"].len(), 1); + assert_eq!(inner["example.com"]["ed25519:key2"], "sig2"); + } + + #[test] + fn test_serialize() { + let mut data = HashMap::new(); + data.insert( + "example.com".to_string(), + make_server_sigs(&[("ed25519:key1", "sig1")]), + ); + let sigs = Signatures::py_new(Some(data)); + + let json = serde_json::to_string(&sigs).unwrap(); + assert_eq!(json, r#"{"example.com":{"ed25519:key1":"sig1"}}"#); + } + + #[test] + fn test_serialize_empty() { + let sigs = Signatures::py_new(None); + let json = serde_json::to_string(&sigs).unwrap(); + assert_eq!(json, "{}"); + } + + #[test] + fn test_deserialize() { + let json = r#"{"example.com":{"ed25519:key1":"sig1"}}"#; + let sigs: Signatures = serde_json::from_str(json).unwrap(); + + let inner = read_inner(&sigs); + assert_eq!(inner.len(), 1); + assert_eq!( + inner.get("example.com").and_then(|m| m.get("ed25519:key1")), + Some(&"sig1".to_string()) + ); + } + + #[test] + fn test_deserialize_empty() { + let sigs: Signatures = serde_json::from_str("{}").unwrap(); + assert!(read_inner(&sigs).is_empty()); + } +} diff --git a/rust/src/events/unsigned.rs b/rust/src/events/unsigned.rs new file mode 100644 index 0000000000..5aa56812c0 --- /dev/null +++ b/rust/src/events/unsigned.rs @@ -0,0 +1,433 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::sync::{Arc, RwLock, RwLockReadGuard}; + +use pyo3::{ + exceptions::{PyKeyError, PyRuntimeError, PyTypeError}, + pyclass, pymethods, + types::{PyAnyMethods, PyList, PyListMethods, PyMapping}, + Bound, IntoPyObjectExt, PyAny, PyResult, Python, +}; +use pythonize::{depythonize, pythonize}; +use serde::{Deserialize, Serialize}; +use serde_json::Number; + +#[pyclass(frozen, skip_from_py_object)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(transparent)] +pub struct Unsigned { + inner: Arc>, +} + +/// The fields in the unsigned data of an event that are persisted in the +/// database. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +struct PersistedUnsignedFields { + #[serde(skip_serializing_if = "Option::is_none")] + age_ts: Option, + #[serde(skip_serializing_if = "Option::is_none")] + replaces_state: Option, + #[serde(skip_serializing_if = "Option::is_none")] + invite_room_state: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + knock_room_state: Option>, +} + +/// The inner representation of the unsigned data of an event, which includes +/// both the fields that are persisted in the database and the fields that are +/// only used in memory. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct UnsignedInner { + #[serde(flatten)] + persisted_fields: PersistedUnsignedFields, + #[serde(skip_serializing_if = "Option::is_none")] + prev_content: Option>, // We use Box to minimise stack space + #[serde(skip_serializing_if = "Option::is_none")] + prev_sender: Option, +} + +/// The fields that exist on the unsigned data of an event. +/// +/// This is used when converting from python to rust, to ensure that if we add a +/// new field we don't forget to add it to all the necessary places. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum UnsignedField { + AgeTs, + ReplacesState, + InviteRoomState, + KnockRoomState, + PrevContent, + PrevSender, +} + +impl std::str::FromStr for UnsignedField { + type Err = (); + + fn from_str(s: &str) -> Result { + match s { + "age_ts" => Ok(Self::AgeTs), + "replaces_state" => Ok(Self::ReplacesState), + "invite_room_state" => Ok(Self::InviteRoomState), + "knock_room_state" => Ok(Self::KnockRoomState), + "prev_content" => Ok(Self::PrevContent), + "prev_sender" => Ok(Self::PrevSender), + _ => Err(()), + } + } +} + +impl Unsigned { + fn py_read(&self) -> PyResult> { + self.inner + .read() + .map_err(|_| PyRuntimeError::new_err("Unsigned lock poisoned")) + } + + fn py_write(&self) -> PyResult> { + self.inner + .write() + .map_err(|_| PyRuntimeError::new_err("Unsigned lock poisoned")) + } +} + +#[pymethods] +impl Unsigned { + #[new] + fn py_new(unsigned: Bound<'_, PyMapping>) -> PyResult { + let inner = depythonize(&unsigned)?; + + Ok(Self { + inner: Arc::new(RwLock::new(inner)), + }) + } + + fn __getitem__<'py>( + &self, + py: Python<'py>, + key: Bound<'_, PyAny>, + ) -> PyResult> { + let key = key + .extract::<&str>() + .map_err(|_| PyTypeError::new_err("Unsigned keys must be strings"))?; + + let field: UnsignedField = key + .parse() + .map_err(|_| PyKeyError::new_err(format!("Unsigned has no key '{key}'")))?; + + let unsigned = self.py_read()?; + + match field { + UnsignedField::AgeTs => { + let age_ts = &unsigned + .persisted_fields + .age_ts + .as_ref() + .ok_or_else(|| PyKeyError::new_err("age_ts"))?; + Ok(pythonize(py, age_ts)?) + } + UnsignedField::ReplacesState => Ok((unsigned.persisted_fields.replaces_state) + .as_ref() + .ok_or_else(|| PyKeyError::new_err("replaces_state"))? + .into_bound_py_any(py)?), + UnsignedField::InviteRoomState => Ok(room_state_to_py( + py, + unsigned + .persisted_fields + .invite_room_state + .as_ref() + .ok_or_else(|| PyKeyError::new_err("invite_room_state"))?, + )?), + UnsignedField::KnockRoomState => Ok(room_state_to_py( + py, + unsigned + .persisted_fields + .knock_room_state + .as_ref() + .ok_or_else(|| PyKeyError::new_err("knock_room_state"))?, + )?), + UnsignedField::PrevContent => Ok(pythonize( + py, + unsigned + .prev_content + .as_ref() + .ok_or_else(|| PyKeyError::new_err("prev_content"))?, + )?), + UnsignedField::PrevSender => Ok((unsigned.prev_sender) + .as_ref() + .ok_or_else(|| PyKeyError::new_err("prev_sender"))? + .into_bound_py_any(py)?), + } + } + + fn __contains__(&self, key: Bound<'_, PyAny>) -> PyResult { + let Ok(key) = key.extract::<&str>() else { + return Ok(false); + }; + + let Ok(field) = key.parse::() else { + return Ok(false); + }; + + let unsigned = self.py_read()?; + + let exists = match field { + UnsignedField::AgeTs => unsigned.persisted_fields.age_ts.is_some(), + UnsignedField::ReplacesState => unsigned.persisted_fields.replaces_state.is_some(), + UnsignedField::InviteRoomState => unsigned.persisted_fields.invite_room_state.is_some(), + UnsignedField::KnockRoomState => unsigned.persisted_fields.knock_room_state.is_some(), + UnsignedField::PrevContent => unsigned.prev_content.is_some(), + UnsignedField::PrevSender => unsigned.prev_sender.is_some(), + }; + + Ok(exists) + } + + fn __setitem__(&self, key: Bound<'_, PyAny>, value: Bound<'_, PyAny>) -> PyResult<()> { + let key = key + .extract::<&str>() + .map_err(|_| PyTypeError::new_err("Unsigned keys must be strings"))?; + + let field: UnsignedField = key + .parse() + .map_err(|_| PyKeyError::new_err(format!("Unsigned has no key '{key}'")))?; + + let mut unsigned = self.py_write()?; + + match field { + UnsignedField::AgeTs => unsigned.persisted_fields.age_ts = Some(depythonize(&value)?), + UnsignedField::ReplacesState => { + unsigned.persisted_fields.replaces_state = Some(value.extract()?) + } + UnsignedField::InviteRoomState => { + unsigned.persisted_fields.invite_room_state = Some(room_state_from_py(value)?) + } + UnsignedField::KnockRoomState => { + unsigned.persisted_fields.knock_room_state = Some(room_state_from_py(value)?) + } + UnsignedField::PrevContent => { + unsigned.prev_content = Some(Box::new(depythonize(&value)?)) + } + UnsignedField::PrevSender => unsigned.prev_sender = Some(value.extract()?), + } + + Ok(()) + } + + fn __delitem__(&self, key: Bound<'_, PyAny>) -> PyResult<()> { + let key = key + .extract::<&str>() + .map_err(|_| PyTypeError::new_err("Unsigned keys must be strings"))?; + + let field: UnsignedField = key + .parse() + .map_err(|_| PyKeyError::new_err(format!("Unsigned has no key '{key}'")))?; + + let mut unsigned = self.py_write()?; + + match field { + UnsignedField::AgeTs => unsigned.persisted_fields.age_ts = None, + UnsignedField::ReplacesState => unsigned.persisted_fields.replaces_state = None, + UnsignedField::InviteRoomState => unsigned.persisted_fields.invite_room_state = None, + UnsignedField::KnockRoomState => unsigned.persisted_fields.knock_room_state = None, + UnsignedField::PrevContent => unsigned.prev_content = None, + UnsignedField::PrevSender => unsigned.prev_sender = None, + } + + Ok(()) + } + + #[pyo3(signature = (key, default=None))] + fn get<'py>( + &self, + py: Python<'py>, + key: Bound<'py, PyAny>, + default: Option>, + ) -> PyResult>> { + match self.__getitem__(py, key) { + Ok(value) => Ok(Some(value)), + Err(err) => { + if err.is_instance_of::(py) { + Ok(default) + } else { + Err(err) + } + } + } + } + + fn for_persistence<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(pythonize(py, &self.py_read()?.persisted_fields)?) + } + + fn for_event<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(pythonize(py, &*self.py_read()?)?) + } +} + +fn room_state_to_py<'py>( + py: Python<'py>, + state: &[serde_json::Value], +) -> PyResult> { + let py_list = PyList::empty(py); + + for item in state { + py_list.append(pythonize(py, item)?)?; + } + + py_list.into_bound_py_any(py) +} + +fn room_state_from_py(value: Bound<'_, PyAny>) -> PyResult> { + let py_list = value.cast::()?; + + let mut state = Vec::with_capacity(py_list.len()); + for item in py_list.iter() { + state.push(pythonize::depythonize(&item)?); + } + + Ok(state) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn test_unsigned_field_from_str_valid() { + assert_eq!("age_ts".parse(), Ok(UnsignedField::AgeTs)); + assert_eq!("replaces_state".parse(), Ok(UnsignedField::ReplacesState)); + assert_eq!( + "invite_room_state".parse(), + Ok(UnsignedField::InviteRoomState) + ); + assert_eq!( + "knock_room_state".parse(), + Ok(UnsignedField::KnockRoomState) + ); + assert_eq!("prev_content".parse(), Ok(UnsignedField::PrevContent)); + assert_eq!("prev_sender".parse(), Ok(UnsignedField::PrevSender)); + } + + #[test] + fn test_unsigned_field_from_str_invalid() { + assert_eq!("".parse::(), Err(())); + assert_eq!("unknown".parse::(), Err(())); + // Case-sensitive: upper-case should not match. + assert_eq!("AGE_TS".parse::(), Err(())); + // Must be an exact match, no whitespace. + assert_eq!(" age_ts".parse::(), Err(())); + } + + #[test] + fn test_persisted_fields_serialize_empty_is_empty_object() { + let fields = PersistedUnsignedFields::default(); + let json = serde_json::to_value(&fields).unwrap(); + assert_eq!(json, json!({})); + } + + #[test] + fn test_persisted_fields_serialize_populated() { + let fields = PersistedUnsignedFields { + age_ts: Some(1234.into()), + replaces_state: Some("$prev:example.com".to_string()), + invite_room_state: Some(vec![json!({"type": "m.room.name"})]), + knock_room_state: Some(vec![json!({"type": "m.room.topic"})]), + }; + let json = serde_json::to_value(&fields).unwrap(); + assert_eq!( + json, + json!({ + "age_ts": 1234, + "replaces_state": "$prev:example.com", + "invite_room_state": [{"type": "m.room.name"}], + "knock_room_state": [{"type": "m.room.topic"}], + }) + ); + } + + #[test] + fn test_unsigned_inner_flattens_persisted_fields() { + let inner = UnsignedInner { + persisted_fields: PersistedUnsignedFields { + age_ts: Some(99.into()), + ..Default::default() + }, + prev_content: Some(Box::new(json!({"body": "hi"}))), + prev_sender: Some("@alice:example.com".to_string()), + }; + + let json = serde_json::to_value(&inner).unwrap(); + assert_eq!( + json, + json!({ + "age_ts": 99, + "prev_content": {"body": "hi"}, + "prev_sender": "@alice:example.com", + }) + ); + } + + #[test] + fn test_unsigned_inner_roundtrip() { + let original = UnsignedInner { + persisted_fields: PersistedUnsignedFields { + age_ts: Some(10.into()), + replaces_state: Some("$state:example.com".to_string()), + invite_room_state: None, + knock_room_state: None, + }, + prev_content: Some(Box::new(json!({"membership": "join"}))), + prev_sender: None, + }; + + let json = serde_json::to_string(&original).unwrap(); + let roundtripped: UnsignedInner = serde_json::from_str(&json).unwrap(); + + assert_eq!(roundtripped.persisted_fields.age_ts, Some(10.into())); + assert_eq!( + roundtripped.persisted_fields.replaces_state.as_deref(), + Some("$state:example.com") + ); + assert_eq!( + roundtripped.prev_content.as_deref(), + Some(&json!({"membership": "join"})) + ); + assert_eq!(roundtripped.prev_sender, None); + } + + #[test] + fn test_unsigned_serializes_transparently() { + // `Unsigned` is `#[serde(transparent)]` over its inner, so serializing + // an empty default should yield an empty object rather than a wrapper. + let unsigned = Unsigned::default(); + let json = serde_json::to_value(&unsigned).unwrap(); + assert_eq!(json, json!({})); + } + + #[test] + fn test_unsigned_deserialize_from_flat_object() { + let json = json!({ + "age_ts": 5, + "prev_sender": "@bob:example.com", + }); + let unsigned: Unsigned = serde_json::from_value(json).unwrap(); + let inner = unsigned.inner.read().unwrap(); + assert_eq!(inner.persisted_fields.age_ts, Some(5.into())); + assert_eq!(inner.prev_sender.as_deref(), Some("@bob:example.com")); + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 3b049a51b7..cc89862e4e 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -5,6 +5,7 @@ use pyo3::prelude::*; use pyo3_log::ResetHandle; pub mod acl; +pub mod canonical_json; pub mod duration; pub mod errors; pub mod events; diff --git a/rust/src/push/mod.rs b/rust/src/push/mod.rs index ac9b9c93e4..780d7a8cbd 100644 --- a/rust/src/push/mod.rs +++ b/rust/src/push/mod.rs @@ -104,7 +104,7 @@ fn get_base_rule_ids() -> HashSet<&'static str> { /// A single push rule for a user. #[derive(Debug, Clone)] -#[pyclass(frozen)] +#[pyclass(frozen, from_py_object)] pub struct PushRule { /// A unique ID for this rule pub rule_id: Cow<'static, str>, @@ -462,7 +462,7 @@ pub struct RelatedEventMatchTypeCondition { /// The collection of push rules for a user. #[derive(Debug, Clone, Default)] -#[pyclass(frozen)] +#[pyclass(frozen, from_py_object)] pub struct PushRules { /// Custom push rules that override a base rule. overridden_base_rules: HashMap, PushRule>, @@ -549,7 +549,7 @@ impl PushRules { /// A wrapper around `PushRules` that checks the enabled state of rules and /// filters out disabled experimental rules. #[derive(Debug, Clone, Default)] -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] pub struct FilteredPushRules { push_rules: PushRules, enabled_map: BTreeMap, diff --git a/rust/src/room_versions.rs b/rust/src/room_versions.rs index dbc962174d..47473cf200 100644 --- a/rust/src/room_versions.rs +++ b/rust/src/room_versions.rs @@ -93,7 +93,7 @@ impl PushRuleRoomFlag { } /// An object which describes the unique attributes of a room version. -#[pyclass(frozen, eq, hash, get_all)] +#[pyclass(frozen, eq, hash, get_all, skip_from_py_object)] #[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct RoomVersion { /// The identifier for this version. @@ -622,7 +622,7 @@ const ROOM_VERSION_MSC4242V12: RoomVersion = RoomVersion { /// Note: room versions can be added to this mapping at startup (allowing /// support for experimental room versions to be behind experimental feature /// flags). -#[pyclass(frozen, mapping)] +#[pyclass(frozen, mapping, skip_from_py_object)] #[derive(Clone)] pub struct KnownRoomVersionsMapping { // Note we use a Vec here to ensure that the order of keys is @@ -637,19 +637,22 @@ pub struct KnownRoomVersionsMapping { impl KnownRoomVersionsMapping { /// Add a new room version to the mapping, indicating that this instance /// supports it. - fn add_room_version(&self, version: RoomVersion) -> PyResult<()> { + fn add_room_version(&self, version: Bound<'_, RoomVersion>) -> PyResult<()> { let mut versions = self .versions .write() .map_err(|_| PyRuntimeError::new_err("KnownRoomVersionsMapping lock poisoned"))?; - if versions.iter().any(|v| v.identifier == version.identifier) { + if versions + .iter() + .any(|v| v.identifier == version.get().identifier) + { // We already have this room version, so we don't add it again (as // otherwise we'd end up with duplicates). return Ok(()); } - versions.push(version); + versions.push(*version.get()); Ok(()) } diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 9385feff29..9484e017d1 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://famedly.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://famedly.github.io/synapse/schema/synapse/v1.152/synapse-config.schema.json +$id: https://famedly.github.io/synapse/schema/synapse/v1.153/synapse-config.schema.json type: object properties: famedly_maximum_refresh_token_lifetime: @@ -110,6 +110,10 @@ properties: Determines whether we should freeze the internal dict object in `FrozenEvent`. Freezing prevents bugs where we accidentally share e.g. signature dicts. However, freezing a dict is expensive. + + + > ⚠️ **Warning** – This option is known to introduce a new class of [comparison + bugs](https://github.com/element-hq/synapse/issues/18117) in Synapse. default: false examples: - true diff --git a/synapse/config/api.py b/synapse/config/api.py index 81bc5f2588..d01b375356 100644 --- a/synapse/config/api.py +++ b/synapse/config/api.py @@ -48,6 +48,9 @@ def _get_prejoin_state_entries( self, config: JsonDict ) -> Iterable[tuple[str, str | None]]: """Get the event types and state keys to include in the prejoin state.""" + # MSC4311: the create event must always be included in invite/knock state. + yield EventTypes.Create, "" + room_prejoin_state_config = config.get("room_prejoin_state") or {} # backwards-compatibility support for room_invite_state_types diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 630a08f868..a1d4eb3ed6 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -386,9 +386,6 @@ def read_config( # MSC3814 (dehydrated devices with SSSS) self.msc3814_enabled: bool = experimental.get("msc3814_enabled", False) - # MSC3266 (room summary api) - self.msc3266_enabled: bool = experimental.get("msc3266_enabled", False) - # MSC2409 (this setting only relates to optionally sending to-device messages). # Presence, typing and read receipt EDUs are already sent to application services that # have opted in to receive them. If enabled, this adds to-device messages to that list. diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index d789c06a9c..823b6288e8 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -236,9 +236,7 @@ def event_needs_resigning( if sender.domain != server_name: return False want_key_id = verify_key.alg + ":" + verify_key.version - signed_with_current_key_id = ev.signatures.get(server_name, {}).get( - want_key_id, None - ) + signed_with_current_key_id = ev.signatures.get_signature(server_name, want_key_id) if signed_with_current_key_id: return False diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 0d4d5e0e17..36736b4559 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -120,8 +120,18 @@ def from_event( ) -> "VerifyJsonRequest": """Create a VerifyJsonRequest to verify all signatures on an event object for the given server. + + Raises immediately if the event doesn't have any signatures from the + given server. """ - key_ids = list(event.signatures.get(server_name, [])) + if server_name not in event.signatures: + raise SynapseError( + 400, + f"Not signed by {server_name}", + Codes.UNAUTHORIZED, + ) + + key_ids = list(event.signatures[server_name]) return VerifyJsonRequest( server_name, # We defer creating the redacted json object, as it uses a lot more diff --git a/synapse/event_auth.py b/synapse/event_auth.py index ca528ae235..fd35da8ba0 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -128,7 +128,7 @@ def validate_event_for_room_version(event: "EventBase") -> None: ) # Check the sender's domain has signed the event - if not event.signatures.get(sender_domain): + if sender_domain not in event.signatures: # We allow invites via 3pid to have a sender from a different # HS, as the sender must match the sender of the original # 3pid invite. This is checked further down with the @@ -141,7 +141,7 @@ def validate_event_for_room_version(event: "EventBase") -> None: event_id_domain = get_domain_from_id(event.event_id) # Check the origin domain has signed the event - if not event.signatures.get(event_id_domain): + if event_id_domain not in event.signatures: raise AuthError(403, "Event not signed by sending server") is_invite_via_allow_rule = ( @@ -154,7 +154,7 @@ def validate_event_for_room_version(event: "EventBase") -> None: authoriser_domain = get_domain_from_id( event.content[EventContentFields.AUTHORISING_USER] ) - if not event.signatures.get(authoriser_domain): + if authoriser_domain not in event.signatures: raise AuthError(403, "Event not signed by authorising server") diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 51e4ca0c45..487bed59bd 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -44,8 +44,12 @@ StickyEvent, ) from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions -from synapse.synapse_rust.events import EventInternalMetadata -from synapse.types import JsonDict, StateKey, StrCollection +from synapse.synapse_rust.events import EventInternalMetadata, Signatures, Unsigned +from synapse.types import ( + JsonDict, + StateKey, + StrCollection, +) from synapse.util.caches import intern_dict from synapse.util.duration import Duration from synapse.util.frozenutils import freeze, unfreeze @@ -63,6 +67,10 @@ NOTE: This is overridden by the configuration by the Synapse worker apps, but for the sake of tests, it is set here because it cannot be configured on the homeserver object itself. + +FIXME: Because of how this option works (changing the underlying types), it causes +subtle downstream bugs that makes type comparisons brittle, tracked by +https://github.com/element-hq/synapse/issues/18117 """ T = TypeVar("T") @@ -199,8 +207,8 @@ def __init__( assert room_version.event_format == self.format_version self.room_version = room_version - self.signatures = signatures - self.unsigned = unsigned + self.signatures = Signatures(signatures) + self.unsigned = Unsigned(unsigned) self.rejected_reason = rejected_reason self._dict = event_dict @@ -250,8 +258,26 @@ def get_state_key(self) -> str | None: return self._dict.get("state_key") def get_dict(self) -> JsonDict: + """Convert the event to a dictionary suitable for serialisation.""" + d = dict(self._dict) + d.update( + { + "signatures": self.signatures.as_dict(), + "unsigned": self.unsigned.for_event(), + } + ) + + return d + + def get_dict_for_persistence(self) -> JsonDict: + """Convert the event to a dictionary suitable for persistence.""" d = dict(self._dict) - d.update({"signatures": self.signatures, "unsigned": dict(self.unsigned)}) + d.update( + { + "signatures": self.signatures.as_dict(), + "unsigned": self.unsigned.for_persistence(), + } + ) return d @@ -397,7 +423,7 @@ def __init__( for name, sigs in event_dict.pop("signatures", {}).items() } - unsigned = dict(event_dict.pop("unsigned", {})) + unsigned = event_dict.pop("unsigned", {}) # We intern these strings because they turn up a lot (especially when # caching). @@ -451,7 +477,7 @@ def __init__( assert "event_id" not in event_dict - unsigned = dict(event_dict.pop("unsigned", {})) + unsigned = event_dict.pop("unsigned", {}) # We intern these strings because they turn up a lot (especially when # caching). diff --git a/synapse/events/utils.py b/synapse/events/utils.py index f038fb5578..926c81b83d 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -47,6 +47,7 @@ from synapse.api.errors import Codes, SynapseError from synapse.api.room_versions import RoomVersion from synapse.logging.opentracing import SynapseTags, set_tag, trace +from synapse.synapse_rust.events import Unsigned from synapse.types import JsonDict, Requester from . import EventBase, FrozenEventV2, StrippedStateEvent, make_event_from_dict @@ -987,7 +988,7 @@ def validate_canonicaljson(value: Any) -> None: def maybe_upsert_event_field( - event: EventBase, container: JsonDict, key: str, value: object + event: EventBase, container: Unsigned, key: str, value: object ) -> bool: """Upsert an event field, but only if this doesn't make the event too large. diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index fe0710a0bf..1631f021ca 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -307,20 +307,27 @@ def _is_invite_via_3pid(event: EventBase) -> bool: def parse_events_from_pdu_json( - pdus_json: Sequence[JsonDict], room_version: RoomVersion + pdus_json: Sequence[JsonDict], + room_version: RoomVersion, + received_time: int | None = None, ) -> list[EventBase]: return [ - event_from_pdu_json(pdu_json, room_version) + event_from_pdu_json(pdu_json, room_version, received_time=received_time) for pdu_json in filter_pdus_for_valid_depth(pdus_json) ] -def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventBase: +def event_from_pdu_json( + pdu_json: JsonDict, room_version: RoomVersion, received_time: int | None = None +) -> EventBase: """Construct an EventBase from an event json received over federation Args: pdu_json: pdu as received over federation room_version: The version of the room this event belongs to + received_time: timestamp in ms that the event was received at. If + `None` then any `age` field in the `unsigned` block will be + dropped. Raises: SynapseError: if the pdu is missing required fields or is otherwise @@ -333,6 +340,25 @@ def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventB if "unsigned" in pdu_json: _strip_unsigned_values(pdu_json) + # Handle the `age` field, which is sent by some servers as part of the + # `unsigned` block. We convert this into an `age_ts` field, which is + # what Synapse uses internally. We also remove the `age` field to avoid + # confusion. + # + # c.f. https://github.com/matrix-org/synapse/issues/8429 + unsigned = pdu_json["unsigned"] + age = unsigned.pop("age", None) + + # We check that the `age` is actually an int before using it below. We + # don't error here as the `age` a) doesn't affect the validity of the + # event, and b) is best effort anyway. + if not isinstance(age, int): + age = None + + unsigned.pop("age_ts", None) + if received_time is not None and age is not None: + unsigned["age_ts"] = received_time - int(age) + depth = pdu_json["depth"] if type(depth) is not int: # noqa: E721 raise SynapseError(400, "Depth %r not an intger" % (depth,), Codes.BAD_JSON) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 78a1900c73..2b5ef5fbac 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -1574,10 +1574,15 @@ async def get_missing_events( min_depth=min_depth, timeout=timeout, ) + received_time = self._clock.time_msec() room_version = await self.store.get_room_version(room_id) - events = parse_events_from_pdu_json(content.get("events", []), room_version) + events = parse_events_from_pdu_json( + content.get("events", []), + room_version, + received_time=received_time, + ) signed_events = await self._check_sigs_and_hash_for_pulled_events_and_fetch( destination, events, room_version=room_version diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index a869f231f3..d8801c7d68 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -20,6 +20,7 @@ # [This file includes modifications made by New Vector Limited] # # +import copy import logging import random from typing import ( @@ -450,16 +451,6 @@ async def _handle_pdus_in_txn( newest_pdu_ts = 0 for p in transaction.pdus: - # FIXME (richardv): I don't think this works: - # https://github.com/matrix-org/synapse/issues/8429 - if "unsigned" in p: - unsigned = p["unsigned"] - if "age" in unsigned: - p["age"] = unsigned["age"] - if "age" in p: - p["age_ts"] = request_time - int(p["age"]) - del p["age"] - # We try and pull out an event ID so that if later checks fail we # can log something sensible. We don't mandate an event ID here in # case future event formats get rid of the key. @@ -487,10 +478,15 @@ async def _handle_pdus_in_txn( continue try: - event = event_from_pdu_json(p, room_version) + event = event_from_pdu_json(p, room_version, received_time=request_time) except SynapseError as e: logger.info("Ignoring PDU for failing to deserialize: %s", e) continue + except Exception as e: + # We catch all exceptions here as we don't want a single bad + # event to cause us to fail the whole transaction. + logger.exception("Error deserializing PDU: %s", e) + continue pdus_by_room.setdefault(room_id, []).append(event) @@ -568,9 +564,58 @@ async def _process_edu(edu_dict: JsonDict) -> None: origin=origin, destination=self.server_name, edu_type=edu_dict["edu_type"], - content=edu_dict["content"], + # Make a deep-copy as we mutate the content down below + content=copy.deepcopy(edu_dict["content"]), ) + try: + # Server ACL's apply to `EduTypes.TYPING` per MSC4163: + # + # > For typing notifications (m.typing), the room_id field inside + # > content should be checked, with the typing notification ignored if + # > the origin of the request is a server which is forbidden by the + # > room's ACL. Ignoring the typing notification means that the EDU + # > MUST be dropped upon receipt. + if edu.edu_type == EduTypes.TYPING: + origin_host, _ = parse_server_name(origin) + room_id = edu.content["room_id"] + try: + await self.check_server_matches_acl(origin_host, room_id) + except AuthError: + logger.warning( + "Ignoring typing EDU for room %s from banned server because of ACL's", + room_id, + ) + return + + # Server ACL's apply to `EduTypes.RECEIPT` per MSC4163: + # + # > For read receipts (m.receipt), all receipts inside a room_id + # > inside content should be ignored if the origin of the request is + # > forbidden by the room's ACL. + if edu.edu_type == EduTypes.RECEIPT: + origin_host, _ = parse_server_name(origin) + to_remove_room_ids = set() + for room_id in edu.content.keys(): + try: + await self.check_server_matches_acl(origin_host, room_id) + except AuthError: + to_remove_room_ids.add(room_id) + + if to_remove_room_ids: + logger.warning( + "Ignoring receipts in EDU for rooms %s from banned server %s because of ACL's", + to_remove_room_ids, + origin_host, + ) + + for room_id in to_remove_room_ids: + edu.content.pop(room_id) + + if not edu.content: + # If we've removed all the rooms, we can just ignore the whole EDU + return + await self.registry.on_edu(edu.edu_type, origin, edu.content) except Exception: # If there was an error handling the EDU, we must reject the diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index f18cfce9e7..c3ff8d26cb 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -105,6 +105,12 @@ ) +NUMBER_OF_EVENTS_TO_BACKFILL = 100 +""" +The number of events we try to backfill from other servers in a single request. +""" + + # TODO: We can refactor this away now that there is only one backfill point again class _BackfillPointType(Enum): # a regular backwards extremity (ie, an event which we don't yet have, but which @@ -253,7 +259,9 @@ async def _maybe_backfill_inner( _BackfillPoint(event_id, depth, _BackfillPointType.BACKWARDS_EXTREMITY) for event_id, depth in await self.store.get_backfill_points_in_room( room_id=room_id, - current_depth=current_depth, + # Per the docstring, it's best to pad the `current_depth` by the + # number of messages you plan to backfill from these points. + nearby_depth=current_depth + NUMBER_OF_EVENTS_TO_BACKFILL, # We only need to end up with 5 extremities combined with the # insertion event extremities to make the `/backfill` request # but fetch an order of magnitude more to make sure there is @@ -297,12 +305,13 @@ async def _maybe_backfill_inner( # likely not to return anything relevant so we backfill in the background. The # only way, this could return something relevant is if we discover a new branch # of history that extends all the way back to where we are currently paginating - # and it's within the 100 events that are returned from `/backfill`. + # and it's within the `NUMBER_OF_EVENTS_TO_BACKFILL` events that are returned + # from `/backfill`. if not sorted_backfill_points and current_depth != MAX_DEPTH: # Check that we actually have later backfill points, if not just return. have_later_backfill_points = await self.store.get_backfill_points_in_room( room_id=room_id, - current_depth=MAX_DEPTH, + nearby_depth=MAX_DEPTH, limit=1, ) if not have_later_backfill_points: @@ -462,7 +471,10 @@ async def try_backfill(domains: StrCollection) -> None: try: await self._federation_event_handler.backfill( - dom, room_id, limit=100, extremities=extremities_to_request + dom, + room_id, + limit=NUMBER_OF_EVENTS_TO_BACKFILL, + extremities=extremities_to_request, ) # If this succeeded then we probably already have the # appropriate stuff. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index bfade58063..a660b9f3f3 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -2089,10 +2089,9 @@ async def persist_and_notify_client_events( returned_invite = await federation_handler.send_invite( invitee.domain, event ) - event.unsigned.pop("room_state", None) # TODO: Make sure the signatures actually are correct. - event.signatures.update(returned_invite.signatures) + event.signatures.update(returned_invite.signatures.as_dict()) if event.content["membership"] == Membership.KNOCK: maybe_upsert_event_field( diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index d4a8e327f2..5bfdb10f26 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -27,6 +27,7 @@ import random import string from collections import OrderedDict +from collections.abc import Mapping from http import HTTPStatus from typing import ( TYPE_CHECKING, @@ -67,7 +68,11 @@ from synapse.event_auth import validate_event_for_room_version from synapse.events import EventBase, event_exists_in_state_dag from synapse.events.snapshot import UnpersistedEventContext -from synapse.events.utils import FilteredEvent, copy_and_fixup_power_levels_contents +from synapse.events.utils import ( + FilteredEvent, + PowerLevelsContent, + copy_and_fixup_power_levels_contents, +) from synapse.handlers.relations import BundledAggregations from synapse.rest.admin._base import assert_user_is_admin from synapse.streams import EventSource @@ -513,10 +518,12 @@ async def _update_upgraded_room_pls( except AuthError as e: logger.warning("Unable to update PLs in old room: %s", e) + power_levels_content: JsonMapping = old_room_pl_state.content + new_room_version = await self.store.get_room_version(new_room_id) if new_room_version.msc4289_creator_power_enabled: - self._remove_creators_from_pl_users_map( - old_room_pl_state.content.get("users", {}), + power_levels_content = self._copy_and_remove_creators_from_pl_users_map( + power_levels_content, requester.user.to_string(), additional_creators, ) @@ -528,9 +535,7 @@ async def _update_upgraded_room_pls( "state_key": "", "room_id": new_room_id, "sender": requester.user.to_string(), - "content": copy_and_fixup_power_levels_contents( - old_room_pl_state.content - ), + "content": copy_and_fixup_power_levels_contents(power_levels_content), }, ratelimit=False, ) @@ -699,11 +704,12 @@ async def clone_existing_room( if new_room_version.msc4289_creator_power_enabled: # the creator(s) cannot be in the users map - self._remove_creators_from_pl_users_map( - user_power_levels, + fixed_power_levels = self._copy_and_remove_creators_from_pl_users_map( + power_levels, user_id, additional_creators, ) + initial_state[(EventTypes.PowerLevels, "")] = fixed_power_levels # We construct a subset of what the body of a call to /createRoom would look like # for passing to the spam checker. We don't include a preset here, as we expect the @@ -1842,12 +1848,19 @@ def _room_preset_config(self, room_config: JsonDict) -> tuple[str, dict]: ) return preset_name, preset_config - def _remove_creators_from_pl_users_map( + def _copy_and_remove_creators_from_pl_users_map( self, - users_map: dict[str, int], + power_levels_content: PowerLevelsContent, creator: str, additional_creators: list[str] | None, - ) -> None: + ) -> PowerLevelsContent: + users_map = power_levels_content.get("users", {}) + if not users_map: + return power_levels_content + + assert isinstance(users_map, Mapping) + users_map = dict(users_map) + creators = [creator] if additional_creators: creators.extend(additional_creators) @@ -1855,6 +1868,9 @@ def _remove_creators_from_pl_users_map( # the creator(s) cannot be in the users map users_map.pop(creator, None) + power_levels_content = {**power_levels_content, "users": users_map} + return power_levels_content + def _generate_room_id(self) -> str: """Generates a random room ID. diff --git a/synapse/handlers/room_policy.py b/synapse/handlers/room_policy.py index 01943e1991..e46e6dc2ef 100644 --- a/synapse/handlers/room_policy.py +++ b/synapse/handlers/room_policy.py @@ -181,9 +181,10 @@ async def is_event_allowed(self, event: EventBase) -> bool: async def _verify_policy_server_signature( self, event: EventBase, policy_server: str, public_key: str ) -> bool: - # check the event is signed with this (via, public_key). - verify_json_req = VerifyJsonRequest.from_event(policy_server, event, 0) try: + # check the event is signed with this (via, public_key). + verify_json_req = VerifyJsonRequest.from_event(policy_server, event, 0) + key_bytes = decode_base64(public_key) verify_key = decode_verify_key_bytes(POLICY_SERVER_KEY_ID, key_bytes) # We would normally use KeyRing.verify_event_for_server but we can't here as we don't @@ -260,9 +261,7 @@ async def ask_policy_server_to_sign_event( # servers need to manually fetch signatures for. This is the code that allows # those events to continue working (because they're legally sent, even if missing # the policy server signature). - event.signatures.setdefault(policy_server.server_name, {}).update( - signature.get(policy_server.server_name, {}) - ) + event.signatures.update(signature) except HttpResponseException as ex: # re-wrap HTTP errors as `SynapseError` so they can be proxied to clients directly raise ex.to_synapse_error() from ex diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 51be3b5084..57792ea53c 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -54,7 +54,18 @@ # will not disappear under our feet as long as we don't delete the room. NEW_EVENT_DURING_PURGE_LOCK_NAME = "new_event_during_purge_lock" -WORKER_LOCK_MAX_RETRY_INTERVAL = Duration(seconds=60) +WORKER_LOCK_MAX_RETRY_INTERVAL = Duration(seconds=5) +""" +The maximum wait time before retrying to acquire the lock. + +Better to retry more quickly than have workers wait around. 5 seconds is still a +reasonable gap in time to not overwhelm the CPU/Database. + +This matters most in cross-worker scenarios. When locks are on the same worker, when the +lock holder releases, we signal to other locks (with the same name/key) that they +should try reacquiring the lock immediately. But locks on other workers only re-check +based on their retry `_timeout_interval`. +""" WORKER_LOCK_EXCESSIVE_WAITING_WARN_DURATION = Duration(minutes=10) diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index 61511b9360..3783211a92 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -367,6 +367,7 @@ def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.room_shutdown_handler = hs.get_room_shutdown_handler() self.pagination_handler = hs.get_pagination_handler() + self._storage_controllers = hs.get_storage_controllers() async def on_GET( self, request: SynapseRequest, room_id: str @@ -383,6 +384,15 @@ async def on_GET( members ) result["forgotten"] = await self.store.is_locally_forgotten_room(room_id) + tombstone_event = await self._storage_controllers.state.get_current_state_event( + room_id, + EventTypes.Tombstone, + "", + ) + result["tombstoned"] = tombstone_event is not None + result["replacement_room"] = ( + tombstone_event.content.get("replacement_room") if tombstone_event else None + ) return HTTPStatus.OK, result diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 83664814a6..48ef42c7d6 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -1718,16 +1718,18 @@ async def on_GET( class RoomSummaryRestServlet(ResolveRoomIdMixin, RestServlet): PATTERNS = ( - # deprecated endpoint, to be removed + # deprecated unstable endpoint, to be removed re.compile( "^/_matrix/client/unstable/im.nheko.summary" "/rooms/(?P[^/]*)/summary$" ), - # recommended endpoint + # recommended unstable endpoint re.compile( "^/_matrix/client/unstable/im.nheko.summary" "/summary/(?P[^/]*)$" ), + # stable endpoint + re.compile("^/_matrix/client/v1/room_summary/(?P[^/]*)$"), ) CATEGORY = "Client API requests" @@ -1775,8 +1777,7 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: RoomTypingRestServlet(hs).register(http_server) RoomEventContextServlet(hs).register(http_server) RoomHierarchyRestServlet(hs).register(http_server) - if hs.config.experimental.msc3266_enabled: - RoomSummaryRestServlet(hs).register(http_server) + RoomSummaryRestServlet(hs).register(http_server) RoomEventServlet(hs).register(http_server) JoinedRoomsRestServlet(hs).register(http_server) RoomAliasListServlet(hs).register(http_server) diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index 415926eb0a..d84c58dcf8 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -1199,34 +1199,72 @@ def _get_auth_chain_difference_txn( async def get_backfill_points_in_room( self, room_id: str, - current_depth: int, + nearby_depth: int, limit: int, ) -> list[tuple[str, int]]: """ Get the backward extremities to backfill from in the room along with the approximate depth. - Only returns events that are at a depth lower than or - equal to the `current_depth`. Sorted by depth, highest to lowest (descending) - so the closest events to the `current_depth` are first in the list. - - We ignore extremities that are newer than the user's current scroll position - (ie, those with depth greater than `current_depth`) as: - 1. we don't really care about getting events that have happened - after our current position; and - 2. by the nature of paginating and scrolling back, we have likely - previously tried and failed to backfill from that extremity, so - to avoid getting "stuck" requesting the same backfill repeatedly - we drop those extremities. + Only returns events that are at a depth lower than or equal to the `nearby_depth`. + Sorted by depth, highest to lowest (descending) so the closest events to the + `nearby_depth` are first in the list. + + ### Why `nearby_depth`? + + We find backfill points from the backward extremities in the DAG. Backward + extremities are the oldest events we know of in the room but we only know of + them because some other event referenced them by prev_event and aren't persisted + in our database yet (meaning we don't know their depth specifically). So we can + only do approximate depth comparisons (use the depth of the known events they're + connected to). And we don't know if those backward extremities point to a long + chain/fork of history that could stretch back far enough to be visible. + + This means a naive homeserver implementation that looks for backward extremities <= + depth of the `/messages?dir=b&from=xxx` token may overlook a backfill point that could + reveal more history in the window the user is currently paginating in. + + We consider "nearby" as anything within range of the number of events you plan + to backfill from the given backfill point. This is a good heuristic as since we + plan to backfill N events, the chain of events from a backfill point could + extend back into the visible window. + + Example: + + - Your pagination token represents a scroll position at a depth of `100`. + - We have a backfill point at an approximate depth of `125` + - You plan to backfill `50` events from that backfill point. + + When we pad the token `depth` with the number of messages we plan to backfill, + `100` + `50` = `150`, we find the backfill point at `125` (because <= `150`, our + `nearby_depth`), backfill `50` events to a depth of `75` in the timeline + (exposing new events that we can return `100` -> `75`). + + When we don't pad our token `depth`, `100` is lower than any of the backfill + points so we don't pick any and miss out on backfilling any events. Without + something like MSC3871 to indicate gaps in the timeline, clients will most + likely never know they are missing any events and never try to paginate again. + + Generally though, we ignore extremities that are newer than the user's current + scroll position (ie, those with depth greater than `nearby_depth`) as: + 1. we don't really care about getting events that have happened after our + current position; and + 2. by the nature of paginating and scrolling back, we have likely previously + tried and failed to backfill from that extremity, so to avoid getting + "stuck" requesting the same backfill repeatedly we drop those + extremities. Although we also have `event_failed_pull_attempts` nowadays + to backoff as well. Args: room_id: Room where we want to find the oldest events - current_depth: The depth at the user's current scrollback position + nearby_depth: Typically, this is depth at the user's current scrollback + position + the number of events you plan to backfill from these backfill + points. limit: The max number of backfill points to return Returns: List of (event_id, depth) tuples. Sorted by depth, highest to lowest - (descending) so the closest events to the `current_depth` are first + (descending) so the closest events to the `nearby_depth` are first in the list. """ @@ -1234,12 +1272,12 @@ def get_backfill_points_in_room_txn( txn: LoggingTransaction, room_id: str ) -> list[tuple[str, int]]: # Assemble a tuple lookup of event_id -> depth for the oldest events - # we know of in the room. Backwards extremeties are the oldest + # we know of in the room. Backwards extremities are the oldest # events we know of in the room but we only know of them because # some other event referenced them by prev_event and aren't # persisted in our database yet (meaning we don't know their depth # specifically). So we need to look for the approximate depth from - # the events connected to the current backwards extremeties. + # the events connected to the current backwards extremities. if isinstance(self.database_engine, PostgresEngine): least_function = "LEAST" @@ -1259,7 +1297,7 @@ def get_backfill_points_in_room_txn( ON edge.event_id = event.event_id /** * We find the "oldest" events in the room by looking for - * events connected to backwards extremeties (oldest events + * events connected to backwards extremities (oldest events * in the room that we know of so far). */ INNER JOIN event_backward_extremities AS backward_extrem @@ -1285,16 +1323,19 @@ def get_backfill_points_in_room_txn( AND edge.is_state is FALSE /** * We only want backwards extremities that are older than or at - * the same position of the given `current_depth` (where older + * the same position of the given `nearby_depth` (where older * means less than the given depth) because we're looking backwards - * from the `current_depth` when backfilling. + * from the `nearby_depth` when backfilling. + * + * Keep in mind that `event.depth` is an approximate depth of the + * backward extremity itself. * - * current_depth (ignore events that come after this, ignore 2-4) + * nearby_depth (ignore events that come after this, ignore 2-4) * | * ▼ * [0]<--[1]<--[2]<--[3]<--[4] */ - AND event.depth <= ? /* current_depth */ + AND event.depth <= ? /* nearby_depth */ /** * Exponential back-off (up to the upper bound) so we don't retry the * same backfill point over and over. ex. 2hr, 4hr, 8hr, 16hr, etc. @@ -1312,7 +1353,7 @@ def get_backfill_points_in_room_txn( ) ) /** - * Sort from highest (closest to the `current_depth`) to the lowest depth + * Sort from highest (closest to the `nearby_depth`) to the lowest depth * because the closest are most relevant to backfill from first. * Then tie-break on alphabetical order of the event_ids so we get a * consistent ordering which is nice when asserting things in tests. @@ -1325,7 +1366,7 @@ def get_backfill_points_in_room_txn( sql, ( room_id, - current_depth, + nearby_depth, self.clock.time_msec(), BACKFILL_EVENT_EXPONENTIAL_BACKOFF_MAXIMUM_DOUBLING_STEPS, BACKFILL_EVENT_EXPONENTIAL_BACKOFF_STEP_MILLISECONDS, diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 12c918eca6..5f6b03a988 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -2711,7 +2711,7 @@ def _store_event_txn( return def event_dict(event: EventBase) -> JsonDict: - d = event.get_dict() + d = event.get_dict_for_persistence() d.pop("redacted", None) d.pop("redacted_because", None) return d diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index d2623f0760..c0d218398d 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -2824,8 +2824,8 @@ def _fetch_next_events_txn( # with the provided old key. if old_verify_key is not None: old_key_id = f"{old_verify_key.alg}:{old_verify_key.version}" - server_sigs = event.signatures.get(self.hs.hostname, {}) - if old_key_id not in server_sigs: + old_sig = event.signatures.get_signature(self.hs.hostname, old_key_id) + if old_sig is None: # Event wasn't signed with this key ID at all, skip. continue diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index cc79b8042b..6f26bd17ce 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -779,14 +779,26 @@ async def get_events_as_list( events.append(event) if get_prev_content: - if "replaces_state" in event.unsigned: + # The `event` here might be in the cache, and so might have + # already had the `prev_content` and `prev_sender` fields added + # to its unsigned. + # + # We check if a) we should add the previous content, and b) if + # we have already added it. + replaces_state = "replaces_state" in event.unsigned + has_prev = ( + "prev_content" in event.unsigned and "prev_sender" in event.unsigned + ) + if replaces_state and not has_prev: prev = await self.get_event( event.unsigned["replaces_state"], get_prev_content=False, allow_none=True, ) if prev: - event.unsigned = dict(event.unsigned) + # This mutates the cached event, but that's fine as the + # previous content/sender will be the same for all + # requests for this event. event.unsigned["prev_content"] = prev.content event.unsigned["prev_sender"] = prev.sender diff --git a/synapse/synapse_rust/events.pyi b/synapse/synapse_rust/events.pyi index b3b4c3d14b..138cdde0be 100644 --- a/synapse/synapse_rust/events.pyi +++ b/synapse/synapse_rust/events.pyi @@ -10,9 +10,9 @@ # See the GNU Affero General Public License for more details: # . -from typing import Mapping +from typing import Any, Mapping -from synapse.types import JsonDict +from synapse.types import JsonDict, JsonMapping class EventInternalMetadata: def __init__(self, internal_metadata_dict: JsonDict): ... @@ -154,3 +154,62 @@ def event_visible_to_server( Returns: Whether the server is allowed to see the unredacted event. """ + +class Signatures: + """A class representing the signatures on an event.""" + + def __init__(self, signatures: Mapping[str, Mapping[str, str]] | None = None): ... + def get_signature(self, server_name: str, key_id: str) -> str | None: ... + """Get the signature for the given server name and key ID, if it exists.""" + + def __getitem__(self, server_name: str) -> Mapping[str, str]: ... + """Get the signatures for the given server name. Raises KeyError if there + are no signatures for that server.""" + + def __contains__(self, server_name: Any) -> bool: ... + """Check if there are signatures for the given server name.""" + + def __len__(self) -> int: ... + """Return the number of servers that have signatures.""" + + def add_signature(self, server_name: str, key_id: str, signature: str) -> None: ... + """Add a signature for the given server name and key ID.""" + + def update(self, signatures: Mapping[str, Mapping[str, str]]) -> None: ... + """Update the signatures with the given signatures. + + Will overwrite all existing signatures for the server names provided. + """ + + def as_dict(self) -> dict[str, dict[str, str]]: ... + """Return a copy of the signatures as a dictionary.""" + +class Unsigned: + """A class representing the unsigned data of an event.""" + + def __init__(self, unsigned_dict: JsonMapping): ... + def __getitem__(self, key: str) -> Any: ... + """Get the value for the given key. + + Raises KeyError if the key is unset or not recognised.""" + + def __setitem__(self, key: str, value: Any) -> None: ... + """Set the value for the given key. + + Raises KeyError if the key is not recognised.""" + + def __delitem__(self, key: str) -> None: ... + """Delete the value for the given key. + + Raises KeyError if the key is unset or not recognised.""" + + def __contains__(self, key: Any) -> bool: ... + def get(self, key: str, default: Any = None) -> Any: ... + """Get the value for the given key, or ``default`` if the key is unset.""" + + def for_persistence(self) -> JsonDict: ... + """Return a dict of the fields that should be persisted to the database.""" + + def for_event(self) -> JsonDict: ... + """Return a dict of all unsigned fields, including those only kept in + memory, suitable for inclusion in an event.""" diff --git a/synapse/util/duration.py b/synapse/util/duration.py index 135b980852..a1abe944b5 100644 --- a/synapse/util/duration.py +++ b/synapse/util/duration.py @@ -32,6 +32,33 @@ class Duration(timedelta): ``` """ + # Using `__new__` (instead of `__init__`) because that's what `timedelta` uses + def __new__( + cls, + # The whole goal of overriding `__new__` is to require keyword-only arguments. + # Without this, `Duration(5)` would create a duration represnting 5 *days* + # (timedelta's default), but callers almost certainly want to specify which unit + # like seconds or hours. + *, + days: float = 0, + seconds: float = 0, + microseconds: float = 0, + milliseconds: float = 0, + minutes: float = 0, + hours: float = 0, + weeks: float = 0, + ) -> "Duration": + return super().__new__( + cls, + days=days, + seconds=seconds, + microseconds=microseconds, + milliseconds=milliseconds, + minutes=minutes, + hours=hours, + weeks=weeks, + ) + def as_millis(self) -> int: """Returns the duration in milliseconds.""" return int(self / _ONE_MILLISECOND) diff --git a/tests/config/test_api.py b/tests/config/test_api.py index e6cc3e21ed..d4fa398a47 100644 --- a/tests/config/test_api.py +++ b/tests/config/test_api.py @@ -5,7 +5,6 @@ from synapse.config import ConfigError from synapse.config._base import RootConfig from synapse.config.api import ApiConfig -from synapse.types.state import StateFilter DEFAULT_PREJOIN_STATE_PAIRS = { ("m.room.join_rules", ""), @@ -38,7 +37,11 @@ def test_disable_default_event_types(self) -> None: disable_default_event_types: true """ ) - self.assertEqual(config.room_prejoin_state, StateFilter.none()) + # MSC4311: m.room.create is always included even when defaults are disabled + self.assertEqual( + set(config.room_prejoin_state.concrete_types()), + {("m.room.create", "")}, + ) def test_event_without_state_key(self) -> None: config = self.read_config( @@ -50,7 +53,11 @@ def test_event_without_state_key(self) -> None: """ ) self.assertEqual(config.room_prejoin_state.wildcard_types(), ["foo"]) - self.assertEqual(config.room_prejoin_state.concrete_types(), []) + # MSC4311: m.room.create is always included + self.assertEqual( + set(config.room_prejoin_state.concrete_types()), + {("m.room.create", "")}, + ) def test_event_with_specific_state_key(self) -> None: config = self.read_config( @@ -62,9 +69,10 @@ def test_event_with_specific_state_key(self) -> None: """ ) self.assertFalse(config.room_prejoin_state.has_wildcards()) + # MSC4311: m.room.create is always included self.assertEqual( set(config.room_prejoin_state.concrete_types()), - {("foo", "bar")}, + {("foo", "bar"), ("m.room.create", "")}, ) def test_repeated_event_with_specific_state_key(self) -> None: @@ -78,9 +86,10 @@ def test_repeated_event_with_specific_state_key(self) -> None: """ ) self.assertFalse(config.room_prejoin_state.has_wildcards()) + # MSC4311: m.room.create is always included self.assertEqual( set(config.room_prejoin_state.concrete_types()), - {("foo", "bar"), ("foo", "baz")}, + {("foo", "bar"), ("foo", "baz"), ("m.room.create", "")}, ) def test_no_specific_state_key_overrides_specific_state_key(self) -> None: @@ -94,7 +103,11 @@ def test_no_specific_state_key_overrides_specific_state_key(self) -> None: """ ) self.assertEqual(config.room_prejoin_state.wildcard_types(), ["foo"]) - self.assertEqual(config.room_prejoin_state.concrete_types(), []) + # MSC4311: m.room.create is always included + self.assertEqual( + set(config.room_prejoin_state.concrete_types()), + {("m.room.create", "")}, + ) config = self.read_config( """ @@ -106,7 +119,11 @@ def test_no_specific_state_key_overrides_specific_state_key(self) -> None: """ ) self.assertEqual(config.room_prejoin_state.wildcard_types(), ["foo"]) - self.assertEqual(config.room_prejoin_state.concrete_types(), []) + # MSC4311: m.room.create is always included + self.assertEqual( + set(config.room_prejoin_state.concrete_types()), + {("m.room.create", "")}, + ) def test_bad_event_type_entry_raises(self) -> None: with self.assertRaises(ConfigError): diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index 12ef42866d..a402dd70d1 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -67,25 +67,31 @@ def MockEvent(**kwargs: Any) -> EventBase: class TestMaybeUpsertEventField(stdlib_unittest.TestCase): def test_update_okay(self) -> None: event = make_event_from_dict({"event_id": "$1234"}) - success = maybe_upsert_event_field(event, event.unsigned, "key", "value") + success = maybe_upsert_event_field( + event, event.unsigned, "replaces_state", "value" + ) self.assertTrue(success) - self.assertEqual(event.unsigned["key"], "value") + self.assertEqual(event.unsigned["replaces_state"], "value") def test_update_not_okay(self) -> None: event = make_event_from_dict({"event_id": "$1234"}) LARGE_STRING = "a" * 100_000 - success = maybe_upsert_event_field(event, event.unsigned, "key", LARGE_STRING) + success = maybe_upsert_event_field( + event, event.unsigned, "replaces_state", LARGE_STRING + ) self.assertFalse(success) - self.assertNotIn("key", event.unsigned) + self.assertNotIn("replaces_state", event.unsigned) def test_update_not_okay_leaves_original_value(self) -> None: event = make_event_from_dict( - {"event_id": "$1234", "unsigned": {"key": "value"}} + {"event_id": "$1234", "unsigned": {"replaces_state": "value"}} ) LARGE_STRING = "a" * 100_000 - success = maybe_upsert_event_field(event, event.unsigned, "key", LARGE_STRING) + success = maybe_upsert_event_field( + event, event.unsigned, "replaces_state", LARGE_STRING + ) self.assertFalse(success) - self.assertEqual(event.unsigned["key"], "value") + self.assertEqual(event.unsigned["replaces_state"], "value") class PruneEventTestCase(stdlib_unittest.TestCase): @@ -623,7 +629,7 @@ def test_unsigned_is_copied(self) -> None: { "type": "A", "event_id": "$test:domain", - "unsigned": {"a": 1, "b": 2}, + "unsigned": {"age_ts": 1, "replaces_state": "2"}, }, RoomVersions.V1, {"txn_id": "txn"}, @@ -634,10 +640,14 @@ def test_unsigned_is_copied(self) -> None: self.assertEqual(original.internal_metadata.instance_name, "worker1") cloned = clone_event(original) - cloned.unsigned["b"] = 3 + cloned.unsigned["age_ts"] = 3 - self.assertEqual(original.unsigned, {"a": 1, "b": 2}) - self.assertEqual(cloned.unsigned, {"a": 1, "b": 3}) + self.assertEqual( + original.unsigned.for_event(), {"age_ts": 1, "replaces_state": "2"} + ) + self.assertEqual( + cloned.unsigned.for_event(), {"age_ts": 3, "replaces_state": "2"} + ) self.assertEqual(cloned.internal_metadata.stream_ordering, 1234) self.assertEqual(cloned.internal_metadata.instance_name, "worker1") self.assertEqual(cloned.internal_metadata.txn_id, "txn") diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index 6ed9e96037..eef3c50cd0 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -814,9 +814,9 @@ def test_strip_event_maintains_allowed_fields(self) -> None: }, } - filtered_event2 = event_from_pdu_json(event2, RoomVersions.V1) - self.assertIn("age", filtered_event2.unsigned) - self.assertEqual(14, filtered_event2.unsigned["age"]) + filtered_event2 = event_from_pdu_json(event2, RoomVersions.V1, received_time=20) + self.assertIn("age_ts", filtered_event2.unsigned) + self.assertEqual(6, filtered_event2.unsigned["age_ts"]) self.assertNotIn("more warez", filtered_event2.unsigned) # Invite_room_state is allowed in events of type m.room.member self.assertIn("invite_room_state", filtered_event2.unsigned) @@ -839,8 +839,8 @@ def test_strip_event_removes_fields_based_on_event_type(self) -> None: "invite_room_state": [], }, } - filtered_event3 = event_from_pdu_json(event3, RoomVersions.V1) - self.assertIn("age", filtered_event3.unsigned) + filtered_event3 = event_from_pdu_json(event3, RoomVersions.V1, received_time=20) + self.assertIn("age_ts", filtered_event3.unsigned) # Invite_room_state field is only permitted in event type m.room.member self.assertNotIn("invite_room_state", filtered_event3.unsigned) self.assertNotIn("more warez", filtered_event3.unsigned) diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index e4a41cf1ae..20ffed68f4 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -43,10 +43,8 @@ from synapse.server import HomeServer from synapse.storage.databases.main.events_worker import EventCacheEntry from synapse.util.clock import Clock -from synapse.util.events import generate_fake_event_id from tests import unittest -from tests.test_utils import event_injection logger = logging.getLogger(__name__) @@ -213,121 +211,6 @@ def test_rejected_state_event_state(self) -> None: self.assertEqual(sg, sg2) - def test_backfill_with_many_backward_extremities(self) -> None: - """ - Check that we can backfill with many backward extremities. - The goal is to make sure that when we only use a portion - of backwards extremities(the magic number is more than 5), - no errors are thrown. - - Regression test, see https://github.com/matrix-org/synapse/pull/11027 - """ - # create the room - user_id = self.register_user("kermit", "test") - tok = self.login("kermit", "test") - - room_id = self.helper.create_room_as(room_creator=user_id, tok=tok) - room_version = self.get_success(self.store.get_room_version(room_id)) - - # we need a user on the remote server to be a member, so that we can send - # extremity-causing events. - remote_server_user_id = f"@user:{self.OTHER_SERVER_NAME}" - self.get_success( - event_injection.inject_member_event( - self.hs, room_id, remote_server_user_id, "join" - ) - ) - - send_result = self.helper.send(room_id, "first message", tok=tok) - ev1 = self.get_success( - self.store.get_event(send_result["event_id"], allow_none=False) - ) - current_state = self.get_success( - self.store.get_events_as_list( - ( - self.get_success(self.store.get_partial_current_state_ids(room_id)) - ).values() - ) - ) - - # Create "many" backward extremities. The magic number we're trying to - # create more than is 5 which corresponds to the number of backward - # extremities we slice off in `_maybe_backfill_inner` - federation_event_handler = self.hs.get_federation_event_handler() - auth_events = [ - ev - for ev in current_state - if (ev.type, ev.state_key) - in {("m.room.create", ""), ("m.room.member", remote_server_user_id)} - ] - for _ in range(8): - event = make_event_from_dict( - self.add_hashes_and_signatures_from_other_server( - { - "origin_server_ts": 1, - "type": "m.room.message", - "content": { - "msgtype": "m.text", - "body": "message connected to fake event", - }, - "room_id": room_id, - "sender": remote_server_user_id, - "prev_events": [ - ev1.event_id, - # We're creating an backward extremity each time thanks - # to this fake event - generate_fake_event_id(), - ], - "auth_events": [ev.event_id for ev in auth_events], - "depth": ev1.depth + 1, - }, - room_version, - ), - room_version, - ) - - # we poke this directly into _process_received_pdu, to avoid the - # federation handler wanting to backfill the fake event. - state_handler = self.hs.get_state_handler() - context = self.get_success( - state_handler.compute_event_context( - event, - state_ids_before_event={ - (e.type, e.state_key): e.event_id for e in current_state - }, - partial_state=False, - ) - ) - self.get_success( - federation_event_handler._process_received_pdu( - self.OTHER_SERVER_NAME, - event, - context, - ) - ) - - # we should now have 8 backwards extremities. - backwards_extremities = self.get_success( - self.store.db_pool.simple_select_list( - "event_backward_extremities", - keyvalues={"room_id": room_id}, - retcols=["event_id"], - ) - ) - self.assertEqual(len(backwards_extremities), 8) - - current_depth = 1 - limit = 100 - - # Make sure backfill still works - self.get_success( - self.hs.get_federation_handler().maybe_backfill( - room_id, - current_depth, - limit, - ) - ) - def test_backfill_ignores_known_events(self) -> None: """ Tests that events that we already know about are ignored when backfilling. @@ -485,7 +368,7 @@ def _build_and_send_join_event( ) # the auth code requires that a signature exists, but doesn't check that # signature... go figure. - join_event.signatures[other_server] = {"x": "y"} + join_event.signatures.update({other_server: {"x": "y"}}) self.get_success( self.hs.get_federation_event_handler().on_send_membership_event( diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index a643c5116e..5152e8fc53 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -316,7 +316,7 @@ async def potentially_slow_update_membership( ) @override_config({"enable_set_displayname": False}) - def test_set_displayname_if_disabled(self) -> None: + def test_set_my_name_if_disabled(self) -> None: # Setting displayname for the first time is allowed self.get_success(self.store.set_profile_displayname(self.frank, "Frank")) @@ -436,7 +436,7 @@ def test_set_my_avatar(self) -> None: ) @override_config({"enable_set_avatar_url": False}) - def test_set_avatar_url_if_disabled(self) -> None: + def test_set_my_avatar_if_disabled(self) -> None: # Setting displayname for the first time is allowed self.get_success( self.store.set_profile_avatar_url(self.frank, "http://my.server/me.png") diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index b32665eb73..507cf10c5d 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -2311,10 +2311,14 @@ def test_single_room(self) -> None: self.assertIn("state_events", channel.json_body) self.assertIn("room_type", channel.json_body) self.assertIn("forgotten", channel.json_body) + self.assertIn("tombstoned", channel.json_body) + self.assertIn("replacement_room", channel.json_body) self.assertEqual(room_id_1, channel.json_body["room_id"]) self.assertIs(True, channel.json_body["federatable"]) self.assertIs(True, channel.json_body["public"]) + self.assertIs(False, channel.json_body["tombstoned"]) + self.assertIs(None, channel.json_body["replacement_room"]) def test_single_room_devices(self) -> None: """Test that `joined_local_devices` can be requested correctly""" diff --git a/tests/rest/client/test_account.py b/tests/rest/client/test_account.py index 2e8089b363..42102230f0 100644 --- a/tests/rest/client/test_account.py +++ b/tests/rest/client/test_account.py @@ -502,13 +502,17 @@ def test_deactivate_account(self) -> None: def test_deactivate_erase_account(self) -> None: """ - Test that deactivating the user with the 'erase' option will remove existing - profile data. + Test that a user account can be signaled for erasure on the Matrix spec endpoint + for client access, `/account/deactivate` and that profile data is erased as part + of the process """ mxid = self.register_user("kermit", "test") user_id = UserID.from_string(mxid) tok = self.login("kermit", "test") + profile_handler = self.hs.get_profile_handler() + + # Set some profile data that can be checked for after the user is erased self.get_success( profile_handler.set_displayname( user_id, create_requester(user_id), "Kermit the Frog" @@ -562,7 +566,9 @@ def test_deactivate_erase_account_with_disabled_profile_changes(self) -> None: mxid = self.register_user("kermit", "test") user_id = UserID.from_string(mxid) tok = self.login("kermit", "test") + profile_handler = self.hs.get_profile_handler() + # Can not use the profile handler to set a display name when it is disabled. Use # the database directly store = self.hs.get_datastores().main @@ -810,6 +816,15 @@ def test_background_update_deletes_deactivated_users_server_side_backup_keys( self.assertEqual(len(res2), 4) def deactivate(self, user_id: str, tok: str, erase: bool = False) -> None: + """ + Helper to deactivate a user using the /account/deactivate endpoint, optionally + with erasure + + Args: + user_id: the string formatted mxid(not a UserID) + tok: the user's access token + erase: bool of if this should be a full erasure request + """ request_data = { "auth": { "type": "m.login.password", diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index e6ada1adb2..74a8678ae9 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -29,6 +29,7 @@ from synapse.api.constants import ( EventContentFields, EventTypes, + JoinRules, ReceiptTypes, RelationTypes, ) @@ -394,6 +395,69 @@ def test_knock_room_state(self) -> None: ) +class SyncCreateEventInPrejoinStateTestCase(unittest.HomeserverTestCase): + """MSC4311: Tests that m.room.create is present in invite_state and knock_state""" + + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + room.register_servlets, + sync.register_servlets, + knock.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + return config + + def test_create_event_present_in_invite_state(self) -> None: + """m.room.create must appear in invite_state.""" + inviter = self.register_user("inviter", "pass") + inviter_tok = self.login("inviter", "pass") + invitee = self.register_user("invitee", "pass") + invitee_tok = self.login("invitee", "pass") + + room_id = self.helper.create_room_as(inviter, tok=inviter_tok) + self.helper.invite(room=room_id, src=inviter, targ=invitee, tok=inviter_tok) + + channel = self.make_request("GET", "/sync", access_token=invitee_tok) + self.assertEqual(channel.code, 200, channel.json_body) + + invite_state_events = channel.json_body["rooms"]["invite"][room_id][ + "invite_state" + ]["events"] + event_types = {stripped_event["type"] for stripped_event in invite_state_events} + self.assertIn(EventTypes.Create, event_types) + + def test_create_event_present_in_knock_state(self) -> None: + """m.room.create must appear in knock_state.""" + host = self.register_user("host", "pass") + host_tok = self.login("host", "pass") + knocker = self.register_user("knocker", "pass") + knocker_tok = self.login("knocker", "pass") + + room_id = self.helper.create_room_as( + host, is_public=False, room_version="7", tok=host_tok + ) + self.helper.send_state( + room_id, + EventTypes.JoinRules, + {"join_rule": JoinRules.KNOCK}, + tok=host_tok, + ) + + self.helper.knock(room_id, knocker, tok=knocker_tok) + + channel = self.make_request("GET", "/sync", access_token=knocker_tok) + self.assertEqual(channel.code, 200, channel.json_body) + + knock_state_events = channel.json_body["rooms"]["knock"][room_id][ + "knock_state" + ]["events"] + event_types = {stripped_event["type"] for stripped_event in knock_state_events} + self.assertIn(EventTypes.Create, event_types) + + class UnreadMessagesTestCase(unittest.HomeserverTestCase): servlets = [ synapse.rest.admin.register_servlets, diff --git a/tests/rest/client/test_upgrade_room.py b/tests/rest/client/test_upgrade_room.py index ee26492909..6cb85c94c4 100644 --- a/tests/rest/client/test_upgrade_room.py +++ b/tests/rest/client/test_upgrade_room.py @@ -23,6 +23,7 @@ from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventContentFields, EventTypes, Membership, RoomTypes +from synapse.api.room_versions import RoomVersions from synapse.config.server import DEFAULT_ROOM_VERSION from synapse.rest import admin from synapse.rest.client import login, room, room_upgrade_rest_servlet @@ -58,6 +59,7 @@ def _upgrade_room( token: str | None = None, room_id: str | None = None, expire_cache: bool = True, + new_version: str = DEFAULT_ROOM_VERSION, ) -> FakeChannel: if expire_cache: # We don't want a cached response. @@ -70,7 +72,7 @@ def _upgrade_room( "POST", f"/_matrix/client/r0/rooms/{room_id}/upgrade", # This will upgrade a room to the same version, but that's fine. - content={"new_version": DEFAULT_ROOM_VERSION}, + content={"new_version": new_version}, access_token=token or self.creator_token, ) @@ -431,3 +433,51 @@ def test_bans(self) -> None: tok=self.creator_token, ) self.assertEqual(content[EventContentFields.MEMBERSHIP], Membership.BAN) + + def test_creator_removed_from_powerlevels_v12(self) -> None: + """ + Test that the creator is removed from the power levels users map when + upgrading to a room version with MSC4289. + """ + # Create a room on room version 11, which doesn't have MSC4289. + room_id = self.helper.create_room_as( + self.creator, tok=self.creator_token, room_version="11" + ) + self.helper.join(room_id, self.other, tok=self.other_token) + + # Retrieve the room's current power levels. + old_power_level_event = self.get_success( + self.hs.get_storage_controllers().state.get_current_state_event( + room_id, "m.room.power_levels", "" + ) + ) + assert old_power_level_event is not None + + # The creator should be in the users map with power level 100. + self.assertEqual(old_power_level_event.content["users"][self.creator], 100) + + # Upgrade the room to version 12, which has MSC4289. + channel = self._upgrade_room( + room_id=room_id, new_version=RoomVersions.V12.identifier + ) + self.assertEqual(200, channel.code, channel.result) + + # Extract the new room ID. + new_room_id = channel.json_body["replacement_room"] + + # Fetch the new room's power level event. + new_power_levels = self.helper.get_state( + new_room_id, + "m.room.power_levels", + tok=self.creator_token, + ) + + # The creator should no longer be in the users map. + self.assertNotIn(self.creator, new_power_levels["users"]) + + # The creator should still be in the old power levels event with power + # level 100. + # + # This is a regression test where previously Synapse would accidentally + # mutate the old power levels event. + self.assertEqual(old_power_level_event.content["users"][self.creator], 100) diff --git a/tests/synapse_rust/test_unsigned.py b/tests/synapse_rust/test_unsigned.py new file mode 100644 index 0000000000..5193188f38 --- /dev/null +++ b/tests/synapse_rust/test_unsigned.py @@ -0,0 +1,52 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from synapse.synapse_rust.events import Unsigned + +from tests import unittest + + +class UnsignedTestCase(unittest.TestCase): + def test_prev_content(self) -> None: + """Test that the prev_content field is correctly exposed as a JsonObject.""" + unsigned = Unsigned({"prev_content": {"key1": "value1", "key2": 42}}) + + self.assert_dict(unsigned["prev_content"], {"key1": "value1", "key2": 42}) + + self.assert_dict( + unsigned.for_event(), {"prev_content": {"key1": "value1", "key2": 42}} + ) + + def test_large_age_ts(self) -> None: + """Test that we can handle integers larger than 2^128, which is larger + than the maximum rust native integer size.""" + + large_int = 2**200 + unsigned = Unsigned({"age_ts": large_int}) + + self.assertEqual(unsigned["age_ts"], large_int) + + self.assert_dict(unsigned.for_event(), {"age_ts": large_int}) + + def test_large_integer_in_prev_content(self) -> None: + """Test that we can handle integers larger than 2^128 in the + prev_content field, which is a JsonObject and thus can contain arbitrary + JSON.""" + + large_int = 2**200 + unsigned = Unsigned({"prev_content": {"some_field": large_int}}) + + self.assertEqual(unsigned["prev_content"]["some_field"], large_int) + self.assert_dict( + unsigned.for_event(), {"prev_content": {"some_field": large_int}} + )