From 3779c59d1b52347c5554d5289a8b645e874438f1 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 19 Nov 2025 11:37:57 +0000 Subject: [PATCH 001/139] move postgres 13 deprecation note to top of changelog --- CHANGES.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b492695196c..5ac37368675 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,12 @@ # synapse 1.143.0rc2 (2025-11-18) +## Dropping support for PostgreSQL 13 + +In line with our [deprecation policy](https://github.com/element-hq/synapse/blob/develop/docs/deprecation_policy.md), we've dropped +support for PostgreSQL 13, as it is no longer supported upstream. +This release of Synapse requires PostgreSQL 14+. + + ## Internal Changes - Fixes docker image creation in the release workflow. @@ -8,12 +15,6 @@ # Synapse 1.143.0rc1 (2025-11-18) -## Dropping support for PostgreSQL 13 - -In line with our [deprecation policy](https://github.com/element-hq/synapse/blob/develop/docs/deprecation_policy.md), we've dropped -support for PostgreSQL 13, as it is no longer supported upstream. -This release of Synapse requires PostgreSQL 14+. - ## Features - Support multiple config files in `register_new_matrix_user`. ([\#18784](https://github.com/element-hq/synapse/issues/18784)) From ea3e08c49cd31762aa9ebb267f8c8ac2097f506c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 15:58:26 +0000 Subject: [PATCH 002/139] Bump attrs from 25.3.0 to 25.4.0 (#19215) Bumps [attrs](https://github.com/sponsors/hynek) from 25.3.0 to 25.4.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=attrs&package-manager=pip&previous-version=25.3.0&new-version=25.4.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/poetry.lock b/poetry.lock index dec659db329..019805c605d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -14,24 +14,16 @@ files = [ [[package]] name = "attrs" -version = "25.3.0" +version = "25.4.0" description = "Classes Without Boilerplate" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3"}, - {file = "attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b"}, + {file = "attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373"}, + {file = "attrs-25.4.0.tar.gz", hash = "sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11"}, ] -[package.extras] -benchmark = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] -cov = ["cloudpickle ; platform_python_implementation == \"CPython\"", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] -dev = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] -docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier"] -tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] -tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\""] - [[package]] name = "authlib" version = "1.6.5" From a5d946bfcbf2cb7f194c0724e47fad7ce489f6ef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 16:29:32 +0000 Subject: [PATCH 003/139] Bump types-bleach from 6.2.0.20250809 to 6.3.0.20251115 (#19217) Bumps [types-bleach](https://github.com/typeshed-internal/stub_uploader) from 6.2.0.20250809 to 6.3.0.20251115.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=types-bleach&package-manager=pip&previous-version=6.2.0.20250809&new-version=6.3.0.20251115)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 019805c605d..be5f1847c46 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2976,14 +2976,14 @@ twisted = "*" [[package]] name = "types-bleach" -version = "6.2.0.20250809" +version = "6.3.0.20251115" description = "Typing stubs for bleach" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types_bleach-6.2.0.20250809-py3-none-any.whl", hash = "sha256:0b372a75117947d9ac8a31ae733fd0f8d92ec75c4772e7b37093ba3fa5b48fb9"}, - {file = "types_bleach-6.2.0.20250809.tar.gz", hash = "sha256:188d7a1119f6c953140b513ed57ba4213755695815472c19d0c22ac09c79b90b"}, + {file = "types_bleach-6.3.0.20251115-py3-none-any.whl", hash = "sha256:f81e7cf4ebac3f3d60b66b3fd5236c324e65037d1b28d22c94d5b457f0b98f42"}, + {file = "types_bleach-6.3.0.20251115.tar.gz", hash = "sha256:96911b20f169a18524d03b61fa7e98a08c411292f7cdb5dc191057f55dad9ae3"}, ] [package.dependencies] From 1b78f0318aa51138c4e5c7c2f87ae69b7431f33a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 16:30:09 +0000 Subject: [PATCH 004/139] Bump rpds-py from 0.28.0 to 0.29.0 (#19216) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [rpds-py](https://github.com/crate-py/rpds) from 0.28.0 to 0.29.0.
Release notes

Sourced from rpds-py's releases.

v0.29.0

What's Changed

Full Changelog: https://github.com/crate-py/rpds/compare/v0.28.0...v0.29.0

Commits
  • 5fb6f35 Prepare for 0.29.0
  • d17dbd1 Add rpds's Stack.
  • 74707af Follow the rpds API more closely for Queue.
  • 41455f3 -> native uv for dpeendency groups.
  • e93532d Use 3.14 by default in nox.
  • 020c41f Remove dead hooks.
  • 6e08b75 Accept zizmor's cooldown suggestions for dependabot.
  • a5d40a9 Merge pull request #197 from crate-py/dependabot/cargo/rpds-1.2.0
  • b830be1 Merge pull request #198 from crate-py/dependabot/github_actions/softprops/act...
  • e7ac330 Merge pull request #199 from crate-py/pre-commit-ci-update-config
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=rpds-py&package-manager=pip&previous-version=0.28.0&new-version=0.29.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 232 ++++++++++++++++++++++++++-------------------------- 1 file changed, 116 insertions(+), 116 deletions(-) diff --git a/poetry.lock b/poetry.lock index be5f1847c46..5a79b932a5d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2356,127 +2356,127 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "rpds-py" -version = "0.28.0" +version = "0.29.0" description = "Python bindings to Rust's persistent data structures (rpds)" optional = false python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "rpds_py-0.28.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:7b6013db815417eeb56b2d9d7324e64fcd4fa289caeee6e7a78b2e11fc9b438a"}, - {file = "rpds_py-0.28.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1a4c6b05c685c0c03f80dabaeb73e74218c49deea965ca63f76a752807397207"}, - {file = "rpds_py-0.28.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4794c6c3fbe8f9ac87699b131a1f26e7b4abcf6d828da46a3a52648c7930eba"}, - {file = "rpds_py-0.28.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2e8456b6ee5527112ff2354dd9087b030e3429e43a74f480d4a5ca79d269fd85"}, - {file = "rpds_py-0.28.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:beb880a9ca0a117415f241f66d56025c02037f7c4efc6fe59b5b8454f1eaa50d"}, - {file = "rpds_py-0.28.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6897bebb118c44b38c9cb62a178e09f1593c949391b9a1a6fe777ccab5934ee7"}, - {file = "rpds_py-0.28.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1b553dd06e875249fd43efd727785efb57a53180e0fde321468222eabbeaafa"}, - {file = "rpds_py-0.28.0-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:f0b2044fdddeea5b05df832e50d2a06fe61023acb44d76978e1b060206a8a476"}, - {file = "rpds_py-0.28.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:05cf1e74900e8da73fa08cc76c74a03345e5a3e37691d07cfe2092d7d8e27b04"}, - {file = "rpds_py-0.28.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:efd489fec7c311dae25e94fe7eeda4b3d06be71c68f2cf2e8ef990ffcd2cd7e8"}, - {file = "rpds_py-0.28.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:ada7754a10faacd4f26067e62de52d6af93b6d9542f0df73c57b9771eb3ba9c4"}, - {file = "rpds_py-0.28.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c2a34fd26588949e1e7977cfcbb17a9a42c948c100cab890c6d8d823f0586457"}, - {file = "rpds_py-0.28.0-cp310-cp310-win32.whl", hash = "sha256:f9174471d6920cbc5e82a7822de8dfd4dcea86eb828b04fc8c6519a77b0ee51e"}, - {file = "rpds_py-0.28.0-cp310-cp310-win_amd64.whl", hash = "sha256:6e32dd207e2c4f8475257a3540ab8a93eff997abfa0a3fdb287cae0d6cd874b8"}, - {file = "rpds_py-0.28.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:03065002fd2e287725d95fbc69688e0c6daf6c6314ba38bdbaa3895418e09296"}, - {file = "rpds_py-0.28.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:28ea02215f262b6d078daec0b45344c89e161eab9526b0d898221d96fdda5f27"}, - {file = "rpds_py-0.28.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25dbade8fbf30bcc551cb352376c0ad64b067e4fc56f90e22ba70c3ce205988c"}, - {file = "rpds_py-0.28.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3c03002f54cc855860bfdc3442928ffdca9081e73b5b382ed0b9e8efe6e5e205"}, - {file = "rpds_py-0.28.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b9699fa7990368b22032baf2b2dce1f634388e4ffc03dfefaaac79f4695edc95"}, - {file = "rpds_py-0.28.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b9b06fe1a75e05e0713f06ea0c89ecb6452210fd60e2f1b6ddc1067b990e08d9"}, - {file = "rpds_py-0.28.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac9f83e7b326a3f9ec3ef84cda98fb0a74c7159f33e692032233046e7fd15da2"}, - {file = "rpds_py-0.28.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:0d3259ea9ad8743a75a43eb7819324cdab393263c91be86e2d1901ee65c314e0"}, - {file = "rpds_py-0.28.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a7548b345f66f6695943b4ef6afe33ccd3f1b638bd9afd0f730dd255c249c9e"}, - {file = "rpds_py-0.28.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c9a40040aa388b037eb39416710fbcce9443498d2eaab0b9b45ae988b53f5c67"}, - {file = "rpds_py-0.28.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8f60c7ea34e78c199acd0d3cda37a99be2c861dd2b8cf67399784f70c9f8e57d"}, - {file = "rpds_py-0.28.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1571ae4292649100d743b26d5f9c63503bb1fedf538a8f29a98dce2d5ba6b4e6"}, - {file = "rpds_py-0.28.0-cp311-cp311-win32.whl", hash = "sha256:5cfa9af45e7c1140af7321fa0bef25b386ee9faa8928c80dc3a5360971a29e8c"}, - {file = "rpds_py-0.28.0-cp311-cp311-win_amd64.whl", hash = "sha256:dd8d86b5d29d1b74100982424ba53e56033dc47720a6de9ba0259cf81d7cecaa"}, - {file = "rpds_py-0.28.0-cp311-cp311-win_arm64.whl", hash = "sha256:4e27d3a5709cc2b3e013bf93679a849213c79ae0573f9b894b284b55e729e120"}, - {file = "rpds_py-0.28.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:6b4f28583a4f247ff60cd7bdda83db8c3f5b05a7a82ff20dd4b078571747708f"}, - {file = "rpds_py-0.28.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d678e91b610c29c4b3d52a2c148b641df2b4676ffe47c59f6388d58b99cdc424"}, - {file = "rpds_py-0.28.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e819e0e37a44a78e1383bf1970076e2ccc4dc8c2bbaa2f9bd1dc987e9afff628"}, - {file = "rpds_py-0.28.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5ee514e0f0523db5d3fb171f397c54875dbbd69760a414dccf9d4d7ad628b5bd"}, - {file = "rpds_py-0.28.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5f3fa06d27fdcee47f07a39e02862da0100cb4982508f5ead53ec533cd5fe55e"}, - {file = "rpds_py-0.28.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:46959ef2e64f9e4a41fc89aa20dbca2b85531f9a72c21099a3360f35d10b0d5a"}, - {file = "rpds_py-0.28.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8455933b4bcd6e83fde3fefc987a023389c4b13f9a58c8d23e4b3f6d13f78c84"}, - {file = "rpds_py-0.28.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:ad50614a02c8c2962feebe6012b52f9802deec4263946cddea37aaf28dd25a66"}, - {file = "rpds_py-0.28.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e5deca01b271492553fdb6c7fd974659dce736a15bae5dad7ab8b93555bceb28"}, - {file = "rpds_py-0.28.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:735f8495a13159ce6a0d533f01e8674cec0c57038c920495f87dcb20b3ddb48a"}, - {file = "rpds_py-0.28.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:961ca621ff10d198bbe6ba4957decca61aa2a0c56695384c1d6b79bf61436df5"}, - {file = "rpds_py-0.28.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2374e16cc9131022e7d9a8f8d65d261d9ba55048c78f3b6e017971a4f5e6353c"}, - {file = "rpds_py-0.28.0-cp312-cp312-win32.whl", hash = "sha256:d15431e334fba488b081d47f30f091e5d03c18527c325386091f31718952fe08"}, - {file = "rpds_py-0.28.0-cp312-cp312-win_amd64.whl", hash = "sha256:a410542d61fc54710f750d3764380b53bf09e8c4edbf2f9141a82aa774a04f7c"}, - {file = "rpds_py-0.28.0-cp312-cp312-win_arm64.whl", hash = "sha256:1f0cfd1c69e2d14f8c892b893997fa9a60d890a0c8a603e88dca4955f26d1edd"}, - {file = "rpds_py-0.28.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:e9e184408a0297086f880556b6168fa927d677716f83d3472ea333b42171ee3b"}, - {file = "rpds_py-0.28.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:edd267266a9b0448f33dc465a97cfc5d467594b600fe28e7fa2f36450e03053a"}, - {file = "rpds_py-0.28.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85beb8b3f45e4e32f6802fb6cd6b17f615ef6c6a52f265371fb916fae02814aa"}, - {file = "rpds_py-0.28.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d2412be8d00a1b895f8ad827cc2116455196e20ed994bb704bf138fe91a42724"}, - {file = "rpds_py-0.28.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cf128350d384b777da0e68796afdcebc2e9f63f0e9f242217754e647f6d32491"}, - {file = "rpds_py-0.28.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a2036d09b363aa36695d1cc1a97b36865597f4478470b0697b5ee9403f4fe399"}, - {file = "rpds_py-0.28.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8e1e9be4fa6305a16be628959188e4fd5cd6f1b0e724d63c6d8b2a8adf74ea6"}, - {file = "rpds_py-0.28.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0a403460c9dd91a7f23fc3188de6d8977f1d9603a351d5db6cf20aaea95b538d"}, - {file = "rpds_py-0.28.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d7366b6553cdc805abcc512b849a519167db8f5e5c3472010cd1228b224265cb"}, - {file = "rpds_py-0.28.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5b43c6a3726efd50f18d8120ec0551241c38785b68952d240c45ea553912ac41"}, - {file = "rpds_py-0.28.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0cb7203c7bc69d7c1585ebb33a2e6074492d2fc21ad28a7b9d40457ac2a51ab7"}, - {file = "rpds_py-0.28.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7a52a5169c664dfb495882adc75c304ae1d50df552fbd68e100fdc719dee4ff9"}, - {file = "rpds_py-0.28.0-cp313-cp313-win32.whl", hash = "sha256:2e42456917b6687215b3e606ab46aa6bca040c77af7df9a08a6dcfe8a4d10ca5"}, - {file = "rpds_py-0.28.0-cp313-cp313-win_amd64.whl", hash = "sha256:e0a0311caedc8069d68fc2bf4c9019b58a2d5ce3cd7cb656c845f1615b577e1e"}, - {file = "rpds_py-0.28.0-cp313-cp313-win_arm64.whl", hash = "sha256:04c1b207ab8b581108801528d59ad80aa83bb170b35b0ddffb29c20e411acdc1"}, - {file = "rpds_py-0.28.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:f296ea3054e11fc58ad42e850e8b75c62d9a93a9f981ad04b2e5ae7d2186ff9c"}, - {file = "rpds_py-0.28.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5a7306c19b19005ad98468fcefeb7100b19c79fc23a5f24a12e06d91181193fa"}, - {file = "rpds_py-0.28.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5d9b86aa501fed9862a443c5c3116f6ead8bc9296185f369277c42542bd646b"}, - {file = "rpds_py-0.28.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e5bbc701eff140ba0e872691d573b3d5d30059ea26e5785acba9132d10c8c31d"}, - {file = "rpds_py-0.28.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a5690671cd672a45aa8616d7374fdf334a1b9c04a0cac3c854b1136e92374fe"}, - {file = "rpds_py-0.28.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9f1d92ecea4fa12f978a367c32a5375a1982834649cdb96539dcdc12e609ab1a"}, - {file = "rpds_py-0.28.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d252db6b1a78d0a3928b6190156042d54c93660ce4d98290d7b16b5296fb7cc"}, - {file = "rpds_py-0.28.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:d61b355c3275acb825f8777d6c4505f42b5007e357af500939d4a35b19177259"}, - {file = "rpds_py-0.28.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:acbe5e8b1026c0c580d0321c8aae4b0a1e1676861d48d6e8c6586625055b606a"}, - {file = "rpds_py-0.28.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:8aa23b6f0fc59b85b4c7d89ba2965af274346f738e8d9fc2455763602e62fd5f"}, - {file = "rpds_py-0.28.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:7b14b0c680286958817c22d76fcbca4800ddacef6f678f3a7c79a1fe7067fe37"}, - {file = "rpds_py-0.28.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:bcf1d210dfee61a6c86551d67ee1031899c0fdbae88b2d44a569995d43797712"}, - {file = "rpds_py-0.28.0-cp313-cp313t-win32.whl", hash = "sha256:3aa4dc0fdab4a7029ac63959a3ccf4ed605fee048ba67ce89ca3168da34a1342"}, - {file = "rpds_py-0.28.0-cp313-cp313t-win_amd64.whl", hash = "sha256:7b7d9d83c942855e4fdcfa75d4f96f6b9e272d42fffcb72cd4bb2577db2e2907"}, - {file = "rpds_py-0.28.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:dcdcb890b3ada98a03f9f2bb108489cdc7580176cb73b4f2d789e9a1dac1d472"}, - {file = "rpds_py-0.28.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f274f56a926ba2dc02976ca5b11c32855cbd5925534e57cfe1fda64e04d1add2"}, - {file = "rpds_py-0.28.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fe0438ac4a29a520ea94c8c7f1754cdd8feb1bc490dfda1bfd990072363d527"}, - {file = "rpds_py-0.28.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8a358a32dd3ae50e933347889b6af9a1bdf207ba5d1a3f34e1a38cd3540e6733"}, - {file = "rpds_py-0.28.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e80848a71c78aa328fefaba9c244d588a342c8e03bda518447b624ea64d1ff56"}, - {file = "rpds_py-0.28.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f586db2e209d54fe177e58e0bc4946bea5fb0102f150b1b2f13de03e1f0976f8"}, - {file = "rpds_py-0.28.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ae8ee156d6b586e4292491e885d41483136ab994e719a13458055bec14cf370"}, - {file = "rpds_py-0.28.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:a805e9b3973f7e27f7cab63a6b4f61d90f2e5557cff73b6e97cd5b8540276d3d"}, - {file = "rpds_py-0.28.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5d3fd16b6dc89c73a4da0b4ac8b12a7ecc75b2864b95c9e5afed8003cb50a728"}, - {file = "rpds_py-0.28.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6796079e5d24fdaba6d49bda28e2c47347e89834678f2bc2c1b4fc1489c0fb01"}, - {file = "rpds_py-0.28.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:76500820c2af232435cbe215e3324c75b950a027134e044423f59f5b9a1ba515"}, - {file = "rpds_py-0.28.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:bbdc5640900a7dbf9dd707fe6388972f5bbd883633eb68b76591044cfe346f7e"}, - {file = "rpds_py-0.28.0-cp314-cp314-win32.whl", hash = "sha256:adc8aa88486857d2b35d75f0640b949759f79dc105f50aa2c27816b2e0dd749f"}, - {file = "rpds_py-0.28.0-cp314-cp314-win_amd64.whl", hash = "sha256:66e6fa8e075b58946e76a78e69e1a124a21d9a48a5b4766d15ba5b06869d1fa1"}, - {file = "rpds_py-0.28.0-cp314-cp314-win_arm64.whl", hash = "sha256:a6fe887c2c5c59413353b7c0caff25d0e566623501ccfff88957fa438a69377d"}, - {file = "rpds_py-0.28.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:7a69df082db13c7070f7b8b1f155fa9e687f1d6aefb7b0e3f7231653b79a067b"}, - {file = "rpds_py-0.28.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b1cde22f2c30ebb049a9e74c5374994157b9b70a16147d332f89c99c5960737a"}, - {file = "rpds_py-0.28.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5338742f6ba7a51012ea470bd4dc600a8c713c0c72adaa0977a1b1f4327d6592"}, - {file = "rpds_py-0.28.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e1460ebde1bcf6d496d80b191d854adedcc619f84ff17dc1c6d550f58c9efbba"}, - {file = "rpds_py-0.28.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e3eb248f2feba84c692579257a043a7699e28a77d86c77b032c1d9fbb3f0219c"}, - {file = "rpds_py-0.28.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3bbba5def70b16cd1c1d7255666aad3b290fbf8d0fe7f9f91abafb73611a91"}, - {file = "rpds_py-0.28.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3114f4db69ac5a1f32e7e4d1cbbe7c8f9cf8217f78e6e002cedf2d54c2a548ed"}, - {file = "rpds_py-0.28.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:4b0cb8a906b1a0196b863d460c0222fb8ad0f34041568da5620f9799b83ccf0b"}, - {file = "rpds_py-0.28.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cf681ac76a60b667106141e11a92a3330890257e6f559ca995fbb5265160b56e"}, - {file = "rpds_py-0.28.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1e8ee6413cfc677ce8898d9cde18cc3a60fc2ba756b0dec5b71eb6eb21c49fa1"}, - {file = "rpds_py-0.28.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:b3072b16904d0b5572a15eb9d31c1954e0d3227a585fc1351aa9878729099d6c"}, - {file = "rpds_py-0.28.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b670c30fd87a6aec281c3c9896d3bae4b205fd75d79d06dc87c2503717e46092"}, - {file = "rpds_py-0.28.0-cp314-cp314t-win32.whl", hash = "sha256:8014045a15b4d2b3476f0a287fcc93d4f823472d7d1308d47884ecac9e612be3"}, - {file = "rpds_py-0.28.0-cp314-cp314t-win_amd64.whl", hash = "sha256:7a4e59c90d9c27c561eb3160323634a9ff50b04e4f7820600a2beb0ac90db578"}, - {file = "rpds_py-0.28.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f5e7101145427087e493b9c9b959da68d357c28c562792300dd21a095118ed16"}, - {file = "rpds_py-0.28.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:31eb671150b9c62409a888850aaa8e6533635704fe2b78335f9aaf7ff81eec4d"}, - {file = "rpds_py-0.28.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48b55c1f64482f7d8bd39942f376bfdf2f6aec637ee8c805b5041e14eeb771db"}, - {file = "rpds_py-0.28.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:24743a7b372e9a76171f6b69c01aedf927e8ac3e16c474d9fe20d552a8cb45c7"}, - {file = "rpds_py-0.28.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:389c29045ee8bbb1627ea190b4976a310a295559eaf9f1464a1a6f2bf84dde78"}, - {file = "rpds_py-0.28.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:23690b5827e643150cf7b49569679ec13fe9a610a15949ed48b85eb7f98f34ec"}, - {file = "rpds_py-0.28.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f0c9266c26580e7243ad0d72fc3e01d6b33866cfab5084a6da7576bcf1c4f72"}, - {file = "rpds_py-0.28.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:4c6c4db5d73d179746951486df97fd25e92396be07fc29ee8ff9a8f5afbdfb27"}, - {file = "rpds_py-0.28.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a3b695a8fa799dd2cfdb4804b37096c5f6dba1ac7f48a7fbf6d0485bcd060316"}, - {file = "rpds_py-0.28.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:6aa1bfce3f83baf00d9c5fcdbba93a3ab79958b4c7d7d1f55e7fe68c20e63912"}, - {file = "rpds_py-0.28.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:7b0f9dceb221792b3ee6acb5438eb1f02b0cb2c247796a72b016dcc92c6de829"}, - {file = "rpds_py-0.28.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:5d0145edba8abd3db0ab22b5300c99dc152f5c9021fab861be0f0544dc3cbc5f"}, - {file = "rpds_py-0.28.0.tar.gz", hash = "sha256:abd4df20485a0983e2ca334a216249b6186d6e3c1627e106651943dbdb791aea"}, + {file = "rpds_py-0.29.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:4ae4b88c6617e1b9e5038ab3fccd7bac0842fdda2b703117b2aa99bc85379113"}, + {file = "rpds_py-0.29.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7d9128ec9d8cecda6f044001fde4fb71ea7c24325336612ef8179091eb9596b9"}, + {file = "rpds_py-0.29.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d37812c3da8e06f2bb35b3cf10e4a7b68e776a706c13058997238762b4e07f4f"}, + {file = "rpds_py-0.29.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:66786c3fb1d8de416a7fa8e1cb1ec6ba0a745b2b0eee42f9b7daa26f1a495545"}, + {file = "rpds_py-0.29.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b58f5c77f1af888b5fd1876c9a0d9858f6f88a39c9dd7c073a88e57e577da66d"}, + {file = "rpds_py-0.29.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:799156ef1f3529ed82c36eb012b5d7a4cf4b6ef556dd7cc192148991d07206ae"}, + {file = "rpds_py-0.29.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:453783477aa4f2d9104c4b59b08c871431647cb7af51b549bbf2d9eb9c827756"}, + {file = "rpds_py-0.29.0-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:24a7231493e3c4a4b30138b50cca089a598e52c34cf60b2f35cebf62f274fdea"}, + {file = "rpds_py-0.29.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7033c1010b1f57bb44d8067e8c25aa6fa2e944dbf46ccc8c92b25043839c3fd2"}, + {file = "rpds_py-0.29.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0248b19405422573621172ab8e3a1f29141362d13d9f72bafa2e28ea0cdca5a2"}, + {file = "rpds_py-0.29.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:f9f436aee28d13b9ad2c764fc273e0457e37c2e61529a07b928346b219fcde3b"}, + {file = "rpds_py-0.29.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24a16cb7163933906c62c272de20ea3c228e4542c8c45c1d7dc2b9913e17369a"}, + {file = "rpds_py-0.29.0-cp310-cp310-win32.whl", hash = "sha256:1a409b0310a566bfd1be82119891fefbdce615ccc8aa558aff7835c27988cbef"}, + {file = "rpds_py-0.29.0-cp310-cp310-win_amd64.whl", hash = "sha256:c5523b0009e7c3c1263471b69d8da1c7d41b3ecb4cb62ef72be206b92040a950"}, + {file = "rpds_py-0.29.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:9b9c764a11fd637e0322a488560533112837f5334ffeb48b1be20f6d98a7b437"}, + {file = "rpds_py-0.29.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3fd2164d73812026ce970d44c3ebd51e019d2a26a4425a5dcbdfa93a34abc383"}, + {file = "rpds_py-0.29.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a097b7f7f7274164566ae90a221fd725363c0e9d243e2e9ed43d195ccc5495c"}, + {file = "rpds_py-0.29.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7cdc0490374e31cedefefaa1520d5fe38e82fde8748cbc926e7284574c714d6b"}, + {file = "rpds_py-0.29.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89ca2e673ddd5bde9b386da9a0aac0cab0e76f40c8f0aaf0d6311b6bbf2aa311"}, + {file = "rpds_py-0.29.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a5d9da3ff5af1ca1249b1adb8ef0573b94c76e6ae880ba1852f033bf429d4588"}, + {file = "rpds_py-0.29.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8238d1d310283e87376c12f658b61e1ee23a14c0e54c7c0ce953efdbdc72deed"}, + {file = "rpds_py-0.29.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:2d6fb2ad1c36f91c4646989811e84b1ea5e0c3cf9690b826b6e32b7965853a63"}, + {file = "rpds_py-0.29.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:534dc9df211387547267ccdb42253aa30527482acb38dd9b21c5c115d66a96d2"}, + {file = "rpds_py-0.29.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d456e64724a075441e4ed648d7f154dc62e9aabff29bcdf723d0c00e9e1d352f"}, + {file = "rpds_py-0.29.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:a738f2da2f565989401bd6fd0b15990a4d1523c6d7fe83f300b7e7d17212feca"}, + {file = "rpds_py-0.29.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a110e14508fd26fd2e472bb541f37c209409876ba601cf57e739e87d8a53cf95"}, + {file = "rpds_py-0.29.0-cp311-cp311-win32.whl", hash = "sha256:923248a56dd8d158389a28934f6f69ebf89f218ef96a6b216a9be6861804d3f4"}, + {file = "rpds_py-0.29.0-cp311-cp311-win_amd64.whl", hash = "sha256:539eb77eb043afcc45314d1be09ea6d6cafb3addc73e0547c171c6d636957f60"}, + {file = "rpds_py-0.29.0-cp311-cp311-win_arm64.whl", hash = "sha256:bdb67151ea81fcf02d8f494703fb728d4d34d24556cbff5f417d74f6f5792e7c"}, + {file = "rpds_py-0.29.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0891cfd8db43e085c0ab93ab7e9b0c8fee84780d436d3b266b113e51e79f954"}, + {file = "rpds_py-0.29.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3897924d3f9a0361472d884051f9a2460358f9a45b1d85a39a158d2f8f1ad71c"}, + {file = "rpds_py-0.29.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a21deb8e0d1571508c6491ce5ea5e25669b1dd4adf1c9d64b6314842f708b5d"}, + {file = "rpds_py-0.29.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9efe71687d6427737a0a2de9ca1c0a216510e6cd08925c44162be23ed7bed2d5"}, + {file = "rpds_py-0.29.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:40f65470919dc189c833e86b2c4bd21bd355f98436a2cef9e0a9a92aebc8e57e"}, + {file = "rpds_py-0.29.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:def48ff59f181130f1a2cb7c517d16328efac3ec03951cca40c1dc2049747e83"}, + {file = "rpds_py-0.29.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad7bd570be92695d89285a4b373006930715b78d96449f686af422debb4d3949"}, + {file = "rpds_py-0.29.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:5a572911cd053137bbff8e3a52d31c5d2dba51d3a67ad902629c70185f3f2181"}, + {file = "rpds_py-0.29.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d583d4403bcbf10cffc3ab5cee23d7643fcc960dff85973fd3c2d6c86e8dbb0c"}, + {file = "rpds_py-0.29.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:070befbb868f257d24c3bb350dbd6e2f645e83731f31264b19d7231dd5c396c7"}, + {file = "rpds_py-0.29.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:fc935f6b20b0c9f919a8ff024739174522abd331978f750a74bb68abd117bd19"}, + {file = "rpds_py-0.29.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8c5a8ecaa44ce2d8d9d20a68a2483a74c07f05d72e94a4dff88906c8807e77b0"}, + {file = "rpds_py-0.29.0-cp312-cp312-win32.whl", hash = "sha256:ba5e1aeaf8dd6d8f6caba1f5539cddda87d511331714b7b5fc908b6cfc3636b7"}, + {file = "rpds_py-0.29.0-cp312-cp312-win_amd64.whl", hash = "sha256:b5f6134faf54b3cb83375db0f113506f8b7770785be1f95a631e7e2892101977"}, + {file = "rpds_py-0.29.0-cp312-cp312-win_arm64.whl", hash = "sha256:b016eddf00dca7944721bf0cd85b6af7f6c4efaf83ee0b37c4133bd39757a8c7"}, + {file = "rpds_py-0.29.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1585648d0760b88292eecab5181f5651111a69d90eff35d6b78aa32998886a61"}, + {file = "rpds_py-0.29.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:521807963971a23996ddaf764c682b3e46459b3c58ccd79fefbe16718db43154"}, + {file = "rpds_py-0.29.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a8896986efaa243ab713c69e6491a4138410f0fe36f2f4c71e18bd5501e8014"}, + {file = "rpds_py-0.29.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1d24564a700ef41480a984c5ebed62b74e6ce5860429b98b1fede76049e953e6"}, + {file = "rpds_py-0.29.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e6596b93c010d386ae46c9fba9bfc9fc5965fa8228edeac51576299182c2e31c"}, + {file = "rpds_py-0.29.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5cc58aac218826d054c7da7f95821eba94125d88be673ff44267bb89d12a5866"}, + {file = "rpds_py-0.29.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de73e40ebc04dd5d9556f50180395322193a78ec247e637e741c1b954810f295"}, + {file = "rpds_py-0.29.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:295ce5ac7f0cf69a651ea75c8f76d02a31f98e5698e82a50a5f4d4982fbbae3b"}, + {file = "rpds_py-0.29.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1ea59b23ea931d494459c8338056fe7d93458c0bf3ecc061cd03916505369d55"}, + {file = "rpds_py-0.29.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f49d41559cebd608042fdcf54ba597a4a7555b49ad5c1c0c03e0af82692661cd"}, + {file = "rpds_py-0.29.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:05a2bd42768ea988294ca328206efbcc66e220d2d9b7836ee5712c07ad6340ea"}, + {file = "rpds_py-0.29.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:33ca7bdfedd83339ca55da3a5e1527ee5870d4b8369456b5777b197756f3ca22"}, + {file = "rpds_py-0.29.0-cp313-cp313-win32.whl", hash = "sha256:20c51ae86a0bb9accc9ad4e6cdeec58d5ebb7f1b09dd4466331fc65e1766aae7"}, + {file = "rpds_py-0.29.0-cp313-cp313-win_amd64.whl", hash = "sha256:6410e66f02803600edb0b1889541f4b5cc298a5ccda0ad789cc50ef23b54813e"}, + {file = "rpds_py-0.29.0-cp313-cp313-win_arm64.whl", hash = "sha256:56838e1cd9174dc23c5691ee29f1d1be9eab357f27efef6bded1328b23e1ced2"}, + {file = "rpds_py-0.29.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:37d94eadf764d16b9a04307f2ab1d7af6dc28774bbe0535c9323101e14877b4c"}, + {file = "rpds_py-0.29.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:d472cf73efe5726a067dce63eebe8215b14beabea7c12606fd9994267b3cfe2b"}, + {file = "rpds_py-0.29.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:72fdfd5ff8992e4636621826371e3ac5f3e3b8323e9d0e48378e9c13c3dac9d0"}, + {file = "rpds_py-0.29.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2549d833abdf8275c901313b9e8ff8fba57e50f6a495035a2a4e30621a2f7cc4"}, + {file = "rpds_py-0.29.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4448dad428f28a6a767c3e3b80cde3446a22a0efbddaa2360f4bb4dc836d0688"}, + {file = "rpds_py-0.29.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:115f48170fd4296a33938d8c11f697f5f26e0472e43d28f35624764173a60e4d"}, + {file = "rpds_py-0.29.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e5bb73ffc029820f4348e9b66b3027493ae00bca6629129cd433fd7a76308ee"}, + {file = "rpds_py-0.29.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:b1581fcde18fcdf42ea2403a16a6b646f8eb1e58d7f90a0ce693da441f76942e"}, + {file = "rpds_py-0.29.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16e9da2bda9eb17ea318b4c335ec9ac1818e88922cbe03a5743ea0da9ecf74fb"}, + {file = "rpds_py-0.29.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:28fd300326dd21198f311534bdb6d7e989dd09b3418b3a91d54a0f384c700967"}, + {file = "rpds_py-0.29.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:2aba991e041d031c7939e1358f583ae405a7bf04804ca806b97a5c0e0af1ea5e"}, + {file = "rpds_py-0.29.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7f437026dbbc3f08c99cc41a5b2570c6e1a1ddbe48ab19a9b814254128d4ea7a"}, + {file = "rpds_py-0.29.0-cp313-cp313t-win32.whl", hash = "sha256:6e97846e9800a5d0fe7be4d008f0c93d0feeb2700da7b1f7528dabafb31dfadb"}, + {file = "rpds_py-0.29.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f49196aec7c4b406495f60e6f947ad71f317a765f956d74bbd83996b9edc0352"}, + {file = "rpds_py-0.29.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:394d27e4453d3b4d82bb85665dc1fcf4b0badc30fc84282defed71643b50e1a1"}, + {file = "rpds_py-0.29.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:55d827b2ae95425d3be9bc9a5838b6c29d664924f98146557f7715e331d06df8"}, + {file = "rpds_py-0.29.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc31a07ed352e5462d3ee1b22e89285f4ce97d5266f6d1169da1142e78045626"}, + {file = "rpds_py-0.29.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c4695dd224212f6105db7ea62197144230b808d6b2bba52238906a2762f1d1e7"}, + {file = "rpds_py-0.29.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcae1770b401167f8b9e1e3f566562e6966ffa9ce63639916248a9e25fa8a244"}, + {file = "rpds_py-0.29.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:90f30d15f45048448b8da21c41703b31c61119c06c216a1bf8c245812a0f0c17"}, + {file = "rpds_py-0.29.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44a91e0ab77bdc0004b43261a4b8cd6d6b451e8d443754cfda830002b5745b32"}, + {file = "rpds_py-0.29.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:4aa195e5804d32c682e453b34474f411ca108e4291c6a0f824ebdc30a91c973c"}, + {file = "rpds_py-0.29.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7971bdb7bf4ee0f7e6f67fa4c7fbc6019d9850cc977d126904392d363f6f8318"}, + {file = "rpds_py-0.29.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8ae33ad9ce580c7a47452c3b3f7d8a9095ef6208e0a0c7e4e2384f9fc5bf8212"}, + {file = "rpds_py-0.29.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:c661132ab2fb4eeede2ef69670fd60da5235209874d001a98f1542f31f2a8a94"}, + {file = "rpds_py-0.29.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:bb78b3a0d31ac1bde132c67015a809948db751cb4e92cdb3f0b242e430b6ed0d"}, + {file = "rpds_py-0.29.0-cp314-cp314-win32.whl", hash = "sha256:f475f103488312e9bd4000bc890a95955a07b2d0b6e8884aef4be56132adbbf1"}, + {file = "rpds_py-0.29.0-cp314-cp314-win_amd64.whl", hash = "sha256:b9cf2359a4fca87cfb6801fae83a76aedf66ee1254a7a151f1341632acf67f1b"}, + {file = "rpds_py-0.29.0-cp314-cp314-win_arm64.whl", hash = "sha256:9ba8028597e824854f0f1733d8b964e914ae3003b22a10c2c664cb6927e0feb9"}, + {file = "rpds_py-0.29.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:e71136fd0612556b35c575dc2726ae04a1669e6a6c378f2240312cf5d1a2ab10"}, + {file = "rpds_py-0.29.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:76fe96632d53f3bf0ea31ede2f53bbe3540cc2736d4aec3b3801b0458499ef3a"}, + {file = "rpds_py-0.29.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9459a33f077130dbb2c7c3cea72ee9932271fb3126404ba2a2661e4fe9eb7b79"}, + {file = "rpds_py-0.29.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5c9546cfdd5d45e562cc0444b6dddc191e625c62e866bf567a2c69487c7ad28a"}, + {file = "rpds_py-0.29.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12597d11d97b8f7e376c88929a6e17acb980e234547c92992f9f7c058f1a7310"}, + {file = "rpds_py-0.29.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28de03cf48b8a9e6ec10318f2197b83946ed91e2891f651a109611be4106ac4b"}, + {file = "rpds_py-0.29.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd7951c964069039acc9d67a8ff1f0a7f34845ae180ca542b17dc1456b1f1808"}, + {file = "rpds_py-0.29.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:c07d107b7316088f1ac0177a7661ca0c6670d443f6fe72e836069025e6266761"}, + {file = "rpds_py-0.29.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1de2345af363d25696969befc0c1688a6cb5e8b1d32b515ef84fc245c6cddba3"}, + {file = "rpds_py-0.29.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:00e56b12d2199ca96068057e1ae7f9998ab6e99cda82431afafd32f3ec98cca9"}, + {file = "rpds_py-0.29.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3919a3bbecee589300ed25000b6944174e07cd20db70552159207b3f4bbb45b8"}, + {file = "rpds_py-0.29.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e7fa2ccc312bbd91e43aa5e0869e46bc03278a3dddb8d58833150a18b0f0283a"}, + {file = "rpds_py-0.29.0-cp314-cp314t-win32.whl", hash = "sha256:97c817863ffc397f1e6a6e9d2d89fe5408c0a9922dac0329672fb0f35c867ea5"}, + {file = "rpds_py-0.29.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2023473f444752f0f82a58dfcbee040d0a1b3d1b3c2ec40e884bd25db6d117d2"}, + {file = "rpds_py-0.29.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:acd82a9e39082dc5f4492d15a6b6c8599aa21db5c35aaf7d6889aea16502c07d"}, + {file = "rpds_py-0.29.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:715b67eac317bf1c7657508170a3e011a1ea6ccb1c9d5f296e20ba14196be6b3"}, + {file = "rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3b1b87a237cb2dba4db18bcfaaa44ba4cd5936b91121b62292ff21df577fc43"}, + {file = "rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1c3c3e8101bb06e337c88eb0c0ede3187131f19d97d43ea0e1c5407ea74c0cbf"}, + {file = "rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2b8e54d6e61f3ecd3abe032065ce83ea63417a24f437e4a3d73d2f85ce7b7cfe"}, + {file = "rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3fbd4e9aebf110473a420dea85a238b254cf8a15acb04b22a5a6b5ce8925b760"}, + {file = "rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80fdf53d36e6c72819993e35d1ebeeb8e8fc688d0c6c2b391b55e335b3afba5a"}, + {file = "rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:ea7173df5d86f625f8dde6d5929629ad811ed8decda3b60ae603903839ac9ac0"}, + {file = "rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:76054d540061eda273274f3d13a21a4abdde90e13eaefdc205db37c05230efce"}, + {file = "rpds_py-0.29.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:9f84c549746a5be3bc7415830747a3a0312573afc9f95785eb35228bb17742ec"}, + {file = "rpds_py-0.29.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:0ea962671af5cb9a260489e311fa22b2e97103e3f9f0caaea6f81390af96a9ed"}, + {file = "rpds_py-0.29.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:f7728653900035fb7b8d06e1e5900545d8088efc9d5d4545782da7df03ec803f"}, + {file = "rpds_py-0.29.0.tar.gz", hash = "sha256:fe55fe686908f50154d1dc599232016e50c243b438c3b7432f24e2895b0e5359"}, ] [[package]] From f3975ce247e8a0a9df6104ec8458deb5af566a72 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 16:30:41 +0000 Subject: [PATCH 005/139] Bump actions/setup-go from 6.0.0 to 6.1.0 (#19214) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/setup-go](https://github.com/actions/setup-go) from 6.0.0 to 6.1.0.
Release notes

Sourced from actions/setup-go's releases.

v6.1.0

What's Changed

Enhancements

Dependency updates

New Contributors

Full Changelog: https://github.com/actions/setup-go/compare/v6...v6.1.0

Commits
  • 4dc6199 Bump semver and @​types/semver (#652)
  • f3787be Add comprehensive breaking changes documentation for v6 (#674)
  • 3a0c2c8 Bump actions/publish-action from 0.3.0 to 0.4.0 (#641)
  • faf5242 Add support for .tool-versions file in setup-go, update workflow (#673)
  • 7bc60db Fall back to downloading from go.dev/dl instead of storage.googleapis.com/gol...
  • c0137ca Bump eslint-config-prettier from 10.0.1 to 10.1.8 and document breaking chang...
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-go&package-manager=github_actions&previous-version=6.0.0&new-version=6.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/latest_deps.yml | 2 +- .github/workflows/tests.yml | 2 +- .github/workflows/twisted_trunk.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 2076a1c1e12..276b6cd3830 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -209,7 +209,7 @@ jobs: - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 + - uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f320e890690..ce01be153c3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -705,7 +705,7 @@ jobs: - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 + - uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 11b7bfe1435..a252816e053 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -182,7 +182,7 @@ jobs: - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 + - uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod From 9fb2b1731bff61fb326d78e5f6ace268b48e70b8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 16:37:55 +0000 Subject: [PATCH 006/139] Bump actions/checkout from 5.0.0 to 6.0.0 (#19213) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/checkout](https://github.com/actions/checkout) from 5.0.0 to 6.0.0.
Release notes

Sourced from actions/checkout's releases.

v6.0.0

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v5.0.0...v6.0.0

v6-beta

What's Changed

Updated persist-credentials to store the credentials under $RUNNER_TEMP instead of directly in the local git config.

This requires a minimum Actions Runner version of v2.329.0 to access the persisted credentials for Docker container action scenarios.

v5.0.1

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v5...v5.0.1

Changelog

Sourced from actions/checkout's changelog.

Changelog

V6.0.0

V5.0.1

V5.0.0

V4.3.1

V4.3.0

v4.2.2

v4.2.1

v4.2.0

v4.1.7

v4.1.6

v4.1.5

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/checkout&package-manager=github_actions&previous-version=5.0.0&new-version=6.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker.yml | 2 +- .github/workflows/docs-pr.yaml | 4 +- .github/workflows/docs.yaml | 2 +- .github/workflows/fix_lint.yaml | 2 +- .github/workflows/latest_deps.yml | 10 ++--- .github/workflows/poetry_lockfile.yaml | 2 +- .github/workflows/push_complement_image.yml | 6 +-- .github/workflows/release-artifacts.yml | 8 ++-- .github/workflows/schema.yaml | 4 +- .github/workflows/tests.yml | 44 ++++++++++----------- .github/workflows/triage_labelled.yml | 2 +- .github/workflows/twisted_trunk.yml | 10 ++--- 12 files changed, 48 insertions(+), 48 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 52a0762efce..d0cdd3acafd 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -31,7 +31,7 @@ jobs: uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Extract version from pyproject.toml # Note: explicitly requesting bash will mean bash is invoked with `-eo pipefail`, see diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index 6a61dd5fb11..d59e0691711 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -13,7 +13,7 @@ jobs: name: GitHub Pages runs-on: ubuntu-latest steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: # Fetch all history so that the schema_versions script works. fetch-depth: 0 @@ -50,7 +50,7 @@ jobs: name: Check links in documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Setup mdbook uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0 diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index f260a4f8042..d02428db19f 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -50,7 +50,7 @@ jobs: needs: - pre steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: # Fetch all history so that the schema_versions script works. fetch-depth: 0 diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index c33481a51e1..a5469be56cc 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 276b6cd3830..eee8dc7e0b8 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -42,7 +42,7 @@ jobs: if: needs.check_repo.outputs.should_run_workflow == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: @@ -77,7 +77,7 @@ jobs: postgres-version: "14" steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -152,7 +152,7 @@ jobs: BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -202,7 +202,7 @@ jobs: steps: - name: Check out synapse codebase - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: path: synapse @@ -234,7 +234,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/poetry_lockfile.yaml b/.github/workflows/poetry_lockfile.yaml index 19468c2d926..62b796287c1 100644 --- a/.github/workflows/poetry_lockfile.yaml +++ b/.github/workflows/poetry_lockfile.yaml @@ -16,7 +16,7 @@ jobs: name: "Check locked dependencies have sdists" runs-on: ubuntu-latest steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: '3.x' diff --git a/.github/workflows/push_complement_image.yml b/.github/workflows/push_complement_image.yml index e08775e588e..c562275a380 100644 --- a/.github/workflows/push_complement_image.yml +++ b/.github/workflows/push_complement_image.yml @@ -33,17 +33,17 @@ jobs: packages: write steps: - name: Checkout specific branch (debug build) - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 if: github.event_name == 'workflow_dispatch' with: ref: ${{ inputs.branch }} - name: Checkout clean copy of develop (scheduled build) - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 if: github.event_name == 'schedule' with: ref: develop - name: Checkout clean copy of master (on-push) - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 if: github.event_name == 'push' with: ref: master diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index c88546c3bf7..e63d65fdf34 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -27,7 +27,7 @@ jobs: name: "Calculate list of debian distros" runs-on: ubuntu-latest steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: "3.x" @@ -55,7 +55,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: path: src @@ -132,7 +132,7 @@ jobs: os: "ubuntu-24.04-arm" steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: @@ -170,7 +170,7 @@ jobs: if: ${{ !startsWith(github.ref, 'refs/pull/') }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: "3.10" diff --git a/.github/workflows/schema.yaml b/.github/workflows/schema.yaml index 6c416e762dd..52b5cd4c53f 100644 --- a/.github/workflows/schema.yaml +++ b/.github/workflows/schema.yaml @@ -14,7 +14,7 @@ jobs: name: Ensure Synapse config schema is valid runs-on: ubuntu-latest steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: "3.x" @@ -40,7 +40,7 @@ jobs: name: Ensure generated documentation is up-to-date runs-on: ubuntu-latest steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: "3.x" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ce01be153c3..771474f74e3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -86,7 +86,7 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: @@ -106,7 +106,7 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: "3.x" @@ -116,7 +116,7 @@ jobs: check-lockfile: runs-on: ubuntu-latest steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: "3.x" @@ -129,7 +129,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -151,7 +151,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -187,7 +187,7 @@ jobs: lint-crlf: runs-on: ubuntu-latest steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Check line endings run: scripts-dev/check_line_terminators.sh @@ -195,7 +195,7 @@ jobs: if: ${{ (github.base_ref == 'develop' || contains(github.base_ref, 'release-')) && github.actor != 'dependabot[bot]' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 @@ -213,7 +213,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -232,7 +232,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -250,7 +250,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -286,7 +286,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -306,7 +306,7 @@ jobs: needs: changes if: ${{ needs.changes.outputs.linting_readme == 'true' }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: "3.x" @@ -354,7 +354,7 @@ jobs: needs: linting-done runs-on: ubuntu-latest steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: "3.x" @@ -375,7 +375,7 @@ jobs: job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - run: sudo apt-get -qq install xmlsec1 - name: Set up PostgreSQL ${{ matrix.job.postgres-version }} if: ${{ matrix.job.postgres-version }} @@ -431,7 +431,7 @@ jobs: - changes runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -496,7 +496,7 @@ jobs: extras: ["all"] steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 # Install libs necessary for PyPy to build binary wheels for dependencies - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -546,7 +546,7 @@ jobs: job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Prepare test blacklist run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers @@ -593,7 +593,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - run: sudo apt-get -qq install xmlsec1 postgresql-client - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -637,7 +637,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Add PostgreSQL apt repository # We need a version of pg_dump that can handle the version of # PostgreSQL being tested against. The Ubuntu package repository lags @@ -692,7 +692,7 @@ jobs: steps: - name: Checkout synapse codebase - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: path: synapse @@ -728,7 +728,7 @@ jobs: - changes steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -748,7 +748,7 @@ jobs: - changes steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master diff --git a/.github/workflows/triage_labelled.yml b/.github/workflows/triage_labelled.yml index d291eea3a1d..34222b7d1bf 100644 --- a/.github/workflows/triage_labelled.yml +++ b/.github/workflows/triage_labelled.yml @@ -22,7 +22,7 @@ jobs: # This field is case-sensitive. TARGET_STATUS: Needs info steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: # Only clone the script file we care about, instead of the whole repo. sparse-checkout: .ci/scripts/triage_labelled_issue.sh diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index a252816e053..b07f98b1cb4 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -70,7 +70,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - run: sudo apt-get -qq install xmlsec1 - name: Install Rust @@ -117,7 +117,7 @@ jobs: - ${{ github.workspace }}:/src steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -175,7 +175,7 @@ jobs: steps: - name: Run actions/checkout@v4 for synapse - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: path: synapse @@ -217,7 +217,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 8b795836439dab16281e84a9404b22ff2438b7c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 18:04:07 +0000 Subject: [PATCH 007/139] Bump sentry-sdk from 2.44.0 to 2.46.0 (#19218) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [sentry-sdk](https://github.com/getsentry/sentry-python) from 2.44.0 to 2.46.0.
Release notes

Sourced from sentry-sdk's releases.

2.46.0

Various fixes & improvements

2.45.0

Various fixes & improvements

  • OTLPIntegration (#4877) by @​sl0thentr0py

    Enable the new OTLP integration with the code snippet below, and your OpenTelemetry instrumentation will be automatically sent to Sentry's OTLP ingestion endpoint.

      import sentry_sdk
  from sentry_sdk.integrations.otlp import OTLPIntegration

    sentry_sdk.init(
    
dsn="<your-dsn>",
    
# Add data like inputs and responses;
    
# see https://docs.sentry.io/platforms/python/data-management/data-collected/
for more info
    
send_default_pii=True,
    
integrations=[
    
OTLPIntegration(),
    
],
    
)

    Under the hood, this will setup:

    • A SpanExporter that will automatically set up the OTLP ingestion endpoint from your DSN
    • A Propagator that ensures Distributed Tracing works
    • Trace/Span linking for all other Sentry events such as Errors, Logs, Crons and Metrics

    If you were using the SentrySpanProcessor before, we recommend migrating over to OTLPIntegration since it's a much simpler setup.

  • feat(integrations): implement context management for invoke_agent spans (#5089) by @​constantinius

  • feat(loguru): Capture extra (#5096) by @​sentrivana

  • feat: Attach server.address to metrics (#5113) by @​alexander-alderman-webb

  • fix: Cast message and detail attributes before appending exception notes (#5114) by @​alexander-alderman-webb

  • fix(integrations): ensure that GEN_AI_AGENT_NAME is properly set for GEN_AI spans under an invoke_agent span (#5030) by @​constantinius

  • fix(logs): Update sentry.origin (#5112) by @​sentrivana

  • chore: Deprecate description truncation option for Redis spans (#5073) by @​alexander-alderman-webb

  • chore: Deprecate max_spans LangChain parameter (#5074) by @​alexander-alderman-webb

  • chore(toxgen): Check availability of pip and add detail to exceptions (#5076) by @​alexander-alderman-webb

... (truncated)

Changelog

Sourced from sentry-sdk's changelog.

2.46.0

Various fixes & improvements

2.45.0

Various fixes & improvements

  • OTLPIntegration (#4877) by @​sl0thentr0py

    Enable the new OTLP integration with the code snippet below, and your OpenTelemetry instrumentation will be automatically sent to Sentry's OTLP ingestion endpoint.

      import sentry_sdk
  from sentry_sdk.integrations.otlp import OTLPIntegration

    sentry_sdk.init(
    
dsn="<your-dsn>",
    
# Add data like inputs and responses;
    
# see https://docs.sentry.io/platforms/python/data-management/data-collected/
for more info
    
send_default_pii=True,
    
integrations=[
    
OTLPIntegration(),
    
],
    
)

    Under the hood, this will setup:

    • A SpanExporter that will automatically set up the OTLP ingestion endpoint from your DSN
    • A Propagator that ensures Distributed Tracing works
    • Trace/Span linking for all other Sentry events such as Errors, Logs, Crons and Metrics

    If you were using the SentrySpanProcessor before, we recommend migrating over to OTLPIntegration since it's a much simpler setup.

  • feat(integrations): implement context management for invoke_agent spans (#5089) by @​constantinius

  • feat(loguru): Capture extra (#5096) by @​sentrivana

  • feat: Attach server.address to metrics (#5113) by @​alexander-alderman-webb

  • fix: Cast message and detail attributes before appending exception notes (#5114) by @​alexander-alderman-webb

  • fix(integrations): ensure that GEN_AI_AGENT_NAME is properly set for GEN_AI spans under an invoke_agent span (#5030) by @​constantinius

  • fix(logs): Update sentry.origin (#5112) by @​sentrivana

  • chore: Deprecate description truncation option for Redis spans (#5073) by @​alexander-alderman-webb

... (truncated)

Commits
  • d3375bc Update CHANGELOG.md
  • 23abfe2 release: 2.46.0
  • ca19d63 feat: Preserve metadata on wrapped coroutines (#5105)
  • cf165e3 build(deps): bump actions/checkout from 5.0.0 to 6.0.0 (#5136)
  • b8d6a57 build(deps): bump actions/create-github-app-token from 2.1.4 to 2.2.0 (#5137)
  • c0c28b8 build(deps): bump supercharge/redis-github-action from 1.8.0 to 1.8.1 (#5138)
  • fb18c21 fix(pydantic-ai): Make imports defensive to avoid ModuleNotFoundError (#5135)
  • f945e38 Fix openai-agents import (#5132)
  • 8596f89 fix(integrations): enhance input handling for embeddings in LiteLLM integrati...
  • 0e6e808 test(openai-agents): Remove MagicMock from mocked ModelResponse (#5126)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=sentry-sdk&package-manager=pip&previous-version=2.44.0&new-version=2.46.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5a79b932a5d..f723322a558 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2543,15 +2543,15 @@ doc = ["Sphinx", "sphinx-rtd-theme"] [[package]] name = "sentry-sdk" -version = "2.44.0" +version = "2.46.0" description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] markers = "extra == \"all\" or extra == \"sentry\"" files = [ - {file = "sentry_sdk-2.44.0-py2.py3-none-any.whl", hash = "sha256:9e36a0372b881e8f92fdbff4564764ce6cec4b7f25424d0a3a8d609c9e4651a7"}, - {file = "sentry_sdk-2.44.0.tar.gz", hash = "sha256:5b1fe54dfafa332e900b07dd8f4dfe35753b64e78e7d9b1655a28fd3065e2493"}, + {file = "sentry_sdk-2.46.0-py2.py3-none-any.whl", hash = "sha256:4eeeb60198074dff8d066ea153fa6f241fef1668c10900ea53a4200abc8da9b1"}, + {file = "sentry_sdk-2.46.0.tar.gz", hash = "sha256:91821a23460725734b7741523021601593f35731808afc0bb2ba46c27b8acd91"}, ] [package.dependencies] @@ -2590,6 +2590,7 @@ openai = ["openai (>=1.0.0)", "tiktoken (>=0.3.0)"] openfeature = ["openfeature-sdk (>=0.7.1)"] opentelemetry = ["opentelemetry-distro (>=0.35b0)"] opentelemetry-experimental = ["opentelemetry-distro"] +opentelemetry-otlp = ["opentelemetry-distro[otlp] (>=0.35b0)"] pure-eval = ["asttokens", "executing", "pure_eval"] pydantic-ai = ["pydantic-ai (>=1.0.0)"] pymongo = ["pymongo (>=3.1)"] From db975ea10d58a89862d15e99170b0f76f409efd6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 25 Nov 2025 10:20:47 +0000 Subject: [PATCH 008/139] Expire sliding sync connections (#19211) We add some logic to expire sliding sync connections if they get old or if there is too much pending data to return. The values of the constants are picked fairly arbitrarily, these are currently: 1. More than 100 rooms with pending events if the connection hasn't been used in over an hour 2. The connection hasn't been used for over a week Reviewable commit-by-commit --------- Co-authored-by: Eric Eastwood --- changelog.d/19211.misc | 1 + synapse/handlers/sliding_sync/room_lists.py | 59 +++++++++- synapse/handlers/sliding_sync/store.py | 2 +- .../storage/databases/main/sliding_sync.py | 68 ++++++++++- synapse/storage/databases/main/stream.py | 14 ++- .../main/delta/93/03_sss_pos_last_used.sql | 27 +++++ synapse/types/handlers/sliding_sync.py | 8 ++ synapse/util/constants.py | 1 + .../sliding_sync/test_connection_tracking.py | 108 ++++++++++++++++++ .../client/sliding_sync/test_sliding_sync.py | 28 ++++- 10 files changed, 304 insertions(+), 12 deletions(-) create mode 100644 changelog.d/19211.misc create mode 100644 synapse/storage/schema/main/delta/93/03_sss_pos_last_used.sql diff --git a/changelog.d/19211.misc b/changelog.d/19211.misc new file mode 100644 index 00000000000..d8a4a44662e --- /dev/null +++ b/changelog.d/19211.misc @@ -0,0 +1 @@ +Expire sliding sync connections that are too old or have too much pending data. diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index 3d119022367..fa4ff22b645 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -34,10 +34,12 @@ EventTypes, Membership, ) +from synapse.api.errors import SlidingSyncUnknownPosition from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.events import StrippedStateEvent from synapse.events.utils import parse_stripped_state_event from synapse.logging.opentracing import start_active_span, trace +from synapse.storage.databases.main.sliding_sync import UPDATE_INTERVAL_LAST_USED_TS_MS from synapse.storage.databases.main.state import ( ROOM_UNKNOWN_SENTINEL, Sentinel as StateSentinel, @@ -68,6 +70,7 @@ ) from synapse.types.state import StateFilter from synapse.util import MutableOverlayMapping +from synapse.util.constants import MILLISECONDS_PER_SECOND, ONE_HOUR_SECONDS from synapse.util.sentinel import Sentinel if TYPE_CHECKING: @@ -77,6 +80,27 @@ logger = logging.getLogger(__name__) +# Minimum time in milliseconds since the last sync before we consider expiring +# the connection due to too many rooms to send. This stops from getting into +# tight loops with clients that request lots of data at once. +# +# c.f. `NUM_ROOMS_THRESHOLD`. These values are somewhat arbitrary picked. +MINIMUM_NOT_USED_AGE_EXPIRY_MS = ONE_HOUR_SECONDS * MILLISECONDS_PER_SECOND + +# How many rooms with updates we allow before we consider the connection expired +# due to too many rooms to send. +# +# c.f. `MINIMUM_NOT_USED_AGE_EXPIRY_MS`. These values are somewhat arbitrary +# picked. +NUM_ROOMS_THRESHOLD = 100 + +# Sanity check that our minimum age is sensible compared to the update interval, +# i.e. if `MINIMUM_NOT_USED_AGE_EXPIRY_MS` is too small then we might expire the +# connection even if it is actively being used (and we're just not updating the +# DB frequently enough). We arbitrarily double the update interval to give some +# wiggle room. +assert 2 * UPDATE_INTERVAL_LAST_USED_TS_MS < MINIMUM_NOT_USED_AGE_EXPIRY_MS + # Helper definition for the types that we might return. We do this to avoid # copying data between types (which can be expensive for many rooms). RoomsForUserType = RoomsForUserStateReset | RoomsForUser | RoomsForUserSlidingSync @@ -176,6 +200,7 @@ def __init__(self, hs: "HomeServer"): self.storage_controllers = hs.get_storage_controllers() self.rooms_to_exclude_globally = hs.config.server.rooms_to_exclude_from_sync self.is_mine_id = hs.is_mine_id + self._clock = hs.get_clock() async def compute_interested_rooms( self, @@ -857,11 +882,41 @@ async def _filter_relevant_rooms_to_send( # We only need to check for new events since any state changes # will also come down as new events. - rooms_that_have_updates = ( - self.store.get_rooms_that_might_have_updates( + + rooms_that_have_updates = await ( + self.store.get_rooms_that_have_updates_since_sliding_sync_table( relevant_room_map.keys(), from_token.room_key ) ) + + # Check if we have lots of updates to send, if so then its + # better for us to tell the client to do a full resync + # instead (to try and avoid long SSS response times when + # there is new data). + # + # Due to the construction of the SSS API, the client is in + # charge of setting the range of rooms to request updates + # for. Generally, it will start with a small range and then + # expand (and occasionally it may contract the range again + # if its been offline for a while). If we know there are a + # lot of updates, it's better to reset the connection and + # wait for the client to start again (with a much smaller + # range) than to try and send down a large number of updates + # (which can take a long time). + # + # We only do this if the last sync was over + # `MINIMUM_NOT_USED_AGE_EXPIRY_MS` to ensure we don't get + # into tight loops with clients that keep requesting large + # sliding sync windows. + if len(rooms_that_have_updates) > NUM_ROOMS_THRESHOLD: + last_sync_ts = previous_connection_state.last_used_ts + if ( + last_sync_ts is not None + and (self._clock.time_msec() - last_sync_ts) + > MINIMUM_NOT_USED_AGE_EXPIRY_MS + ): + raise SlidingSyncUnknownPosition() + rooms_should_send.update(rooms_that_have_updates) relevant_rooms_to_send_map = { room_id: room_sync_config diff --git a/synapse/handlers/sliding_sync/store.py b/synapse/handlers/sliding_sync/store.py index 7bcd5f27eae..d01fab271f6 100644 --- a/synapse/handlers/sliding_sync/store.py +++ b/synapse/handlers/sliding_sync/store.py @@ -75,7 +75,7 @@ async def get_and_clear_connection_positions( """ # If this is our first request, there is no previous connection state to fetch out of the database if from_token is None or from_token.connection_position == 0: - return PerConnectionState() + return PerConnectionState(last_used_ts=None) conn_id = sync_config.conn_id or "" diff --git a/synapse/storage/databases/main/sliding_sync.py b/synapse/storage/databases/main/sliding_sync.py index 2b67e75ac47..8cd3de8f40c 100644 --- a/synapse/storage/databases/main/sliding_sync.py +++ b/synapse/storage/databases/main/sliding_sync.py @@ -20,6 +20,7 @@ from synapse.api.errors import SlidingSyncUnknownPosition from synapse.logging.opentracing import log_kv +from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.storage._base import SQLBaseStore, db_to_json from synapse.storage.database import ( DatabasePool, @@ -36,6 +37,12 @@ RoomSyncConfig, ) from synapse.util.caches.descriptors import cached +from synapse.util.constants import ( + MILLISECONDS_PER_SECOND, + ONE_DAY_SECONDS, + ONE_HOUR_SECONDS, + ONE_MINUTE_SECONDS, +) from synapse.util.json import json_encoder if TYPE_CHECKING: @@ -45,6 +52,21 @@ logger = logging.getLogger(__name__) +# How often to update the `last_used_ts` column on +# `sliding_sync_connection_positions` when the client uses a connection +# position. We don't want to update it on every use to avoid excessive +# writes, but we want it to be reasonably up-to-date to help with +# cleaning up old connection positions. +UPDATE_INTERVAL_LAST_USED_TS_MS = 5 * ONE_MINUTE_SECONDS * MILLISECONDS_PER_SECOND + +# Time in milliseconds the connection hasn't been used before we consider it +# expired and delete it. +CONNECTION_EXPIRY_MS = 7 * ONE_DAY_SECONDS * MILLISECONDS_PER_SECOND + +# How often we run the background process to delete old sliding sync connections. +CONNECTION_EXPIRY_FREQUENCY_MS = ONE_HOUR_SECONDS * MILLISECONDS_PER_SECOND + + class SlidingSyncStore(SQLBaseStore): def __init__( self, @@ -76,6 +98,12 @@ def __init__( replaces_index="sliding_sync_membership_snapshots_user_id", ) + if self.hs.config.worker.run_background_tasks: + self.clock.looping_call( + self.delete_old_sliding_sync_connections, + CONNECTION_EXPIRY_FREQUENCY_MS, + ) + async def get_latest_bump_stamp_for_room( self, room_id: str, @@ -202,6 +230,7 @@ def persist_per_connection_state_txn( "effective_device_id": device_id, "conn_id": conn_id, "created_ts": self.clock.time_msec(), + "last_used_ts": self.clock.time_msec(), }, returning=("connection_key",), ) @@ -384,7 +413,7 @@ def _get_and_clear_connection_positions_txn( # The `previous_connection_position` is a user-supplied value, so we # need to make sure that the one they supplied is actually theirs. sql = """ - SELECT connection_key + SELECT connection_key, last_used_ts FROM sliding_sync_connection_positions INNER JOIN sliding_sync_connections USING (connection_key) WHERE @@ -396,7 +425,20 @@ def _get_and_clear_connection_positions_txn( if row is None: raise SlidingSyncUnknownPosition() - (connection_key,) = row + (connection_key, last_used_ts) = row + + # Update the `last_used_ts` if it's due to be updated. We don't update + # every time to avoid excessive writes. + now = self.clock.time_msec() + if last_used_ts is None or now - last_used_ts > UPDATE_INTERVAL_LAST_USED_TS_MS: + self.db_pool.simple_update_txn( + txn, + table="sliding_sync_connections", + keyvalues={ + "connection_key": connection_key, + }, + updatevalues={"last_used_ts": now}, + ) # Now that we have seen the client has received and used the connection # position, we can delete all the other connection positions. @@ -480,12 +522,30 @@ def _get_and_clear_connection_positions_txn( logger.warning("Unrecognized sliding sync stream in DB %r", stream) return PerConnectionStateDB( + last_used_ts=last_used_ts, rooms=RoomStatusMap(rooms), receipts=RoomStatusMap(receipts), account_data=RoomStatusMap(account_data), room_configs=room_configs, ) + @wrap_as_background_process("delete_old_sliding_sync_connections") + async def delete_old_sliding_sync_connections(self) -> None: + """Delete sliding sync connections that have not been used for a long time.""" + cutoff_ts = self.clock.time_msec() - CONNECTION_EXPIRY_MS + + def delete_old_sliding_sync_connections_txn(txn: LoggingTransaction) -> None: + sql = """ + DELETE FROM sliding_sync_connections + WHERE last_used_ts IS NOT NULL AND last_used_ts < ? + """ + txn.execute(sql, (cutoff_ts,)) + + await self.db_pool.runInteraction( + "delete_old_sliding_sync_connections", + delete_old_sliding_sync_connections_txn, + ) + @attr.s(auto_attribs=True, frozen=True) class PerConnectionStateDB: @@ -498,6 +558,8 @@ class PerConnectionStateDB: When persisting this *only* contains updates to the state. """ + last_used_ts: int | None + rooms: "RoomStatusMap[str]" receipts: "RoomStatusMap[str]" account_data: "RoomStatusMap[str]" @@ -553,6 +615,7 @@ async def from_state( ) return PerConnectionStateDB( + last_used_ts=per_connection_state.last_used_ts, rooms=RoomStatusMap(rooms), receipts=RoomStatusMap(receipts), account_data=RoomStatusMap(account_data), @@ -596,6 +659,7 @@ async def to_state(self, store: "DataStore") -> "PerConnectionState": } return PerConnectionState( + last_used_ts=self.last_used_ts, rooms=RoomStatusMap(rooms), receipts=RoomStatusMap(receipts), account_data=RoomStatusMap(account_data), diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index 8644ff412ec..8fa1e2e5a97 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -740,7 +740,14 @@ async def get_rooms_that_have_updates_since_sliding_sync_table( from_key: RoomStreamToken, ) -> StrCollection: """Return the rooms that probably have had updates since the given - token (changes that are > `from_key`).""" + token (changes that are > `from_key`). + + May return false positives, but must not return false negatives. + + If `have_finished_sliding_sync_background_jobs` is False, then we return + all the room IDs, as we can't be sure that the sliding sync table is + fully populated. + """ # If the stream change cache is valid for the stream token, we can just # use the result of that. if from_key.stream >= self._events_stream_cache.get_earliest_known_position(): @@ -748,6 +755,11 @@ async def get_rooms_that_have_updates_since_sliding_sync_table( room_ids, from_key.stream ) + if not self.have_finished_sliding_sync_background_jobs(): + # If the table hasn't been populated yet, we have to assume all rooms + # have updates. + return room_ids + def get_rooms_that_have_updates_since_sliding_sync_table_txn( txn: LoggingTransaction, ) -> StrCollection: diff --git a/synapse/storage/schema/main/delta/93/03_sss_pos_last_used.sql b/synapse/storage/schema/main/delta/93/03_sss_pos_last_used.sql new file mode 100644 index 00000000000..747ba7a144b --- /dev/null +++ b/synapse/storage/schema/main/delta/93/03_sss_pos_last_used.sql @@ -0,0 +1,27 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 Element Creations, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Add a timestamp for when the sliding sync connection position was last used, +-- only updated with a small granularity. +-- +-- This should be NOT NULL, but we need to consider existing rows. In future we +-- may want to either backfill this or delete all rows with a NULL value (and +-- then make it NOT NULL). +ALTER TABLE sliding_sync_connections ADD COLUMN last_used_ts BIGINT; + +-- Note: We don't add an index on this column to allow HOT updates on PostgreSQL +-- to reduce the cost of the updates to the column. c.f. +-- https://www.postgresql.org/docs/current/storage-hot.html +-- +-- We do query this column directly to find expired connections, but we expect +-- that to be an infrequent operation and a sequential scan should be fine. diff --git a/synapse/types/handlers/sliding_sync.py b/synapse/types/handlers/sliding_sync.py index 494e3570d05..03b3bcb3caf 100644 --- a/synapse/types/handlers/sliding_sync.py +++ b/synapse/types/handlers/sliding_sync.py @@ -850,12 +850,16 @@ class PerConnectionState: since the last time you made a sync request. Attributes: + last_used_ts: The time this connection was last used, in milliseconds. + This is only accurate to `UPDATE_CONNECTION_STATE_EVERY_MS`. rooms: The status of each room for the events stream. receipts: The status of each room for the receipts stream. room_configs: Map from room_id to the `RoomSyncConfig` of all rooms that we have previously sent down. """ + last_used_ts: int | None = None + rooms: RoomStatusMap[RoomStreamToken] = attr.Factory(RoomStatusMap) receipts: RoomStatusMap[MultiWriterStreamToken] = attr.Factory(RoomStatusMap) account_data: RoomStatusMap[int] = attr.Factory(RoomStatusMap) @@ -867,6 +871,7 @@ def get_mutable(self) -> "MutablePerConnectionState": room_configs = cast(MutableMapping[str, RoomSyncConfig], self.room_configs) return MutablePerConnectionState( + last_used_ts=self.last_used_ts, rooms=self.rooms.get_mutable(), receipts=self.receipts.get_mutable(), account_data=self.account_data.get_mutable(), @@ -875,6 +880,7 @@ def get_mutable(self) -> "MutablePerConnectionState": def copy(self) -> "PerConnectionState": return PerConnectionState( + last_used_ts=self.last_used_ts, rooms=self.rooms.copy(), receipts=self.receipts.copy(), account_data=self.account_data.copy(), @@ -889,6 +895,8 @@ def __len__(self) -> int: class MutablePerConnectionState(PerConnectionState): """A mutable version of `PerConnectionState`""" + last_used_ts: int | None + rooms: MutableRoomStatusMap[RoomStreamToken] receipts: MutableRoomStatusMap[MultiWriterStreamToken] account_data: MutableRoomStatusMap[int] diff --git a/synapse/util/constants.py b/synapse/util/constants.py index 7a3d073df55..f4491b58856 100644 --- a/synapse/util/constants.py +++ b/synapse/util/constants.py @@ -18,5 +18,6 @@ # readability and catching bugs. ONE_MINUTE_SECONDS = 60 ONE_HOUR_SECONDS = 60 * ONE_MINUTE_SECONDS +ONE_DAY_SECONDS = 24 * ONE_HOUR_SECONDS MILLISECONDS_PER_SECOND = 1000 diff --git a/tests/rest/client/sliding_sync/test_connection_tracking.py b/tests/rest/client/sliding_sync/test_connection_tracking.py index 16d13fcc860..cdf63317e39 100644 --- a/tests/rest/client/sliding_sync/test_connection_tracking.py +++ b/tests/rest/client/sliding_sync/test_connection_tracking.py @@ -12,6 +12,7 @@ # . # import logging +from unittest.mock import patch from parameterized import parameterized, parameterized_class @@ -19,8 +20,11 @@ import synapse.rest.admin from synapse.api.constants import EventTypes +from synapse.api.errors import Codes +from synapse.handlers.sliding_sync import room_lists from synapse.rest.client import login, room, sync from synapse.server import HomeServer +from synapse.storage.databases.main.sliding_sync import CONNECTION_EXPIRY_MS from synapse.util.clock import Clock from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase @@ -395,3 +399,107 @@ def test_rooms_timeline_incremental_sync_NEVER(self) -> None: ) self.assertEqual(response_body["rooms"][room_id1]["limited"], True) self.assertEqual(response_body["rooms"][room_id1]["initial"], True) + + @patch("synapse.handlers.sliding_sync.room_lists.NUM_ROOMS_THRESHOLD", new=5) + def test_sliding_sync_connection_expires_with_too_much_data(self) -> None: + """ + Test that if we have too much data to send down for incremental sync, + we expire the connection and ask the client to do a full resync. + + Connections are only expired if they have not been used for a minimum + amount of time (MINIMUM_NOT_USED_AGE_EXPIRY_MS) to avoid expiring + connections that are actively being used. + """ + + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create enough rooms that we can later trigger the too much data case + room_ids = [] + for _ in range(room_lists.NUM_ROOMS_THRESHOLD + 2): + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id, user1_id, tok=user1_tok) + room_ids.append(room_id) + + # Make sure we don't hit ratelimits + self.reactor.advance(60 * 1000) + + # Make the Sliding Sync request + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1000]], + "required_state": [], + "timeline_limit": 1, + } + } + } + + response_body, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Check we got all the rooms down + for room_id in room_ids: + self.assertIn(room_id, response_body["rooms"]) + + # Send a lot of events to cause the connection to expire + for room_id in room_ids: + self.helper.send(room_id, "msg", tok=user2_tok) + + # If we don't advance the clock then we won't expire the connection. + response_body, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Send some more events. + for room_id in room_ids: + self.helper.send(room_id, "msg", tok=user2_tok) + + # Advance the clock to ensure that the last_used_ts is old enough + self.reactor.advance(2 * room_lists.MINIMUM_NOT_USED_AGE_EXPIRY_MS / 1000) + + # This sync should now raise SlidingSyncUnknownPosition + channel = self.make_sync_request(sync_body, since=from_token, tok=user1_tok) + self.assertEqual(channel.code, 400) + self.assertEqual(channel.json_body["errcode"], Codes.UNKNOWN_POS) + + def test_sliding_sync_connection_expires_after_time(self) -> None: + """ + Test that if we don't use a sliding sync connection for a long time, + we expire the connection and ask the client to do a full resync. + """ + + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Make the Sliding Sync request + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1000]], + "required_state": [], + "timeline_limit": 1, + } + } + } + + _, from_token = self.do_sync(sync_body, tok=user1_tok) + + # We can keep syncing so long as the interval between requests is less + # than CONNECTION_EXPIRY_MS + for _ in range(5): + self.reactor.advance(0.5 * CONNECTION_EXPIRY_MS / 1000) + + _, from_token = self.do_sync(sync_body, tok=user1_tok) + + # ... but if we wait too long, the connection expires + self.reactor.advance(1 + CONNECTION_EXPIRY_MS / 1000) + + # This sync should now raise SlidingSyncUnknownPosition + channel = self.make_sync_request(sync_body, since=from_token, tok=user1_tok) + self.assertEqual(channel.code, 400) + self.assertEqual(channel.json_body["errcode"], Codes.UNKNOWN_POS) diff --git a/tests/rest/client/sliding_sync/test_sliding_sync.py b/tests/rest/client/sliding_sync/test_sliding_sync.py index c27a712088d..bcd22d15ca6 100644 --- a/tests/rest/client/sliding_sync/test_sliding_sync.py +++ b/tests/rest/client/sliding_sync/test_sliding_sync.py @@ -46,7 +46,7 @@ from synapse.util.stringutils import random_string from tests import unittest -from tests.server import TimedOutException +from tests.server import FakeChannel, TimedOutException from tests.test_utils.event_injection import create_event logger = logging.getLogger(__name__) @@ -80,12 +80,10 @@ def default_config(self) -> JsonDict: config["experimental_features"] = {"msc3575_enabled": True} return config - def do_sync( + def make_sync_request( self, sync_body: JsonDict, *, since: str | None = None, tok: str - ) -> tuple[JsonDict, str]: - """Do a sliding sync request with given body. - - Asserts the request was successful. + ) -> FakeChannel: + """Make a sliding sync request with given body. Attributes: sync_body: The full request body to use @@ -106,6 +104,24 @@ def do_sync( content=sync_body, access_token=tok, ) + return channel + + def do_sync( + self, sync_body: JsonDict, *, since: str | None = None, tok: str + ) -> tuple[JsonDict, str]: + """Do a sliding sync request with given body. + + Asserts the request was successful. + + Attributes: + sync_body: The full request body to use + since: Optional since token + tok: Access token to use + + Returns: + A tuple of the response body and the `pos` field. + """ + channel = self.make_sync_request(sync_body, since=since, tok=tok) self.assertEqual(channel.code, 200, channel.json_body) return channel.json_body, channel.json_body["pos"] From b7e592a88c0f8c0b2ceaa9e88c53de661d22e6f2 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 25 Nov 2025 14:09:48 +0000 Subject: [PATCH 009/139] Allow `ruff` to auto-fix trailing spaces in multi-line comments (#19221) --- changelog.d/19221.misc | 1 + pyproject.toml | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 changelog.d/19221.misc diff --git a/changelog.d/19221.misc b/changelog.d/19221.misc new file mode 100644 index 00000000000..d1faf9cb72c --- /dev/null +++ b/changelog.d/19221.misc @@ -0,0 +1 @@ +Auto-fix trailing spaces in multi-line strings and comments when running the lint script. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 3e238233625..b795cba2383 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -269,6 +269,8 @@ extend-safe-fixes = [ "UP007", # pyupgrade rules compatible with Python >= 3.10 "UP045", + # Allow ruff to automatically fix trailing spaces within a multi-line string/comment. + "W293" ] [tool.ruff.lint.isort] From 2eb76b47414c72c64cd5ddcf6097a0e0ba4f6528 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 25 Nov 2025 08:48:04 -0700 Subject: [PATCH 010/139] 1.143.0 --- CHANGES.md | 13 +++++++++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 5ac37368675..83bb9dd6bee 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,16 @@ +# synapse 1.143.0 (2025-11-25) + +## Dropping support for PostgreSQL 13 + +In line with our [deprecation policy](https://github.com/element-hq/synapse/blob/develop/docs/deprecation_policy.md), we've dropped +support for PostgreSQL 13, as it is no longer supported upstream. +This release of Synapse requires PostgreSQL 14+. + +No significant changes since 1.143.0rc2. + + + + # synapse 1.143.0rc2 (2025-11-18) ## Dropping support for PostgreSQL 13 diff --git a/debian/changelog b/debian/changelog index 24b23ad52e8..f8cf0c86f9c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.143.0) stable; urgency=medium + + * New Synapse release 1.143.0. + + -- Synapse Packaging team Tue, 25 Nov 2025 08:44:56 -0700 + matrix-synapse-py3 (1.143.0~rc2) stable; urgency=medium * New Synapse release 1.143.0rc2. diff --git a/pyproject.toml b/pyproject.toml index 3e238233625..246a1d283e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.143.0rc2" +version = "1.143.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 87d6e27057c4bf939a371622006cbb3032b7d838 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 25 Nov 2025 08:48:42 -0700 Subject: [PATCH 011/139] Capitalize Synapse in changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 83bb9dd6bee..848cdc26a96 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,4 @@ -# synapse 1.143.0 (2025-11-25) +# Synapse 1.143.0 (2025-11-25) ## Dropping support for PostgreSQL 13 From ba65d8c351245090b87f3004ab64d708e6a08e67 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 25 Nov 2025 18:03:33 +0000 Subject: [PATCH 012/139] Put MSC2666 endpoint behind an experimental flag (#19219) --- changelog.d/19219.misc | 1 + docs/upgrade.md | 11 +++++++++++ synapse/config/experimental.py | 3 +++ synapse/rest/client/mutual_rooms.py | 3 ++- synapse/rest/client/versions.py | 2 +- tests/rest/client/test_mutual_rooms.py | 21 +++++++++++++++++++++ 6 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 changelog.d/19219.misc diff --git a/changelog.d/19219.misc b/changelog.d/19219.misc new file mode 100644 index 00000000000..83557293587 --- /dev/null +++ b/changelog.d/19219.misc @@ -0,0 +1 @@ +Require an experimental feature flag to be enabled in order for the unstable [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666) endpoint (`/_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms`) to be available. \ No newline at end of file diff --git a/docs/upgrade.md b/docs/upgrade.md index 20b7e952b27..350b71fe479 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -117,6 +117,17 @@ each upgrade are complete before moving on to the next upgrade, to avoid stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). +# Upgrading to v1.144.0 + +## Unstable mutual rooms endpoint is now behind an experimental feature flag + +The unstable mutual rooms endpoint from +[MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666) +(`/_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms`) is now +disabled by default. If you rely on this unstable endpoint, you must now set +`experimental_features.msc2666_enabled: true` in your configuration to keep +using it. + # Upgrading to v1.143.0 ## Dropping support for PostgreSQL 13 diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 52c3ec0da29..566071eef31 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -438,6 +438,9 @@ def read_config( # previously calculated push actions. self.msc2654_enabled: bool = experimental.get("msc2654_enabled", False) + # MSC2666: Query mutual rooms between two users. + self.msc2666_enabled: bool = experimental.get("msc2666_enabled", False) + # MSC2815 (allow room moderators to view redacted event content) self.msc2815_enabled: bool = experimental.get("msc2815_enabled", False) diff --git a/synapse/rest/client/mutual_rooms.py b/synapse/rest/client/mutual_rooms.py index bda6ed1f70a..3e5316c4b7f 100644 --- a/synapse/rest/client/mutual_rooms.py +++ b/synapse/rest/client/mutual_rooms.py @@ -90,4 +90,5 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - UserMutualRoomsServlet(hs).register(http_server) + if hs.config.experimental.msc2666_enabled: + UserMutualRoomsServlet(hs).register(http_server) diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index dee2cdb637b..a0178e473d1 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -124,7 +124,7 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: # Implements additional endpoints as described in MSC2432 "org.matrix.msc2432": True, # Implements additional endpoints as described in MSC2666 - "uk.half-shot.msc2666.query_mutual_rooms": True, + "uk.half-shot.msc2666.query_mutual_rooms": self.config.experimental.msc2666_enabled, # Whether new rooms will be set to encrypted or not (based on presets). "io.element.e2ee_forced.public": self.e2ee_forced_public, "io.element.e2ee_forced.private": self.e2ee_forced_private, diff --git a/tests/rest/client/test_mutual_rooms.py b/tests/rest/client/test_mutual_rooms.py index 8580d090064..ea063707aa3 100644 --- a/tests/rest/client/test_mutual_rooms.py +++ b/tests/rest/client/test_mutual_rooms.py @@ -43,6 +43,12 @@ class UserMutualRoomsTest(unittest.HomeserverTestCase): mutual_rooms.register_servlets, ] + def default_config(self) -> dict: + config = super().default_config() + experimental = config.setdefault("experimental_features", {}) + experimental.setdefault("msc2666_enabled", True) + return config + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: config = self.default_config() return self.setup_test_homeserver(config=config) @@ -58,6 +64,21 @@ def _get_mutual_rooms(self, token: str, other_user: str) -> FakeChannel: access_token=token, ) + @unittest.override_config({"experimental_features": {"msc2666_enabled": False}}) + def test_mutual_rooms_no_experimental_flag(self) -> None: + """ + The endpoint should 404 if the experimental flag is not enabled. + """ + # Register a user. + u1 = self.register_user("user1", "pass") + u1_token = self.login(u1, "pass") + + # Check that we're unable to query the endpoint due to the endpoint + # being unrecognised. + channel = self._get_mutual_rooms(u1_token, "@not-used:test") + self.assertEqual(404, channel.code, channel.result) + self.assertEqual("M_UNRECOGNIZED", channel.json_body["errcode"], channel.result) + def test_shared_room_list_public(self) -> None: """ A room should show up in the shared list of rooms between two users From 2741ead56903dafd33126a7b970222f9bc9c202f Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 26 Nov 2025 10:32:39 +0000 Subject: [PATCH 013/139] Stop building wheels for MacOS (#19225) --- changelog.d/19225.removal | 1 + pyproject.toml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19225.removal diff --git a/changelog.d/19225.removal b/changelog.d/19225.removal new file mode 100644 index 00000000000..bed5db07e81 --- /dev/null +++ b/changelog.d/19225.removal @@ -0,0 +1 @@ +Stop building release wheels for MacOS. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index b483c903f6a..fabc483b3c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -395,7 +395,8 @@ build-backend = "poetry.core.masonry.api" # We skip: # - free-threaded cpython builds: these are not currently supported. # - i686: We don't support 32-bit platforms. -skip = "cp3??t-* *i686*" +# - *macosx*: we don't support building wheels for MacOS. +skip = "cp3??t-* *i686* *macosx*" # Enable non-default builds. See the list of available options: # https://cibuildwheel.pypa.io/en/stable/options#enable # From b74c29f6948d9f2de971fe4bf61798a58b2796dc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 26 Nov 2025 10:56:59 +0000 Subject: [PATCH 014/139] Move towards a dedicated `Duration` class (#19223) We have various constants to try and avoid mistyping of durations, e.g. `ONE_HOUR_SECONDS * MILLISECONDS_PER_SECOND`, however this can get a little verbose and doesn't help with typing. Instead, let's move towards a dedicated `Duration` class (basically a [`timedelta`](https://docs.python.org/3/library/datetime.html#timedelta-objects) with helper methods). This PR introduces the new types and converts all usages of the existing constants with it. Future PRs may work to move the clock methods to also use it (e.g. `call_later` and `looping_call`). Reviewable commit-by-commit. --- changelog.d/19223.misc | 1 + synapse/app/phone_stats_home.py | 18 ++++----- synapse/handlers/sliding_sync/room_lists.py | 10 ++--- synapse/handlers/worker_lock.py | 6 +-- synapse/rest/client/transactions.py | 7 ++-- synapse/storage/databases/main/deviceinbox.py | 13 +++--- .../storage/databases/main/sliding_sync.py | 22 +++++----- synapse/util/__init__.py | 9 ----- synapse/util/background_queue.py | 6 +-- synapse/util/constants.py | 23 ----------- synapse/util/duration.py | 40 +++++++++++++++++++ tests/metrics/test_phone_home_stats.py | 4 +- .../sliding_sync/test_connection_tracking.py | 12 +++--- tests/rest/client/test_transactions.py | 9 ++--- .../databases/main/test_deviceinbox.py | 8 ++-- 15 files changed, 95 insertions(+), 93 deletions(-) create mode 100644 changelog.d/19223.misc delete mode 100644 synapse/util/constants.py create mode 100644 synapse/util/duration.py diff --git a/changelog.d/19223.misc b/changelog.d/19223.misc new file mode 100644 index 00000000000..8caebead72a --- /dev/null +++ b/changelog.d/19223.misc @@ -0,0 +1 @@ +Move towards using a dedicated `Duration` type. diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index 13a0e3db7cd..d278e30850b 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -30,24 +30,20 @@ from synapse.metrics import SERVER_NAME_LABEL from synapse.types import JsonDict -from synapse.util.constants import ( - MILLISECONDS_PER_SECOND, - ONE_HOUR_SECONDS, - ONE_MINUTE_SECONDS, -) +from synapse.util.duration import Duration if TYPE_CHECKING: from synapse.server import HomeServer logger = logging.getLogger("synapse.app.homeserver") -INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS = 5 * ONE_MINUTE_SECONDS +INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME = Duration(minutes=5) """ We wait 5 minutes to send the first set of stats as the server can be quite busy the first few minutes """ -PHONE_HOME_INTERVAL_SECONDS = 3 * ONE_HOUR_SECONDS +PHONE_HOME_INTERVAL = Duration(hours=3) """ Phone home stats are sent every 3 hours """ @@ -222,13 +218,13 @@ def performance_stats_init() -> None: # table will decrease clock.looping_call( hs.get_datastores().main.generate_user_daily_visits, - 5 * ONE_MINUTE_SECONDS * MILLISECONDS_PER_SECOND, + Duration(minutes=5).as_millis(), ) # monthly active user limiting functionality clock.looping_call( hs.get_datastores().main.reap_monthly_active_users, - ONE_HOUR_SECONDS * MILLISECONDS_PER_SECOND, + Duration(hours=1).as_millis(), ) hs.get_datastores().main.reap_monthly_active_users() @@ -274,7 +270,7 @@ async def _generate_monthly_active_users() -> None: logger.info("Scheduling stats reporting for 3 hour intervals") clock.looping_call( phone_stats_home, - PHONE_HOME_INTERVAL_SECONDS * MILLISECONDS_PER_SECOND, + PHONE_HOME_INTERVAL.as_millis(), hs, stats, ) @@ -289,7 +285,7 @@ async def _generate_monthly_active_users() -> None: # We wait 5 minutes to send the first set of stats as the server can # be quite busy the first few minutes clock.call_later( - INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS, + INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME.as_secs(), phone_stats_home, hs, stats, diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index fa4ff22b645..8969d915836 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -39,7 +39,7 @@ from synapse.events import StrippedStateEvent from synapse.events.utils import parse_stripped_state_event from synapse.logging.opentracing import start_active_span, trace -from synapse.storage.databases.main.sliding_sync import UPDATE_INTERVAL_LAST_USED_TS_MS +from synapse.storage.databases.main.sliding_sync import UPDATE_INTERVAL_LAST_USED_TS from synapse.storage.databases.main.state import ( ROOM_UNKNOWN_SENTINEL, Sentinel as StateSentinel, @@ -70,7 +70,7 @@ ) from synapse.types.state import StateFilter from synapse.util import MutableOverlayMapping -from synapse.util.constants import MILLISECONDS_PER_SECOND, ONE_HOUR_SECONDS +from synapse.util.duration import Duration from synapse.util.sentinel import Sentinel if TYPE_CHECKING: @@ -85,7 +85,7 @@ # tight loops with clients that request lots of data at once. # # c.f. `NUM_ROOMS_THRESHOLD`. These values are somewhat arbitrary picked. -MINIMUM_NOT_USED_AGE_EXPIRY_MS = ONE_HOUR_SECONDS * MILLISECONDS_PER_SECOND +MINIMUM_NOT_USED_AGE_EXPIRY = Duration(hours=1) # How many rooms with updates we allow before we consider the connection expired # due to too many rooms to send. @@ -99,7 +99,7 @@ # connection even if it is actively being used (and we're just not updating the # DB frequently enough). We arbitrarily double the update interval to give some # wiggle room. -assert 2 * UPDATE_INTERVAL_LAST_USED_TS_MS < MINIMUM_NOT_USED_AGE_EXPIRY_MS +assert 2 * UPDATE_INTERVAL_LAST_USED_TS < MINIMUM_NOT_USED_AGE_EXPIRY # Helper definition for the types that we might return. We do this to avoid # copying data between types (which can be expensive for many rooms). @@ -913,7 +913,7 @@ async def _filter_relevant_rooms_to_send( if ( last_sync_ts is not None and (self._clock.time_msec() - last_sync_ts) - > MINIMUM_NOT_USED_AGE_EXPIRY_MS + > MINIMUM_NOT_USED_AGE_EXPIRY.as_millis() ): raise SlidingSyncUnknownPosition() diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 0e3fab292f9..4f9c632f5cf 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -39,7 +39,7 @@ from synapse.storage.databases.main.lock import Lock, LockStore from synapse.util.async_helpers import timeout_deferred from synapse.util.clock import Clock -from synapse.util.constants import ONE_MINUTE_SECONDS +from synapse.util.duration import Duration if TYPE_CHECKING: from synapse.logging.opentracing import opentracing @@ -276,7 +276,7 @@ async def __aexit__( def _get_next_retry_interval(self) -> float: next = self._retry_interval self._retry_interval = max(5, next * 2) - if self._retry_interval > 10 * ONE_MINUTE_SECONDS: # >7 iterations + if self._retry_interval > Duration(minutes=10).as_secs(): # >7 iterations logger.warning( "Lock timeout is getting excessive: %ss. There may be a deadlock.", self._retry_interval, @@ -363,7 +363,7 @@ async def __aexit__( def _get_next_retry_interval(self) -> float: next = self._retry_interval self._retry_interval = max(5, next * 2) - if self._retry_interval > 10 * ONE_MINUTE_SECONDS: # >7 iterations + if self._retry_interval > Duration(minutes=10).as_secs(): # >7 iterations logger.warning( "Lock timeout is getting excessive: %ss. There may be a deadlock.", self._retry_interval, diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py index 4b3656a5971..0c1ac1f11b4 100644 --- a/synapse/rest/client/transactions.py +++ b/synapse/rest/client/transactions.py @@ -34,13 +34,14 @@ from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.types import JsonDict, Requester from synapse.util.async_helpers import ObservableDeferred +from synapse.util.duration import Duration if TYPE_CHECKING: from synapse.server import HomeServer logger = logging.getLogger(__name__) -CLEANUP_PERIOD_MS = 1000 * 60 * 30 # 30 mins +CLEANUP_PERIOD = Duration(minutes=30) P = ParamSpec("P") @@ -56,7 +57,7 @@ def __init__(self, hs: "HomeServer"): ] = {} # Try to clean entries every 30 mins. This means entries will exist # for at *LEAST* 30 mins, and at *MOST* 60 mins. - self.clock.looping_call(self._cleanup, CLEANUP_PERIOD_MS) + self.clock.looping_call(self._cleanup, CLEANUP_PERIOD.as_millis()) def _get_transaction_key(self, request: IRequest, requester: Requester) -> Hashable: """A helper function which returns a transaction key that can be used @@ -145,5 +146,5 @@ def _cleanup(self) -> None: now = self.clock.time_msec() for key in list(self.transactions): ts = self.transactions[key][1] - if now > (ts + CLEANUP_PERIOD_MS): # after cleanup period + if now > (ts + CLEANUP_PERIOD.as_millis()): # after cleanup period del self.transactions[key] diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index a12411d7238..28e706d5c34 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -48,9 +48,9 @@ ) from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.types import JsonDict, StrCollection -from synapse.util import Duration from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.caches.stream_change_cache import StreamChangeCache +from synapse.util.duration import Duration from synapse.util.iterutils import batch_iter from synapse.util.json import json_encoder from synapse.util.stringutils import parse_and_validate_server_name @@ -62,10 +62,10 @@ # How long to keep messages in the device federation inbox before deleting them. -DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS = 7 * Duration.DAY_MS +DEVICE_FEDERATION_INBOX_CLEANUP_DELAY = Duration(days=7) # How often to run the task to clean up old device_federation_inbox rows. -DEVICE_FEDERATION_INBOX_CLEANUP_INTERVAL_MS = 5 * Duration.MINUTE_MS +DEVICE_FEDERATION_INBOX_CLEANUP_INTERVAL = Duration(minutes=5) # Update name for the device federation inbox received timestamp index. DEVICE_FEDERATION_INBOX_RECEIVED_INDEX_UPDATE = ( @@ -152,7 +152,7 @@ def __init__( if hs.config.worker.run_background_tasks: self.clock.looping_call( run_as_background_process, - DEVICE_FEDERATION_INBOX_CLEANUP_INTERVAL_MS, + DEVICE_FEDERATION_INBOX_CLEANUP_INTERVAL.as_millis(), "_delete_old_federation_inbox_rows", self.server_name, self._delete_old_federation_inbox_rows, @@ -996,9 +996,10 @@ async def _delete_old_federation_inbox_rows(self, batch_size: int = 1000) -> Non def _delete_old_federation_inbox_rows_txn(txn: LoggingTransaction) -> bool: # We delete at most 100 rows that are older than - # DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS + # DEVICE_FEDERATION_INBOX_CLEANUP_DELAY delete_before_ts = ( - self.clock.time_msec() - DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS + self.clock.time_msec() + - DEVICE_FEDERATION_INBOX_CLEANUP_DELAY.as_millis() ) sql = """ WITH to_delete AS ( diff --git a/synapse/storage/databases/main/sliding_sync.py b/synapse/storage/databases/main/sliding_sync.py index 8cd3de8f40c..6f87308cde6 100644 --- a/synapse/storage/databases/main/sliding_sync.py +++ b/synapse/storage/databases/main/sliding_sync.py @@ -37,12 +37,7 @@ RoomSyncConfig, ) from synapse.util.caches.descriptors import cached -from synapse.util.constants import ( - MILLISECONDS_PER_SECOND, - ONE_DAY_SECONDS, - ONE_HOUR_SECONDS, - ONE_MINUTE_SECONDS, -) +from synapse.util.duration import Duration from synapse.util.json import json_encoder if TYPE_CHECKING: @@ -57,14 +52,14 @@ # position. We don't want to update it on every use to avoid excessive # writes, but we want it to be reasonably up-to-date to help with # cleaning up old connection positions. -UPDATE_INTERVAL_LAST_USED_TS_MS = 5 * ONE_MINUTE_SECONDS * MILLISECONDS_PER_SECOND +UPDATE_INTERVAL_LAST_USED_TS = Duration(minutes=5) # Time in milliseconds the connection hasn't been used before we consider it # expired and delete it. -CONNECTION_EXPIRY_MS = 7 * ONE_DAY_SECONDS * MILLISECONDS_PER_SECOND +CONNECTION_EXPIRY = Duration(days=7) # How often we run the background process to delete old sliding sync connections. -CONNECTION_EXPIRY_FREQUENCY_MS = ONE_HOUR_SECONDS * MILLISECONDS_PER_SECOND +CONNECTION_EXPIRY_FREQUENCY = Duration(hours=1) class SlidingSyncStore(SQLBaseStore): @@ -101,7 +96,7 @@ def __init__( if self.hs.config.worker.run_background_tasks: self.clock.looping_call( self.delete_old_sliding_sync_connections, - CONNECTION_EXPIRY_FREQUENCY_MS, + CONNECTION_EXPIRY_FREQUENCY.as_millis(), ) async def get_latest_bump_stamp_for_room( @@ -430,7 +425,10 @@ def _get_and_clear_connection_positions_txn( # Update the `last_used_ts` if it's due to be updated. We don't update # every time to avoid excessive writes. now = self.clock.time_msec() - if last_used_ts is None or now - last_used_ts > UPDATE_INTERVAL_LAST_USED_TS_MS: + if ( + last_used_ts is None + or now - last_used_ts > UPDATE_INTERVAL_LAST_USED_TS.as_millis() + ): self.db_pool.simple_update_txn( txn, table="sliding_sync_connections", @@ -532,7 +530,7 @@ def _get_and_clear_connection_positions_txn( @wrap_as_background_process("delete_old_sliding_sync_connections") async def delete_old_sliding_sync_connections(self) -> None: """Delete sliding sync connections that have not been used for a long time.""" - cutoff_ts = self.clock.time_msec() - CONNECTION_EXPIRY_MS + cutoff_ts = self.clock.time_msec() - CONNECTION_EXPIRY.as_millis() def delete_old_sliding_sync_connections_txn(txn: LoggingTransaction) -> None: sql = """ diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index f937080f9ec..fbd01914d56 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -41,15 +41,6 @@ logger = logging.getLogger(__name__) -class Duration: - """Helper class that holds constants for common time durations in - milliseconds.""" - - MINUTE_MS = 60 * 1000 - HOUR_MS = 60 * MINUTE_MS - DAY_MS = 24 * HOUR_MS - - def unwrapFirstError(failure: Failure) -> Failure: # Deprecated: you probably just want to catch defer.FirstError and reraise # the subFailure's value, which will do a better job of preserving stacktraces. diff --git a/synapse/util/background_queue.py b/synapse/util/background_queue.py index 93ffd9f2719..dfea7247f40 100644 --- a/synapse/util/background_queue.py +++ b/synapse/util/background_queue.py @@ -27,7 +27,7 @@ from twisted.internet import defer from synapse.util.async_helpers import DeferredEvent -from synapse.util.constants import MILLISECONDS_PER_SECOND +from synapse.util.duration import Duration if TYPE_CHECKING: from synapse.server import HomeServer @@ -67,7 +67,7 @@ def __init__( self._hs = hs self._name = name self._callback = callback - self._timeout_ms = timeout_ms + self._timeout_ms = Duration(milliseconds=timeout_ms) # The queue of items to process. self._queue: collections.deque[T] = collections.deque() @@ -125,7 +125,7 @@ async def _process_queue(self) -> None: # just loop round, clear the event, recheck the queue, and then # wait here again. new_data = await self._wakeup_event.wait( - timeout_seconds=self._timeout_ms / MILLISECONDS_PER_SECOND + timeout_seconds=self._timeout_ms.as_secs() ) if not new_data: # Timed out waiting for new data, so exit the loop diff --git a/synapse/util/constants.py b/synapse/util/constants.py deleted file mode 100644 index f4491b58856..00000000000 --- a/synapse/util/constants.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright (C) 2025 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# - -# Time-based constants. -# -# Laying these out incrementally, even if only some are required, helps with -# readability and catching bugs. -ONE_MINUTE_SECONDS = 60 -ONE_HOUR_SECONDS = 60 * ONE_MINUTE_SECONDS -ONE_DAY_SECONDS = 24 * ONE_HOUR_SECONDS - -MILLISECONDS_PER_SECOND = 1000 diff --git a/synapse/util/duration.py b/synapse/util/duration.py new file mode 100644 index 00000000000..3419f6dda60 --- /dev/null +++ b/synapse/util/duration.py @@ -0,0 +1,40 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from datetime import timedelta + +# Constant so we don't keep creating new timedelta objects when calling +# `.as_millis()`. +_ONE_MILLISECOND = timedelta(milliseconds=1) + + +class Duration(timedelta): + """A subclass of timedelta that adds a convenience method for getting + the duration in milliseconds. + + Examples: + + ``` + duration = Duration(hours=2) + print(duration.as_millis()) # Outputs: 7200000 + ``` + """ + + def as_millis(self) -> int: + """Returns the duration in milliseconds.""" + return int(self / _ONE_MILLISECOND) + + def as_secs(self) -> int: + """Returns the duration in seconds.""" + return int(self.total_seconds()) diff --git a/tests/metrics/test_phone_home_stats.py b/tests/metrics/test_phone_home_stats.py index 4462385dae0..dfb88588cdf 100644 --- a/tests/metrics/test_phone_home_stats.py +++ b/tests/metrics/test_phone_home_stats.py @@ -17,7 +17,7 @@ from twisted.internet.testing import MemoryReactor from synapse.app.phone_stats_home import ( - PHONE_HOME_INTERVAL_SECONDS, + PHONE_HOME_INTERVAL, start_phone_stats_home, ) from synapse.rest import admin, login, register, room @@ -78,7 +78,7 @@ def prepare( def _get_latest_phone_home_stats(self) -> JsonDict: # Wait for `phone_stats_home` to be called again + a healthy margin (50s). - self.reactor.advance(2 * PHONE_HOME_INTERVAL_SECONDS + 50) + self.reactor.advance(2 * PHONE_HOME_INTERVAL.as_secs() + 50) # Extract the reported stats from our http client mock mock_calls = self.put_json_mock.call_args_list diff --git a/tests/rest/client/sliding_sync/test_connection_tracking.py b/tests/rest/client/sliding_sync/test_connection_tracking.py index cdf63317e39..44e7fa47262 100644 --- a/tests/rest/client/sliding_sync/test_connection_tracking.py +++ b/tests/rest/client/sliding_sync/test_connection_tracking.py @@ -24,7 +24,7 @@ from synapse.handlers.sliding_sync import room_lists from synapse.rest.client import login, room, sync from synapse.server import HomeServer -from synapse.storage.databases.main.sliding_sync import CONNECTION_EXPIRY_MS +from synapse.storage.databases.main.sliding_sync import CONNECTION_EXPIRY from synapse.util.clock import Clock from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase @@ -407,7 +407,7 @@ def test_sliding_sync_connection_expires_with_too_much_data(self) -> None: we expire the connection and ask the client to do a full resync. Connections are only expired if they have not been used for a minimum - amount of time (MINIMUM_NOT_USED_AGE_EXPIRY_MS) to avoid expiring + amount of time (MINIMUM_NOT_USED_AGE_EXPIRY) to avoid expiring connections that are actively being used. """ @@ -455,7 +455,7 @@ def test_sliding_sync_connection_expires_with_too_much_data(self) -> None: self.helper.send(room_id, "msg", tok=user2_tok) # Advance the clock to ensure that the last_used_ts is old enough - self.reactor.advance(2 * room_lists.MINIMUM_NOT_USED_AGE_EXPIRY_MS / 1000) + self.reactor.advance(2 * room_lists.MINIMUM_NOT_USED_AGE_EXPIRY.as_secs()) # This sync should now raise SlidingSyncUnknownPosition channel = self.make_sync_request(sync_body, since=from_token, tok=user1_tok) @@ -490,14 +490,14 @@ def test_sliding_sync_connection_expires_after_time(self) -> None: _, from_token = self.do_sync(sync_body, tok=user1_tok) # We can keep syncing so long as the interval between requests is less - # than CONNECTION_EXPIRY_MS + # than CONNECTION_EXPIRY for _ in range(5): - self.reactor.advance(0.5 * CONNECTION_EXPIRY_MS / 1000) + self.reactor.advance(0.5 * CONNECTION_EXPIRY.as_secs()) _, from_token = self.do_sync(sync_body, tok=user1_tok) # ... but if we wait too long, the connection expires - self.reactor.advance(1 + CONNECTION_EXPIRY_MS / 1000) + self.reactor.advance(1 + CONNECTION_EXPIRY.as_secs()) # This sync should now raise SlidingSyncUnknownPosition channel = self.make_sync_request(sync_body, since=from_token, tok=user1_tok) diff --git a/tests/rest/client/test_transactions.py b/tests/rest/client/test_transactions.py index 64d22d485ab..0407bb53477 100644 --- a/tests/rest/client/test_transactions.py +++ b/tests/rest/client/test_transactions.py @@ -26,12 +26,9 @@ from twisted.internet import defer, reactor as _reactor from synapse.logging.context import SENTINEL_CONTEXT, LoggingContext, current_context -from synapse.rest.client.transactions import CLEANUP_PERIOD_MS, HttpTransactionCache +from synapse.rest.client.transactions import CLEANUP_PERIOD, HttpTransactionCache from synapse.types import ISynapseReactor, JsonDict from synapse.util.clock import Clock -from synapse.util.constants import ( - MILLISECONDS_PER_SECOND, -) from tests import unittest from tests.server import get_clock @@ -187,7 +184,7 @@ def test_cleans_up(self) -> Generator["defer.Deferred[Any]", object, None]: ) # Advance time just under the cleanup period. # Should NOT have cleaned up yet - self.reactor.advance((CLEANUP_PERIOD_MS - 1) / MILLISECONDS_PER_SECOND) + self.reactor.advance(CLEANUP_PERIOD.as_secs() - 1) yield self.cache.fetch_or_execute_request( self.mock_request, self.mock_requester, cb, "an arg" @@ -196,7 +193,7 @@ def test_cleans_up(self) -> Generator["defer.Deferred[Any]", object, None]: cb.assert_called_once_with("an arg") # Advance time just after the cleanup period. - self.reactor.advance(2 / MILLISECONDS_PER_SECOND) + self.reactor.advance(2) yield self.cache.fetch_or_execute_request( self.mock_request, self.mock_requester, cb, "an arg" diff --git a/tests/storage/databases/main/test_deviceinbox.py b/tests/storage/databases/main/test_deviceinbox.py index dbf362a3cc4..be585068fe2 100644 --- a/tests/storage/databases/main/test_deviceinbox.py +++ b/tests/storage/databases/main/test_deviceinbox.py @@ -28,7 +28,7 @@ from synapse.rest.client import devices from synapse.server import HomeServer from synapse.storage.databases.main.deviceinbox import ( - DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS, + DEVICE_FEDERATION_INBOX_CLEANUP_DELAY, ) from synapse.util.clock import Clock @@ -191,7 +191,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.db_pool = self.store.db_pool # Advance time to ensure we are past the cleanup delay - self.reactor.advance(DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS * 2 / 1000) + self.reactor.advance(DEVICE_FEDERATION_INBOX_CLEANUP_DELAY.as_secs() * 2) def test_delete_old_federation_inbox_rows_skips_if_no_index(self) -> None: """Test that we don't delete rows if the index hasn't been created yet.""" @@ -245,7 +245,7 @@ def test_delete_old_federation_inbox_rows(self) -> None: ) ) - self.reactor.advance(2 * DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS / 1000) + self.reactor.advance(2 * DEVICE_FEDERATION_INBOX_CLEANUP_DELAY.as_secs()) # Insert new messages for i in range(5): @@ -293,7 +293,7 @@ def test_delete_old_federation_inbox_rows_batch_limit(self) -> None: ) # Advance time to ensure we are past the cleanup delay - self.reactor.advance(2 * DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS / 1000) + self.reactor.advance(2 * DEVICE_FEDERATION_INBOX_CLEANUP_DELAY.as_millis()) # Run the cleanup - it should delete in batches and sleep between them deferred = defer.ensureDeferred( From c928347779cfd1dd028354c4d50674514903b7df Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 26 Nov 2025 16:12:14 +0000 Subject: [PATCH 015/139] Implement MSC4380: Invite blocking (#19203) MSC4380 aims to be a simplified implementation of MSC4155; the hope is that we can get it specced and rolled out rapidly, so that we can resolve the fact that `matrix.org` has enabled MSC4155. The implementation leans heavily on what's already there for MSC4155. It has its own `experimental_features` flag. If both MSC4155 and MSC4380 are enabled, and a user has both configurations set, then we prioritise the MSC4380 one. Contributed wearing my :tophat: Spec Core Team hat. --- changelog.d/19203.feature | 1 + synapse/api/constants.py | 4 + synapse/api/errors.py | 2 +- synapse/config/experimental.py | 3 + synapse/rest/client/versions.py | 2 + .../storage/databases/main/account_data.py | 36 +++-- synapse/storage/invite_rule.py | 44 +++++- tests/handlers/test_room_member.py | 146 +++++++++++++++++- tests/storage/test_invite_rule.py | 28 ++-- 9 files changed, 239 insertions(+), 27 deletions(-) create mode 100644 changelog.d/19203.feature diff --git a/changelog.d/19203.feature b/changelog.d/19203.feature new file mode 100644 index 00000000000..d192781b20e --- /dev/null +++ b/changelog.d/19203.feature @@ -0,0 +1 @@ +Add experimentatal implememntation of [MSC4380](https://github.com/matrix-org/matrix-spec-proposals/pull/4380) (invite blocking). diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 7a8f546d6bf..d41e44b1541 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -307,6 +307,10 @@ class AccountDataTypes: MSC4155_INVITE_PERMISSION_CONFIG: Final = ( "org.matrix.msc4155.invite_permission_config" ) + # MSC4380: Invite blocking + MSC4380_INVITE_PERMISSION_CONFIG: Final = ( + "org.matrix.msc4380.invite_permission_config" + ) # Synapse-specific behaviour. See "Client-Server API Extensions" documentation # in Admin API for more information. SYNAPSE_ADMIN_CLIENT_CONFIG: Final = "io.element.synapse.admin_client_config" diff --git a/synapse/api/errors.py b/synapse/api/errors.py index c4339ebef89..37b909a1a7e 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -137,7 +137,7 @@ class Codes(str, Enum): PROFILE_TOO_LARGE = "M_PROFILE_TOO_LARGE" KEY_TOO_LARGE = "M_KEY_TOO_LARGE" - # Part of MSC4155 + # Part of MSC4155/MSC4380 INVITE_BLOCKED = "ORG.MATRIX.MSC4155.M_INVITE_BLOCKED" # Part of MSC4190 diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 566071eef31..dc5e096791a 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -596,3 +596,6 @@ def read_config( # MSC4306: Thread Subscriptions # (and MSC4308: Thread Subscriptions extension to Sliding Sync) self.msc4306_enabled: bool = experimental.get("msc4306_enabled", False) + + # MSC4380: Invite blocking + self.msc4380_enabled: bool = experimental.get("msc4380_enabled", False) diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index a0178e473d1..75f27c98dea 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -182,6 +182,8 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: "org.matrix.msc4306": self.config.experimental.msc4306_enabled, # MSC4169: Backwards-compatible redaction sending using `/send` "com.beeper.msc4169": self.config.experimental.msc4169_enabled, + # MSC4380: Invite blocking + "org.matrix.msc4380": self.config.experimental.msc4380_enabled, }, }, ) diff --git a/synapse/storage/databases/main/account_data.py b/synapse/storage/databases/main/account_data.py index 15728cf6181..71182cdab2b 100644 --- a/synapse/storage/databases/main/account_data.py +++ b/synapse/storage/databases/main/account_data.py @@ -40,7 +40,12 @@ ) from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.databases.main.push_rule import PushRulesWorkerStore -from synapse.storage.invite_rule import InviteRulesConfig +from synapse.storage.invite_rule import ( + AllowAllInviteRulesConfig, + InviteRulesConfig, + MSC4155InviteRulesConfig, + MSC4380InviteRulesConfig, +) from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.types import JsonDict, JsonMapping from synapse.util.caches.descriptors import cached @@ -104,6 +109,7 @@ def __init__( ) self._msc4155_enabled = hs.config.experimental.msc4155_enabled + self._msc4380_enabled = hs.config.experimental.msc4380_enabled def get_max_account_data_stream_id(self) -> int: """Get the current max stream ID for account data stream @@ -562,20 +568,28 @@ async def ignored_users(self, user_id: str) -> frozenset[str]: async def get_invite_config_for_user(self, user_id: str) -> InviteRulesConfig: """ - Get the invite configuration for the current user. + Get the invite configuration for the given user. Args: - user_id: + user_id: The user whose invite configuration should be returned. """ + if self._msc4380_enabled: + data = await self.get_global_account_data_by_type_for_user( + user_id, AccountDataTypes.MSC4380_INVITE_PERMISSION_CONFIG + ) + # If the user has an MSC4380-style config setting, prioritise that + # above an MSC4155 one + if data is not None: + return MSC4380InviteRulesConfig.from_account_data(data) + + if self._msc4155_enabled: + data = await self.get_global_account_data_by_type_for_user( + user_id, AccountDataTypes.MSC4155_INVITE_PERMISSION_CONFIG + ) + if data is not None: + return MSC4155InviteRulesConfig(data) - if not self._msc4155_enabled: - # This equates to allowing all invites, as if the setting was off. - return InviteRulesConfig(None) - - data = await self.get_global_account_data_by_type_for_user( - user_id, AccountDataTypes.MSC4155_INVITE_PERMISSION_CONFIG - ) - return InviteRulesConfig(data) + return AllowAllInviteRulesConfig() async def get_admin_client_config_for_user(self, user_id: str) -> AdminClientConfig: """ diff --git a/synapse/storage/invite_rule.py b/synapse/storage/invite_rule.py index 3de77e8c21e..489533a9f4d 100644 --- a/synapse/storage/invite_rule.py +++ b/synapse/storage/invite_rule.py @@ -1,7 +1,9 @@ import logging +from abc import abstractmethod from enum import Enum from typing import Pattern +import attr from matrix_common.regex import glob_to_regex from synapse.types import JsonMapping, UserID @@ -18,9 +20,29 @@ class InviteRule(Enum): class InviteRulesConfig: - """Class to determine if a given user permits an invite from another user, and the action to take.""" + """An object encapsulating a given user's choices about whether to accept invites.""" - def __init__(self, account_data: JsonMapping | None): + @abstractmethod + def get_invite_rule(self, inviter_user_id: str) -> InviteRule: + """Get the invite rule that matches this user. Will return InviteRule.ALLOW if no rules match + + Args: + inviter_user_id: The user ID of the inviting user. + """ + + +@attr.s(slots=True) +class AllowAllInviteRulesConfig(InviteRulesConfig): + """An `InviteRulesConfig` implementation which will accept all invites.""" + + def get_invite_rule(self, inviter_user_id: str) -> InviteRule: + return InviteRule.ALLOW + + +class MSC4155InviteRulesConfig(InviteRulesConfig): + """An object encapsulating [MSC4155](https://github.com/matrix-org/matrix-spec-proposals/pull/4155) invite rules.""" + + def __init__(self, account_data: JsonMapping): self.allowed_users: list[Pattern[str]] = [] self.ignored_users: list[Pattern[str]] = [] self.blocked_users: list[Pattern[str]] = [] @@ -110,3 +132,21 @@ def get_invite_rule(self, user_id: str) -> InviteRule: return rule return InviteRule.ALLOW + + +@attr.s(slots=True, auto_attribs=True) +class MSC4380InviteRulesConfig(InviteRulesConfig): + default_invite_rule: InviteRule + """The invite rule to apply to all invites.""" + + @classmethod + def from_account_data(cls, data: JsonMapping) -> "MSC4380InviteRulesConfig": + default = data.get("default_action") + + default_invite_rule = ( + InviteRule.BLOCK if default == "block" else InviteRule.ALLOW + ) + return cls(default_invite_rule=default_invite_rule) + + def get_invite_rule(self, inviter_user_id: str) -> InviteRule: + return self.default_invite_rule diff --git a/tests/handlers/test_room_member.py b/tests/handlers/test_room_member.py index 8f9e27603eb..d8d7caaf1bd 100644 --- a/tests/handlers/test_room_member.py +++ b/tests/handlers/test_room_member.py @@ -458,7 +458,9 @@ def test_deduplicate_joins(self) -> None: self.assertEqual(initial_count, new_count) -class TestInviteFiltering(FederatingHomeserverTestCase): +class TestMSC4155InviteFiltering(FederatingHomeserverTestCase): + """Tests for MSC4155-style invite filtering.""" + servlets = [ synapse.rest.admin.register_servlets, synapse.rest.client.login.register_servlets, @@ -618,3 +620,145 @@ def test_msc4155_block_invite_remote_server(self) -> None: ).value self.assertEqual(f.code, 403) self.assertEqual(f.errcode, "ORG.MATRIX.MSC4155.M_INVITE_BLOCKED") + + +class TestMSC4380InviteBlocking(FederatingHomeserverTestCase): + """Tests for MSC4380-style invite filtering.""" + + servlets = [ + synapse.rest.admin.register_servlets, + synapse.rest.client.login.register_servlets, + synapse.rest.client.room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.handler = hs.get_room_member_handler() + self.fed_handler = hs.get_federation_handler() + self.store = hs.get_datastores().main + + # Create two users. + self.alice = self.register_user("alice", "pass") + self.alice_token = self.login("alice", "pass") + self.bob = self.register_user("bob", "pass") + self.bob_token = self.login("bob", "pass") + + @override_config({"experimental_features": {"msc4380_enabled": True}}) + def test_misc4380_block_invite_local(self) -> None: + """Test that MSC4380 will block a user from being invited to a room""" + room_id = self.helper.create_room_as(self.alice, tok=self.alice_token) + + self.get_success( + self.store.add_account_data_for_user( + self.bob, + AccountDataTypes.MSC4380_INVITE_PERMISSION_CONFIG, + { + "default_action": "block", + }, + ) + ) + + f = self.get_failure( + self.handler.update_membership( + requester=create_requester(self.alice), + target=UserID.from_string(self.bob), + room_id=room_id, + action=Membership.INVITE, + ), + SynapseError, + ).value + self.assertEqual(f.code, 403) + self.assertEqual(f.errcode, "ORG.MATRIX.MSC4155.M_INVITE_BLOCKED") + + @override_config({"experimental_features": {"msc4380_enabled": True}}) + def test_misc4380_non_string_setting(self) -> None: + """Test that `default_action` being set to something non-stringy is the same as "accept".""" + room_id = self.helper.create_room_as(self.alice, tok=self.alice_token) + + self.get_success( + self.store.add_account_data_for_user( + self.bob, + AccountDataTypes.MSC4380_INVITE_PERMISSION_CONFIG, + { + "default_action": 1, + }, + ) + ) + + self.get_success( + self.handler.update_membership( + requester=create_requester(self.alice), + target=UserID.from_string(self.bob), + room_id=room_id, + action=Membership.INVITE, + ) + ) + + @override_config({"experimental_features": {"msc4380_enabled": False}}) + def test_msc4380_disabled_allow_invite_local(self) -> None: + """Test that, when MSC4380 is not enabled, invites are accepted as normal""" + room_id = self.helper.create_room_as(self.alice, tok=self.alice_token) + + self.get_success( + self.store.add_account_data_for_user( + self.bob, + AccountDataTypes.MSC4380_INVITE_PERMISSION_CONFIG, + { + "default_action": "block", + }, + ) + ) + + self.get_success( + self.handler.update_membership( + requester=create_requester(self.alice), + target=UserID.from_string(self.bob), + room_id=room_id, + action=Membership.INVITE, + ), + ) + + @override_config({"experimental_features": {"msc4380_enabled": True}}) + def test_msc4380_block_invite_remote(self) -> None: + """Test that MSC4380 will block a user from being invited to a room by a remote user.""" + # A remote user who sends the invite + remote_server = "otherserver" + remote_user = "@otheruser:" + remote_server + + self.get_success( + self.store.add_account_data_for_user( + self.bob, + AccountDataTypes.MSC4380_INVITE_PERMISSION_CONFIG, + {"default_action": "block"}, + ) + ) + + room_id = self.helper.create_room_as( + room_creator=self.alice, tok=self.alice_token + ) + room_version = self.get_success(self.store.get_room_version(room_id)) + + invite_event = event_from_pdu_json( + { + "type": EventTypes.Member, + "content": {"membership": "invite"}, + "room_id": room_id, + "sender": remote_user, + "state_key": self.bob, + "depth": 32, + "prev_events": [], + "auth_events": [], + "origin_server_ts": self.clock.time_msec(), + }, + room_version, + ) + + f = self.get_failure( + self.fed_handler.on_invite_request( + remote_server, + invite_event, + invite_event.room_version, + ), + SynapseError, + ).value + self.assertEqual(f.code, 403) + self.assertEqual(f.errcode, "ORG.MATRIX.MSC4155.M_INVITE_BLOCKED") diff --git a/tests/storage/test_invite_rule.py b/tests/storage/test_invite_rule.py index 38c97ecaa37..ae999077045 100644 --- a/tests/storage/test_invite_rule.py +++ b/tests/storage/test_invite_rule.py @@ -1,4 +1,8 @@ -from synapse.storage.invite_rule import InviteRule, InviteRulesConfig +from synapse.storage.invite_rule import ( + AllowAllInviteRulesConfig, + InviteRule, + MSC4155InviteRulesConfig, +) from synapse.types import UserID from tests import unittest @@ -10,23 +14,23 @@ class InviteFilterTestCase(unittest.TestCase): - def test_empty(self) -> None: + def test_allow_all(self) -> None: """Permit by default""" - config = InviteRulesConfig(None) + config = AllowAllInviteRulesConfig() self.assertEqual( config.get_invite_rule(regular_user.to_string()), InviteRule.ALLOW ) def test_ignore_invalid(self) -> None: """Invalid strings are ignored""" - config = InviteRulesConfig({"blocked_users": ["not a user"]}) + config = MSC4155InviteRulesConfig({"blocked_users": ["not a user"]}) self.assertEqual( config.get_invite_rule(blocked_user.to_string()), InviteRule.ALLOW ) def test_user_blocked(self) -> None: """Permit all, except explicitly blocked users""" - config = InviteRulesConfig({"blocked_users": [blocked_user.to_string()]}) + config = MSC4155InviteRulesConfig({"blocked_users": [blocked_user.to_string()]}) self.assertEqual( config.get_invite_rule(blocked_user.to_string()), InviteRule.BLOCK ) @@ -36,7 +40,7 @@ def test_user_blocked(self) -> None: def test_user_ignored(self) -> None: """Permit all, except explicitly ignored users""" - config = InviteRulesConfig({"ignored_users": [ignored_user.to_string()]}) + config = MSC4155InviteRulesConfig({"ignored_users": [ignored_user.to_string()]}) self.assertEqual( config.get_invite_rule(ignored_user.to_string()), InviteRule.IGNORE ) @@ -46,7 +50,7 @@ def test_user_ignored(self) -> None: def test_user_precedence(self) -> None: """Always take allowed over ignored, ignored over blocked, and then block.""" - config = InviteRulesConfig( + config = MSC4155InviteRulesConfig( { "allowed_users": [allowed_user.to_string()], "ignored_users": [allowed_user.to_string(), ignored_user.to_string()], @@ -70,7 +74,7 @@ def test_user_precedence(self) -> None: def test_server_blocked(self) -> None: """Block all users on the server except those allowed.""" user_on_same_server = UserID("blocked", allowed_user.domain) - config = InviteRulesConfig( + config = MSC4155InviteRulesConfig( { "allowed_users": [allowed_user.to_string()], "blocked_servers": [allowed_user.domain], @@ -86,7 +90,7 @@ def test_server_blocked(self) -> None: def test_server_ignored(self) -> None: """Ignore all users on the server except those allowed.""" user_on_same_server = UserID("ignored", allowed_user.domain) - config = InviteRulesConfig( + config = MSC4155InviteRulesConfig( { "allowed_users": [allowed_user.to_string()], "ignored_servers": [allowed_user.domain], @@ -104,7 +108,7 @@ def test_server_allow(self) -> None: blocked_user_on_same_server = UserID("blocked", allowed_user.domain) ignored_user_on_same_server = UserID("ignored", allowed_user.domain) allowed_user_on_same_server = UserID("another", allowed_user.domain) - config = InviteRulesConfig( + config = MSC4155InviteRulesConfig( { "ignored_users": [ignored_user_on_same_server.to_string()], "blocked_users": [blocked_user_on_same_server.to_string()], @@ -129,7 +133,7 @@ def test_server_allow(self) -> None: def test_server_precedence(self) -> None: """Always take allowed over ignored, ignored over blocked, and then block.""" - config = InviteRulesConfig( + config = MSC4155InviteRulesConfig( { "allowed_servers": [allowed_user.domain], "ignored_servers": [allowed_user.domain, ignored_user.domain], @@ -152,7 +156,7 @@ def test_server_precedence(self) -> None: def test_server_glob(self) -> None: """Test that glob patterns match""" - config = InviteRulesConfig({"blocked_servers": ["*.example.org"]}) + config = MSC4155InviteRulesConfig({"blocked_servers": ["*.example.org"]}) self.assertEqual( config.get_invite_rule(allowed_user.to_string()), InviteRule.BLOCK ) From 703464c1f78418c5f145a22e3a2f58d197be7270 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 26 Nov 2025 17:17:04 +0000 Subject: [PATCH 016/139] Fix case where `get_partial_current_state_deltas` could return >100 rows (#18960) --- changelog.d/18960.bugfix | 1 + synapse/storage/controllers/state.py | 2 +- .../storage/databases/main/state_deltas.py | 97 ++++-- tests/storage/test_state.py | 314 ++++++++++++++++++ 4 files changed, 380 insertions(+), 34 deletions(-) create mode 100644 changelog.d/18960.bugfix diff --git a/changelog.d/18960.bugfix b/changelog.d/18960.bugfix new file mode 100644 index 00000000000..909089f8092 --- /dev/null +++ b/changelog.d/18960.bugfix @@ -0,0 +1 @@ +Fix a bug in the database function for fetching state deltas that could result in unnecessarily long query times. \ No newline at end of file diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index 9c5e837ab07..4885268305d 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -683,7 +683,7 @@ async def get_current_state_deltas( # https://github.com/matrix-org/synapse/issues/13008 return await self.stores.main.get_partial_current_state_deltas( - prev_stream_id, max_stream_id + prev_stream_id, max_stream_id, limit=100 ) @trace diff --git a/synapse/storage/databases/main/state_deltas.py b/synapse/storage/databases/main/state_deltas.py index cd8f286d085..a5d5407327c 100644 --- a/synapse/storage/databases/main/state_deltas.py +++ b/synapse/storage/databases/main/state_deltas.py @@ -78,27 +78,41 @@ def __init__( ) async def get_partial_current_state_deltas( - self, prev_stream_id: int, max_stream_id: int + self, prev_stream_id: int, max_stream_id: int, limit: int = 100 ) -> tuple[int, list[StateDelta]]: - """Fetch a list of room state changes since the given stream id + """Fetch a list of room state changes since the given stream id. This may be the partial state if we're lazy joining the room. + This method takes care to handle state deltas that share the same + `stream_id`. That can happen when persisting state in a batch, + potentially as the result of state resolution (both adding new state and + undo'ing previous state). + + State deltas are grouped by `stream_id`. When hitting the given `limit` + would return only part of a "group" of state deltas, that entire group + is omitted. Thus, this function may return *up to* `limit` state deltas, + or slightly more when a single group itself exceeds `limit`. + Args: prev_stream_id: point to get changes since (exclusive) max_stream_id: the point that we know has been correctly persisted - ie, an upper limit to return changes from. + limit: the maximum number of rows to return. Returns: A tuple consisting of: - the stream id which these results go up to - list of current_state_delta_stream rows. If it is empty, we are up to date. - - A maximum of 100 rows will be returned. """ prev_stream_id = int(prev_stream_id) + if limit <= 0: + raise ValueError( + "Invalid `limit` passed to `get_partial_current_state_deltas" + ) + # check we're not going backwards assert prev_stream_id <= max_stream_id, ( f"New stream id {max_stream_id} is smaller than prev stream id {prev_stream_id}" @@ -115,45 +129,62 @@ async def get_partial_current_state_deltas( def get_current_state_deltas_txn( txn: LoggingTransaction, ) -> tuple[int, list[StateDelta]]: - # First we calculate the max stream id that will give us less than - # N results. - # We arbitrarily limit to 100 stream_id entries to ensure we don't - # select toooo many. - sql = """ - SELECT stream_id, count(*) + # First we group state deltas by `stream_id` and calculate which + # groups can be returned without exceeding the provided `limit`. + sql_grouped = """ + SELECT stream_id, COUNT(*) AS c FROM current_state_delta_stream WHERE stream_id > ? AND stream_id <= ? GROUP BY stream_id - ORDER BY stream_id ASC - LIMIT 100 + ORDER BY stream_id + LIMIT ? """ - txn.execute(sql, (prev_stream_id, max_stream_id)) - - total = 0 - - for stream_id, count in txn: - total += count - if total > 100: - # We arbitrarily limit to 100 entries to ensure we don't - # select toooo many. - logger.debug( - "Clipping current_state_delta_stream rows to stream_id %i", - stream_id, - ) - clipped_stream_id = stream_id + group_limit = limit + 1 + txn.execute(sql_grouped, (prev_stream_id, max_stream_id, group_limit)) + grouped_rows = txn.fetchall() + + if not grouped_rows: + # Nothing to return in the range; we are up to date through max_stream_id. + return max_stream_id, [] + + # Always retrieve the first group, at the bare minimum. This ensures the + # caller always makes progress, even if a single group exceeds `limit`. + fetch_upto_stream_id, included_rows = grouped_rows[0] + + # Determine which other groups we can retrieve at the same time, + # without blowing the budget. + included_all_groups = True + for stream_id, count in grouped_rows[1:]: + if included_rows + count > limit: + included_all_groups = False break - else: - # if there's no problem, we may as well go right up to the max_stream_id - clipped_stream_id = max_stream_id + included_rows += count + fetch_upto_stream_id = stream_id + + # If we retrieved fewer groups than the limit *and* we didn't hit the + # `LIMIT ?` cap on the grouping query, we know we've caught up with + # the stream. + caught_up_with_stream = ( + included_all_groups and len(grouped_rows) < group_limit + ) + + # At this point we should have advanced, or bailed out early above. + assert fetch_upto_stream_id != prev_stream_id - # Now actually get the deltas - sql = """ + # 2) Fetch the actual rows for only the included stream_id groups. + sql_rows = """ SELECT stream_id, room_id, type, state_key, event_id, prev_event_id FROM current_state_delta_stream WHERE ? < stream_id AND stream_id <= ? ORDER BY stream_id ASC """ - txn.execute(sql, (prev_stream_id, clipped_stream_id)) + txn.execute(sql_rows, (prev_stream_id, fetch_upto_stream_id)) + rows = txn.fetchall() + + clipped_stream_id = ( + max_stream_id if caught_up_with_stream else fetch_upto_stream_id + ) + return clipped_stream_id, [ StateDelta( stream_id=row[0], @@ -163,7 +194,7 @@ def get_current_state_deltas_txn( event_id=row[4], prev_event_id=row[5], ) - for row in txn.fetchall() + for row in rows ] return await self.db_pool.runInteraction( diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py index 8e821c6d183..dbbede812d9 100644 --- a/tests/storage/test_state.py +++ b/tests/storage/test_state.py @@ -19,6 +19,7 @@ # # +import json import logging from typing import cast @@ -33,6 +34,7 @@ from synapse.types import JsonDict, RoomID, StateMap, UserID from synapse.types.state import StateFilter from synapse.util.clock import Clock +from synapse.util.stringutils import random_string from tests.unittest import HomeserverTestCase @@ -643,3 +645,315 @@ def test_batched_state_group_storing(self) -> None: ), ) self.assertEqual(context.state_group_before_event, groups[0][0]) + + +class CurrentStateDeltaStreamTestCase(HomeserverTestCase): + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + self.store = hs.get_datastores().main + self.storage = hs.get_storage_controllers() + self.state_datastore = self.storage.state.stores.state + self.event_creation_handler = hs.get_event_creation_handler() + self.event_builder_factory = hs.get_event_builder_factory() + + # Create a made-up room and a user. + self.alice_user_id = UserID.from_string("@alice:test") + self.room = RoomID.from_string("!abc1234:test") + + self.get_success( + self.store.store_room( + self.room.to_string(), + room_creator_user_id="@creator:text", + is_public=True, + room_version=RoomVersions.V1, + ) + ) + + def inject_state_event( + self, room: RoomID, sender: UserID, typ: str, state_key: str, content: JsonDict + ) -> EventBase: + builder = self.event_builder_factory.for_room_version( + RoomVersions.V1, + { + "type": typ, + "sender": sender.to_string(), + "state_key": state_key, + "room_id": room.to_string(), + "content": content, + }, + ) + + event, unpersisted_context = self.get_success( + self.event_creation_handler.create_new_client_event(builder) + ) + + context = self.get_success(unpersisted_context.persist(event)) + + assert self.storage.persistence is not None + self.get_success(self.storage.persistence.persist_event(event, context)) + + return event + + def test_get_partial_current_state_deltas_limit(self) -> None: + """ + Tests that `get_partial_current_state_deltas` actually returns `limit` rows. + + Regression test for https://github.com/element-hq/synapse/pull/18960. + """ + # Inject a create event which other events can auth with. + self.inject_state_event( + self.room, self.alice_user_id, EventTypes.Create, "", {} + ) + + limit = 2 + + # Make N*2 state changes in the room, resulting in 2N+1 total state + # events (including the create event) in the room. + for i in range(limit * 2): + self.inject_state_event( + self.room, + self.alice_user_id, + EventTypes.Name, + "", + {"name": f"rename #{i}"}, + ) + + # Call the function under test. This must return <= `limit` rows. + max_stream_id = self.store.get_room_max_stream_ordering() + clipped_stream_id, deltas = self.get_success( + self.store.get_partial_current_state_deltas( + prev_stream_id=0, + max_stream_id=max_stream_id, + limit=limit, + ) + ) + + self.assertLessEqual( + len(deltas), limit, f"Returned {len(deltas)} rows, expected at most {limit}" + ) + + # Advancing from the clipped point should eventually drain the remainder. + # Make sure we make progress and don’t get stuck. + if deltas: + next_prev = clipped_stream_id + next_clipped, next_deltas = self.get_success( + self.store.get_partial_current_state_deltas( + prev_stream_id=next_prev, max_stream_id=max_stream_id, limit=limit + ) + ) + self.assertNotEqual( + next_clipped, clipped_stream_id, "Did not advance clipped_stream_id" + ) + # Still should respect the limit. + self.assertLessEqual(len(next_deltas), limit) + + def test_non_unique_stream_ids_in_current_state_delta_stream(self) -> None: + """ + Tests that `get_partial_current_state_deltas` always returns entire + groups of state deltas (grouped by `stream_id`), and never part of one. + + We check by passing a `limit` that to the function that, if followed + blindly, would split a group of state deltas that share a `stream_id`. + The test passes if that group is not returned at all (because doing so + would overshoot the limit of returned state deltas). + + Regression test for https://github.com/element-hq/synapse/pull/18960. + """ + # Inject a create event to start with. + self.inject_state_event( + self.room, self.alice_user_id, EventTypes.Create, "", {} + ) + + # Then inject one "real" m.room.name event. This will give us a stream_id that + # we can create some more (fake) events with. + self.inject_state_event( + self.room, + self.alice_user_id, + EventTypes.Name, + "", + {"name": "rename #1"}, + ) + + # Get the stream_id of the last-inserted event. + max_stream_id = self.store.get_room_max_stream_ordering() + + # Make 3 more state changes in the room, resulting in 5 total state + # events (including the create event, and the first name update) in + # the room. + # + # All of these state deltas have the same `stream_id` as the original name event. + # Do so by editing the table directly as that's the simplest way to have + # all share the same `stream_id`. + self.get_success( + self.store.db_pool.simple_insert_many( + "current_state_delta_stream", + keys=( + "stream_id", + "room_id", + "type", + "state_key", + "event_id", + "prev_event_id", + "instance_name", + ), + values=[ + ( + max_stream_id, + self.room.to_string(), + EventTypes.Name, + "", + f"${random_string(5)}:test", + json.dumps({"name": f"rename #{i}"}), + "master", + ) + for i in range(3) + ], + desc="inject_room_name_state_events", + ) + ) + + # Call the function under test with a limit of 4. Without the limit, we + # would return 5 state deltas: + # + # C N N N N + # 1 2 3 4 5 + # + # C = m.room.create + # N = m.room.name + # + # With the limit, we should return only the create event, as returning 4 + # state deltas would result in splitting a group: + # + # 2 3 3 3 3 - state IDs/groups + # C N N N N + # 1 2 3 4 X + + clipped_stream_id, deltas = self.get_success( + self.store.get_partial_current_state_deltas( + prev_stream_id=0, + max_stream_id=max_stream_id, + limit=4, + ) + ) + + # 2 is the stream ID of the m.room.create event. + self.assertEqual(clipped_stream_id, 2) + self.assertEqual( + len(deltas), + 1, + f"Returned {len(deltas)} rows, expected only one (the create event): {deltas}", + ) + + # Advance once more with our limit of 4. We should now get all 4 + # `m.room.name` state deltas as they can fit under the limit. + clipped_stream_id, next_deltas = self.get_success( + self.store.get_partial_current_state_deltas( + prev_stream_id=clipped_stream_id, max_stream_id=max_stream_id, limit=4 + ) + ) + self.assertEqual( + clipped_stream_id, 3 + ) # The stream ID of the 4 m.room.name events. + + self.assertEqual( + len(next_deltas), + 4, + f"Returned {len(next_deltas)} rows, expected all 4 m.room.name events: {next_deltas}", + ) + + def test_get_partial_current_state_deltas_does_not_enter_infinite_loop( + self, + ) -> None: + """ + Tests that `get_partial_current_state_deltas` does not repeatedly return + zero entries due to the passed `limit` parameter being less than the + size of the next group of state deltas from the given `prev_stream_id`. + """ + # Inject a create event to start with. + self.inject_state_event( + self.room, self.alice_user_id, EventTypes.Create, "", {} + ) + + # Then inject one "real" m.room.name event. This will give us a stream_id that + # we can create some more (fake) events with. + self.inject_state_event( + self.room, + self.alice_user_id, + EventTypes.Name, + "", + {"name": "rename #1"}, + ) + + # Get the stream_id of the last-inserted event. + max_stream_id = self.store.get_room_max_stream_ordering() + + # Make 3 more state changes in the room, resulting in 5 total state + # events (including the create event, and the first name update) in + # the room. + # + # All of these state deltas have the same `stream_id` as the original name event. + # Do so by editing the table directly as that's the simplest way to have + # all share the same `stream_id`. + self.get_success( + self.store.db_pool.simple_insert_many( + "current_state_delta_stream", + keys=( + "stream_id", + "room_id", + "type", + "state_key", + "event_id", + "prev_event_id", + "instance_name", + ), + values=[ + ( + max_stream_id, + self.room.to_string(), + EventTypes.Name, + "", + f"${random_string(5)}:test", + json.dumps({"name": f"rename #{i}"}), + "master", + ) + for i in range(3) + ], + desc="inject_room_name_state_events", + ) + ) + + # Call the function under test with a limit of 4. Without the limit, we would return + # 5 state deltas: + # + # C N N N N + # 1 2 3 4 5 + # + # C = m.room.create + # N = m.room.name + # + # With the limit, we should return only the create event, as returning 4 + # state deltas would result in splitting a group: + # + # 2 3 3 3 3 - state IDs/groups + # C N N N N + # 1 2 3 4 X + + clipped_stream_id, deltas = self.get_success( + self.store.get_partial_current_state_deltas( + prev_stream_id=2, # Start after the create event (which has stream_id 2). + max_stream_id=max_stream_id, + limit=2, # Less than the size of the next group (which is 4). + ) + ) + + self.assertEqual( + clipped_stream_id, 3 + ) # The stream ID of the 4 m.room.name events. + + # We should get all 4 `m.room.name` state deltas, instead of 0, which + # would result in the caller entering an infinite loop. + self.assertEqual( + len(deltas), + 4, + f"Returned {len(deltas)} rows, expected 4 even though it broke our limit: {deltas}", + ) From 52089f1f790174908963233922329c9afe558ccc Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 27 Nov 2025 16:15:06 +0000 Subject: [PATCH 017/139] Prevent `lint-newsfile` job activating when fixing dependabot PR branches (#19220) --- .github/workflows/tests.yml | 3 ++- changelog.d/19220.misc | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19220.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 771474f74e3..dd183840efc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -192,7 +192,8 @@ jobs: run: scripts-dev/check_line_terminators.sh lint-newsfile: - if: ${{ (github.base_ref == 'develop' || contains(github.base_ref, 'release-')) && github.actor != 'dependabot[bot]' }} + # Only run on pull_request events, targeting develop/release branches, and skip when the PR author is dependabot[bot]. + if: ${{ github.event_name == 'pull_request' && (github.base_ref == 'develop' || contains(github.base_ref, 'release-')) && github.event.pull_request.user.login != 'dependabot[bot]' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 diff --git a/changelog.d/19220.misc b/changelog.d/19220.misc new file mode 100644 index 00000000000..e98f5ade610 --- /dev/null +++ b/changelog.d/19220.misc @@ -0,0 +1 @@ +Prevent changelog check CI running on @dependabot's PRs even when a human has modified the branch. \ No newline at end of file From 566670c363915691826b5b435c4aa7acde61b408 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 27 Nov 2025 16:44:17 +0000 Subject: [PATCH 018/139] Move `RestartDelayedEventServlet` to workers (#19207) --- changelog.d/19207.feature | 1 + docker/configure_workers_and_start.py | 1 + docs/upgrade.md | 8 ++++++ docs/workers.md | 5 +++- synapse/handlers/delayed_events.py | 25 ++++++++++++------- synapse/rest/client/delayed_events.py | 4 +-- .../storage/databases/main/delayed_events.py | 19 ++++++++++++-- 7 files changed, 49 insertions(+), 14 deletions(-) create mode 100644 changelog.d/19207.feature diff --git a/changelog.d/19207.feature b/changelog.d/19207.feature new file mode 100644 index 00000000000..e64562c350b --- /dev/null +++ b/changelog.d/19207.feature @@ -0,0 +1 @@ +Allow restarting delayed event timeouts on workers. diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index e19b0a00392..e7cbd701b8e 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -196,6 +196,7 @@ "^/_matrix/client/(api/v1|r0|v3|unstable)/keys/upload", "^/_matrix/client/(api/v1|r0|v3|unstable)/keys/device_signing/upload$", "^/_matrix/client/(api/v1|r0|v3|unstable)/keys/signatures/upload$", + "^/_matrix/client/unstable/org.matrix.msc4140/delayed_events(/.*/restart)?$", ], "shared_extra_conf": {}, "worker_extra_conf": "", diff --git a/docs/upgrade.md b/docs/upgrade.md index 350b71fe479..5e7fa31580c 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -119,6 +119,14 @@ stacking them up. You can monitor the currently running background updates with # Upgrading to v1.144.0 +## Worker support for unstable MSC4140 `/restart` endpoint + +The following unstable endpoint pattern may now be routed to worker processes: + +``` +^/_matrix/client/unstable/org.matrix.msc4140/delayed_events/.*/restart$ +``` + ## Unstable mutual rooms endpoint is now behind an experimental feature flag The unstable mutual rooms endpoint from diff --git a/docs/workers.md b/docs/workers.md index f766b402513..2bc8afa74f6 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -285,10 +285,13 @@ information. # User directory search requests ^/_matrix/client/(r0|v3|unstable)/user_directory/search$ + # Unstable MSC4140 support + ^/_matrix/client/unstable/org.matrix.msc4140/delayed_events(/.*/restart)?$ + Additionally, the following REST endpoints can be handled for GET requests: + # Push rules requests ^/_matrix/client/(api/v1|r0|v3|unstable)/pushrules/ - ^/_matrix/client/unstable/org.matrix.msc4140/delayed_events # Account data requests ^/_matrix/client/(r0|v3|unstable)/.*/tags diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index de21e3abbb7..8817b653161 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -96,16 +96,18 @@ async def _schedule_db_events() -> None: self.notify_new_event, ) - # Delayed events that are already marked as processed on startup might not have been - # sent properly on the last run of the server, so unmark them to send them again. + # Now process any delayed events that are due to be sent. + # + # We set `reprocess_events` to True in case any events had been + # marked as processed, but had not yet actually been sent, + # before the homeserver stopped. + # # Caveat: this will double-send delayed events that successfully persisted, but failed # to be removed from the DB table of delayed events. # TODO: To avoid double-sending, scan the timeline to find which of these events were # already sent. To do so, must store delay_ids in sent events to retrieve them later. - await self._store.unprocess_delayed_events() - events, next_send_ts = await self._store.process_timeout_delayed_events( - self._get_current_ts() + self._get_current_ts(), reprocess_events=True ) if next_send_ts: @@ -423,18 +425,23 @@ async def restart(self, request: SynapseRequest, delay_id: str) -> None: Raises: NotFoundError: if no matching delayed event could be found. """ - assert self._is_master await self._delayed_event_mgmt_ratelimiter.ratelimit( None, request.getClientAddress().host ) - await make_deferred_yieldable(self._initialized_from_db) + + # Note: We don't need to wait on `self._initialized_from_db` here as the + # events that deals with are already marked as processed. + # + # `restart_delayed_events` will skip over such events entirely. next_send_ts = await self._store.restart_delayed_event( delay_id, self._get_current_ts() ) - if self._next_send_ts_changed(next_send_ts): - self._schedule_next_at(next_send_ts) + # Only the main process handles sending delayed events. + if self._is_master: + if self._next_send_ts_changed(next_send_ts): + self._schedule_next_at(next_send_ts) async def send(self, request: SynapseRequest, delay_id: str) -> None: """ diff --git a/synapse/rest/client/delayed_events.py b/synapse/rest/client/delayed_events.py index 69d1013e728..7afecffe2d0 100644 --- a/synapse/rest/client/delayed_events.py +++ b/synapse/rest/client/delayed_events.py @@ -156,10 +156,10 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - # The following can't currently be instantiated on workers. + # Most of the following can't currently be instantiated on workers. if hs.config.worker.worker_app is None: UpdateDelayedEventServlet(hs).register(http_server) CancelDelayedEventServlet(hs).register(http_server) - RestartDelayedEventServlet(hs).register(http_server) SendDelayedEventServlet(hs).register(http_server) + RestartDelayedEventServlet(hs).register(http_server) DelayedEventsServlet(hs).register(http_server) diff --git a/synapse/storage/databases/main/delayed_events.py b/synapse/storage/databases/main/delayed_events.py index 7f72be46f57..55471505154 100644 --- a/synapse/storage/databases/main/delayed_events.py +++ b/synapse/storage/databases/main/delayed_events.py @@ -259,7 +259,7 @@ async def get_all_delayed_events_for_user( ] async def process_timeout_delayed_events( - self, current_ts: Timestamp + self, current_ts: Timestamp, reprocess_events: bool = False ) -> tuple[ list[DelayedEventDetails], Timestamp | None, @@ -268,6 +268,16 @@ async def process_timeout_delayed_events( Marks for processing all delayed events that should have been sent prior to the provided time that haven't already been marked as such. + Args: + current_ts: The current timestamp. + reprocess_events: Whether to reprocess already-processed delayed + events. If set to True, events which are marked as processed + will have their `send_ts` re-checked. + + This is mainly useful for recovering from a server restart; + which could have occurred between an event being marked as + processed and the event actually being sent. + Returns: The details of all newly-processed delayed events, and the send time of the next delayed event to be sent, if any. """ @@ -292,7 +302,12 @@ def process_timeout_delayed_events_txn( ) ) sql_update = "UPDATE delayed_events SET is_processed = TRUE" - sql_where = "WHERE send_ts <= ? AND NOT is_processed" + sql_where = "WHERE send_ts <= ?" + + if not reprocess_events: + # Skip already-processed events. + sql_where += " AND NOT is_processed" + sql_args = (current_ts,) sql_order = "ORDER BY send_ts" if isinstance(self.database_engine, PostgresEngine): From 78ec3043d645a546411ef994ace7370653baa77c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 28 Nov 2025 15:49:15 +0000 Subject: [PATCH 019/139] Use sqlglot to properly check SQL delta files (#19224) Rather than using dodgy regexes which keep breaking. Also fixes a regression where it looks like we didn't fail CI if the delta was in the wrong place. --- .github/workflows/tests.yml | 2 +- changelog.d/19224.misc | 1 + poetry.lock | 62 ++++++++----- pyproject.toml | 3 + scripts-dev/check_schema_delta.py | 145 ++++++++++++++++++------------ 5 files changed, 131 insertions(+), 82 deletions(-) create mode 100644 changelog.d/19224.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dd183840efc..c32d018a640 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -110,7 +110,7 @@ jobs: - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: "3.x" - - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'" + - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20' 'sqlglot>=28.0.0'" - run: scripts-dev/check_schema_delta.py --force-colors check-lockfile: diff --git a/changelog.d/19224.misc b/changelog.d/19224.misc new file mode 100644 index 00000000000..3f8f630c5ee --- /dev/null +++ b/changelog.d/19224.misc @@ -0,0 +1 @@ +Improve robustness of the SQL schema linting in CI. diff --git a/poetry.lock b/poetry.lock index f723322a558..35c642fdeb6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,7 +31,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" +markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" files = [ {file = "authlib-1.6.5-py2.py3-none-any.whl", hash = "sha256:3e0e0507807f842b02175507bdee8957a1d5707fd4afb17c32fb43fee90b6e3a"}, {file = "authlib-1.6.5.tar.gz", hash = "sha256:6aaf9c79b7cc96c900f0b284061691c5d4e61221640a948fe690b556a6d6d10b"}, @@ -446,7 +446,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -471,7 +471,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "elementpath-4.1.5-py3-none-any.whl", hash = "sha256:2ac1a2fb31eb22bbbf817f8cf6752f844513216263f0e3892c8e79782fe4bb55"}, {file = "elementpath-4.1.5.tar.gz", hash = "sha256:c2d6dc524b29ef751ecfc416b0627668119d8812441c555d7471da41d4bacb8d"}, @@ -521,7 +521,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "hiredis-3.3.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:9937d9b69321b393fbace69f55423480f098120bc55a3316e1ca3508c4dbbd6f"}, {file = "hiredis-3.3.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:50351b77f89ba6a22aff430b993653847f36b71d444509036baa0f2d79d1ebf4"}, @@ -844,7 +844,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -982,7 +982,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -998,7 +998,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"url-preview\"" +markers = "extra == \"url-preview\" or extra == \"all\"" files = [ {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e77dd455b9a16bbd2a5036a63ddbd479c19572af81b624e79ef422f929eef388"}, {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d444858b9f07cefff6455b983aea9a67f7462ba1f6cbe4a21e8bf6791bf2153"}, @@ -1284,7 +1284,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "matrix-synapse-ldap3-0.3.0.tar.gz", hash = "sha256:8bb6517173164d4b9cc44f49de411d8cebdb2e705d5dd1ea1f38733c4a009e1d"}, {file = "matrix_synapse_ldap3-0.3.0-py3-none-any.whl", hash = "sha256:8b4d701f8702551e98cc1d8c20dbed532de5613584c08d0df22de376ba99159d"}, @@ -1526,7 +1526,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1716,7 +1716,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"postgres\"" +markers = "extra == \"postgres\" or extra == \"all\"" files = [ {file = "psycopg2-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:103e857f46bb76908768ead4e2d0ba1d1a130e7b8ed77d3ae91e8b33481813e8"}, {file = "psycopg2-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:210daed32e18f35e3140a1ebe059ac29209dd96468f2f7559aa59f75ee82a5cb"}, @@ -1734,7 +1734,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -1750,7 +1750,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2031,7 +2031,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"cache-memory\"" +markers = "extra == \"cache-memory\" or extra == \"all\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2091,7 +2091,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2116,7 +2116,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, @@ -2144,7 +2144,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, @@ -2548,7 +2548,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"sentry\"" +markers = "extra == \"sentry\" or extra == \"all\"" files = [ {file = "sentry_sdk-2.46.0-py2.py3-none-any.whl", hash = "sha256:4eeeb60198074dff8d066ea153fa6f241fef1668c10900ea53a4200abc8da9b1"}, {file = "sentry_sdk-2.46.0.tar.gz", hash = "sha256:91821a23460725734b7741523021601593f35731808afc0bb2ba46c27b8acd91"}, @@ -2723,6 +2723,22 @@ files = [ {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, ] +[[package]] +name = "sqlglot" +version = "28.0.0" +description = "An easily customizable SQL parser and transpiler" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "sqlglot-28.0.0-py3-none-any.whl", hash = "sha256:ac1778e7fa4812f4f7e5881b260632fc167b00ca4c1226868891fb15467122e4"}, + {file = "sqlglot-28.0.0.tar.gz", hash = "sha256:cc9a651ef4182e61dac58aa955e5fb21845a5865c6a4d7d7b5a7857450285ad4"}, +] + +[package.extras] +dev = ["duckdb (>=0.6)", "maturin (>=1.4,<2.0)", "mypy", "pandas", "pandas-stubs", "pdoc", "pre-commit", "pyperf", "python-dateutil", "pytz", "ruff (==0.7.2)", "types-python-dateutil", "types-pytz", "typing_extensions"] +rs = ["sqlglotrs (==0.7.3)"] + [[package]] name = "systemd-python" version = "235" @@ -2742,7 +2758,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -2758,7 +2774,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, ] @@ -2831,7 +2847,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "tornado-6.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:f81067dad2e4443b015368b24e802d0083fecada4f0a4572fdb72fc06e54a9a6"}, {file = "tornado-6.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9ac1cbe1db860b3cbb251e795c701c41d343f06a96049d6274e7c77559117e41"}, @@ -2965,7 +2981,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3211,7 +3227,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "xmlschema-2.4.0-py3-none-any.whl", hash = "sha256:dc87be0caaa61f42649899189aab2fd8e0d567f2cf548433ba7b79278d231a4a"}, {file = "xmlschema-2.4.0.tar.gz", hash = "sha256:d74cd0c10866ac609e1ef94a5a69b018ad16e39077bc6393408b40c6babee793"}, @@ -3346,4 +3362,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<4.0.0" -content-hash = "4f8d98723236eaf3d13f440dce95ec6cc3c4dc49ba3a0e45bf9cfbb51aca899c" +content-hash = "98b9062f48205a3bcc99b43ae665083d360a15d4a208927fa978df9c36fd5315" diff --git a/pyproject.toml b/pyproject.toml index fabc483b3c3..5ee843365d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -370,6 +370,9 @@ towncrier = ">=18.6.0rc1" # Used for checking the Poetry lockfile tomli = ">=1.2.3" +# Used for checking the schema delta files +sqlglot = ">=28.0.0" + [build-system] # The upper bounds here are defensive, intended to prevent situations like diff --git a/scripts-dev/check_schema_delta.py b/scripts-dev/check_schema_delta.py index dd96c904bb3..d3440831487 100755 --- a/scripts-dev/check_schema_delta.py +++ b/scripts-dev/check_schema_delta.py @@ -9,15 +9,11 @@ import click import git +import sqlglot +import sqlglot.expressions SCHEMA_FILE_REGEX = re.compile(r"^synapse/storage/schema/(.*)/delta/(.*)/(.*)$") -INDEX_CREATION_REGEX = re.compile( - r"CREATE .*INDEX .*ON ([a-z_0-9]+)", flags=re.IGNORECASE -) -INDEX_DELETION_REGEX = re.compile(r"DROP .*INDEX ([a-z_0-9]+)", flags=re.IGNORECASE) -TABLE_CREATION_REGEX = re.compile( - r"CREATE .*TABLE.* ([a-z_0-9]+)\s*\(", flags=re.IGNORECASE -) + # The base branch we want to check against. We use the main development branch # on the assumption that is what we are developing against. @@ -141,6 +137,9 @@ def main(force_colors: bool) -> None: color=force_colors, ) + # Mark this run as not successful, but continue so that we report *all* + # errors. + return_code = 1 else: click.secho( f"All deltas are in the correct folder: {current_schema_version}!", @@ -153,60 +152,90 @@ def main(force_colors: bool) -> None: # and delta files are also numbered in order. changed_delta_files.sort() - # Now check that we're not trying to create or drop indices. If we want to - # do that they should be in background updates. The exception is when we - # create indices on tables we've just created. - created_tables = set() - for delta_file in changed_delta_files: - with open(delta_file) as fd: - delta_lines = fd.readlines() - - for line in delta_lines: - # Strip SQL comments - line = line.split("--", maxsplit=1)[0] - - # Check and track any tables we create - match = TABLE_CREATION_REGEX.search(line) - if match: - table_name = match.group(1) - created_tables.add(table_name) - - # Check for dropping indices, these are always banned - match = INDEX_DELETION_REGEX.search(line) - if match: - clause = match.group() - - click.secho( - f"Found delta with index deletion: '{clause}' in {delta_file}", - fg="red", - bold=True, - color=force_colors, - ) - click.secho( - " ↪ These should be in background updates.", - ) - return_code = 1 - - # Check for index creation, which is only allowed for tables we've - # created. - match = INDEX_CREATION_REGEX.search(line) - if match: - clause = match.group() - table_name = match.group(1) - if table_name not in created_tables: - click.secho( - f"Found delta with index creation for existing table: '{clause}' in {delta_file}", - fg="red", - bold=True, - color=force_colors, - ) - click.secho( - " ↪ These should be in background updates (or the table should be created in the same delta).", - ) - return_code = 1 + success = check_schema_delta(changed_delta_files, force_colors) + if not success: + return_code = 1 click.get_current_context().exit(return_code) +def check_schema_delta(delta_files: list[str], force_colors: bool) -> bool: + """Check that the given schema delta files do not create or drop indices + inappropriately. + + Index creation is only allowed on tables created in the same set of deltas. + + Index deletion is never allowed and should be done in background updates. + + Returns: + True if all checks succeeded, False if at least one failed. + """ + + # The tables created in this delta + created_tables = set[str]() + + # The indices created/dropped in this delta, each a tuple of (table_name, sql) + created_indices = list[tuple[str, str]]() + + # The indices dropped in this delta, just the sql + dropped_indices = list[str]() + + for delta_file in delta_files: + with open(delta_file) as fd: + delta_contents = fd.read() + + # Assume the SQL dialect from the file extension, defaulting to Postgres. + sql_lang = "postgres" + if delta_file.endswith(".sqlite"): + sql_lang = "sqlite" + + statements = sqlglot.parse(delta_contents, read=sql_lang) + + for statement in statements: + if isinstance(statement, sqlglot.expressions.Create): + if statement.kind == "TABLE": + assert isinstance(statement.this, sqlglot.expressions.Schema) + assert isinstance(statement.this.this, sqlglot.expressions.Table) + + table_name = statement.this.this.name + created_tables.add(table_name) + elif statement.kind == "INDEX": + assert isinstance(statement.this, sqlglot.expressions.Index) + + table_name = statement.this.args["table"].name + created_indices.append((table_name, statement.sql())) + elif isinstance(statement, sqlglot.expressions.Drop): + if statement.kind == "INDEX": + dropped_indices.append(statement.sql()) + + success = True + for table_name, clause in created_indices: + if table_name not in created_tables: + click.secho( + f"Found delta with index creation for existing table: '{clause}'", + fg="red", + bold=True, + color=force_colors, + ) + click.secho( + " ↪ These should be in background updates (or the table should be created in the same delta).", + ) + success = False + + for clause in dropped_indices: + click.secho( + f"Found delta with index deletion: '{clause}'", + fg="red", + bold=True, + color=force_colors, + ) + click.secho( + " ↪ These should be in background updates.", + ) + success = False + + return success + + if __name__ == "__main__": main() From 778897a4e9a43522a1698a205f68feb3c601ee2d Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 28 Nov 2025 17:01:15 +0000 Subject: [PATCH 020/139] Add a unit test that ensures that deleting a device purges the associated refresh token (#19230) --- changelog.d/19230.misc | 1 + tests/handlers/test_device.py | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 changelog.d/19230.misc diff --git a/changelog.d/19230.misc b/changelog.d/19230.misc new file mode 100644 index 00000000000..06704db25b0 --- /dev/null +++ b/changelog.d/19230.misc @@ -0,0 +1 @@ +Add a unit test for ensuring associated refresh tokens are erased when a device is delted. \ No newline at end of file diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index acd37a1c711..fa6bb4970be 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -449,6 +449,33 @@ def test_on_federation_query_user_devices_appservice(self) -> None: ], ) + def test_delete_device_removes_refresh_tokens(self) -> None: + """Deleting a device should also purge any refresh tokens for it.""" + self._record_users() + + self.get_success( + self.store.add_refresh_token_to_user( + user_id=user1, + token="refresh_token", + device_id="abc", + expiry_ts=None, + ultimate_session_expiry_ts=None, + ) + ) + + self.get_success(self.handler.delete_devices(user1, ["abc"])) + + remaining_refresh_token = self.get_success( + self.store.db_pool.simple_select_one( + table="refresh_tokens", + keyvalues={"user_id": user1, "device_id": "abc"}, + retcols=("id",), + desc="get_refresh_token_for_device", + allow_none=True, + ) + ) + self.assertIsNone(remaining_refresh_token) + class DehydrationTestCase(unittest.HomeserverTestCase): servlets = [ From 034c5e625c0ae63033a49a5288ee0b65b53dc0f4 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 28 Nov 2025 17:41:56 +0000 Subject: [PATCH 021/139] Move call invite filtering logic to `filter_events_for_client` (#17782) --- changelog.d/17782.misc | 1 + synapse/handlers/sliding_sync/__init__.py | 2 -- synapse/handlers/sync.py | 12 +------ synapse/visibility.py | 38 +++++++++++++++++++++-- 4 files changed, 38 insertions(+), 15 deletions(-) create mode 100644 changelog.d/17782.misc diff --git a/changelog.d/17782.misc b/changelog.d/17782.misc new file mode 100644 index 00000000000..d7321470d01 --- /dev/null +++ b/changelog.d/17782.misc @@ -0,0 +1 @@ +Improve event filtering for Simplified Sliding Sync. \ No newline at end of file diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index 6a5d5c7b3cc..68135e9cd3a 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -761,8 +761,6 @@ async def get_room_sync_data( != Membership.JOIN, filter_send_to_client=True, ) - # TODO: Filter out `EventTypes.CallInvite` in public rooms, - # see https://github.com/element-hq/synapse/issues/17359 # TODO: Handle timeline gaps (`get_timeline_gaps()`) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index b534e24698f..60d88274255 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -36,7 +36,6 @@ Direction, EventContentFields, EventTypes, - JoinRules, Membership, ) from synapse.api.filtering import FilterCollection @@ -790,22 +789,13 @@ async def _load_filtered_recents( ) ) - filtered_recents = await filter_events_for_client( + loaded_recents = await filter_events_for_client( self._storage_controllers, sync_config.user.to_string(), loaded_recents, always_include_ids=current_state_ids, ) - loaded_recents = [] - for event in filtered_recents: - if event.type == EventTypes.CallInvite: - room_info = await self.store.get_room_with_stats(event.room_id) - assert room_info is not None - if room_info.join_rules == JoinRules.PUBLIC: - continue - loaded_recents.append(event) - log_kv({"loaded_recents_after_client_filtering": len(loaded_recents)}) loaded_recents.extend(recents) diff --git a/synapse/visibility.py b/synapse/visibility.py index 16b39e62008..bfa0db5670d 100644 --- a/synapse/visibility.py +++ b/synapse/visibility.py @@ -33,6 +33,7 @@ EventTypes, EventUnsignedContentFields, HistoryVisibility, + JoinRules, Membership, ) from synapse.events import EventBase @@ -111,7 +112,17 @@ async def filter_events_for_client( # happen within the function. events_before_filtering = events.copy() # Default case is to *exclude* soft-failed events - events = [e for e in events if not e.internal_metadata.is_soft_failed()] + events = [] + found_call_invite = False + for event in events_before_filtering: + if event.internal_metadata.is_soft_failed(): + continue + + if event.type == EventTypes.CallInvite and not event.is_state(): + found_call_invite = True + + events.append(event) + client_config = await storage.main.get_admin_client_config_for_user(user_id) if filter_send_to_client and await storage.main.is_server_admin(user_id): if client_config.return_soft_failed_events: @@ -139,7 +150,11 @@ async def filter_events_for_client( [event.event_id for event in events], ) - types = (_HISTORY_VIS_KEY, (EventTypes.Member, user_id)) + types = [_HISTORY_VIS_KEY, (EventTypes.Member, user_id)] + if found_call_invite: + # We need to fetch the room's join rules state to determine + # whether to allow call invites in public rooms. + types.append((EventTypes.JoinRules, "")) # we exclude outliers at this point, and then handle them separately later event_id_to_state = await storage.state.get_state_for_events( @@ -178,6 +193,25 @@ def allowed(event: EventBase) -> EventBase | None: if filtered is None: return None + # Filter out call invites in public rooms, as this would potentially + # ring a lot of users. + if event.type == EventTypes.CallInvite and not event.is_state(): + # `state_after_event` should only be None if the event is an outlier, + # and earlier code should filter out outliers entirely. + # + # In addition, we only create outliers locally for out-of-band + # invite rejections, invites received over federation, or state + # events needed to authorise other events. None of this applies to + # call invites. + assert state_after_event is not None + + room_join_rules = state_after_event.get((EventTypes.JoinRules, "")) + if ( + room_join_rules is not None + and room_join_rules.content.get("join_rule") == JoinRules.PUBLIC + ): + return None + # Annotate the event with the user's membership after the event. # # Normally we just look in `state_after_event`, but if the event is an outlier From d143276bda3b14cf1f9a05dfcb26a2380d472847 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 1 Dec 2025 13:34:21 +0000 Subject: [PATCH 022/139] Fix rust source check when using .egg-info (#19251) We have checks to try and catch the case where Synapse is being run from a source directory, but the compiled Rust code is out-of-date. This commonly happens when Synapse is updated without running `poetry install` (or equivalent). These checks did not correctly handle `.egg-info` installs, and so were not run. Currently, the `.egg-info` directory is created automatically by poetry (due to using setuptools to build Rust). --- changelog.d/19251.misc | 1 + synapse/util/rust.py | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19251.misc diff --git a/changelog.d/19251.misc b/changelog.d/19251.misc new file mode 100644 index 00000000000..9d0501c3d4e --- /dev/null +++ b/changelog.d/19251.misc @@ -0,0 +1 @@ +Fix check of the Rust compiled code being outdated when using source checkout and `.egg-info`. diff --git a/synapse/util/rust.py b/synapse/util/rust.py index 63b53b917f0..d1e1a259e42 100644 --- a/synapse/util/rust.py +++ b/synapse/util/rust.py @@ -111,7 +111,38 @@ def get_synapse_source_directory() -> str | None: # c.f. https://packaging.python.org/en/latest/specifications/direct-url/ direct_url_json = package.read_text("direct_url.json") if direct_url_json is None: - return None + # No direct url metadata. Check if this is an egg-info install. + # + # An egg-info install is when there exists a `matrix_synapse.egg-info` + # directory alongside the source tree, containing the package metadata. + # This allows discovering packages in the current directory, without + # installing them properly to the environment wide `site-packages` + # directory. + # + # When searching for a package, Python will look for `.egg-info` files + # in the current working directory before looking in `site-packages`. + # This means that when running Synapse (or the tests) from the source + # tree Python will pick up the synapse package from the egg-info + # install. + # + # Poetry will create an egg-info install when running `poetry install`. + # + # The combination of the above means that it is very common for + # developers (e.g. running tests) to encounter egg-info installs. + # + # In this case we can find the source tree by looking for the + # `matrix_synapse.egg-info/PKG-INFO` file, and going up two directories + # from there. + + metadata_path = package.locate_file("matrix_synapse.egg-info/PKG-INFO") + if not os.path.exists(str(metadata_path)): + # Not an egg-info install. + return None + + # `metadata_path` points to the egg-info/PKG-INFO file, so go up two + # directories to get the root of the source tree. + source_dir = metadata_path.parent.parent + return os.fspath(source_dir) # c.f. https://packaging.python.org/en/latest/specifications/direct-url/ for # the format From 1bddd25a85d82b2ef4a2a42f6ecd476108d7dd96 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 1 Dec 2025 13:55:06 +0000 Subject: [PATCH 023/139] Port `Clock` functions to use `Duration` class (#19229) This changes the arguments in clock functions to be `Duration` and converts call sites and constants into `Duration`. There are still some more functions around that should be converted (e.g. `timeout_deferred`), but we leave that to another PR. We also changes `.as_secs()` to return a float, as the rounding broke things subtly. The only reason to keep it (its the same as `timedelta.total_seconds()`) is for symmetry with `as_millis()`. Follows on from https://github.com/element-hq/synapse/pull/19223 --- changelog.d/19229.misc | 1 + rust/src/duration.rs | 56 +++++++++++++ rust/src/lib.rs | 1 + rust/src/rendezvous/mod.rs | 5 +- synapse/api/ratelimiting.py | 3 +- synapse/app/phone_stats_home.py | 12 +-- synapse/appservice/scheduler.py | 3 +- synapse/federation/federation_client.py | 3 +- synapse/federation/federation_server.py | 7 +- synapse/federation/send_queue.py | 3 +- synapse/federation/sender/__init__.py | 13 +-- synapse/handlers/account_validity.py | 3 +- synapse/handlers/auth.py | 3 +- synapse/handlers/delayed_events.py | 9 ++- synapse/handlers/device.py | 13 ++- synapse/handlers/e2e_keys.py | 3 +- synapse/handlers/federation.py | 5 +- synapse/handlers/federation_event.py | 3 +- synapse/handlers/message.py | 18 ++--- synapse/handlers/pagination.py | 3 +- synapse/handlers/presence.py | 17 ++-- synapse/handlers/profile.py | 3 +- synapse/handlers/room.py | 3 +- synapse/handlers/room_member.py | 9 ++- synapse/handlers/stats.py | 3 +- synapse/handlers/typing.py | 26 +++--- synapse/handlers/user_directory.py | 19 +++-- synapse/handlers/worker_lock.py | 4 +- synapse/http/client.py | 5 +- .../http/federation/well_known_resolver.py | 3 +- synapse/http/server.py | 3 +- synapse/media/media_repository.py | 9 ++- synapse/media/media_storage.py | 5 +- synapse/media/url_previewer.py | 5 +- synapse/metrics/common_usage_metrics.py | 3 +- synapse/module_api/__init__.py | 8 +- synapse/notifier.py | 11 ++- synapse/push/emailpusher.py | 3 +- synapse/push/httppusher.py | 5 +- synapse/replication/http/_base.py | 5 +- synapse/replication/tcp/client.py | 3 +- synapse/replication/tcp/protocol.py | 5 +- synapse/replication/tcp/redis.py | 3 +- synapse/replication/tcp/resource.py | 3 +- synapse/rest/client/account.py | 13 ++- synapse/rest/client/register.py | 9 ++- synapse/rest/client/transactions.py | 2 +- synapse/state/__init__.py | 3 +- synapse/state/v2.py | 15 ++-- synapse/storage/background_updates.py | 3 +- synapse/storage/controllers/purge_events.py | 3 +- synapse/storage/database.py | 7 +- synapse/storage/databases/main/cache.py | 13 +-- .../storage/databases/main/censor_events.py | 3 +- synapse/storage/databases/main/client_ips.py | 5 +- synapse/storage/databases/main/deviceinbox.py | 4 +- synapse/storage/databases/main/devices.py | 3 +- .../databases/main/event_federation.py | 7 +- .../databases/main/event_push_actions.py | 11 ++- .../storage/databases/main/events_worker.py | 3 +- synapse/storage/databases/main/lock.py | 11 +-- synapse/storage/databases/main/metrics.py | 3 +- .../storage/databases/main/registration.py | 9 +-- synapse/storage/databases/main/roommember.py | 5 +- synapse/storage/databases/main/session.py | 3 +- .../storage/databases/main/sliding_sync.py | 2 +- .../storage/databases/main/transactions.py | 3 +- synapse/util/async_helpers.py | 7 +- synapse/util/batching_queue.py | 3 +- synapse/util/caches/expiringcache.py | 3 +- synapse/util/caches/lrucache.py | 7 +- synapse/util/caches/response_cache.py | 7 +- synapse/util/clock.py | 33 ++++---- synapse/util/duration.py | 81 ++++++++++++++++++- synapse/util/ratelimitutils.py | 7 +- synapse/util/task_scheduler.py | 15 ++-- synmark/suites/logging.py | 3 +- .../federation/transport/server/test__base.py | 5 +- tests/handlers/test_device.py | 2 +- tests/handlers/test_typing.py | 2 +- tests/http/test_servlet.py | 5 +- tests/logging/test_opentracing.py | 9 ++- tests/replication/http/test__base.py | 5 +- tests/rest/admin/test_background_updates.py | 3 +- tests/rest/admin/test_room.py | 7 +- tests/rest/client/test_transactions.py | 3 +- tests/server_notices/__init__.py | 3 +- tests/state/test_v2.py | 3 +- tests/state/test_v21.py | 3 +- tests/storage/databases/main/test_lock.py | 4 +- tests/storage/test_background_update.py | 5 +- tests/test_server.py | 9 ++- tests/util/caches/test_response_cache.py | 5 +- tests/util/test_logcontext.py | 55 +++++++------ tests/util/test_task_scheduler.py | 7 +- 95 files changed, 511 insertions(+), 260 deletions(-) create mode 100644 changelog.d/19229.misc create mode 100644 rust/src/duration.rs diff --git a/changelog.d/19229.misc b/changelog.d/19229.misc new file mode 100644 index 00000000000..8caebead72a --- /dev/null +++ b/changelog.d/19229.misc @@ -0,0 +1 @@ +Move towards using a dedicated `Duration` type. diff --git a/rust/src/duration.rs b/rust/src/duration.rs new file mode 100644 index 00000000000..a3dbe919b28 --- /dev/null +++ b/rust/src/duration.rs @@ -0,0 +1,56 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2025 Element Creations, Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + */ + +use once_cell::sync::OnceCell; +use pyo3::{ + types::{IntoPyDict, PyAnyMethods}, + Bound, BoundObject, IntoPyObject, Py, PyAny, PyErr, PyResult, Python, +}; + +/// A reference to the `synapse.util.duration` module. +static DURATION: OnceCell> = OnceCell::new(); + +/// Access to the `synapse.util.duration` module. +fn duration_module(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + Ok(DURATION + .get_or_try_init(|| py.import("synapse.util.duration").map(Into::into))? + .bind(py)) +} + +/// Mirrors the `synapse.util.duration.Duration` Python class. +pub struct SynapseDuration { + microseconds: u64, +} + +impl SynapseDuration { + /// For now we only need to create durations from milliseconds. + pub fn from_milliseconds(milliseconds: u64) -> Self { + Self { + microseconds: milliseconds * 1_000, + } + } +} + +impl<'py> IntoPyObject<'py> for &SynapseDuration { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let duration_module = duration_module(py)?; + let kwargs = [("microseconds", self.microseconds)].into_py_dict(py)?; + let duration_instance = duration_module.call_method("Duration", (), Some(&kwargs))?; + Ok(duration_instance.into_bound()) + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 6522148fa15..fe880af2eae 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -5,6 +5,7 @@ use pyo3::prelude::*; use pyo3_log::ResetHandle; pub mod acl; +pub mod duration; pub mod errors; pub mod events; pub mod http; diff --git a/rust/src/rendezvous/mod.rs b/rust/src/rendezvous/mod.rs index 848b5035bbc..9a6da9fcc32 100644 --- a/rust/src/rendezvous/mod.rs +++ b/rust/src/rendezvous/mod.rs @@ -35,6 +35,7 @@ use ulid::Ulid; use self::session::Session; use crate::{ + duration::SynapseDuration, errors::{NotFoundError, SynapseError}, http::{http_request_from_twisted, http_response_to_twisted, HeaderMapPyExt}, UnwrapInfallible, @@ -132,6 +133,8 @@ impl RendezvousHandler { .unwrap_infallible() .unbind(); + let eviction_duration = SynapseDuration::from_milliseconds(eviction_interval); + // Construct a Python object so that we can get a reference to the // evict method and schedule it to run. let self_ = Py::new( @@ -149,7 +152,7 @@ impl RendezvousHandler { let evict = self_.getattr(py, "_evict")?; homeserver.call_method0("get_clock")?.call_method( "looping_call", - (evict, eviction_interval), + (evict, &eviction_duration), None, )?; diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py index df884d47d75..d6cc3d26b5f 100644 --- a/synapse/api/ratelimiting.py +++ b/synapse/api/ratelimiting.py @@ -27,6 +27,7 @@ from synapse.storage.databases.main import DataStore from synapse.types import Requester from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.wheel_timer import WheelTimer if TYPE_CHECKING: @@ -100,7 +101,7 @@ def __init__( # and doesn't affect correctness. self._timer: WheelTimer[Hashable] = WheelTimer() - self.clock.looping_call(self._prune_message_counts, 15 * 1000) + self.clock.looping_call(self._prune_message_counts, Duration(seconds=15)) def _get_key(self, requester: Requester | None, key: Hashable | None) -> Hashable: """Use the requester's MXID as a fallback key if no key is provided.""" diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index d278e30850b..7b4bf25c280 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -218,13 +218,13 @@ def performance_stats_init() -> None: # table will decrease clock.looping_call( hs.get_datastores().main.generate_user_daily_visits, - Duration(minutes=5).as_millis(), + Duration(minutes=5), ) # monthly active user limiting functionality clock.looping_call( hs.get_datastores().main.reap_monthly_active_users, - Duration(hours=1).as_millis(), + Duration(hours=1), ) hs.get_datastores().main.reap_monthly_active_users() @@ -263,14 +263,14 @@ async def _generate_monthly_active_users() -> None: if hs.config.server.limit_usage_by_mau or hs.config.server.mau_stats_only: generate_monthly_active_users() - clock.looping_call(generate_monthly_active_users, 5 * 60 * 1000) + clock.looping_call(generate_monthly_active_users, Duration(minutes=5)) # End of monthly active user settings if hs.config.metrics.report_stats: logger.info("Scheduling stats reporting for 3 hour intervals") clock.looping_call( phone_stats_home, - PHONE_HOME_INTERVAL.as_millis(), + PHONE_HOME_INTERVAL, hs, stats, ) @@ -278,14 +278,14 @@ async def _generate_monthly_active_users() -> None: # We need to defer this init for the cases that we daemonize # otherwise the process ID we get is that of the non-daemon process clock.call_later( - 0, + Duration(seconds=0), performance_stats_init, ) # We wait 5 minutes to send the first set of stats as the server can # be quite busy the first few minutes clock.call_later( - INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME.as_secs(), + INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME, phone_stats_home, hs, stats, diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 250f84d6445..befb4ae44b0 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -77,6 +77,7 @@ from synapse.storage.databases.main import DataStore from synapse.types import DeviceListUpdates, JsonMapping from synapse.util.clock import Clock, DelayedCallWrapper +from synapse.util.duration import Duration if TYPE_CHECKING: from synapse.server import HomeServer @@ -504,7 +505,7 @@ def __init__( self.scheduled_recovery: DelayedCallWrapper | None = None def recover(self) -> None: - delay = 2**self.backoff_counter + delay = Duration(seconds=2**self.backoff_counter) logger.info("Scheduling retries on %s in %fs", self.service.id, delay) self.scheduled_recovery = self.clock.call_later( delay, diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 4110a90ed6d..ba738ad65e7 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -75,6 +75,7 @@ from synapse.types.handlers.policy_server import RECOMMENDATION_OK, RECOMMENDATION_SPAM from synapse.util.async_helpers import concurrently_execute from synapse.util.caches.expiringcache import ExpiringCache +from synapse.util.duration import Duration from synapse.util.retryutils import NotRetryingDestination if TYPE_CHECKING: @@ -132,7 +133,7 @@ def __init__(self, hs: "HomeServer"): super().__init__(hs) self.pdu_destination_tried: dict[str, dict[str, int]] = {} - self._clock.looping_call(self._clear_tried_cache, 60 * 1000) + self._clock.looping_call(self._clear_tried_cache, Duration(minutes=1)) self.state = hs.get_state_handler() self.transport_layer = hs.get_federation_transport_client() diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 34abac1cec9..b909f1e5956 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -89,6 +89,7 @@ from synapse.util import unwrapFirstError from synapse.util.async_helpers import Linearizer, concurrently_execute, gather_results from synapse.util.caches.response_cache import ResponseCache +from synapse.util.duration import Duration from synapse.util.stringutils import parse_server_name if TYPE_CHECKING: @@ -226,7 +227,7 @@ async def _handle_old_staged_events(self) -> None: ) # We pause a bit so that we don't start handling all rooms at once. - await self._clock.sleep(random.uniform(0, 0.1)) + await self._clock.sleep(Duration(seconds=random.uniform(0, 0.1))) async def on_backfill_request( self, origin: str, room_id: str, versions: list[str], limit: int @@ -301,7 +302,9 @@ async def on_incoming_transaction( # Start a periodic check for old staged events. This is to handle # the case where locks time out, e.g. if another process gets killed # without dropping its locks. - self._clock.looping_call(self._handle_old_staged_events, 60 * 1000) + self._clock.looping_call( + self._handle_old_staged_events, Duration(minutes=1) + ) # keep this as early as possible to make the calculated origin ts as # accurate as possible. diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py index cf70e10a58f..4a6d155217c 100644 --- a/synapse/federation/send_queue.py +++ b/synapse/federation/send_queue.py @@ -53,6 +53,7 @@ from synapse.metrics import SERVER_NAME_LABEL, LaterGauge from synapse.replication.tcp.streams.federation import FederationStream from synapse.types import JsonDict, ReadReceipt, RoomStreamToken, StrCollection +from synapse.util.duration import Duration from synapse.util.metrics import Measure from .units import Edu @@ -137,7 +138,7 @@ def register(queue_name: QueueNames, queue: Sized) -> None: assert isinstance(queue, Sized) register(queue_name, queue=queue) - self.clock.looping_call(self._clear_queue, 30 * 1000) + self.clock.looping_call(self._clear_queue, Duration(seconds=30)) def shutdown(self) -> None: """Stops this federation sender instance from sending further transactions.""" diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 0bd97c25dfb..f7240c2f7f9 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -174,6 +174,7 @@ get_domain_from_id, ) from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.metrics import Measure from synapse.util.retryutils import filter_destinations_by_retry_limiter @@ -218,12 +219,12 @@ # Please note that rate limiting still applies, so while the loop is # executed every X seconds the destinations may not be woken up because # they are being rate limited following previous attempt failures. -WAKEUP_RETRY_PERIOD_SEC = 60 +WAKEUP_RETRY_PERIOD = Duration(minutes=1) -# Time (in s) to wait in between waking up each destination, i.e. one destination +# Time to wait in between waking up each destination, i.e. one destination # will be woken up every seconds until we have woken every destination # has outstanding catch-up. -WAKEUP_INTERVAL_BETWEEN_DESTINATIONS_SEC = 5 +WAKEUP_INTERVAL_BETWEEN_DESTINATIONS = Duration(seconds=5) class AbstractFederationSender(metaclass=abc.ABCMeta): @@ -379,7 +380,7 @@ async def _handle(self) -> None: queue.attempt_new_transaction() - await self.clock.sleep(current_sleep_seconds) + await self.clock.sleep(Duration(seconds=current_sleep_seconds)) if not self.queue: break @@ -468,7 +469,7 @@ def __init__(self, hs: "HomeServer"): # Regularly wake up destinations that have outstanding PDUs to be caught up self.clock.looping_call_now( self.hs.run_as_background_process, - WAKEUP_RETRY_PERIOD_SEC * 1000.0, + WAKEUP_RETRY_PERIOD, "wake_destinations_needing_catchup", self._wake_destinations_needing_catchup, ) @@ -1161,4 +1162,4 @@ async def _wake_destinations_needing_catchup(self) -> None: last_processed, ) self.wake_destination(destination) - await self.clock.sleep(WAKEUP_INTERVAL_BETWEEN_DESTINATIONS_SEC) + await self.clock.sleep(WAKEUP_INTERVAL_BETWEEN_DESTINATIONS) diff --git a/synapse/handlers/account_validity.py b/synapse/handlers/account_validity.py index bc50efa1a7b..ba40d5763ef 100644 --- a/synapse/handlers/account_validity.py +++ b/synapse/handlers/account_validity.py @@ -28,6 +28,7 @@ from synapse.types import UserID from synapse.util import stringutils from synapse.util.async_helpers import delay_cancellation +from synapse.util.duration import Duration if TYPE_CHECKING: from synapse.server import HomeServer @@ -73,7 +74,7 @@ def __init__(self, hs: "HomeServer"): # Check the renewal emails to send and send them every 30min. if hs.config.worker.run_background_tasks: - self.clock.looping_call(self._send_renewal_emails, 30 * 60 * 1000) + self.clock.looping_call(self._send_renewal_emails, Duration(minutes=30)) async def is_user_expired(self, user_id: str) -> bool: """Checks if a user has expired against third-party modules. diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index d9355d33da9..b5c0cbdba23 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -74,6 +74,7 @@ from synapse.types import JsonDict, Requester, StrCollection, UserID from synapse.util import stringutils as stringutils from synapse.util.async_helpers import delay_cancellation, maybe_awaitable +from synapse.util.duration import Duration from synapse.util.msisdn import phone_number_to_msisdn from synapse.util.stringutils import base62_encode from synapse.util.threepids import canonicalise_email @@ -242,7 +243,7 @@ def __init__(self, hs: "HomeServer"): if hs.config.worker.run_background_tasks: self._clock.looping_call( run_as_background_process, - 5 * 60 * 1000, + Duration(minutes=5), "expire_old_sessions", self.server_name, self._expire_old_sessions, diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index 8817b653161..cb0a4dd6b23 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -42,6 +42,7 @@ UserID, create_requester, ) +from synapse.util.duration import Duration from synapse.util.events import generate_fake_event_id from synapse.util.metrics import Measure from synapse.util.sentinel import Sentinel @@ -92,7 +93,7 @@ async def _schedule_db_events() -> None: # Kick off again (without blocking) to catch any missed notifications # that may have fired before the callback was added. self._clock.call_later( - 0, + Duration(seconds=0), self.notify_new_event, ) @@ -508,17 +509,17 @@ def _schedule_next_at_or_none(self, next_send_ts: Timestamp | None) -> None: def _schedule_next_at(self, next_send_ts: Timestamp) -> None: delay = next_send_ts - self._get_current_ts() - delay_sec = delay / 1000 if delay > 0 else 0 + delay_duration = Duration(milliseconds=max(delay, 0)) if self._next_delayed_event_call is None: self._next_delayed_event_call = self._clock.call_later( - delay_sec, + delay_duration, self.hs.run_as_background_process, "_send_on_timeout", self._send_on_timeout, ) else: - self._next_delayed_event_call.reset(delay_sec) + self._next_delayed_event_call.reset(delay_duration.as_secs()) async def get_all_for_user(self, requester: Requester) -> list[JsonDict]: """Return all pending delayed events requested by the given user.""" diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 3f1a5fe6d6e..1b7de57ab93 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -71,6 +71,7 @@ from synapse.util.async_helpers import Linearizer from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.cancellation import cancellable +from synapse.util.duration import Duration from synapse.util.metrics import measure_func from synapse.util.retryutils import ( NotRetryingDestination, @@ -85,7 +86,7 @@ DELETE_DEVICE_MSGS_TASK_NAME = "delete_device_messages" MAX_DEVICE_DISPLAY_NAME_LEN = 100 -DELETE_STALE_DEVICES_INTERVAL_MS = 24 * 60 * 60 * 1000 +DELETE_STALE_DEVICES_INTERVAL = Duration(days=1) def _check_device_name_length(name: str | None) -> None: @@ -186,7 +187,7 @@ def __init__(self, hs: "HomeServer"): ): self.clock.looping_call( self.hs.run_as_background_process, - DELETE_STALE_DEVICES_INTERVAL_MS, + DELETE_STALE_DEVICES_INTERVAL, desc="delete_stale_devices", func=self._delete_stale_devices, ) @@ -915,7 +916,7 @@ async def handle_new_device_update(self) -> None: ) DEVICE_MSGS_DELETE_BATCH_LIMIT = 1000 - DEVICE_MSGS_DELETE_SLEEP_MS = 100 + DEVICE_MSGS_DELETE_SLEEP = Duration(milliseconds=100) async def _delete_device_messages( self, @@ -941,9 +942,7 @@ async def _delete_device_messages( if from_stream_id is None: return TaskStatus.COMPLETE, None, None - await self.clock.sleep( - DeviceWriterHandler.DEVICE_MSGS_DELETE_SLEEP_MS / 1000.0 - ) + await self.clock.sleep(DeviceWriterHandler.DEVICE_MSGS_DELETE_SLEEP) class DeviceWriterHandler(DeviceHandler): @@ -1469,7 +1468,7 @@ def __init__(self, hs: "HomeServer", device_handler: DeviceWriterHandler): self._resync_retry_lock = Lock() self.clock.looping_call( self.hs.run_as_background_process, - 30 * 1000, + Duration(seconds=30), func=self._maybe_retry_device_resync, desc="_maybe_retry_device_resync", ) diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index 41d27d47da1..64f705a3dad 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -46,6 +46,7 @@ ) from synapse.util.async_helpers import Linearizer, concurrently_execute from synapse.util.cancellation import cancellable +from synapse.util.duration import Duration from synapse.util.json import json_decoder from synapse.util.retryutils import ( NotRetryingDestination, @@ -1634,7 +1635,7 @@ async def _delete_old_one_time_keys_task( # matrix.org has about 15M users in the e2e_one_time_keys_json table # (comprising 20M devices). We want this to take about a week, so we need # to do about one batch of 100 users every 4 seconds. - await self.clock.sleep(4) + await self.clock.sleep(Duration(seconds=4)) def _check_cross_signing_key( diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 1bba3fc758e..7808f8928b0 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -72,6 +72,7 @@ from synapse.types import JsonDict, StrCollection, get_domain_from_id from synapse.types.state import StateFilter from synapse.util.async_helpers import Linearizer +from synapse.util.duration import Duration from synapse.util.retryutils import NotRetryingDestination from synapse.visibility import filter_events_for_server @@ -1972,7 +1973,9 @@ async def _sync_partial_state_room( logger.warning( "%s; waiting for %d ms...", e, e.retry_after_ms ) - await self.clock.sleep(e.retry_after_ms / 1000) + await self.clock.sleep( + Duration(milliseconds=e.retry_after_ms) + ) # Success, no need to try the rest of the destinations. break diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 01e98f60ada..e314180e122 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -91,6 +91,7 @@ ) from synapse.types.state import StateFilter from synapse.util.async_helpers import Linearizer, concurrently_execute +from synapse.util.duration import Duration from synapse.util.iterutils import batch_iter, partition, sorted_topologically from synapse.util.retryutils import NotRetryingDestination from synapse.util.stringutils import shortstr @@ -1802,7 +1803,7 @@ async def prep(event: EventBase) -> None: # the reactor. For large rooms let's yield to the reactor # occasionally to ensure we don't block other work. if (i + 1) % 1000 == 0: - await self._clock.sleep(0) + await self._clock.sleep(Duration(seconds=0)) # Also persist the new event in batches for similar reasons as above. for batch in batch_iter(events_and_contexts_to_persist, 1000): diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 7679303a36d..bac4bd9361b 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -83,6 +83,7 @@ from synapse.util import log_failure, unwrapFirstError from synapse.util.async_helpers import Linearizer, gather_results from synapse.util.caches.expiringcache import ExpiringCache +from synapse.util.duration import Duration from synapse.util.json import json_decoder, json_encoder from synapse.util.metrics import measure_func from synapse.visibility import get_effective_room_visibility_from_state @@ -433,14 +434,11 @@ def _schedule_expiry_for_event(self, event_id: str, expiry_ts: int) -> None: # Figure out how many seconds we need to wait before expiring the event. now_ms = self.clock.time_msec() - delay = (expiry_ts - now_ms) / 1000 + delay = Duration(milliseconds=max(expiry_ts - now_ms, 0)) - # callLater doesn't support negative delays, so trim the delay to 0 if we're - # in that case. - if delay < 0: - delay = 0 - - logger.info("Scheduling expiry for event %s in %.3fs", event_id, delay) + logger.info( + "Scheduling expiry for event %s in %.3fs", event_id, delay.as_secs() + ) self._scheduled_expiry = self.clock.call_later( delay, @@ -551,7 +549,7 @@ def __init__(self, hs: "HomeServer"): "send_dummy_events_to_fill_extremities", self._send_dummy_events_to_fill_extremities, ), - 5 * 60 * 1000, + Duration(minutes=5), ) self._message_handler = hs.get_message_handler() @@ -1012,7 +1010,7 @@ async def create_and_send_nonmember_event( if not ignore_shadow_ban and requester.shadow_banned: # We randomly sleep a bit just to annoy the requester. - await self.clock.sleep(random.randint(1, 10)) + await self.clock.sleep(Duration(seconds=random.randint(1, 10))) raise ShadowBanError() room_version = None @@ -1515,7 +1513,7 @@ async def handle_new_client_event( and requester.shadow_banned ): # We randomly sleep a bit just to annoy the requester. - await self.clock.sleep(random.randint(1, 10)) + await self.clock.sleep(Duration(seconds=random.randint(1, 10))) raise ShadowBanError() if event.is_state(): diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index a90ed3193cd..f869a41c5ed 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -42,6 +42,7 @@ from synapse.types.handlers import ShutdownRoomParams, ShutdownRoomResponse from synapse.types.state import StateFilter from synapse.util.async_helpers import ReadWriteLock +from synapse.util.duration import Duration from synapse.visibility import filter_events_for_client if TYPE_CHECKING: @@ -116,7 +117,7 @@ def __init__(self, hs: "HomeServer"): self.clock.looping_call( self.hs.run_as_background_process, - job.interval, + Duration(milliseconds=job.interval), "purge_history_for_rooms_in_range", self.purge_history_for_rooms_in_range, job.shortest_max_lifetime, diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index ca5002cab35..4c3adca46e3 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -121,6 +121,7 @@ get_domain_from_id, ) from synapse.util.async_helpers import Linearizer +from synapse.util.duration import Duration from synapse.util.metrics import Measure from synapse.util.wheel_timer import WheelTimer @@ -203,7 +204,7 @@ # Delay before a worker tells the presence handler that a user has stopped # syncing. -UPDATE_SYNCING_USERS_MS = 10 * 1000 +UPDATE_SYNCING_USERS = Duration(seconds=10) assert LAST_ACTIVE_GRANULARITY < IDLE_TIMER @@ -528,7 +529,7 @@ def __init__(self, hs: "HomeServer"): self._bump_active_client = ReplicationBumpPresenceActiveTime.make_client(hs) self._set_state_client = ReplicationPresenceSetState.make_client(hs) - self.clock.looping_call(self.send_stop_syncing, UPDATE_SYNCING_USERS_MS) + self.clock.looping_call(self.send_stop_syncing, UPDATE_SYNCING_USERS) hs.register_async_shutdown_handler( phase="before", @@ -581,7 +582,7 @@ def send_stop_syncing(self) -> None: for (user_id, device_id), last_sync_ms in list( self._user_devices_going_offline.items() ): - if now - last_sync_ms > UPDATE_SYNCING_USERS_MS: + if now - last_sync_ms > UPDATE_SYNCING_USERS.as_millis(): self._user_devices_going_offline.pop((user_id, device_id), None) self.send_user_sync(user_id, device_id, False, last_sync_ms) @@ -861,20 +862,20 @@ def __init__(self, hs: "HomeServer"): # The initial delay is to allow disconnected clients a chance to # reconnect before we treat them as offline. self.clock.call_later( - 30, + Duration(seconds=30), self.clock.looping_call, self._handle_timeouts, - 5000, + Duration(seconds=5), ) # Presence information is persisted, whether or not it is being tracked # internally. if self._presence_enabled: self.clock.call_later( - 60, + Duration(minutes=1), self.clock.looping_call, self._persist_unpersisted_changes, - 60 * 1000, + Duration(minutes=1), ) presence_wheel_timer_size_gauge.register_hook( @@ -2430,7 +2431,7 @@ class PresenceFederationQueue: _KEEP_ITEMS_IN_QUEUE_FOR_MS = 5 * 60 * 1000 # How often to check if we can expire entries from the queue. - _CLEAR_ITEMS_EVERY_MS = 60 * 1000 + _CLEAR_ITEMS_EVERY_MS = Duration(minutes=1) def __init__(self, hs: "HomeServer", presence_handler: BasePresenceHandler): self._clock = hs.get_clock() diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 59904cd995a..8f16ae6dece 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -34,6 +34,7 @@ from synapse.storage.databases.main.media_repository import LocalMedia, RemoteMedia from synapse.types import JsonDict, JsonValue, Requester, UserID, create_requester from synapse.util.caches.descriptors import cached +from synapse.util.duration import Duration from synapse.util.stringutils import parse_and_validate_mxc_uri if TYPE_CHECKING: @@ -583,7 +584,7 @@ async def _update_join_states( # Do not actually update the room state for shadow-banned users. if requester.shadow_banned: # We randomly sleep a bit just to annoy the requester. - await self.clock.sleep(random.randint(1, 10)) + await self.clock.sleep(Duration(seconds=random.randint(1, 10))) return room_ids = await self.store.get_rooms_for_user(target_user.to_string()) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index d62ad5393f1..1026bfd8766 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -92,6 +92,7 @@ from synapse.util import stringutils from synapse.util.async_helpers import concurrently_execute from synapse.util.caches.response_cache import ResponseCache +from synapse.util.duration import Duration from synapse.util.iterutils import batch_iter from synapse.util.stringutils import parse_and_validate_server_name from synapse.visibility import filter_events_for_client @@ -1179,7 +1180,7 @@ async def create_room( if (invite_list or invite_3pid_list) and requester.shadow_banned: # We randomly sleep a bit just to annoy the requester. - await self.clock.sleep(random.randint(1, 10)) + await self.clock.sleep(Duration(seconds=random.randint(1, 10))) # Allow the request to go through, but remove any associated invites. invite_3pid_list = [] diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index d5f72c1732d..6f8481de9ae 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -66,6 +66,7 @@ from synapse.types.state import StateFilter from synapse.util.async_helpers import Linearizer from synapse.util.distributor import user_left_room +from synapse.util.duration import Duration if TYPE_CHECKING: from synapse.server import HomeServer @@ -642,7 +643,7 @@ async def update_membership( if action == Membership.INVITE and requester.shadow_banned: # We randomly sleep a bit just to annoy the requester. - await self.clock.sleep(random.randint(1, 10)) + await self.clock.sleep(Duration(seconds=random.randint(1, 10))) raise ShadowBanError() key = (room_id,) @@ -1647,7 +1648,7 @@ async def do_3pid_invite( if requester.shadow_banned: # We randomly sleep a bit just to annoy the requester. - await self.clock.sleep(random.randint(1, 10)) + await self.clock.sleep(Duration(seconds=random.randint(1, 10))) raise ShadowBanError() # We need to rate limit *before* we send out any 3PID invites, so we @@ -2190,7 +2191,7 @@ def __init__(self, hs: "HomeServer"): # We kick this off to pick up outstanding work from before the last restart. self._clock.call_later( - 0, + Duration(seconds=0), self.notify_new_event, ) @@ -2232,7 +2233,7 @@ async def _unsafe_process(self) -> None: # # We wait for a short time so that we don't "tight" loop just # keeping the table up to date. - await self._clock.sleep(0.5) + await self._clock.sleep(Duration(milliseconds=500)) self.pos = self._store.get_room_max_stream_ordering() await self._store.update_room_forgetter_stream_pos(self.pos) diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index 6d661453ac8..c87b5f854a5 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -32,6 +32,7 @@ from synapse.metrics import SERVER_NAME_LABEL, event_processing_positions from synapse.storage.databases.main.state_deltas import StateDelta from synapse.types import JsonDict +from synapse.util.duration import Duration from synapse.util.events import get_plain_text_topic_from_event_content if TYPE_CHECKING: @@ -72,7 +73,7 @@ def __init__(self, hs: "HomeServer"): # We kick this off so that we don't have to wait for a change before # we start populating stats self.clock.call_later( - 0, + Duration(seconds=0), self.notify_new_event, ) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 8b577d5d585..e66396fecc2 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -41,6 +41,7 @@ UserID, ) from synapse.util.caches.stream_change_cache import StreamChangeCache +from synapse.util.duration import Duration from synapse.util.metrics import Measure from synapse.util.retryutils import filter_destinations_by_retry_limiter from synapse.util.wheel_timer import WheelTimer @@ -60,15 +61,15 @@ class RoomMember: # How often we expect remote servers to resend us presence. -FEDERATION_TIMEOUT = 60 * 1000 +FEDERATION_TIMEOUT = Duration(minutes=1) # How often to resend typing across federation. -FEDERATION_PING_INTERVAL = 40 * 1000 +FEDERATION_PING_INTERVAL = Duration(seconds=40) # How long to remember a typing notification happened in a room before # forgetting about it. -FORGET_TIMEOUT = 10 * 60 * 1000 +FORGET_TIMEOUT = Duration(minutes=10) class FollowerTypingHandler: @@ -106,7 +107,7 @@ def __init__(self, hs: "HomeServer"): self._rooms_updated: set[str] = set() - self.clock.looping_call(self._handle_timeouts, 5000) + self.clock.looping_call(self._handle_timeouts, Duration(seconds=5)) self.clock.looping_call(self._prune_old_typing, FORGET_TIMEOUT) def _reset(self) -> None: @@ -141,7 +142,10 @@ def _handle_timeout_for_member(self, now: int, member: RoomMember) -> None: # user. if self.federation and self.is_mine_id(member.user_id): last_fed_poke = self._member_last_federation_poke.get(member, None) - if not last_fed_poke or last_fed_poke + FEDERATION_PING_INTERVAL <= now: + if ( + not last_fed_poke + or last_fed_poke + FEDERATION_PING_INTERVAL.as_millis() <= now + ): self.hs.run_as_background_process( "typing._push_remote", self._push_remote, @@ -165,7 +169,7 @@ async def _push_remote(self, member: RoomMember, typing: bool) -> None: now = self.clock.time_msec() self.wheel_timer.insert( - now=now, obj=member, then=now + FEDERATION_PING_INTERVAL + now=now, obj=member, then=now + FEDERATION_PING_INTERVAL.as_millis() ) hosts: StrCollection = ( @@ -315,7 +319,7 @@ async def started_typing( if requester.shadow_banned: # We randomly sleep a bit just to annoy the requester. - await self.clock.sleep(random.randint(1, 10)) + await self.clock.sleep(Duration(seconds=random.randint(1, 10))) raise ShadowBanError() await self.auth.check_user_in_room(room_id, requester) @@ -350,7 +354,7 @@ async def stopped_typing( if requester.shadow_banned: # We randomly sleep a bit just to annoy the requester. - await self.clock.sleep(random.randint(1, 10)) + await self.clock.sleep(Duration(seconds=random.randint(1, 10))) raise ShadowBanError() await self.auth.check_user_in_room(room_id, requester) @@ -428,8 +432,10 @@ async def _recv_edu(self, origin: str, content: JsonDict) -> None: if user.domain in domains: logger.info("Got typing update from %s: %r", user_id, content) now = self.clock.time_msec() - self._member_typing_until[member] = now + FEDERATION_TIMEOUT - self.wheel_timer.insert(now=now, obj=member, then=now + FEDERATION_TIMEOUT) + self._member_typing_until[member] = now + FEDERATION_TIMEOUT.as_millis() + self.wheel_timer.insert( + now=now, obj=member, then=now + FEDERATION_TIMEOUT.as_millis() + ) self._push_update_local(member=member, typing=content["typing"]) def _push_update_local(self, member: RoomMember, typing: bool) -> None: diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index e5210a3e978..36b037e8e1f 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -40,6 +40,7 @@ from synapse.storage.databases.main.user_directory import SearchResult from synapse.storage.roommember import ProfileInfo from synapse.types import UserID +from synapse.util.duration import Duration from synapse.util.metrics import Measure from synapse.util.retryutils import NotRetryingDestination from synapse.util.stringutils import non_null_str_or_none @@ -52,7 +53,7 @@ # Don't refresh a stale user directory entry, using a Federation /profile request, # for 60 seconds. This gives time for other state events to arrive (which will # then be coalesced such that only one /profile request is made). -USER_DIRECTORY_STALE_REFRESH_TIME_MS = 60 * 1000 +USER_DIRECTORY_STALE_REFRESH_TIME = Duration(minutes=1) # Maximum number of remote servers that we will attempt to refresh profiles for # in one go. @@ -60,7 +61,7 @@ # As long as we have servers to refresh (without backoff), keep adding more # every 15 seconds. -INTERVAL_TO_ADD_MORE_SERVERS_TO_REFRESH_PROFILES = 15 +INTERVAL_TO_ADD_MORE_SERVERS_TO_REFRESH_PROFILES = Duration(seconds=15) def calculate_time_of_next_retry(now_ts: int, retry_count: int) -> int: @@ -137,13 +138,13 @@ def __init__(self, hs: "HomeServer"): # We kick this off so that we don't have to wait for a change before # we start populating the user directory self.clock.call_later( - 0, + Duration(seconds=0), self.notify_new_event, ) # Kick off the profile refresh process on startup self._refresh_remote_profiles_call_later = self.clock.call_later( - 10, + Duration(seconds=10), self.kick_off_remote_profile_refresh_process, ) @@ -550,7 +551,7 @@ async def _handle_possible_remote_profile_change( now_ts = self.clock.time_msec() await self.store.set_remote_user_profile_in_user_dir_stale( user_id, - next_try_at_ms=now_ts + USER_DIRECTORY_STALE_REFRESH_TIME_MS, + next_try_at_ms=now_ts + USER_DIRECTORY_STALE_REFRESH_TIME.as_millis(), retry_counter=0, ) # Schedule a wake-up to refresh the user directory for this server. @@ -558,13 +559,13 @@ async def _handle_possible_remote_profile_change( # other servers ahead of it in the queue to get in the way of updating # the profile if the server only just sent us an event. self.clock.call_later( - USER_DIRECTORY_STALE_REFRESH_TIME_MS // 1000 + 1, + USER_DIRECTORY_STALE_REFRESH_TIME + Duration(seconds=1), self.kick_off_remote_profile_refresh_process_for_remote_server, UserID.from_string(user_id).domain, ) # Schedule a wake-up to handle any backoffs that may occur in the future. self.clock.call_later( - 2 * USER_DIRECTORY_STALE_REFRESH_TIME_MS // 1000 + 1, + USER_DIRECTORY_STALE_REFRESH_TIME * 2 + Duration(seconds=1), self.kick_off_remote_profile_refresh_process, ) return @@ -656,7 +657,9 @@ async def _unsafe_refresh_remote_profiles(self) -> None: if not users: return _, _, next_try_at_ts = users[0] - delay = ((next_try_at_ts - self.clock.time_msec()) // 1000) + 2 + delay = Duration( + milliseconds=next_try_at_ts - self.clock.time_msec() + ) + Duration(seconds=2) self._refresh_remote_profiles_call_later = self.clock.call_later( delay, self.kick_off_remote_profile_refresh_process, diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 4f9c632f5cf..1537a18cc05 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -72,7 +72,7 @@ def __init__(self, hs: "HomeServer") -> None: # that lock. self._locks: dict[tuple[str, str], WeakSet[WaitingLock | WaitingMultiLock]] = {} - self._clock.looping_call(self._cleanup_locks, 30_000) + self._clock.looping_call(self._cleanup_locks, Duration(seconds=30)) self._notifier.add_lock_released_callback(self._on_lock_released) @@ -187,7 +187,7 @@ def _wake_all_locks( lock.release_lock() self._clock.call_later( - 0, + Duration(seconds=0), _wake_all_locks, locks, ) diff --git a/synapse/http/client.py b/synapse/http/client.py index cb9b8cd6831..f0b9201086d 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -87,6 +87,7 @@ from synapse.types import ISynapseReactor, StrSequence from synapse.util.async_helpers import timeout_deferred from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.json import json_decoder if TYPE_CHECKING: @@ -161,7 +162,9 @@ def _is_ip_blocked( return False -_EPSILON = 0.00000001 +# The delay used by the scheduler to schedule tasks "as soon as possible", while +# still allowing other tasks to run between runs. +_EPSILON = Duration(microseconds=1) def _make_scheduler(clock: Clock) -> Callable[[Callable[[], object]], IDelayedCall]: diff --git a/synapse/http/federation/well_known_resolver.py b/synapse/http/federation/well_known_resolver.py index ec72e178c9a..303b3856a27 100644 --- a/synapse/http/federation/well_known_resolver.py +++ b/synapse/http/federation/well_known_resolver.py @@ -37,6 +37,7 @@ from synapse.types import ISynapseThreadlessReactor from synapse.util.caches.ttlcache import TTLCache from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.json import json_decoder from synapse.util.metrics import Measure @@ -315,7 +316,7 @@ async def _make_well_known_request( logger.info("Error fetching %s: %s. Retrying", uri_str, e) # Sleep briefly in the hopes that they come back up - await self._clock.sleep(0.5) + await self._clock.sleep(Duration(milliseconds=500)) def _cache_period_from_headers( diff --git a/synapse/http/server.py b/synapse/http/server.py index 5f4e7484fd7..226cb008317 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -76,6 +76,7 @@ from synapse.util.caches import intern_dict from synapse.util.cancellation import is_function_cancellable from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.iterutils import chunk_seq from synapse.util.json import json_encoder @@ -334,7 +335,7 @@ async def _async_render_wrapper(self, request: "SynapseRequest") -> None: callback_return = await self._async_render(request) except LimitExceededError as e: if e.pause: - await self._clock.sleep(e.pause) + await self._clock.sleep(Duration(seconds=e.pause)) raise if callback_return is not None: diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 7b4408b2bce..29c5e66ec49 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -70,6 +70,7 @@ from synapse.storage.databases.main.media_repository import LocalMedia, RemoteMedia from synapse.types import UserID from synapse.util.async_helpers import Linearizer +from synapse.util.duration import Duration from synapse.util.retryutils import NotRetryingDestination from synapse.util.stringutils import random_string @@ -80,10 +81,10 @@ # How often to run the background job to update the "recently accessed" # attribute of local and remote media. -UPDATE_RECENTLY_ACCESSED_TS = 60 * 1000 # 1 minute +UPDATE_RECENTLY_ACCESSED_TS = Duration(minutes=1) # How often to run the background job to check for local and remote media # that should be purged according to the configured media retention settings. -MEDIA_RETENTION_CHECK_PERIOD_MS = 60 * 60 * 1000 # 1 hour +MEDIA_RETENTION_CHECK_PERIOD = Duration(hours=1) class MediaRepository: @@ -166,7 +167,7 @@ def __init__(self, hs: "HomeServer"): # with the duration between runs dictated by the homeserver config. self.clock.looping_call( self._start_apply_media_retention_rules, - MEDIA_RETENTION_CHECK_PERIOD_MS, + MEDIA_RETENTION_CHECK_PERIOD, ) if hs.config.media.url_preview_enabled: @@ -485,7 +486,7 @@ async def get_local_media_info( if now >= wait_until: break - await self.clock.sleep(0.5) + await self.clock.sleep(Duration(milliseconds=500)) logger.info("Media %s has not yet been uploaded", media_id) self.respond_not_yet_uploaded(request) diff --git a/synapse/media/media_storage.py b/synapse/media/media_storage.py index bc12212c46a..e83869bf4d6 100644 --- a/synapse/media/media_storage.py +++ b/synapse/media/media_storage.py @@ -51,6 +51,7 @@ from synapse.logging.opentracing import start_active_span, trace, trace_with_opname from synapse.media._base import ThreadedFileSender from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.file_consumer import BackgroundFileConsumer from ..types import JsonDict @@ -457,7 +458,7 @@ async def write_chunks_to(self, callback: Callable[[bytes], object]) -> None: callback(chunk) # We yield to the reactor by sleeping for 0 seconds. - await self.clock.sleep(0) + await self.clock.sleep(Duration(seconds=0)) @implementer(interfaces.IConsumer) @@ -652,7 +653,7 @@ async def _resumeProducingRepeatedly(self) -> None: self.paused = False while not self.paused: producer.resumeProducing() - await self.clock.sleep(0) + await self.clock.sleep(Duration(seconds=0)) class Header: diff --git a/synapse/media/url_previewer.py b/synapse/media/url_previewer.py index bbd8017b130..2c5e518918b 100644 --- a/synapse/media/url_previewer.py +++ b/synapse/media/url_previewer.py @@ -47,6 +47,7 @@ from synapse.types import JsonDict, UserID from synapse.util.async_helpers import ObservableDeferred from synapse.util.caches.expiringcache import ExpiringCache +from synapse.util.duration import Duration from synapse.util.json import json_encoder from synapse.util.stringutils import random_string @@ -208,7 +209,9 @@ def __init__( ) if self._worker_run_media_background_jobs: - self.clock.looping_call(self._start_expire_url_cache_data, 10 * 1000) + self.clock.looping_call( + self._start_expire_url_cache_data, Duration(seconds=10) + ) async def preview(self, url: str, user: UserID, ts: int) -> bytes: # the in-memory cache: diff --git a/synapse/metrics/common_usage_metrics.py b/synapse/metrics/common_usage_metrics.py index 3f38412fa7a..bd27a9ca9fc 100644 --- a/synapse/metrics/common_usage_metrics.py +++ b/synapse/metrics/common_usage_metrics.py @@ -23,6 +23,7 @@ import attr from synapse.metrics import SERVER_NAME_LABEL +from synapse.util.duration import Duration if TYPE_CHECKING: from synapse.server import HomeServer @@ -70,7 +71,7 @@ def setup(self) -> None: ) self._clock.looping_call( self._hs.run_as_background_process, - 5 * 60 * 1000, + Duration(minutes=5), desc="common_usage_metrics_update_gauges", func=self._update_gauges, ) diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 6a2d152e3f6..0580f3665cf 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -158,6 +158,7 @@ from synapse.util.async_helpers import maybe_awaitable from synapse.util.caches.descriptors import CachedFunction, cached as _cached from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.frozenutils import freeze if TYPE_CHECKING: @@ -1389,7 +1390,7 @@ def looping_background_call( if self._hs.config.worker.run_background_tasks or run_on_all_instances: self._clock.looping_call( self._hs.run_as_background_process, - msec, + Duration(milliseconds=msec), desc, lambda: maybe_awaitable(f(*args, **kwargs)), ) @@ -1444,8 +1445,7 @@ def delayed_background_call( desc = f.__name__ return self._clock.call_later( - # convert ms to seconds as needed by call_later. - msec * 0.001, + Duration(milliseconds=msec), self._hs.run_as_background_process, desc, lambda: maybe_awaitable(f(*args, **kwargs)), @@ -1457,7 +1457,7 @@ async def sleep(self, seconds: float) -> None: Added in Synapse v1.49.0. """ - await self._clock.sleep(seconds) + await self._clock.sleep(Duration(seconds=seconds)) async def send_http_push_notification( self, diff --git a/synapse/notifier.py b/synapse/notifier.py index 260a2c0d87f..d8d2db17f12 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -61,6 +61,7 @@ from synapse.util.async_helpers import ( timeout_deferred, ) +from synapse.util.duration import Duration from synapse.util.stringutils import shortstr from synapse.visibility import filter_events_for_client @@ -235,7 +236,7 @@ class Notifier: Primarily used from the /events stream. """ - UNUSED_STREAM_EXPIRY_MS = 10 * 60 * 1000 + UNUSED_STREAM_EXPIRY = Duration(minutes=10) def __init__(self, hs: "HomeServer"): self.user_to_user_stream: dict[str, _NotifierUserStream] = {} @@ -269,9 +270,7 @@ def __init__(self, hs: "HomeServer"): self.state_handler = hs.get_state_handler() - self.clock.looping_call( - self.remove_expired_streams, self.UNUSED_STREAM_EXPIRY_MS - ) + self.clock.looping_call(self.remove_expired_streams, self.UNUSED_STREAM_EXPIRY) # This is not a very cheap test to perform, but it's only executed # when rendering the metrics page, which is likely once per minute at @@ -861,7 +860,7 @@ async def wait_for_stream_token(self, stream_token: StreamToken) -> bool: logged = True # TODO: be better - await self.clock.sleep(0.5) + await self.clock.sleep(Duration(milliseconds=500)) async def _get_room_ids( self, user: UserID, explicit_room_id: str | None @@ -889,7 +888,7 @@ async def _is_world_readable(self, room_id: str) -> bool: def remove_expired_streams(self) -> None: time_now_ms = self.clock.time_msec() expired_streams = [] - expire_before_ts = time_now_ms - self.UNUSED_STREAM_EXPIRY_MS + expire_before_ts = time_now_ms - self.UNUSED_STREAM_EXPIRY.as_millis() for stream in self.user_to_user_stream.values(): if stream.count_listeners(): continue diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 36dc9bf6fcc..ce4a2102e4d 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -29,6 +29,7 @@ from synapse.push.mailer import Mailer from synapse.push.push_types import EmailReason from synapse.storage.databases.main.event_push_actions import EmailPushAction +from synapse.util.duration import Duration from synapse.util.threepids import validate_email if TYPE_CHECKING: @@ -229,7 +230,7 @@ async def _unsafe_process(self) -> None: if soonest_due_at is not None: delay = self.seconds_until(soonest_due_at) self.timed_call = self.hs.get_clock().call_later( - delay, + Duration(seconds=delay), self.on_timer, ) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index edcabf0c295..1e7e742dddd 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -40,6 +40,7 @@ if TYPE_CHECKING: from synapse.server import HomeServer +from synapse.util.duration import Duration logger = logging.getLogger(__name__) @@ -336,7 +337,7 @@ async def _unsafe_process(self) -> None: else: logger.info("Push failed: delaying for %ds", self.backoff_delay) self.timed_call = self.hs.get_clock().call_later( - self.backoff_delay, + Duration(seconds=self.backoff_delay), self.on_timer, ) self.backoff_delay = min( @@ -371,7 +372,7 @@ async def _process_one(self, push_action: HttpPushAction) -> bool: delay_ms = random.randint(1, self.push_jitter_delay_ms) diff_ms = event.origin_server_ts + delay_ms - self.clock.time_msec() if diff_ms > 0: - await self.clock.sleep(diff_ms / 1000) + await self.clock.sleep(Duration(milliseconds=diff_ms)) rejected = await self.dispatch_push_event(event, tweaks, badge) if rejected is False: diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py index d76b40cf39b..2bab9c2d713 100644 --- a/synapse/replication/http/_base.py +++ b/synapse/replication/http/_base.py @@ -42,6 +42,7 @@ from synapse.types import JsonDict from synapse.util.caches.response_cache import ResponseCache from synapse.util.cancellation import is_function_cancellable +from synapse.util.duration import Duration from synapse.util.stringutils import random_string if TYPE_CHECKING: @@ -317,7 +318,7 @@ async def send_request( # If we timed out we probably don't need to worry about backing # off too much, but lets just wait a little anyway. - await clock.sleep(1) + await clock.sleep(Duration(seconds=1)) except (ConnectError, DNSLookupError) as e: if not cls.RETRY_ON_CONNECT_ERROR: raise @@ -332,7 +333,7 @@ async def send_request( e, ) - await clock.sleep(delay) + await clock.sleep(Duration(seconds=delay)) attempts += 1 except HttpResponseException as e: # We convert to SynapseError as we know that it was a SynapseError diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 297feb00497..fdda932ead2 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -55,6 +55,7 @@ ) from synapse.types import PersistedEventPosition, ReadReceipt, StreamKeyType, UserID from synapse.util.async_helpers import Linearizer, timeout_deferred +from synapse.util.duration import Duration from synapse.util.iterutils import batch_iter from synapse.util.metrics import Measure @@ -173,7 +174,7 @@ async def on_rdata( ) # Yield to reactor so that we don't block. - await self._clock.sleep(0) + await self._clock.sleep(Duration(seconds=0)) elif stream_name == PushersStream.NAME: for row in rows: if row.deleted: diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index 3068e60af0a..489a2c76a62 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -55,6 +55,7 @@ parse_command_from_line, ) from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.stringutils import random_string if TYPE_CHECKING: @@ -193,7 +194,9 @@ def connectionMade(self) -> None: self._send_pending_commands() # Starts sending pings - self._send_ping_loop = self.clock.looping_call(self.send_ping, 5000) + self._send_ping_loop = self.clock.looping_call( + self.send_ping, Duration(seconds=5) + ) # Always send the initial PING so that the other side knows that they # can time us out. diff --git a/synapse/replication/tcp/redis.py b/synapse/replication/tcp/redis.py index 27d43e6fba8..93ba48b4065 100644 --- a/synapse/replication/tcp/redis.py +++ b/synapse/replication/tcp/redis.py @@ -53,6 +53,7 @@ tcp_inbound_commands_counter, tcp_outbound_commands_counter, ) +from synapse.util.duration import Duration if TYPE_CHECKING: from synapse.replication.tcp.handler import ReplicationCommandHandler @@ -317,7 +318,7 @@ def __init__( self.hs = hs # nb must be called this for @wrap_as_background_process self.server_name = hs.hostname - hs.get_clock().looping_call(self._send_ping, 30 * 1000) + hs.get_clock().looping_call(self._send_ping, Duration(seconds=30)) @wrap_as_background_process("redis_ping") async def _send_ping(self) -> None: diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index 134d8d921f9..36dd39ed672 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -34,6 +34,7 @@ from synapse.replication.tcp.protocol import ServerReplicationStreamProtocol from synapse.replication.tcp.streams import EventsStream from synapse.replication.tcp.streams._base import CachesStream, StreamRow, Token +from synapse.util.duration import Duration from synapse.util.metrics import Measure if TYPE_CHECKING: @@ -116,7 +117,7 @@ def __init__(self, hs: "HomeServer"): # # Note that if the position hasn't advanced then we won't send anything. if any(EventsStream.NAME == s.NAME for s in self.streams): - self.clock.looping_call(self.on_notifier_poke, 1000) + self.clock.looping_call(self.on_notifier_poke, Duration(seconds=1)) def on_notifier_poke(self) -> None: """Checks if there is actually any new data and sends it to the diff --git a/synapse/rest/client/account.py b/synapse/rest/client/account.py index b052052be05..3cb1e09f44c 100644 --- a/synapse/rest/client/account.py +++ b/synapse/rest/client/account.py @@ -58,6 +58,7 @@ EmailRequestTokenBody, MsisdnRequestTokenBody, ) +from synapse.util.duration import Duration from synapse.util.msisdn import phone_number_to_msisdn from synapse.util.stringutils import assert_valid_client_secret, random_string from synapse.util.threepids import check_3pid_allowed, validate_email @@ -125,7 +126,9 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: # comments for request_token_inhibit_3pid_errors. # Also wait for some random amount of time between 100ms and 1s to make it # look like we did something. - await self.hs.get_clock().sleep(random.randint(1, 10) / 10) + await self.hs.get_clock().sleep( + Duration(milliseconds=random.randint(100, 1000)) + ) return 200, {"sid": random_string(16)} raise SynapseError(400, "Email not found", Codes.THREEPID_NOT_FOUND) @@ -383,7 +386,9 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: # comments for request_token_inhibit_3pid_errors. # Also wait for some random amount of time between 100ms and 1s to make it # look like we did something. - await self.hs.get_clock().sleep(random.randint(1, 10) / 10) + await self.hs.get_clock().sleep( + Duration(milliseconds=random.randint(100, 1000)) + ) return 200, {"sid": random_string(16)} raise SynapseError(400, "Email is already in use", Codes.THREEPID_IN_USE) @@ -449,7 +454,9 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: # comments for request_token_inhibit_3pid_errors. # Also wait for some random amount of time between 100ms and 1s to make it # look like we did something. - await self.hs.get_clock().sleep(random.randint(1, 10) / 10) + await self.hs.get_clock().sleep( + Duration(milliseconds=random.randint(100, 1000)) + ) return 200, {"sid": random_string(16)} logger.info("MSISDN %s is already in use by %s", msisdn, existing_user_id) diff --git a/synapse/rest/client/register.py b/synapse/rest/client/register.py index 9503446b92f..fdd2f1985a3 100644 --- a/synapse/rest/client/register.py +++ b/synapse/rest/client/register.py @@ -59,6 +59,7 @@ from synapse.metrics import SERVER_NAME_LABEL, threepid_send_requests from synapse.push.mailer import Mailer from synapse.types import JsonDict +from synapse.util.duration import Duration from synapse.util.msisdn import phone_number_to_msisdn from synapse.util.ratelimitutils import FederationRateLimiter from synapse.util.stringutils import assert_valid_client_secret, random_string @@ -150,7 +151,9 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: # Also wait for some random amount of time between 100ms and 1s to make it # look like we did something. await self.already_in_use_mailer.send_already_in_use_mail(email) - await self.hs.get_clock().sleep(random.randint(1, 10) / 10) + await self.hs.get_clock().sleep( + Duration(milliseconds=random.randint(100, 1000)) + ) return 200, {"sid": random_string(16)} raise SynapseError(400, "Email is already in use", Codes.THREEPID_IN_USE) @@ -219,7 +222,9 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: # comments for request_token_inhibit_3pid_errors. # Also wait for some random amount of time between 100ms and 1s to make it # look like we did something. - await self.hs.get_clock().sleep(random.randint(1, 10) / 10) + await self.hs.get_clock().sleep( + Duration(milliseconds=random.randint(100, 1000)) + ) return 200, {"sid": random_string(16)} raise SynapseError( diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py index 0c1ac1f11b4..43c7b6f9938 100644 --- a/synapse/rest/client/transactions.py +++ b/synapse/rest/client/transactions.py @@ -57,7 +57,7 @@ def __init__(self, hs: "HomeServer"): ] = {} # Try to clean entries every 30 mins. This means entries will exist # for at *LEAST* 30 mins, and at *MOST* 60 mins. - self.clock.looping_call(self._cleanup, CLEANUP_PERIOD.as_millis()) + self.clock.looping_call(self._cleanup, CLEANUP_PERIOD) def _get_transaction_key(self, request: IRequest, requester: Requester) -> Hashable: """A helper function which returns a transaction key that can be used diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 9fc49be4b13..a92233c863e 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -54,6 +54,7 @@ from synapse.types.state import StateFilter from synapse.util.async_helpers import Linearizer from synapse.util.caches.expiringcache import ExpiringCache +from synapse.util.duration import Duration from synapse.util.metrics import Measure, measure_func from synapse.util.stringutils import shortstr @@ -663,7 +664,7 @@ def __init__(self, hs: "HomeServer"): _StateResMetrics ) - self.clock.looping_call(self._report_metrics, 120 * 1000) + self.clock.looping_call(self._report_metrics, Duration(minutes=2)) async def resolve_state_groups( self, diff --git a/synapse/state/v2.py b/synapse/state/v2.py index c410c3a7ec2..1241a4d66e5 100644 --- a/synapse/state/v2.py +++ b/synapse/state/v2.py @@ -40,6 +40,7 @@ from synapse.events import EventBase, is_creator from synapse.storage.databases.main.event_federation import StateDifference from synapse.types import MutableStateMap, StateMap, StrCollection +from synapse.util.duration import Duration logger = logging.getLogger(__name__) @@ -48,7 +49,7 @@ class Clock(Protocol): # This is usually synapse.util.Clock, but it's replaced with a FakeClock in tests. # We only ever sleep(0) though, so that other async functions can make forward # progress without waiting for stateres to complete. - async def sleep(self, duration_ms: float) -> None: ... + async def sleep(self, duration: Duration) -> None: ... class StateResolutionStore(Protocol): @@ -639,7 +640,7 @@ async def _reverse_topological_power_sort( # We await occasionally when we're working with large data sets to # ensure that we don't block the reactor loop for too long. if idx % _AWAIT_AFTER_ITERATIONS == 0: - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) event_to_pl = {} for idx, event_id in enumerate(graph, start=1): @@ -651,7 +652,7 @@ async def _reverse_topological_power_sort( # We await occasionally when we're working with large data sets to # ensure that we don't block the reactor loop for too long. if idx % _AWAIT_AFTER_ITERATIONS == 0: - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) def _get_power_order(event_id: str) -> tuple[int, int, str]: ev = event_map[event_id] @@ -745,7 +746,7 @@ async def _iterative_auth_checks( # We await occasionally when we're working with large data sets to # ensure that we don't block the reactor loop for too long. if idx % _AWAIT_AFTER_ITERATIONS == 0: - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) return resolved_state @@ -796,7 +797,7 @@ async def _mainline_sort( # We await occasionally when we're working with large data sets to # ensure that we don't block the reactor loop for too long. if idx != 0 and idx % _AWAIT_AFTER_ITERATIONS == 0: - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) idx += 1 @@ -814,7 +815,7 @@ async def _mainline_sort( # We await occasionally when we're working with large data sets to # ensure that we don't block the reactor loop for too long. if idx % _AWAIT_AFTER_ITERATIONS == 0: - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) event_ids.sort(key=lambda ev_id: order_map[ev_id]) @@ -865,7 +866,7 @@ async def _get_mainline_depth_for_event( idx += 1 if idx % _AWAIT_AFTER_ITERATIONS == 0: - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) # Didn't find a power level auth event, so we just return 0 return 0 diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index c71bcdb7fb1..311534c5e7b 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -40,6 +40,7 @@ from synapse.storage.types import Connection, Cursor from synapse.types import JsonDict, StrCollection from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.json import json_encoder from . import engines @@ -162,7 +163,7 @@ def __init__( async def __aenter__(self) -> int: if self._sleep: - await self._clock.sleep(self._sleep_duration_ms / 1000) + await self._clock.sleep(Duration(milliseconds=self._sleep_duration_ms)) return self._update_duration_ms diff --git a/synapse/storage/controllers/purge_events.py b/synapse/storage/controllers/purge_events.py index 4ca3f8f4e1a..8a2053d25af 100644 --- a/synapse/storage/controllers/purge_events.py +++ b/synapse/storage/controllers/purge_events.py @@ -32,6 +32,7 @@ from synapse.storage.database import LoggingTransaction from synapse.storage.databases import Databases from synapse.types.storage import _BackgroundUpdates +from synapse.util.duration import Duration from synapse.util.stringutils import shortstr if TYPE_CHECKING: @@ -50,7 +51,7 @@ def __init__(self, hs: "HomeServer", stores: Databases): if hs.config.worker.run_background_tasks: self._delete_state_loop_call = hs.get_clock().looping_call( - self._delete_state_groups_loop, 60 * 1000 + self._delete_state_groups_loop, Duration(minutes=1) ) self.stores.state.db_pool.updates.register_background_update_handler( diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 18f0eac5852..2d5e1d3c482 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -62,6 +62,7 @@ from synapse.storage.types import Connection, Cursor, SQLQueryParameters from synapse.types import StrCollection from synapse.util.async_helpers import delay_cancellation +from synapse.util.duration import Duration from synapse.util.iterutils import batch_iter if TYPE_CHECKING: @@ -631,7 +632,7 @@ def __init__( # Check ASAP (and then later, every 1s) to see if we have finished # background updates of tables that aren't safe to update. self._clock.call_later( - 0.0, + Duration(seconds=0), self.hs.run_as_background_process, "upsert_safety_check", self._check_safe_to_upsert, @@ -679,7 +680,7 @@ async def _check_safe_to_upsert(self) -> None: # If there's any updates still running, reschedule to run. if background_update_names: self._clock.call_later( - 15.0, + Duration(seconds=15), self.hs.run_as_background_process, "upsert_safety_check", self._check_safe_to_upsert, @@ -706,7 +707,7 @@ def loop() -> None: "Total database time: %.3f%% {%s}", ratio * 100, top_three_counters ) - self._clock.looping_call(loop, 10000) + self._clock.looping_call(loop, Duration(seconds=10)) def new_transaction( self, diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py index b7b9b424618..a4530796f20 100644 --- a/synapse/storage/databases/main/cache.py +++ b/synapse/storage/databases/main/cache.py @@ -45,6 +45,7 @@ from synapse.storage.engines import PostgresEngine from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.util.caches.descriptors import CachedFunction +from synapse.util.duration import Duration from synapse.util.iterutils import batch_iter if TYPE_CHECKING: @@ -71,11 +72,11 @@ # How long between cache invalidation table cleanups, once we have caught up # with the backlog. -REGULAR_CLEANUP_INTERVAL_MS = Config.parse_duration("1h") +REGULAR_CLEANUP_INTERVAL = Duration(hours=1) # How long between cache invalidation table cleanups, before we have caught # up with the backlog. -CATCH_UP_CLEANUP_INTERVAL_MS = Config.parse_duration("1m") +CATCH_UP_CLEANUP_INTERVAL = Duration(minutes=1) # Maximum number of cache invalidation rows to delete at once. CLEAN_UP_MAX_BATCH_SIZE = 20_000 @@ -139,7 +140,7 @@ def __init__( self.database_engine, PostgresEngine ): self.hs.get_clock().call_later( - CATCH_UP_CLEANUP_INTERVAL_MS / 1000, + CATCH_UP_CLEANUP_INTERVAL, self._clean_up_cache_invalidation_wrapper, ) @@ -825,12 +826,12 @@ async def _clean_up_cache_invalidation_wrapper(self) -> None: # Vary how long we wait before calling again depending on whether we # are still sifting through backlog or we have caught up. if in_backlog: - next_interval = CATCH_UP_CLEANUP_INTERVAL_MS + next_interval = CATCH_UP_CLEANUP_INTERVAL else: - next_interval = REGULAR_CLEANUP_INTERVAL_MS + next_interval = REGULAR_CLEANUP_INTERVAL self.hs.get_clock().call_later( - next_interval / 1000, + next_interval, self._clean_up_cache_invalidation_wrapper, ) diff --git a/synapse/storage/databases/main/censor_events.py b/synapse/storage/databases/main/censor_events.py index 5d667a53453..a5ae4bf506a 100644 --- a/synapse/storage/databases/main/censor_events.py +++ b/synapse/storage/databases/main/censor_events.py @@ -32,6 +32,7 @@ ) from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.databases.main.events_worker import EventsWorkerStore +from synapse.util.duration import Duration from synapse.util.json import json_encoder if TYPE_CHECKING: @@ -54,7 +55,7 @@ def __init__( hs.config.worker.run_background_tasks and self.hs.config.server.redaction_retention_period is not None ): - hs.get_clock().looping_call(self._censor_redactions, 5 * 60 * 1000) + hs.get_clock().looping_call(self._censor_redactions, Duration(minutes=5)) @wrap_as_background_process("_censor_redactions") async def _censor_redactions(self) -> None: diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py index 4948d0c286b..7cd3667a2b0 100644 --- a/synapse/storage/databases/main/client_ips.py +++ b/synapse/storage/databases/main/client_ips.py @@ -42,6 +42,7 @@ ) from synapse.types import JsonDict, UserID from synapse.util.caches.lrucache import LruCache +from synapse.util.duration import Duration if TYPE_CHECKING: from synapse.server import HomeServer @@ -437,7 +438,7 @@ def __init__( ) if hs.config.worker.run_background_tasks and self.user_ips_max_age: - self.clock.looping_call(self._prune_old_user_ips, 5 * 1000) + self.clock.looping_call(self._prune_old_user_ips, Duration(seconds=5)) if self._update_on_this_worker: # This is the designated worker that can write to the client IP @@ -448,7 +449,7 @@ def __init__( tuple[str, str, str], tuple[str, str | None, int] ] = {} - self.clock.looping_call(self._update_client_ips_batch, 5 * 1000) + self.clock.looping_call(self._update_client_ips_batch, Duration(seconds=5)) hs.register_async_shutdown_handler( phase="before", eventType="shutdown", diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index 28e706d5c34..fc61f46c1c5 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -152,7 +152,7 @@ def __init__( if hs.config.worker.run_background_tasks: self.clock.looping_call( run_as_background_process, - DEVICE_FEDERATION_INBOX_CLEANUP_INTERVAL.as_millis(), + DEVICE_FEDERATION_INBOX_CLEANUP_INTERVAL, "_delete_old_federation_inbox_rows", self.server_name, self._delete_old_federation_inbox_rows, @@ -1029,7 +1029,7 @@ def _delete_old_federation_inbox_rows_txn(txn: LoggingTransaction) -> bool: # We sleep a bit so that we don't hammer the database in a tight # loop first time we run this. - await self.clock.sleep(1) + await self.clock.sleep(Duration(seconds=1)) async def get_devices_with_messages( self, user_id: str, device_ids: StrCollection diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index caae2a0648d..cbad40faf75 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -62,6 +62,7 @@ from synapse.util.caches.descriptors import cached, cachedList from synapse.util.caches.stream_change_cache import StreamChangeCache from synapse.util.cancellation import cancellable +from synapse.util.duration import Duration from synapse.util.iterutils import batch_iter from synapse.util.json import json_decoder, json_encoder from synapse.util.stringutils import shortstr @@ -191,7 +192,7 @@ def __init__( if hs.config.worker.run_background_tasks: self.clock.looping_call( - self._prune_old_outbound_device_pokes, 60 * 60 * 1000 + self._prune_old_outbound_device_pokes, Duration(hours=1) ) def process_replication_rows( diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index b2f0aeaf581..cc7083b605d 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -56,6 +56,7 @@ from synapse.util.caches.descriptors import cached from synapse.util.caches.lrucache import LruCache from synapse.util.cancellation import cancellable +from synapse.util.duration import Duration from synapse.util.iterutils import batch_iter from synapse.util.json import json_encoder @@ -155,7 +156,7 @@ def __init__( if hs.config.worker.run_background_tasks: hs.get_clock().looping_call( - self._delete_old_forward_extrem_cache, 60 * 60 * 1000 + self._delete_old_forward_extrem_cache, Duration(hours=1) ) # Cache of event ID to list of auth event IDs and their depths. @@ -171,7 +172,9 @@ def __init__( # index. self.tests_allow_no_chain_cover_index = True - self.clock.looping_call(self._get_stats_for_federation_staging, 30 * 1000) + self.clock.looping_call( + self._get_stats_for_federation_staging, Duration(seconds=30) + ) if isinstance(self.database_engine, PostgresEngine): self.db_pool.updates.register_background_validate_constraint_and_delete_rows( diff --git a/synapse/storage/databases/main/event_push_actions.py b/synapse/storage/databases/main/event_push_actions.py index 2e99d7314e8..a66caa672cd 100644 --- a/synapse/storage/databases/main/event_push_actions.py +++ b/synapse/storage/databases/main/event_push_actions.py @@ -105,6 +105,7 @@ from synapse.storage.databases.main.stream import StreamWorkerStore from synapse.types import JsonDict, StrCollection from synapse.util.caches.descriptors import cached +from synapse.util.duration import Duration from synapse.util.json import json_encoder if TYPE_CHECKING: @@ -270,15 +271,17 @@ def __init__( self._find_stream_orderings_for_times_txn(cur) cur.close() - self.clock.looping_call(self._find_stream_orderings_for_times, 10 * 60 * 1000) + self.clock.looping_call( + self._find_stream_orderings_for_times, Duration(minutes=10) + ) self._rotate_count = 10000 self._doing_notif_rotation = False if hs.config.worker.run_background_tasks: - self.clock.looping_call(self._rotate_notifs, 30 * 1000) + self.clock.looping_call(self._rotate_notifs, Duration(seconds=30)) self.clock.looping_call( - self._clear_old_push_actions_staging, 30 * 60 * 1000 + self._clear_old_push_actions_staging, Duration(minutes=30) ) self.db_pool.updates.register_background_index_update( @@ -1817,7 +1820,7 @@ def _clear_old_push_actions_staging_txn(txn: LoggingTransaction) -> bool: return # We sleep to ensure that we don't overwhelm the DB. - await self.clock.sleep(1.0) + await self.clock.sleep(Duration(seconds=1)) async def get_push_actions_for_user( self, diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 4cf708442d4..ae6ee50dc24 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -92,6 +92,7 @@ from synapse.util.caches.lrucache import AsyncLruCache from synapse.util.caches.stream_change_cache import StreamChangeCache from synapse.util.cancellation import cancellable +from synapse.util.duration import Duration from synapse.util.iterutils import batch_iter from synapse.util.metrics import Measure @@ -278,7 +279,7 @@ def __init__( # We periodically clean out old transaction ID mappings self.clock.looping_call( self._cleanup_old_transaction_ids, - 5 * 60 * 1000, + Duration(minutes=5), ) self._get_event_cache: AsyncLruCache[tuple[str], EventCacheEntry] = ( diff --git a/synapse/storage/databases/main/lock.py b/synapse/storage/databases/main/lock.py index 51f04acbcb1..dd49f98366e 100644 --- a/synapse/storage/databases/main/lock.py +++ b/synapse/storage/databases/main/lock.py @@ -38,6 +38,7 @@ ) from synapse.types import ISynapseReactor from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.stringutils import random_string if TYPE_CHECKING: @@ -49,11 +50,13 @@ # How often to renew an acquired lock by updating the `last_renewed_ts` time in # the lock table. -_RENEWAL_INTERVAL_MS = 30 * 1000 +_RENEWAL_INTERVAL = Duration(seconds=30) # How long before an acquired lock times out. _LOCK_TIMEOUT_MS = 2 * 60 * 1000 +_LOCK_REAP_INTERVAL = Duration(milliseconds=_LOCK_TIMEOUT_MS / 10.0) + class LockStore(SQLBaseStore): """Provides a best effort distributed lock between worker instances. @@ -106,9 +109,7 @@ def __init__( self._acquiring_locks: set[tuple[str, str]] = set() - self.clock.looping_call( - self._reap_stale_read_write_locks, _LOCK_TIMEOUT_MS / 10.0 - ) + self.clock.looping_call(self._reap_stale_read_write_locks, _LOCK_REAP_INTERVAL) @wrap_as_background_process("LockStore._on_shutdown") async def _on_shutdown(self) -> None: @@ -410,7 +411,7 @@ def __init__( def _setup_looping_call(self) -> None: self._looping_call = self._clock.looping_call( self._renew, - _RENEWAL_INTERVAL_MS, + _RENEWAL_INTERVAL, self._server_name, self._store, self._hs, diff --git a/synapse/storage/databases/main/metrics.py b/synapse/storage/databases/main/metrics.py index dc8e2c16165..b2b45612478 100644 --- a/synapse/storage/databases/main/metrics.py +++ b/synapse/storage/databases/main/metrics.py @@ -34,6 +34,7 @@ from synapse.storage.databases.main.event_push_actions import ( EventPushActionsWorkerStore, ) +from synapse.util.duration import Duration if TYPE_CHECKING: from synapse.server import HomeServer @@ -78,7 +79,7 @@ def __init__( # Read the extrems every 60 minutes if hs.config.worker.run_background_tasks: - self.clock.looping_call(self._read_forward_extremities, 60 * 60 * 1000) + self.clock.looping_call(self._read_forward_extremities, Duration(hours=1)) # Used in _generate_user_daily_visits to keep track of progress self._last_user_visit_update = self._get_start_of_day() diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index 545b0f11c42..9a9c0fffc7e 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -49,13 +49,12 @@ from synapse.storage.util.sequence import build_sequence_generator from synapse.types import JsonDict, StrCollection, UserID, UserInfo from synapse.util.caches.descriptors import cached +from synapse.util.duration import Duration from synapse.util.iterutils import batch_iter if TYPE_CHECKING: from synapse.server import HomeServer -THIRTY_MINUTES_IN_MS = 30 * 60 * 1000 - logger = logging.getLogger(__name__) @@ -213,7 +212,7 @@ def __init__( if hs.config.worker.run_background_tasks: self.clock.call_later( - 0.0, + Duration(seconds=0), self._set_expiration_date_when_missing, ) @@ -227,7 +226,7 @@ def __init__( # Create a background job for culling expired 3PID validity tokens if hs.config.worker.run_background_tasks: self.clock.looping_call( - self.cull_expired_threepid_validation_tokens, THIRTY_MINUTES_IN_MS + self.cull_expired_threepid_validation_tokens, Duration(minutes=30) ) async def register_user( @@ -2739,7 +2738,7 @@ def __init__( # Create a background job for removing expired login tokens if hs.config.worker.run_background_tasks: self.clock.looping_call( - self._delete_expired_login_tokens, THIRTY_MINUTES_IN_MS + self._delete_expired_login_tokens, Duration(minutes=30) ) async def add_access_token_to_user( diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 4fb7779d38d..9b06ab69fed 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -63,6 +63,7 @@ get_domain_from_id, ) from synapse.util.caches.descriptors import _CacheContext, cached, cachedList +from synapse.util.duration import Duration from synapse.util.iterutils import batch_iter from synapse.util.metrics import Measure @@ -110,10 +111,10 @@ def __init__( self._known_servers_count = 1 self.hs.get_clock().looping_call( self._count_known_servers, - 60 * 1000, + Duration(minutes=1), ) self.hs.get_clock().call_later( - 1, + Duration(seconds=1), self._count_known_servers, ) federation_known_servers_gauge.register_hook( diff --git a/synapse/storage/databases/main/session.py b/synapse/storage/databases/main/session.py index 1154bb2d599..f088a8d88cf 100644 --- a/synapse/storage/databases/main/session.py +++ b/synapse/storage/databases/main/session.py @@ -30,6 +30,7 @@ LoggingTransaction, ) from synapse.types import JsonDict +from synapse.util.duration import Duration from synapse.util.json import json_encoder if TYPE_CHECKING: @@ -55,7 +56,7 @@ def __init__( # Create a background job for culling expired sessions. if hs.config.worker.run_background_tasks: - self.clock.looping_call(self._delete_expired_sessions, 30 * 60 * 1000) + self.clock.looping_call(self._delete_expired_sessions, Duration(minutes=30)) async def create_session( self, session_type: str, value: JsonDict, expiry_ms: int diff --git a/synapse/storage/databases/main/sliding_sync.py b/synapse/storage/databases/main/sliding_sync.py index 6f87308cde6..828eed3a73c 100644 --- a/synapse/storage/databases/main/sliding_sync.py +++ b/synapse/storage/databases/main/sliding_sync.py @@ -96,7 +96,7 @@ def __init__( if self.hs.config.worker.run_background_tasks: self.clock.looping_call( self.delete_old_sliding_sync_connections, - CONNECTION_EXPIRY_FREQUENCY.as_millis(), + CONNECTION_EXPIRY_FREQUENCY, ) async def get_latest_bump_stamp_for_room( diff --git a/synapse/storage/databases/main/transactions.py b/synapse/storage/databases/main/transactions.py index 70c5b928fdc..2fdd27d3da1 100644 --- a/synapse/storage/databases/main/transactions.py +++ b/synapse/storage/databases/main/transactions.py @@ -37,6 +37,7 @@ from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.types import JsonDict, StrCollection from synapse.util.caches.descriptors import cached, cachedList +from synapse.util.duration import Duration if TYPE_CHECKING: from synapse.server import HomeServer @@ -81,7 +82,7 @@ def __init__( super().__init__(database, db_conn, hs) if hs.config.worker.run_background_tasks: - self.clock.looping_call(self._cleanup_transactions, 30 * 60 * 1000) + self.clock.looping_call(self._cleanup_transactions, Duration(minutes=30)) @wrap_as_background_process("cleanup_transactions") async def _cleanup_transactions(self) -> None: diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 6f9bbcac678..818f8b1a69b 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -58,6 +58,7 @@ run_in_background, ) from synapse.util.clock import Clock +from synapse.util.duration import Duration logger = logging.getLogger(__name__) @@ -640,7 +641,7 @@ async def _acquire_lock(self, key: Hashable) -> _LinearizerEntry: # This needs to happen while we hold the lock. We could put it on the # exit path, but that would slow down the uncontended case. try: - await self._clock.sleep(0) + await self._clock.sleep(Duration(seconds=0)) except CancelledError: self._release_lock(key, entry) raise @@ -818,7 +819,9 @@ def time_it_out() -> None: # We don't track these calls since they are short. delayed_call = clock.call_later( - timeout, time_it_out, call_later_cancel_on_shutdown=cancel_on_shutdown + Duration(seconds=timeout), + time_it_out, + call_later_cancel_on_shutdown=cancel_on_shutdown, ) def convert_cancelled(value: Failure) -> Failure: diff --git a/synapse/util/batching_queue.py b/synapse/util/batching_queue.py index 514abcbec16..43eefcb7f18 100644 --- a/synapse/util/batching_queue.py +++ b/synapse/util/batching_queue.py @@ -36,6 +36,7 @@ from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable from synapse.metrics import SERVER_NAME_LABEL from synapse.util.clock import Clock +from synapse.util.duration import Duration if TYPE_CHECKING: from synapse.server import HomeServer @@ -175,7 +176,7 @@ async def _process_queue(self, key: Hashable) -> None: # pattern is to call `add_to_queue` multiple times at once, and # deferring to the next reactor tick allows us to batch all of # those up. - await self._clock.sleep(0) + await self._clock.sleep(Duration(seconds=0)) next_values = self._next_values.pop(key, []) if not next_values: diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py index 528e4bb852c..87870f42237 100644 --- a/synapse/util/caches/expiringcache.py +++ b/synapse/util/caches/expiringcache.py @@ -38,6 +38,7 @@ from synapse.config import cache as cache_config from synapse.util.caches import EvictionReason, register_cache from synapse.util.clock import Clock +from synapse.util.duration import Duration if TYPE_CHECKING: from synapse.server import HomeServer @@ -112,7 +113,7 @@ def __init__( def f() -> "defer.Deferred[None]": return hs.run_as_background_process("prune_cache", self._prune_cache) - self._clock.looping_call(f, self._expiry_ms / 2) + self._clock.looping_call(f, Duration(milliseconds=self._expiry_ms / 2)) def __setitem__(self, key: KT, value: VT) -> None: now = self._clock.time_msec() diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index d304e804e94..a3e7bd4d03e 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -50,6 +50,7 @@ iterate_tree_cache_items, ) from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.linked_list import ListNode if TYPE_CHECKING: @@ -202,9 +203,9 @@ async def _internal_expire_old_entries( if (i + 1) % 10000 == 0: logger.debug("Waiting during drop") if node.last_access_ts_secs > now - expiry_seconds: - await clock.sleep(0.5) + await clock.sleep(Duration(milliseconds=500)) else: - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) logger.debug("Waking during drop") node = next_node @@ -248,7 +249,7 @@ def setup_expire_lru_cache_entries(hs: "HomeServer") -> None: clock = hs.get_clock() clock.looping_call( _expire_old_entries, - 30 * 1000, + Duration(seconds=30), server_name, hs, clock, diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py index b1cdc81dda2..0289e13f6a0 100644 --- a/synapse/util/caches/response_cache.py +++ b/synapse/util/caches/response_cache.py @@ -42,6 +42,7 @@ from synapse.util.async_helpers import AbstractObservableDeferred, ObservableDeferred from synapse.util.caches import EvictionReason, register_cache from synapse.util.clock import Clock +from synapse.util.duration import Duration logger = logging.getLogger(__name__) @@ -120,7 +121,7 @@ def __init__( self._result_cache: dict[KV, ResponseCacheEntry] = {} self.clock = clock - self.timeout_sec = timeout_ms / 1000.0 + self.timeout = Duration(milliseconds=timeout_ms) self._name = name self._metrics = register_cache( @@ -195,9 +196,9 @@ def on_complete(r: RV) -> RV: # if this cache has a non-zero timeout, and the callback has not cleared # the should_cache bit, we leave it in the cache for now and schedule # its removal later. - if self.timeout_sec and context.should_cache: + if self.timeout and context.should_cache: self.clock.call_later( - self.timeout_sec, + self.timeout, self._entry_timeout, key, # We don't need to track these calls since they don't hold any strong diff --git a/synapse/util/clock.py b/synapse/util/clock.py index 65f71648967..6fd31864b75 100644 --- a/synapse/util/clock.py +++ b/synapse/util/clock.py @@ -31,6 +31,7 @@ from synapse.logging import context from synapse.types import ISynapseThreadlessReactor from synapse.util import log_failure +from synapse.util.duration import Duration from synapse.util.stringutils import random_string_insecure_fast P = ParamSpec("P") @@ -84,14 +85,14 @@ def shutdown(self) -> None: self.cancel_all_looping_calls() self.cancel_all_delayed_calls() - async def sleep(self, seconds: float) -> None: + async def sleep(self, duration: Duration) -> None: d: defer.Deferred[float] = defer.Deferred() # Start task in the `sentinel` logcontext, to avoid leaking the current context # into the reactor once it finishes. with context.PreserveLoggingContext(): # We can ignore the lint here since this class is the one location callLater should # be called. - self._reactor.callLater(seconds, d.callback, seconds) # type: ignore[call-later-not-tracked] + self._reactor.callLater(duration.as_secs(), d.callback, duration.as_secs()) # type: ignore[call-later-not-tracked] await d def time(self) -> float: @@ -105,13 +106,13 @@ def time_msec(self) -> int: def looping_call( self, f: Callable[P, object], - msec: float, + duration: Duration, *args: P.args, **kwargs: P.kwargs, ) -> LoopingCall: """Call a function repeatedly. - Waits `msec` initially before calling `f` for the first time. + Waits `duration` initially before calling `f` for the first time. If the function given to `looping_call` returns an awaitable/deferred, the next call isn't scheduled until after the returned awaitable has finished. We get @@ -124,16 +125,16 @@ def looping_call( Args: f: The function to call repeatedly. - msec: How long to wait between calls in milliseconds. + duration: How long to wait between calls. *args: Positional arguments to pass to function. **kwargs: Key arguments to pass to function. """ - return self._looping_call_common(f, msec, False, *args, **kwargs) + return self._looping_call_common(f, duration, False, *args, **kwargs) def looping_call_now( self, f: Callable[P, object], - msec: float, + duration: Duration, *args: P.args, **kwargs: P.kwargs, ) -> LoopingCall: @@ -148,16 +149,16 @@ def looping_call_now( Args: f: The function to call repeatedly. - msec: How long to wait between calls in milliseconds. + duration: How long to wait between calls. *args: Positional arguments to pass to function. **kwargs: Key arguments to pass to function. """ - return self._looping_call_common(f, msec, True, *args, **kwargs) + return self._looping_call_common(f, duration, True, *args, **kwargs) def _looping_call_common( self, f: Callable[P, object], - msec: float, + duration: Duration, now: bool, *args: P.args, **kwargs: P.kwargs, @@ -217,7 +218,7 @@ def wrapped_f(*args: P.args, **kwargs: P.kwargs) -> Deferred: # We want to start the task in the `sentinel` logcontext, to avoid leaking the # current context into the reactor after the function finishes. with context.PreserveLoggingContext(): - d = call.start(msec / 1000.0, now=now) + d = call.start(duration.as_secs(), now=now) d.addErrback(log_failure, "Looping call died", consumeErrors=False) self._looping_calls.append(call) @@ -225,7 +226,7 @@ def wrapped_f(*args: P.args, **kwargs: P.kwargs) -> Deferred: "%s(%s): Scheduled looping call every %sms later", looping_call_context_string, instance_id, - msec, + duration.as_millis(), # Find out who is scheduling the call which makes it easy to follow in the # logs. stack_info=True, @@ -251,7 +252,7 @@ def cancel_all_looping_calls(self, consumeErrors: bool = True) -> None: def call_later( self, - delay: float, + delay: Duration, callback: Callable, *args: Any, call_later_cancel_on_shutdown: bool = True, @@ -264,7 +265,7 @@ def call_later( `run_as_background_process` to give it more specific label and track metrics. Args: - delay: How long to wait in seconds. + delay: How long to wait. callback: Function to call *args: Postional arguments to pass to function. call_later_cancel_on_shutdown: Whether this call should be tracked for cleanup during @@ -322,7 +323,9 @@ def wrapped_callback(*args: Any, **kwargs: Any) -> None: # We can ignore the lint here since this class is the one location callLater should # be called. - call = self._reactor.callLater(delay, wrapped_callback, *args, **kwargs) # type: ignore[call-later-not-tracked] + call = self._reactor.callLater( + delay.as_secs(), wrapped_callback, *args, **kwargs + ) # type: ignore[call-later-not-tracked] logger.debug( "call_later(%s): Scheduled call for %ss later (tracked for shutdown: %s)", diff --git a/synapse/util/duration.py b/synapse/util/duration.py index 3419f6dda60..135b9808520 100644 --- a/synapse/util/duration.py +++ b/synapse/util/duration.py @@ -13,6 +13,7 @@ # from datetime import timedelta +from typing import overload # Constant so we don't keep creating new timedelta objects when calling # `.as_millis()`. @@ -35,6 +36,82 @@ def as_millis(self) -> int: """Returns the duration in milliseconds.""" return int(self / _ONE_MILLISECOND) - def as_secs(self) -> int: + def as_secs(self) -> float: """Returns the duration in seconds.""" - return int(self.total_seconds()) + return self.total_seconds() + + # Override arithmetic operations to return Duration instances + + def __add__(self, other: timedelta) -> "Duration": + """Add two durations together, returning a Duration.""" + result = super().__add__(other) + return Duration(seconds=result.total_seconds()) + + def __radd__(self, other: timedelta) -> "Duration": + """Add two durations together (reversed), returning a Duration.""" + result = super().__radd__(other) + return Duration(seconds=result.total_seconds()) + + def __sub__(self, other: timedelta) -> "Duration": + """Subtract two durations, returning a Duration.""" + result = super().__sub__(other) + return Duration(seconds=result.total_seconds()) + + def __rsub__(self, other: timedelta) -> "Duration": + """Subtract two durations (reversed), returning a Duration.""" + result = super().__rsub__(other) + return Duration(seconds=result.total_seconds()) + + def __mul__(self, other: float) -> "Duration": + """Multiply a duration by a scalar, returning a Duration.""" + result = super().__mul__(other) + return Duration(seconds=result.total_seconds()) + + def __rmul__(self, other: float) -> "Duration": + """Multiply a duration by a scalar (reversed), returning a Duration.""" + result = super().__rmul__(other) + return Duration(seconds=result.total_seconds()) + + @overload + def __truediv__(self, other: timedelta) -> float: ... + + @overload + def __truediv__(self, other: float) -> "Duration": ... + + def __truediv__(self, other: float | timedelta) -> "Duration | float": + """Divide a duration by a scalar or another duration. + + If dividing by a scalar, returns a Duration. + If dividing by a timedelta, returns a float ratio. + """ + result = super().__truediv__(other) + if isinstance(other, timedelta): + # Dividing by a timedelta gives a float ratio + assert isinstance(result, float) + return result + else: + # Dividing by a scalar gives a Duration + assert isinstance(result, timedelta) + return Duration(seconds=result.total_seconds()) + + @overload + def __floordiv__(self, other: timedelta) -> int: ... + + @overload + def __floordiv__(self, other: int) -> "Duration": ... + + def __floordiv__(self, other: int | timedelta) -> "Duration | int": + """Floor divide a duration by a scalar or another duration. + + If dividing by a scalar, returns a Duration. + If dividing by a timedelta, returns an int ratio. + """ + result = super().__floordiv__(other) + if isinstance(other, timedelta): + # Dividing by a timedelta gives an int ratio + assert isinstance(result, int) + return result + else: + # Dividing by a scalar gives a Duration + assert isinstance(result, timedelta) + return Duration(seconds=result.total_seconds()) diff --git a/synapse/util/ratelimitutils.py b/synapse/util/ratelimitutils.py index 024706d9cfd..d1053d227be 100644 --- a/synapse/util/ratelimitutils.py +++ b/synapse/util/ratelimitutils.py @@ -48,6 +48,7 @@ from synapse.logging.opentracing import start_active_span from synapse.metrics import SERVER_NAME_LABEL, Histogram, LaterGauge from synapse.util.clock import Clock +from synapse.util.duration import Duration if typing.TYPE_CHECKING: from contextlib import _GeneratorContextManager @@ -353,7 +354,9 @@ def queue_request() -> "defer.Deferred[None]": rate_limiter_name=self.metrics_name, **{SERVER_NAME_LABEL: self.our_server_name}, ).inc() - ret_defer = run_in_background(self.clock.sleep, self.sleep_sec) + ret_defer = run_in_background( + self.clock.sleep, Duration(seconds=self.sleep_sec) + ) self.sleeping_requests.add(request_id) @@ -414,6 +417,6 @@ def start_next_request() -> None: pass self.clock.call_later( - 0.0, + Duration(seconds=0), start_next_request, ) diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index 3b4423a1ff5..353ddb70bc3 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -35,6 +35,7 @@ wrap_as_background_process, ) from synapse.types import JsonMapping, ScheduledTask, TaskStatus +from synapse.util.duration import Duration from synapse.util.stringutils import random_string if TYPE_CHECKING: @@ -92,8 +93,8 @@ class TaskScheduler: """ # Precision of the scheduler, evaluation of tasks to run will only happen - # every `SCHEDULE_INTERVAL_MS` ms - SCHEDULE_INTERVAL_MS = 1 * 60 * 1000 # 1mn + # every `SCHEDULE_INTERVAL` + SCHEDULE_INTERVAL = Duration(minutes=1) # How often to clean up old tasks. CLEANUP_INTERVAL_MS = 30 * 60 * 1000 # Time before a complete or failed task is deleted from the DB @@ -103,7 +104,7 @@ class TaskScheduler: # Time from the last task update after which we will log a warning LAST_UPDATE_BEFORE_WARNING_MS = 24 * 60 * 60 * 1000 # 24hrs # Report a running task's status and usage every so often. - OCCASIONAL_REPORT_INTERVAL_MS = 5 * 60 * 1000 # 5 minutes + OCCASIONAL_REPORT_INTERVAL = Duration(minutes=5) def __init__(self, hs: "HomeServer"): self.hs = hs # nb must be called this for @wrap_as_background_process @@ -127,11 +128,11 @@ def __init__(self, hs: "HomeServer"): if self._run_background_tasks: self._clock.looping_call( self._launch_scheduled_tasks, - TaskScheduler.SCHEDULE_INTERVAL_MS, + TaskScheduler.SCHEDULE_INTERVAL, ) self._clock.looping_call( self._clean_scheduled_tasks, - TaskScheduler.SCHEDULE_INTERVAL_MS, + TaskScheduler.SCHEDULE_INTERVAL, ) running_tasks_gauge.register_hook( @@ -433,7 +434,7 @@ async def wrapper() -> None: start_time = self._clock.time() occasional_status_call = self._clock.looping_call( _occasional_report, - TaskScheduler.OCCASIONAL_REPORT_INTERVAL_MS, + TaskScheduler.OCCASIONAL_REPORT_INTERVAL, log_context, start_time, ) @@ -468,7 +469,7 @@ async def wrapper() -> None: # Try launch a new task since we've finished with this one. self._clock.call_later( - 0.1, + Duration(milliseconds=100), self._launch_scheduled_tasks, ) diff --git a/synmark/suites/logging.py b/synmark/suites/logging.py index d89f487d3dc..243f9dbca09 100644 --- a/synmark/suites/logging.py +++ b/synmark/suites/logging.py @@ -37,6 +37,7 @@ from synapse.synapse_rust import reset_logging_config from synapse.types import ISynapseReactor from synapse.util.clock import Clock +from synapse.util.duration import Duration class LineCounter(LineOnlyReceiver): @@ -141,7 +142,7 @@ class _logging: if len(handler._buffer) == handler.maximum_buffer: while len(handler._buffer) > handler.maximum_buffer / 2: - await clock.sleep(0.01) + await clock.sleep(Duration(milliseconds=10)) await logger_factory.on_done diff --git a/tests/federation/transport/server/test__base.py b/tests/federation/transport/server/test__base.py index 3c553e6e402..00a9c2064c6 100644 --- a/tests/federation/transport/server/test__base.py +++ b/tests/federation/transport/server/test__base.py @@ -30,6 +30,7 @@ from synapse.server import HomeServer from synapse.types import JsonDict from synapse.util.cancellation import cancellable +from synapse.util.duration import Duration from synapse.util.ratelimitutils import FederationRateLimiter from tests import unittest @@ -53,13 +54,13 @@ def __init__( async def on_GET( self, origin: str, content: None, query: dict[bytes, list[bytes]] ) -> tuple[int, JsonDict]: - await self.clock.sleep(1.0) + await self.clock.sleep(Duration(seconds=1)) return HTTPStatus.OK, {"result": True} async def on_POST( self, origin: str, content: JsonDict, query: dict[bytes, list[bytes]] ) -> tuple[int, JsonDict]: - await self.clock.sleep(1.0) + await self.clock.sleep(Duration(seconds=1)) return HTTPStatus.OK, {"result": True} diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index fa6bb4970be..183234b8a0a 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -250,7 +250,7 @@ def test_delete_device_and_big_device_inbox(self) -> None: self.assertEqual(10, len(res)) # wait for the task scheduler to do a second delete pass - self.reactor.advance(TaskScheduler.SCHEDULE_INTERVAL_MS / 1000) + self.reactor.advance(TaskScheduler.SCHEDULE_INTERVAL.as_secs()) # remaining messages should now be deleted res = self.get_success( diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 70557a4a5fa..623eef0ecb6 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -544,7 +544,7 @@ def test_prune_typing_replication(self) -> None: ) self.assertEqual(rows, [(2, [ROOM_ID, []])]) - self.reactor.advance(FORGET_TIMEOUT) + self.reactor.advance(FORGET_TIMEOUT.as_secs()) rows, _, _ = self.get_success( self.handler.get_all_typing_updates( diff --git a/tests/http/test_servlet.py b/tests/http/test_servlet.py index 5bf8305d05e..2f1c8f03c6c 100644 --- a/tests/http/test_servlet.py +++ b/tests/http/test_servlet.py @@ -34,6 +34,7 @@ from synapse.server import HomeServer from synapse.types import JsonDict from synapse.util.cancellation import cancellable +from synapse.util.duration import Duration from tests import unittest from tests.http.server._base import test_disconnect @@ -108,11 +109,11 @@ def __init__(self, hs: HomeServer): @cancellable async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: - await self.clock.sleep(1.0) + await self.clock.sleep(Duration(seconds=1)) return HTTPStatus.OK, {"result": True} async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: - await self.clock.sleep(1.0) + await self.clock.sleep(Duration(seconds=1)) return HTTPStatus.OK, {"result": True} diff --git a/tests/logging/test_opentracing.py b/tests/logging/test_opentracing.py index 3aaa7432654..d5e643585d5 100644 --- a/tests/logging/test_opentracing.py +++ b/tests/logging/test_opentracing.py @@ -37,6 +37,7 @@ ) from synapse.metrics.background_process_metrics import run_as_background_process from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests.server import get_clock @@ -184,7 +185,7 @@ async def task(i: int) -> None: scopes.append(scope) self.assertEqual(self._tracer.active_span, scope.span) - await clock.sleep(4) + await clock.sleep(Duration(seconds=4)) self.assertEqual(self._tracer.active_span, scope.span) scope.close() @@ -194,7 +195,7 @@ async def root() -> None: scopes.append(root_scope) d1 = run_in_background(task, 1) - await clock.sleep(2) + await clock.sleep(Duration(seconds=2)) d2 = run_in_background(task, 2) # because we did run_in_background, the active span should still be the @@ -351,7 +352,7 @@ async def bg_task() -> None: # Now wait for the background process to finish while not callback_finished: - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self.assertTrue( callback_finished, @@ -418,7 +419,7 @@ async def bg_task() -> None: # Now wait for the background process to finish while not callback_finished: - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self.assertTrue( callback_finished, diff --git a/tests/replication/http/test__base.py b/tests/replication/http/test__base.py index b757c6428a7..1c7e7e997bb 100644 --- a/tests/replication/http/test__base.py +++ b/tests/replication/http/test__base.py @@ -30,6 +30,7 @@ from synapse.server import HomeServer from synapse.types import JsonDict from synapse.util.cancellation import cancellable +from synapse.util.duration import Duration from tests import unittest from tests.http.server._base import test_disconnect @@ -52,7 +53,7 @@ async def _serialize_payload(**kwargs: ReplicationEndpoint) -> JsonDict: async def _handle_request( # type: ignore[override] self, request: Request, content: JsonDict ) -> tuple[int, JsonDict]: - await self.clock.sleep(1.0) + await self.clock.sleep(Duration(seconds=1)) return HTTPStatus.OK, {"result": True} @@ -73,7 +74,7 @@ async def _serialize_payload(**kwargs: ReplicationEndpoint) -> JsonDict: async def _handle_request( # type: ignore[override] self, request: Request, content: JsonDict ) -> tuple[int, JsonDict]: - await self.clock.sleep(1.0) + await self.clock.sleep(Duration(seconds=1)) return HTTPStatus.OK, {"result": True} diff --git a/tests/rest/admin/test_background_updates.py b/tests/rest/admin/test_background_updates.py index 25112baaa2b..a4a3112e20e 100644 --- a/tests/rest/admin/test_background_updates.py +++ b/tests/rest/admin/test_background_updates.py @@ -31,6 +31,7 @@ from synapse.storage.background_updates import BackgroundUpdater from synapse.types import JsonDict from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests import unittest @@ -105,7 +106,7 @@ def _register_bg_update(self) -> None: "Adds a bg update but doesn't start it" async def _fake_update(progress: JsonDict, batch_size: int) -> int: - await self.clock.sleep(0.2) + await self.clock.sleep(Duration(milliseconds=200)) return batch_size self.store.db_pool.updates.register_background_update_handler( diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index 7daf13ad220..1c340efa0cd 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -44,6 +44,7 @@ ) from synapse.types import UserID from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.task_scheduler import TaskScheduler from tests import unittest @@ -1161,7 +1162,7 @@ def test_delete_same_room_twice(self) -> None: # Mock PaginationHandler.purge_room to sleep for 100s, so we have time to do a second call # before the purge is over. Note that it doesn't purge anymore, but we don't care. async def purge_room(room_id: str, force: bool) -> None: - await self.hs.get_clock().sleep(100) + await self.hs.get_clock().sleep(Duration(seconds=100)) self.pagination_handler.purge_room = AsyncMock(side_effect=purge_room) # type: ignore[method-assign] @@ -1464,7 +1465,7 @@ def test_scheduled_purge_room(self) -> None: self._is_purged(room_id) # Wait for next scheduler run - self.reactor.advance(TaskScheduler.SCHEDULE_INTERVAL_MS) + self.reactor.advance(TaskScheduler.SCHEDULE_INTERVAL.as_secs()) self._is_purged(room_id) @@ -1501,7 +1502,7 @@ def test_schedule_shutdown_room(self) -> None: self._is_purged(room_id) # Wait for next scheduler run - self.reactor.advance(TaskScheduler.SCHEDULE_INTERVAL_MS) + self.reactor.advance(TaskScheduler.SCHEDULE_INTERVAL.as_secs()) # Test that all users has been kicked (room is shutdown) self._has_no_members(room_id) diff --git a/tests/rest/client/test_transactions.py b/tests/rest/client/test_transactions.py index 0407bb53477..31586a451ff 100644 --- a/tests/rest/client/test_transactions.py +++ b/tests/rest/client/test_transactions.py @@ -29,6 +29,7 @@ from synapse.rest.client.transactions import CLEANUP_PERIOD, HttpTransactionCache from synapse.types import ISynapseReactor, JsonDict from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests import unittest from tests.server import get_clock @@ -93,7 +94,7 @@ def cb() -> Generator["defer.Deferred[object]", object, tuple[int, JsonDict]]: # Ignore `multiple-internal-clocks` linter error here since we are creating a `Clock` # for testing purposes. yield defer.ensureDeferred( - Clock(reactor, server_name="test_server").sleep(0) # type: ignore[multiple-internal-clocks] + Clock(reactor, server_name="test_server").sleep(Duration(seconds=0)) # type: ignore[multiple-internal-clocks] ) return 1, {} diff --git a/tests/server_notices/__init__.py b/tests/server_notices/__init__.py index eca52930db2..19bda218e30 100644 --- a/tests/server_notices/__init__.py +++ b/tests/server_notices/__init__.py @@ -20,6 +20,7 @@ from synapse.server import HomeServer from synapse.types import JsonDict from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests import unittest from tests.unittest import override_config @@ -131,7 +132,7 @@ def _check_user_received_server_notice( break # Sleep and try again. - self.get_success(self.clock.sleep(0.1)) + self.get_success(self.clock.sleep(Duration(milliseconds=100))) else: self.fail( f"Failed to join the server notices room. No 'join' field in sync_body['rooms']: {sync_body['rooms']}" diff --git a/tests/state/test_v2.py b/tests/state/test_v2.py index 7db710846d3..85ce5bede25 100644 --- a/tests/state/test_v2.py +++ b/tests/state/test_v2.py @@ -42,6 +42,7 @@ ) from synapse.storage.databases.main.event_federation import StateDifference from synapse.types import EventID, StateMap +from synapse.util.duration import Duration from tests import unittest @@ -61,7 +62,7 @@ class FakeClock: - async def sleep(self, msec: float) -> None: + async def sleep(self, duration: Duration) -> None: return None diff --git a/tests/state/test_v21.py b/tests/state/test_v21.py index b17773fb562..58d800f921d 100644 --- a/tests/state/test_v21.py +++ b/tests/state/test_v21.py @@ -39,6 +39,7 @@ ) from synapse.types import StateMap from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests import unittest from tests.state.test_v2 import TestStateResolutionStore @@ -66,7 +67,7 @@ def monotonic_timestamp() -> int: class FakeClock: - async def sleep(self, duration_ms: float) -> None: + async def sleep(self, duration: Duration) -> None: defer.succeed(None) diff --git a/tests/storage/databases/main/test_lock.py b/tests/storage/databases/main/test_lock.py index 3743a4a386f..622eb96ded0 100644 --- a/tests/storage/databases/main/test_lock.py +++ b/tests/storage/databases/main/test_lock.py @@ -26,7 +26,7 @@ from twisted.internet.testing import MemoryReactor from synapse.server import HomeServer -from synapse.storage.databases.main.lock import _LOCK_TIMEOUT_MS, _RENEWAL_INTERVAL_MS +from synapse.storage.databases.main.lock import _LOCK_TIMEOUT_MS, _RENEWAL_INTERVAL from synapse.util.clock import Clock from tests import unittest @@ -377,7 +377,7 @@ def test_maintain_lock(self) -> None: # Wait for ages with the lock, we should not be able to get the lock. for _ in range(10): - self.reactor.advance((_RENEWAL_INTERVAL_MS / 1000)) + self.reactor.advance((_RENEWAL_INTERVAL.as_secs())) lock2 = self.get_success( self.store.try_acquire_read_write_lock("name", "key", write=True) diff --git a/tests/storage/test_background_update.py b/tests/storage/test_background_update.py index 3505423691f..e3f79d76707 100644 --- a/tests/storage/test_background_update.py +++ b/tests/storage/test_background_update.py @@ -38,6 +38,7 @@ from synapse.storage.engines import PostgresEngine, Sqlite3Engine from synapse.types import JsonDict from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests import unittest from tests.unittest import override_config @@ -59,7 +60,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: async def update(self, progress: JsonDict, count: int) -> int: duration_ms = 10 - await self.clock.sleep((count * duration_ms) / 1000) + await self.clock.sleep(Duration(milliseconds=count * duration_ms)) progress = {"my_key": progress["my_key"] + 1} await self.store.db_pool.runInteraction( "update_progress", @@ -309,7 +310,7 @@ def test_background_update_min_batch_set_in_config(self) -> None: # Run the update with the long-running update item async def update_long(progress: JsonDict, count: int) -> int: - await self.clock.sleep((count * duration_ms) / 1000) + await self.clock.sleep(Duration(milliseconds=count * duration_ms)) progress = {"my_key": progress["my_key"] + 1} await self.store.db_pool.runInteraction( "update_progress", diff --git a/tests/test_server.py b/tests/test_server.py index 2df6bdfa44a..ec31b6cc5f6 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -38,6 +38,7 @@ from synapse.types import JsonDict from synapse.util.cancellation import cancellable from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests import unittest from tests.http.server._base import test_disconnect @@ -406,11 +407,11 @@ def __init__(self, clock: Clock): @cancellable async def _async_render_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: - await self.clock.sleep(1.0) + await self.clock.sleep(Duration(seconds=1)) return HTTPStatus.OK, {"result": True} async def _async_render_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: - await self.clock.sleep(1.0) + await self.clock.sleep(Duration(seconds=1)) return HTTPStatus.OK, {"result": True} @@ -423,11 +424,11 @@ def __init__(self, clock: Clock): @cancellable async def _async_render_GET(self, request: SynapseRequest) -> tuple[int, bytes]: - await self.clock.sleep(1.0) + await self.clock.sleep(Duration(seconds=1)) return HTTPStatus.OK, b"ok" async def _async_render_POST(self, request: SynapseRequest) -> tuple[int, bytes]: - await self.clock.sleep(1.0) + await self.clock.sleep(Duration(seconds=1)) return HTTPStatus.OK, b"ok" diff --git a/tests/util/caches/test_response_cache.py b/tests/util/caches/test_response_cache.py index 30cd6ef0e48..def5c817db2 100644 --- a/tests/util/caches/test_response_cache.py +++ b/tests/util/caches/test_response_cache.py @@ -26,6 +26,7 @@ from twisted.internet import defer from synapse.util.caches.response_cache import ResponseCache, ResponseCacheContext +from synapse.util.duration import Duration from tests.server import get_clock from tests.unittest import TestCase @@ -55,7 +56,7 @@ async def instant_return(o: str) -> str: return o async def delayed_return(self, o: str) -> str: - await self.clock.sleep(1) + await self.clock.sleep(Duration(seconds=1)) return o def test_cache_hit(self) -> None: @@ -182,7 +183,7 @@ def test_cache_context_nocache(self, should_cache: bool) -> None: async def non_caching(o: str, cache_context: ResponseCacheContext[int]) -> str: nonlocal call_count call_count += 1 - await self.clock.sleep(1) + await self.clock.sleep(Duration(seconds=1)) cache_context.should_cache = should_cache return o diff --git a/tests/util/test_logcontext.py b/tests/util/test_logcontext.py index ca805bb20a0..a4114cdfcc5 100644 --- a/tests/util/test_logcontext.py +++ b/tests/util/test_logcontext.py @@ -37,6 +37,7 @@ ) from synapse.types import ISynapseReactor from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests import unittest from tests.unittest import logcontext_clean @@ -82,7 +83,7 @@ async def competing_callback() -> None: self._check_test_key("sentinel") with LoggingContext(name="competing", server_name="test_server"): - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self._check_test_key("competing") self._check_test_key("sentinel") @@ -94,9 +95,9 @@ async def competing_callback() -> None: reactor.callLater(0, lambda: defer.ensureDeferred(competing_callback())) # type: ignore[call-later-not-tracked] with LoggingContext(name="foo", server_name="test_server"): - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self._check_test_key("foo") - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self._check_test_key("foo") self.assertTrue( @@ -128,7 +129,7 @@ async def competing_callback() -> None: self._check_test_key("looping_call") with LoggingContext(name="competing", server_name="test_server"): - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self._check_test_key("competing") self._check_test_key("looping_call") @@ -139,12 +140,12 @@ async def competing_callback() -> None: with LoggingContext(name="foo", server_name="test_server"): lc = clock.looping_call( - lambda: defer.ensureDeferred(competing_callback()), 0 + lambda: defer.ensureDeferred(competing_callback()), Duration(seconds=0) ) self._check_test_key("foo") - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self._check_test_key("foo") - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self._check_test_key("foo") self.assertTrue( @@ -179,7 +180,7 @@ async def competing_callback() -> None: self._check_test_key("looping_call") with LoggingContext(name="competing", server_name="test_server"): - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self._check_test_key("competing") self._check_test_key("looping_call") @@ -190,10 +191,10 @@ async def competing_callback() -> None: with LoggingContext(name="foo", server_name="test_server"): lc = clock.looping_call_now( - lambda: defer.ensureDeferred(competing_callback()), 0 + lambda: defer.ensureDeferred(competing_callback()), Duration(seconds=0) ) self._check_test_key("foo") - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self._check_test_key("foo") self.assertTrue( @@ -228,7 +229,7 @@ async def competing_callback() -> None: self._check_test_key("call_later") with LoggingContext(name="competing", server_name="test_server"): - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self._check_test_key("competing") self._check_test_key("call_later") @@ -238,11 +239,13 @@ async def competing_callback() -> None: callback_finished = True with LoggingContext(name="foo", server_name="test_server"): - clock.call_later(0, lambda: defer.ensureDeferred(competing_callback())) + clock.call_later( + Duration(seconds=0), lambda: defer.ensureDeferred(competing_callback()) + ) self._check_test_key("foo") - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self._check_test_key("foo") - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self._check_test_key("foo") self.assertTrue( @@ -280,7 +283,7 @@ async def competing_callback() -> None: self._check_test_key("foo") with LoggingContext(name="competing", server_name="test_server"): - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self._check_test_key("competing") self._check_test_key("foo") @@ -303,7 +306,7 @@ async def competing_callback() -> None: await d self._check_test_key("foo") - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self.assertTrue( callback_finished, @@ -338,7 +341,7 @@ async def competing_callback() -> None: self._check_test_key("sentinel") with LoggingContext(name="competing", server_name="test_server"): - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self._check_test_key("competing") self._check_test_key("sentinel") @@ -364,7 +367,7 @@ async def competing_callback() -> None: d.callback(None) self._check_test_key("foo") - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self.assertTrue( callback_finished, @@ -400,7 +403,7 @@ async def competing_callback() -> None: self._check_test_key("foo") with LoggingContext(name="competing", server_name="test_server"): - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self._check_test_key("competing") self._check_test_key("foo") @@ -446,7 +449,7 @@ async def competing_callback() -> None: run_in_background(lambda: (d.callback(None), d)[1]) # type: ignore[call-overload, func-returns-value] self._check_test_key("foo") - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self.assertTrue( callback_finished, @@ -486,7 +489,7 @@ def callback(result: object) -> object: # Now wait for the function under test to have run, and check that # the logcontext is left in a sane state. while not callback_finished: - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self._check_test_key("foo") self.assertTrue( @@ -501,7 +504,7 @@ def callback(result: object) -> object: async def test_run_in_background_with_blocking_fn(self) -> None: async def blocking_function() -> None: # Ignore linter error since we are creating a `Clock` for testing purposes. - await Clock(reactor, server_name="test_server").sleep(0) # type: ignore[multiple-internal-clocks] + await Clock(reactor, server_name="test_server").sleep(Duration(seconds=0)) # type: ignore[multiple-internal-clocks] await self._test_run_in_background(blocking_function) @@ -535,7 +538,9 @@ async def test_run_in_background_with_coroutine(self) -> None: async def testfunc() -> None: self._check_test_key("foo") # Ignore linter error since we are creating a `Clock` for testing purposes. - d = defer.ensureDeferred(Clock(reactor, server_name="test_server").sleep(0)) # type: ignore[multiple-internal-clocks] + d = defer.ensureDeferred( + Clock(reactor, server_name="test_server").sleep(Duration(seconds=0)) # type: ignore[multiple-internal-clocks] + ) self.assertIs(current_context(), SENTINEL_CONTEXT) await d self._check_test_key("foo") @@ -579,7 +584,7 @@ async def competing_callback() -> None: self._check_test_key("foo") with LoggingContext(name="competing", server_name="test_server"): - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self._check_test_key("competing") self._check_test_key("foo") @@ -591,7 +596,7 @@ async def competing_callback() -> None: with LoggingContext(name="foo", server_name="test_server"): run_coroutine_in_background(competing_callback()) self._check_test_key("foo") - await clock.sleep(0) + await clock.sleep(Duration(seconds=0)) self._check_test_key("foo") self.assertTrue( diff --git a/tests/util/test_task_scheduler.py b/tests/util/test_task_scheduler.py index e33ded8a7f3..2c8e21b3398 100644 --- a/tests/util/test_task_scheduler.py +++ b/tests/util/test_task_scheduler.py @@ -26,6 +26,7 @@ from synapse.server import HomeServer from synapse.types import JsonMapping, ScheduledTask, TaskStatus from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.task_scheduler import TaskScheduler from tests.replication._base import BaseMultiWorkerStreamTestCase @@ -68,7 +69,7 @@ def test_schedule_task(self) -> None: # The timestamp being 30s after now the task should been executed # after the first scheduling loop is run - self.reactor.advance(TaskScheduler.SCHEDULE_INTERVAL_MS / 1000) + self.reactor.advance(TaskScheduler.SCHEDULE_INTERVAL.as_secs()) task = self.get_success(self.task_scheduler.get_task(task_id)) assert task is not None @@ -87,7 +88,7 @@ async def _sleeping_task( self, task: ScheduledTask ) -> tuple[TaskStatus, JsonMapping | None, str | None]: # Sleep for a second - await self.hs.get_clock().sleep(1) + await self.hs.get_clock().sleep(Duration(seconds=1)) return TaskStatus.COMPLETE, None, None def test_schedule_lot_of_tasks(self) -> None: @@ -187,7 +188,7 @@ def test_schedule_resumable_task(self) -> None: # Simulate a synapse restart by emptying the list of running tasks self.task_scheduler._running_tasks = set() - self.reactor.advance((TaskScheduler.SCHEDULE_INTERVAL_MS / 1000)) + self.reactor.advance((TaskScheduler.SCHEDULE_INTERVAL.as_secs())) task = self.get_success(self.task_scheduler.get_task(task_id)) assert task is not None From 119f02e3b385652c85c8acbf9d125c1f406e914e Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Mon, 1 Dec 2025 15:24:26 +0000 Subject: [PATCH 024/139] Return 400 when canonical_alias content invalid (#19240) Fixes #19198 Returns HTTP 400 when `alias` or `alt_alias` inside of `m.room.canonical_alias` `content` are not of type string. Previously this resulted in HTTP 500 errors as Synapse assumed they were strings and would raise an exception when it tried to treat them as such if they actually weren't. With the changes implemented: Screenshot from 2025-11-28 16-48-06 Screenshot from 2025-11-28 16-47-42 ### Pull Request Checklist * [X] Pull request is based on the develop branch * [X] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [X] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/19240.bugfix | 1 + synapse/handlers/message.py | 12 ++++++++++++ tests/rest/client/test_rooms.py | 2 ++ 3 files changed, 15 insertions(+) create mode 100644 changelog.d/19240.bugfix diff --git a/changelog.d/19240.bugfix b/changelog.d/19240.bugfix new file mode 100644 index 00000000000..d8490bcc1f8 --- /dev/null +++ b/changelog.d/19240.bugfix @@ -0,0 +1 @@ +Fix bug where invalid `canonical_alias` content would return 500 instead of 400. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index bac4bd9361b..a6499de3a87 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1955,6 +1955,12 @@ async def persist_and_notify_client_events( room_alias_str = event.content.get("alias", None) directory_handler = self.hs.get_directory_handler() if room_alias_str and room_alias_str != original_alias: + if not isinstance(room_alias_str, str): + raise SynapseError( + 400, + "The alias must be of type string.", + Codes.INVALID_PARAM, + ) await self._validate_canonical_alias( directory_handler, room_alias_str, event.room_id ) @@ -1978,6 +1984,12 @@ async def persist_and_notify_client_events( new_alt_aliases = set(alt_aliases) - set(original_alt_aliases) if new_alt_aliases: for alias_str in new_alt_aliases: + if not isinstance(alias_str, str): + raise SynapseError( + 400, + "Each alt_alias must be of type string.", + Codes.INVALID_PARAM, + ) await self._validate_canonical_alias( directory_handler, alias_str, event.room_id ) diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 68e09afc54e..926560afd6b 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -3880,9 +3880,11 @@ def test_bad_data(self) -> None: self._set_canonical_alias({"alt_aliases": False}, expected_code=400) self._set_canonical_alias({"alt_aliases": True}, expected_code=400) self._set_canonical_alias({"alt_aliases": {}}, expected_code=400) + self._set_canonical_alias({"alt_aliases": [0]}, expected_code=400) def test_bad_alias(self) -> None: """An alias which does not point to the room raises a SynapseError.""" + self._set_canonical_alias({"alias": {"@unknown:test": "a"}}, expected_code=400) self._set_canonical_alias({"alias": "@unknown:test"}, expected_code=400) self._set_canonical_alias({"alt_aliases": ["@unknown:test"]}, expected_code=400) From b4ee0bf71e7082abda612a36299e9a73f8e99ac0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Dec 2025 15:38:19 +0000 Subject: [PATCH 025/139] Bump actions/setup-python from 6.0.0 to 6.1.0 (#19245) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/setup-python](https://github.com/actions/setup-python) from 6.0.0 to 6.1.0.
Release notes

Sourced from actions/setup-python's releases.

v6.1.0

What's Changed

Enhancements:

Dependency and Documentation updates:

New Contributors

Full Changelog: https://github.com/actions/setup-python/compare/v6...v6.1.0

Commits
  • 83679a8 Bump @​types/node from 24.1.0 to 24.9.1 and update macos-13 to macos-15-intel ...
  • bfc4944 Bump prettier from 3.5.3 to 3.6.2 (#1234)
  • 97aeb3e Bump requests from 2.32.2 to 2.32.4 in /tests/data (#1130)
  • 443da59 Bump actions/publish-action from 0.3.0 to 0.4.0 & Documentation update for pi...
  • cfd55ca graalpy: add graalpy early-access and windows builds (#880)
  • bba65e5 Bump typescript from 5.4.2 to 5.9.3 and update docs/advanced-usage.md (#1094)
  • 18566f8 Improve wording and "fix example" (remove 3.13) on testing against pre-releas...
  • 2e3e4b1 Add support for pip-install input (#1201)
  • 4267e28 Bump urllib3 from 1.26.19 to 2.5.0 in /tests/data and document breaking c...
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-python&package-manager=github_actions&previous-version=6.0.0&new-version=6.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docs-pr.yaml | 2 +- .github/workflows/docs.yaml | 2 +- .github/workflows/latest_deps.yml | 2 +- .github/workflows/poetry_lockfile.yaml | 2 +- .github/workflows/release-artifacts.yml | 8 ++++---- .github/workflows/schema.yaml | 4 ++-- .github/workflows/tests.yml | 12 ++++++------ 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index d59e0691711..4d28533a272 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -24,7 +24,7 @@ jobs: mdbook-version: '0.4.17' - name: Setup python - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.x" diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index d02428db19f..51944b13e8c 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -64,7 +64,7 @@ jobs: run: echo 'window.SYNAPSE_VERSION = "${{ needs.pre.outputs.branch-version }}";' > ./docs/website_files/version.js - name: Setup python - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.x" diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index eee8dc7e0b8..a4865fea811 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -93,7 +93,7 @@ jobs: -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ postgres:${{ matrix.postgres-version }} - - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.x" - run: pip install .[all,test] diff --git a/.github/workflows/poetry_lockfile.yaml b/.github/workflows/poetry_lockfile.yaml index 62b796287c1..5c139bf5745 100644 --- a/.github/workflows/poetry_lockfile.yaml +++ b/.github/workflows/poetry_lockfile.yaml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: '3.x' - run: pip install tomli diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index e63d65fdf34..03c2b0a3266 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.x" - id: set-distros @@ -74,7 +74,7 @@ jobs: ${{ runner.os }}-buildx- - name: Set up python - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.x" @@ -134,7 +134,7 @@ jobs: steps: - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: # setup-python@v4 doesn't impose a default python version. Need to use 3.x # here, because `python` on osx points to Python 2.7. @@ -171,7 +171,7 @@ jobs: steps: - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.10" diff --git a/.github/workflows/schema.yaml b/.github/workflows/schema.yaml index 52b5cd4c53f..0755a5f0234 100644 --- a/.github/workflows/schema.yaml +++ b/.github/workflows/schema.yaml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.x" - name: Install check-jsonschema @@ -41,7 +41,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.x" - name: Install PyYAML diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c32d018a640..7138268a2b7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -107,7 +107,7 @@ jobs: steps: - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.x" - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20' 'sqlglot>=28.0.0'" @@ -117,7 +117,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.x" - run: .ci/scripts/check_lockfile.py @@ -200,7 +200,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 - - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.x" - run: "pip install 'towncrier>=18.6.0rc1'" @@ -308,7 +308,7 @@ jobs: if: ${{ needs.changes.outputs.linting_readme == 'true' }} steps: - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.x" - run: "pip install rstcheck" @@ -356,7 +356,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.x" - id: get-matrix @@ -447,7 +447,7 @@ jobs: sudo apt-get -qq install build-essential libffi-dev python3-dev \ libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev - - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: '3.10' From bf6163c8bf43c5d1e33ee48ae06ec44ce43681c9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Dec 2025 15:38:50 +0000 Subject: [PATCH 026/139] Bump docker/metadata-action from 5.9.0 to 5.10.0 (#19246) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [docker/metadata-action](https://github.com/docker/metadata-action) from 5.9.0 to 5.10.0.
Release notes

Sourced from docker/metadata-action's releases.

v5.10.0

Full Changelog: https://github.com/docker/metadata-action/compare/v5.9.0...v5.10.0

Commits
  • c299e40 Merge pull request #569 from docker/dependabot/npm_and_yarn/docker/actions-to...
  • f015d79 chore: update generated content
  • 121bcc2 chore(deps): Bump @​docker/actions-toolkit from 0.67.0 to 0.68.0
  • f7b6bf4 Merge pull request #564 from docker/dependabot/npm_and_yarn/js-yaml-3.14.2
  • 0b95c6b Merge pull request #565 from docker/dependabot/github_actions/actions/checkout-6
  • 17f70d7 Merge pull request #568 from motoki317/docs/fix-to-24h-schedule-pattern
  • afd7e6d docs(README): Fix date format from 12h to 24h in schedule pattern
  • 602aff8 chore(deps): Bump actions/checkout from 5 to 6
  • aecb1a4 chore(deps): Bump js-yaml from 3.14.1 to 3.14.2
  • 8d8c7c1 Merge pull request #559 from docker/dependabot/npm_and_yarn/docker/actions-to...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=docker/metadata-action&package-manager=github_actions&previous-version=5.9.0&new-version=5.10.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker.yml | 2 +- .github/workflows/push_complement_image.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d0cdd3acafd..aaf1e22d3c4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -123,7 +123,7 @@ jobs: uses: sigstore/cosign-installer@faadad0cce49287aee09b3a48701e75088a2c6ad # v4.0.0 - name: Calculate docker image tag - uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 # v5.9.0 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 with: images: ${{ matrix.repository }} flavor: | diff --git a/.github/workflows/push_complement_image.yml b/.github/workflows/push_complement_image.yml index c562275a380..ed82482505a 100644 --- a/.github/workflows/push_complement_image.yml +++ b/.github/workflows/push_complement_image.yml @@ -55,7 +55,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Work out labels for complement image id: meta - uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 # v5.9.0 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 with: images: ghcr.io/${{ github.repository }}/complement-synapse tags: | From 58dd25976cfb8f485c4b106c3edab28309bdbcfa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Dec 2025 15:41:40 +0000 Subject: [PATCH 027/139] Bump http from 1.3.1 to 1.4.0 (#19249) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [http](https://github.com/hyperium/http) from 1.3.1 to 1.4.0.
Release notes

Sourced from http's releases.

v1.4.0

Highlights

  • Add StatusCode::EARLY_HINTS constant for 103 Early Hints.
  • Make StatusCode::from_u16 now a const fn.
  • Make Authority::from_static now a const fn.
  • Make PathAndQuery::from_static now a const fn.
  • MSRV increased to 1.57 (allows legible const fn panic messages).

What's Changed

New Contributors

Full Changelog: https://github.com/hyperium/http/compare/v1.3.1...v1.4.0

Changelog

Sourced from http's changelog.

1.4.0 (November 24, 2025)

  • Add StatusCode::EARLY_HINTS constant for 103 Early Hints.
  • Make StatusCode::from_u16 now a const fn.
  • Make Authority::from_static now a const fn.
  • Make PathAndQuery::from_static now a const fn.
  • MSRV increased to 1.57 (allows legible const fn panic messages).
Commits
  • b9625d8 v1.4.0
  • 50b009c refactor(header): inline FNV hasher to reduce dependencies (#796)
  • b370d36 feat(uri): make Authority/PathAndQuery::from_static const (#786)
  • 0d74251 chore(ci): update to actions/checkout@v5 (#800)
  • a760767 docs: remove unnecessary extern crate sentence (#799)
  • fb1d457 refactor(header): use better panic message in const HeaderName and HeaderValu...
  • 20dbd6e feat(status): Add 103 EARLY_HINTS status code (#758)
  • e7a7337 chore: bump MSRV to 1.57
  • 1888e28 tests: downgrade rand back to 0.8 for now
  • 918bbc3 chore: minor improvement for docs (#790)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=http&package-manager=cargo&previous-version=1.3.1&new-version=1.4.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c89d0829ba2..007428a3802 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -374,12 +374,11 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" dependencies = [ "bytes", - "fnv", "itoa", ] From d435cfc125f49ccf4c8897923b498b4fb49b4e54 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Mon, 1 Dec 2025 15:47:36 +0000 Subject: [PATCH 028/139] Add mention of future deprecations to release script (#19239) Small improvement to the release script to prompt the user to consider upcoming deprecations that should be mentioned in the changelog. ### Pull Request Checklist * [X] Pull request is based on the develop branch * [X] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [X] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Olivier 'reivilibre' --- changelog.d/19239.misc | 1 + scripts-dev/release.py | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 changelog.d/19239.misc diff --git a/changelog.d/19239.misc b/changelog.d/19239.misc new file mode 100644 index 00000000000..fd5757eb770 --- /dev/null +++ b/changelog.d/19239.misc @@ -0,0 +1 @@ +Prompt user to consider adding future deprecations to the changelog in release script. diff --git a/scripts-dev/release.py b/scripts-dev/release.py index ba95a19382e..17eadbf6c3b 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -291,6 +291,12 @@ def _prepare() -> None: synapse_repo.git.add("-u") subprocess.run("git diff --cached", shell=True) + print( + "Consider any upcoming platform deprecations that should be mentioned in the changelog. (e.g. upcoming Python, PostgreSQL or SQLite deprecations)" + ) + print( + "Platform deprecations should be mentioned at least 1 release prior to being unsupported." + ) if click.confirm("Edit changelog?", default=False): click.edit(filename="CHANGES.md") From c20dd888bd7e15d84e98ef59fd3228d3578d0db2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andre=20Kl=C3=A4rner?= Date: Mon, 1 Dec 2025 17:05:07 +0100 Subject: [PATCH 029/139] Document how merging config files works - see #11203 (#19243) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --------- Signed-off-by: Andre Klärner Co-authored-by: Olivier 'reivilibre --- changelog.d/19243.doc | 1 + synapse/config/_base.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19243.doc diff --git a/changelog.d/19243.doc b/changelog.d/19243.doc new file mode 100644 index 00000000000..3a396c88d5d --- /dev/null +++ b/changelog.d/19243.doc @@ -0,0 +1 @@ +Document in the `--config-path` help how multiple files are merged - by merging them shallowly. diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 95a00c67180..43dece4a08a 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -672,7 +672,8 @@ def load_or_generate_config( action="append", metavar="CONFIG_FILE", help="Specify config file. Can be given multiple times and" - " may specify directories containing *.yaml files.", + " may specify directories containing *.yaml files." + " Top-level keys in later files overwrite ones in earlier files.", ) parser.add_argument( "--no-secrets-in-config", From 38588f9462c06adcc4b020e0da48164af01a58a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Dec 2025 16:25:31 +0000 Subject: [PATCH 030/139] Bump Swatinem/rust-cache from 2.8.1 to 2.8.2 (#19244) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [Swatinem/rust-cache](https://github.com/swatinem/rust-cache) from 2.8.1 to 2.8.2.
Release notes

Sourced from Swatinem/rust-cache's releases.

v2.8.2

What's Changed

New Contributors

Full Changelog: https://github.com/Swatinem/rust-cache/compare/v2.8.1...v2.8.2

Changelog

Sourced from Swatinem/rust-cache's changelog.

Changelog

2.8.2

  • Don't overwrite env for cargo-metadata call

2.8.1

  • Set empty CARGO_ENCODED_RUSTFLAGS when retrieving metadata
  • Various dependency updates

2.8.0

  • Add support for warpbuild cache provider
  • Add new cache-workspace-crates feature

2.7.8

  • Include CPU arch in the cache key

2.7.7

  • Also cache cargo install metadata

2.7.6

  • Allow opting out of caching $CARGO_HOME/bin
  • Add runner OS in cache key
  • Adds an option to do lookup-only of the cache

2.7.5

  • Support Cargo.lock format cargo-lock v4
  • Only run macOsWorkaround() on macOS

2.7.3

  • Work around upstream problem that causes cache saving to hang for minutes.

2.7.2

  • Only key by Cargo.toml and Cargo.lock files of workspace members.

2.7.1

  • Update toml parser to fix parsing errors.

2.7.0

  • Properly cache trybuild tests.

... (truncated)

Commits
  • 779680d 2.8.2
  • 2ea64ef Bump smol-toml from 1.4.2 to 1.5.2 in the prd-minor group (#287)
  • 8930d9c Bump the actions group with 3 updates (#288)
  • c071727 Bump @​actions/io from 1.1.3 to 2.0.0 in the prd-major group (#281)
  • f2a41b7 Bump @​types/node from 24.9.0 to 24.10.0 in the dev-minor group (#282)
  • e306f83 Don't overwrite env for cargo-metadata call (#285)
  • c911900 Merge pull request #284 from Swatinem/dependabot/github_actions/actions-baeb0...
  • 3aaed55 Bump the actions group with 2 updates
  • 972b315 Merge pull request #283 from Swatinem/dependabot/github_actions/actions-b360d...
  • 07caf06 Bump taiki-e/install-action from 2.62.45 to 2.62.49 in the actions group
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Swatinem/rust-cache&package-manager=github_actions&previous-version=2.8.1&new-version=2.8.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/fix_lint.yaml | 2 +- .github/workflows/latest_deps.yml | 6 +++--- .github/workflows/tests.yml | 24 ++++++++++++------------ .github/workflows/twisted_trunk.yml | 6 +++--- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index a5469be56cc..9daea3f3783 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -25,7 +25,7 @@ jobs: with: toolchain: ${{ env.RUST_VERSION }} components: clippy, rustfmt - - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index a4865fea811..c356ee8e3dd 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -47,7 +47,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 # The dev dependencies aren't exposed in the wheel metadata (at least with current # poetry-core versions), so we install with poetry. @@ -83,7 +83,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - run: sudo apt-get -qq install xmlsec1 - name: Set up PostgreSQL ${{ matrix.postgres-version }} @@ -158,7 +158,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - name: Ensure sytest runs `pip install` # Delete the lockfile so sytest will `pip install` rather than `poetry install` diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7138268a2b7..aff2832b943 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -91,7 +91,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: "3.x" @@ -157,7 +157,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -221,7 +221,7 @@ jobs: with: components: clippy toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - run: cargo clippy -- -D warnings @@ -240,7 +240,7 @@ jobs: with: toolchain: nightly-2025-04-23 components: clippy - - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - run: cargo clippy --all-features -- -D warnings @@ -257,7 +257,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -296,7 +296,7 @@ jobs: # `.rustfmt.toml`. toolchain: nightly-2025-04-23 components: rustfmt - - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - run: cargo fmt --check @@ -394,7 +394,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -438,7 +438,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 # There aren't wheels for some of the older deps, so we need to install # their build dependencies @@ -555,7 +555,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - name: Run SyTest run: /bootstrap.sh synapse @@ -701,7 +701,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh @@ -735,7 +735,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - run: cargo test @@ -755,7 +755,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: nightly-2022-12-01 - - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - run: cargo bench --no-run diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index b07f98b1cb4..325902f131b 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -49,7 +49,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -77,7 +77,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -123,7 +123,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - name: Patch dependencies # Note: The poetry commands want to create a virtualenv in /src/.venv/, From c09298eeaf40438980b76f46ce0ef9c883963160 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Dec 2025 16:45:41 +0000 Subject: [PATCH 031/139] Bump pydantic from 2.12.4 to 2.12.5 (#19250) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [pydantic](https://github.com/pydantic/pydantic) from 2.12.4 to 2.12.5.
Release notes

Sourced from pydantic's releases.

v2.12.5 2025-11-26

v2.12.5 (2025-11-26)

This is the fifth 2.12 patch release, addressing an issue with the MISSING sentinel and providing several documentation improvements.

The next 2.13 minor release will be published in a couple weeks, and will include a new polymorphic serialization feature addressing the remaining unexpected changes to the serialize as any behavior.

  • Fix pickle error when using model_construct() on a model with MISSING as a default value by @​ornariece in #12522.
  • Several updates to the documentation by @​Viicos.

Full Changelog: https://github.com/pydantic/pydantic/compare/v2.12.4...v2.12.5

Changelog

Sourced from pydantic's changelog.

v2.12.5 (2025-11-26)

GitHub release

This is the fifth 2.12 patch release, addressing an issue with the MISSING sentinel and providing several documentation improvements.

The next 2.13 minor release will be published in a couple weeks, and will include a new polymorphic serialization feature addressing the remaining unexpected changes to the serialize as any behavior.

  • Fix pickle error when using model_construct() on a model with MISSING as a default value by @​ornariece in #12522.
  • Several updates to the documentation by @​Viicos.
Commits
  • bd2d0dd Prepare release v2.12.5
  • 7d0302e Document security implications when using create_model()
  • e9ef980 Fix typo in Standard Library Types documentation
  • f2c20c0 Add pydantic-docs dev dependency, make use of versioning blocks
  • a76c1aa Update documentation about JSON Schema
  • 8cbc72c Add documentation about custom __init__()
  • 99eba59 Add additional test for FieldInfo.get_default()
  • c710769 Special case MISSING sentinel in smart_deepcopy()
  • 20a9d77 Do not delete mock validator/serializer in rebuild_dataclass()
  • c86515a Update parts of the model and revalidate_instances documentation
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=pydantic&package-manager=pip&previous-version=2.12.4&new-version=2.12.5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/poetry.lock b/poetry.lock index 35c642fdeb6..ee45f3e4642 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,7 +31,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" files = [ {file = "authlib-1.6.5-py2.py3-none-any.whl", hash = "sha256:3e0e0507807f842b02175507bdee8957a1d5707fd4afb17c32fb43fee90b6e3a"}, {file = "authlib-1.6.5.tar.gz", hash = "sha256:6aaf9c79b7cc96c900f0b284061691c5d4e61221640a948fe690b556a6d6d10b"}, @@ -446,7 +446,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -471,7 +471,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "elementpath-4.1.5-py3-none-any.whl", hash = "sha256:2ac1a2fb31eb22bbbf817f8cf6752f844513216263f0e3892c8e79782fe4bb55"}, {file = "elementpath-4.1.5.tar.gz", hash = "sha256:c2d6dc524b29ef751ecfc416b0627668119d8812441c555d7471da41d4bacb8d"}, @@ -521,7 +521,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "hiredis-3.3.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:9937d9b69321b393fbace69f55423480f098120bc55a3316e1ca3508c4dbbd6f"}, {file = "hiredis-3.3.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:50351b77f89ba6a22aff430b993653847f36b71d444509036baa0f2d79d1ebf4"}, @@ -844,7 +844,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -982,7 +982,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -998,7 +998,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"url-preview\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"url-preview\"" files = [ {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e77dd455b9a16bbd2a5036a63ddbd479c19572af81b624e79ef422f929eef388"}, {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d444858b9f07cefff6455b983aea9a67f7462ba1f6cbe4a21e8bf6791bf2153"}, @@ -1284,7 +1284,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "matrix-synapse-ldap3-0.3.0.tar.gz", hash = "sha256:8bb6517173164d4b9cc44f49de411d8cebdb2e705d5dd1ea1f38733c4a009e1d"}, {file = "matrix_synapse_ldap3-0.3.0-py3-none-any.whl", hash = "sha256:8b4d701f8702551e98cc1d8c20dbed532de5613584c08d0df22de376ba99159d"}, @@ -1526,7 +1526,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1716,7 +1716,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"postgres\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"postgres\"" files = [ {file = "psycopg2-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:103e857f46bb76908768ead4e2d0ba1d1a130e7b8ed77d3ae91e8b33481813e8"}, {file = "psycopg2-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:210daed32e18f35e3140a1ebe059ac29209dd96468f2f7559aa59f75ee82a5cb"}, @@ -1734,7 +1734,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -1750,7 +1750,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -1799,14 +1799,14 @@ files = [ [[package]] name = "pydantic" -version = "2.12.4" +version = "2.12.5" description = "Data validation using Python type hints" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "pydantic-2.12.4-py3-none-any.whl", hash = "sha256:92d3d202a745d46f9be6df459ac5a064fdaa3c1c4cd8adcfa332ccf3c05f871e"}, - {file = "pydantic-2.12.4.tar.gz", hash = "sha256:0f8cb9555000a4b5b617f66bfd2566264c4984b27589d3b845685983e8ea85ac"}, + {file = "pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d"}, + {file = "pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49"}, ] [package.dependencies] @@ -2031,7 +2031,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"cache-memory\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"cache-memory\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2091,7 +2091,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2116,7 +2116,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, @@ -2144,7 +2144,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, @@ -2548,7 +2548,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"sentry\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"sentry\"" files = [ {file = "sentry_sdk-2.46.0-py2.py3-none-any.whl", hash = "sha256:4eeeb60198074dff8d066ea153fa6f241fef1668c10900ea53a4200abc8da9b1"}, {file = "sentry_sdk-2.46.0.tar.gz", hash = "sha256:91821a23460725734b7741523021601593f35731808afc0bb2ba46c27b8acd91"}, @@ -2758,7 +2758,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -2774,7 +2774,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, ] @@ -2847,7 +2847,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "tornado-6.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:f81067dad2e4443b015368b24e802d0083fecada4f0a4572fdb72fc06e54a9a6"}, {file = "tornado-6.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9ac1cbe1db860b3cbb251e795c701c41d343f06a96049d6274e7c77559117e41"}, @@ -2981,7 +2981,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3227,7 +3227,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "xmlschema-2.4.0-py3-none-any.whl", hash = "sha256:dc87be0caaa61f42649899189aab2fd8e0d567f2cf548433ba7b79278d231a4a"}, {file = "xmlschema-2.4.0.tar.gz", hash = "sha256:d74cd0c10866ac609e1ef94a5a69b018ad16e39077bc6393408b40c6babee793"}, From e0e7a44fe99a96ef264d4594f9606e659f5c637a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Dec 2025 16:55:16 +0000 Subject: [PATCH 032/139] Bump pyopenssl from 25.1.0 to 25.3.0 (#19248) --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index ee45f3e4642..d5e5942119a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2066,18 +2066,18 @@ tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] [[package]] name = "pyopenssl" -version = "25.1.0" +version = "25.3.0" description = "Python wrapper module around the OpenSSL library" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "pyopenssl-25.1.0-py3-none-any.whl", hash = "sha256:2b11f239acc47ac2e5aca04fd7fa829800aeee22a2eb30d744572a157bd8a1ab"}, - {file = "pyopenssl-25.1.0.tar.gz", hash = "sha256:8d031884482e0c67ee92bf9a4d8cceb08d92aba7136432ffb0703c5280fc205b"}, + {file = "pyopenssl-25.3.0-py3-none-any.whl", hash = "sha256:1fda6fc034d5e3d179d39e59c1895c9faeaf40a79de5fc4cbbfbe0d36f4a77b6"}, + {file = "pyopenssl-25.3.0.tar.gz", hash = "sha256:c981cb0a3fd84e8602d7afc209522773b94c1c2446a3c710a75b06fe1beae329"}, ] [package.dependencies] -cryptography = ">=41.0.5,<46" +cryptography = ">=45.0.7,<47" typing-extensions = {version = ">=4.9", markers = "python_version < \"3.13\" and python_version >= \"3.8\""} [package.extras] From 3cf21bc64979d7b0cb56270dd641f01b5ff86984 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Dec 2025 16:55:36 +0000 Subject: [PATCH 033/139] Bump rpds-py from 0.29.0 to 0.30.0 (#19247) --- poetry.lock | 232 ++++++++++++++++++++++++++-------------------------- 1 file changed, 116 insertions(+), 116 deletions(-) diff --git a/poetry.lock b/poetry.lock index d5e5942119a..ed0374be321 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2356,127 +2356,127 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "rpds-py" -version = "0.29.0" +version = "0.30.0" description = "Python bindings to Rust's persistent data structures (rpds)" optional = false python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "rpds_py-0.29.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:4ae4b88c6617e1b9e5038ab3fccd7bac0842fdda2b703117b2aa99bc85379113"}, - {file = "rpds_py-0.29.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7d9128ec9d8cecda6f044001fde4fb71ea7c24325336612ef8179091eb9596b9"}, - {file = "rpds_py-0.29.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d37812c3da8e06f2bb35b3cf10e4a7b68e776a706c13058997238762b4e07f4f"}, - {file = "rpds_py-0.29.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:66786c3fb1d8de416a7fa8e1cb1ec6ba0a745b2b0eee42f9b7daa26f1a495545"}, - {file = "rpds_py-0.29.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b58f5c77f1af888b5fd1876c9a0d9858f6f88a39c9dd7c073a88e57e577da66d"}, - {file = "rpds_py-0.29.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:799156ef1f3529ed82c36eb012b5d7a4cf4b6ef556dd7cc192148991d07206ae"}, - {file = "rpds_py-0.29.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:453783477aa4f2d9104c4b59b08c871431647cb7af51b549bbf2d9eb9c827756"}, - {file = "rpds_py-0.29.0-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:24a7231493e3c4a4b30138b50cca089a598e52c34cf60b2f35cebf62f274fdea"}, - {file = "rpds_py-0.29.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7033c1010b1f57bb44d8067e8c25aa6fa2e944dbf46ccc8c92b25043839c3fd2"}, - {file = "rpds_py-0.29.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0248b19405422573621172ab8e3a1f29141362d13d9f72bafa2e28ea0cdca5a2"}, - {file = "rpds_py-0.29.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:f9f436aee28d13b9ad2c764fc273e0457e37c2e61529a07b928346b219fcde3b"}, - {file = "rpds_py-0.29.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24a16cb7163933906c62c272de20ea3c228e4542c8c45c1d7dc2b9913e17369a"}, - {file = "rpds_py-0.29.0-cp310-cp310-win32.whl", hash = "sha256:1a409b0310a566bfd1be82119891fefbdce615ccc8aa558aff7835c27988cbef"}, - {file = "rpds_py-0.29.0-cp310-cp310-win_amd64.whl", hash = "sha256:c5523b0009e7c3c1263471b69d8da1c7d41b3ecb4cb62ef72be206b92040a950"}, - {file = "rpds_py-0.29.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:9b9c764a11fd637e0322a488560533112837f5334ffeb48b1be20f6d98a7b437"}, - {file = "rpds_py-0.29.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3fd2164d73812026ce970d44c3ebd51e019d2a26a4425a5dcbdfa93a34abc383"}, - {file = "rpds_py-0.29.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a097b7f7f7274164566ae90a221fd725363c0e9d243e2e9ed43d195ccc5495c"}, - {file = "rpds_py-0.29.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7cdc0490374e31cedefefaa1520d5fe38e82fde8748cbc926e7284574c714d6b"}, - {file = "rpds_py-0.29.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89ca2e673ddd5bde9b386da9a0aac0cab0e76f40c8f0aaf0d6311b6bbf2aa311"}, - {file = "rpds_py-0.29.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a5d9da3ff5af1ca1249b1adb8ef0573b94c76e6ae880ba1852f033bf429d4588"}, - {file = "rpds_py-0.29.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8238d1d310283e87376c12f658b61e1ee23a14c0e54c7c0ce953efdbdc72deed"}, - {file = "rpds_py-0.29.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:2d6fb2ad1c36f91c4646989811e84b1ea5e0c3cf9690b826b6e32b7965853a63"}, - {file = "rpds_py-0.29.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:534dc9df211387547267ccdb42253aa30527482acb38dd9b21c5c115d66a96d2"}, - {file = "rpds_py-0.29.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d456e64724a075441e4ed648d7f154dc62e9aabff29bcdf723d0c00e9e1d352f"}, - {file = "rpds_py-0.29.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:a738f2da2f565989401bd6fd0b15990a4d1523c6d7fe83f300b7e7d17212feca"}, - {file = "rpds_py-0.29.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a110e14508fd26fd2e472bb541f37c209409876ba601cf57e739e87d8a53cf95"}, - {file = "rpds_py-0.29.0-cp311-cp311-win32.whl", hash = "sha256:923248a56dd8d158389a28934f6f69ebf89f218ef96a6b216a9be6861804d3f4"}, - {file = "rpds_py-0.29.0-cp311-cp311-win_amd64.whl", hash = "sha256:539eb77eb043afcc45314d1be09ea6d6cafb3addc73e0547c171c6d636957f60"}, - {file = "rpds_py-0.29.0-cp311-cp311-win_arm64.whl", hash = "sha256:bdb67151ea81fcf02d8f494703fb728d4d34d24556cbff5f417d74f6f5792e7c"}, - {file = "rpds_py-0.29.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0891cfd8db43e085c0ab93ab7e9b0c8fee84780d436d3b266b113e51e79f954"}, - {file = "rpds_py-0.29.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3897924d3f9a0361472d884051f9a2460358f9a45b1d85a39a158d2f8f1ad71c"}, - {file = "rpds_py-0.29.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a21deb8e0d1571508c6491ce5ea5e25669b1dd4adf1c9d64b6314842f708b5d"}, - {file = "rpds_py-0.29.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9efe71687d6427737a0a2de9ca1c0a216510e6cd08925c44162be23ed7bed2d5"}, - {file = "rpds_py-0.29.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:40f65470919dc189c833e86b2c4bd21bd355f98436a2cef9e0a9a92aebc8e57e"}, - {file = "rpds_py-0.29.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:def48ff59f181130f1a2cb7c517d16328efac3ec03951cca40c1dc2049747e83"}, - {file = "rpds_py-0.29.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad7bd570be92695d89285a4b373006930715b78d96449f686af422debb4d3949"}, - {file = "rpds_py-0.29.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:5a572911cd053137bbff8e3a52d31c5d2dba51d3a67ad902629c70185f3f2181"}, - {file = "rpds_py-0.29.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d583d4403bcbf10cffc3ab5cee23d7643fcc960dff85973fd3c2d6c86e8dbb0c"}, - {file = "rpds_py-0.29.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:070befbb868f257d24c3bb350dbd6e2f645e83731f31264b19d7231dd5c396c7"}, - {file = "rpds_py-0.29.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:fc935f6b20b0c9f919a8ff024739174522abd331978f750a74bb68abd117bd19"}, - {file = "rpds_py-0.29.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8c5a8ecaa44ce2d8d9d20a68a2483a74c07f05d72e94a4dff88906c8807e77b0"}, - {file = "rpds_py-0.29.0-cp312-cp312-win32.whl", hash = "sha256:ba5e1aeaf8dd6d8f6caba1f5539cddda87d511331714b7b5fc908b6cfc3636b7"}, - {file = "rpds_py-0.29.0-cp312-cp312-win_amd64.whl", hash = "sha256:b5f6134faf54b3cb83375db0f113506f8b7770785be1f95a631e7e2892101977"}, - {file = "rpds_py-0.29.0-cp312-cp312-win_arm64.whl", hash = "sha256:b016eddf00dca7944721bf0cd85b6af7f6c4efaf83ee0b37c4133bd39757a8c7"}, - {file = "rpds_py-0.29.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1585648d0760b88292eecab5181f5651111a69d90eff35d6b78aa32998886a61"}, - {file = "rpds_py-0.29.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:521807963971a23996ddaf764c682b3e46459b3c58ccd79fefbe16718db43154"}, - {file = "rpds_py-0.29.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a8896986efaa243ab713c69e6491a4138410f0fe36f2f4c71e18bd5501e8014"}, - {file = "rpds_py-0.29.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1d24564a700ef41480a984c5ebed62b74e6ce5860429b98b1fede76049e953e6"}, - {file = "rpds_py-0.29.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e6596b93c010d386ae46c9fba9bfc9fc5965fa8228edeac51576299182c2e31c"}, - {file = "rpds_py-0.29.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5cc58aac218826d054c7da7f95821eba94125d88be673ff44267bb89d12a5866"}, - {file = "rpds_py-0.29.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de73e40ebc04dd5d9556f50180395322193a78ec247e637e741c1b954810f295"}, - {file = "rpds_py-0.29.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:295ce5ac7f0cf69a651ea75c8f76d02a31f98e5698e82a50a5f4d4982fbbae3b"}, - {file = "rpds_py-0.29.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1ea59b23ea931d494459c8338056fe7d93458c0bf3ecc061cd03916505369d55"}, - {file = "rpds_py-0.29.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f49d41559cebd608042fdcf54ba597a4a7555b49ad5c1c0c03e0af82692661cd"}, - {file = "rpds_py-0.29.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:05a2bd42768ea988294ca328206efbcc66e220d2d9b7836ee5712c07ad6340ea"}, - {file = "rpds_py-0.29.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:33ca7bdfedd83339ca55da3a5e1527ee5870d4b8369456b5777b197756f3ca22"}, - {file = "rpds_py-0.29.0-cp313-cp313-win32.whl", hash = "sha256:20c51ae86a0bb9accc9ad4e6cdeec58d5ebb7f1b09dd4466331fc65e1766aae7"}, - {file = "rpds_py-0.29.0-cp313-cp313-win_amd64.whl", hash = "sha256:6410e66f02803600edb0b1889541f4b5cc298a5ccda0ad789cc50ef23b54813e"}, - {file = "rpds_py-0.29.0-cp313-cp313-win_arm64.whl", hash = "sha256:56838e1cd9174dc23c5691ee29f1d1be9eab357f27efef6bded1328b23e1ced2"}, - {file = "rpds_py-0.29.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:37d94eadf764d16b9a04307f2ab1d7af6dc28774bbe0535c9323101e14877b4c"}, - {file = "rpds_py-0.29.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:d472cf73efe5726a067dce63eebe8215b14beabea7c12606fd9994267b3cfe2b"}, - {file = "rpds_py-0.29.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:72fdfd5ff8992e4636621826371e3ac5f3e3b8323e9d0e48378e9c13c3dac9d0"}, - {file = "rpds_py-0.29.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2549d833abdf8275c901313b9e8ff8fba57e50f6a495035a2a4e30621a2f7cc4"}, - {file = "rpds_py-0.29.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4448dad428f28a6a767c3e3b80cde3446a22a0efbddaa2360f4bb4dc836d0688"}, - {file = "rpds_py-0.29.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:115f48170fd4296a33938d8c11f697f5f26e0472e43d28f35624764173a60e4d"}, - {file = "rpds_py-0.29.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e5bb73ffc029820f4348e9b66b3027493ae00bca6629129cd433fd7a76308ee"}, - {file = "rpds_py-0.29.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:b1581fcde18fcdf42ea2403a16a6b646f8eb1e58d7f90a0ce693da441f76942e"}, - {file = "rpds_py-0.29.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16e9da2bda9eb17ea318b4c335ec9ac1818e88922cbe03a5743ea0da9ecf74fb"}, - {file = "rpds_py-0.29.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:28fd300326dd21198f311534bdb6d7e989dd09b3418b3a91d54a0f384c700967"}, - {file = "rpds_py-0.29.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:2aba991e041d031c7939e1358f583ae405a7bf04804ca806b97a5c0e0af1ea5e"}, - {file = "rpds_py-0.29.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7f437026dbbc3f08c99cc41a5b2570c6e1a1ddbe48ab19a9b814254128d4ea7a"}, - {file = "rpds_py-0.29.0-cp313-cp313t-win32.whl", hash = "sha256:6e97846e9800a5d0fe7be4d008f0c93d0feeb2700da7b1f7528dabafb31dfadb"}, - {file = "rpds_py-0.29.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f49196aec7c4b406495f60e6f947ad71f317a765f956d74bbd83996b9edc0352"}, - {file = "rpds_py-0.29.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:394d27e4453d3b4d82bb85665dc1fcf4b0badc30fc84282defed71643b50e1a1"}, - {file = "rpds_py-0.29.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:55d827b2ae95425d3be9bc9a5838b6c29d664924f98146557f7715e331d06df8"}, - {file = "rpds_py-0.29.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc31a07ed352e5462d3ee1b22e89285f4ce97d5266f6d1169da1142e78045626"}, - {file = "rpds_py-0.29.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c4695dd224212f6105db7ea62197144230b808d6b2bba52238906a2762f1d1e7"}, - {file = "rpds_py-0.29.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcae1770b401167f8b9e1e3f566562e6966ffa9ce63639916248a9e25fa8a244"}, - {file = "rpds_py-0.29.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:90f30d15f45048448b8da21c41703b31c61119c06c216a1bf8c245812a0f0c17"}, - {file = "rpds_py-0.29.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44a91e0ab77bdc0004b43261a4b8cd6d6b451e8d443754cfda830002b5745b32"}, - {file = "rpds_py-0.29.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:4aa195e5804d32c682e453b34474f411ca108e4291c6a0f824ebdc30a91c973c"}, - {file = "rpds_py-0.29.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7971bdb7bf4ee0f7e6f67fa4c7fbc6019d9850cc977d126904392d363f6f8318"}, - {file = "rpds_py-0.29.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8ae33ad9ce580c7a47452c3b3f7d8a9095ef6208e0a0c7e4e2384f9fc5bf8212"}, - {file = "rpds_py-0.29.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:c661132ab2fb4eeede2ef69670fd60da5235209874d001a98f1542f31f2a8a94"}, - {file = "rpds_py-0.29.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:bb78b3a0d31ac1bde132c67015a809948db751cb4e92cdb3f0b242e430b6ed0d"}, - {file = "rpds_py-0.29.0-cp314-cp314-win32.whl", hash = "sha256:f475f103488312e9bd4000bc890a95955a07b2d0b6e8884aef4be56132adbbf1"}, - {file = "rpds_py-0.29.0-cp314-cp314-win_amd64.whl", hash = "sha256:b9cf2359a4fca87cfb6801fae83a76aedf66ee1254a7a151f1341632acf67f1b"}, - {file = "rpds_py-0.29.0-cp314-cp314-win_arm64.whl", hash = "sha256:9ba8028597e824854f0f1733d8b964e914ae3003b22a10c2c664cb6927e0feb9"}, - {file = "rpds_py-0.29.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:e71136fd0612556b35c575dc2726ae04a1669e6a6c378f2240312cf5d1a2ab10"}, - {file = "rpds_py-0.29.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:76fe96632d53f3bf0ea31ede2f53bbe3540cc2736d4aec3b3801b0458499ef3a"}, - {file = "rpds_py-0.29.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9459a33f077130dbb2c7c3cea72ee9932271fb3126404ba2a2661e4fe9eb7b79"}, - {file = "rpds_py-0.29.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5c9546cfdd5d45e562cc0444b6dddc191e625c62e866bf567a2c69487c7ad28a"}, - {file = "rpds_py-0.29.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12597d11d97b8f7e376c88929a6e17acb980e234547c92992f9f7c058f1a7310"}, - {file = "rpds_py-0.29.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28de03cf48b8a9e6ec10318f2197b83946ed91e2891f651a109611be4106ac4b"}, - {file = "rpds_py-0.29.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd7951c964069039acc9d67a8ff1f0a7f34845ae180ca542b17dc1456b1f1808"}, - {file = "rpds_py-0.29.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:c07d107b7316088f1ac0177a7661ca0c6670d443f6fe72e836069025e6266761"}, - {file = "rpds_py-0.29.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1de2345af363d25696969befc0c1688a6cb5e8b1d32b515ef84fc245c6cddba3"}, - {file = "rpds_py-0.29.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:00e56b12d2199ca96068057e1ae7f9998ab6e99cda82431afafd32f3ec98cca9"}, - {file = "rpds_py-0.29.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3919a3bbecee589300ed25000b6944174e07cd20db70552159207b3f4bbb45b8"}, - {file = "rpds_py-0.29.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e7fa2ccc312bbd91e43aa5e0869e46bc03278a3dddb8d58833150a18b0f0283a"}, - {file = "rpds_py-0.29.0-cp314-cp314t-win32.whl", hash = "sha256:97c817863ffc397f1e6a6e9d2d89fe5408c0a9922dac0329672fb0f35c867ea5"}, - {file = "rpds_py-0.29.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2023473f444752f0f82a58dfcbee040d0a1b3d1b3c2ec40e884bd25db6d117d2"}, - {file = "rpds_py-0.29.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:acd82a9e39082dc5f4492d15a6b6c8599aa21db5c35aaf7d6889aea16502c07d"}, - {file = "rpds_py-0.29.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:715b67eac317bf1c7657508170a3e011a1ea6ccb1c9d5f296e20ba14196be6b3"}, - {file = "rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3b1b87a237cb2dba4db18bcfaaa44ba4cd5936b91121b62292ff21df577fc43"}, - {file = "rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1c3c3e8101bb06e337c88eb0c0ede3187131f19d97d43ea0e1c5407ea74c0cbf"}, - {file = "rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2b8e54d6e61f3ecd3abe032065ce83ea63417a24f437e4a3d73d2f85ce7b7cfe"}, - {file = "rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3fbd4e9aebf110473a420dea85a238b254cf8a15acb04b22a5a6b5ce8925b760"}, - {file = "rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80fdf53d36e6c72819993e35d1ebeeb8e8fc688d0c6c2b391b55e335b3afba5a"}, - {file = "rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:ea7173df5d86f625f8dde6d5929629ad811ed8decda3b60ae603903839ac9ac0"}, - {file = "rpds_py-0.29.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:76054d540061eda273274f3d13a21a4abdde90e13eaefdc205db37c05230efce"}, - {file = "rpds_py-0.29.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:9f84c549746a5be3bc7415830747a3a0312573afc9f95785eb35228bb17742ec"}, - {file = "rpds_py-0.29.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:0ea962671af5cb9a260489e311fa22b2e97103e3f9f0caaea6f81390af96a9ed"}, - {file = "rpds_py-0.29.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:f7728653900035fb7b8d06e1e5900545d8088efc9d5d4545782da7df03ec803f"}, - {file = "rpds_py-0.29.0.tar.gz", hash = "sha256:fe55fe686908f50154d1dc599232016e50c243b438c3b7432f24e2895b0e5359"}, + {file = "rpds_py-0.30.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:679ae98e00c0e8d68a7fda324e16b90fd5260945b45d3b824c892cec9eea3288"}, + {file = "rpds_py-0.30.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4cc2206b76b4f576934f0ed374b10d7ca5f457858b157ca52064bdfc26b9fc00"}, + {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:389a2d49eded1896c3d48b0136ead37c48e221b391c052fba3f4055c367f60a6"}, + {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:32c8528634e1bf7121f3de08fa85b138f4e0dc47657866630611b03967f041d7"}, + {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f207f69853edd6f6700b86efb84999651baf3789e78a466431df1331608e5324"}, + {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:67b02ec25ba7a9e8fa74c63b6ca44cf5707f2fbfadae3ee8e7494297d56aa9df"}, + {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c0e95f6819a19965ff420f65578bacb0b00f251fefe2c8b23347c37174271f3"}, + {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:a452763cc5198f2f98898eb98f7569649fe5da666c2dc6b5ddb10fde5a574221"}, + {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e0b65193a413ccc930671c55153a03ee57cecb49e6227204b04fae512eb657a7"}, + {file = "rpds_py-0.30.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:858738e9c32147f78b3ac24dc0edb6610000e56dc0f700fd5f651d0a0f0eb9ff"}, + {file = "rpds_py-0.30.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:da279aa314f00acbb803da1e76fa18666778e8a8f83484fba94526da5de2cba7"}, + {file = "rpds_py-0.30.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7c64d38fb49b6cdeda16ab49e35fe0da2e1e9b34bc38bd78386530f218b37139"}, + {file = "rpds_py-0.30.0-cp310-cp310-win32.whl", hash = "sha256:6de2a32a1665b93233cde140ff8b3467bdb9e2af2b91079f0333a0974d12d464"}, + {file = "rpds_py-0.30.0-cp310-cp310-win_amd64.whl", hash = "sha256:1726859cd0de969f88dc8673bdd954185b9104e05806be64bcd87badbe313169"}, + {file = "rpds_py-0.30.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a2bffea6a4ca9f01b3f8e548302470306689684e61602aa3d141e34da06cf425"}, + {file = "rpds_py-0.30.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dc4f992dfe1e2bc3ebc7444f6c7051b4bc13cd8e33e43511e8ffd13bf407010d"}, + {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:422c3cb9856d80b09d30d2eb255d0754b23e090034e1deb4083f8004bd0761e4"}, + {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:07ae8a593e1c3c6b82ca3292efbe73c30b61332fd612e05abee07c79359f292f"}, + {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12f90dd7557b6bd57f40abe7747e81e0c0b119bef015ea7726e69fe550e394a4"}, + {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99b47d6ad9a6da00bec6aabe5a6279ecd3c06a329d4aa4771034a21e335c3a97"}, + {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33f559f3104504506a44bb666b93a33f5d33133765b0c216a5bf2f1e1503af89"}, + {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:946fe926af6e44f3697abbc305ea168c2c31d3e3ef1058cf68f379bf0335a78d"}, + {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:495aeca4b93d465efde585977365187149e75383ad2684f81519f504f5c13038"}, + {file = "rpds_py-0.30.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9a0ca5da0386dee0655b4ccdf46119df60e0f10da268d04fe7cc87886872ba7"}, + {file = "rpds_py-0.30.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8d6d1cc13664ec13c1b84241204ff3b12f9bb82464b8ad6e7a5d3486975c2eed"}, + {file = "rpds_py-0.30.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3896fa1be39912cf0757753826bc8bdc8ca331a28a7c4ae46b7a21280b06bb85"}, + {file = "rpds_py-0.30.0-cp311-cp311-win32.whl", hash = "sha256:55f66022632205940f1827effeff17c4fa7ae1953d2b74a8581baaefb7d16f8c"}, + {file = "rpds_py-0.30.0-cp311-cp311-win_amd64.whl", hash = "sha256:a51033ff701fca756439d641c0ad09a41d9242fa69121c7d8769604a0a629825"}, + {file = "rpds_py-0.30.0-cp311-cp311-win_arm64.whl", hash = "sha256:47b0ef6231c58f506ef0b74d44e330405caa8428e770fec25329ed2cb971a229"}, + {file = "rpds_py-0.30.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a161f20d9a43006833cd7068375a94d035714d73a172b681d8881820600abfad"}, + {file = "rpds_py-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6abc8880d9d036ecaafe709079969f56e876fcf107f7a8e9920ba6d5a3878d05"}, + {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca28829ae5f5d569bb62a79512c842a03a12576375d5ece7d2cadf8abe96ec28"}, + {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1010ed9524c73b94d15919ca4d41d8780980e1765babf85f9a2f90d247153dd"}, + {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8d1736cfb49381ba528cd5baa46f82fdc65c06e843dab24dd70b63d09121b3f"}, + {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d948b135c4693daff7bc2dcfc4ec57237a29bd37e60c2fabf5aff2bbacf3e2f1"}, + {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47f236970bccb2233267d89173d3ad2703cd36a0e2a6e92d0560d333871a3d23"}, + {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:2e6ecb5a5bcacf59c3f912155044479af1d0b6681280048b338b28e364aca1f6"}, + {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a8fa71a2e078c527c3e9dc9fc5a98c9db40bcc8a92b4e8858e36d329f8684b51"}, + {file = "rpds_py-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73c67f2db7bc334e518d097c6d1e6fed021bbc9b7d678d6cc433478365d1d5f5"}, + {file = "rpds_py-0.30.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5ba103fb455be00f3b1c2076c9d4264bfcb037c976167a6047ed82f23153f02e"}, + {file = "rpds_py-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee9c752c0364588353e627da8a7e808a66873672bcb5f52890c33fd965b394"}, + {file = "rpds_py-0.30.0-cp312-cp312-win32.whl", hash = "sha256:1ab5b83dbcf55acc8b08fc62b796ef672c457b17dbd7820a11d6c52c06839bdf"}, + {file = "rpds_py-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:a090322ca841abd453d43456ac34db46e8b05fd9b3b4ac0c78bcde8b089f959b"}, + {file = "rpds_py-0.30.0-cp312-cp312-win_arm64.whl", hash = "sha256:669b1805bd639dd2989b281be2cfd951c6121b65e729d9b843e9639ef1fd555e"}, + {file = "rpds_py-0.30.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f83424d738204d9770830d35290ff3273fbb02b41f919870479fab14b9d303b2"}, + {file = "rpds_py-0.30.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7536cd91353c5273434b4e003cbda89034d67e7710eab8761fd918ec6c69cf8"}, + {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2771c6c15973347f50fece41fc447c054b7ac2ae0502388ce3b6738cd366e3d4"}, + {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0a59119fc6e3f460315fe9d08149f8102aa322299deaa5cab5b40092345c2136"}, + {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:76fec018282b4ead0364022e3c54b60bf368b9d926877957a8624b58419169b7"}, + {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:692bef75a5525db97318e8cd061542b5a79812d711ea03dbc1f6f8dbb0c5f0d2"}, + {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9027da1ce107104c50c81383cae773ef5c24d296dd11c99e2629dbd7967a20c6"}, + {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:9cf69cdda1f5968a30a359aba2f7f9aa648a9ce4b580d6826437f2b291cfc86e"}, + {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a4796a717bf12b9da9d3ad002519a86063dcac8988b030e405704ef7d74d2d9d"}, + {file = "rpds_py-0.30.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d4c2aa7c50ad4728a094ebd5eb46c452e9cb7edbfdb18f9e1221f597a73e1e7"}, + {file = "rpds_py-0.30.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba81a9203d07805435eb06f536d95a266c21e5b2dfbf6517748ca40c98d19e31"}, + {file = "rpds_py-0.30.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:945dccface01af02675628334f7cf49c2af4c1c904748efc5cf7bbdf0b579f95"}, + {file = "rpds_py-0.30.0-cp313-cp313-win32.whl", hash = "sha256:b40fb160a2db369a194cb27943582b38f79fc4887291417685f3ad693c5a1d5d"}, + {file = "rpds_py-0.30.0-cp313-cp313-win_amd64.whl", hash = "sha256:806f36b1b605e2d6a72716f321f20036b9489d29c51c91f4dd29a3e3afb73b15"}, + {file = "rpds_py-0.30.0-cp313-cp313-win_arm64.whl", hash = "sha256:d96c2086587c7c30d44f31f42eae4eac89b60dabbac18c7669be3700f13c3ce1"}, + {file = "rpds_py-0.30.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:eb0b93f2e5c2189ee831ee43f156ed34e2a89a78a66b98cadad955972548be5a"}, + {file = "rpds_py-0.30.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:922e10f31f303c7c920da8981051ff6d8c1a56207dbdf330d9047f6d30b70e5e"}, + {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdc62c8286ba9bf7f47befdcea13ea0e26bf294bda99758fd90535cbaf408000"}, + {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47f9a91efc418b54fb8190a6b4aa7813a23fb79c51f4bb84e418f5476c38b8db"}, + {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3587eb9b17f3789ad50824084fa6f81921bbf9a795826570bda82cb3ed91f2"}, + {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39c02563fc592411c2c61d26b6c5fe1e51eaa44a75aa2c8735ca88b0d9599daa"}, + {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a1234d8febafdfd33a42d97da7a43f5dcb120c1060e352a3fbc0c6d36e2083"}, + {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:eb2c4071ab598733724c08221091e8d80e89064cd472819285a9ab0f24bcedb9"}, + {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bdfdb946967d816e6adf9a3d8201bfad269c67efe6cefd7093ef959683c8de0"}, + {file = "rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c77afbd5f5250bf27bf516c7c4a016813eb2d3e116139aed0096940c5982da94"}, + {file = "rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:61046904275472a76c8c90c9ccee9013d70a6d0f73eecefd38c1ae7c39045a08"}, + {file = "rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c5f36a861bc4b7da6516dbdf302c55313afa09b81931e8280361a4f6c9a2d27"}, + {file = "rpds_py-0.30.0-cp313-cp313t-win32.whl", hash = "sha256:3d4a69de7a3e50ffc214ae16d79d8fbb0922972da0356dcf4d0fdca2878559c6"}, + {file = "rpds_py-0.30.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f14fc5df50a716f7ece6a80b6c78bb35ea2ca47c499e422aa4463455dd96d56d"}, + {file = "rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:68f19c879420aa08f61203801423f6cd5ac5f0ac4ac82a2368a9fcd6a9a075e0"}, + {file = "rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ec7c4490c672c1a0389d319b3a9cfcd098dcdc4783991553c332a15acf7249be"}, + {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f251c812357a3fed308d684a5079ddfb9d933860fc6de89f2b7ab00da481e65f"}, + {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac98b175585ecf4c0348fd7b29c3864bda53b805c773cbf7bfdaffc8070c976f"}, + {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3e62880792319dbeb7eb866547f2e35973289e7d5696c6e295476448f5b63c87"}, + {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e7fc54e0900ab35d041b0601431b0a0eb495f0851a0639b6ef90f7741b39a18"}, + {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47e77dc9822d3ad616c3d5759ea5631a75e5809d5a28707744ef79d7a1bcfcad"}, + {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b4dc1a6ff022ff85ecafef7979a2c6eb423430e05f1165d6688234e62ba99a07"}, + {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4559c972db3a360808309e06a74628b95eaccbf961c335c8fe0d590cf587456f"}, + {file = "rpds_py-0.30.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0ed177ed9bded28f8deb6ab40c183cd1192aa0de40c12f38be4d59cd33cb5c65"}, + {file = "rpds_py-0.30.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ad1fa8db769b76ea911cb4e10f049d80bf518c104f15b3edb2371cc65375c46f"}, + {file = "rpds_py-0.30.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:46e83c697b1f1c72b50e5ee5adb4353eef7406fb3f2043d64c33f20ad1c2fc53"}, + {file = "rpds_py-0.30.0-cp314-cp314-win32.whl", hash = "sha256:ee454b2a007d57363c2dfd5b6ca4a5d7e2c518938f8ed3b706e37e5d470801ed"}, + {file = "rpds_py-0.30.0-cp314-cp314-win_amd64.whl", hash = "sha256:95f0802447ac2d10bcc69f6dc28fe95fdf17940367b21d34e34c737870758950"}, + {file = "rpds_py-0.30.0-cp314-cp314-win_arm64.whl", hash = "sha256:613aa4771c99f03346e54c3f038e4cc574ac09a3ddfb0e8878487335e96dead6"}, + {file = "rpds_py-0.30.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:7e6ecfcb62edfd632e56983964e6884851786443739dbfe3582947e87274f7cb"}, + {file = "rpds_py-0.30.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a1d0bc22a7cdc173fedebb73ef81e07faef93692b8c1ad3733b67e31e1b6e1b8"}, + {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d08f00679177226c4cb8c5265012eea897c8ca3b93f429e546600c971bcbae7"}, + {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5965af57d5848192c13534f90f9dd16464f3c37aaf166cc1da1cae1fd5a34898"}, + {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a4e86e34e9ab6b667c27f3211ca48f73dba7cd3d90f8d5b11be56e5dbc3fb4e"}, + {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d3e6b26f2c785d65cc25ef1e5267ccbe1b069c5c21b8cc724efee290554419"}, + {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:626a7433c34566535b6e56a1b39a7b17ba961e97ce3b80ec62e6f1312c025551"}, + {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:acd7eb3f4471577b9b5a41baf02a978e8bdeb08b4b355273994f8b87032000a8"}, + {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe5fa731a1fa8a0a56b0977413f8cacac1768dad38d16b3a296712709476fbd5"}, + {file = "rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:74a3243a411126362712ee1524dfc90c650a503502f135d54d1b352bd01f2404"}, + {file = "rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3e8eeb0544f2eb0d2581774be4c3410356eba189529a6b3e36bbbf9696175856"}, + {file = "rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dbd936cde57abfee19ab3213cf9c26be06d60750e60a8e4dd85d1ab12c8b1f40"}, + {file = "rpds_py-0.30.0-cp314-cp314t-win32.whl", hash = "sha256:dc824125c72246d924f7f796b4f63c1e9dc810c7d9e2355864b3c3a73d59ade0"}, + {file = "rpds_py-0.30.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27f4b0e92de5bfbc6f86e43959e6edd1425c33b5e69aab0984a72047f2bcf1e3"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c2262bdba0ad4fc6fb5545660673925c2d2a5d9e2e0fb603aad545427be0fc58"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ee6af14263f25eedc3bb918a3c04245106a42dfd4f5c2285ea6f997b1fc3f89a"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3adbb8179ce342d235c31ab8ec511e66c73faa27a47e076ccc92421add53e2bb"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:250fa00e9543ac9b97ac258bd37367ff5256666122c2d0f2bc97577c60a1818c"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9854cf4f488b3d57b9aaeb105f06d78e5529d3145b1e4a41750167e8c213c6d3"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:993914b8e560023bc0a8bf742c5f303551992dcb85e247b1e5c7f4a7d145bda5"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58edca431fb9b29950807e301826586e5bbf24163677732429770a697ffe6738"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:dea5b552272a944763b34394d04577cf0f9bd013207bc32323b5a89a53cf9c2f"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ba3af48635eb83d03f6c9735dfb21785303e73d22ad03d489e88adae6eab8877"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:dff13836529b921e22f15cb099751209a60009731a68519630a24d61f0b1b30a"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:1b151685b23929ab7beec71080a8889d4d6d9fa9a983d213f07121205d48e2c4"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:ac37f9f516c51e5753f27dfdef11a88330f04de2d564be3991384b2f3535d02e"}, + {file = "rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84"}, ] [[package]] From afdf9af6b543a8e3b3ae4f90842a23b04d519659 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Dec 2025 17:55:33 +0000 Subject: [PATCH 034/139] Bump types-jsonschema from 4.25.1.20250822 to 4.25.1.20251009 (#19252) Bumps [types-jsonschema](https://github.com/typeshed-internal/stub_uploader) from 4.25.1.20250822 to 4.25.1.20251009.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=types-jsonschema&package-manager=pip&previous-version=4.25.1.20250822&new-version=4.25.1.20251009)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index ed0374be321..a6de2a86f08 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3035,14 +3035,14 @@ files = [ [[package]] name = "types-jsonschema" -version = "4.25.1.20250822" +version = "4.25.1.20251009" description = "Typing stubs for jsonschema" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types_jsonschema-4.25.1.20250822-py3-none-any.whl", hash = "sha256:f82c2d7fa1ce1c0b84ba1de4ed6798469768188884db04e66421913a4e181294"}, - {file = "types_jsonschema-4.25.1.20250822.tar.gz", hash = "sha256:aac69ed4b23f49aaceb7fcb834141d61b9e4e6a7f6008cb2f0d3b831dfa8464a"}, + {file = "types_jsonschema-4.25.1.20251009-py3-none-any.whl", hash = "sha256:f30b329037b78e7a60146b1146feb0b6fb0b71628637584409bada83968dad3e"}, + {file = "types_jsonschema-4.25.1.20251009.tar.gz", hash = "sha256:75d0f5c5dd18dc23b664437a0c1a625743e8d2e665ceaf3aecb29841f3a5f97f"}, ] [package.dependencies] From 08e1b63b30d889ba16f94d286b453aa79b0fead4 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Mon, 1 Dec 2025 18:26:43 +0000 Subject: [PATCH 035/139] Fix v12 rooms when using frozen dicts (#19235) Fix #19233 Synapse fails to handle events in v12 rooms when the server is run with the `{use_frozen_dicts: True}` config. This PR fixes the issue, and adds tests which cover room creation, joining, and joining over federation, with both frozen and not frozen config settings, by extending the existing `test_send_join` federation tests. This approach to testing was chosen as it is a simple way to get high level integration style test coverage, without going through all our existing tests and trying to retroactively add in coverage when using frozen dicts. This should provide an easy place for future room versions to extend the suite of tests and reduce the chance of introducing subtle bugs like this in the future. ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/19235.bugfix | 1 + synapse/events/__init__.py | 2 +- tests/federation/test_federation_server.py | 56 ++++++++++++++++++---- 3 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 changelog.d/19235.bugfix diff --git a/changelog.d/19235.bugfix b/changelog.d/19235.bugfix new file mode 100644 index 00000000000..1c312351a41 --- /dev/null +++ b/changelog.d/19235.bugfix @@ -0,0 +1 @@ +Fix v12 rooms when running with `use_frozen_dicts: True`. diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 5f786037824..c7eaf7eda2b 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -548,7 +548,7 @@ def auth_event_ids(self) -> StrCollection: assert create_event_id not in self._dict["auth_events"] if self.type == EventTypes.Create and self.get_state_key() == "": return self._dict["auth_events"] # should be [] - return self._dict["auth_events"] + [create_event_id] + return [*self._dict["auth_events"], create_event_id] def _event_type_from_format_version( diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index 0d747912906..c4491d5b3c9 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -30,6 +30,7 @@ from synapse.api.errors import FederationError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.config.server import DEFAULT_ROOM_VERSION +from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.events import EventBase, make_event_from_dict from synapse.federation.federation_base import event_from_pdu_json from synapse.http.types import QueryParams @@ -356,19 +357,44 @@ def _make_join(self, user_id: str) -> JsonDict: self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) return channel.json_body - def test_send_join(self) -> None: + def _test_send_join_common(self, room_version: str) -> None: """happy-path test of send_join""" + creator_user_id = self.register_user(f"kermit_v{room_version}", "test") + tok = self.login(f"kermit_v{room_version}", "test") + room_id = self.helper.create_room_as( + room_creator=creator_user_id, tok=tok, room_version=room_version + ) + + # Second member joins + second_member_user_id = self.register_user(f"fozzie_v{room_version}", "bear") + tok2 = self.login(f"fozzie_v{room_version}", "bear") + self.helper.join(room_id, second_member_user_id, tok=tok2) + + # Make join for remote user joining_user = "@misspiggy:" + self.OTHER_SERVER_NAME - join_result = self._make_join(joining_user) + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/make_join/{room_id}/{joining_user}?ver={room_version}", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + join_result = channel.json_body + # Sign and send the join join_event_dict = join_result["event"] self.add_hashes_and_signatures_from_other_server( join_event_dict, - KNOWN_ROOM_VERSIONS[DEFAULT_ROOM_VERSION], + KNOWN_ROOM_VERSIONS[room_version], ) + if room_version in ["1", "2"]: + add_hashes_and_signatures( + KNOWN_ROOM_VERSIONS[room_version], + join_event_dict, + signature_name=self.hs.hostname, + signing_key=self.hs.signing_key, + ) channel = self.make_signed_federation_request( "PUT", - f"/_matrix/federation/v2/send_join/{self._room_id}/x", + f"/_matrix/federation/v2/send_join/{room_id}/x", content=join_event_dict, ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) @@ -384,8 +410,8 @@ def test_send_join(self) -> None: ("m.room.power_levels", ""), ("m.room.join_rules", ""), ("m.room.history_visibility", ""), - ("m.room.member", "@kermit:test"), - ("m.room.member", "@fozzie:test"), + ("m.room.member", f"@kermit_v{room_version}:test"), + ("m.room.member", f"@fozzie_v{room_version}:test"), # nb: *not* the joining user ], ) @@ -398,18 +424,28 @@ def test_send_join(self) -> None: returned_auth_chain_events, [ ("m.room.create", ""), - ("m.room.member", "@kermit:test"), + ("m.room.member", f"@kermit_v{room_version}:test"), ("m.room.power_levels", ""), ("m.room.join_rules", ""), ], ) # the room should show that the new user is a member - r = self.get_success( - self._storage_controllers.state.get_current_state(self._room_id) - ) + r = self.get_success(self._storage_controllers.state.get_current_state(room_id)) self.assertEqual(r[("m.room.member", joining_user)].membership, "join") + @parameterized.expand([(k,) for k in KNOWN_ROOM_VERSIONS.keys()]) + @override_config({"use_frozen_dicts": True}) + def test_send_join_with_frozen_dicts(self, room_version: str) -> None: + """Test send_join with USE_FROZEN_DICTS=True""" + self._test_send_join_common(room_version) + + @parameterized.expand([(k,) for k in KNOWN_ROOM_VERSIONS.keys()]) + @override_config({"use_frozen_dicts": False}) + def test_send_join_without_frozen_dicts(self, room_version: str) -> None: + """Test send_join with USE_FROZEN_DICTS=False""" + self._test_send_join_common(room_version) + def test_send_join_partial_state(self) -> None: """/send_join should return partial state, if requested""" joining_user = "@misspiggy:" + self.OTHER_SERVER_NAME From 88310fe7eddde237f50b17b109d1feed4e38b299 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 1 Dec 2025 17:10:22 -0600 Subject: [PATCH 036/139] Add log to determine whether clients are using `/messages` as expected (#19226) Spawning from wanting some better homeserver logs to debug https://github.com/element-hq/synapse/issues/19153. We can check whether we are returning a `/messages` response with an `end` pagination token and then check to see whether the client is making another `/messages` request with that token. Although clients should also have similar logs and debugging capabilities to determine this info as well. This just makes it easier for us when someone creates an issue claiming backend issue and we can ask them for homeserver logs. --- changelog.d/19226.misc | 1 + synapse/handlers/pagination.py | 116 ++++++++++++++++++++++----------- synapse/rest/admin/rooms.py | 35 +++++++++- synapse/rest/client/room.py | 100 +++++++++++++++++++++++++++- 4 files changed, 210 insertions(+), 42 deletions(-) create mode 100644 changelog.d/19226.misc diff --git a/changelog.d/19226.misc b/changelog.d/19226.misc new file mode 100644 index 00000000000..c38d1d3ef69 --- /dev/null +++ b/changelog.d/19226.misc @@ -0,0 +1 @@ +Add log to determine whether clients are using `/messages` as expected. diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index f869a41c5ed..63e5dfa70c5 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -21,22 +21,25 @@ import logging from typing import TYPE_CHECKING, cast +import attr + from twisted.python.failure import Failure from synapse.api.constants import Direction, EventTypes, Membership from synapse.api.errors import SynapseError from synapse.api.filtering import Filter -from synapse.events.utils import SerializeEventConfig +from synapse.events import EventBase +from synapse.handlers.relations import BundledAggregations from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.logging.opentracing import trace from synapse.rest.admin._base import assert_user_is_admin from synapse.streams.config import PaginationConfig from synapse.types import ( - JsonDict, JsonMapping, Requester, ScheduledTask, StreamKeyType, + StreamToken, TaskStatus, ) from synapse.types.handlers import ShutdownRoomParams, ShutdownRoomResponse @@ -70,6 +73,58 @@ SHUTDOWN_AND_PURGE_ROOM_ACTION_NAME = "shutdown_and_purge_room" +@attr.s(slots=True, frozen=True, auto_attribs=True) +class GetMessagesResult: + """ + Everything needed to serialize a `/messages` response. + """ + + messages_chunk: list[EventBase] + """ + A list of room events. + + - When the request is `Direction.FORWARDS`, events will be in the range: + `start_token` < x <= `end_token`, (ascending topological_order) + - When the request is `Direction.BACKWARDS`, events will be in the range: + `start_token` >= x > `end_token`, (descending topological_order) + + Note that an empty chunk does not necessarily imply that no more events are + available. Clients should continue to paginate until no `end_token` property is returned. + """ + + bundled_aggregations: dict[str, BundledAggregations] + """ + A map of event ID to the bundled aggregations for the events in the chunk. + + If an event doesn't have any bundled aggregations, it may not appear in the map. + """ + + state: list[EventBase] | None + """ + A list of state events relevant to showing the chunk. For example, if + lazy_load_members is enabled in the filter then this may contain the membership + events for the senders of events in the chunk. + + Omitted from the response when `None`. + """ + + start_token: StreamToken + """ + Token corresponding to the start of chunk. This will be the same as the value given + in `from` query parameter of the `/messages` request. + """ + + end_token: StreamToken | None + """ + A token corresponding to the end of chunk. This token can be passed back to this + endpoint to request further events. + + If no further events are available (either because we have reached the start of the + timeline, or because the user does not have permission to see any more events), this + property is omitted from the response. + """ + + class PaginationHandler: """Handles pagination and purge history requests. @@ -418,7 +473,7 @@ async def get_messages( as_client_event: bool = True, event_filter: Filter | None = None, use_admin_priviledge: bool = False, - ) -> JsonDict: + ) -> GetMessagesResult: """Get messages in a room. Args: @@ -617,10 +672,13 @@ async def get_messages( # In that case we do not return end, to tell the client # there is no need for further queries. if not events: - return { - "chunk": [], - "start": await from_token.to_string(self.store), - } + return GetMessagesResult( + messages_chunk=[], + bundled_aggregations={}, + state=None, + start_token=from_token, + end_token=None, + ) if event_filter: events = await event_filter.filter(events) @@ -636,11 +694,13 @@ async def get_messages( # if after the filter applied there are no more events # return immediately - but there might be more in next_token batch if not events: - return { - "chunk": [], - "start": await from_token.to_string(self.store), - "end": await next_token.to_string(self.store), - } + return GetMessagesResult( + messages_chunk=[], + bundled_aggregations={}, + state=None, + start_token=from_token, + end_token=next_token, + ) state = None if event_filter and event_filter.lazy_load_members and len(events) > 0: @@ -657,38 +717,20 @@ async def get_messages( if state_ids: state_dict = await self.store.get_events(list(state_ids.values())) - state = state_dict.values() + state = list(state_dict.values()) aggregations = await self._relations_handler.get_bundled_aggregations( events, user_id ) - time_now = self.clock.time_msec() - - serialize_options = SerializeEventConfig( - as_client_event=as_client_event, requester=requester + return GetMessagesResult( + messages_chunk=events, + bundled_aggregations=aggregations, + state=state, + start_token=from_token, + end_token=next_token, ) - chunk = { - "chunk": ( - await self._event_serializer.serialize_events( - events, - time_now, - config=serialize_options, - bundle_aggregations=aggregations, - ) - ), - "start": await from_token.to_string(self.store), - "end": await next_token.to_string(self.store), - } - - if state: - chunk["state"] = await self._event_serializer.serialize_events( - state, time_now, config=serialize_options - ) - - return chunk - async def _shutdown_and_purge_room( self, task: ScheduledTask, diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index cf24bc628ab..a886859ffab 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -28,9 +28,13 @@ from synapse.api.constants import Direction, EventTypes, JoinRules, Membership from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError from synapse.api.filtering import Filter +from synapse.events.utils import ( + SerializeEventConfig, +) from synapse.handlers.pagination import ( PURGE_ROOM_ACTION_NAME, SHUTDOWN_AND_PURGE_ROOM_ACTION_NAME, + GetMessagesResult, ) from synapse.http.servlet import ( ResolveRoomIdMixin, @@ -44,11 +48,13 @@ parse_string, ) from synapse.http.site import SynapseRequest +from synapse.logging.opentracing import trace from synapse.rest.admin._base import ( admin_patterns, assert_requester_is_admin, assert_user_is_admin, ) +from synapse.rest.client.room import SerializeMessagesDeps, encode_messages_response from synapse.storage.databases.main.room import RoomSortOrder from synapse.streams.config import PaginationConfig from synapse.types import JsonDict, RoomID, ScheduledTask, UserID, create_requester @@ -976,6 +982,7 @@ def __init__(self, hs: "HomeServer"): self._pagination_handler = hs.get_pagination_handler() self._auth = hs.get_auth() self._store = hs.get_datastores().main + self._event_serializer = hs.get_event_client_serializer() async def on_GET( self, request: SynapseRequest, room_id: str @@ -999,7 +1006,11 @@ async def on_GET( ): as_client_event = False - msgs = await self._pagination_handler.get_messages( + serialize_options = SerializeEventConfig( + as_client_event=as_client_event, requester=requester + ) + + get_messages_result = await self._pagination_handler.get_messages( room_id=room_id, requester=requester, pagin_config=pagination_config, @@ -1008,7 +1019,27 @@ async def on_GET( use_admin_priviledge=True, ) - return HTTPStatus.OK, msgs + response_content = await self.encode_response( + get_messages_result, serialize_options + ) + + return HTTPStatus.OK, response_content + + @trace + async def encode_response( + self, + get_messages_result: GetMessagesResult, + serialize_options: SerializeEventConfig, + ) -> JsonDict: + return await encode_messages_response( + get_messages_result=get_messages_result, + serialize_options=serialize_options, + serialize_deps=SerializeMessagesDeps( + clock=self._clock, + event_serializer=self._event_serializer, + store=self._store, + ), + ) class RoomTimestampToEventRestServlet(RestServlet): diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 81a6bd57fc3..5e7dcb01911 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -28,6 +28,7 @@ from typing import TYPE_CHECKING, Awaitable from urllib import parse as urlparse +import attr from prometheus_client.core import Histogram from twisted.web.server import Request @@ -45,10 +46,12 @@ ) from synapse.api.filtering import Filter from synapse.events.utils import ( + EventClientSerializer, SerializeEventConfig, format_event_for_client_v2, serialize_event, ) +from synapse.handlers.pagination import GetMessagesResult from synapse.http.server import HttpServer from synapse.http.servlet import ( ResolveRoomIdMixin, @@ -64,15 +67,17 @@ ) from synapse.http.site import SynapseRequest from synapse.logging.context import make_deferred_yieldable, run_in_background -from synapse.logging.opentracing import set_tag +from synapse.logging.opentracing import set_tag, trace from synapse.metrics import SERVER_NAME_LABEL from synapse.rest.client._base import client_patterns from synapse.rest.client.transactions import HttpTransactionCache from synapse.state import CREATE_KEY, POWER_KEY +from synapse.storage.databases.main import DataStore from synapse.streams.config import PaginationConfig from synapse.types import JsonDict, Requester, StreamToken, ThirdPartyInstanceID, UserID from synapse.types.state import StateFilter from synapse.util.cancellation import cancellable +from synapse.util.clock import Clock from synapse.util.events import generate_fake_event_id from synapse.util.stringutils import parse_and_validate_server_name @@ -790,6 +795,56 @@ async def on_GET( return 200, {"joined": users_with_profile} +@attr.s(slots=True, frozen=True, auto_attribs=True) +class SerializeMessagesDeps: + clock: Clock + event_serializer: EventClientSerializer + store: DataStore + + +@trace +async def encode_messages_response( + *, + get_messages_result: GetMessagesResult, + serialize_options: SerializeEventConfig, + serialize_deps: SerializeMessagesDeps, +) -> JsonDict: + """ + Serialize a `GetMessagesResult` into the JSON response format for the `/messages` + endpoint. + + This logic is shared between the client API and Synapse admin API. + """ + + time_now = serialize_deps.clock.time_msec() + + serialized_result = { + "chunk": ( + await serialize_deps.event_serializer.serialize_events( + get_messages_result.messages_chunk, + time_now, + config=serialize_options, + bundle_aggregations=get_messages_result.bundled_aggregations, + ) + ), + "start": await get_messages_result.start_token.to_string(serialize_deps.store), + } + + if get_messages_result.end_token is not None: + serialized_result["end"] = await get_messages_result.end_token.to_string( + serialize_deps.store + ) + + if get_messages_result.state is not None: + serialized_result[ + "state" + ] = await serialize_deps.event_serializer.serialize_events( + get_messages_result.state, time_now, config=serialize_options + ) + + return serialized_result + + # TODO: Needs better unit testing class RoomMessageListRestServlet(RestServlet): PATTERNS = client_patterns("/rooms/(?P[^/]*)/messages$", v1=True) @@ -806,6 +861,7 @@ def __init__(self, hs: "HomeServer"): self.pagination_handler = hs.get_pagination_handler() self.auth = hs.get_auth() self.store = hs.get_datastores().main + self.event_serializer = hs.get_event_client_serializer() async def on_GET( self, request: SynapseRequest, room_id: str @@ -839,7 +895,11 @@ async def on_GET( ): as_client_event = False - msgs = await self.pagination_handler.get_messages( + serialize_options = SerializeEventConfig( + as_client_event=as_client_event, requester=requester + ) + + get_messages_result = await self.pagination_handler.get_messages( room_id=room_id, requester=requester, pagin_config=pagination_config, @@ -847,6 +907,24 @@ async def on_GET( event_filter=event_filter, ) + # Useful for debugging timeline/pagination issues. For example, if a client + # isn't seeing the full history, we can check the homeserver logs to see if the + # client just never made the next request with the given `end` token. + logger.info( + "Responding to `/messages` request: {%s} %s %s -> %d messages with end_token=%s", + requester.user.to_string(), + request.get_method(), + request.get_redacted_uri(), + len(get_messages_result.messages_chunk), + (await get_messages_result.end_token.to_string(self.store)) + if get_messages_result.end_token + else None, + ) + + response_content = await self.encode_response( + get_messages_result, serialize_options + ) + processing_end_time = self.clock.time_msec() room_member_count = await make_deferred_yieldable(room_member_count_deferred) messsages_response_timer.labels( @@ -854,7 +932,23 @@ async def on_GET( **{SERVER_NAME_LABEL: self.server_name}, ).observe((processing_end_time - processing_start_time) / 1000) - return 200, msgs + return 200, response_content + + @trace + async def encode_response( + self, + get_messages_result: GetMessagesResult, + serialize_options: SerializeEventConfig, + ) -> JsonDict: + return await encode_messages_response( + get_messages_result=get_messages_result, + serialize_options=serialize_options, + serialize_deps=SerializeMessagesDeps( + clock=self.clock, + event_serializer=self.event_serializer, + store=self.store, + ), + ) # TODO: Needs unit testing From a8e5c319ab952db39b74cba87e9a7344bffdf204 Mon Sep 17 00:00:00 2001 From: Patrick Maier Date: Tue, 2 Dec 2025 12:09:18 +0100 Subject: [PATCH 037/139] Simplify README and add ESS Getting started section (#19228) - Add reference to ESS for getting started easily with Synapse/Matrix/Element - Remove details on standalone installations and link to the docs - Other updates like copyright --- README.rst | 202 +++++++++++------------------------------ changelog.d/19228.misc | 1 + 2 files changed, 55 insertions(+), 148 deletions(-) create mode 100644 changelog.d/19228.misc diff --git a/README.rst b/README.rst index d10b662d1ad..9ea397cd0f4 100644 --- a/README.rst +++ b/README.rst @@ -7,81 +7,61 @@ Synapse is an open source `Matrix `__ homeserver implementation, written and maintained by `Element `_. -`Matrix `__ is the open standard for -secure and interoperable real-time communications. You can directly run -and manage the source code in this repository, available under an AGPL -license (or alternatively under a commercial license from Element). -There is no support provided by Element unless you have a -subscription from Element. +`Matrix `__ is the open standard for secure and +interoperable real-time communications. You can directly run and manage the +source code in this repository, available under an AGPL license (or +alternatively under a commercial license from Element). -Subscription -============ +There is no support provided by Element unless you have a subscription from +Element. -For those that need an enterprise-ready solution, Element -Server Suite (ESS) is `available via subscription `_. -ESS builds on Synapse to offer a complete Matrix-based backend including the full -`Admin Console product `_, -giving admins the power to easily manage an organization-wide -deployment. It includes advanced identity management, auditing, -moderation and data retention options as well as Long-Term Support and -SLAs. ESS supports any Matrix-compatible client. +🚀 Getting started +================== -.. contents:: +This component is developed and maintained by `Element `_. +It gets shipped as part of the **Element Server Suite (ESS)** which provides the +official means of deployment. -🛠️ Installation and configuration -================================== +ESS is a Matrix distribution from Element with focus on quality and ease of use. +It ships a full Matrix stack tailored to the respective use case. -The Synapse documentation describes `how to install Synapse `_. We recommend using -`Docker images `_ or `Debian packages from Matrix.org -`_. +There are three editions of ESS: -.. _federation: +- `ESS Community `_ - the free Matrix + distribution from Element tailored to small-/mid-scale, non-commercial + community use cases +- `ESS Pro `_ - the commercial Matrix + distribution from Element for professional use +- `ESS TI-M `_ - a special version + of ESS Pro focused on the requirements of TI-Messenger Pro and ePA as + specified by the German National Digital Health Agency Gematik -Synapse has a variety of `config options -`_ -which can be used to customise its behaviour after installation. -There are additional details on how to `configure Synapse for federation here -`_. -.. _reverse-proxy: +🛠️ Standalone installation and configuration +============================================ -Using a reverse proxy with Synapse ----------------------------------- - -It is recommended to put a reverse proxy such as -`nginx `_, -`Apache `_, -`Caddy `_, -`HAProxy `_ or -`relayd `_ in front of Synapse. One advantage of -doing so is that it means that you can expose the default https port (443) to -Matrix clients without needing to run Synapse with root privileges. -For information on configuring one, see `the reverse proxy docs -`_. - -Upgrading an existing Synapse ------------------------------ - -The instructions for upgrading Synapse are in `the upgrade notes`_. -Please check these instructions as upgrading may require extra steps for some -versions of Synapse. - -.. _the upgrade notes: https://element-hq.github.io/synapse/develop/upgrade.html +The Synapse documentation describes `options for installing Synapse standalone +`_. See +below for more useful documenation links. +- `Synapse configuration options `_ +- `Synapse configuration for federation `_ +- `Using a reverse proxy with Synapse `_ +- `Upgrading Synapse `_ Platform dependencies --------------------- Synapse uses a number of platform dependencies such as Python and PostgreSQL, -and aims to follow supported upstream versions. See the -`deprecation policy `_ -for more details. +and aims to follow supported upstream versions. See the `deprecation policy +`_ for more +details. Security note ------------- -Matrix serves raw, user-supplied data in some APIs -- specifically the `content +Matrix serves raw, user-supplied data in some APIs — specifically the `content repository endpoints`_. .. _content repository endpoints: https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid @@ -118,60 +98,6 @@ mentioned in MXIDs hosted on that server. Following this advice ensures that even if an XSS is found in Synapse, the impact to other applications will be minimal. - -🧪 Testing a new installation -============================= - -The easiest way to try out your new Synapse installation is by connecting to it -from a web client. - -Unless you are running a test instance of Synapse on your local machine, in -general, you will need to enable TLS support before you can successfully -connect from a client: see -`TLS certificates `_. - -An easy way to get started is to login or register via Element at -https://app.element.io/#/login or https://app.element.io/#/register respectively. -You will need to change the server you are logging into from ``matrix.org`` -and instead specify a homeserver URL of ``https://:8448`` -(or just ``https://`` if you are using a reverse proxy). -If you prefer to use another client, refer to our -`client breakdown `_. - -If all goes well you should at least be able to log in, create a room, and -start sending messages. - -.. _`client-user-reg`: - -Registering a new user from a client ------------------------------------- - -By default, registration of new users via Matrix clients is disabled. To enable -it: - -1. In the - `registration config section `_ - set ``enable_registration: true`` in ``homeserver.yaml``. -2. Then **either**: - - a. set up a `CAPTCHA `_, or - b. set ``enable_registration_without_verification: true`` in ``homeserver.yaml``. - -We **strongly** recommend using a CAPTCHA, particularly if your homeserver is exposed to -the public internet. Without it, anyone can freely register accounts on your homeserver. -This can be exploited by attackers to create spambots targeting the rest of the Matrix -federation. - -Your new Matrix ID will be formed partly from the ``server_name``, and partly -from a localpart you specify when you create the account in the form of:: - - @localpart:my.domain.name - -(pronounced "at localpart on my dot domain dot name"). - -As when logging in, you will need to specify a "Custom server". Specify your -desired ``localpart`` in the 'Username' box. - 🎯 Troubleshooting and support ============================== @@ -182,7 +108,7 @@ Enterprise quality support for Synapse including SLAs is available as part of an `Element Server Suite (ESS) `_ subscription. If you are an existing ESS subscriber then you can raise a `support request `_ -and access the `knowledge base `_. +and access the `Element product documentation `_. 🤝 Community support -------------------- @@ -201,35 +127,6 @@ issues for support requests, only for bug reports and feature requests. .. |docs| replace:: ``docs`` .. _docs: docs -🪪 Identity Servers -=================== - -Identity servers have the job of mapping email addresses and other 3rd Party -IDs (3PIDs) to Matrix user IDs, as well as verifying the ownership of 3PIDs -before creating that mapping. - -**Identity servers do not store accounts or credentials - these are stored and managed on homeservers. -Identity Servers are just for mapping 3rd Party IDs to Matrix IDs.** - -This process is highly security-sensitive, as there is an obvious risk of spam if it -is too easy to sign up for Matrix accounts or harvest 3PID data. In the longer -term, we hope to create a decentralised system to manage it (`matrix-doc #712 -`_), but in the meantime, -the role of managing trusted identity in the Matrix ecosystem is farmed out to -a cluster of known trusted ecosystem partners, who run 'Matrix Identity -Servers' such as `Sydent `_, whose role -is purely to authenticate and track 3PID logins and publish end-user public -keys. - -You can host your own copy of Sydent, but this will prevent you reaching other -users in the Matrix ecosystem via their email address, and prevent them finding -you. We therefore recommend that you use one of the centralised identity servers -at ``https://matrix.org`` or ``https://vector.im`` for now. - -To reiterate: the Identity server will only be used if you choose to associate -an email address with your account, or send an invite to another user via their -email address. - 🛠️ Development ============== @@ -252,20 +149,29 @@ Alongside all that, join our developer community on Matrix: Copyright and Licensing ======================= -| Copyright 2014-2017 OpenMarket Ltd -| Copyright 2017 Vector Creations Ltd -| Copyright 2017-2025 New Vector Ltd -| + | Copyright 2014–2017 OpenMarket Ltd + | Copyright 2017 Vector Creations Ltd + | Copyright 2017–2025 New Vector Ltd + | Copyright 2025 Element Creations Ltd -This software is dual-licensed by New Vector Ltd (Element). It can be used either: +This software is dual-licensed by Element Creations Ltd (Element). It can be +used either: -(1) for free under the terms of the GNU Affero General Public License (as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version); OR +(1) for free under the terms of the GNU Affero General Public License (as + published by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version); OR -(2) under the terms of a paid-for Element Commercial License agreement between you and Element (the terms of which may vary depending on what you and Element have agreed to). +(2) under the terms of a paid-for Element Commercial License agreement between + you and Element (the terms of which may vary depending on what you and + Element have agreed to). -Unless required by applicable law or agreed to in writing, software distributed under the Licenses is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the Licenses for the specific language governing permissions and limitations under the Licenses. +Unless required by applicable law or agreed to in writing, software distributed +under the Licenses is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the Licenses for the +specific language governing permissions and limitations under the Licenses. -Please contact `licensing@element.io `_ to purchase an Element commercial license for this software. +Please contact `licensing@element.io `_ to purchase +an Element commercial license for this software. .. |support| image:: https://img.shields.io/badge/matrix-community%20support-success diff --git a/changelog.d/19228.misc b/changelog.d/19228.misc new file mode 100644 index 00000000000..bee72bff9a6 --- /dev/null +++ b/changelog.d/19228.misc @@ -0,0 +1 @@ +Simplify README and add ESS Getting started section. From 022e56cce34e0b383dfe2ada426da36c67ff92ad Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Dec 2025 14:25:12 +0000 Subject: [PATCH 038/139] Move security note from README into the docs (#19259) This is a) to simplify the README and b) so that we can easily link to the security page from e.g. the installation guide. Follows on from https://github.com/element-hq/synapse/pull/19228 --- README.rst | 50 +------------------------------------- changelog.d/19259.misc | 1 + docs/SUMMARY.md | 1 + docs/setup/installation.md | 7 ++++++ docs/setup/security.md | 41 +++++++++++++++++++++++++++++++ 5 files changed, 51 insertions(+), 49 deletions(-) create mode 100644 changelog.d/19259.misc create mode 100644 docs/setup/security.md diff --git a/README.rst b/README.rst index 9ea397cd0f4..e95aa0f8aa5 100644 --- a/README.rst +++ b/README.rst @@ -42,61 +42,13 @@ There are three editions of ESS: The Synapse documentation describes `options for installing Synapse standalone `_. See -below for more useful documenation links. +below for more useful documentation links. - `Synapse configuration options `_ - `Synapse configuration for federation `_ - `Using a reverse proxy with Synapse `_ - `Upgrading Synapse `_ -Platform dependencies ---------------------- - -Synapse uses a number of platform dependencies such as Python and PostgreSQL, -and aims to follow supported upstream versions. See the `deprecation policy -`_ for more -details. - - -Security note -------------- - -Matrix serves raw, user-supplied data in some APIs — specifically the `content -repository endpoints`_. - -.. _content repository endpoints: https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid - -Whilst we make a reasonable effort to mitigate against XSS attacks (for -instance, by using `CSP`_), a Matrix homeserver should not be hosted on a -domain hosting other web applications. This especially applies to sharing -the domain with Matrix web clients and other sensitive applications like -webmail. See -https://developer.github.com/changes/2014-04-25-user-content-security for more -information. - -.. _CSP: https://github.com/matrix-org/synapse/pull/1021 - -Ideally, the homeserver should not simply be on a different subdomain, but on -a completely different `registered domain`_ (also known as top-level site or -eTLD+1). This is because `some attacks`_ are still possible as long as the two -applications share the same registered domain. - -.. _registered domain: https://tools.ietf.org/html/draft-ietf-httpbis-rfc6265bis-03#section-2.3 - -.. _some attacks: https://en.wikipedia.org/wiki/Session_fixation#Attacks_using_cross-subdomain_cookie - -To illustrate this with an example, if your Element Web or other sensitive web -application is hosted on ``A.example1.com``, you should ideally host Synapse on -``example2.com``. Some amount of protection is offered by hosting on -``B.example1.com`` instead, so this is also acceptable in some scenarios. -However, you should *not* host your Synapse on ``A.example1.com``. - -Note that all of the above refers exclusively to the domain used in Synapse's -``public_baseurl`` setting. In particular, it has no bearing on the domain -mentioned in MXIDs hosted on that server. - -Following this advice ensures that even if an XSS is found in Synapse, the -impact to other applications will be minimal. 🎯 Troubleshooting and support ============================== diff --git a/changelog.d/19259.misc b/changelog.d/19259.misc new file mode 100644 index 00000000000..bee72bff9a6 --- /dev/null +++ b/changelog.d/19259.misc @@ -0,0 +1 @@ +Simplify README and add ESS Getting started section. diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 926a6eb8489..980f51d078c 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -5,6 +5,7 @@ # Setup - [Installation](setup/installation.md) + - [Security](setup/security.md) - [Using Postgres](postgres.md) - [Configuring a Reverse Proxy](reverse_proxy.md) - [Configuring a Forward/Outbound Proxy](setup/forward_proxy.md) diff --git a/docs/setup/installation.md b/docs/setup/installation.md index 786672c6890..a48662362af 100644 --- a/docs/setup/installation.md +++ b/docs/setup/installation.md @@ -16,8 +16,15 @@ that your email address is probably `user@example.com` rather than `user@email.example.com`) - but doing so may require more advanced setup: see [Setting up Federation](../federate.md). +⚠️ Before setting up Synapse please consult the [security page](security.md) for +best practices. ⚠️ + ## Installing Synapse +Note: Synapse uses a number of platform dependencies such as Python and PostgreSQL, +and aims to follow supported upstream versions. See the [deprecation +policy](../deprecation_policy.md) for more details. + ### Prebuilt packages Prebuilt packages are available for a number of platforms. These are recommended diff --git a/docs/setup/security.md b/docs/setup/security.md new file mode 100644 index 00000000000..2c21b494e51 --- /dev/null +++ b/docs/setup/security.md @@ -0,0 +1,41 @@ +# Security + +This page lays out security best-practices when running Synapse. + +If you believe you have encountered a security issue, see our [Security +Disclosure Policy](https://element.io/en/security/security-disclosure-policy). + +## Content repository + +Matrix serves raw, user-supplied data in some APIs — specifically the [content +repository endpoints](https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid). + +Whilst we make a reasonable effort to mitigate against XSS attacks (for +instance, by using [CSP](https://github.com/matrix-org/synapse/pull/1021)), a +Matrix homeserver should not be hosted on a domain hosting other web +applications. This especially applies to sharing the domain with Matrix web +clients and other sensitive applications like webmail. See +https://developer.github.com/changes/2014-04-25-user-content-security for more +information. + +Ideally, the homeserver should not simply be on a different subdomain, but on a +completely different [registered +domain](https://tools.ietf.org/html/draft-ietf-httpbis-rfc6265bis-03#section-2.3) +(also known as top-level site or eTLD+1). This is because [some +attacks](https://en.wikipedia.org/wiki/Session_fixation#Attacks_using_cross-subdomain_cookie) +are still possible as long as the two applications share the same registered +domain. + + +To illustrate this with an example, if your Element Web or other sensitive web +application is hosted on `A.example1.com`, you should ideally host Synapse on +`example2.com`. Some amount of protection is offered by hosting on +`B.example1.com` instead, so this is also acceptable in some scenarios. +However, you should *not* host your Synapse on `A.example1.com`. + +Note that all of the above refers exclusively to the domain used in Synapse's +`public_baseurl` setting. In particular, it has no bearing on the domain +mentioned in MXIDs hosted on that server. + +Following this advice ensures that even if an XSS is found in Synapse, the +impact to other applications will be minimal. From 2862c77837cf806230965897d74ccb31e94c6a99 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 2 Dec 2025 15:59:27 +0000 Subject: [PATCH 039/139] Remove macos wheels from CI (#19263) Follows #19225 and stops building macos wheels in CI. ### Pull Request Checklist * [X] Pull request is based on the develop branch * [X] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [X] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- .github/workflows/release-artifacts.yml | 7 ------- changelog.d/19263.misc | 1 + pyproject.toml | 4 ---- 3 files changed, 1 insertion(+), 11 deletions(-) create mode 100644 changelog.d/19263.misc diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 03c2b0a3266..531680b989d 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -114,19 +114,12 @@ jobs: os: - ubuntu-24.04 - ubuntu-24.04-arm - - macos-14 # This uses arm64 - - macos-15-intel # This uses x86-64 # is_pr is a flag used to exclude certain jobs from the matrix on PRs. # It is not read by the rest of the workflow. is_pr: - ${{ startsWith(github.ref, 'refs/pull/') }} exclude: - # Don't build macos wheels on PR CI. - - is_pr: true - os: "macos-15-intel" - - is_pr: true - os: "macos-14" # Don't build aarch64 wheels on PR CI. - is_pr: true os: "ubuntu-24.04-arm" diff --git a/changelog.d/19263.misc b/changelog.d/19263.misc new file mode 100644 index 00000000000..62b75941085 --- /dev/null +++ b/changelog.d/19263.misc @@ -0,0 +1 @@ +Stop building macos wheels in CI pipeline. diff --git a/pyproject.toml b/pyproject.toml index 5ee843365d5..8cee32ecfa0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -424,7 +424,3 @@ test-command = "python -c 'from synapse.synapse_rust import sum_as_string; print [tool.cibuildwheel.linux] # Wrap the repair command to correctly rename the built cpython wheels as ABI3. repair-wheel-command = "./.ci/scripts/auditwheel_wrapper.py -w {dest_dir} {wheel}" - -[tool.cibuildwheel.macos] -# Wrap the repair command to correctly rename the built cpython wheels as ABI3. -repair-wheel-command = "./.ci/scripts/auditwheel_wrapper.py --require-archs {delocate_archs} -w {dest_dir} {wheel}" From 5fe4b7ed60d22f7d7b5c50c4c74f1785c65943df Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 2 Dec 2025 09:21:08 -0700 Subject: [PATCH 040/139] 1.144.0rc1 --- CHANGES.md | 60 +++++++++++++++++++++++++++++++ changelog.d/17782.misc | 1 - changelog.d/18960.bugfix | 1 - changelog.d/19203.feature | 1 - changelog.d/19207.feature | 1 - changelog.d/19208.misc | 1 - changelog.d/19209.misc | 1 - changelog.d/19211.misc | 1 - changelog.d/19219.misc | 1 - changelog.d/19220.misc | 1 - changelog.d/19221.misc | 1 - changelog.d/19223.misc | 1 - changelog.d/19224.misc | 1 - changelog.d/19225.removal | 1 - changelog.d/19226.misc | 1 - changelog.d/19228.misc | 1 - changelog.d/19229.misc | 1 - changelog.d/19230.misc | 1 - changelog.d/19235.bugfix | 1 - changelog.d/19239.misc | 1 - changelog.d/19240.bugfix | 1 - changelog.d/19243.doc | 1 - changelog.d/19251.misc | 1 - changelog.d/19259.misc | 1 - changelog.d/19263.misc | 1 - debian/changelog | 6 ++++ pyproject.toml | 2 +- schema/synapse-config.schema.yaml | 2 +- 28 files changed, 68 insertions(+), 26 deletions(-) delete mode 100644 changelog.d/17782.misc delete mode 100644 changelog.d/18960.bugfix delete mode 100644 changelog.d/19203.feature delete mode 100644 changelog.d/19207.feature delete mode 100644 changelog.d/19208.misc delete mode 100644 changelog.d/19209.misc delete mode 100644 changelog.d/19211.misc delete mode 100644 changelog.d/19219.misc delete mode 100644 changelog.d/19220.misc delete mode 100644 changelog.d/19221.misc delete mode 100644 changelog.d/19223.misc delete mode 100644 changelog.d/19224.misc delete mode 100644 changelog.d/19225.removal delete mode 100644 changelog.d/19226.misc delete mode 100644 changelog.d/19228.misc delete mode 100644 changelog.d/19229.misc delete mode 100644 changelog.d/19230.misc delete mode 100644 changelog.d/19235.bugfix delete mode 100644 changelog.d/19239.misc delete mode 100644 changelog.d/19240.bugfix delete mode 100644 changelog.d/19243.doc delete mode 100644 changelog.d/19251.misc delete mode 100644 changelog.d/19259.misc delete mode 100644 changelog.d/19263.misc diff --git a/CHANGES.md b/CHANGES.md index 53d0ae36746..a58f3cea210 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,63 @@ +# synapse 1.144.0rc1 (2025-12-02) + +Admins using the unstable [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666) endpoint (`/_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms`), please check [the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/develop/docs/upgrade.md#upgrading-to-v11440) as this release contains changes that disable that endpoint by default. + +## Features + +- Add experimentatal implememntation of [MSC4380](https://github.com/matrix-org/matrix-spec-proposals/pull/4380) (invite blocking). ([\#19203](https://github.com/element-hq/synapse/issues/19203)) +- Allow restarting delayed event timeouts on workers. ([\#19207](https://github.com/element-hq/synapse/issues/19207)) + +## Bugfixes + +- Fix a bug in the database function for fetching state deltas that could result in unnecessarily long query times. ([\#18960](https://github.com/element-hq/synapse/issues/18960)) +- Fix v12 rooms when running with `use_frozen_dicts: True`. ([\#19235](https://github.com/element-hq/synapse/issues/19235)) +- Fix bug where invalid `canonical_alias` content would return 500 instead of 400. ([\#19240](https://github.com/element-hq/synapse/issues/19240)) + +## Improved Documentation + +- Document in the `--config-path` help how multiple files are merged - by merging them shallowly. ([\#19243](https://github.com/element-hq/synapse/issues/19243)) + +## Deprecations and Removals + +- Stop building release wheels for MacOS. ([\#19225](https://github.com/element-hq/synapse/issues/19225)) + +## Internal Changes + +- Improve event filtering for Simplified Sliding Sync. ([\#17782](https://github.com/element-hq/synapse/issues/17782)) +- Export `SYNAPSE_SUPPORTED_COMPLEMENT_TEST_PACKAGES` environment variable from `scripts-dev/complement.sh`. ([\#19208](https://github.com/element-hq/synapse/issues/19208)) +- Refactor `scripts-dev/complement.sh` logic to avoid `exit` to facilitate being able to source it from other scripts (composable). ([\#19209](https://github.com/element-hq/synapse/issues/19209)) +- Expire sliding sync connections that are too old or have too much pending data. ([\#19211](https://github.com/element-hq/synapse/issues/19211)) +- Require an experimental feature flag to be enabled in order for the unstable [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666) endpoint (`/_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms`) to be available. ([\#19219](https://github.com/element-hq/synapse/issues/19219)) +- Prevent changelog check CI running on @dependabot's PRs even when a human has modified the branch. ([\#19220](https://github.com/element-hq/synapse/issues/19220)) +- Auto-fix trailing spaces in multi-line strings and comments when running the lint script. ([\#19221](https://github.com/element-hq/synapse/issues/19221)) +- Move towards using a dedicated `Duration` type. ([\#19223](https://github.com/element-hq/synapse/issues/19223), [\#19229](https://github.com/element-hq/synapse/issues/19229)) +- Improve robustness of the SQL schema linting in CI. ([\#19224](https://github.com/element-hq/synapse/issues/19224)) +- Add log to determine whether clients are using `/messages` as expected. ([\#19226](https://github.com/element-hq/synapse/issues/19226)) +- Simplify README and add ESS Getting started section. ([\#19228](https://github.com/element-hq/synapse/issues/19228), [\#19259](https://github.com/element-hq/synapse/issues/19259)) +- Add a unit test for ensuring associated refresh tokens are erased when a device is deleted. ([\#19230](https://github.com/element-hq/synapse/issues/19230)) +- Prompt user to consider adding future deprecations to the changelog in release script. ([\#19239](https://github.com/element-hq/synapse/issues/19239)) +- Fix check of the Rust compiled code being outdated when using source checkout and `.egg-info`. ([\#19251](https://github.com/element-hq/synapse/issues/19251)) +- Stop building macos wheels in CI pipeline. ([\#19263](https://github.com/element-hq/synapse/issues/19263)) + + + +### Updates to locked dependencies + +* Bump Swatinem/rust-cache from 2.8.1 to 2.8.2. ([\#19244](https://github.com/element-hq/synapse/issues/19244)) +* Bump actions/checkout from 5.0.0 to 6.0.0. ([\#19213](https://github.com/element-hq/synapse/issues/19213)) +* Bump actions/setup-go from 6.0.0 to 6.1.0. ([\#19214](https://github.com/element-hq/synapse/issues/19214)) +* Bump actions/setup-python from 6.0.0 to 6.1.0. ([\#19245](https://github.com/element-hq/synapse/issues/19245)) +* Bump attrs from 25.3.0 to 25.4.0. ([\#19215](https://github.com/element-hq/synapse/issues/19215)) +* Bump docker/metadata-action from 5.9.0 to 5.10.0. ([\#19246](https://github.com/element-hq/synapse/issues/19246)) +* Bump http from 1.3.1 to 1.4.0. ([\#19249](https://github.com/element-hq/synapse/issues/19249)) +* Bump pydantic from 2.12.4 to 2.12.5. ([\#19250](https://github.com/element-hq/synapse/issues/19250)) +* Bump pyopenssl from 25.1.0 to 25.3.0. ([\#19248](https://github.com/element-hq/synapse/issues/19248)) +* Bump rpds-py from 0.28.0 to 0.29.0. ([\#19216](https://github.com/element-hq/synapse/issues/19216)) +* Bump rpds-py from 0.29.0 to 0.30.0. ([\#19247](https://github.com/element-hq/synapse/issues/19247)) +* Bump sentry-sdk from 2.44.0 to 2.46.0. ([\#19218](https://github.com/element-hq/synapse/issues/19218)) +* Bump types-bleach from 6.2.0.20250809 to 6.3.0.20251115. ([\#19217](https://github.com/element-hq/synapse/issues/19217)) +* Bump types-jsonschema from 4.25.1.20250822 to 4.25.1.20251009. ([\#19252](https://github.com/element-hq/synapse/issues/19252)) + # Synapse 1.143.0 (2025-11-25) ## Dropping support for PostgreSQL 13 diff --git a/changelog.d/17782.misc b/changelog.d/17782.misc deleted file mode 100644 index d7321470d01..00000000000 --- a/changelog.d/17782.misc +++ /dev/null @@ -1 +0,0 @@ -Improve event filtering for Simplified Sliding Sync. \ No newline at end of file diff --git a/changelog.d/18960.bugfix b/changelog.d/18960.bugfix deleted file mode 100644 index 909089f8092..00000000000 --- a/changelog.d/18960.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug in the database function for fetching state deltas that could result in unnecessarily long query times. \ No newline at end of file diff --git a/changelog.d/19203.feature b/changelog.d/19203.feature deleted file mode 100644 index d192781b20e..00000000000 --- a/changelog.d/19203.feature +++ /dev/null @@ -1 +0,0 @@ -Add experimentatal implememntation of [MSC4380](https://github.com/matrix-org/matrix-spec-proposals/pull/4380) (invite blocking). diff --git a/changelog.d/19207.feature b/changelog.d/19207.feature deleted file mode 100644 index e64562c350b..00000000000 --- a/changelog.d/19207.feature +++ /dev/null @@ -1 +0,0 @@ -Allow restarting delayed event timeouts on workers. diff --git a/changelog.d/19208.misc b/changelog.d/19208.misc deleted file mode 100644 index 1948be309bf..00000000000 --- a/changelog.d/19208.misc +++ /dev/null @@ -1 +0,0 @@ -Export `SYNAPSE_SUPPORTED_COMPLEMENT_TEST_PACKAGES` environment variable from `scripts-dev/complement.sh`. diff --git a/changelog.d/19209.misc b/changelog.d/19209.misc deleted file mode 100644 index e64ca85d1d3..00000000000 --- a/changelog.d/19209.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor `scripts-dev/complement.sh` logic to avoid `exit` to facilitate being able to source it from other scripts (composable). diff --git a/changelog.d/19211.misc b/changelog.d/19211.misc deleted file mode 100644 index d8a4a44662e..00000000000 --- a/changelog.d/19211.misc +++ /dev/null @@ -1 +0,0 @@ -Expire sliding sync connections that are too old or have too much pending data. diff --git a/changelog.d/19219.misc b/changelog.d/19219.misc deleted file mode 100644 index 83557293587..00000000000 --- a/changelog.d/19219.misc +++ /dev/null @@ -1 +0,0 @@ -Require an experimental feature flag to be enabled in order for the unstable [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666) endpoint (`/_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms`) to be available. \ No newline at end of file diff --git a/changelog.d/19220.misc b/changelog.d/19220.misc deleted file mode 100644 index e98f5ade610..00000000000 --- a/changelog.d/19220.misc +++ /dev/null @@ -1 +0,0 @@ -Prevent changelog check CI running on @dependabot's PRs even when a human has modified the branch. \ No newline at end of file diff --git a/changelog.d/19221.misc b/changelog.d/19221.misc deleted file mode 100644 index d1faf9cb72c..00000000000 --- a/changelog.d/19221.misc +++ /dev/null @@ -1 +0,0 @@ -Auto-fix trailing spaces in multi-line strings and comments when running the lint script. \ No newline at end of file diff --git a/changelog.d/19223.misc b/changelog.d/19223.misc deleted file mode 100644 index 8caebead72a..00000000000 --- a/changelog.d/19223.misc +++ /dev/null @@ -1 +0,0 @@ -Move towards using a dedicated `Duration` type. diff --git a/changelog.d/19224.misc b/changelog.d/19224.misc deleted file mode 100644 index 3f8f630c5ee..00000000000 --- a/changelog.d/19224.misc +++ /dev/null @@ -1 +0,0 @@ -Improve robustness of the SQL schema linting in CI. diff --git a/changelog.d/19225.removal b/changelog.d/19225.removal deleted file mode 100644 index bed5db07e81..00000000000 --- a/changelog.d/19225.removal +++ /dev/null @@ -1 +0,0 @@ -Stop building release wheels for MacOS. \ No newline at end of file diff --git a/changelog.d/19226.misc b/changelog.d/19226.misc deleted file mode 100644 index c38d1d3ef69..00000000000 --- a/changelog.d/19226.misc +++ /dev/null @@ -1 +0,0 @@ -Add log to determine whether clients are using `/messages` as expected. diff --git a/changelog.d/19228.misc b/changelog.d/19228.misc deleted file mode 100644 index bee72bff9a6..00000000000 --- a/changelog.d/19228.misc +++ /dev/null @@ -1 +0,0 @@ -Simplify README and add ESS Getting started section. diff --git a/changelog.d/19229.misc b/changelog.d/19229.misc deleted file mode 100644 index 8caebead72a..00000000000 --- a/changelog.d/19229.misc +++ /dev/null @@ -1 +0,0 @@ -Move towards using a dedicated `Duration` type. diff --git a/changelog.d/19230.misc b/changelog.d/19230.misc deleted file mode 100644 index 06704db25b0..00000000000 --- a/changelog.d/19230.misc +++ /dev/null @@ -1 +0,0 @@ -Add a unit test for ensuring associated refresh tokens are erased when a device is delted. \ No newline at end of file diff --git a/changelog.d/19235.bugfix b/changelog.d/19235.bugfix deleted file mode 100644 index 1c312351a41..00000000000 --- a/changelog.d/19235.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix v12 rooms when running with `use_frozen_dicts: True`. diff --git a/changelog.d/19239.misc b/changelog.d/19239.misc deleted file mode 100644 index fd5757eb770..00000000000 --- a/changelog.d/19239.misc +++ /dev/null @@ -1 +0,0 @@ -Prompt user to consider adding future deprecations to the changelog in release script. diff --git a/changelog.d/19240.bugfix b/changelog.d/19240.bugfix deleted file mode 100644 index d8490bcc1f8..00000000000 --- a/changelog.d/19240.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where invalid `canonical_alias` content would return 500 instead of 400. diff --git a/changelog.d/19243.doc b/changelog.d/19243.doc deleted file mode 100644 index 3a396c88d5d..00000000000 --- a/changelog.d/19243.doc +++ /dev/null @@ -1 +0,0 @@ -Document in the `--config-path` help how multiple files are merged - by merging them shallowly. diff --git a/changelog.d/19251.misc b/changelog.d/19251.misc deleted file mode 100644 index 9d0501c3d4e..00000000000 --- a/changelog.d/19251.misc +++ /dev/null @@ -1 +0,0 @@ -Fix check of the Rust compiled code being outdated when using source checkout and `.egg-info`. diff --git a/changelog.d/19259.misc b/changelog.d/19259.misc deleted file mode 100644 index bee72bff9a6..00000000000 --- a/changelog.d/19259.misc +++ /dev/null @@ -1 +0,0 @@ -Simplify README and add ESS Getting started section. diff --git a/changelog.d/19263.misc b/changelog.d/19263.misc deleted file mode 100644 index 62b75941085..00000000000 --- a/changelog.d/19263.misc +++ /dev/null @@ -1 +0,0 @@ -Stop building macos wheels in CI pipeline. diff --git a/debian/changelog b/debian/changelog index f8cf0c86f9c..901c210d609 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.144.0~rc1) stable; urgency=medium + + * New Synapse release 1.144.0rc1. + + -- Synapse Packaging team Tue, 02 Dec 2025 09:11:19 -0700 + matrix-synapse-py3 (1.143.0) stable; urgency=medium * New Synapse release 1.143.0. diff --git a/pyproject.toml b/pyproject.toml index 8cee32ecfa0..ce2ecf63634 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.143.0" +version = "1.144.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index f2d51fdb953..bf9346995da 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/schema/synapse/v1.143/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.144/synapse-config.schema.json type: object properties: modules: From ffd0b4c07958548844291d58e603e9b6d72f8b76 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 2 Dec 2025 16:45:28 +0000 Subject: [PATCH 041/139] Add a 14-day cooldown for dependency updates (#19258) --- .github/dependabot.yml | 16 ++++++++++++++++ changelog.d/19258.misc | 1 + 2 files changed, 17 insertions(+) create mode 100644 changelog.d/19258.misc diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 7ce353ed640..34484438c86 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,19 +5,35 @@ updates: directory: "/" schedule: interval: "weekly" + # Prevent pulling packages that were recently updated to help mitigate + # supply chain attacks. 14 days was taken from the recommendation at + # https://blog.yossarian.net/2025/11/21/We-should-all-be-using-dependency-cooldowns + # where the author noted that 9/10 attacks would have been mitigated by a + # two week cooldown. + # + # The cooldown only applies to general updates; security updates will still + # be pulled in as soon as possible. + cooldown: + default-days: 14 - package-ecosystem: "docker" directory: "/docker" schedule: interval: "weekly" + cooldown: + default-days: 14 - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" + cooldown: + default-days: 14 - package-ecosystem: "cargo" directory: "/" versioning-strategy: "lockfile-only" schedule: interval: "weekly" + cooldown: + default-days: 14 diff --git a/changelog.d/19258.misc b/changelog.d/19258.misc new file mode 100644 index 00000000000..9155f9d20fe --- /dev/null +++ b/changelog.d/19258.misc @@ -0,0 +1 @@ +Require 14 days to pass before pulling in general dependency updates to help mitigate upstream supply chain attacks. \ No newline at end of file From 0dfc21ca9f877806bfa9d3cd49432da78466757c Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 2 Dec 2025 16:45:41 +0000 Subject: [PATCH 042/139] Remove "Updates to locked dependencies" section from changelog (#19254) --- changelog.d/19254.removal | 1 + scripts-dev/release.py | 50 +-------------------------------------- 2 files changed, 2 insertions(+), 49 deletions(-) create mode 100644 changelog.d/19254.removal diff --git a/changelog.d/19254.removal b/changelog.d/19254.removal new file mode 100644 index 00000000000..ee527cef994 --- /dev/null +++ b/changelog.d/19254.removal @@ -0,0 +1 @@ +Remove the "Updates to locked dependencies" section from the changelog due to lack of use and the maintenance burden. \ No newline at end of file diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 17eadbf6c3b..3aed4b2f769 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -32,7 +32,7 @@ import urllib.request from os import path from tempfile import TemporaryDirectory -from typing import Any, Match +from typing import Any import attr import click @@ -968,10 +968,6 @@ def generate_and_write_changelog( new_changes = new_changes.replace( "No significant changes.", f"No significant changes since {current_version}." ) - new_changes += build_dependabot_changelog( - repo, - current_version, - ) # Prepend changes to changelog with open("CHANGES.md", "r+") as f: @@ -986,49 +982,5 @@ def generate_and_write_changelog( os.remove(filename) -def build_dependabot_changelog(repo: Repo, current_version: version.Version) -> str: - """Summarise dependabot commits between `current_version` and `release_branch`. - - Returns an empty string if there have been no such commits; otherwise outputs a - third-level markdown header followed by an unordered list.""" - last_release_commit = repo.tag("v" + str(current_version)).commit - rev_spec = f"{last_release_commit.hexsha}.." - commits = list(git.objects.Commit.iter_items(repo, rev_spec)) - messages = [] - for commit in reversed(commits): - if commit.author.name == "dependabot[bot]": - message: str | bytes = commit.message - if isinstance(message, bytes): - message = message.decode("utf-8") - messages.append(message.split("\n", maxsplit=1)[0]) - - if not messages: - print(f"No dependabot commits in range {rev_spec}", file=sys.stderr) - return "" - - messages.sort() - - def replacer(match: Match[str]) -> str: - desc = match.group(1) - number = match.group(2) - return f"* {desc}. ([\\#{number}](https://github.com/element-hq/synapse/issues/{number}))" - - for i, message in enumerate(messages): - messages[i] = re.sub(r"(.*) \(#(\d+)\)$", replacer, message) - messages.insert(0, "### Updates to locked dependencies\n") - # Add an extra blank line to the bottom of the section - messages.append("") - return "\n".join(messages) - - -@cli.command() -@click.argument("since") -def test_dependabot_changelog(since: str) -> None: - """Test building the dependabot changelog. - - Summarises all dependabot commits between the SINCE tag and the current git HEAD.""" - print(build_dependabot_changelog(git.Repo("."), version.Version(since))) - - if __name__ == "__main__": cli() From 3d28e2213f3b074eb598381b6b6ee33bd3051384 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 2 Dec 2025 16:45:54 +0000 Subject: [PATCH 043/139] Dependabot: allow 10 open PRs for general updates (#19253) --- .github/dependabot.yml | 7 +++++++ changelog.d/19253.misc | 1 + 2 files changed, 8 insertions(+) create mode 100644 changelog.d/19253.misc diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 34484438c86..cfaa3c04e33 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,8 +1,12 @@ version: 2 +# As dependabot is currently only run on a weekly basis, we raise the +# open-pull-requests-limit to 10 (from the default of 5) to better ensure we +# don't continuously grow a backlog of updates. updates: - # "pip" is the correct setting for poetry, per https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem package-ecosystem: "pip" directory: "/" + open-pull-requests-limit: 10 schedule: interval: "weekly" # Prevent pulling packages that were recently updated to help mitigate @@ -18,6 +22,7 @@ updates: - package-ecosystem: "docker" directory: "/docker" + open-pull-requests-limit: 10 schedule: interval: "weekly" cooldown: @@ -25,6 +30,7 @@ updates: - package-ecosystem: "github-actions" directory: "/" + open-pull-requests-limit: 10 schedule: interval: "weekly" cooldown: @@ -32,6 +38,7 @@ updates: - package-ecosystem: "cargo" directory: "/" + open-pull-requests-limit: 10 versioning-strategy: "lockfile-only" schedule: interval: "weekly" diff --git a/changelog.d/19253.misc b/changelog.d/19253.misc new file mode 100644 index 00000000000..1d45f936f64 --- /dev/null +++ b/changelog.d/19253.misc @@ -0,0 +1 @@ +Raise the limit for concurrently-open non-security @dependabot PRs from 5 to 10. \ No newline at end of file From f86918e5622406ae73e94d78d45544dcaccde766 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 2 Dec 2025 16:46:08 +0000 Subject: [PATCH 044/139] Remove the currently broken netlify GHA workflow (#19262) --- .github/workflows/docs-pr-netlify.yaml | 34 -------------------------- changelog.d/19262.misc | 1 + 2 files changed, 1 insertion(+), 34 deletions(-) delete mode 100644 .github/workflows/docs-pr-netlify.yaml create mode 100644 changelog.d/19262.misc diff --git a/.github/workflows/docs-pr-netlify.yaml b/.github/workflows/docs-pr-netlify.yaml deleted file mode 100644 index 53a2d6b5974..00000000000 --- a/.github/workflows/docs-pr-netlify.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: Deploy documentation PR preview - -on: - workflow_run: - workflows: [ "Prepare documentation PR preview" ] - types: - - completed - -jobs: - netlify: - if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request' - runs-on: ubuntu-latest - steps: - # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action - # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess: - - name: 📥 Download artifact - uses: dawidd6/action-download-artifact@ac66b43f0e6a346234dd65d4d0c8fbb31cb316e5 # v11 - with: - workflow: docs-pr.yaml - run_id: ${{ github.event.workflow_run.id }} - name: book - path: book - - - name: 📤 Deploy to Netlify - uses: matrix-org/netlify-pr-preview@9805cd123fc9a7e421e35340a05e1ebc5dee46b5 # v3 - with: - path: book - owner: ${{ github.event.workflow_run.head_repository.owner.login }} - branch: ${{ github.event.workflow_run.head_branch }} - revision: ${{ github.event.workflow_run.head_sha }} - token: ${{ secrets.NETLIFY_AUTH_TOKEN }} - site_id: ${{ secrets.NETLIFY_SITE_ID }} - desc: Documentation preview - deployment_env: PR Documentation Preview diff --git a/changelog.d/19262.misc b/changelog.d/19262.misc new file mode 100644 index 00000000000..31906e6623b --- /dev/null +++ b/changelog.d/19262.misc @@ -0,0 +1 @@ +Drop the broken netlify documentation workflow until a new one is implemented. \ No newline at end of file From 39316672da675458ba2a5809f71114a5844d8906 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 2 Dec 2025 10:58:06 -0600 Subject: [PATCH 045/139] Be able to `shutdown` homeserver that hasn't `setup` (#19187) For example, a homeserver can fail to `setup` if it fails to connect to the database. Fix https://github.com/element-hq/synapse/issues/19188 Follow-up to https://github.com/element-hq/synapse/pull/18828 ### Background As part of Element's plan to support a light form of vhosting (virtual host) (multiple instances of Synapse in the same Python process) (c.f Synapse Pro for small hosts), we're currently diving into the details and implications of running multiple instances of Synapse in the same Python process. "Clean tenant deprovisioning" tracked internally by https://github.com/element-hq/synapse-small-hosts/issues/50 --- changelog.d/19187.misc | 1 + synapse/api/errors.py | 6 + synapse/crypto/keyring.py | 135 +++++++++++----- synapse/server.py | 35 +++- tests/app/test_homeserver_shutdown.py | 221 +++++++++++++++++--------- tests/crypto/test_keyring.py | 15 +- tests/server.py | 125 +++++++++------ 7 files changed, 358 insertions(+), 180 deletions(-) create mode 100644 changelog.d/19187.misc diff --git a/changelog.d/19187.misc b/changelog.d/19187.misc new file mode 100644 index 00000000000..d831de38c8f --- /dev/null +++ b/changelog.d/19187.misc @@ -0,0 +1 @@ +Fix `HomeServer.shutdown()` failing if the homeserver hasn't been setup yet. diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 37b909a1a7e..c299ca84d94 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -856,6 +856,12 @@ def to_synapse_error(self) -> SynapseError: return ProxiedRequestError(self.code, errmsg, errcode, j) +class HomeServerNotSetupException(Exception): + """ + Raised when an operation is attempted on the HomeServer before setup() has been called. + """ + + class ShadowBanError(Exception): """ Raised when a shadow-banned user attempts to perform an action. diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 3abb644df5d..883f682e776 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -21,6 +21,7 @@ import abc import logging +from contextlib import ExitStack from typing import TYPE_CHECKING, Callable, Iterable import attr @@ -150,57 +151,81 @@ class Keyring: """ def __init__( - self, hs: "HomeServer", key_fetchers: "Iterable[KeyFetcher] | None" = None + self, + hs: "HomeServer", + test_only_key_fetchers: "list[KeyFetcher] | None" = None, ): - self.server_name = hs.hostname + """ + Args: + hs: The HomeServer instance + test_only_key_fetchers: Dependency injection for tests only. If provided, + these key fetchers will be used instead of the default ones. + """ + # Clean-up to avoid partial initialization leaving behind references. + with ExitStack() as exit: + self.server_name = hs.hostname + + self._key_fetchers: list[KeyFetcher] = [] + if test_only_key_fetchers is None: + # Always fetch keys from the database. + store_key_fetcher = StoreKeyFetcher(hs) + exit.callback(store_key_fetcher.shutdown) + self._key_fetchers.append(store_key_fetcher) + + # Fetch keys from configured trusted key servers, if any exist. + key_servers = hs.config.key.key_servers + if key_servers: + perspectives_key_fetcher = PerspectivesKeyFetcher(hs) + exit.callback(perspectives_key_fetcher.shutdown) + self._key_fetchers.append(perspectives_key_fetcher) + + # Finally, fetch keys from the origin server directly. + server_key_fetcher = ServerKeyFetcher(hs) + exit.callback(server_key_fetcher.shutdown) + self._key_fetchers.append(server_key_fetcher) + else: + self._key_fetchers = test_only_key_fetchers + + self._fetch_keys_queue: BatchingQueue[ + _FetchKeyRequest, dict[str, dict[str, FetchKeyResult]] + ] = BatchingQueue( + name="keyring_server", + hs=hs, + clock=hs.get_clock(), + # The method called to fetch each key + process_batch_callback=self._inner_fetch_key_requests, + ) + exit.callback(self._fetch_keys_queue.shutdown) - if key_fetchers is None: - # Always fetch keys from the database. - mutable_key_fetchers: list[KeyFetcher] = [StoreKeyFetcher(hs)] - # Fetch keys from configured trusted key servers, if any exist. - key_servers = hs.config.key.key_servers - if key_servers: - mutable_key_fetchers.append(PerspectivesKeyFetcher(hs)) - # Finally, fetch keys from the origin server directly. - mutable_key_fetchers.append(ServerKeyFetcher(hs)) - - self._key_fetchers: Iterable[KeyFetcher] = tuple(mutable_key_fetchers) - else: - self._key_fetchers = key_fetchers - - self._fetch_keys_queue: BatchingQueue[ - _FetchKeyRequest, dict[str, dict[str, FetchKeyResult]] - ] = BatchingQueue( - name="keyring_server", - hs=hs, - clock=hs.get_clock(), - # The method called to fetch each key - process_batch_callback=self._inner_fetch_key_requests, - ) + self._is_mine_server_name = hs.is_mine_server_name - self._is_mine_server_name = hs.is_mine_server_name + # build a FetchKeyResult for each of our own keys, to shortcircuit the + # fetcher. + self._local_verify_keys: dict[str, FetchKeyResult] = {} + for key_id, key in hs.config.key.old_signing_keys.items(): + self._local_verify_keys[key_id] = FetchKeyResult( + verify_key=key, valid_until_ts=key.expired + ) - # build a FetchKeyResult for each of our own keys, to shortcircuit the - # fetcher. - self._local_verify_keys: dict[str, FetchKeyResult] = {} - for key_id, key in hs.config.key.old_signing_keys.items(): - self._local_verify_keys[key_id] = FetchKeyResult( - verify_key=key, valid_until_ts=key.expired + vk = get_verify_key(hs.signing_key) + self._local_verify_keys[f"{vk.alg}:{vk.version}"] = FetchKeyResult( + verify_key=vk, + valid_until_ts=2**63, # fake future timestamp ) - vk = get_verify_key(hs.signing_key) - self._local_verify_keys[f"{vk.alg}:{vk.version}"] = FetchKeyResult( - verify_key=vk, - valid_until_ts=2**63, # fake future timestamp - ) + # We reached the end of the block which means everything was successful, so + # no exit handlers are needed (remove them all). + exit.pop_all() def shutdown(self) -> None: """ Prepares the KeyRing for garbage collection by shutting down it's queues. """ self._fetch_keys_queue.shutdown() + for key_fetcher in self._key_fetchers: key_fetcher.shutdown() + self._key_fetchers.clear() async def verify_json_for_server( self, @@ -521,9 +546,21 @@ class StoreKeyFetcher(KeyFetcher): """KeyFetcher impl which fetches keys from our data store""" def __init__(self, hs: "HomeServer"): - super().__init__(hs) - - self.store = hs.get_datastores().main + # Clean-up to avoid partial initialization leaving behind references. + with ExitStack() as exit: + super().__init__(hs) + # `KeyFetcher` keeps a reference to `hs` which we need to clean up if + # something goes wrong so we can cleanly shutdown the homeserver. + exit.callback(super().shutdown) + + # An error can be raised here if someone tried to create a `StoreKeyFetcher` + # before the homeserver is fully set up (`HomeServerNotSetupException: + # HomeServer.setup must be called before getting datastores`). + self.store = hs.get_datastores().main + + # We reached the end of the block which means everything was successful, so + # no exit handlers are needed (remove them all). + exit.pop_all() async def _fetch_keys( self, keys_to_fetch: list[_FetchKeyRequest] @@ -543,9 +580,21 @@ async def _fetch_keys( class BaseV2KeyFetcher(KeyFetcher): def __init__(self, hs: "HomeServer"): - super().__init__(hs) - - self.store = hs.get_datastores().main + # Clean-up to avoid partial initialization leaving behind references. + with ExitStack() as exit: + super().__init__(hs) + # `KeyFetcher` keeps a reference to `hs` which we need to clean up if + # something goes wrong so we can cleanly shutdown the homeserver. + exit.callback(super().shutdown) + + # An error can be raised here if someone tried to create a `StoreKeyFetcher` + # before the homeserver is fully set up (`HomeServerNotSetupException: + # HomeServer.setup must be called before getting datastores`). + self.store = hs.get_datastores().main + + # We reached the end of the block which means everything was successful, so + # no exit handlers are needed (remove them all). + exit.pop_all() async def process_v2_response( self, from_server: str, response_json: JsonDict, time_added_ms: int diff --git a/synapse/server.py b/synapse/server.py index de0a2b098c6..88662c5b28e 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -54,6 +54,7 @@ from synapse.api.auth.internal import InternalAuth from synapse.api.auth.mas import MasDelegatedAuth from synapse.api.auth_blocking import AuthBlocking +from synapse.api.errors import HomeServerNotSetupException from synapse.api.filtering import Filtering from synapse.api.ratelimiting import Ratelimiter, RequestRatelimiter from synapse.app._base import unregister_sighups @@ -399,7 +400,7 @@ def run_as_background_process( """ if self._is_shutdown: raise Exception( - f"Cannot start background process. HomeServer has been shutdown {len(self._background_processes)} {len(self.get_clock()._looping_calls)} {len(self.get_clock()._call_id_to_delayed_call)}" + "Cannot start background process. HomeServer has been shutdown" ) # Ignore linter error as this is the one location this should be called. @@ -466,7 +467,17 @@ async def shutdown(self) -> None: # TODO: Cleanup replication pieces - self.get_keyring().shutdown() + keyring: Keyring | None = None + try: + keyring = self.get_keyring() + except HomeServerNotSetupException: + # If the homeserver wasn't fully setup, keyring won't have existed before + # this and will fail to be initialized but it cleans itself up for any + # partial initialization problem. + pass + + if keyring: + keyring.shutdown() # Cleanup metrics associated with the homeserver for later_gauge in all_later_gauges_to_clean_up_on_shutdown.values(): @@ -478,8 +489,12 @@ async def shutdown(self) -> None: self.config.server.server_name ) - for db in self.get_datastores().databases: - db.stop_background_updates() + try: + for db in self.get_datastores().databases: + db.stop_background_updates() + except HomeServerNotSetupException: + # If the homeserver wasn't fully setup, the datastores won't exist + pass if self.should_send_federation(): try: @@ -513,8 +528,12 @@ async def shutdown(self) -> None: pass self._background_processes.clear() - for db in self.get_datastores().databases: - db._db_pool.close() + try: + for db in self.get_datastores().databases: + db._db_pool.close() + except HomeServerNotSetupException: + # If the homeserver wasn't fully setup, the datastores won't exist + pass def register_async_shutdown_handler( self, @@ -677,7 +696,9 @@ def get_clock(self) -> Clock: def get_datastores(self) -> Databases: if not self.datastores: - raise Exception("HomeServer.setup must be called before getting datastores") + raise HomeServerNotSetupException( + "HomeServer.setup must be called before getting datastores" + ) return self.datastores diff --git a/tests/app/test_homeserver_shutdown.py b/tests/app/test_homeserver_shutdown.py index f127e5571de..0f5d1c73387 100644 --- a/tests/app/test_homeserver_shutdown.py +++ b/tests/app/test_homeserver_shutdown.py @@ -19,7 +19,10 @@ # import gc +import sys import weakref +from typing import Any +from unittest.mock import patch from synapse.app.homeserver import SynapseHomeServer from synapse.logging.context import LoggingContext @@ -81,45 +84,12 @@ async def shutdown() -> None: # Ensure the `HomeServer` hs been garbage collected by attempting to use the # weakref to it. - if hs_ref() is not None: - self.fail("HomeServer reference should not be valid at this point") - - # To help debug this test when it fails, it is useful to leverage the - # `objgraph` module. - # The following code serves as an example of what I have found to be useful - # when tracking down references holding the `SynapseHomeServer` in memory: - # - # all_objects = gc.get_objects() - # for obj in all_objects: - # try: - # # These are a subset of types that are typically involved with - # # holding the `HomeServer` in memory. You may want to inspect - # # other types as well. - # if isinstance(obj, DataStore): - # print(sys.getrefcount(obj), "refs to", obj) - # if not isinstance(obj, weakref.ProxyType): - # db_obj = obj - # if isinstance(obj, SynapseHomeServer): - # print(sys.getrefcount(obj), "refs to", obj) - # if not isinstance(obj, weakref.ProxyType): - # synapse_hs = obj - # if isinstance(obj, SynapseSite): - # print(sys.getrefcount(obj), "refs to", obj) - # if not isinstance(obj, weakref.ProxyType): - # sysite = obj - # if isinstance(obj, DatabasePool): - # print(sys.getrefcount(obj), "refs to", obj) - # if not isinstance(obj, weakref.ProxyType): - # dbpool = obj - # except Exception: - # pass - # - # print(sys.getrefcount(hs_ref()), "refs to", hs_ref()) - # - # # The following values for `max_depth` and `too_many` have been found to - # # render a useful amount of information without taking an overly long time - # # to generate the result. - # objgraph.show_backrefs(synapse_hs, max_depth=10, too_many=10) + hs_after_shutdown = hs_ref() + if hs_after_shutdown is not None: + self.fail( + "HomeServer reference should not be valid at this point " + f"{get_memory_debug_info_for_object(hs_after_shutdown)}", + ) @logcontext_clean def test_clean_homeserver_shutdown_mid_background_updates(self) -> None: @@ -165,42 +135,137 @@ async def shutdown() -> None: # Ensure the `HomeServer` hs been garbage collected by attempting to use the # weakref to it. - if hs_ref() is not None: - self.fail("HomeServer reference should not be valid at this point") - - # To help debug this test when it fails, it is useful to leverage the - # `objgraph` module. - # The following code serves as an example of what I have found to be useful - # when tracking down references holding the `SynapseHomeServer` in memory: - # - # all_objects = gc.get_objects() - # for obj in all_objects: - # try: - # # These are a subset of types that are typically involved with - # # holding the `HomeServer` in memory. You may want to inspect - # # other types as well. - # if isinstance(obj, DataStore): - # print(sys.getrefcount(obj), "refs to", obj) - # if not isinstance(obj, weakref.ProxyType): - # db_obj = obj - # if isinstance(obj, SynapseHomeServer): - # print(sys.getrefcount(obj), "refs to", obj) - # if not isinstance(obj, weakref.ProxyType): - # synapse_hs = obj - # if isinstance(obj, SynapseSite): - # print(sys.getrefcount(obj), "refs to", obj) - # if not isinstance(obj, weakref.ProxyType): - # sysite = obj - # if isinstance(obj, DatabasePool): - # print(sys.getrefcount(obj), "refs to", obj) - # if not isinstance(obj, weakref.ProxyType): - # dbpool = obj - # except Exception: - # pass - # - # print(sys.getrefcount(hs_ref()), "refs to", hs_ref()) - # - # # The following values for `max_depth` and `too_many` have been found to - # # render a useful amount of information without taking an overly long time - # # to generate the result. - # objgraph.show_backrefs(synapse_hs, max_depth=10, too_many=10) + hs_after_shutdown = hs_ref() + if hs_after_shutdown is not None: + self.fail( + "HomeServer reference should not be valid at this point " + f"{get_memory_debug_info_for_object(hs_after_shutdown)}", + ) + + @logcontext_clean + def test_clean_homeserver_shutdown_when_failed_to_setup(self) -> None: + """ + Ensure the `SynapseHomeServer` can be fully shutdown and garbage collected if it + fails to be `setup`. + """ + self.reactor, self.clock = get_clock() + + # Patch `hs.setup()` to do nothing, so that the homeserver is not fully setup. + with patch.object(SynapseHomeServer, "setup", return_value=None) as mock_setup: + # Patch out the call to `start_test_homeserver` since we want access to the + # homeserver even before the server is setup (let alone started) + with patch("tests.server.start_test_homeserver", return_value=None): + self.hs = setup_test_homeserver( + cleanup_func=self.addCleanup, + reactor=self.reactor, + homeserver_to_use=SynapseHomeServer, + clock=self.clock, + ) + # Sanity check that we patched the correct method (make sure it was the + # thing that was called) + mock_setup.assert_called_once_with() + + hs_ref = weakref.ref(self.hs) + + # Run the reactor so any `callWhenRunning` functions can be cleared out. + self.reactor.run() + # This would normally happen as part of `HomeServer.shutdown` but the `MemoryReactor` + # we use in tests doesn't handle this properly (see doc comment) + cleanup_test_reactor_system_event_triggers(self.reactor) + + async def shutdown() -> None: + # Use a logcontext just to double-check that we don't mangle the logcontext + # during shutdown. + with LoggingContext(name="hs_shutdown", server_name=self.hs.hostname): + await self.hs.shutdown() + + self.get_success(shutdown()) + + # Cleanup the internal reference in our test case + del self.hs + + # Force garbage collection. + gc.collect() + + # Ensure the `HomeServer` hs been garbage collected by attempting to use the + # weakref to it. + hs_after_shutdown = hs_ref() + if hs_after_shutdown is not None: + self.fail( + "HomeServer reference should not be valid at this point " + f"{get_memory_debug_info_for_object(hs_after_shutdown)}", + ) + + +def get_memory_debug_info_for_object(object: Any) -> dict[str, Any]: + """ + Gathers some useful information to make it easier to figure out why the `object` is + still in memory. + + Args: + object: The object to gather debug information for. + """ + debug: dict[str, Any] = {} + if object is not None: + # The simplest tracing we can do is show the reference count for the object. + debug["reference_count"] = sys.getrefcount(object) + + # Find the list of objects that directly refer to the object. + # + # Note: The `ref_count` can be >0 but `referrers` can be empty because + # the all of the objects were frozen. Look at the + # `frozen_object_count` to detect this scenario. + referrers = gc.get_referrers(object) + debug["gc_referrer_count"] = len(referrers) + debug["gc_referrers"] = referrers + + # We don't expect to see frozen objects in normal operation of the + # `multi_synapse` shard. + # + # We can see frozen objects if you forget to `freeze=False` when + # starting the `SynapseHomeServer`. Frozen objects mean they are + # never considered for garbage collection. If the + # `SynapseHomeServer` (or anything that references the homeserver) + # is frozen, the homeserver can never be garbage collected and will + # linger in memory forever. + freeze_count = gc.get_freeze_count() + debug["gc_global_frozen_object_count"] = freeze_count + + # To help debug this test when it fails, it is useful to leverage the + # `objgraph` module. + # The following code serves as an example of what I have found to be useful + # when tracking down references holding the `SynapseHomeServer` in memory: + # + # all_objects = gc.get_objects() + # for obj in all_objects: + # try: + # # These are a subset of types that are typically involved with + # # holding the `HomeServer` in memory. You may want to inspect + # # other types as well. + # if isinstance(obj, DataStore): + # print(sys.getrefcount(obj), "refs to", obj) + # if not isinstance(obj, weakref.ProxyType): + # db_obj = obj + # if isinstance(obj, SynapseHomeServer): + # print(sys.getrefcount(obj), "refs to", obj) + # if not isinstance(obj, weakref.ProxyType): + # synapse_hs = obj + # if isinstance(obj, SynapseSite): + # print(sys.getrefcount(obj), "refs to", obj) + # if not isinstance(obj, weakref.ProxyType): + # sysite = obj + # if isinstance(obj, DatabasePool): + # print(sys.getrefcount(obj), "refs to", obj) + # if not isinstance(obj, weakref.ProxyType): + # dbpool = obj + # except Exception: + # pass + # + # print(sys.getrefcount(hs_ref()), "refs to", hs_ref()) + # + # # The following values for `max_depth` and `too_many` have been found to + # # render a useful amount of information without taking an overly long time + # # to generate the result. + # objgraph.show_backrefs(synapse_hs, max_depth=10, too_many=10) + + return debug diff --git a/tests/crypto/test_keyring.py b/tests/crypto/test_keyring.py index d3e8da97f84..3cc905f699a 100644 --- a/tests/crypto/test_keyring.py +++ b/tests/crypto/test_keyring.py @@ -95,7 +95,12 @@ def check_context( def test_verify_json_objects_for_server_awaits_previous_requests(self) -> None: mock_fetcher = Mock() mock_fetcher.get_keys = Mock() - kr = keyring.Keyring(self.hs, key_fetchers=(mock_fetcher,)) + kr = keyring.Keyring( + self.hs, + test_only_key_fetchers=[ + mock_fetcher, + ], + ) # a signed object that we are going to try to validate key1 = signedjson.key.generate_signing_key("1") @@ -286,7 +291,7 @@ async def get_keys( mock_fetcher = Mock() mock_fetcher.get_keys = Mock(side_effect=get_keys) kr = keyring.Keyring( - self.hs, key_fetchers=(StoreKeyFetcher(self.hs), mock_fetcher) + self.hs, test_only_key_fetchers=[StoreKeyFetcher(self.hs), mock_fetcher] ) # sign the json @@ -313,7 +318,7 @@ async def get_keys( mock_fetcher = Mock() mock_fetcher.get_keys = Mock(side_effect=get_keys) - kr = keyring.Keyring(self.hs, key_fetchers=(mock_fetcher,)) + kr = keyring.Keyring(self.hs, test_only_key_fetchers=[mock_fetcher]) json1: JsonDict = {} signedjson.sign.sign_json(json1, "server1", key1) @@ -363,7 +368,9 @@ async def get_keys2( mock_fetcher1.get_keys = Mock(side_effect=get_keys1) mock_fetcher2 = Mock() mock_fetcher2.get_keys = Mock(side_effect=get_keys2) - kr = keyring.Keyring(self.hs, key_fetchers=(mock_fetcher1, mock_fetcher2)) + kr = keyring.Keyring( + self.hs, test_only_key_fetchers=[mock_fetcher1, mock_fetcher2] + ) json1: JsonDict = {} signedjson.sign.sign_json(json1, "server1", key1) diff --git a/tests/server.py b/tests/server.py index 30337f3e38e..4fb7dea5ec0 100644 --- a/tests/server.py +++ b/tests/server.py @@ -1074,10 +1074,10 @@ def setup_test_homeserver( If no datastore is supplied, one is created and given to the homeserver. Args: - cleanup_func : The function used to register a cleanup routine for - after the test. If the function returns a Deferred, the - test case will wait until the Deferred has fired before - proceeding to the next cleanup function. + cleanup_func: The function used to register a cleanup routine for + after the test. If the function returns a Deferred, the + test case will wait until the Deferred has fired before + proceeding to the next cleanup function. server_name: Homeserver name config: Homeserver config reactor: Twisted reactor @@ -1190,6 +1190,53 @@ def setup_test_homeserver( cur.close() db_conn.close() + def cleanup() -> None: + import psycopg2 + + dropped = False + + # Drop the test database + db_conn = db_engine.module.connect( + dbname=POSTGRES_BASE_DB, + user=POSTGRES_USER, + host=POSTGRES_HOST, + port=POSTGRES_PORT, + password=POSTGRES_PASSWORD, + ) + db_engine.attempt_to_set_autocommit(db_conn, True) + cur = db_conn.cursor() + + # Try a few times to drop the DB. Some things may hold on to the + # database for a few more seconds due to flakiness, preventing + # us from dropping it when the test is over. If we can't drop + # it, warn and move on. + for _ in range(5): + try: + cur.execute("DROP DATABASE IF EXISTS %s;" % (test_db,)) + db_conn.commit() + dropped = True + except psycopg2.OperationalError as e: + warnings.warn( + "Couldn't drop old db: " + str(e), + category=UserWarning, + stacklevel=2, + ) + time.sleep(0.5) + + cur.close() + db_conn.close() + + if not dropped: + warnings.warn( + "Failed to drop old DB.", + category=UserWarning, + stacklevel=2, + ) + + if not LEAVE_DB: + # Register the cleanup hook + cleanup_func(cleanup) + hs = homeserver_to_use( server_name, config=config, @@ -1224,6 +1271,32 @@ def shutdown_hs_on_cleanup() -> "Deferred[None]": with patch("synapse.storage.database.make_pool", side_effect=make_fake_db_pool): hs.setup() + # Ideally, setup/start would be separated but since this is historically used + # throughout tests, we keep the existing behavior for now. We probably just need to + # rename this function. + start_test_homeserver(hs=hs, cleanup_func=cleanup_func, reactor=reactor) + + return hs + + +def start_test_homeserver( + *, + hs: HomeServer, + cleanup_func: Callable[[Callable[[], Optional["Deferred[None]"]]], None], + reactor: ISynapseReactor, +) -> None: + """ + Start a homeserver for testing. + + Args: + hs: The homeserver to start. + cleanup_func: The function used to register a cleanup routine for + after the test. If the function returns a Deferred, the + test case will wait until the Deferred has fired before + proceeding to the next cleanup function. + reactor: Twisted reactor + """ + # Register background tasks required by this server. This must be done # somewhat manually due to the background tasks not being registered # unless handlers are instantiated. @@ -1245,53 +1318,11 @@ def shutdown_hs_on_cleanup() -> "Deferred[None]": # We need to do cleanup on PostgreSQL def cleanup() -> None: - import psycopg2 - # Close all the db pools db_pool = database_pool() if db_pool is not None: db_pool._db_pool.close() - dropped = False - - # Drop the test database - db_conn = db_engine.module.connect( - dbname=POSTGRES_BASE_DB, - user=POSTGRES_USER, - host=POSTGRES_HOST, - port=POSTGRES_PORT, - password=POSTGRES_PASSWORD, - ) - db_engine.attempt_to_set_autocommit(db_conn, True) - cur = db_conn.cursor() - - # Try a few times to drop the DB. Some things may hold on to the - # database for a few more seconds due to flakiness, preventing - # us from dropping it when the test is over. If we can't drop - # it, warn and move on. - for _ in range(5): - try: - cur.execute("DROP DATABASE IF EXISTS %s;" % (test_db,)) - db_conn.commit() - dropped = True - except psycopg2.OperationalError as e: - warnings.warn( - "Couldn't drop old db: " + str(e), - category=UserWarning, - stacklevel=2, - ) - time.sleep(0.5) - - cur.close() - db_conn.close() - - if not dropped: - warnings.warn( - "Failed to drop old DB.", - category=UserWarning, - stacklevel=2, - ) - if not LEAVE_DB: # Register the cleanup hook cleanup_func(cleanup) @@ -1330,5 +1361,3 @@ def thread_pool() -> threadpool.ThreadPool: load_legacy_third_party_event_rules(hs) load_legacy_presence_router(hs) load_legacy_password_auth_providers(hs) - - return hs From 83023ce1e01646fe99bd76664e99907f09ad25a0 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 2 Dec 2025 11:28:46 -0600 Subject: [PATCH 046/139] Be able to `shutdown` homeserver that failed to `start` (#19232) For example, a homeserver can fail to `start` if the port is already in use or the port number is invalid (not 0-65535) Fix https://github.com/element-hq/synapse/issues/19189 Follow-up to https://github.com/element-hq/synapse/pull/18828 ### Background As part of Element's plan to support a light form of vhosting (virtual host) (multiple instances of Synapse in the same Python process) (c.f [Synapse Pro for small hosts](https://docs.element.io/latest/element-server-suite-pro/synapse-pro-for-small-hosts/overview/)), we're currently diving into the details and implications of running multiple instances of Synapse in the same Python process. "Clean tenant deprovisioning" tracked internally by https://github.com/element-hq/synapse-small-hosts/issues/50 --- changelog.d/19232.misc | 1 + synapse/app/_base.py | 114 ++++++++++++++++++++++++++++++----------- synapse/http/site.py | 7 +++ 3 files changed, 91 insertions(+), 31 deletions(-) create mode 100644 changelog.d/19232.misc diff --git a/changelog.d/19232.misc b/changelog.d/19232.misc new file mode 100644 index 00000000000..6e3e2ff6494 --- /dev/null +++ b/changelog.d/19232.misc @@ -0,0 +1 @@ +Fix `HomeServer.shutdown()` failing if the homeserver failed to `start`. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 52bdb9e0d71..d1ed1201e5a 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -41,7 +41,7 @@ from wsgiref.simple_server import WSGIServer from cryptography.utils import CryptographyDeprecationWarning -from typing_extensions import ParamSpec +from typing_extensions import ParamSpec, assert_never import twisted from twisted.internet import defer, error, reactor as _reactor @@ -64,7 +64,12 @@ from synapse.config import ConfigError from synapse.config._base import format_config_error from synapse.config.homeserver import HomeServerConfig -from synapse.config.server import ListenerConfig, ManholeConfig, TCPListenerConfig +from synapse.config.server import ( + ListenerConfig, + ManholeConfig, + TCPListenerConfig, + UnixListenerConfig, +) from synapse.crypto import context_factory from synapse.events.auto_accept_invites import InviteAutoAccepter from synapse.events.presence_router import load_legacy_presence_router @@ -413,6 +418,37 @@ def listen_unix( ] +class ListenerException(RuntimeError): + """ + An exception raised when we fail to listen with the given `ListenerConfig`. + + Attributes: + listener_config: The listener config that caused the exception. + """ + + def __init__( + self, + listener_config: ListenerConfig, + ): + listener_human_name = "" + port = "" + if isinstance(listener_config, TCPListenerConfig): + listener_human_name = "TCP port" + port = str(listener_config.port) + elif isinstance(listener_config, UnixListenerConfig): + listener_human_name = "unix socket" + port = listener_config.path + else: + assert_never(listener_config) + + super().__init__( + "Failed to listen on %s (%s) with the given listener config: %s" + % (listener_human_name, port, listener_config) + ) + + self.listener_config = listener_config + + def listen_http( hs: "HomeServer", listener_config: ListenerConfig, @@ -447,39 +483,55 @@ def listen_http( hs=hs, ) - if isinstance(listener_config, TCPListenerConfig): - if listener_config.is_tls(): - # refresh_certificate should have been called before this. - assert context_factory is not None - ports = listen_ssl( - listener_config.bind_addresses, - listener_config.port, - site, - context_factory, - reactor=reactor, + try: + if isinstance(listener_config, TCPListenerConfig): + if listener_config.is_tls(): + # refresh_certificate should have been called before this. + assert context_factory is not None + ports = listen_ssl( + listener_config.bind_addresses, + listener_config.port, + site, + context_factory, + reactor=reactor, + ) + logger.info( + "Synapse now listening on TCP port %d (TLS)", listener_config.port + ) + else: + ports = listen_tcp( + listener_config.bind_addresses, + listener_config.port, + site, + reactor=reactor, + ) + logger.info( + "Synapse now listening on TCP port %d", listener_config.port + ) + + elif isinstance(listener_config, UnixListenerConfig): + ports = listen_unix( + listener_config.path, listener_config.mode, site, reactor=reactor ) + # getHost() returns a UNIXAddress which contains an instance variable of 'name' + # encoded as a byte string. Decode as utf-8 so pretty. logger.info( - "Synapse now listening on TCP port %d (TLS)", listener_config.port + "Synapse now listening on Unix Socket at: %s", + ports[0].getHost().name.decode("utf-8"), ) else: - ports = listen_tcp( - listener_config.bind_addresses, - listener_config.port, - site, - reactor=reactor, - ) - logger.info("Synapse now listening on TCP port %d", listener_config.port) - - else: - ports = listen_unix( - listener_config.path, listener_config.mode, site, reactor=reactor - ) - # getHost() returns a UNIXAddress which contains an instance variable of 'name' - # encoded as a byte string. Decode as utf-8 so pretty. - logger.info( - "Synapse now listening on Unix Socket at: %s", - ports[0].getHost().name.decode("utf-8"), - ) + assert_never(listener_config) + except Exception as exc: + # The Twisted interface says that "Users should not call this function + # themselves!" but this appears to be the correct/only way handle proper cleanup + # of the site when things go wrong. In the normal case, a `Port` is created + # which we can call `Port.stopListening()` on to do the same thing (but no + # `Port` is created when an error occurs). + # + # We use `site.stopFactory()` instead of `site.doStop()` as the latter assumes + # that `site.doStart()` was called (which won't be the case if an error occurs). + site.stopFactory() + raise ListenerException(listener_config) from exc return ports diff --git a/synapse/http/site.py b/synapse/http/site.py index 03d5d048b11..a1b0b8d9c2f 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -815,6 +815,13 @@ def stopFactory(self) -> None: protocol.transport.loseConnection() self.connections.clear() + # Replace the resource tree with an empty resource to break circular references + # to the resource tree which holds a bunch of homeserver references. This is + # important if we try to call `hs.shutdown()` after `start` fails. For some + # reason, this doesn't seem to be necessary in the normal case where `start` + # succeeds and we call `hs.shutdown()` later. + self.resource = Resource() + def log(self, request: SynapseRequest) -> None: # type: ignore[override] pass From aff90a52453174ababf1f5571cc1724369785a3a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Dec 2025 20:03:07 +0000 Subject: [PATCH 047/139] Bump bleach from 6.2.0 to 6.3.0 (#19265) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index a6de2a86f08..44368c4903d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -134,14 +134,14 @@ typecheck = ["mypy"] [[package]] name = "bleach" -version = "6.2.0" +version = "6.3.0" description = "An easy safelist-based HTML-sanitizing tool." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "bleach-6.2.0-py3-none-any.whl", hash = "sha256:117d9c6097a7c3d22fd578fcd8d35ff1e125df6736f554da4e432fdd63f31e5e"}, - {file = "bleach-6.2.0.tar.gz", hash = "sha256:123e894118b8a599fd80d3ec1a6d4cc7ce4e5882b1317a7e1ba69b56e95f991f"}, + {file = "bleach-6.3.0-py3-none-any.whl", hash = "sha256:fe10ec77c93ddf3d13a73b035abaac7a9f5e436513864ccdad516693213c65d6"}, + {file = "bleach-6.3.0.tar.gz", hash = "sha256:6f3b91b1c0a02bb9a78b5a454c92506aa0fdf197e1d5e114d2e00c6f64306d22"}, ] [package.dependencies] From d688daf41c372f0d5790a583b5706fab45cdce92 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 2 Dec 2025 20:08:32 +0000 Subject: [PATCH 048/139] Fix bug where `Duration` was logged incorrectly (#19267) ### Pull Request Checklist * [X] Pull request is based on the develop branch * [X] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [X] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/19267.bugfix | 1 + synapse/appservice/scheduler.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19267.bugfix diff --git a/changelog.d/19267.bugfix b/changelog.d/19267.bugfix new file mode 100644 index 00000000000..6c7ed750eca --- /dev/null +++ b/changelog.d/19267.bugfix @@ -0,0 +1 @@ +Fix bug where `Duration` was logged incorrectly. diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index befb4ae44b0..c3a83d140c9 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -506,7 +506,7 @@ def __init__( def recover(self) -> None: delay = Duration(seconds=2**self.backoff_counter) - logger.info("Scheduling retries on %s in %fs", self.service.id, delay) + logger.info("Scheduling retries on %s in %fs", self.service.id, delay.as_secs()) self.scheduled_recovery = self.clock.call_later( delay, self.hs.run_as_background_process, From 4cd05baaecb4eb93c64f41785f5224160f388ef1 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 2 Dec 2025 20:08:32 +0000 Subject: [PATCH 049/139] Fix bug where `Duration` was logged incorrectly (#19267) ### Pull Request Checklist * [X] Pull request is based on the develop branch * [X] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [X] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/19267.bugfix | 1 + synapse/appservice/scheduler.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19267.bugfix diff --git a/changelog.d/19267.bugfix b/changelog.d/19267.bugfix new file mode 100644 index 00000000000..6c7ed750eca --- /dev/null +++ b/changelog.d/19267.bugfix @@ -0,0 +1 @@ +Fix bug where `Duration` was logged incorrectly. diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index befb4ae44b0..c3a83d140c9 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -506,7 +506,7 @@ def __init__( def recover(self) -> None: delay = Duration(seconds=2**self.backoff_counter) - logger.info("Scheduling retries on %s in %fs", self.service.id, delay) + logger.info("Scheduling retries on %s in %fs", self.service.id, delay.as_secs()) self.scheduled_recovery = self.clock.call_later( delay, self.hs.run_as_background_process, From 989c4d2585003a7245dff1f5d3131fbbfaad2488 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 2 Dec 2025 13:11:50 -0700 Subject: [PATCH 050/139] Update changelog --- CHANGES.md | 3 ++- changelog.d/19267.bugfix | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/19267.bugfix diff --git a/CHANGES.md b/CHANGES.md index a58f3cea210..37c1edcba8a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,4 @@ -# synapse 1.144.0rc1 (2025-12-02) +# Synapse 1.144.0rc1 (2025-12-02) Admins using the unstable [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666) endpoint (`/_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms`), please check [the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/develop/docs/upgrade.md#upgrading-to-v11440) as this release contains changes that disable that endpoint by default. @@ -12,6 +12,7 @@ Admins using the unstable [MSC2666](https://github.com/matrix-org/matrix-spec-pr - Fix a bug in the database function for fetching state deltas that could result in unnecessarily long query times. ([\#18960](https://github.com/element-hq/synapse/issues/18960)) - Fix v12 rooms when running with `use_frozen_dicts: True`. ([\#19235](https://github.com/element-hq/synapse/issues/19235)) - Fix bug where invalid `canonical_alias` content would return 500 instead of 400. ([\#19240](https://github.com/element-hq/synapse/issues/19240)) +- Fix bug where `Duration` was logged incorrectly. ([\#19267](https://github.com/element-hq/synapse/issues/19267)) ## Improved Documentation diff --git a/changelog.d/19267.bugfix b/changelog.d/19267.bugfix deleted file mode 100644 index 6c7ed750eca..00000000000 --- a/changelog.d/19267.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where `Duration` was logged incorrectly. From 93e658bd13860c0fde4e9b80aab7b332f3bd6a0a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Dec 2025 20:27:05 +0000 Subject: [PATCH 051/139] Bump cryptography from 45.0.7 to 46.0.3 (#19266) Bumps [cryptography](https://github.com/pyca/cryptography) from 45.0.7 to 46.0.3.
Changelog

Sourced from cryptography's changelog.

46.0.3 - 2025-10-15


* Fixed compilation when using LibreSSL 4.2.0.

.. _v46-0-2:


46.0.2 - 2025-09-30

  • Updated Windows, macOS, and Linux wheels to be compiled with OpenSSL 3.5.4.

.. _v46-0-1:

46.0.1 - 2025-09-16


* Fixed an issue where users installing via ``pip`` on Python 3.14
development
  versions would not properly install a dependency.
* Fixed an issue building the free-threaded macOS 3.14 wheels.

.. _v46-0-0:


46.0.0 - 2025-09-16

  • BACKWARDS INCOMPATIBLE: Support for Python 3.7 has been removed.
  • Support for OpenSSL < 3.0 is deprecated and will be removed in the next release.
  • Support for x86_64 macOS (including publishing wheels) is deprecated and will be removed in two releases. We will switch to publishing an arm64 only wheel for macOS.
  • Support for 32-bit Windows (including publishing wheels) is deprecated and will be removed in two releases. Users should move to a 64-bit Python installation.
  • Updated Windows, macOS, and Linux wheels to be compiled with OpenSSL 3.5.3.
  • We now build ppc64le manylinux wheels and publish them to PyPI.
  • We now build win_arm64 (Windows on Arm) wheels and publish them to PyPI.
  • Added support for free-threaded Python 3.14.
  • Removed the deprecated get_attribute_for_oid method on :class:~cryptography.x509.CertificateSigningRequest. Users should use :meth:~cryptography.x509.Attributes.get_attribute_for_oid instead.
  • Removed the deprecated CAST5, SEED, IDEA, and Blowfish classes from the cipher module. These are still available in :doc:/hazmat/decrepit/index.
  • In X.509, when performing a PSS signature with a SHA-3 hash, it is now encoded with the official NIST SHA3 OID.

.. _v45-0-7:

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=cryptography&package-manager=pip&previous-version=45.0.7&new-version=46.0.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 264 +++++++++++++++++++++++++++++----------------------- 1 file changed, 150 insertions(+), 114 deletions(-) diff --git a/poetry.lock b/poetry.lock index 44368c4903d..271be1456e4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -176,83 +176,100 @@ files = [ [[package]] name = "cffi" -version = "1.17.1" +version = "2.0.0" description = "Foreign Function Interface for Python calling C code." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, - {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, - {file = "cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382"}, - {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702"}, - {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3"}, - {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6"}, - {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17"}, - {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8"}, - {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e"}, - {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be"}, - {file = "cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c"}, - {file = "cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15"}, - {file = "cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401"}, - {file = "cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf"}, - {file = "cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4"}, - {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41"}, - {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1"}, - {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6"}, - {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d"}, - {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6"}, - {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f"}, - {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b"}, - {file = "cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655"}, - {file = "cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0"}, - {file = "cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4"}, - {file = "cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c"}, - {file = "cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36"}, - {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5"}, - {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff"}, - {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99"}, - {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93"}, - {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3"}, - {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8"}, - {file = "cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65"}, - {file = "cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903"}, - {file = "cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e"}, - {file = "cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2"}, - {file = "cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3"}, - {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683"}, - {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5"}, - {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4"}, - {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd"}, - {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed"}, - {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9"}, - {file = "cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d"}, - {file = "cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a"}, - {file = "cffi-1.17.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b"}, - {file = "cffi-1.17.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964"}, - {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9"}, - {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc"}, - {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c"}, - {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1"}, - {file = "cffi-1.17.1-cp38-cp38-win32.whl", hash = "sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8"}, - {file = "cffi-1.17.1-cp38-cp38-win_amd64.whl", hash = "sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1"}, - {file = "cffi-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16"}, - {file = "cffi-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36"}, - {file = "cffi-1.17.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8"}, - {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca74b8dbe6e8e8263c0ffd60277de77dcee6c837a3d0881d8c1ead7268c9e576"}, - {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87"}, - {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98e3969bcff97cae1b2def8ba499ea3d6f31ddfdb7635374834cf89a1a08ecf0"}, - {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdf5ce3acdfd1661132f2a9c19cac174758dc2352bfe37d98aa7512c6b7178b3"}, - {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9755e4345d1ec879e3849e62222a18c7174d65a6a92d5b346b1863912168b595"}, - {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f1e22e8c4419538cb197e4dd60acc919d7696e5ef98ee4da4e01d3f8cfa4cc5a"}, - {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c03e868a0b3bc35839ba98e74211ed2b05d2119be4e8a0f224fba9384f1fe02e"}, - {file = "cffi-1.17.1-cp39-cp39-win32.whl", hash = "sha256:e31ae45bc2e29f6b2abd0de1cc3b9d5205aa847cafaecb8af1476a609a2f6eb7"}, - {file = "cffi-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662"}, - {file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"}, + {file = "cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44"}, + {file = "cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453"}, + {file = "cffi-2.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495"}, + {file = "cffi-2.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5"}, + {file = "cffi-2.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb"}, + {file = "cffi-2.0.0-cp310-cp310-win32.whl", hash = "sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a"}, + {file = "cffi-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739"}, + {file = "cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe"}, + {file = "cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26"}, + {file = "cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9"}, + {file = "cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414"}, + {file = "cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743"}, + {file = "cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5"}, + {file = "cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5"}, + {file = "cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d"}, + {file = "cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d"}, + {file = "cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba"}, + {file = "cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94"}, + {file = "cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187"}, + {file = "cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18"}, + {file = "cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5"}, + {file = "cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6"}, + {file = "cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb"}, + {file = "cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26"}, + {file = "cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c"}, + {file = "cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b"}, + {file = "cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27"}, + {file = "cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75"}, + {file = "cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91"}, + {file = "cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5"}, + {file = "cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775"}, + {file = "cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205"}, + {file = "cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1"}, + {file = "cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f"}, + {file = "cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25"}, + {file = "cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad"}, + {file = "cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9"}, + {file = "cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592"}, + {file = "cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512"}, + {file = "cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4"}, + {file = "cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e"}, + {file = "cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6"}, + {file = "cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9"}, + {file = "cffi-2.0.0-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf"}, + {file = "cffi-2.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:de8dad4425a6ca6e4e5e297b27b5c824ecc7581910bf9aee86cb6835e6812aa7"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:4647afc2f90d1ddd33441e5b0e85b16b12ddec4fca55f0d9671fef036ecca27c"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3f4d46d8b35698056ec29bca21546e1551a205058ae1a181d871e278b0b28165"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e6e73b9e02893c764e7e8d5bb5ce277f1a009cd5243f8228f75f842bf937c534"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:cb527a79772e5ef98fb1d700678fe031e353e765d1ca2d409c92263c6d43e09f"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:61d028e90346df14fedc3d1e5441df818d095f3b87d286825dfcbd6459b7ef63"}, + {file = "cffi-2.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0f6084a0ea23d05d20c3edcda20c3d006f9b6f3fefeac38f59262e10cef47ee2"}, + {file = "cffi-2.0.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:1cd13c99ce269b3ed80b417dcd591415d3372bcac067009b6e0f59c7d4015e65"}, + {file = "cffi-2.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:89472c9762729b5ae1ad974b777416bfda4ac5642423fa93bd57a09204712322"}, + {file = "cffi-2.0.0-cp39-cp39-win32.whl", hash = "sha256:2081580ebb843f759b9f617314a24ed5738c51d2aee65d31e02f6f7a2b97707a"}, + {file = "cffi-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9"}, + {file = "cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529"}, ] [package.dependencies] -pycparser = "*" +pycparser = {version = "*", markers = "implementation_name != \"PyPy\""} [[package]] name = "charset-normalizer" @@ -381,62 +398,80 @@ files = [ [[package]] name = "cryptography" -version = "45.0.7" +version = "46.0.3" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false -python-versions = "!=3.9.0,!=3.9.1,>=3.7" +python-versions = "!=3.9.0,!=3.9.1,>=3.8" groups = ["main", "dev"] files = [ - {file = "cryptography-45.0.7-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:3be4f21c6245930688bd9e162829480de027f8bf962ede33d4f8ba7d67a00cee"}, - {file = "cryptography-45.0.7-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:67285f8a611b0ebc0857ced2081e30302909f571a46bfa7a3cc0ad303fe015c6"}, - {file = "cryptography-45.0.7-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:577470e39e60a6cd7780793202e63536026d9b8641de011ed9d8174da9ca5339"}, - {file = "cryptography-45.0.7-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4bd3e5c4b9682bc112d634f2c6ccc6736ed3635fc3319ac2bb11d768cc5a00d8"}, - {file = "cryptography-45.0.7-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:465ccac9d70115cd4de7186e60cfe989de73f7bb23e8a7aa45af18f7412e75bf"}, - {file = "cryptography-45.0.7-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:16ede8a4f7929b4b7ff3642eba2bf79aa1d71f24ab6ee443935c0d269b6bc513"}, - {file = "cryptography-45.0.7-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:8978132287a9d3ad6b54fcd1e08548033cc09dc6aacacb6c004c73c3eb5d3ac3"}, - {file = "cryptography-45.0.7-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:b6a0e535baec27b528cb07a119f321ac024592388c5681a5ced167ae98e9fff3"}, - {file = "cryptography-45.0.7-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a24ee598d10befaec178efdff6054bc4d7e883f615bfbcd08126a0f4931c83a6"}, - {file = "cryptography-45.0.7-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:fa26fa54c0a9384c27fcdc905a2fb7d60ac6e47d14bc2692145f2b3b1e2cfdbd"}, - {file = "cryptography-45.0.7-cp311-abi3-win32.whl", hash = "sha256:bef32a5e327bd8e5af915d3416ffefdbe65ed975b646b3805be81b23580b57b8"}, - {file = "cryptography-45.0.7-cp311-abi3-win_amd64.whl", hash = "sha256:3808e6b2e5f0b46d981c24d79648e5c25c35e59902ea4391a0dcb3e667bf7443"}, - {file = "cryptography-45.0.7-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bfb4c801f65dd61cedfc61a83732327fafbac55a47282e6f26f073ca7a41c3b2"}, - {file = "cryptography-45.0.7-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:81823935e2f8d476707e85a78a405953a03ef7b7b4f55f93f7c2d9680e5e0691"}, - {file = "cryptography-45.0.7-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3994c809c17fc570c2af12c9b840d7cea85a9fd3e5c0e0491f4fa3c029216d59"}, - {file = "cryptography-45.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dad43797959a74103cb59c5dac71409f9c27d34c8a05921341fb64ea8ccb1dd4"}, - {file = "cryptography-45.0.7-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ce7a453385e4c4693985b4a4a3533e041558851eae061a58a5405363b098fcd3"}, - {file = "cryptography-45.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b04f85ac3a90c227b6e5890acb0edbaf3140938dbecf07bff618bf3638578cf1"}, - {file = "cryptography-45.0.7-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:48c41a44ef8b8c2e80ca4527ee81daa4c527df3ecbc9423c41a420a9559d0e27"}, - {file = "cryptography-45.0.7-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:f3df7b3d0f91b88b2106031fd995802a2e9ae13e02c36c1fc075b43f420f3a17"}, - {file = "cryptography-45.0.7-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:dd342f085542f6eb894ca00ef70236ea46070c8a13824c6bde0dfdcd36065b9b"}, - {file = "cryptography-45.0.7-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1993a1bb7e4eccfb922b6cd414f072e08ff5816702a0bdb8941c247a6b1b287c"}, - {file = "cryptography-45.0.7-cp37-abi3-win32.whl", hash = "sha256:18fcf70f243fe07252dcb1b268a687f2358025ce32f9f88028ca5c364b123ef5"}, - {file = "cryptography-45.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:7285a89df4900ed3bfaad5679b1e668cb4b38a8de1ccbfc84b05f34512da0a90"}, - {file = "cryptography-45.0.7-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:de58755d723e86175756f463f2f0bddd45cc36fbd62601228a3f8761c9f58252"}, - {file = "cryptography-45.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a20e442e917889d1a6b3c570c9e3fa2fdc398c20868abcea268ea33c024c4083"}, - {file = "cryptography-45.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:258e0dff86d1d891169b5af222d362468a9570e2532923088658aa866eb11130"}, - {file = "cryptography-45.0.7-pp310-pypy310_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:d97cf502abe2ab9eff8bd5e4aca274da8d06dd3ef08b759a8d6143f4ad65d4b4"}, - {file = "cryptography-45.0.7-pp310-pypy310_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:c987dad82e8c65ebc985f5dae5e74a3beda9d0a2a4daf8a1115f3772b59e5141"}, - {file = "cryptography-45.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c13b1e3afd29a5b3b2656257f14669ca8fa8d7956d509926f0b130b600b50ab7"}, - {file = "cryptography-45.0.7-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:4a862753b36620af6fc54209264f92c716367f2f0ff4624952276a6bbd18cbde"}, - {file = "cryptography-45.0.7-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:06ce84dc14df0bf6ea84666f958e6080cdb6fe1231be2a51f3fc1267d9f3fb34"}, - {file = "cryptography-45.0.7-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d0c5c6bac22b177bf8da7435d9d27a6834ee130309749d162b26c3105c0795a9"}, - {file = "cryptography-45.0.7-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:2f641b64acc00811da98df63df7d59fd4706c0df449da71cb7ac39a0732b40ae"}, - {file = "cryptography-45.0.7-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:f5414a788ecc6ee6bc58560e85ca624258a55ca434884445440a810796ea0e0b"}, - {file = "cryptography-45.0.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:1f3d56f73595376f4244646dd5c5870c14c196949807be39e79e7bd9bac3da63"}, - {file = "cryptography-45.0.7.tar.gz", hash = "sha256:4b1654dfc64ea479c242508eb8c724044f1e964a47d1d1cacc5132292d851971"}, + {file = "cryptography-46.0.3-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:109d4ddfadf17e8e7779c39f9b18111a09efb969a301a31e987416a0191ed93a"}, + {file = "cryptography-46.0.3-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:09859af8466b69bc3c27bdf4f5d84a665e0f7ab5088412e9e2ec49758eca5cbc"}, + {file = "cryptography-46.0.3-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:01ca9ff2885f3acc98c29f1860552e37f6d7c7d013d7334ff2a9de43a449315d"}, + {file = "cryptography-46.0.3-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6eae65d4c3d33da080cff9c4ab1f711b15c1d9760809dad6ea763f3812d254cb"}, + {file = "cryptography-46.0.3-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5bf0ed4490068a2e72ac03d786693adeb909981cc596425d09032d372bcc849"}, + {file = "cryptography-46.0.3-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:5ecfccd2329e37e9b7112a888e76d9feca2347f12f37918facbb893d7bb88ee8"}, + {file = "cryptography-46.0.3-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a2c0cd47381a3229c403062f764160d57d4d175e022c1df84e168c6251a22eec"}, + {file = "cryptography-46.0.3-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:549e234ff32571b1f4076ac269fcce7a808d3bf98b76c8dd560e42dbc66d7d91"}, + {file = "cryptography-46.0.3-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:c0a7bb1a68a5d3471880e264621346c48665b3bf1c3759d682fc0864c540bd9e"}, + {file = "cryptography-46.0.3-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:10b01676fc208c3e6feeb25a8b83d81767e8059e1fe86e1dc62d10a3018fa926"}, + {file = "cryptography-46.0.3-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0abf1ffd6e57c67e92af68330d05760b7b7efb243aab8377e583284dbab72c71"}, + {file = "cryptography-46.0.3-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a04bee9ab6a4da801eb9b51f1b708a1b5b5c9eb48c03f74198464c66f0d344ac"}, + {file = "cryptography-46.0.3-cp311-abi3-win32.whl", hash = "sha256:f260d0d41e9b4da1ed1e0f1ce571f97fe370b152ab18778e9e8f67d6af432018"}, + {file = "cryptography-46.0.3-cp311-abi3-win_amd64.whl", hash = "sha256:a9a3008438615669153eb86b26b61e09993921ebdd75385ddd748702c5adfddb"}, + {file = "cryptography-46.0.3-cp311-abi3-win_arm64.whl", hash = "sha256:5d7f93296ee28f68447397bf5198428c9aeeab45705a55d53a6343455dcb2c3c"}, + {file = "cryptography-46.0.3-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:00a5e7e87938e5ff9ff5447ab086a5706a957137e6e433841e9d24f38a065217"}, + {file = "cryptography-46.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c8daeb2d2174beb4575b77482320303f3d39b8e81153da4f0fb08eb5fe86a6c5"}, + {file = "cryptography-46.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:39b6755623145ad5eff1dab323f4eae2a32a77a7abef2c5089a04a3d04366715"}, + {file = "cryptography-46.0.3-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:db391fa7c66df6762ee3f00c95a89e6d428f4d60e7abc8328f4fe155b5ac6e54"}, + {file = "cryptography-46.0.3-cp314-cp314t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:78a97cf6a8839a48c49271cdcbd5cf37ca2c1d6b7fdd86cc864f302b5e9bf459"}, + {file = "cryptography-46.0.3-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:dfb781ff7eaa91a6f7fd41776ec37c5853c795d3b358d4896fdbb5df168af422"}, + {file = "cryptography-46.0.3-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:6f61efb26e76c45c4a227835ddeae96d83624fb0d29eb5df5b96e14ed1a0afb7"}, + {file = "cryptography-46.0.3-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:23b1a8f26e43f47ceb6d6a43115f33a5a37d57df4ea0ca295b780ae8546e8044"}, + {file = "cryptography-46.0.3-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:b419ae593c86b87014b9be7396b385491ad7f320bde96826d0dd174459e54665"}, + {file = "cryptography-46.0.3-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:50fc3343ac490c6b08c0cf0d704e881d0d660be923fd3076db3e932007e726e3"}, + {file = "cryptography-46.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:22d7e97932f511d6b0b04f2bfd818d73dcd5928db509460aaf48384778eb6d20"}, + {file = "cryptography-46.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d55f3dffadd674514ad19451161118fd010988540cee43d8bc20675e775925de"}, + {file = "cryptography-46.0.3-cp314-cp314t-win32.whl", hash = "sha256:8a6e050cb6164d3f830453754094c086ff2d0b2f3a897a1d9820f6139a1f0914"}, + {file = "cryptography-46.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:760f83faa07f8b64e9c33fc963d790a2edb24efb479e3520c14a45741cd9b2db"}, + {file = "cryptography-46.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:516ea134e703e9fe26bcd1277a4b59ad30586ea90c365a87781d7887a646fe21"}, + {file = "cryptography-46.0.3-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:cb3d760a6117f621261d662bccc8ef5bc32ca673e037c83fbe565324f5c46936"}, + {file = "cryptography-46.0.3-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4b7387121ac7d15e550f5cb4a43aef2559ed759c35df7336c402bb8275ac9683"}, + {file = "cryptography-46.0.3-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:15ab9b093e8f09daab0f2159bb7e47532596075139dd74365da52ecc9cb46c5d"}, + {file = "cryptography-46.0.3-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:46acf53b40ea38f9c6c229599a4a13f0d46a6c3fa9ef19fc1a124d62e338dfa0"}, + {file = "cryptography-46.0.3-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10ca84c4668d066a9878890047f03546f3ae0a6b8b39b697457b7757aaf18dbc"}, + {file = "cryptography-46.0.3-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:36e627112085bb3b81b19fed209c05ce2a52ee8b15d161b7c643a7d5a88491f3"}, + {file = "cryptography-46.0.3-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1000713389b75c449a6e979ffc7dcc8ac90b437048766cef052d4d30b8220971"}, + {file = "cryptography-46.0.3-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:b02cf04496f6576afffef5ddd04a0cb7d49cf6be16a9059d793a30b035f6b6ac"}, + {file = "cryptography-46.0.3-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:71e842ec9bc7abf543b47cf86b9a743baa95f4677d22baa4c7d5c69e49e9bc04"}, + {file = "cryptography-46.0.3-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:402b58fc32614f00980b66d6e56a5b4118e6cb362ae8f3fda141ba4689bd4506"}, + {file = "cryptography-46.0.3-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ef639cb3372f69ec44915fafcd6698b6cc78fbe0c2ea41be867f6ed612811963"}, + {file = "cryptography-46.0.3-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3b51b8ca4f1c6453d8829e1eb7299499ca7f313900dd4d89a24b8b87c0a780d4"}, + {file = "cryptography-46.0.3-cp38-abi3-win32.whl", hash = "sha256:6276eb85ef938dc035d59b87c8a7dc559a232f954962520137529d77b18ff1df"}, + {file = "cryptography-46.0.3-cp38-abi3-win_amd64.whl", hash = "sha256:416260257577718c05135c55958b674000baef9a1c7d9e8f306ec60d71db850f"}, + {file = "cryptography-46.0.3-cp38-abi3-win_arm64.whl", hash = "sha256:d89c3468de4cdc4f08a57e214384d0471911a3830fcdaf7a8cc587e42a866372"}, + {file = "cryptography-46.0.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a23582810fedb8c0bc47524558fb6c56aac3fc252cb306072fd2815da2a47c32"}, + {file = "cryptography-46.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:e7aec276d68421f9574040c26e2a7c3771060bc0cff408bae1dcb19d3ab1e63c"}, + {file = "cryptography-46.0.3-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7ce938a99998ed3c8aa7e7272dca1a610401ede816d36d0693907d863b10d9ea"}, + {file = "cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:191bb60a7be5e6f54e30ba16fdfae78ad3a342a0599eb4193ba88e3f3d6e185b"}, + {file = "cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c70cc23f12726be8f8bc72e41d5065d77e4515efae3690326764ea1b07845cfb"}, + {file = "cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:9394673a9f4de09e28b5356e7fff97d778f8abad85c9d5ac4a4b7e25a0de7717"}, + {file = "cryptography-46.0.3-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:94cd0549accc38d1494e1f8de71eca837d0509d0d44bf11d158524b0e12cebf9"}, + {file = "cryptography-46.0.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6b5063083824e5509fdba180721d55909ffacccc8adbec85268b48439423d78c"}, + {file = "cryptography-46.0.3.tar.gz", hash = "sha256:a8b17438104fed022ce745b362294d9ce35b4c2e45c1d958ad4a4b019285f4a1"}, ] [package.dependencies] -cffi = {version = ">=1.14", markers = "platform_python_implementation != \"PyPy\""} +cffi = {version = ">=2.0.0", markers = "python_full_version >= \"3.9.0\" and platform_python_implementation != \"PyPy\""} +typing-extensions = {version = ">=4.13.2", markers = "python_full_version < \"3.11.0\""} [package.extras] -docs = ["sphinx (>=5.3.0)", "sphinx-inline-tabs ; python_full_version >= \"3.8.0\"", "sphinx-rtd-theme (>=3.0.0) ; python_full_version >= \"3.8.0\""] +docs = ["sphinx (>=5.3.0)", "sphinx-inline-tabs", "sphinx-rtd-theme (>=3.0.0)"] docstest = ["pyenchant (>=3)", "readme-renderer (>=30.0)", "sphinxcontrib-spelling (>=7.3.1)"] -nox = ["nox (>=2024.4.15)", "nox[uv] (>=2024.3.2) ; python_full_version >= \"3.8.0\""] -pep8test = ["check-sdist ; python_full_version >= \"3.8.0\"", "click (>=8.0.1)", "mypy (>=1.4)", "ruff (>=0.3.6)"] +nox = ["nox[uv] (>=2024.4.15)"] +pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.14)", "ruff (>=0.11.11)"] sdist = ["build (>=1.0.0)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi (>=2024)", "cryptography-vectors (==45.0.7)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] +test = ["certifi (>=2024)", "cryptography-vectors (==46.0.3)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] test-randomorder = ["pytest-randomly"] [[package]] @@ -1792,6 +1827,7 @@ description = "C parser in Python" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" groups = ["main", "dev"] +markers = "implementation_name != \"PyPy\"" files = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, From e8710e7c5e45ff511141ba17facda276ecb861b0 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 4 Dec 2025 23:49:24 +0000 Subject: [PATCH 052/139] Don't include debug logs in `Clock` unless explicitly enabled (#19278) Fixes #19276 This log with stack traces results in a ton of noise in the logs and is confusing to users since it looks like it's an error in the logs. This PR removes the stack trace from the log. This can be re-enabled on demand if it is deemed necessary in the future. ### Pull Request Checklist * [X] Pull request is based on the develop branch * [X] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [X] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- changelog.d/19278.misc | 1 + synapse/util/clock.py | 42 ++++++++++++++++++++++++++++++++---------- 2 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 changelog.d/19278.misc diff --git a/changelog.d/19278.misc b/changelog.d/19278.misc new file mode 100644 index 00000000000..d1425ff38c2 --- /dev/null +++ b/changelog.d/19278.misc @@ -0,0 +1 @@ +Don't include debug logs in `Clock` unless explicitly enabled. diff --git a/synapse/util/clock.py b/synapse/util/clock.py index 6fd31864b75..4355704f8a2 100644 --- a/synapse/util/clock.py +++ b/synapse/util/clock.py @@ -29,6 +29,7 @@ from twisted.internet.task import LoopingCall from synapse.logging import context +from synapse.logging.loggers import ExplicitlyConfiguredLogger from synapse.types import ISynapseThreadlessReactor from synapse.util import log_failure from synapse.util.duration import Duration @@ -39,6 +40,25 @@ logger = logging.getLogger(__name__) +original_logger_class = logging.getLoggerClass() +logging.setLoggerClass(ExplicitlyConfiguredLogger) +clock_debug_logger = logging.getLogger("synapse.util.clock.debug") +""" +A logger for debugging what is scheduling calls. + +Ideally, these wouldn't be gated behind an `ExplicitlyConfiguredLogger` as including logs +from this logger would be helpful to track when things are being scheduled. However, for +these logs to be meaningful, they need to include a stack trace to show what initiated the +call in the first place. + +Since the stack traces can create a lot of noise and make the logs hard to read (unless you're +specifically debugging scheduling issues) we want users to opt-in to seeing these logs. To enable +this, they must explicitly set `synapse.util.clock.debug` in the logging configuration. Note that +this setting won't inherit the log level from the parent logger. +""" +# Restore the original logger class +logging.setLoggerClass(original_logger_class) + class Clock: """ @@ -174,7 +194,7 @@ def _looping_call_common( looping_call_context_string = "looping_call_now" def wrapped_f(*args: P.args, **kwargs: P.kwargs) -> Deferred: - logger.debug( + clock_debug_logger.debug( "%s(%s): Executing callback", looping_call_context_string, instance_id ) @@ -222,7 +242,7 @@ def wrapped_f(*args: P.args, **kwargs: P.kwargs) -> Deferred: d.addErrback(log_failure, "Looping call died", consumeErrors=False) self._looping_calls.append(call) - logger.debug( + clock_debug_logger.debug( "%s(%s): Scheduled looping call every %sms later", looping_call_context_string, instance_id, @@ -283,7 +303,7 @@ def call_later( raise Exception("Cannot start delayed call. Clock has been shutdown") def wrapped_callback(*args: Any, **kwargs: Any) -> None: - logger.debug("call_later(%s): Executing callback", call_id) + clock_debug_logger.debug("call_later(%s): Executing callback", call_id) assert context.current_context() is context.SENTINEL_CONTEXT, ( "Expected `call_later` callback from the reactor to start with the sentinel logcontext " @@ -327,7 +347,7 @@ def wrapped_callback(*args: Any, **kwargs: Any) -> None: delay.as_secs(), wrapped_callback, *args, **kwargs ) # type: ignore[call-later-not-tracked] - logger.debug( + clock_debug_logger.debug( "call_later(%s): Scheduled call for %ss later (tracked for shutdown: %s)", call_id, delay, @@ -347,7 +367,7 @@ def cancel_call_later( self, wrapped_call: "DelayedCallWrapper", ignore_errs: bool = False ) -> None: try: - logger.debug( + clock_debug_logger.debug( "cancel_call_later: cancelling scheduled call %s", wrapped_call.call_id ) wrapped_call.delayed_call.cancel() @@ -367,7 +387,7 @@ def cancel_all_delayed_calls(self, ignore_errs: bool = True) -> None: # will result in the call removing itself from the map mid-iteration. for call_id, call in list(self._call_id_to_delayed_call.items()): try: - logger.debug( + clock_debug_logger.debug( "cancel_all_delayed_calls: cancelling scheduled call %s", call_id ) call.cancel() @@ -396,7 +416,9 @@ def call_when_running( instance_id = random_string_insecure_fast(5) def wrapped_callback(*args: Any, **kwargs: Any) -> None: - logger.debug("call_when_running(%s): Executing callback", instance_id) + clock_debug_logger.debug( + "call_when_running(%s): Executing callback", instance_id + ) # Since this callback can be invoked immediately if the reactor is already # running, we can't always assume that we're running in the sentinel @@ -436,7 +458,7 @@ def wrapped_callback(*args: Any, **kwargs: Any) -> None: # callWhenRunning should be called. self._reactor.callWhenRunning(wrapped_callback, *args, **kwargs) # type: ignore[prefer-synapse-clock-call-when-running] - logger.debug( + clock_debug_logger.debug( "call_when_running(%s): Scheduled call", instance_id, # Find out who is scheduling the call which makes it easy to follow in the @@ -472,7 +494,7 @@ def add_system_event_trigger( instance_id = random_string_insecure_fast(5) def wrapped_callback(*args: Any, **kwargs: Any) -> None: - logger.debug( + clock_debug_logger.debug( "add_system_event_trigger(%s): Executing %s %s callback", instance_id, phase, @@ -509,7 +531,7 @@ def wrapped_callback(*args: Any, **kwargs: Any) -> None: # logcontext to the reactor context.run_in_background(callback, *args, **kwargs) - logger.debug( + clock_debug_logger.debug( "add_system_event_trigger(%s) for %s %s", instance_id, phase, From a096fba969f43fb89d946d14a8b41ed23b7538f6 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 5 Dec 2025 10:48:01 +0000 Subject: [PATCH 053/139] Group non-breaking dependabot PRs together to reduce review load (#18402) --- .github/dependabot.yml | 46 ++++++++++++++++++++++++++++++++++++++++++ changelog.d/18402.misc | 1 + 2 files changed, 47 insertions(+) create mode 100644 changelog.d/18402.misc diff --git a/.github/dependabot.yml b/.github/dependabot.yml index cfaa3c04e33..38920ead7ab 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,6 +9,22 @@ updates: open-pull-requests-limit: 10 schedule: interval: "weekly" + # Group patch updates to packages together into a single PR, as they rarely + # if ever contain breaking changes that need to be reviewed separately. + # + # Less PRs means a streamlined review process. + # + # Python packages follow semantic versioning, and tend to only introduce + # breaking changes in major version bumps. Thus, we'll group minor and patch + # versions together. + groups: + minor-and-patches: + applies-to: version-updates + patterns: + - "*" + update-types: + - "minor" + - "patch" # Prevent pulling packages that were recently updated to help mitigate # supply chain attacks. 14 days was taken from the recommendation at # https://blog.yossarian.net/2025/11/21/We-should-all-be-using-dependency-cooldowns @@ -25,6 +41,16 @@ updates: open-pull-requests-limit: 10 schedule: interval: "weekly" + # For container versions, breaking changes are also typically only introduced in major + # package bumps. + groups: + minor-and-patches: + applies-to: version-updates + patterns: + - "*" + update-types: + - "minor" + - "patch" cooldown: default-days: 14 @@ -33,6 +59,16 @@ updates: open-pull-requests-limit: 10 schedule: interval: "weekly" + # Similarly for GitHub Actions, breaking changes are typically only introduced in major + # package bumps. + groups: + minor-and-patches: + applies-to: version-updates + patterns: + - "*" + update-types: + - "minor" + - "patch" cooldown: default-days: 14 @@ -42,5 +78,15 @@ updates: versioning-strategy: "lockfile-only" schedule: interval: "weekly" + # The Rust ecosystem is special in that breaking changes are often introduced + # in minor version bumps, as packages typically stay pre-1.0 for a long time. + # Thus we specifically keep minor version bumps separate in their own PRs. + groups: + patches: + applies-to: version-updates + patterns: + - "*" + update-types: + - "patch" cooldown: default-days: 14 diff --git a/changelog.d/18402.misc b/changelog.d/18402.misc new file mode 100644 index 00000000000..4b13652845c --- /dev/null +++ b/changelog.d/18402.misc @@ -0,0 +1 @@ +Group together dependabot update PRs to reduce the review load. \ No newline at end of file From 891983f3f42b12f7dedb84768c0bfa7b5dc49473 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Dec 2025 22:11:58 +0000 Subject: [PATCH 054/139] Bump the minor-and-patches group with 3 updates (#19280) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps the minor-and-patches group with 3 updates: [mypy](https://github.com/python/mypy), [mypy-zope](https://github.com/Shoobx/mypy-zope) and [phonenumbers](https://github.com/daviddrysdale/python-phonenumbers). Updates `mypy` from 1.17.1 to 1.18.2
Changelog

Sourced from mypy's changelog.

Mypy 1.18.2

  • Fix crash on recursive alias (Ivan Levkivskyi, PR 19845)
  • Add additional guidance for stubtest errors when runtime is object.__init__ (Stephen Morton, PR 19733)
  • Fix handling of None values in f-string expressions in mypyc (BobTheBuidler, PR 19846)

Acknowledgements

Thanks to all mypy contributors who contributed to this release:

  • Ali Hamdan
  • Anthony Sottile
  • BobTheBuidler
  • Brian Schubert
  • Chainfire
  • Charlie Denton
  • Christoph Tyralla
  • CoolCat467
  • Daniel Hnyk
  • Emily
  • Emma Smith
  • Ethan Sarp
  • Ivan Levkivskyi
  • Jahongir Qurbonov
  • Jelle Zijlstra
  • Joren Hammudoglu
  • Jukka Lehtosalo
  • Marc Mueller
  • Omer Hadari
  • Piotr Sawicki
  • PrinceNaroliya
  • Randolf Scholz
  • Robsdedude
  • Saul Shanabrook
  • Shantanu
  • Stanislav Terliakov
  • Stephen Morton
  • wyattscarpenter

I’d also like to thank my employer, Dropbox, for supporting mypy development.

Mypy 1.17

We’ve just uploaded mypy 1.17 to the Python Package Index (PyPI). Mypy is a static type checker for Python. This release includes new features and bug fixes. You can install it as follows:

python3 -m pip install -U mypy

You can read the full documentation for this release on Read the Docs.

... (truncated)

Commits
  • df05f05 remove +dev from version
  • 01a7a12 Update changelog for 1.18.2 (#19873)
  • ca5abf0 Typeshed cherry-pick: Make type of unitest.mock.Any a subclass of Any (#1...
  • 9d794b5 [mypyc] fix: inappropriate Nones in f-strings (#19846)
  • 2c0510c stubtest: additional guidance on errors when runtime is object.init (#19733)
  • 2f3f03c Bump version to 1.18.2+dev for point release
  • 7669841 Fix crash on recursive alias in indirection.py (#19845)
  • 03fbaa9 bump version to 1.18.1 due to wheels failure
  • b44a1fb removed +dev from version
  • 7197a99 Removed Unreleased in the Changelog for Release 1.18 (#19827)
  • Additional commits viewable in compare view

Updates `mypy-zope` from 1.0.13 to 1.0.14
Changelog

Sourced from mypy-zope's changelog.

1.0.14 (2025-12-01)

  • Support mypy-1.19
  • Support mypy-1.18
Commits
  • 38d22f3 Preparing release 1.0.14
  • 76762ec Maintain changelog
  • 4971d98 Merge pull request #134 from Shoobx/dependabot/pip/mypy-gte-1.0.0-and-lt-1.20.0
  • 47af89d Update mypy requirement from <1.19.0,>=1.0.0 to >=1.0.0,<1.20.0
  • 0c596ff Maintain changelog
  • dcaa278 Merge pull request #132 from Shoobx/dependabot/pip/mypy-gte-1.0.0-and-lt-1.19.0
  • 8f7b677 Update mypy requirement from <1.18.0,>=1.0.0 to >=1.0.0,<1.19.0
  • 91b275b Back to development: 1.0.14
  • See full diff in compare view

Updates `phonenumbers` from 9.0.18 to 9.0.19
Commits

**Does not** update `pysaml2` from 7.5.0 to 7.5.4 since this would downgrade pyOpenSSL
Release notes

Sourced from pysaml2's releases.

Version v7.5.4

v7.5.4 (2025-10-07)

  • Minor refactor to handle shelve.open and dbm errors
  • Remove import of deprecated cgi module
  • Replace deprecated datetime.utcnow() by datetime.now(timezone.utc)
  • deps: Remove the importlib_metadata dependency
  • deps: Remove the importlib_resources dependency
  • deps: Update dependency versions and lockfile
  • build: Update pyproject and lockfile to be compatible with PEP 621
  • docs: Correct spelling mistakes
  • docs: Fix interal references/links
  • docs: Clarify units for accepted_time_diff config param
  • docs: Correct documentation for contact_person

Version 7.5.3

7.5.3 (2025-10-04)

  • #973 Fix prepare_for_negotiated_authenticate to avoid double signing redirect requests

Version 7.5.2

7.5.2 (2025-02-10)

  • Include the XSD of the XML Encryption Syntax and Processing Version 1.1 to the schema validator

Version 7.5.1

7.5.1 (2025-02-10)

  • deps: restrict pyOpenSSL up to v24.2.1 until it is replaced
  • deps: update dependncies for the lockfile and examples
Changelog

Sourced from pysaml2's changelog.

v7.5.4 (2025-10-07)

  • Minor refactor to handle shelve.open and dbm errors
  • Remove import of deprecated cgi module
  • Replace deprecated datetime.utcnow() by datetime.now(timezone.utc)
  • deps: Remove the importlib_metadata dependency
  • deps: Remove the importlib_resources dependency
  • deps: Update dependency versions and lockfile
  • build: Update pyproject and lockfile to be compatible with PEP 621
  • docs: Correct spelling mistakes
  • docs: Fix interal references/links
  • docs: Clarify units for accepted_time_diff config param
  • docs: Correct documentation for contact_person

7.5.3 (2025-10-04)

  • #973 Fix prepare_for_negotiated_authenticate to avoid double signing redirect requests

7.5.2 (2025-02-10)

  • Include the XSD of the XML Encryption Syntax and Processing Version 1.1 to the schema validator

7.5.1 (2025-02-10)

  • deps: restrict pyOpenSSL up to v24.2.1 until it is replaced
  • deps: update dependencies for the lockfile and examples
Commits
  • 9cf71f7 Release version 7.5.4
  • c3ec719 Refactor _shelve_compat
  • 1d6ea60 Remove import of deprecated cgi module
  • c45eb9d Replace deprecated datetime.utcnow() by datetime.now(timezone.utc)
  • 178f6d1 Remove unneeded dependencies
  • 1f0a25a remove importlib_metadata import
  • 099f716 remove importlib_resources imports
  • 3fa11ee spelling updates.
  • 4b7887f update link.
  • bc8d3b4 update link.
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Devon Hudson --- poetry.lock | 229 ++++++++++++++++-------- synapse/handlers/room_member.py | 2 +- synapse/storage/databases/main/state.py | 2 +- synapse/util/iterutils.py | 3 +- tests/rest/admin/test_device.py | 7 +- 5 files changed, 164 insertions(+), 79 deletions(-) diff --git a/poetry.lock b/poetry.lock index 271be1456e4..5fe740a0255 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,7 +31,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" +markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" files = [ {file = "authlib-1.6.5-py2.py3-none-any.whl", hash = "sha256:3e0e0507807f842b02175507bdee8957a1d5707fd4afb17c32fb43fee90b6e3a"}, {file = "authlib-1.6.5.tar.gz", hash = "sha256:6aaf9c79b7cc96c900f0b284061691c5d4e61221640a948fe690b556a6d6d10b"}, @@ -481,7 +481,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -506,7 +506,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "elementpath-4.1.5-py3-none-any.whl", hash = "sha256:2ac1a2fb31eb22bbbf817f8cf6752f844513216263f0e3892c8e79782fe4bb55"}, {file = "elementpath-4.1.5.tar.gz", hash = "sha256:c2d6dc524b29ef751ecfc416b0627668119d8812441c555d7471da41d4bacb8d"}, @@ -556,7 +556,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "hiredis-3.3.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:9937d9b69321b393fbace69f55423480f098120bc55a3316e1ca3508c4dbbd6f"}, {file = "hiredis-3.3.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:50351b77f89ba6a22aff430b993653847f36b71d444509036baa0f2d79d1ebf4"}, @@ -879,7 +879,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1017,7 +1017,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1026,6 +1026,92 @@ files = [ [package.dependencies] pyasn1 = ">=0.4.6" +[[package]] +name = "librt" +version = "0.6.3" +description = "Mypyc runtime library" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "librt-0.6.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:45660d26569cc22ed30adf583389d8a0d1b468f8b5e518fcf9bfe2cd298f9dd1"}, + {file = "librt-0.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:54f3b2177fb892d47f8016f1087d21654b44f7fc4cf6571c1c6b3ea531ab0fcf"}, + {file = "librt-0.6.3-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:c5b31bed2c2f2fa1fcb4815b75f931121ae210dc89a3d607fb1725f5907f1437"}, + {file = "librt-0.6.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f8ed5053ef9fb08d34f1fd80ff093ccbd1f67f147633a84cf4a7d9b09c0f089"}, + {file = "librt-0.6.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3f0e4bd9bcb0ee34fa3dbedb05570da50b285f49e52c07a241da967840432513"}, + {file = "librt-0.6.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d8f89c8d20dfa648a3f0a56861946eb00e5b00d6b00eea14bc5532b2fcfa8ef1"}, + {file = "librt-0.6.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:ecc2c526547eacd20cb9fbba19a5268611dbc70c346499656d6cf30fae328977"}, + {file = "librt-0.6.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fbedeb9b48614d662822ee514567d2d49a8012037fc7b4cd63f282642c2f4b7d"}, + {file = "librt-0.6.3-cp310-cp310-win32.whl", hash = "sha256:0765b0fe0927d189ee14b087cd595ae636bef04992e03fe6dfdaa383866c8a46"}, + {file = "librt-0.6.3-cp310-cp310-win_amd64.whl", hash = "sha256:8c659f9fb8a2f16dc4131b803fa0144c1dadcb3ab24bb7914d01a6da58ae2457"}, + {file = "librt-0.6.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:61348cc488b18d1b1ff9f3e5fcd5ac43ed22d3e13e862489d2267c2337285c08"}, + {file = "librt-0.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:64645b757d617ad5f98c08e07620bc488d4bced9ced91c6279cec418f16056fa"}, + {file = "librt-0.6.3-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:26b8026393920320bb9a811b691d73c5981385d537ffc5b6e22e53f7b65d4122"}, + {file = "librt-0.6.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d998b432ed9ffccc49b820e913c8f327a82026349e9c34fa3690116f6b70770f"}, + {file = "librt-0.6.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e18875e17ef69ba7dfa9623f2f95f3eda6f70b536079ee6d5763ecdfe6cc9040"}, + {file = "librt-0.6.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a218f85081fc3f70cddaed694323a1ad7db5ca028c379c214e3a7c11c0850523"}, + {file = "librt-0.6.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1ef42ff4edd369e84433ce9b188a64df0837f4f69e3d34d3b34d4955c599d03f"}, + {file = "librt-0.6.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0e0f2b79993fec23a685b3e8107ba5f8675eeae286675a216da0b09574fa1e47"}, + {file = "librt-0.6.3-cp311-cp311-win32.whl", hash = "sha256:fd98cacf4e0fabcd4005c452cb8a31750258a85cab9a59fb3559e8078da408d7"}, + {file = "librt-0.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:e17b5b42c8045867ca9d1f54af00cc2275198d38de18545edaa7833d7e9e4ac8"}, + {file = "librt-0.6.3-cp311-cp311-win_arm64.whl", hash = "sha256:87597e3d57ec0120a3e1d857a708f80c02c42ea6b00227c728efbc860f067c45"}, + {file = "librt-0.6.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:74418f718083009108dc9a42c21bf2e4802d49638a1249e13677585fcc9ca176"}, + {file = "librt-0.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:514f3f363d1ebc423357d36222c37e5c8e6674b6eae8d7195ac9a64903722057"}, + {file = "librt-0.6.3-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cf1115207a5049d1f4b7b4b72de0e52f228d6c696803d94843907111cbf80610"}, + {file = "librt-0.6.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ad8ba80cdcea04bea7b78fcd4925bfbf408961e9d8397d2ee5d3ec121e20c08c"}, + {file = "librt-0.6.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4018904c83eab49c814e2494b4e22501a93cdb6c9f9425533fe693c3117126f9"}, + {file = "librt-0.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8983c5c06ac9c990eac5eb97a9f03fe41dc7e9d7993df74d9e8682a1056f596c"}, + {file = "librt-0.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d7769c579663a6f8dbf34878969ac71befa42067ce6bf78e6370bf0d1194997c"}, + {file = "librt-0.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d3c9a07eafdc70556f8c220da4a538e715668c0c63cabcc436a026e4e89950bf"}, + {file = "librt-0.6.3-cp312-cp312-win32.whl", hash = "sha256:38320386a48a15033da295df276aea93a92dfa94a862e06893f75ea1d8bbe89d"}, + {file = "librt-0.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:c0ecf4786ad0404b072196b5df774b1bb23c8aacdcacb6c10b4128bc7b00bd01"}, + {file = "librt-0.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:9f2a6623057989ebc469cd9cc8fe436c40117a0147627568d03f84aef7854c55"}, + {file = "librt-0.6.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e716f9012148a81f02f46a04fc4c663420c6fbfeacfac0b5e128cf43b4413d3"}, + {file = "librt-0.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:669ff2495728009a96339c5ad2612569c6d8be4474e68f3f3ac85d7c3261f5f5"}, + {file = "librt-0.6.3-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:349b6873ebccfc24c9efd244e49da9f8a5c10f60f07575e248921aae2123fc42"}, + {file = "librt-0.6.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c74c26736008481c9f6d0adf1aedb5a52aff7361fea98276d1f965c0256ee70"}, + {file = "librt-0.6.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:408a36ddc75e91918cb15b03460bdc8a015885025d67e68c6f78f08c3a88f522"}, + {file = "librt-0.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e61ab234624c9ffca0248a707feffe6fac2343758a36725d8eb8a6efef0f8c30"}, + {file = "librt-0.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:324462fe7e3896d592b967196512491ec60ca6e49c446fe59f40743d08c97917"}, + {file = "librt-0.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:36b2ec8c15030002c7f688b4863e7be42820d7c62d9c6eece3db54a2400f0530"}, + {file = "librt-0.6.3-cp313-cp313-win32.whl", hash = "sha256:25b1b60cb059471c0c0c803e07d0dfdc79e41a0a122f288b819219ed162672a3"}, + {file = "librt-0.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:10a95ad074e2a98c9e4abc7f5b7d40e5ecbfa84c04c6ab8a70fabf59bd429b88"}, + {file = "librt-0.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:17000df14f552e86877d67e4ab7966912224efc9368e998c96a6974a8d609bf9"}, + {file = "librt-0.6.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8e695f25d1a425ad7a272902af8ab8c8d66c1998b177e4b5f5e7b4e215d0c88a"}, + {file = "librt-0.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3e84a4121a7ae360ca4da436548a9c1ca8ca134a5ced76c893cc5944426164bd"}, + {file = "librt-0.6.3-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:05f385a414de3f950886ea0aad8f109650d4b712cf9cc14cc17f5f62a9ab240b"}, + {file = "librt-0.6.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:36a8e337461150b05ca2c7bdedb9e591dfc262c5230422cea398e89d0c746cdc"}, + {file = "librt-0.6.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dcbe48f6a03979384f27086484dc2a14959be1613cb173458bd58f714f2c48f3"}, + {file = "librt-0.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4bca9e4c260233fba37b15c4ec2f78aa99c1a79fbf902d19dd4a763c5c3fb751"}, + {file = "librt-0.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:760c25ed6ac968e24803eb5f7deb17ce026902d39865e83036bacbf5cf242aa8"}, + {file = "librt-0.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4aa4a93a353ccff20df6e34fa855ae8fd788832c88f40a9070e3ddd3356a9f0e"}, + {file = "librt-0.6.3-cp314-cp314-win32.whl", hash = "sha256:cb92741c2b4ea63c09609b064b26f7f5d9032b61ae222558c55832ec3ad0bcaf"}, + {file = "librt-0.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:fdcd095b1b812d756fa5452aca93b962cf620694c0cadb192cec2bb77dcca9a2"}, + {file = "librt-0.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:822ca79e28720a76a935c228d37da6579edef048a17cd98d406a2484d10eda78"}, + {file = "librt-0.6.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:078cd77064d1640cb7b0650871a772956066174d92c8aeda188a489b58495179"}, + {file = "librt-0.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5cc22f7f5c0cc50ed69f4b15b9c51d602aabc4500b433aaa2ddd29e578f452f7"}, + {file = "librt-0.6.3-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:14b345eb7afb61b9fdcdfda6738946bd11b8e0f6be258666b0646af3b9bb5916"}, + {file = "librt-0.6.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d46aa46aa29b067f0b8b84f448fd9719aaf5f4c621cc279164d76a9dc9ab3e8"}, + {file = "librt-0.6.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1b51ba7d9d5d9001494769eca8c0988adce25d0a970c3ba3f2eb9df9d08036fc"}, + {file = "librt-0.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ced0925a18fddcff289ef54386b2fc230c5af3c83b11558571124bfc485b8c07"}, + {file = "librt-0.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:6bac97e51f66da2ca012adddbe9fd656b17f7368d439de30898f24b39512f40f"}, + {file = "librt-0.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b2922a0e8fa97395553c304edc3bd36168d8eeec26b92478e292e5d4445c1ef0"}, + {file = "librt-0.6.3-cp314-cp314t-win32.whl", hash = "sha256:f33462b19503ba68d80dac8a1354402675849259fb3ebf53b67de86421735a3a"}, + {file = "librt-0.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:04f8ce401d4f6380cfc42af0f4e67342bf34c820dae01343f58f472dbac75dcf"}, + {file = "librt-0.6.3-cp314-cp314t-win_arm64.whl", hash = "sha256:afb39550205cc5e5c935762c6bf6a2bb34f7d21a68eadb25e2db7bf3593fecc0"}, + {file = "librt-0.6.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:09262cb2445b6f15d09141af20b95bb7030c6f13b00e876ad8fdd1a9045d6aa5"}, + {file = "librt-0.6.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:57705e8eec76c5b77130d729c0f70190a9773366c555c5457c51eace80afd873"}, + {file = "librt-0.6.3-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3ac2a7835434b31def8ed5355dd9b895bbf41642d61967522646d1d8b9681106"}, + {file = "librt-0.6.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:71f0a5918aebbea1e7db2179a8fe87e8a8732340d9e8b8107401fb407eda446e"}, + {file = "librt-0.6.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aa346e202e6e1ebc01fe1c69509cffe486425884b96cb9ce155c99da1ecbe0e9"}, + {file = "librt-0.6.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:92267f865c7bbd12327a0d394666948b9bf4b51308b52947c0cc453bfa812f5d"}, + {file = "librt-0.6.3-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:86605d5bac340beb030cbc35859325982a79047ebdfba1e553719c7126a2389d"}, + {file = "librt-0.6.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:98e4bbecbef8d2a60ecf731d735602feee5ac0b32117dbbc765e28b054bac912"}, + {file = "librt-0.6.3-cp39-cp39-win32.whl", hash = "sha256:3caa0634c02d5ff0b2ae4a28052e0d8c5f20d497623dc13f629bd4a9e2a6efad"}, + {file = "librt-0.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:b47395091e7e0ece1e6ebac9b98bf0c9084d1e3d3b2739aa566be7e56e3f7bf2"}, + {file = "librt-0.6.3.tar.gz", hash = "sha256:c724a884e642aa2bbad52bb0203ea40406ad742368a5f90da1b220e970384aae"}, +] + [[package]] name = "lxml" version = "6.0.2" @@ -1033,7 +1119,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"url-preview\"" +markers = "extra == \"url-preview\" or extra == \"all\"" files = [ {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e77dd455b9a16bbd2a5036a63ddbd479c19572af81b624e79ef422f929eef388"}, {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d444858b9f07cefff6455b983aea9a67f7462ba1f6cbe4a21e8bf6791bf2153"}, @@ -1319,7 +1405,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "matrix-synapse-ldap3-0.3.0.tar.gz", hash = "sha256:8bb6517173164d4b9cc44f49de411d8cebdb2e705d5dd1ea1f38733c4a009e1d"}, {file = "matrix_synapse_ldap3-0.3.0-py3-none-any.whl", hash = "sha256:8b4d701f8702551e98cc1d8c20dbed532de5613584c08d0df22de376ba99159d"}, @@ -1448,53 +1534,54 @@ docs = ["sphinx (>=8,<9)", "sphinx-autobuild"] [[package]] name = "mypy" -version = "1.17.1" +version = "1.19.0" description = "Optional static typing for Python" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "mypy-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3fbe6d5555bf608c47203baa3e72dbc6ec9965b3d7c318aa9a4ca76f465bd972"}, - {file = "mypy-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:80ef5c058b7bce08c83cac668158cb7edea692e458d21098c7d3bce35a5d43e7"}, - {file = "mypy-1.17.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4a580f8a70c69e4a75587bd925d298434057fe2a428faaf927ffe6e4b9a98df"}, - {file = "mypy-1.17.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd86bb649299f09d987a2eebb4d52d10603224500792e1bee18303bbcc1ce390"}, - {file = "mypy-1.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a76906f26bd8d51ea9504966a9c25419f2e668f012e0bdf3da4ea1526c534d94"}, - {file = "mypy-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:e79311f2d904ccb59787477b7bd5d26f3347789c06fcd7656fa500875290264b"}, - {file = "mypy-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ad37544be07c5d7fba814eb370e006df58fed8ad1ef33ed1649cb1889ba6ff58"}, - {file = "mypy-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:064e2ff508e5464b4bd807a7c1625bc5047c5022b85c70f030680e18f37273a5"}, - {file = "mypy-1.17.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70401bbabd2fa1aa7c43bb358f54037baf0586f41e83b0ae67dd0534fc64edfd"}, - {file = "mypy-1.17.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e92bdc656b7757c438660f775f872a669b8ff374edc4d18277d86b63edba6b8b"}, - {file = "mypy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c1fdf4abb29ed1cb091cf432979e162c208a5ac676ce35010373ff29247bcad5"}, - {file = "mypy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:ff2933428516ab63f961644bc49bc4cbe42bbffb2cd3b71cc7277c07d16b1a8b"}, - {file = "mypy-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:69e83ea6553a3ba79c08c6e15dbd9bfa912ec1e493bf75489ef93beb65209aeb"}, - {file = "mypy-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1b16708a66d38abb1e6b5702f5c2c87e133289da36f6a1d15f6a5221085c6403"}, - {file = "mypy-1.17.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89e972c0035e9e05823907ad5398c5a73b9f47a002b22359b177d40bdaee7056"}, - {file = "mypy-1.17.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03b6d0ed2b188e35ee6d5c36b5580cffd6da23319991c49ab5556c023ccf1341"}, - {file = "mypy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c837b896b37cd103570d776bda106eabb8737aa6dd4f248451aecf53030cdbeb"}, - {file = "mypy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:665afab0963a4b39dff7c1fa563cc8b11ecff7910206db4b2e64dd1ba25aed19"}, - {file = "mypy-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:93378d3203a5c0800c6b6d850ad2f19f7a3cdf1a3701d3416dbf128805c6a6a7"}, - {file = "mypy-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:15d54056f7fe7a826d897789f53dd6377ec2ea8ba6f776dc83c2902b899fee81"}, - {file = "mypy-1.17.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:209a58fed9987eccc20f2ca94afe7257a8f46eb5df1fb69958650973230f91e6"}, - {file = "mypy-1.17.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:099b9a5da47de9e2cb5165e581f158e854d9e19d2e96b6698c0d64de911dd849"}, - {file = "mypy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa6ffadfbe6994d724c5a1bb6123a7d27dd68fc9c059561cd33b664a79578e14"}, - {file = "mypy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:9a2b7d9180aed171f033c9f2fc6c204c1245cf60b0cb61cf2e7acc24eea78e0a"}, - {file = "mypy-1.17.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:15a83369400454c41ed3a118e0cc58bd8123921a602f385cb6d6ea5df050c733"}, - {file = "mypy-1.17.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:55b918670f692fc9fba55c3298d8a3beae295c5cded0a55dccdc5bbead814acd"}, - {file = "mypy-1.17.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:62761474061feef6f720149d7ba876122007ddc64adff5ba6f374fda35a018a0"}, - {file = "mypy-1.17.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c49562d3d908fd49ed0938e5423daed8d407774a479b595b143a3d7f87cdae6a"}, - {file = "mypy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:397fba5d7616a5bc60b45c7ed204717eaddc38f826e3645402c426057ead9a91"}, - {file = "mypy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:9d6b20b97d373f41617bd0708fd46aa656059af57f2ef72aa8c7d6a2b73b74ed"}, - {file = "mypy-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5d1092694f166a7e56c805caaf794e0585cabdbf1df36911c414e4e9abb62ae9"}, - {file = "mypy-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:79d44f9bfb004941ebb0abe8eff6504223a9c1ac51ef967d1263c6572bbebc99"}, - {file = "mypy-1.17.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b01586eed696ec905e61bd2568f48740f7ac4a45b3a468e6423a03d3788a51a8"}, - {file = "mypy-1.17.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43808d9476c36b927fbcd0b0255ce75efe1b68a080154a38ae68a7e62de8f0f8"}, - {file = "mypy-1.17.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:feb8cc32d319edd5859da2cc084493b3e2ce5e49a946377663cc90f6c15fb259"}, - {file = "mypy-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d7598cf74c3e16539d4e2f0b8d8c318e00041553d83d4861f87c7a72e95ac24d"}, - {file = "mypy-1.17.1-py3-none-any.whl", hash = "sha256:a9f52c0351c21fe24c21d8c0eb1f62967b262d6729393397b6f443c3b773c3b9"}, - {file = "mypy-1.17.1.tar.gz", hash = "sha256:25e01ec741ab5bb3eec8ba9cdb0f769230368a22c959c4937360efb89b7e9f01"}, + {file = "mypy-1.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6148ede033982a8c5ca1143de34c71836a09f105068aaa8b7d5edab2b053e6c8"}, + {file = "mypy-1.19.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a9ac09e52bb0f7fb912f5d2a783345c72441a08ef56ce3e17c1752af36340a39"}, + {file = "mypy-1.19.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:11f7254c15ab3f8ed68f8e8f5cbe88757848df793e31c36aaa4d4f9783fd08ab"}, + {file = "mypy-1.19.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318ba74f75899b0e78b847d8c50821e4c9637c79d9a59680fc1259f29338cb3e"}, + {file = "mypy-1.19.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cf7d84f497f78b682edd407f14a7b6e1a2212b433eedb054e2081380b7395aa3"}, + {file = "mypy-1.19.0-cp310-cp310-win_amd64.whl", hash = "sha256:c3385246593ac2b97f155a0e9639be906e73534630f663747c71908dfbf26134"}, + {file = "mypy-1.19.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a31e4c28e8ddb042c84c5e977e28a21195d086aaffaf08b016b78e19c9ef8106"}, + {file = "mypy-1.19.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:34ec1ac66d31644f194b7c163d7f8b8434f1b49719d403a5d26c87fff7e913f7"}, + {file = "mypy-1.19.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb64b0ba5980466a0f3f9990d1c582bcab8db12e29815ecb57f1408d99b4bff7"}, + {file = "mypy-1.19.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:120cffe120cca5c23c03c77f84abc0c14c5d2e03736f6c312480020082f1994b"}, + {file = "mypy-1.19.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7a500ab5c444268a70565e374fc803972bfd1f09545b13418a5174e29883dab7"}, + {file = "mypy-1.19.0-cp311-cp311-win_amd64.whl", hash = "sha256:c14a98bc63fd867530e8ec82f217dae29d0550c86e70debc9667fff1ec83284e"}, + {file = "mypy-1.19.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0fb3115cb8fa7c5f887c8a8d81ccdcb94cff334684980d847e5a62e926910e1d"}, + {file = "mypy-1.19.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3e19e3b897562276bb331074d64c076dbdd3e79213f36eed4e592272dabd760"}, + {file = "mypy-1.19.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b9d491295825182fba01b6ffe2c6fe4e5a49dbf4e2bb4d1217b6ced3b4797bc6"}, + {file = "mypy-1.19.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6016c52ab209919b46169651b362068f632efcd5eb8ef9d1735f6f86da7853b2"}, + {file = "mypy-1.19.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f188dcf16483b3e59f9278c4ed939ec0254aa8a60e8fc100648d9ab5ee95a431"}, + {file = "mypy-1.19.0-cp312-cp312-win_amd64.whl", hash = "sha256:0e3c3d1e1d62e678c339e7ade72746a9e0325de42cd2cccc51616c7b2ed1a018"}, + {file = "mypy-1.19.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7686ed65dbabd24d20066f3115018d2dce030d8fa9db01aa9f0a59b6813e9f9e"}, + {file = "mypy-1.19.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fd4a985b2e32f23bead72e2fb4bbe5d6aceee176be471243bd831d5b2644672d"}, + {file = "mypy-1.19.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fc51a5b864f73a3a182584b1ac75c404396a17eced54341629d8bdcb644a5bba"}, + {file = "mypy-1.19.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:37af5166f9475872034b56c5efdcf65ee25394e9e1d172907b84577120714364"}, + {file = "mypy-1.19.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:510c014b722308c9bd377993bcbf9a07d7e0692e5fa8fc70e639c1eb19fc6bee"}, + {file = "mypy-1.19.0-cp313-cp313-win_amd64.whl", hash = "sha256:cabbee74f29aa9cd3b444ec2f1e4fa5a9d0d746ce7567a6a609e224429781f53"}, + {file = "mypy-1.19.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:f2e36bed3c6d9b5f35d28b63ca4b727cb0228e480826ffc8953d1892ddc8999d"}, + {file = "mypy-1.19.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a18d8abdda14035c5718acb748faec09571432811af129bf0d9e7b2d6699bf18"}, + {file = "mypy-1.19.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f75e60aca3723a23511948539b0d7ed514dda194bc3755eae0bfc7a6b4887aa7"}, + {file = "mypy-1.19.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f44f2ae3c58421ee05fe609160343c25f70e3967f6e32792b5a78006a9d850f"}, + {file = "mypy-1.19.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:63ea6a00e4bd6822adbfc75b02ab3653a17c02c4347f5bb0cf1d5b9df3a05835"}, + {file = "mypy-1.19.0-cp314-cp314-win_amd64.whl", hash = "sha256:3ad925b14a0bb99821ff6f734553294aa6a3440a8cb082fe1f5b84dfb662afb1"}, + {file = "mypy-1.19.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0dde5cb375cb94deff0d4b548b993bec52859d1651e073d63a1386d392a95495"}, + {file = "mypy-1.19.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1cf9c59398db1c68a134b0b5354a09a1e124523f00bacd68e553b8bd16ff3299"}, + {file = "mypy-1.19.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3210d87b30e6af9c8faed61be2642fcbe60ef77cec64fa1ef810a630a4cf671c"}, + {file = "mypy-1.19.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2c1101ab41d01303103ab6ef82cbbfedb81c1a060c868fa7cc013d573d37ab5"}, + {file = "mypy-1.19.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:0ea4fd21bb48f0da49e6d3b37ef6bd7e8228b9fe41bbf4d80d9364d11adbd43c"}, + {file = "mypy-1.19.0-cp39-cp39-win_amd64.whl", hash = "sha256:16f76ff3f3fd8137aadf593cb4607d82634fca675e8211ad75c43d86033ee6c6"}, + {file = "mypy-1.19.0-py3-none-any.whl", hash = "sha256:0c01c99d626380752e527d5ce8e69ffbba2046eb8a060db0329690849cf9b6f9"}, + {file = "mypy-1.19.0.tar.gz", hash = "sha256:f6b874ca77f733222641e5c46e4711648c4037ea13646fd0cdc814c2eaec2528"}, ] [package.dependencies] +librt = ">=0.6.2" mypy_extensions = ">=1.0.0" pathspec = ">=0.9.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} @@ -1521,18 +1608,18 @@ files = [ [[package]] name = "mypy-zope" -version = "1.0.13" +version = "1.0.14" description = "Plugin for mypy to support zope interfaces" optional = false python-versions = "*" groups = ["dev"] files = [ - {file = "mypy_zope-1.0.13-py3-none-any.whl", hash = "sha256:13740c4cbc910cca2c143c6709e1c483c991abeeeb7b629ad6f73d8ac1edad15"}, - {file = "mypy_zope-1.0.13.tar.gz", hash = "sha256:63fb4d035ea874baf280dc69e714dcde4bd2a4a4837a0fd8d90ce91bea510f99"}, + {file = "mypy_zope-1.0.14-py3-none-any.whl", hash = "sha256:8842ade93630421dbec0c9906d6515f6e65c6407ef8b9b2eb7f4f73ae1e8a42a"}, + {file = "mypy_zope-1.0.14.tar.gz", hash = "sha256:42555ad4703f2e50c912de3ebe0c7197619c3f71864817fabc5385ecea0f8449"}, ] [package.dependencies] -mypy = ">=1.0.0,<1.18.0" +mypy = ">=1.0.0,<1.20.0" "zope.interface" = "*" "zope.schema" = "*" @@ -1561,7 +1648,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1610,14 +1697,14 @@ files = [ [[package]] name = "phonenumbers" -version = "9.0.18" +version = "9.0.19" description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers." optional = false python-versions = "*" groups = ["main"] files = [ - {file = "phonenumbers-9.0.18-py2.py3-none-any.whl", hash = "sha256:d3354454ac31c97f8a08121df97a7145b8dca641f734c6f1518a41c2f60c5764"}, - {file = "phonenumbers-9.0.18.tar.gz", hash = "sha256:5537c61ba95b11b992c95e804da6e49193cc06b1224f632ade64631518a48ed1"}, + {file = "phonenumbers-9.0.19-py2.py3-none-any.whl", hash = "sha256:004abdfe2010518c2383f148515664a742e8a5d5540e07c049735c139d7e8b09"}, + {file = "phonenumbers-9.0.19.tar.gz", hash = "sha256:e0674e31554362f4d95383558f7aefde738ef2e7bf96d28a10afd3e87d63a65c"}, ] [[package]] @@ -1751,7 +1838,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"postgres\"" +markers = "extra == \"postgres\" or extra == \"all\"" files = [ {file = "psycopg2-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:103e857f46bb76908768ead4e2d0ba1d1a130e7b8ed77d3ae91e8b33481813e8"}, {file = "psycopg2-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:210daed32e18f35e3140a1ebe059ac29209dd96468f2f7559aa59f75ee82a5cb"}, @@ -1769,7 +1856,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -1785,7 +1872,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2067,7 +2154,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"cache-memory\"" +markers = "extra == \"cache-memory\" or extra == \"all\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2127,7 +2214,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2152,7 +2239,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, @@ -2175,15 +2262,15 @@ files = [ [[package]] name = "pytz" -version = "2022.7.1" +version = "2025.2" description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ - {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, - {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, + {file = "pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00"}, + {file = "pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3"}, ] [[package]] @@ -2584,7 +2671,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"sentry\"" +markers = "extra == \"sentry\" or extra == \"all\"" files = [ {file = "sentry_sdk-2.46.0-py2.py3-none-any.whl", hash = "sha256:4eeeb60198074dff8d066ea153fa6f241fef1668c10900ea53a4200abc8da9b1"}, {file = "sentry_sdk-2.46.0.tar.gz", hash = "sha256:91821a23460725734b7741523021601593f35731808afc0bb2ba46c27b8acd91"}, @@ -2794,7 +2881,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -2810,7 +2897,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, ] @@ -2883,7 +2970,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "tornado-6.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:f81067dad2e4443b015368b24e802d0083fecada4f0a4572fdb72fc06e54a9a6"}, {file = "tornado-6.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9ac1cbe1db860b3cbb251e795c701c41d343f06a96049d6274e7c77559117e41"}, @@ -3017,7 +3104,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3263,7 +3350,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "xmlschema-2.4.0-py3-none-any.whl", hash = "sha256:dc87be0caaa61f42649899189aab2fd8e0d567f2cf548433ba7b79278d231a4a"}, {file = "xmlschema-2.4.0.tar.gz", hash = "sha256:d74cd0c10866ac609e1ef94a5a69b018ad16e39077bc6393408b40c6babee793"}, diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 6f8481de9ae..a8935fded65 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -874,7 +874,7 @@ async def update_membership_locked( if target_id == self._server_notices_mxid: raise SynapseError(HTTPStatus.FORBIDDEN, "Cannot invite this user") - block_invite_result = None + block_invite_result: tuple[Codes, dict] | None = None if ( self._server_notices_mxid is not None diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index a0aea4975ca..cfde107b486 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -983,7 +983,7 @@ def __getitem__(self, key: StateKey) -> str: raise Exception("State map was filtered and doesn't include: %s", key) return super().__getitem__(key) - @overload # type: ignore[override] + @overload def get(self, key: StateKey, default: None = None, /) -> str | None: ... @overload def get(self, key: StateKey, default: str, /) -> str: ... diff --git a/synapse/util/iterutils.py b/synapse/util/iterutils.py index 19789a4666d..cb3b9960731 100644 --- a/synapse/util/iterutils.py +++ b/synapse/util/iterutils.py @@ -22,6 +22,7 @@ import heapq from itertools import islice from typing import ( + Any, Callable, Collection, Generator, @@ -33,7 +34,7 @@ TypeVar, ) -T = TypeVar("T") +T = TypeVar("T", bound=Any) S = TypeVar("S", bound="_SelfSlice") diff --git a/tests/rest/admin/test_device.py b/tests/rest/admin/test_device.py index 4dff59e1806..d85d1694766 100644 --- a/tests/rest/admin/test_device.py +++ b/tests/rest/admin/test_device.py @@ -26,7 +26,7 @@ import synapse.rest.admin from synapse.api.errors import Codes -from synapse.handlers.device import DeviceWriterHandler +from synapse.handlers.device import MAX_DEVICE_DISPLAY_NAME_LEN, DeviceWriterHandler from synapse.rest.client import devices, login from synapse.server import HomeServer from synapse.util.clock import Clock @@ -175,10 +175,7 @@ def test_update_device_too_long_display_name(self) -> None: ) # Request to update a device display name with a new value that is longer than allowed. - update = { - "display_name": "a" - * (synapse.handlers.device.MAX_DEVICE_DISPLAY_NAME_LEN + 1) - } + update = {"display_name": "a" * (MAX_DEVICE_DISPLAY_NAME_LEN + 1)} channel = self.make_request( "PUT", From 09fd2645c21350e859663c60a8090a28683baa62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Dec 2025 23:51:29 +0000 Subject: [PATCH 055/139] Bump urllib3 from 2.5.0 to 2.6.0 (#19282) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.5.0 to 2.6.0.
Release notes

Sourced from urllib3's releases.

2.6.0

🚀 urllib3 is fundraising for HTTP/2 support

urllib3 is raising ~$40,000 USD to release HTTP/2 support and ensure long-term sustainable maintenance of the project after a sharp decline in financial support. If your company or organization uses Python and would benefit from HTTP/2 support in Requests, pip, cloud SDKs, and thousands of other projects please consider contributing financially to ensure HTTP/2 support is developed sustainably and maintained for the long-haul.

Thank you for your support.

Security

  • Fixed a security issue where streaming API could improperly handle highly compressed HTTP content ("decompression bombs") leading to excessive resource consumption even when a small amount of data was requested. Reading small chunks of compressed data is safer and much more efficient now. (CVE-2025-66471 reported by @​Cycloctane, 8.9 High, GHSA-2xpw-w6gg-jr37)
  • Fixed a security issue where an attacker could compose an HTTP response with virtually unlimited links in the Content-Encoding header, potentially leading to a denial of service (DoS) attack by exhausting system resources during decoding. The number of allowed chained encodings is now limited to 5. (CVE-2025-66418 reported by @​illia-v, 8.9 High, GHSA-gm62-xv2j-4w53)

[!IMPORTANT]

  • If urllib3 is not installed with the optional urllib3[brotli] extra, but your environment contains a Brotli/brotlicffi/brotlipy package anyway, make sure to upgrade it to at least Brotli 1.2.0 or brotlicffi 1.2.0.0 to benefit from the security fixes and avoid warnings. Prefer using urllib3[brotli] to install a compatible Brotli package automatically.
  • If you use custom decompressors, please make sure to update them to respect the changed API of urllib3.response.ContentDecoder.

Features

  • Enabled retrieval, deletion, and membership testing in HTTPHeaderDict using bytes keys. (#3653)
  • Added host and port information to string representations of HTTPConnection. (#3666)
  • Added support for Python 3.14 free-threading builds explicitly. (#3696)

Removals

  • Removed the HTTPResponse.getheaders() method in favor of HTTPResponse.headers. Removed the HTTPResponse.getheader(name, default) method in favor of HTTPResponse.headers.get(name, default). (#3622)

Bugfixes

  • Fixed redirect handling in urllib3.PoolManager when an integer is passed for the retries parameter. (#3649)
  • Fixed HTTPConnectionPool when used in Emscripten with no explicit port. (#3664)
  • Fixed handling of SSLKEYLOGFILE with expandable variables. (#3700)

Misc

  • Changed the zstd extra to install backports.zstd instead of zstandard on Python 3.13 and before. (#3693)
  • Improved the performance of content decoding by optimizing BytesQueueBuffer class. (#3710)
  • Allowed building the urllib3 package with newer setuptools-scm v9.x. (#3652)
  • Ensured successful urllib3 builds by setting Hatchling requirement to ≥ 1.27.0. (#3638)
Changelog

Sourced from urllib3's changelog.

2.6.0 (2025-12-05)

Security

  • Fixed a security issue where streaming API could improperly handle highly compressed HTTP content ("decompression bombs") leading to excessive resource consumption even when a small amount of data was requested. Reading small chunks of compressed data is safer and much more efficient now. (GHSA-2xpw-w6gg-jr37 <https://github.com/urllib3/urllib3/security/advisories/GHSA-2xpw-w6gg-jr37>__)
  • Fixed a security issue where an attacker could compose an HTTP response with virtually unlimited links in the Content-Encoding header, potentially leading to a denial of service (DoS) attack by exhausting system resources during decoding. The number of allowed chained encodings is now limited to 5. (GHSA-gm62-xv2j-4w53 <https://github.com/urllib3/urllib3/security/advisories/GHSA-gm62-xv2j-4w53>__)

.. caution::

  • If urllib3 is not installed with the optional urllib3[brotli] extra, but your environment contains a Brotli/brotlicffi/brotlipy package anyway, make sure to upgrade it to at least Brotli 1.2.0 or brotlicffi 1.2.0.0 to benefit from the security fixes and avoid warnings. Prefer using urllib3[brotli] to install a compatible Brotli package automatically.

  • If you use custom decompressors, please make sure to update them to respect the changed API of urllib3.response.ContentDecoder.

Features

  • Enabled retrieval, deletion, and membership testing in HTTPHeaderDict using bytes keys. ([#3653](https://github.com/urllib3/urllib3/issues/3653) <https://github.com/urllib3/urllib3/issues/3653>__)
  • Added host and port information to string representations of HTTPConnection. ([#3666](https://github.com/urllib3/urllib3/issues/3666) <https://github.com/urllib3/urllib3/issues/3666>__)
  • Added support for Python 3.14 free-threading builds explicitly. ([#3696](https://github.com/urllib3/urllib3/issues/3696) <https://github.com/urllib3/urllib3/issues/3696>__)

Removals

  • Removed the HTTPResponse.getheaders() method in favor of HTTPResponse.headers. Removed the HTTPResponse.getheader(name, default) method in favor of HTTPResponse.headers.get(name, default). ([#3622](https://github.com/urllib3/urllib3/issues/3622) <https://github.com/urllib3/urllib3/issues/3622>__)

Bugfixes

  • Fixed redirect handling in urllib3.PoolManager when an integer is passed for the retries parameter. ([#3649](https://github.com/urllib3/urllib3/issues/3649) <https://github.com/urllib3/urllib3/issues/3649>__)
  • Fixed HTTPConnectionPool when used in Emscripten with no explicit port. ([#3664](https://github.com/urllib3/urllib3/issues/3664) <https://github.com/urllib3/urllib3/issues/3664>__)
  • Fixed handling of SSLKEYLOGFILE with expandable variables. ([#3700](https://github.com/urllib3/urllib3/issues/3700) <https://github.com/urllib3/urllib3/issues/3700>__)

... (truncated)

Commits
  • 720f484 Release 2.6.0
  • 24d7b67 Merge commit from fork
  • c19571d Merge commit from fork
  • 816fcf0 Bump actions/setup-python from 6.0.0 to 6.1.0 (#3725)
  • 18af0a1 Improve speed of BytesQueueBuffer.get() by using memoryview (#3711)
  • 1f6abac Bump versions of pre-commit hooks (#3716)
  • 1c8fbf7 Bump actions/checkout from 5.0.0 to 6.0.0 (#3722)
  • 7784b9e Add Python 3.15 to CI (#3717)
  • 0241c9e Updated docs to reflect change in optional zstd dependency from zstandard t...
  • 7afcabb Expand environment variable of SSLKEYLOGFILE (#3705)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=urllib3&package-manager=pip&previous-version=2.5.0&new-version=2.6.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/element-hq/synapse/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 54 ++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5fe740a0255..14b1f085b8a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,7 +31,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" files = [ {file = "authlib-1.6.5-py2.py3-none-any.whl", hash = "sha256:3e0e0507807f842b02175507bdee8957a1d5707fd4afb17c32fb43fee90b6e3a"}, {file = "authlib-1.6.5.tar.gz", hash = "sha256:6aaf9c79b7cc96c900f0b284061691c5d4e61221640a948fe690b556a6d6d10b"}, @@ -481,7 +481,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -506,7 +506,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "elementpath-4.1.5-py3-none-any.whl", hash = "sha256:2ac1a2fb31eb22bbbf817f8cf6752f844513216263f0e3892c8e79782fe4bb55"}, {file = "elementpath-4.1.5.tar.gz", hash = "sha256:c2d6dc524b29ef751ecfc416b0627668119d8812441c555d7471da41d4bacb8d"}, @@ -556,7 +556,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "hiredis-3.3.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:9937d9b69321b393fbace69f55423480f098120bc55a3316e1ca3508c4dbbd6f"}, {file = "hiredis-3.3.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:50351b77f89ba6a22aff430b993653847f36b71d444509036baa0f2d79d1ebf4"}, @@ -879,7 +879,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1017,7 +1017,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1119,7 +1119,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"url-preview\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"url-preview\"" files = [ {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e77dd455b9a16bbd2a5036a63ddbd479c19572af81b624e79ef422f929eef388"}, {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d444858b9f07cefff6455b983aea9a67f7462ba1f6cbe4a21e8bf6791bf2153"}, @@ -1405,7 +1405,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "matrix-synapse-ldap3-0.3.0.tar.gz", hash = "sha256:8bb6517173164d4b9cc44f49de411d8cebdb2e705d5dd1ea1f38733c4a009e1d"}, {file = "matrix_synapse_ldap3-0.3.0-py3-none-any.whl", hash = "sha256:8b4d701f8702551e98cc1d8c20dbed532de5613584c08d0df22de376ba99159d"}, @@ -1648,7 +1648,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1838,7 +1838,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"postgres\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"postgres\"" files = [ {file = "psycopg2-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:103e857f46bb76908768ead4e2d0ba1d1a130e7b8ed77d3ae91e8b33481813e8"}, {file = "psycopg2-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:210daed32e18f35e3140a1ebe059ac29209dd96468f2f7559aa59f75ee82a5cb"}, @@ -1856,7 +1856,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -1872,7 +1872,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2154,7 +2154,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"cache-memory\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"cache-memory\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2214,7 +2214,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2239,7 +2239,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, @@ -2267,7 +2267,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00"}, {file = "pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3"}, @@ -2671,7 +2671,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"sentry\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"sentry\"" files = [ {file = "sentry_sdk-2.46.0-py2.py3-none-any.whl", hash = "sha256:4eeeb60198074dff8d066ea153fa6f241fef1668c10900ea53a4200abc8da9b1"}, {file = "sentry_sdk-2.46.0.tar.gz", hash = "sha256:91821a23460725734b7741523021601593f35731808afc0bb2ba46c27b8acd91"}, @@ -2881,7 +2881,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -2897,7 +2897,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, ] @@ -2970,7 +2970,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "tornado-6.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:f81067dad2e4443b015368b24e802d0083fecada4f0a4572fdb72fc06e54a9a6"}, {file = "tornado-6.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9ac1cbe1db860b3cbb251e795c701c41d343f06a96049d6274e7c77559117e41"}, @@ -3104,7 +3104,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3315,21 +3315,21 @@ files = [ [[package]] name = "urllib3" -version = "2.5.0" +version = "2.6.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, - {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, + {file = "urllib3-2.6.0-py3-none-any.whl", hash = "sha256:c90f7a39f716c572c4e3e58509581ebd83f9b59cced005b7db7ad2d22b0db99f"}, + {file = "urllib3-2.6.0.tar.gz", hash = "sha256:cb9bcef5a4b345d5da5d145dc3e30834f58e8018828cbc724d30b4cb7d4d49f1"}, ] [package.extras] -brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] +brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] +zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "webencodings" @@ -3350,7 +3350,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "xmlschema-2.4.0-py3-none-any.whl", hash = "sha256:dc87be0caaa61f42649899189aab2fd8e0d567f2cf548433ba7b79278d231a4a"}, {file = "xmlschema-2.4.0.tar.gz", hash = "sha256:d74cd0c10866ac609e1ef94a5a69b018ad16e39077bc6393408b40c6babee793"}, From 8b0083cad97ca27e1bc6ce2753a75abfae8166ab Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Mon, 8 Dec 2025 21:39:18 +0000 Subject: [PATCH 056/139] Respond with useful error codes when `Content-Length` header/s are invalid (#19212) Related to https://github.com/element-hq/synapse/issues/17035, when Synapse receives a request that is larger than the maximum size allowed, it aborts the connection without ever sending back a HTTP response. I dug into our usage of twisted and how best to try and report such an error and this is what I came up with. It would be ideal to be able to report the status from within `handleContentChunk` but that is called too early on in the twisted http handling code, before things have been setup enough to be able to properly write a response. I tested this change out locally (both with C-S and S-S apis) and they do receive a 413 response now in addition to the connection being closed. Hopefully this will aid in being able to quickly detect when https://github.com/element-hq/synapse/issues/17035 is occurring as the current situation makes it very hard to narrow things down to that specific issue without making a lot of assumptions. This PR also responds with more meaningful error codes now in the case of: - multiple `Content-Length` headers - invalid `Content-Length` header value - request content size being larger than the `Content-Length` value ### Pull Request Checklist * [X] Pull request is based on the develop branch * [X] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [X] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Eric Eastwood --- changelog.d/19212.misc | 1 + synapse/api/constants.py | 13 +++ synapse/app/_base.py | 15 +-- synapse/http/site.py | 158 ++++++++++++++++++++++++++++---- tests/http/test_site.py | 102 +++++++++++++++++++++ tests/rest/client/test_login.py | 6 -- tests/rest/client/test_media.py | 4 - tests/rest/client/utils.py | 2 - tests/server.py | 25 +++-- tests/test_server.py | 60 ++++++++++++ 10 files changed, 336 insertions(+), 50 deletions(-) create mode 100644 changelog.d/19212.misc diff --git a/changelog.d/19212.misc b/changelog.d/19212.misc new file mode 100644 index 00000000000..83158ce2d9c --- /dev/null +++ b/changelog.d/19212.misc @@ -0,0 +1 @@ +Respond with useful error codes with `Content-Length` header/s are invalid. diff --git a/synapse/api/constants.py b/synapse/api/constants.py index d41e44b1541..9b6a68e929c 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -29,6 +29,19 @@ # the max size of a (canonical-json-encoded) event MAX_PDU_SIZE = 65536 +# The maximum allowed size of an HTTP request. +# Other than media uploads, the biggest request we expect to see is a fully-loaded +# /federation/v1/send request. +# +# The main thing in such a request is up to 50 PDUs, and up to 100 EDUs. PDUs are +# limited to 65536 bytes (possibly slightly more if the sender didn't use canonical +# json encoding); there is no specced limit to EDUs (see +# https://github.com/matrix-org/matrix-doc/issues/3121). +# +# in short, we somewhat arbitrarily limit requests to 200 * 64K (about 12.5M) +# +MAX_REQUEST_SIZE = 200 * MAX_PDU_SIZE + # Max/min size of ints in canonical JSON CANONICALJSON_MAX_INT = (2**53) - 1 CANONICALJSON_MIN_INT = -CANONICALJSON_MAX_INT diff --git a/synapse/app/_base.py b/synapse/app/_base.py index d1ed1201e5a..98d051bf04d 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -59,7 +59,7 @@ from twisted.web.resource import Resource import synapse.util.caches -from synapse.api.constants import MAX_PDU_SIZE +from synapse.api.constants import MAX_REQUEST_SIZE from synapse.app import check_bind_error from synapse.config import ConfigError from synapse.config._base import format_config_error @@ -895,17 +895,8 @@ def sdnotify(state: bytes) -> None: def max_request_body_size(config: HomeServerConfig) -> int: """Get a suitable maximum size for incoming HTTP requests""" - # Other than media uploads, the biggest request we expect to see is a fully-loaded - # /federation/v1/send request. - # - # The main thing in such a request is up to 50 PDUs, and up to 100 EDUs. PDUs are - # limited to 65536 bytes (possibly slightly more if the sender didn't use canonical - # json encoding); there is no specced limit to EDUs (see - # https://github.com/matrix-org/matrix-doc/issues/3121). - # - # in short, we somewhat arbitrarily limit requests to 200 * 64K (about 12.5M) - # - max_request_size = 200 * MAX_PDU_SIZE + # Baseline default for any request that isn't configured in the homeserver config + max_request_size = MAX_REQUEST_SIZE # if we have a media repo enabled, we may need to allow larger uploads than that if config.media.can_load_media_repo: diff --git a/synapse/http/site.py b/synapse/http/site.py index a1b0b8d9c2f..6ced5b98b3c 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -19,6 +19,7 @@ # # import contextlib +import json import logging import time from http import HTTPStatus @@ -36,6 +37,7 @@ from twisted.web.resource import IResource, Resource from twisted.web.server import Request +from synapse.api.errors import Codes, SynapseError from synapse.config.server import ListenerConfig from synapse.http import get_request_user_agent, redact_uri from synapse.http.proxy import ProxySite @@ -59,6 +61,10 @@ _next_request_seq = 0 +class ContentLengthError(SynapseError): + """Raised when content-length validation fails.""" + + class SynapseRequest(Request): """Class which encapsulates an HTTP request to synapse. @@ -144,36 +150,150 @@ def __repr__(self) -> str: self.synapse_site.site_tag, ) + def _respond_with_error(self, synapse_error: SynapseError) -> None: + """Send an error response and close the connection.""" + self.setResponseCode(synapse_error.code) + error_response_bytes = json.dumps(synapse_error.error_dict(None)).encode() + + self.responseHeaders.setRawHeaders(b"Content-Type", [b"application/json"]) + self.responseHeaders.setRawHeaders( + b"Content-Length", [f"{len(error_response_bytes)}"] + ) + self.write(error_response_bytes) + self.loseConnection() + + def _get_content_length_from_headers(self) -> int | None: + """Attempts to obtain the `Content-Length` value from the request's headers. + + Returns: + Content length as `int` if present. Otherwise `None`. + + Raises: + ContentLengthError: if multiple `Content-Length` headers are present or the + value is not an `int`. + """ + content_length_headers = self.requestHeaders.getRawHeaders(b"Content-Length") + if content_length_headers is None: + return None + + # If there are multiple `Content-Length` headers return an error. + # We don't want to even try to pick the right one if there are multiple + # as we could run into problems similar to request smuggling vulnerabilities + # which rely on the mismatch of how different systems interpret information. + if len(content_length_headers) != 1: + raise ContentLengthError( + HTTPStatus.BAD_REQUEST, + "Multiple Content-Length headers received", + Codes.UNKNOWN, + ) + + try: + return int(content_length_headers[0]) + except (ValueError, TypeError): + raise ContentLengthError( + HTTPStatus.BAD_REQUEST, + "Content-Length header value is not a valid integer", + Codes.UNKNOWN, + ) + + def _validate_content_length(self) -> None: + """Validate Content-Length header and actual content size. + + Raises: + ContentLengthError: If validation fails. + """ + # we should have a `content` by now. + assert self.content, "_validate_content_length() called before gotLength()" + content_length = self._get_content_length_from_headers() + + if content_length is None: + return + + actual_content_length = self.content.tell() + + if content_length > self._max_request_body_size: + logger.info( + "Rejecting request from %s because Content-Length %d exceeds maximum size %d: %s %s", + self.client, + content_length, + self._max_request_body_size, + self.get_method(), + self.get_redacted_uri(), + ) + raise ContentLengthError( + HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + f"Request content is too large (>{self._max_request_body_size})", + Codes.TOO_LARGE, + ) + + if content_length != actual_content_length: + comparison = ( + "smaller" if content_length < actual_content_length else "larger" + ) + logger.info( + "Rejecting request from %s because Content-Length %d is %s than the request content size %d: %s %s", + self.client, + content_length, + comparison, + actual_content_length, + self.get_method(), + self.get_redacted_uri(), + ) + raise ContentLengthError( + HTTPStatus.BAD_REQUEST, + f"Rejecting request as the Content-Length header value {content_length} " + f"is {comparison} than the actual request content size {actual_content_length}", + Codes.UNKNOWN, + ) + # Twisted machinery: this method is called by the Channel once the full request has # been received, to dispatch the request to a resource. - # - # We're patching Twisted to bail/abort early when we see someone trying to upload - # `multipart/form-data` so we can avoid Twisted parsing the entire request body into - # in-memory (specific problem of this specific `Content-Type`). This protects us - # from an attacker uploading something bigger than the available RAM and crashing - # the server with a `MemoryError`, or carefully block just enough resources to cause - # all other requests to fail. - # - # FIXME: This can be removed once we Twisted releases a fix and we update to a - # version that is patched def requestReceived(self, command: bytes, path: bytes, version: bytes) -> None: + # In the case of a Content-Length header being present, and it's value being too + # large, throw a proper error to make debugging issues due to overly large requests much + # easier. Currently we handle such cases in `handleContentChunk` and abort the + # connection without providing a proper HTTP response. + # + # Attempting to write an HTTP response from within `handleContentChunk` does not + # work, so the code here has been added to at least provide a response in the + # case of the Content-Length header being present. + self.method, self.uri = command, path + self.clientproto = version + + try: + self._validate_content_length() + except ContentLengthError as e: + self._respond_with_error(e) + return + + # We're patching Twisted to bail/abort early when we see someone trying to upload + # `multipart/form-data` so we can avoid Twisted parsing the entire request body into + # in-memory (specific problem of this specific `Content-Type`). This protects us + # from an attacker uploading something bigger than the available RAM and crashing + # the server with a `MemoryError`, or carefully block just enough resources to cause + # all other requests to fail. + # + # FIXME: This can be removed once Twisted releases a fix and we update to a + # version that is patched + # See: https://github.com/element-hq/synapse/security/advisories/GHSA-rfq8-j7rh-8hf2 if command == b"POST": ctype = self.requestHeaders.getRawHeaders(b"content-type") if ctype and b"multipart/form-data" in ctype[0]: - self.method, self.uri = command, path - self.clientproto = version + logger.warning( + "Aborting connection from %s because `content-type: multipart/form-data` is unsupported: %s %s", + self.client, + self.get_method(), + self.get_redacted_uri(), + ) + self.code = HTTPStatus.UNSUPPORTED_MEDIA_TYPE.value self.code_message = bytes( HTTPStatus.UNSUPPORTED_MEDIA_TYPE.phrase, "ascii" ) - self.responseHeaders.setRawHeaders(b"content-length", [b"0"]) - logger.warning( - "Aborting connection from %s because `content-type: multipart/form-data` is unsupported: %s %s", - self.client, - command, - path, - ) + # FIXME: Return a better error response here similar to the + # `error_response_json` returned in other code paths here. + self.responseHeaders.setRawHeaders(b"Content-Length", [b"0"]) self.write(b"") self.loseConnection() return diff --git a/tests/http/test_site.py b/tests/http/test_site.py index 9e6d929c9ef..654ec3190be 100644 --- a/tests/http/test_site.py +++ b/tests/http/test_site.py @@ -22,6 +22,7 @@ from twisted.internet.address import IPv6Address from twisted.internet.testing import MemoryReactor, StringTransport +from synapse.app._base import max_request_body_size from synapse.app.homeserver import SynapseHomeServer from synapse.server import HomeServer from synapse.util.clock import Clock @@ -143,3 +144,104 @@ def test_content_type_multipart(self) -> None: # we should get a 415 self.assertRegex(transport.value().decode(), r"^HTTP/1\.1 415 ") + + def test_content_length_too_large(self) -> None: + """HTTP requests with Content-Length exceeding max size should be rejected with 413""" + self.hs.start_listening() + + # find the HTTP server which is configured to listen on port 0 + (port, factory, _backlog, interface) = self.reactor.tcpServers[0] + self.assertEqual(interface, "::") + self.assertEqual(port, 0) + + # complete the connection and wire it up to a fake transport + client_address = IPv6Address("TCP", "::1", 2345) + protocol = factory.buildProtocol(client_address) + transport = StringTransport() + protocol.makeConnection(transport) + + # Send a request with Content-Length header that exceeds the limit. + # Default max is 50MB (from media max_upload_size), so send something larger. + oversized_length = 1 + max_request_body_size(self.hs.config) + protocol.dataReceived( + b"POST / HTTP/1.1\r\n" + b"Connection: close\r\n" + b"Content-Length: " + str(oversized_length).encode() + b"\r\n" + b"\r\n" + b"" + b"x" * oversized_length + b"\r\n" + b"\r\n" + ) + + # Advance the reactor to process the request + while not transport.disconnecting: + self.reactor.advance(1) + + # We should get a 413 Content Too Large + response = transport.value().decode() + self.assertRegex(response, r"^HTTP/1\.1 413 ") + self.assertSubstring("M_TOO_LARGE", response) + + def test_too_many_content_length_headers(self) -> None: + """HTTP requests with multiple Content-Length headers should be rejected with 400""" + self.hs.start_listening() + + # find the HTTP server which is configured to listen on port 0 + (port, factory, _backlog, interface) = self.reactor.tcpServers[0] + self.assertEqual(interface, "::") + self.assertEqual(port, 0) + + # complete the connection and wire it up to a fake transport + client_address = IPv6Address("TCP", "::1", 2345) + protocol = factory.buildProtocol(client_address) + transport = StringTransport() + protocol.makeConnection(transport) + + protocol.dataReceived( + b"POST / HTTP/1.1\r\n" + b"Connection: close\r\n" + b"Content-Length: " + str(5).encode() + b"\r\n" + b"Content-Length: " + str(5).encode() + b"\r\n" + b"\r\n" + b"" + b"xxxxx" + b"\r\n" + b"\r\n" + ) + + # Advance the reactor to process the request + while not transport.disconnecting: + self.reactor.advance(1) + + # We should get a 400 + response = transport.value().decode() + self.assertRegex(response, r"^HTTP/1\.1 400 ") + + def test_invalid_content_length_headers(self) -> None: + """HTTP requests with invalid Content-Length header should be rejected with 400""" + self.hs.start_listening() + + # find the HTTP server which is configured to listen on port 0 + (port, factory, _backlog, interface) = self.reactor.tcpServers[0] + self.assertEqual(interface, "::") + self.assertEqual(port, 0) + + # complete the connection and wire it up to a fake transport + client_address = IPv6Address("TCP", "::1", 2345) + protocol = factory.buildProtocol(client_address) + transport = StringTransport() + protocol.makeConnection(transport) + + protocol.dataReceived( + b"POST / HTTP/1.1\r\n" + b"Connection: close\r\n" + b"Content-Length: eight\r\n" + b"\r\n" + b"" + b"xxxxx" + b"\r\n" + b"\r\n" + ) + + # Advance the reactor to process the request + while not transport.disconnecting: + self.reactor.advance(1) + + # We should get a 400 + response = transport.value().decode() + self.assertRegex(response, r"^HTTP/1\.1 400 ") diff --git a/tests/rest/client/test_login.py b/tests/rest/client/test_login.py index d599351df78..d83604a6961 100644 --- a/tests/rest/client/test_login.py +++ b/tests/rest/client/test_login.py @@ -1728,9 +1728,6 @@ def test_username_picker_use_displayname_avatar_and_email(self) -> None: content_is_form=True, custom_headers=[ ("Cookie", "username_mapping_session=" + session_id), - # old versions of twisted don't do form-parsing without a valid - # content-length header. - ("Content-Length", str(len(content))), ], ) self.assertEqual(chan.code, 302, chan.result) @@ -1818,9 +1815,6 @@ def test_username_picker_dont_use_displayname_avatar_or_email(self) -> None: content_is_form=True, custom_headers=[ ("Cookie", "username_mapping_session=" + session_id), - # old versions of twisted don't do form-parsing without a valid - # content-length header. - ("Content-Length", str(len(content))), ], ) self.assertEqual(chan.code, 302, chan.result) diff --git a/tests/rest/client/test_media.py b/tests/rest/client/test_media.py index 33172f930ed..ec81b1413c2 100644 --- a/tests/rest/client/test_media.py +++ b/tests/rest/client/test_media.py @@ -2590,7 +2590,6 @@ def test_authenticated_media(self) -> None: self.tok, shorthand=False, content_type=b"image/png", - custom_headers=[("Content-Length", str(67))], ) self.assertEqual(channel.code, 200) res = channel.json_body.get("content_uri") @@ -2750,7 +2749,6 @@ def test_authenticated_media_etag(self) -> None: self.tok, shorthand=False, content_type=b"image/png", - custom_headers=[("Content-Length", str(67))], ) self.assertEqual(channel.code, 200) res = channel.json_body.get("content_uri") @@ -2909,7 +2907,6 @@ def upload_media(self, size: int) -> FakeChannel: access_token=self.tok, shorthand=False, content_type=b"text/plain", - custom_headers=[("Content-Length", str(size))], ) def test_upload_under_limit(self) -> None: @@ -3074,7 +3071,6 @@ def upload_media(self, size: int, tok: str) -> FakeChannel: access_token=tok, shorthand=False, content_type=b"text/plain", - custom_headers=[("Content-Length", str(size))], ) def test_upload_under_limit(self) -> None: diff --git a/tests/rest/client/utils.py b/tests/rest/client/utils.py index 613c317b8a6..b3808d75bb9 100644 --- a/tests/rest/client/utils.py +++ b/tests/rest/client/utils.py @@ -612,7 +612,6 @@ def upload_media( filename: The filename of the media to be uploaded expect_code: The return code to expect from attempting to upload the media """ - image_length = len(image_data) path = "/_matrix/media/r0/upload?filename=%s" % (filename,) channel = make_request( self.reactor, @@ -621,7 +620,6 @@ def upload_media( path, content=image_data, access_token=tok, - custom_headers=[("Content-Length", str(image_length))], ) assert channel.code == expect_code, "Expected: %d, got: %d, resp: %r" % ( diff --git a/tests/server.py b/tests/server.py index 4fb7dea5ec0..ce31a4162ad 100644 --- a/tests/server.py +++ b/tests/server.py @@ -81,6 +81,7 @@ from twisted.web.resource import IResource from twisted.web.server import Request, Site +from synapse.api.constants import MAX_REQUEST_SIZE from synapse.config.database import DatabaseConnectionConfig from synapse.config.homeserver import HomeServerConfig from synapse.events.auto_accept_invites import InviteAutoAccepter @@ -241,7 +242,6 @@ def writeSequence(self, data: Iterable[bytes]) -> None: def loseConnection(self) -> None: self.unregisterProducer() - self.transport.loseConnection() # Type ignore: mypy doesn't like the fact that producer isn't an IProducer. def registerProducer(self, producer: IProducer, streaming: bool) -> None: @@ -428,18 +428,29 @@ def make_request( channel = FakeChannel(site, reactor, ip=client_ip) - req = request(channel, site, our_server_name="test_server") + req = request( + channel, + site, + our_server_name="test_server", + max_request_body_size=MAX_REQUEST_SIZE, + ) channel.request = req req.content = BytesIO(content) # Twisted expects to be at the end of the content when parsing the request. req.content.seek(0, SEEK_END) - # Old version of Twisted (<20.3.0) have issues with parsing x-www-form-urlencoded - # bodies if the Content-Length header is missing - req.requestHeaders.addRawHeader( - b"Content-Length", str(len(content)).encode("ascii") - ) + # If `Content-Length` was passed in as a custom header, don't automatically add it + # here. + if custom_headers is None or not any( + (k if isinstance(k, bytes) else k.encode("ascii")) == b"Content-Length" + for k, _ in custom_headers + ): + # Old version of Twisted (<20.3.0) have issues with parsing x-www-form-urlencoded + # bodies if the Content-Length header is missing + req.requestHeaders.addRawHeader( + b"Content-Length", str(len(content)).encode("ascii") + ) if access_token: req.requestHeaders.addRawHeader( diff --git a/tests/test_server.py b/tests/test_server.py index ec31b6cc5f6..2a36dd4b304 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -212,6 +212,66 @@ def _callback( self.assertEqual(channel.code, 200) self.assertNotIn("body", channel.result) + def test_content_larger_than_content_length(self) -> None: + """ + HTTP requests with content size exceeding Content-Length should be rejected with 400. + """ + + def _callback( + request: SynapseRequest, **kwargs: object + ) -> tuple[int, JsonDict]: + return 200, {} + + res = JsonResource(self.homeserver) + res.register_paths( + "POST", [re.compile("^/_matrix/foo$")], _callback, "test_servlet" + ) + + channel = make_request( + self.reactor, + FakeSite(res, self.reactor), + b"POST", + b"/_matrix/foo", + {}, + # Set the `Content-Length` value to be smaller than the actual content size + custom_headers=[("Content-Length", "1")], + # The request should disconnect early so don't await the result + await_result=False, + ) + + self.reactor.advance(0.1) + self.assertEqual(channel.code, 400) + + def test_content_smaller_than_content_length(self) -> None: + """ + HTTP requests with content size smaller than Content-Length should be rejected with 400. + """ + + def _callback( + request: SynapseRequest, **kwargs: object + ) -> tuple[int, JsonDict]: + return 200, {} + + res = JsonResource(self.homeserver) + res.register_paths( + "POST", [re.compile("^/_matrix/foo$")], _callback, "test_servlet" + ) + + channel = make_request( + self.reactor, + FakeSite(res, self.reactor), + b"POST", + b"/_matrix/foo", + {}, + # Set the `Content-Length` value to be larger than the actual content size + custom_headers=[("Content-Length", "10")], + # The request should disconnect early so don't await the result + await_result=False, + ) + + self.reactor.advance(0.1) + self.assertEqual(channel.code, 400) + class OptionsResourceTests(unittest.TestCase): def setUp(self) -> None: From 1aeb34a1e14e4161d450b31cb6aeaba3adea9ed1 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 9 Dec 2025 08:32:23 -0700 Subject: [PATCH 057/139] 1.144.0 --- CHANGES.md | 7 +++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 37c1edcba8a..ef0b41bd857 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +# Synapse 1.144.0 (2025-12-09) + +No significant changes since 1.144.0rc1. + + + + # Synapse 1.144.0rc1 (2025-12-02) Admins using the unstable [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666) endpoint (`/_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms`), please check [the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/develop/docs/upgrade.md#upgrading-to-v11440) as this release contains changes that disable that endpoint by default. diff --git a/debian/changelog b/debian/changelog index 901c210d609..15ff7cbd9d1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.144.0) stable; urgency=medium + + * New Synapse release 1.144.0. + + -- Synapse Packaging team Tue, 09 Dec 2025 08:30:40 -0700 + matrix-synapse-py3 (1.144.0~rc1) stable; urgency=medium * New Synapse release 1.144.0rc1. diff --git a/pyproject.toml b/pyproject.toml index ce2ecf63634..38f5990cc70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.144.0rc1" +version = "1.144.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 1bfcc9acf102fb902daa546b810786f707f4aa96 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 9 Dec 2025 08:36:59 -0700 Subject: [PATCH 058/139] Lift important notes to top of changelog --- CHANGES.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index ef0b41bd857..83ab1849e61 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,19 @@ # Synapse 1.144.0 (2025-12-09) +## Deprecation of MacOS Python wheels + +The team has decided to deprecate and stop publishing python wheels for MacOS. +Synapse docker images will continue to work on MacOS, as will building Synapse +from source (though note this requires a Rust compiler). + +## Unstable mutual rooms endpoint is now behind an experimental feature flag + +Admins using the unstable [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666) endpoint (`/_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms`), +please check [the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/develop/docs/upgrade.md#upgrading-to-v11440) as this release contains changes +that disable that endpoint by default. + + + No significant changes since 1.144.0rc1. From ba774e2311022b997687c76d4bbed1743974c1f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Dec 2025 23:01:56 +0000 Subject: [PATCH 059/139] Bump ruff from 0.14.5 to 0.14.6 in the minor-and-patches group across 1 directory (#19296) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps the minor-and-patches group with 1 update in the / directory: [ruff](https://github.com/astral-sh/ruff). Updates `ruff` from 0.14.5 to 0.14.6
Release notes

Sourced from ruff's releases.

0.14.6

Release Notes

Released on 2025-11-21.

Preview features

  • [flake8-bandit] Support new PySNMP API paths (S508, S509) (#21374)

Bug fixes

  • Adjust own-line comment placement between branches (#21185)
  • Avoid syntax error when formatting attribute expressions with outer parentheses, parenthesized value, and trailing comment on value (#20418)
  • Fix panic when formatting comments in unary expressions (#21501)
  • Respect fmt: skip for compound statements on a single line (#20633)
  • [refurb] Fix FURB103 autofix (#21454)
  • [ruff] Fix false positive for complex conversion specifiers in logging-eager-conversion (RUF065) (#21464)

Rule changes

  • [ruff] Avoid false positive on ClassVar reassignment (RUF012) (#21478)

CLI

  • Render hyperlinks for lint errors (#21514)
  • Add a ruff analyze option to skip over imports in TYPE_CHECKING blocks (#21472)

Documentation

  • Limit eglot-format hook to eglot-managed Python buffers (#21459)
  • Mention force-exclude in "Configuration > Python file discovery" (#21500)

Contributors

Install ruff 0.14.6

Install prebuilt binaries via shell script

curl --proto '=https' --tlsv1.2 -LsSf
https://github.com/astral-sh/ruff/releases/download/0.14.6/ruff-installer.sh
| sh
</tr></table>

... (truncated)

Changelog

Sourced from ruff's changelog.

0.14.6

Released on 2025-11-21.

Preview features

  • [flake8-bandit] Support new PySNMP API paths (S508, S509) (#21374)

Bug fixes

  • Adjust own-line comment placement between branches (#21185)
  • Avoid syntax error when formatting attribute expressions with outer parentheses, parenthesized value, and trailing comment on value (#20418)
  • Fix panic when formatting comments in unary expressions (#21501)
  • Respect fmt: skip for compound statements on a single line (#20633)
  • [refurb] Fix FURB103 autofix (#21454)
  • [ruff] Fix false positive for complex conversion specifiers in logging-eager-conversion (RUF065) (#21464)

Rule changes

  • [ruff] Avoid false positive on ClassVar reassignment (RUF012) (#21478)

CLI

  • Render hyperlinks for lint errors (#21514)
  • Add a ruff analyze option to skip over imports in TYPE_CHECKING blocks (#21472)

Documentation

  • Limit eglot-format hook to eglot-managed Python buffers (#21459)
  • Mention force-exclude in "Configuration > Python file discovery" (#21500)

Contributors

Commits
  • 59c6cb5 Bump 0.14.6 (#21558)
  • 54dba15 [ty] Improve debug messages when imports fail (#21555)
  • 1af3185 [ty] Add support for relative import completions
  • 553e568 [ty] Refactor detection of import statements for completions
  • cdef3f5 [ty] Use dedicated collector for completions
  • 6178822 [ty] Attach subdiagnostics to unresolved-import errors for relative imports...
  • 6b7adb0 [ty] support PEP 613 type aliases (#21394)
  • 06941c1 [ty] More low-hanging fruit for inlay hint goto-definition (#21548)
  • eb7c098 [ty] implement TypedDict structural assignment (#21467)
  • 1b28fc1 [ty] Add more random TypeDetails and tests (#21546)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=ruff&package-manager=pip&previous-version=0.14.5&new-version=0.14.6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 42 +++++++++++++++++++++--------------------- pyproject.toml | 2 +- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/poetry.lock b/poetry.lock index 14b1f085b8a..8c9256c8929 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2604,31 +2604,31 @@ files = [ [[package]] name = "ruff" -version = "0.14.5" +version = "0.14.6" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "ruff-0.14.5-py3-none-linux_armv6l.whl", hash = "sha256:f3b8248123b586de44a8018bcc9fefe31d23dda57a34e6f0e1e53bd51fd63594"}, - {file = "ruff-0.14.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:f7a75236570318c7a30edd7f5491945f0169de738d945ca8784500b517163a72"}, - {file = "ruff-0.14.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:6d146132d1ee115f8802356a2dc9a634dbf58184c51bff21f313e8cd1c74899a"}, - {file = "ruff-0.14.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2380596653dcd20b057794d55681571a257a42327da8894b93bbd6111aa801f"}, - {file = "ruff-0.14.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2d1fa985a42b1f075a098fa1ab9d472b712bdb17ad87a8ec86e45e7fa6273e68"}, - {file = "ruff-0.14.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88f0770d42b7fa02bbefddde15d235ca3aa24e2f0137388cc15b2dcbb1f7c7a7"}, - {file = "ruff-0.14.5-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:3676cb02b9061fee7294661071c4709fa21419ea9176087cb77e64410926eb78"}, - {file = "ruff-0.14.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b595bedf6bc9cab647c4a173a61acf4f1ac5f2b545203ba82f30fcb10b0318fb"}, - {file = "ruff-0.14.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f55382725ad0bdb2e8ee2babcbbfb16f124f5a59496a2f6a46f1d9d99d93e6e2"}, - {file = "ruff-0.14.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7497d19dce23976bdaca24345ae131a1d38dcfe1b0850ad8e9e6e4fa321a6e19"}, - {file = "ruff-0.14.5-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:410e781f1122d6be4f446981dd479470af86537fb0b8857f27a6e872f65a38e4"}, - {file = "ruff-0.14.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c01be527ef4c91a6d55e53b337bfe2c0f82af024cc1a33c44792d6844e2331e1"}, - {file = "ruff-0.14.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:f66e9bb762e68d66e48550b59c74314168ebb46199886c5c5aa0b0fbcc81b151"}, - {file = "ruff-0.14.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d93be8f1fa01022337f1f8f3bcaa7ffee2d0b03f00922c45c2207954f351f465"}, - {file = "ruff-0.14.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:c135d4b681f7401fe0e7312017e41aba9b3160861105726b76cfa14bc25aa367"}, - {file = "ruff-0.14.5-py3-none-win32.whl", hash = "sha256:c83642e6fccfb6dea8b785eb9f456800dcd6a63f362238af5fc0c83d027dd08b"}, - {file = "ruff-0.14.5-py3-none-win_amd64.whl", hash = "sha256:9d55d7af7166f143c94eae1db3312f9ea8f95a4defef1979ed516dbb38c27621"}, - {file = "ruff-0.14.5-py3-none-win_arm64.whl", hash = "sha256:4b700459d4649e2594b31f20a9de33bc7c19976d4746d8d0798ad959621d64a4"}, - {file = "ruff-0.14.5.tar.gz", hash = "sha256:8d3b48d7d8aad423d3137af7ab6c8b1e38e4de104800f0d596990f6ada1a9fc1"}, + {file = "ruff-0.14.6-py3-none-linux_armv6l.whl", hash = "sha256:d724ac2f1c240dbd01a2ae98db5d1d9a5e1d9e96eba999d1c48e30062df578a3"}, + {file = "ruff-0.14.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9f7539ea257aa4d07b7ce87aed580e485c40143f2473ff2f2b75aee003186004"}, + {file = "ruff-0.14.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7f6007e55b90a2a7e93083ba48a9f23c3158c433591c33ee2e99a49b889c6332"}, + {file = "ruff-0.14.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a8e7b9d73d8728b68f632aa8e824ef041d068d231d8dbc7808532d3629a6bef"}, + {file = "ruff-0.14.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d50d45d4553a3ebcbd33e7c5e0fe6ca4aafd9a9122492de357205c2c48f00775"}, + {file = "ruff-0.14.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:118548dd121f8a21bfa8ab2c5b80e5b4aed67ead4b7567790962554f38e598ce"}, + {file = "ruff-0.14.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:57256efafbfefcb8748df9d1d766062f62b20150691021f8ab79e2d919f7c11f"}, + {file = "ruff-0.14.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff18134841e5c68f8e5df1999a64429a02d5549036b394fafbe410f886e1989d"}, + {file = "ruff-0.14.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:29c4b7ec1e66a105d5c27bd57fa93203637d66a26d10ca9809dc7fc18ec58440"}, + {file = "ruff-0.14.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:167843a6f78680746d7e226f255d920aeed5e4ad9c03258094a2d49d3028b105"}, + {file = "ruff-0.14.6-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:16a33af621c9c523b1ae006b1b99b159bf5ac7e4b1f20b85b2572455018e0821"}, + {file = "ruff-0.14.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:1432ab6e1ae2dc565a7eea707d3b03a0c234ef401482a6f1621bc1f427c2ff55"}, + {file = "ruff-0.14.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:4c55cfbbe7abb61eb914bfd20683d14cdfb38a6d56c6c66efa55ec6570ee4e71"}, + {file = "ruff-0.14.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:efea3c0f21901a685fff4befda6d61a1bf4cb43de16da87e8226a281d614350b"}, + {file = "ruff-0.14.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:344d97172576d75dc6afc0e9243376dbe1668559c72de1864439c4fc95f78185"}, + {file = "ruff-0.14.6-py3-none-win32.whl", hash = "sha256:00169c0c8b85396516fdd9ce3446c7ca20c2a8f90a77aa945ba6b8f2bfe99e85"}, + {file = "ruff-0.14.6-py3-none-win_amd64.whl", hash = "sha256:390e6480c5e3659f8a4c8d6a0373027820419ac14fa0d2713bd8e6c3e125b8b9"}, + {file = "ruff-0.14.6-py3-none-win_arm64.whl", hash = "sha256:d43c81fbeae52cfa8728d8766bbf46ee4298c888072105815b392da70ca836b2"}, + {file = "ruff-0.14.6.tar.gz", hash = "sha256:6f0c742ca6a7783a736b867a263b9a7a80a45ce9bee391eeda296895f1b4e1cc"}, ] [[package]] @@ -3485,4 +3485,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<4.0.0" -content-hash = "98b9062f48205a3bcc99b43ae665083d360a15d4a208927fa978df9c36fd5315" +content-hash = "960ddae65fde8574f0f36b6988622fc4baf7646823c36699c5cd4773cad8b0ed" diff --git a/pyproject.toml b/pyproject.toml index 38f5990cc70..70d5e3d5730 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -330,7 +330,7 @@ generate-setup-file = true # failing on new releases. Keeping lower bounds loose here means that dependabot # can bump versions without having to update the content-hash in the lockfile. # This helps prevents merge conflicts when running a batch of dependabot updates. -ruff = "0.14.5" +ruff = "0.14.6" # Typechecking lxml-stubs = ">=0.4.0" From 3aaa2e80b2eb43bed618b9f13db30f3d4585dae9 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 10 Dec 2025 14:46:47 +0000 Subject: [PATCH 060/139] Switch the build backend from `poetry-core` to `maturin` (#19234) --- .ci/scripts/auditwheel_wrapper.py | 146 ---------------------- changelog.d/19234.misc | 1 + pyproject.toml | 193 ++++++++++++++++-------------- 3 files changed, 105 insertions(+), 235 deletions(-) delete mode 100755 .ci/scripts/auditwheel_wrapper.py create mode 100644 changelog.d/19234.misc diff --git a/.ci/scripts/auditwheel_wrapper.py b/.ci/scripts/auditwheel_wrapper.py deleted file mode 100755 index 98328212217..00000000000 --- a/.ci/scripts/auditwheel_wrapper.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# - -# Wraps `auditwheel repair` to first check if we're repairing a potentially abi3 -# compatible wheel, if so rename the wheel before repairing it. - -import argparse -import os -import subprocess -from zipfile import ZipFile - -from packaging.tags import Tag -from packaging.utils import parse_wheel_filename -from packaging.version import Version - - -def check_is_abi3_compatible(wheel_file: str) -> None: - """Check the contents of the built wheel for any `.so` files that are *not* - abi3 compatible. - """ - - with ZipFile(wheel_file, "r") as wheel: - for file in wheel.namelist(): - if not file.endswith(".so"): - continue - - if not file.endswith(".abi3.so"): - raise Exception(f"Found non-abi3 lib: {file}") - - -def cpython(wheel_file: str, name: str, version: Version, tag: Tag) -> str: - """Replaces the cpython wheel file with a ABI3 compatible wheel""" - - if tag.abi == "abi3": - # Nothing to do. - return wheel_file - - check_is_abi3_compatible(wheel_file) - - # HACK: it seems that some older versions of pip will consider a wheel marked - # as macosx_11_0 as incompatible with Big Sur. I haven't done the full archaeology - # here; there are some clues in - # https://github.com/pantsbuild/pants/pull/12857 - # https://github.com/pypa/pip/issues/9138 - # https://github.com/pypa/packaging/pull/319 - # Empirically this seems to work, note that macOS 11 and 10.16 are the same, - # both versions are valid for backwards compatibility. - platform = tag.platform.replace("macosx_11_0", "macosx_10_16") - abi3_tag = Tag(tag.interpreter, "abi3", platform) - - dirname = os.path.dirname(wheel_file) - new_wheel_file = os.path.join( - dirname, - f"{name}-{version}-{abi3_tag}.whl", - ) - - os.rename(wheel_file, new_wheel_file) - - print("Renamed wheel to", new_wheel_file) - - return new_wheel_file - - -def main(wheel_file: str, dest_dir: str, archs: str | None) -> None: - """Entry point""" - - # Parse the wheel file name into its parts. Note that `parse_wheel_filename` - # normalizes the package name (i.e. it converts matrix_synapse -> - # matrix-synapse), which is not what we want. - _, version, build, tags = parse_wheel_filename(os.path.basename(wheel_file)) - name = os.path.basename(wheel_file).split("-")[0] - - if len(tags) != 1: - # We expect only a wheel file with only a single tag - raise Exception(f"Unexpectedly found multiple tags: {tags}") - - tag = next(iter(tags)) - - if build: - # We don't use build tags in Synapse - raise Exception(f"Unexpected build tag: {build}") - - # If the wheel is for cpython then convert it into an abi3 wheel. - if tag.interpreter.startswith("cp"): - wheel_file = cpython(wheel_file, name, version, tag) - - # Finally, repair the wheel. - if archs is not None: - # If we are given archs then we are on macos and need to use - # `delocate-listdeps`. - subprocess.run(["delocate-listdeps", wheel_file], check=True) - subprocess.run( - ["delocate-wheel", "--require-archs", archs, "-w", dest_dir, wheel_file], - check=True, - ) - else: - subprocess.run(["auditwheel", "repair", "-w", dest_dir, wheel_file], check=True) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Tag wheel as abi3 and repair it.") - - parser.add_argument( - "--wheel-dir", - "-w", - metavar="WHEEL_DIR", - help="Directory to store delocated wheels", - required=True, - ) - - parser.add_argument( - "--require-archs", - metavar="archs", - default=None, - ) - - parser.add_argument( - "wheel_file", - metavar="WHEEL_FILE", - ) - - args = parser.parse_args() - - wheel_file = args.wheel_file - wheel_dir = args.wheel_dir - archs = args.require_archs - - main(wheel_file, wheel_dir, archs) diff --git a/changelog.d/19234.misc b/changelog.d/19234.misc new file mode 100644 index 00000000000..d79bc0b19fc --- /dev/null +++ b/changelog.d/19234.misc @@ -0,0 +1 @@ +Switch the build backend from `poetry-core` to `maturin`. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 70d5e3d5730..c009ecbf9ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,7 +120,7 @@ redis = ["txredisapi>=1.4.7", "hiredis"] # Required to use experimental `caches.track_memory_usage` config option. cache-memory = ["pympler"] # If this is updated, don't forget to update the equivalent lines in -# tool.poetry.group.dev.dependencies. +# `dependency-groups.dev` below. test = ["parameterized>=0.9.0", "idna>=3.3"] # The duplication here is awful. @@ -177,6 +177,85 @@ synapse_port_db = "synapse._scripts.synapse_port_db:main" synapse_review_recent_signups = "synapse._scripts.review_recent_signups:main" update_synapse_database = "synapse._scripts.update_synapse_database:main" +[tool.poetry] +packages = [{ include = "synapse" }] + +[tool.poetry.build] +# Compile our rust module when using `poetry install`. This is still required +# while using `poetry` as the build frontend. Saves the developer from needing +# to run both: +# +# $ poetry install +# $ maturin develop +script = "build_rust.py" +# Create a `setup.py` file which will call the `build` method in our build +# script. +# +# Our build script currently uses the "old" build method, where we define a +# `build` method and `setup.py` calls it. Poetry developers have mentioned that +# this will eventually be removed: +# https://github.com/matrix-org/synapse/pull/14949#issuecomment-1418001859 +# +# The new build method is defined here: +# https://python-poetry.org/docs/building-extension-modules/#maturin-build-script +# but is still marked as "unstable" at the time of writing. This would also +# bump our minimum `poetry-core` version to 1.5.0. +# +# We can just drop this work-around entirely if migrating away from +# Poetry, thus there's little motivation to update the build script. +generate-setup-file = true + +# Dependencies used for developing Synapse itself. +# +# Hold off on migrating these to `dev-dependencies` (PEP 735) for now until +# Poetry 2.2.0+, pip 25.1+ are more widely available. +[tool.poetry.group.dev.dependencies] +# We pin development dependencies in poetry.lock so that our tests don't start +# failing on new releases. Keeping lower bounds loose here means that dependabot +# can bump versions without having to update the content-hash in the lockfile. +# This helps prevents merge conflicts when running a batch of dependabot updates. +ruff = "0.14.6" + +# Typechecking +lxml-stubs = ">=0.4.0" +mypy = "*" +mypy-zope = "*" +types-bleach = ">=4.1.0" +types-jsonschema = ">=3.2.0" +types-netaddr = ">=0.8.0.6" +types-opentracing = ">=2.4.2" +types-Pillow = ">=8.3.4" +types-psycopg2 = ">=2.9.9" +types-pyOpenSSL = ">=20.0.7" +types-PyYAML = ">=5.4.10" +types-requests = ">=2.26.0" +types-setuptools = ">=57.4.0" + +# Dependencies which are exclusively required by unit test code. This is +# NOT a list of all modules that are necessary to run the unit tests. +# Tests assume that all optional dependencies are installed. +# +# If this is updated, don't forget to update the equivalent lines in +# project.optional-dependencies.test. +parameterized = ">=0.9.0" +idna = ">=3.3" + +# The following are used by the release script +click = ">=8.1.3" +# GitPython was == 3.1.14; bumped to 3.1.20, the first release with type hints. +GitPython = ">=3.1.20" +markdown-it-py = ">=3.0.0" +pygithub = ">=1.59" +# The following are executed as commands by the release script. +twine = "*" +# Towncrier min version comes from https://github.com/matrix-org/synapse/pull/3425. Rationale unclear. +towncrier = ">=18.6.0rc1" + +# Used for checking the Poetry lockfile +tomli = ">=1.2.3" + +# Used for checking the schema delta files +sqlglot = ">=28.0.0" [tool.towncrier] package = "synapse" @@ -291,88 +370,29 @@ line-ending = "auto" [tool.maturin] manifest-path = "rust/Cargo.toml" module-name = "synapse.synapse_rust" - -[tool.poetry] -packages = [ - { include = "synapse" }, -] -include = [ - { path = "AUTHORS.rst", format = "sdist" }, - { path = "book.toml", format = "sdist" }, - { path = "changelog.d", format = "sdist" }, - { path = "CHANGES.md", format = "sdist" }, - { path = "CONTRIBUTING.md", format = "sdist" }, - { path = "demo", format = "sdist" }, - { path = "docs", format = "sdist" }, - { path = "INSTALL.md", format = "sdist" }, - { path = "mypy.ini", format = "sdist" }, - { path = "scripts-dev", format = "sdist" }, - { path = "synmark", format="sdist" }, - { path = "sytest-blacklist", format = "sdist" }, - { path = "tests", format = "sdist" }, - { path = "UPGRADE.rst", format = "sdist" }, - { path = "Cargo.toml", format = "sdist" }, - { path = "Cargo.lock", format = "sdist" }, - { path = "rust/Cargo.toml", format = "sdist" }, - { path = "rust/build.rs", format = "sdist" }, - { path = "rust/src/**", format = "sdist" }, +python-source = "." +sdist-include = [ + "AUTHORS.rst", + "book.toml", + "changelog.d", + "CHANGES.md", + "CONTRIBUTING.md", + "demo", + "docs", + "INSTALL.md", + "mypy.ini", + "scripts-dev", + "synmark", + "sytest-blacklist", + "tests", + "UPGRADE.rst", + "Cargo.toml", + "Cargo.lock", + "rust/Cargo.toml", + "rust/build.rs", + "rust/src/**", ] -exclude = [ - { path = "synapse/*.so", format = "sdist"} -] - -[tool.poetry.build] -script = "build_rust.py" -generate-setup-file = true - -[tool.poetry.group.dev.dependencies] -# We pin development dependencies in poetry.lock so that our tests don't start -# failing on new releases. Keeping lower bounds loose here means that dependabot -# can bump versions without having to update the content-hash in the lockfile. -# This helps prevents merge conflicts when running a batch of dependabot updates. -ruff = "0.14.6" - -# Typechecking -lxml-stubs = ">=0.4.0" -mypy = "*" -mypy-zope = "*" -types-bleach = ">=4.1.0" -types-jsonschema = ">=3.2.0" -types-netaddr = ">=0.8.0.6" -types-opentracing = ">=2.4.2" -types-Pillow = ">=8.3.4" -types-psycopg2 = ">=2.9.9" -types-pyOpenSSL = ">=20.0.7" -types-PyYAML = ">=5.4.10" -types-requests = ">=2.26.0" -types-setuptools = ">=57.4.0" - -# Dependencies which are exclusively required by unit test code. This is -# NOT a list of all modules that are necessary to run the unit tests. -# Tests assume that all optional dependencies are installed. -# -# If this is updated, don't forget to update the equivalent lines in -# project.optional-dependencies.test. -parameterized = ">=0.9.0" -idna = ">=3.3" - -# The following are used by the release script -click = ">=8.1.3" -# GitPython was == 3.1.14; bumped to 3.1.20, the first release with type hints. -GitPython = ">=3.1.20" -markdown-it-py = ">=3.0.0" -pygithub = ">=1.59" -# The following are executed as commands by the release script. -twine = "*" -# Towncrier min version comes from https://github.com/matrix-org/synapse/pull/3425. Rationale unclear. -towncrier = ">=18.6.0rc1" - -# Used for checking the Poetry lockfile -tomli = ">=1.2.3" - -# Used for checking the schema delta files -sqlglot = ">=28.0.0" - +sdist-exclude = ["synapse/*.so"] [build-system] # The upper bounds here are defensive, intended to prevent situations like @@ -381,8 +401,8 @@ sqlglot = ">=28.0.0" # runtime errors caused by build system changes. # We are happy to raise these upper bounds upon request, # provided we check that it's safe to do so (i.e. that CI passes). -requires = ["poetry-core>=2.0.0,<=2.1.3", "setuptools_rust>=1.3,<=1.11.1"] -build-backend = "poetry.core.masonry.api" +requires = ["maturin>=1.0,<2.0"] +build-backend = "maturin" [tool.cibuildwheel] @@ -419,8 +439,3 @@ environment= { PATH = "$PATH:$HOME/.cargo/bin" } before-build = "rm -rf {project}/build" build-frontend = "build" test-command = "python -c 'from synapse.synapse_rust import sum_as_string; print(sum_as_string(1, 2))'" - - -[tool.cibuildwheel.linux] -# Wrap the repair command to correctly rename the built cpython wheels as ABI3. -repair-wheel-command = "./.ci/scripts/auditwheel_wrapper.py -w {dest_dir} {wheel}" From cdf286d405c606ab2f9133d660289097f70f5322 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 11 Dec 2025 17:58:27 +0000 Subject: [PATCH 061/139] Use `uv` to test full set of minimum deps in CI (#19289) Stemming from #19274 this updates the `olddeps` CI to test against not just the minimum version of our explicit dependencies, but also the minimum version of all implicit (transitive) dependencies that are pulled in from the explicit dependencies themselves. ### Pull Request Checklist * [X] Pull request is based on the develop branch * [X] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [X] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- .ci/scripts/prepare_old_deps.sh | 39 ------------ .github/workflows/tests.yml | 14 ++-- changelog.d/19289.misc | 1 + poetry.lock | 109 ++++++++++++++++++++++++-------- pyproject.toml | 58 +++++++++++++++-- 5 files changed, 141 insertions(+), 80 deletions(-) delete mode 100755 .ci/scripts/prepare_old_deps.sh create mode 100644 changelog.d/19289.misc diff --git a/.ci/scripts/prepare_old_deps.sh b/.ci/scripts/prepare_old_deps.sh deleted file mode 100755 index 29d281dc23a..00000000000 --- a/.ci/scripts/prepare_old_deps.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -# this script is run by GitHub Actions in a plain `jammy` container; it -# - installs the minimal system requirements, and poetry; -# - patches the project definition file to refer to old versions only; -# - creates a venv with these old versions using poetry; and finally -# - invokes `trial` to run the tests with old deps. - -set -ex - -# Prevent virtualenv from auto-updating pip to an incompatible version -export VIRTUALENV_NO_DOWNLOAD=1 - -# TODO: in the future, we could use an implementation of -# https://github.com/python-poetry/poetry/issues/3527 -# https://github.com/pypa/pip/issues/8085 -# to select the lowest possible versions, rather than resorting to this sed script. - -# Patch the project definitions in-place: -# - `-E` use extended regex syntax. -# - Don't modify the line that defines required Python versions. -# - Replace all lower and tilde bounds with exact bounds. -# - Replace all caret bounds with exact bounds. -# - Delete all lines referring to psycopg2 - so no testing of postgres support. -# - Use pyopenssl 17.0, which is the oldest version that works with -# a `cryptography` compiled against OpenSSL 1.1. -# - Omit systemd: we're not logging to journal here. - -sed -i -E ' - /^\s*requires-python\s*=/b - s/[~>]=/==/g - s/\^/==/g - /psycopg2/d - s/pyOpenSSL\s*==\s*16\.0\.0"/pyOpenSSL==17.0.0"/ - /systemd/d -' pyproject.toml - -echo "::group::Patched pyproject.toml" -cat pyproject.toml -echo "::endgroup::" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index aff2832b943..cab6bbdefee 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -452,14 +452,12 @@ jobs: python-version: '3.10' - name: Prepare old deps - if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true' - run: .ci/scripts/prepare_old_deps.sh - - # Note: we install using `pip` here, not poetry. `poetry install` ignores the - # build-system section (https://github.com/python-poetry/poetry/issues/6154), but - # we explicitly want to test that you can `pip install` using the oldest version - # of poetry-core and setuptools-rust. - - run: pip install .[all,test] + # Note: we install using `uv` here, not poetry or pip to allow us to test with the + # minimum version of all dependencies, both those explicitly specified and those + # implicitly brought in by the explicit dependencies. + run: | + pip install uv + uv pip install --system --resolution=lowest .[all,test] # We nuke the local copy, as we've installed synapse into the virtualenv # (rather than use an editable install, which we no longer support). If we diff --git a/changelog.d/19289.misc b/changelog.d/19289.misc new file mode 100644 index 00000000000..4ad0dbc4300 --- /dev/null +++ b/changelog.d/19289.misc @@ -0,0 +1 @@ +Use `uv` to test olddeps to ensure all transitive dependencies use minimum versions. diff --git a/poetry.lock b/poetry.lock index 8c9256c8929..4dacae38a47 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,7 +31,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" +markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" files = [ {file = "authlib-1.6.5-py2.py3-none-any.whl", hash = "sha256:3e0e0507807f842b02175507bdee8957a1d5707fd4afb17c32fb43fee90b6e3a"}, {file = "authlib-1.6.5.tar.gz", hash = "sha256:6aaf9c79b7cc96c900f0b284061691c5d4e61221640a948fe690b556a6d6d10b"}, @@ -481,7 +481,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -506,7 +506,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "elementpath-4.1.5-py3-none-any.whl", hash = "sha256:2ac1a2fb31eb22bbbf817f8cf6752f844513216263f0e3892c8e79782fe4bb55"}, {file = "elementpath-4.1.5.tar.gz", hash = "sha256:c2d6dc524b29ef751ecfc416b0627668119d8812441c555d7471da41d4bacb8d"}, @@ -556,7 +556,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "hiredis-3.3.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:9937d9b69321b393fbace69f55423480f098120bc55a3316e1ca3508c4dbbd6f"}, {file = "hiredis-3.3.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:50351b77f89ba6a22aff430b993653847f36b71d444509036baa0f2d79d1ebf4"}, @@ -879,7 +879,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1017,7 +1017,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1119,7 +1119,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"url-preview\"" +markers = "extra == \"url-preview\" or extra == \"all\"" files = [ {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e77dd455b9a16bbd2a5036a63ddbd479c19572af81b624e79ef422f929eef388"}, {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d444858b9f07cefff6455b983aea9a67f7462ba1f6cbe4a21e8bf6791bf2153"}, @@ -1405,7 +1405,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "matrix-synapse-ldap3-0.3.0.tar.gz", hash = "sha256:8bb6517173164d4b9cc44f49de411d8cebdb2e705d5dd1ea1f38733c4a009e1d"}, {file = "matrix_synapse_ldap3-0.3.0-py3-none-any.whl", hash = "sha256:8b4d701f8702551e98cc1d8c20dbed532de5613584c08d0df22de376ba99159d"}, @@ -1648,7 +1648,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1838,7 +1838,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"postgres\"" +markers = "extra == \"postgres\" or extra == \"all\"" files = [ {file = "psycopg2-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:103e857f46bb76908768ead4e2d0ba1d1a130e7b8ed77d3ae91e8b33481813e8"}, {file = "psycopg2-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:210daed32e18f35e3140a1ebe059ac29209dd96468f2f7559aa59f75ee82a5cb"}, @@ -1856,7 +1856,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -1872,7 +1872,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2154,7 +2154,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"cache-memory\"" +markers = "extra == \"cache-memory\" or extra == \"all\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2207,6 +2207,63 @@ typing-extensions = {version = ">=4.9", markers = "python_version < \"3.13\" and docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx_rtd_theme"] test = ["pretend", "pytest (>=3.0.1)", "pytest-rerunfailures"] +[[package]] +name = "pyparsing" +version = "3.2.5" +description = "pyparsing - Classes and methods to define and execute parsing grammars" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pyparsing-3.2.5-py3-none-any.whl", hash = "sha256:e38a4f02064cf41fe6593d328d0512495ad1f3d8a91c4f73fc401b3079a59a5e"}, + {file = "pyparsing-3.2.5.tar.gz", hash = "sha256:2df8d5b7b2802ef88e8d016a2eb9c7aeaa923529cd251ed0fe4608275d4105b6"}, +] + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + [[package]] name = "pysaml2" version = "7.5.0" @@ -2214,7 +2271,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2239,7 +2296,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, @@ -2267,7 +2324,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00"}, {file = "pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3"}, @@ -2671,7 +2728,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"sentry\"" +markers = "extra == \"sentry\" or extra == \"all\"" files = [ {file = "sentry_sdk-2.46.0-py2.py3-none-any.whl", hash = "sha256:4eeeb60198074dff8d066ea153fa6f241fef1668c10900ea53a4200abc8da9b1"}, {file = "sentry_sdk-2.46.0.tar.gz", hash = "sha256:91821a23460725734b7741523021601593f35731808afc0bb2ba46c27b8acd91"}, @@ -2881,7 +2938,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -2897,7 +2954,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, ] @@ -2970,7 +3027,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "tornado-6.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:f81067dad2e4443b015368b24e802d0083fecada4f0a4572fdb72fc06e54a9a6"}, {file = "tornado-6.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9ac1cbe1db860b3cbb251e795c701c41d343f06a96049d6274e7c77559117e41"}, @@ -3104,7 +3161,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3350,7 +3407,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "xmlschema-2.4.0-py3-none-any.whl", hash = "sha256:dc87be0caaa61f42649899189aab2fd8e0d567f2cf548433ba7b79278d231a4a"}, {file = "xmlschema-2.4.0.tar.gz", hash = "sha256:d74cd0c10866ac609e1ef94a5a69b018ad16e39077bc6393408b40c6babee793"}, @@ -3468,15 +3525,15 @@ docs = ["Sphinx", "repoze.sphinx.autointerface"] test = ["zope.i18nmessageid", "zope.testing", "zope.testrunner"] [extras] -all = ["authlib", "hiredis", "jaeger-client", "lxml", "matrix-synapse-ldap3", "opentracing", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pympler", "pysaml2", "sentry-sdk", "txredisapi"] +all = ["authlib", "defusedxml", "hiredis", "jaeger-client", "lxml", "matrix-synapse-ldap3", "opentracing", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pympler", "pysaml2", "pytz", "sentry-sdk", "thrift", "tornado", "txredisapi"] cache-memory = ["pympler"] jwt = ["authlib"] matrix-synapse-ldap3 = ["matrix-synapse-ldap3"] oidc = ["authlib"] -opentracing = ["jaeger-client", "opentracing"] +opentracing = ["jaeger-client", "opentracing", "thrift", "tornado"] postgres = ["psycopg2", "psycopg2cffi", "psycopg2cffi-compat"] redis = ["hiredis", "txredisapi"] -saml2 = ["pysaml2"] +saml2 = ["defusedxml", "pysaml2", "pytz"] sentry = ["sentry-sdk"] systemd = ["systemd-python"] test = ["idna", "parameterized"] @@ -3485,4 +3542,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<4.0.0" -content-hash = "960ddae65fde8574f0f36b6988622fc4baf7646823c36699c5cd4773cad8b0ed" +content-hash = "abbbdff591a306b56cc8890dbb2f477ac5f1a2d328baa6409e01084abc655bbf" diff --git a/pyproject.toml b/pyproject.toml index c009ecbf9ba..182861ca39b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,8 @@ dependencies = [ "Twisted[tls]>=21.2.0", "treq>=21.5.0", # Twisted has required pyopenssl 16.0 since about Twisted 16.6. - "pyOpenSSL>=16.0.0", + # pyOpenSSL 16.2.0 fixes compatibility with OpenSSL 1.1.0. + "pyOpenSSL>=16.2.0", "PyYAML>=5.3", "pyasn1>=0.1.9", "pyasn1-modules>=0.0.7", @@ -95,6 +96,20 @@ dependencies = [ # This is used for parsing multipart responses "python-multipart>=0.0.9", + + # Transitive dependency constraints + # These dependencies aren't directly required by Synapse. + # However, in order for Synapse to build, Synapse requires a higher minimum version + # for these dependencies than the minimum specified by the direct dependency. + # We should periodically check to see if these dependencies are still necessary and + # remove any that are no longer required. + "cffi>=1.15", # via cryptography + "pynacl>=1.3", # via signedjson + "pyparsing>=2.4", # via packaging + "pyrsistent>=0.18.0", # via jsonschema + "requests>=2.16.0", # 2.16.0+ no longer vendors urllib3, avoiding Python 3.10+ incompatibility + "urllib3>=1.26.5", # via treq; 1.26.5 fixes Python 3.10+ collections.abc compatibility + "zope-interface>=6.2", # via twisted ] [project.optional-dependencies] @@ -104,7 +119,16 @@ postgres = [ "psycopg2cffi>=2.8;platform_python_implementation == 'PyPy'", "psycopg2cffi-compat==1.1;platform_python_implementation == 'PyPy'", ] -saml2 = ["pysaml2>=4.5.0"] +saml2 = [ + "pysaml2>=4.5.0", + + # Transitive dependencies from pysaml2 + # These dependencies aren't directly required by Synapse. + # However, in order for Synapse to build, Synapse requires a higher minimum version + # for these dependencies than the minimum specified by the direct dependency. + "defusedxml>=0.7.1", # via pysaml2 + "pytz>=2018.3", # via pysaml2 +] oidc = ["authlib>=0.15.1"] # systemd-python is necessary for logging to the systemd journal via # `systemd.journal.JournalHandler`, as is documented in @@ -112,13 +136,23 @@ oidc = ["authlib>=0.15.1"] systemd = ["systemd-python>=231"] url-preview = ["lxml>=4.6.3"] sentry = ["sentry-sdk>=0.7.2"] -opentracing = ["jaeger-client>=4.2.0", "opentracing>=2.2.0"] +opentracing = [ + "jaeger-client>=4.2.0", + "opentracing>=2.2.0", + + # Transitive dependencies from jaeger-client + # These dependencies aren't directly required by Synapse. + # However, in order for Synapse to build, Synapse requires a higher minimum version + # for these dependencies than the minimum specified by the direct dependency. + "thrift>=0.10", # via jaeger-client + "tornado>=6.0", # via jaeger-client +] jwt = ["authlib"] # hiredis is not a *strict* dependency, but it makes things much faster. # (if it is not installed, we fall back to slow code.) -redis = ["txredisapi>=1.4.7", "hiredis"] +redis = ["txredisapi>=1.4.7", "hiredis>=0.3"] # Required to use experimental `caches.track_memory_usage` config option. -cache-memory = ["pympler"] +cache-memory = ["pympler>=1.0"] # If this is updated, don't forget to update the equivalent lines in # `dependency-groups.dev` below. test = ["parameterized>=0.9.0", "idna>=3.3"] @@ -149,12 +183,22 @@ all = [ # opentracing "jaeger-client>=4.2.0", "opentracing>=2.2.0", # redis - "txredisapi>=1.4.7", "hiredis", + "txredisapi>=1.4.7", "hiredis>=0.3", # cache-memory - "pympler", + # 1.0 added support for python 3.10, our current minimum supported python version + "pympler>=1.0", # omitted: # - test: it's useful to have this separate from dev deps in the olddeps job # - systemd: this is a system-based requirement + + # Transitive dependencies + # These dependencies aren't directly required by Synapse. + # However, in order for Synapse to build, Synapse requires a higher minimum version + # for these dependencies than the minimum specified by the direct dependency. + "defusedxml>=0.7.1", # via pysaml2 + "pytz>=2018.3", # via pysaml2 + "thrift>=0.10", # via jaeger-client + "tornado>=6.0", # via jaeger-client ] [project.urls] From dfd00a986fe12208dc4bb0fe0cc4e9770b621197 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Dec 2025 10:02:57 +0000 Subject: [PATCH 062/139] Fix sliding sync performance slow down for long lived connections. (#19206) Fixes https://github.com/element-hq/synapse/issues/19175 This PR moves tracking of what lazy loaded membership we've sent to each room out of the required state table. This avoids that table from continuously growing, which massively helps performance as we pull out all matching rows for the connection when we receive a request. The new table is only read when we have data in a room to send, so we end up reading a lot fewer rows from the DB. Though we now read from that table for every room we have events to return in, rather than once at the start of the request. For an explanation of how the new table works, see the [comment](https://github.com/element-hq/synapse/blob/erikj/sss_better_membership_storage2/synapse/storage/schema/main/delta/93/02_sliding_sync_members.sql#L15-L38) on the table schema. The table is designed so that we can later prune old entries if we wish, but that is not implemented in this PR. Reviewable commit-by-commit. --------- Co-authored-by: Eric Eastwood --- changelog.d/19206.bugfix | 1 + scripts-dev/check_schema_delta.py | 1 - synapse/handlers/sliding_sync/__init__.py | 394 ++++++++++-- synapse/handlers/sliding_sync/store.py | 10 +- .../storage/databases/main/sliding_sync.py | 179 +++++- .../main/delta/93/02_sliding_sync_members.sql | 60 ++ synapse/types/handlers/sliding_sync.py | 90 ++- tests/handlers/test_sliding_sync.py | 561 +++++++++++------ .../client/sliding_sync/test_lists_filters.py | 12 +- .../sliding_sync/test_rooms_required_state.py | 569 +++++++++++++++++- .../client/sliding_sync/test_sliding_sync.py | 10 +- tests/storage/test_sliding_sync_tables.py | 220 ++++--- 12 files changed, 1748 insertions(+), 359 deletions(-) create mode 100644 changelog.d/19206.bugfix create mode 100644 synapse/storage/schema/main/delta/93/02_sliding_sync_members.sql diff --git a/changelog.d/19206.bugfix b/changelog.d/19206.bugfix new file mode 100644 index 00000000000..9cdfaa2571b --- /dev/null +++ b/changelog.d/19206.bugfix @@ -0,0 +1 @@ +Fix sliding sync performance slow down for long lived connections. diff --git a/scripts-dev/check_schema_delta.py b/scripts-dev/check_schema_delta.py index d3440831487..ba8aff36283 100755 --- a/scripts-dev/check_schema_delta.py +++ b/scripts-dev/check_schema_delta.py @@ -14,7 +14,6 @@ SCHEMA_FILE_REGEX = re.compile(r"^synapse/storage/schema/(.*)/delta/(.*)/(.*)$") - # The base branch we want to check against. We use the main development branch # on the assumption that is what we are developing against. DEVELOP_BRANCH = "develop" diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index 68135e9cd3a..bb2e785cfa1 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -17,6 +17,7 @@ from itertools import chain from typing import TYPE_CHECKING, AbstractSet, Mapping +import attr from prometheus_client import Histogram from typing_extensions import assert_never @@ -62,6 +63,7 @@ HaveSentRoomFlag, MutablePerConnectionState, PerConnectionState, + RoomLazyMembershipChanges, RoomSyncConfig, SlidingSyncConfig, SlidingSyncResult, @@ -106,7 +108,7 @@ def __init__(self, hs: "HomeServer"): self.rooms_to_exclude_globally = hs.config.server.rooms_to_exclude_from_sync self.is_mine_id = hs.is_mine_id - self.connection_store = SlidingSyncConnectionStore(self.store) + self.connection_store = SlidingSyncConnectionStore(self.clock, self.store) self.extensions = SlidingSyncExtensionHandler(hs) self.room_lists = SlidingSyncRoomLists(hs) @@ -981,14 +983,15 @@ async def get_room_sync_data( # # Calculate the `StateFilter` based on the `required_state` for the room required_state_filter = StateFilter.none() - # The requested `required_state_map` with the lazy membership expanded and - # `$ME` replaced with the user's ID. This allows us to see what membership we've - # sent down to the client in the next request. - # - # Make a copy so we can modify it. Still need to be careful to make a copy of - # the state key sets if we want to add/remove from them. We could make a deep - # copy but this saves us some work. - expanded_required_state_map = dict(room_sync_config.required_state_map) + + # Keep track of which users' state we may need to fetch. We split this + # into explicit users and lazy loaded users. + explicit_user_state = set() + lazy_load_user_ids = set() + + # Whether lazy-loading of room members is enabled. + lazy_load_room_members = False + if room_membership_for_user_at_to_token.membership not in ( Membership.INVITE, Membership.KNOCK, @@ -1036,7 +1039,6 @@ async def get_room_sync_data( else: required_state_types: list[tuple[str, str | None]] = [] num_wild_state_keys = 0 - lazy_load_room_members = False num_others = 0 for ( state_type, @@ -1068,43 +1070,60 @@ async def get_room_sync_data( timeline_event.state_key ) + # The client needs to know the membership of everyone in + # the timeline we're returning. + lazy_load_user_ids.update(timeline_membership) + # Update the required state filter so we pick up the new # membership - for user_id in timeline_membership: - required_state_types.append( - (EventTypes.Member, user_id) + if limited or initial: + # If the timeline is limited, we only need to + # return the membership changes for people in + # the timeline. + for user_id in timeline_membership: + required_state_types.append( + (EventTypes.Member, user_id) + ) + else: + # For non-limited timelines we always return all + # membership changes. This is so that clients + # who have fetched the full membership list + # already can continue to maintain it for + # non-limited syncs. + # + # This assumes that for non-limited syncs there + # won't be many membership changes that wouldn't + # have been included already (this can only + # happen if membership state was rolled back due + # to state resolution anyway). + # + # `None` is a wildcard in the `StateFilter` + required_state_types.append((EventTypes.Member, None)) + + # Record the extra members we're returning. + lazy_load_user_ids.update( + state_key + for event_type, state_key in room_state_delta_id_map + if event_type == EventTypes.Member ) - - # Add an explicit entry for each user in the timeline - # - # Make a new set or copy of the state key set so we can - # modify it without affecting the original - # `required_state_map` - expanded_required_state_map[EventTypes.Member] = ( - expanded_required_state_map.get( - EventTypes.Member, set() - ) - | timeline_membership - ) - elif state_key == StateValues.ME: + else: num_others += 1 - required_state_types.append((state_type, user.to_string())) + # Replace `$ME` with the user's ID so we can deduplicate # when someone requests the same state with `$ME` or with # their user ID. - # - # Make a new set or copy of the state key set so we can - # modify it without affecting the original - # `required_state_map` - expanded_required_state_map[EventTypes.Member] = ( - expanded_required_state_map.get( - EventTypes.Member, set() - ) - | {user.to_string()} + normalized_state_key = state_key + if state_key == StateValues.ME: + normalized_state_key = user.to_string() + + if state_type == EventTypes.Member: + # Also track explicitly requested member state for + # lazy membership tracking. + explicit_user_state.add(normalized_state_key) + + required_state_types.append( + (state_type, normalized_state_key) ) - else: - num_others += 1 - required_state_types.append((state_type, state_key)) set_tag( SynapseTags.FUNC_ARG_PREFIX @@ -1122,6 +1141,10 @@ async def get_room_sync_data( required_state_filter = StateFilter.from_types(required_state_types) + # Remove any explicitly requested user state from the lazy-loaded set, + # as we track them separately. + lazy_load_user_ids -= explicit_user_state + # We need this base set of info for the response so let's just fetch it along # with the `required_state` for the room hero_room_state = [ @@ -1149,6 +1172,22 @@ async def get_room_sync_data( # We can return all of the state that was requested if this was the first # time we've sent the room down this connection. room_state: StateMap[EventBase] = {} + + # Includes the state for the heroes if we need them (may contain other + # state as well). + hero_membership_state: StateMap[EventBase] = {} + + # By default, we mark all `lazy_load_user_ids` as being sent down + # for the first time in this sync. We later check if we sent any of them + # down previously and update `returned_user_id_to_last_seen_ts_map` if + # we have. + returned_user_id_to_last_seen_ts_map = {} + if lazy_load_room_members: + returned_user_id_to_last_seen_ts_map = dict.fromkeys(lazy_load_user_ids) + new_connection_state.room_lazy_membership[room_id] = RoomLazyMembershipChanges( + returned_user_id_to_last_seen_ts_map=returned_user_id_to_last_seen_ts_map + ) + if initial: room_state = await self.get_current_state_at( room_id=room_id, @@ -1156,28 +1195,97 @@ async def get_room_sync_data( state_filter=state_filter, to_token=to_token, ) + + # The `room_state` includes the hero membership state if needed. + # We'll later filter this down so we don't need to do so here. + hero_membership_state = room_state else: + assert from_token is not None assert from_bound is not None if prev_room_sync_config is not None: + # Define `all_required_user_state` as all user state we want, which + # is the explicitly requested members, any needed for lazy + # loading, and users whose membership has changed. + all_required_user_state = explicit_user_state | lazy_load_user_ids + for state_type, state_key in room_state_delta_id_map: + if state_type == EventTypes.Member: + all_required_user_state.add(state_key) + + # We need to know what user state we previously sent down the + # connection so we can determine what has changed. + # + # We need to fetch all users whose memberships we may want + # to send down this sync. This includes (and matches + # `all_required_user_state`): + # 1. Explicitly requested user state + # 2. Lazy loaded members, i.e. users who appear in the + # timeline. + # 3. The users whose membership has changed in the room, i.e. + # in the state deltas. + # + # This is to correctly handle the cases where a user was + # previously sent down as a lazy loaded member: + # - and is now explicitly requested (so shouldn't be sent down + # again); or + # - their membership has changed (so we need to invalidate + # their entry in the lazy loaded table if we don't send the + # change down). + if all_required_user_state: + previously_returned_user_to_last_seen = ( + await self.store.get_sliding_sync_connection_lazy_members( + connection_position=from_token.connection_position, + room_id=room_id, + user_ids=all_required_user_state, + ) + ) + + # Update the room lazy membership changes to track which + # lazy loaded members were needed for this sync. This is so + # that we can correctly track the last time we sent down + # users' membership (and so can evict old membership state + # from the DB tables). + returned_user_id_to_last_seen_ts_map.update( + (user_id, timestamp) + for user_id, timestamp in previously_returned_user_to_last_seen.items() + if user_id in lazy_load_user_ids + ) + else: + previously_returned_user_to_last_seen = {} + # Check if there are any changes to the required state config # that we need to handle. - changed_required_state_map, added_state_filter = ( - _required_state_changes( - user.to_string(), - prev_required_state_map=prev_room_sync_config.required_state_map, - request_required_state_map=expanded_required_state_map, - state_deltas=room_state_delta_id_map, - ) + changes_return = _required_state_changes( + user.to_string(), + prev_required_state_map=prev_room_sync_config.required_state_map, + request_required_state_map=room_sync_config.required_state_map, + previously_returned_lazy_user_ids=previously_returned_user_to_last_seen.keys(), + request_lazy_load_user_ids=lazy_load_user_ids, + state_deltas=room_state_delta_id_map, ) + changed_required_state_map = changes_return.changed_required_state_map - if added_state_filter: + new_connection_state.room_lazy_membership[ + room_id + ].invalidated_user_ids = changes_return.lazy_members_invalidated + + # Add any previously returned explicit memberships to the lazy + # loaded table. This happens when a client requested explicit + # members and then converted them to lazy loading. + for user_id in changes_return.extra_users_to_add_to_lazy_cache: + # We don't know the right timestamp to use here, as we don't + # know the last time we would have sent the membership down. + # So we don't overwrite it if we have a timestamp already, + # and fallback to `None` (which means now) if we don't. + returned_user_id_to_last_seen_ts_map.setdefault(user_id, None) + + if changes_return.added_state_filter: # Some state entries got added, so we pull out the current # state for them. If we don't do this we'd only send down new deltas. state_ids = await self.get_current_state_ids_at( room_id=room_id, room_membership_for_user_at_to_token=room_membership_for_user_at_to_token, - state_filter=added_state_filter, + state_filter=changes_return.added_state_filter, to_token=to_token, ) room_state_delta_id_map.update(state_ids) @@ -1189,6 +1297,7 @@ async def get_room_sync_data( # If the membership changed and we have to get heroes, get the remaining # heroes from the state + hero_membership_state = {} if hero_user_ids: hero_membership_state = await self.get_current_state_at( room_id=room_id, @@ -1196,7 +1305,6 @@ async def get_room_sync_data( state_filter=StateFilter.from_types(hero_room_state), to_token=to_token, ) - room_state.update(hero_membership_state) required_room_state: StateMap[EventBase] = {} if required_state_filter != StateFilter.none(): @@ -1219,7 +1327,7 @@ async def get_room_sync_data( # Assemble heroes: extract the info from the state we just fetched heroes: list[SlidingSyncResult.RoomResult.StrippedHero] = [] for hero_user_id in hero_user_ids: - member_event = room_state.get((EventTypes.Member, hero_user_id)) + member_event = hero_membership_state.get((EventTypes.Member, hero_user_id)) if member_event is not None: heroes.append( SlidingSyncResult.RoomResult.StrippedHero( @@ -1281,7 +1389,7 @@ async def get_room_sync_data( bump_stamp = 0 room_sync_required_state_map_to_persist: Mapping[str, AbstractSet[str]] = ( - expanded_required_state_map + room_sync_config.required_state_map ) if changed_required_state_map: room_sync_required_state_map_to_persist = changed_required_state_map @@ -1471,13 +1579,37 @@ async def _get_bump_stamp( return None +@attr.s(auto_attribs=True) +class _RequiredStateChangesReturn: + """Return type for _required_state_changes.""" + + changed_required_state_map: Mapping[str, AbstractSet[str]] | None + """The updated required state map to store in the room config, or None if + there is no change.""" + + added_state_filter: StateFilter + """The state filter to use to fetch any additional current state that needs + to be returned to the client.""" + + extra_users_to_add_to_lazy_cache: AbstractSet[str] = frozenset() + """The set of user IDs we should add to the lazy members cache that we had + previously returned. Handles the case where a user was previously sent down + explicitly but is now being lazy loaded.""" + + lazy_members_invalidated: AbstractSet[str] = frozenset() + """The set of user IDs whose membership has changed but we didn't send down, + so we need to invalidate them from the cache.""" + + def _required_state_changes( user_id: str, *, prev_required_state_map: Mapping[str, AbstractSet[str]], request_required_state_map: Mapping[str, AbstractSet[str]], + previously_returned_lazy_user_ids: AbstractSet[str], + request_lazy_load_user_ids: AbstractSet[str], state_deltas: StateMap[str], -) -> tuple[Mapping[str, AbstractSet[str]] | None, StateFilter]: +) -> _RequiredStateChangesReturn: """Calculates the changes between the required state room config from the previous requests compared with the current request. @@ -1491,14 +1623,62 @@ def _required_state_changes( added, removed and then added again to the required state. In that case we only want to re-send that entry down sync if it has changed. - Returns: - A 2-tuple of updated required state config (or None if there is no update) - and the state filter to use to fetch extra current state that we need to - return. + Args: + user_id: The user ID of the user making the request. + prev_required_state_map: The required state map from the previous + request. + request_required_state_map: The required state map from the current + request. + previously_returned_lazy_user_ids: The set of user IDs whose membership + we have previously returned to the client due to lazy loading. This + is filtered to only include users who have either sent events in the + `timeline`, `required_state` or whose membership changed. + request_lazy_load_user_ids: The set of user IDs whose lazy-loaded + membership is required for this request. + state_deltas: The state deltas in the room in the request token range, + considering user membership. See `get_current_state_deltas_for_room` + for more details. """ + + # First we find any lazy members that have been invalidated due to state + # changes that we are not sending down. + lazy_members_invalidated = set() + for event_type, state_key in state_deltas: + if event_type != EventTypes.Member: + continue + + if state_key in request_lazy_load_user_ids: + # Because it's part of the `request_lazy_load_user_ids`, we're going to + # send this member change down. + continue + + if state_key not in previously_returned_lazy_user_ids: + # We've not previously returned this member so nothing to + # invalidate. + continue + + lazy_members_invalidated.add(state_key) + if prev_required_state_map == request_required_state_map: - # There has been no change. Return immediately. - return None, StateFilter.none() + # There has been no change in state, just need to check lazy members. + newly_returned_lazy_members = ( + request_lazy_load_user_ids - previously_returned_lazy_user_ids + ) + if newly_returned_lazy_members: + # There are some new lazy members we need to fetch. + added_types: list[tuple[str, str | None]] = [] + for new_user_id in newly_returned_lazy_members: + added_types.append((EventTypes.Member, new_user_id)) + + added_state_filter = StateFilter.from_types(added_types) + else: + added_state_filter = StateFilter.none() + + return _RequiredStateChangesReturn( + changed_required_state_map=None, + added_state_filter=added_state_filter, + lazy_members_invalidated=lazy_members_invalidated, + ) prev_wildcard = prev_required_state_map.get(StateValues.WILDCARD, set()) request_wildcard = request_required_state_map.get(StateValues.WILDCARD, set()) @@ -1508,17 +1688,29 @@ def _required_state_changes( # already fetching everything, we don't have to fetch anything now that they've # narrowed. if StateValues.WILDCARD in prev_wildcard: - return request_required_state_map, StateFilter.none() + return _RequiredStateChangesReturn( + changed_required_state_map=request_required_state_map, + added_state_filter=StateFilter.none(), + lazy_members_invalidated=lazy_members_invalidated, + ) # If a event type wildcard has been added or removed we don't try and do # anything fancy, and instead always update the effective room required # state config to match the request. if request_wildcard - prev_wildcard: # Some keys were added, so we need to fetch everything - return request_required_state_map, StateFilter.all() + return _RequiredStateChangesReturn( + changed_required_state_map=request_required_state_map, + added_state_filter=StateFilter.all(), + lazy_members_invalidated=lazy_members_invalidated, + ) if prev_wildcard - request_wildcard: # Keys were only removed, so we don't have to fetch everything. - return request_required_state_map, StateFilter.none() + return _RequiredStateChangesReturn( + changed_required_state_map=request_required_state_map, + added_state_filter=StateFilter.none(), + lazy_members_invalidated=lazy_members_invalidated, + ) # Contains updates to the required state map compared with the previous room # config. This has the same format as `RoomSyncConfig.required_state` @@ -1550,6 +1742,17 @@ def _required_state_changes( # Nothing *added*, so we skip. Removals happen below. continue + # Handle the special case of adding `$LAZY` membership, where we want to + # always record the change to be lazy loading, as we immediately start + # using the lazy loading tables so there is no point *not* recording the + # change to lazy load in the effective room config. + if event_type == EventTypes.Member: + old_state_key_lazy = StateValues.LAZY in old_state_keys + request_state_key_lazy = StateValues.LAZY in request_state_keys + if not old_state_key_lazy and request_state_key_lazy: + changes[event_type] = request_state_keys + continue + # We only remove state keys from the effective state if they've been # removed from the request *and* the state has changed. This ensures # that if a client removes and then re-adds a state key, we only send @@ -1620,9 +1823,31 @@ def _required_state_changes( # LAZY values should also be ignore for event types that are # not membership. pass + elif event_type == EventTypes.Member: + if state_key not in previously_returned_lazy_user_ids: + # Only add *explicit* members we haven't previously sent + # down. + added.append((event_type, state_key)) else: added.append((event_type, state_key)) + previously_required_state_members = set( + prev_required_state_map.get(EventTypes.Member, ()) + ) + if StateValues.ME in previously_required_state_members: + previously_required_state_members.add(user_id) + + # We also need to pull out any lazy members that are now required but + # haven't previously been returned. + for required_user_id in ( + request_lazy_load_user_ids + # Remove previously returned users + - previously_returned_lazy_user_ids + # Exclude previously explicitly requested members. + - previously_required_state_members + ): + added.append((EventTypes.Member, required_user_id)) + added_state_filter = StateFilter.from_types(added) # Figure out what changes we need to apply to the effective required state @@ -1663,13 +1888,25 @@ def _required_state_changes( changes[event_type] = request_state_keys continue + # When handling $LAZY membership, we want to either a) not update the + # state or b) update it to match the request. This is to avoid churn of + # the effective required state for rooms (we deduplicate required state + # between rooms), and because we can store the previously returned + # explicit memberships with the lazy loaded memberships. if event_type == EventTypes.Member: old_state_key_lazy = StateValues.LAZY in old_state_keys request_state_key_lazy = StateValues.LAZY in request_state_keys + has_lazy = old_state_key_lazy or request_state_key_lazy + # If a "$LAZY" has been added or removed we always update to match + # the request. if old_state_key_lazy != request_state_key_lazy: - # If a "$LAZY" has been added or removed we always update the effective room - # required state config to match the request. + changes[event_type] = request_state_keys + continue + + # Or if we have lazy membership and there are invalidated + # explicit memberships. + if has_lazy and invalidated_state_keys: changes[event_type] = request_state_keys continue @@ -1684,6 +1921,28 @@ def _required_state_changes( if invalidated_state_keys: changes[event_type] = old_state_keys - invalidated_state_keys + # Check for any explicit membership changes that were removed that we can + # add to the lazy members previously returned. This is so that we don't + # return a user due to lazy loading if they were previously returned as an + # explicit membership. + users_to_add_to_lazy_cache: set[str] = set() + + membership_changes = changes.get(EventTypes.Member, set()) + if membership_changes and StateValues.LAZY in request_state_keys: + for state_key in prev_required_state_map.get(EventTypes.Member, set()): + if state_key == StateValues.WILDCARD or state_key == StateValues.LAZY: + # Ignore non-user IDs. + continue + + if state_key == StateValues.ME: + # Normalize to proper user ID + state_key = user_id + + # We remember the user if they haven't been invalidated + if (EventTypes.Member, state_key) not in state_deltas: + users_to_add_to_lazy_cache.add(state_key) + + new_required_state_map = None if changes: # Update the required state config based on the changes. new_required_state_map = dict(prev_required_state_map) @@ -1694,6 +1953,9 @@ def _required_state_changes( # Remove entries with empty state keys. new_required_state_map.pop(event_type, None) - return new_required_state_map, added_state_filter - else: - return None, added_state_filter + return _RequiredStateChangesReturn( + changed_required_state_map=new_required_state_map, + added_state_filter=added_state_filter, + lazy_members_invalidated=lazy_members_invalidated, + extra_users_to_add_to_lazy_cache=users_to_add_to_lazy_cache, + ) diff --git a/synapse/handlers/sliding_sync/store.py b/synapse/handlers/sliding_sync/store.py index d01fab271f6..65febe58aaa 100644 --- a/synapse/handlers/sliding_sync/store.py +++ b/synapse/handlers/sliding_sync/store.py @@ -13,7 +13,6 @@ # import logging -from typing import TYPE_CHECKING import attr @@ -25,9 +24,7 @@ PerConnectionState, SlidingSyncConfig, ) - -if TYPE_CHECKING: - pass +from synapse.util.clock import Clock logger = logging.getLogger(__name__) @@ -61,7 +58,8 @@ class SlidingSyncConnectionStore: to mapping of room ID to `HaveSentRoom`. """ - store: "DataStore" + clock: Clock + store: DataStore async def get_and_clear_connection_positions( self, @@ -101,7 +99,7 @@ async def record_new_state( If there are no changes to the state this may return the same token as the existing per-connection state. """ - if not new_connection_state.has_updates(): + if not new_connection_state.has_updates(self.clock): if from_token is not None: return from_token.connection_position else: diff --git a/synapse/storage/databases/main/sliding_sync.py b/synapse/storage/databases/main/sliding_sync.py index 828eed3a73c..c66002dae47 100644 --- a/synapse/storage/databases/main/sliding_sync.py +++ b/synapse/storage/databases/main/sliding_sync.py @@ -14,7 +14,7 @@ import logging -from typing import TYPE_CHECKING, Mapping, cast +from typing import TYPE_CHECKING, AbstractSet, Mapping, cast import attr @@ -26,13 +26,16 @@ DatabasePool, LoggingDatabaseConnection, LoggingTransaction, + make_in_list_sql_clause, ) +from synapse.storage.engines import PostgresEngine from synapse.types import MultiWriterStreamToken, RoomStreamToken from synapse.types.handlers.sliding_sync import ( HaveSentRoom, HaveSentRoomFlag, MutablePerConnectionState, PerConnectionState, + RoomLazyMembershipChanges, RoomStatusMap, RoomSyncConfig, ) @@ -373,6 +376,13 @@ def persist_per_connection_state_txn( value_values=values, ) + self._persist_sliding_sync_connection_lazy_members_txn( + txn, + connection_key, + connection_position, + per_connection_state.room_lazy_membership, + ) + return connection_position @cached(iterable=True, max_entries=100000) @@ -446,6 +456,23 @@ def _get_and_clear_connection_positions_txn( """ txn.execute(sql, (connection_key, connection_position)) + # Move any lazy membership entries for this connection position to have + # `NULL` connection position, indicating that it applies to all future + # positions on this connection. This is safe because we have deleted all + # other (potentially forked) connection positions, and so all future + # positions in this connection will be a continuation of the current + # position. Thus any lazy membership entries we have sent down will still + # be valid. + self.db_pool.simple_update_txn( + txn, + table="sliding_sync_connection_lazy_members", + keyvalues={ + "connection_key": connection_key, + "connection_position": connection_position, + }, + updatevalues={"connection_position": None}, + ) + # Fetch and create a mapping from required state ID to the actual # required state for the connection. rows = self.db_pool.simple_select_list_txn( @@ -525,8 +552,153 @@ def _get_and_clear_connection_positions_txn( receipts=RoomStatusMap(receipts), account_data=RoomStatusMap(account_data), room_configs=room_configs, + room_lazy_membership={}, ) + async def get_sliding_sync_connection_lazy_members( + self, + connection_position: int, + room_id: str, + user_ids: AbstractSet[str], + ) -> Mapping[str, int]: + """Get which user IDs in the room we have previously sent lazy + membership for. + + Args: + connection_position: The sliding sync connection position. + room_id: The room ID to get lazy members for. + user_ids: The user IDs to check whether we've previously sent + because of lazy membership. + + Returns: + The mapping of user IDs to the last seen timestamp for those user + IDs. Only includes user IDs that we have previously sent lazy + membership for, and so may be a subset of the `user_ids` passed in. + """ + + def get_sliding_sync_connection_lazy_members_txn( + txn: LoggingTransaction, + ) -> Mapping[str, int]: + user_clause, user_args = make_in_list_sql_clause( + txn.database_engine, "user_id", user_ids + ) + + # Fetch all the lazy membership entries for the given connection, + # room and user IDs. We don't have the `connection_key` here, so we + # join against `sliding_sync_connection_positions` to get it. + # + # Beware that there are two `connection_position` columns in the + # query which are different, the one in + # `sliding_sync_connection_positions` is the one we match to get the + # connection_key, whereas the one in + # `sliding_sync_connection_lazy_members` is what we filter against + # (it may be null or the same as the one passed in). + # + # FIXME: We should pass in `connection_key` here to avoid the join. + # We don't do this currently as the caller doesn't have it handy. + sql = f""" + SELECT user_id, members.connection_position, last_seen_ts + FROM sliding_sync_connection_lazy_members AS members + INNER JOIN sliding_sync_connection_positions AS pos USING (connection_key) + WHERE pos.connection_position = ? AND room_id = ? AND {user_clause} + """ + + txn.execute(sql, (connection_position, room_id, *user_args)) + + # Filter out any cache entries that only apply to forked connection + # positions. Entries with `NULL` `connection_position` apply to all + # positions on the connection. + return { + user_id: last_seen_ts + for user_id, db_connection_position, last_seen_ts in txn + if db_connection_position == connection_position + or db_connection_position is None + } + + return await self.db_pool.runInteraction( + "get_sliding_sync_connection_lazy_members", + get_sliding_sync_connection_lazy_members_txn, + db_autocommit=True, # Avoid transaction for single read + ) + + def _persist_sliding_sync_connection_lazy_members_txn( + self, + txn: LoggingTransaction, + connection_key: int, + new_connection_position: int, + all_changes: dict[str, RoomLazyMembershipChanges], + ) -> None: + """Persist that we have sent lazy membership for the given user IDs.""" + + now = self.clock.time_msec() + + # Figure out which cache entries to add or update. + # + # These are either a) new entries we've never sent before (i.e. with a + # None last_seen_ts), or b) where the `last_seen_ts` is old enough that + # we want to update it. + # + # We don't update the timestamp every time to avoid hammering the DB + # with writes, and we don't need the timestamp to be precise. It is used + # to evict old entries that haven't been used in a while. + to_update: list[tuple[str, str]] = [] + for room_id, room_changes in all_changes.items(): + user_ids_to_update = room_changes.get_returned_user_ids_to_update( + self.clock + ) + to_update.extend((room_id, user_id) for user_id in user_ids_to_update) + + if to_update: + # Upsert the new/updated entries. + # + # Ignore conflicts where the existing entry has a different + # connection position (i.e. from a forked connection position). This + # may mean that we lose some updates, but that's acceptable as this + # is a cache and its fine for it to *not* include rows. (Downstream + # this will cause us to maybe send a few extra lazy members down + # sync, but we're allowed to send extra members). + sql = """ + INSERT INTO sliding_sync_connection_lazy_members + (connection_key, connection_position, room_id, user_id, last_seen_ts) + VALUES {value_placeholder} + ON CONFLICT (connection_key, room_id, user_id) + DO UPDATE SET last_seen_ts = EXCLUDED.last_seen_ts + WHERE sliding_sync_connection_lazy_members.connection_position IS NULL + OR sliding_sync_connection_lazy_members.connection_position = EXCLUDED.connection_position + """ + + args = [ + (connection_key, new_connection_position, room_id, user_id, now) + for room_id, user_id in to_update + ] + + if isinstance(self.database_engine, PostgresEngine): + sql = sql.format(value_placeholder="?") + txn.execute_values(sql, args, fetch=False) + else: + sql = sql.format(value_placeholder="(?, ?, ?, ?, ?)") + txn.execute_batch(sql, args) + + # Remove any invalidated entries. + to_remove: list[tuple[str, str]] = [] + for room_id, room_changes in all_changes.items(): + for user_id in room_changes.invalidated_user_ids: + to_remove.append((room_id, user_id)) + + if to_remove: + # We don't try and match on connection position here: it's fine to + # remove it from all forks. This is a cache so it's fine to expire + # arbitrary entries, the worst that happens is we send a few extra + # lazy members down sync. + self.db_pool.simple_delete_many_batch_txn( + txn, + table="sliding_sync_connection_lazy_members", + keys=("connection_key", "room_id", "user_id"), + values=[ + (connection_key, room_id, user_id) for room_id, user_id in to_remove + ], + ) + @wrap_as_background_process("delete_old_sliding_sync_connections") async def delete_old_sliding_sync_connections(self) -> None: """Delete sliding sync connections that have not been used for a long time.""" @@ -564,6 +736,10 @@ class PerConnectionStateDB: room_configs: Mapping[str, "RoomSyncConfig"] + room_lazy_membership: dict[str, RoomLazyMembershipChanges] + """Lazy membership changes to persist alongside this state. Only used + when persisting.""" + @staticmethod async def from_state( per_connection_state: "MutablePerConnectionState", store: "DataStore" @@ -618,6 +794,7 @@ async def from_state( receipts=RoomStatusMap(receipts), account_data=RoomStatusMap(account_data), room_configs=per_connection_state.room_configs.maps[0], + room_lazy_membership=per_connection_state.room_lazy_membership, ) async def to_state(self, store: "DataStore") -> "PerConnectionState": diff --git a/synapse/storage/schema/main/delta/93/02_sliding_sync_members.sql b/synapse/storage/schema/main/delta/93/02_sliding_sync_members.sql new file mode 100644 index 00000000000..279458d7108 --- /dev/null +++ b/synapse/storage/schema/main/delta/93/02_sliding_sync_members.sql @@ -0,0 +1,60 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 Element Creations Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + + +-- Tracks which member states have been sent to the client for lazy-loaded +-- members in sliding sync. This is a *cache* as it doesn't matter if we send +-- down members we've previously sent down, i.e. it's safe to delete any rows. +-- +-- We could have tracked these as part of the +-- `sliding_sync_connection_required_state` table, but that would bloat that +-- table significantly as most rooms will have lazy-loaded members. We want to +-- keep that table small as we always pull out all rows for the connection for +-- every request, so storing lots of data there would be bad for performance. To +-- keep that table small we also deduplicate the requested state across +-- different rooms, which if we stored lazy members there would prevent. +-- +-- We track a *rough* `last_seen_ts` for each user in each room which indicates +-- when we last would've sent their member state to the client. `last_seen_ts` +-- is used so that we can remove members which haven't been seen for a while to +-- save space. This is a *rough* timestamp as we don't want to update the +-- timestamp every time to avoid hammering the DB with writes, and we don't need +-- the timestamp to be precise (as it is used to evict old entries that haven't +-- been used in a while). +-- +-- Care must be taken when handling "forked" positions, i.e. we have responded +-- to a request with a position and then get another different request using the +-- previous position as a base. We track this by including a +-- `connection_position` for newly inserted rows. When we advance the position +-- we set this to NULL for all rows which were present at that position, and +-- delete all other rows. When reading rows we can then filter out any rows +-- which have a non-NULL `connection_position` which is not the current +-- position. +-- +-- I.e. `connection_position` is NULL for rows which are valid for *all* +-- positions on the connection, and is non-NULL for rows which are only valid +-- for a specific position. +-- +-- When invalidating rows, we can just delete them. Technically this could +-- invalidate for a forked position, but this is acceptable as equivalent to a +-- cache eviction. +CREATE TABLE sliding_sync_connection_lazy_members ( + connection_key BIGINT NOT NULL REFERENCES sliding_sync_connections(connection_key) ON DELETE CASCADE, + connection_position BIGINT REFERENCES sliding_sync_connection_positions(connection_position) ON DELETE CASCADE, + room_id TEXT NOT NULL, + user_id TEXT NOT NULL, + last_seen_ts BIGINT NOT NULL +); + +CREATE UNIQUE INDEX sliding_sync_connection_lazy_members_idx ON sliding_sync_connection_lazy_members (connection_key, room_id, user_id); +CREATE INDEX sliding_sync_connection_lazy_members_pos_idx ON sliding_sync_connection_lazy_members (connection_key, connection_position) WHERE connection_position IS NOT NULL; diff --git a/synapse/types/handlers/sliding_sync.py b/synapse/types/handlers/sliding_sync.py index 03b3bcb3caf..dcb125c494a 100644 --- a/synapse/types/handlers/sliding_sync.py +++ b/synapse/types/handlers/sliding_sync.py @@ -49,12 +49,21 @@ UserID, ) from synapse.types.rest.client import SlidingSyncBody +from synapse.util.clock import Clock +from synapse.util.duration import Duration if TYPE_CHECKING: from synapse.handlers.relations import BundledAggregations logger = logging.getLogger(__name__) +# How often to update the last seen timestamp for lazy members. +# +# We don't update the timestamp every time to avoid hammering the DB with +# writes, and we don't need the timestamp to be precise (as it is used to evict +# old entries that haven't been used in a while). +LAZY_MEMBERS_UPDATE_INTERVAL = Duration(hours=1) + class SlidingSyncConfig(SlidingSyncBody): """ @@ -891,6 +900,69 @@ def __len__(self) -> int: return len(self.rooms) + len(self.receipts) + len(self.room_configs) +@attr.s(auto_attribs=True) +class RoomLazyMembershipChanges: + """Changes to lazily-loaded room memberships for a given room.""" + + returned_user_id_to_last_seen_ts_map: Mapping[str, int | None] = attr.Factory(dict) + """Map from user ID to timestamp for users whose membership we have lazily + loaded in this room an request. The timestamp indicates the time we + previously needed the membership, or None if we sent it down for the first + time in this request. + + We track a *rough* `last_seen_ts` for each user in each room which indicates + when we last would've sent their member state to the client. This is used so + that we can remove members which haven't been seen for a while to save + space. + + Note: this will include users whose membership we would have sent down but + didn't due to us having previously sent them. + """ + + invalidated_user_ids: AbstractSet[str] = attr.Factory(set) + """Set of user IDs whose latest membership we have *not* sent down""" + + def get_returned_user_ids_to_update(self, clock: Clock) -> StrCollection: + """Get the user IDs whose last seen timestamp we need to update in the + database. + + This is a subset of user IDs in `returned_user_id_to_last_seen_ts_map`, + whose timestamp is either None (first time we've sent them) or older + than `LAZY_MEMBERS_UPDATE_INTERVAL`. + + We only update the timestamp in the database every so often to avoid + hammering the DB with writes. We don't need the timestamp to be precise, + as the timestamp is used to evict old entries that haven't been used in + a while. + """ + + now_ms = clock.time_msec() + return [ + user_id + for user_id, last_seen_ts in self.returned_user_id_to_last_seen_ts_map.items() + if last_seen_ts is None + or now_ms - last_seen_ts >= LAZY_MEMBERS_UPDATE_INTERVAL.as_millis() + ] + + def has_updates(self, clock: Clock) -> bool: + """Check if there are any updates to the lazy membership changes. + + Called to check if we need to persist changes to the lazy membership + state for the room. We want to avoid persisting the state if there are + no changes, to avoid unnecessary writes (and cache misses due to new + connection position). + """ + + # We consider there to be updates if there are any invalidated user + # IDs... + if self.invalidated_user_ids: + return True + + # ...or if any of the returned user IDs need their last seen timestamp + # updating in the database. + return bool(self.get_returned_user_ids_to_update(clock)) + + @attr.s(auto_attribs=True) class MutablePerConnectionState(PerConnectionState): """A mutable version of `PerConnectionState`""" @@ -903,12 +975,28 @@ class MutablePerConnectionState(PerConnectionState): room_configs: typing.ChainMap[str, RoomSyncConfig] - def has_updates(self) -> bool: + # A map from room ID to the lazily-loaded memberships needed for the + # request in that room. + room_lazy_membership: dict[str, RoomLazyMembershipChanges] = attr.Factory(dict) + + def has_updates(self, clock: Clock) -> bool: + """Check if there are any updates to the per-connection state that need + persisting. + + It is important that we don't spuriously do persistence, as that will + always generate a new connection position which will invalidate some of + the caches. It doesn't need to be perfect, but we should avoid always + generating new connection positions when doing lazy loading + """ return ( bool(self.rooms.get_updates()) or bool(self.receipts.get_updates()) or bool(self.account_data.get_updates()) or bool(self.get_room_config_updates()) + or any( + change.has_updates(clock) + for change in self.room_lazy_membership.values() + ) ) def get_room_config_updates(self) -> Mapping[str, RoomSyncConfig]: diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py index 45829064415..22939880368 100644 --- a/tests/handlers/test_sliding_sync.py +++ b/tests/handlers/test_sliding_sync.py @@ -18,7 +18,7 @@ # # import logging -from typing import AbstractSet, Mapping +from typing import AbstractSet from unittest.mock import patch import attr @@ -38,13 +38,17 @@ RoomSyncConfig, StateValues, _required_state_changes, + _RequiredStateChangesReturn, ) from synapse.rest import admin from synapse.rest.client import knock, login, room from synapse.server import HomeServer from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.types import JsonDict, StateMap, StreamToken, UserID, create_requester -from synapse.types.handlers.sliding_sync import PerConnectionState, SlidingSyncConfig +from synapse.types.handlers.sliding_sync import ( + PerConnectionState, + SlidingSyncConfig, +) from synapse.types.state import StateFilter from synapse.util.clock import Clock @@ -3827,12 +3831,11 @@ class RequiredStateChangesTestParameters: previous_required_state_map: dict[str, set[str]] request_required_state_map: dict[str, set[str]] state_deltas: StateMap[str] - expected_with_state_deltas: tuple[ - Mapping[str, AbstractSet[str]] | None, StateFilter - ] - expected_without_state_deltas: tuple[ - Mapping[str, AbstractSet[str]] | None, StateFilter - ] + expected_with_state_deltas: _RequiredStateChangesReturn + expected_without_state_deltas: _RequiredStateChangesReturn + + previously_returned_lazy_user_ids: AbstractSet[str] = frozenset() + request_lazy_load_user_ids: AbstractSet[str] = frozenset() class RequiredStateChangesTestCase(unittest.TestCase): @@ -3848,8 +3851,12 @@ class RequiredStateChangesTestCase(unittest.TestCase): request_required_state_map={"type1": {"state_key"}}, state_deltas={("type1", "state_key"): "$event_id"}, # No changes - expected_with_state_deltas=(None, StateFilter.none()), - expected_without_state_deltas=(None, StateFilter.none()), + expected_with_state_deltas=_RequiredStateChangesReturn( + None, StateFilter.none() + ), + expected_without_state_deltas=_RequiredStateChangesReturn( + None, StateFilter.none() + ), ), ), ( @@ -3862,14 +3869,14 @@ class RequiredStateChangesTestCase(unittest.TestCase): "type2": {"state_key"}, }, state_deltas={("type2", "state_key"): "$event_id"}, - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( # We've added a type so we should persist the changed required state # config. {"type1": {"state_key"}, "type2": {"state_key"}}, # We should see the new type added StateFilter.from_types([("type2", "state_key")]), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( {"type1": {"state_key"}, "type2": {"state_key"}}, StateFilter.from_types([("type2", "state_key")]), ), @@ -3885,7 +3892,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): "type2": {"state_key"}, }, state_deltas={("type2", "state_key"): "$event_id"}, - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( # We've added a type so we should persist the changed required state # config. {"type1": {"state_key"}, "type2": {"state_key"}}, @@ -3894,7 +3901,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): [("type1", "state_key"), ("type2", "state_key")] ), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( {"type1": {"state_key"}, "type2": {"state_key"}}, StateFilter.from_types( [("type1", "state_key"), ("type2", "state_key")] @@ -3909,14 +3916,14 @@ class RequiredStateChangesTestCase(unittest.TestCase): previous_required_state_map={"type": {"state_key1"}}, request_required_state_map={"type": {"state_key1", "state_key2"}}, state_deltas={("type", "state_key2"): "$event_id"}, - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( # We've added a key so we should persist the changed required state # config. {"type": {"state_key1", "state_key2"}}, # We should see the new state_keys added StateFilter.from_types([("type", "state_key2")]), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( {"type": {"state_key1", "state_key2"}}, StateFilter.from_types([("type", "state_key2")]), ), @@ -3929,7 +3936,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): previous_required_state_map={"type": {"state_key1"}}, request_required_state_map={"type": {"state_key2", "state_key3"}}, state_deltas={("type", "state_key2"): "$event_id"}, - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( # We've added a key so we should persist the changed required state # config. # @@ -3940,7 +3947,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): [("type", "state_key2"), ("type", "state_key3")] ), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( {"type": {"state_key1", "state_key2", "state_key3"}}, StateFilter.from_types( [("type", "state_key2"), ("type", "state_key3")] @@ -3964,7 +3971,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): }, request_required_state_map={"type1": {"state_key"}}, state_deltas={("type2", "state_key"): "$event_id"}, - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( # Remove `type2` since there's been a change to that state, # (persist the change to required state). That way next time, # they request `type2`, we see that we haven't sent it before @@ -3975,7 +3982,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): # less state now StateFilter.none(), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( # `type2` is no longer requested but since that state hasn't # changed, nothing should change (we should still keep track # that we've sent `type2` before). @@ -3998,7 +4005,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): }, request_required_state_map={}, state_deltas={("type2", "state_key"): "$event_id"}, - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( # Remove `type2` since there's been a change to that state, # (persist the change to required state). That way next time, # they request `type2`, we see that we haven't sent it before @@ -4009,7 +4016,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): # less state now StateFilter.none(), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( # `type2` is no longer requested but since that state hasn't # changed, nothing should change (we should still keep track # that we've sent `type2` before). @@ -4029,7 +4036,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): previous_required_state_map={"type": {"state_key1", "state_key2"}}, request_required_state_map={"type": {"state_key1"}}, state_deltas={("type", "state_key2"): "$event_id"}, - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( # Remove `(type, state_key2)` since there's been a change # to that state (persist the change to required state). # That way next time, they request `(type, state_key2)`, we see @@ -4041,7 +4048,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): # less state now StateFilter.none(), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( # `(type, state_key2)` is no longer requested but since that # state hasn't changed, nothing should change (we should still # keep track that we've sent `(type, state_key1)` and `(type, @@ -4073,11 +4080,11 @@ class RequiredStateChangesTestCase(unittest.TestCase): ("other_type", "state_key"): "$event_id", }, # We've added a wildcard, so we persist the change and request everything - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( {"type1": {"state_key2"}, StateValues.WILDCARD: {"state_key"}}, StateFilter.all(), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( {"type1": {"state_key2"}, StateValues.WILDCARD: {"state_key"}}, StateFilter.all(), ), @@ -4103,13 +4110,13 @@ class RequiredStateChangesTestCase(unittest.TestCase): ("other_type", "state_key"): "$event_id", }, # We've removed a type wildcard, so we persist the change but don't request anything - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( {"type1": {"state_key2"}}, # We don't need to request anything more if they are requesting # less state now StateFilter.none(), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( {"type1": {"state_key2"}}, # We don't need to request anything more if they are requesting # less state now @@ -4129,11 +4136,11 @@ class RequiredStateChangesTestCase(unittest.TestCase): state_deltas={("type2", "state_key"): "$event_id"}, # We've added a wildcard state_key, so we persist the change and # request all of the state for that type - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( {"type1": {"state_key"}, "type2": {StateValues.WILDCARD}}, StateFilter.from_types([("type2", None)]), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( {"type1": {"state_key"}, "type2": {StateValues.WILDCARD}}, StateFilter.from_types([("type2", None)]), ), @@ -4151,7 +4158,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): state_deltas={("type2", "state_key"): "$event_id"}, # We've removed a state_key wildcard, so we persist the change and # request nothing - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( {"type1": {"state_key"}}, # We don't need to request anything more if they are requesting # less state now @@ -4160,7 +4167,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): # We've removed a state_key wildcard but there have been no matching # state changes, so no changes needed, just persist the # `request_required_state_map` as-is. - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( None, # We don't need to request anything more if they are requesting # less state now @@ -4180,7 +4187,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): }, request_required_state_map={"type1": {"state_key1"}}, state_deltas={("type1", "state_key3"): "$event_id"}, - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( # We've removed some state keys from the type, but only state_key3 was # changed so only that one should be removed. {"type1": {"state_key1", "state_key2"}}, @@ -4188,7 +4195,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): # less state now StateFilter.none(), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( # No changes needed, just persist the # `request_required_state_map` as-is None, @@ -4207,14 +4214,14 @@ class RequiredStateChangesTestCase(unittest.TestCase): previous_required_state_map={}, request_required_state_map={"type1": {StateValues.ME}}, state_deltas={("type1", "@user:test"): "$event_id"}, - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( # We've added a type so we should persist the changed required state # config. {"type1": {StateValues.ME}}, # We should see the new state_keys added StateFilter.from_types([("type1", "@user:test")]), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( {"type1": {StateValues.ME}}, StateFilter.from_types([("type1", "@user:test")]), ), @@ -4229,7 +4236,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): previous_required_state_map={"type1": {StateValues.ME}}, request_required_state_map={}, state_deltas={("type1", "@user:test"): "$event_id"}, - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( # Remove `type1` since there's been a change to that state, # (persist the change to required state). That way next time, # they request `type1`, we see that we haven't sent it before @@ -4240,7 +4247,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): # less state now StateFilter.none(), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( # `type1` is no longer requested but since that state hasn't # changed, nothing should change (we should still keep track # that we've sent `type1` before). @@ -4260,14 +4267,14 @@ class RequiredStateChangesTestCase(unittest.TestCase): previous_required_state_map={}, request_required_state_map={"type1": {"@user:test"}}, state_deltas={("type1", "@user:test"): "$event_id"}, - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( # We've added a type so we should persist the changed required state # config. {"type1": {"@user:test"}}, # We should see the new state_keys added StateFilter.from_types([("type1", "@user:test")]), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( {"type1": {"@user:test"}}, StateFilter.from_types([("type1", "@user:test")]), ), @@ -4282,7 +4289,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): previous_required_state_map={"type1": {"@user:test"}}, request_required_state_map={}, state_deltas={("type1", "@user:test"): "$event_id"}, - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( # Remove `type1` since there's been a change to that state, # (persist the change to required state). That way next time, # they request `type1`, we see that we haven't sent it before @@ -4293,7 +4300,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): # less state now StateFilter.none(), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( # `type1` is no longer requested but since that state hasn't # changed, nothing should change (we should still keep track # that we've sent `type1` before). @@ -4313,13 +4320,13 @@ class RequiredStateChangesTestCase(unittest.TestCase): previous_required_state_map={}, request_required_state_map={EventTypes.Member: {StateValues.LAZY}}, state_deltas={(EventTypes.Member, "@user:test"): "$event_id"}, - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( # If a "$LAZY" has been added or removed we always update the # required state to what was requested for simplicity. {EventTypes.Member: {StateValues.LAZY}}, StateFilter.none(), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( {EventTypes.Member: {StateValues.LAZY}}, StateFilter.none(), ), @@ -4334,7 +4341,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): previous_required_state_map={EventTypes.Member: {StateValues.LAZY}}, request_required_state_map={}, state_deltas={(EventTypes.Member, "@user:test"): "$event_id"}, - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( # If a "$LAZY" has been added or removed we always update the # required state to what was requested for simplicity. {}, @@ -4342,7 +4349,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): # less state now StateFilter.none(), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( # `EventTypes.Member` is no longer requested but since that # state hasn't changed, nothing should change (we should still # keep track that we've sent `EventTypes.Member` before). @@ -4361,41 +4368,40 @@ class RequiredStateChangesTestCase(unittest.TestCase): we're sending down another response without any timeline events. """, RequiredStateChangesTestParameters( - previous_required_state_map={ - EventTypes.Member: { - StateValues.LAZY, - "@user2:test", - "@user3:test", - } - }, + previous_required_state_map={EventTypes.Member: {StateValues.LAZY}}, request_required_state_map={EventTypes.Member: {StateValues.LAZY}}, + previously_returned_lazy_user_ids={"@user2:test", "@user3:test"}, + request_lazy_load_user_ids=set(), state_deltas={(EventTypes.Member, "@user2:test"): "$event_id"}, - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( + # The `request_required_state_map` hasn't changed + None, + # We don't need to request anything more if they are requesting + # less state now + StateFilter.none(), + # Previous request did not include any explicit members, + # so there is no extra users to add to the lazy cache. + extra_users_to_add_to_lazy_cache=frozenset(), # Remove "@user2:test" since that state has changed and is no # longer being requested anymore. Since something was removed, # we should persist the changed to required state. That way next # time, they request "@user2:test", we see that we haven't sent # it before and send the new state. (we should still keep track # that we've sent specific `EventTypes.Member` before) - { - EventTypes.Member: { - StateValues.LAZY, - "@user3:test", - } - }, - # We don't need to request anything more if they are requesting - # less state now - StateFilter.none(), + lazy_members_invalidated={"@user2:test"}, ), - expected_without_state_deltas=( - # We're not requesting any specific `EventTypes.Member` now but - # since that state hasn't changed, nothing should change (we - # should still keep track that we've sent specific - # `EventTypes.Member` before). + expected_without_state_deltas=_RequiredStateChangesReturn( + # The `request_required_state_map` hasn't changed None, # We don't need to request anything more if they are requesting # less state now StateFilter.none(), + # Previous request did not include any explicit members, + # so there is no extra users to add to the lazy cache. + extra_users_to_add_to_lazy_cache=frozenset(), + # Nothing should change (we should still keep track that + # we've sent specific `EventTypes.Member` before). + lazy_members_invalidated=frozenset(), ), ), ), @@ -4407,50 +4413,37 @@ class RequiredStateChangesTestCase(unittest.TestCase): we're sending down another response with a new event from user4. """, RequiredStateChangesTestParameters( - previous_required_state_map={ - EventTypes.Member: { - StateValues.LAZY, - "@user2:test", - "@user3:test", - } - }, - request_required_state_map={ - EventTypes.Member: {StateValues.LAZY, "@user4:test"} - }, + previous_required_state_map={EventTypes.Member: {StateValues.LAZY}}, + request_required_state_map={EventTypes.Member: {StateValues.LAZY}}, + previously_returned_lazy_user_ids={"@user2:test", "@user3:test"}, + request_lazy_load_user_ids={"@user4:test"}, state_deltas={(EventTypes.Member, "@user2:test"): "$event_id"}, - expected_with_state_deltas=( - # Since "@user4:test" was added, we should persist the changed - # required state config. - # - # Also remove "@user2:test" since that state has changed and is no - # longer being requested anymore. Since something was removed, - # we also should persist the changed to required state. That way next - # time, they request "@user2:test", we see that we haven't sent - # it before and send the new state. (we should still keep track - # that we've sent specific `EventTypes.Member` before) - { - EventTypes.Member: { - StateValues.LAZY, - "@user3:test", - "@user4:test", - } - }, + expected_with_state_deltas=_RequiredStateChangesReturn( + # The `request_required_state_map` hasn't changed + None, # We should see the new state_keys added StateFilter.from_types([(EventTypes.Member, "@user4:test")]), + # Previous request did not include any explicit members, + # so there is no extra users to add to the lazy cache. + extra_users_to_add_to_lazy_cache=frozenset(), + # Remove "@user2:test" since that state has changed and + # is no longer being requested anymore. Since something + # was removed, we also should persist the changed to + # required state. That way next time, they request + # "@user2:test", we see that we haven't sent it before + # and send the new state. (we should still keep track + # that we've sent specific `EventTypes.Member` before) + lazy_members_invalidated={"@user2:test"}, ), - expected_without_state_deltas=( - # Since "@user4:test" was added, we should persist the changed - # required state config. - { - EventTypes.Member: { - StateValues.LAZY, - "@user2:test", - "@user3:test", - "@user4:test", - } - }, + expected_without_state_deltas=_RequiredStateChangesReturn( + # The `request_required_state_map` hasn't changed + None, # We should see the new state_keys added StateFilter.from_types([(EventTypes.Member, "@user4:test")]), + # Previous request did not include any explicit members, + # so there is no extra users to add to the lazy cache. + extra_users_to_add_to_lazy_cache=frozenset(), + lazy_members_invalidated=frozenset(), ), ), ), @@ -4464,40 +4457,81 @@ class RequiredStateChangesTestCase(unittest.TestCase): EventTypes.Member: {"@user2:test", "@user3:test"} }, request_required_state_map={EventTypes.Member: {StateValues.LAZY}}, + previously_returned_lazy_user_ids=frozenset(), + request_lazy_load_user_ids=frozenset(), + state_deltas={(EventTypes.Member, "@user2:test"): "$event_id"}, + expected_with_state_deltas=_RequiredStateChangesReturn( + # Since `StateValues.LAZY` was added, we should persist the + # changed required state config. + {EventTypes.Member: {StateValues.LAZY}}, + # No users are being lazy loaded, so nothing to request. + StateFilter.none(), + # Remember the fact that we've sent @user3 down before, + # but not @user2 as that has been invalidated. + extra_users_to_add_to_lazy_cache={"@user3:test"}, + # Nothing to invalidate as there are no existing lazy members. + lazy_members_invalidated=frozenset(), + ), + expected_without_state_deltas=_RequiredStateChangesReturn( + # Since `StateValues.LAZY` was added, we should persist the + # changed required state config. + {EventTypes.Member: {StateValues.LAZY}}, + # No users are being lazy loaded, so nothing to request. + StateFilter.none(), + # Remember the fact that we've sent the users down before. + extra_users_to_add_to_lazy_cache={"@user2:test", "@user3:test"}, + # Nothing to invalidate as there are no existing lazy members. + lazy_members_invalidated=frozenset(), + ), + ), + ), + ( + "state_key_expand_lazy_keep_previous_memberships_need_previous_sent", + """ + Test expanding the `required_state` to lazy-loading room + members. If a previously explicit membership is requested then + we should not send it again (as it was already sent before). + """, + RequiredStateChangesTestParameters( + previous_required_state_map={ + EventTypes.Member: {"@user2:test", "@user3:test"} + }, + request_required_state_map={EventTypes.Member: {StateValues.LAZY}}, + previously_returned_lazy_user_ids=frozenset(), + request_lazy_load_user_ids={"@user3:test"}, state_deltas={(EventTypes.Member, "@user2:test"): "$event_id"}, - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( # Since `StateValues.LAZY` was added, we should persist the # changed required state config. + {EventTypes.Member: {StateValues.LAZY}}, + # We have already sent @user3 down before. # - # Also remove "@user2:test" since that state has changed and is no - # longer being requested anymore. Since something was removed, - # we also should persist the changed to required state. That way next - # time, they request "@user2:test", we see that we haven't sent - # it before and send the new state. (we should still keep track - # that we've sent specific `EventTypes.Member` before) - { - EventTypes.Member: { - StateValues.LAZY, - "@user3:test", - } - }, - # We don't need to request anything more if they are requesting - # less state now + # `@user3:test` is required for lazy loading, but we've + # already sent it down before (due to it being in + # `previous_required_state_map`), so we don't need to + # request it again. StateFilter.none(), + # Remember the fact that we've sent @user3 down before, + # but not @user2 as that has been invalidated. + extra_users_to_add_to_lazy_cache={"@user3:test"}, + # Nothing to invalidate as there are no existing lazy members. + lazy_members_invalidated=frozenset(), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( # Since `StateValues.LAZY` was added, we should persist the # changed required state config. - { - EventTypes.Member: { - StateValues.LAZY, - "@user2:test", - "@user3:test", - } - }, - # We don't need to request anything more if they are requesting - # less state now + {EventTypes.Member: {StateValues.LAZY}}, + # We have already sent @user3 down before. + # + # `@user3:test` is required for lazy loading, but we've + # already sent it down before (due to it being in + # `previous_required_state_map`), so we don't need to + # request it again. StateFilter.none(), + # Remember the fact that we've sent the users down before. + extra_users_to_add_to_lazy_cache={"@user2:test", "@user3:test"}, + # Nothing to invalidate as there are no existing lazy members. + lazy_members_invalidated=frozenset(), ), ), ), @@ -4507,36 +4541,33 @@ class RequiredStateChangesTestCase(unittest.TestCase): Test retracting the `required_state` to no longer lazy-loading room members. """, RequiredStateChangesTestParameters( - previous_required_state_map={ - EventTypes.Member: { - StateValues.LAZY, - "@user2:test", - "@user3:test", - } - }, + previous_required_state_map={EventTypes.Member: {StateValues.LAZY}}, request_required_state_map={}, + previously_returned_lazy_user_ids={"@user2:test", "@user3:test"}, + request_lazy_load_user_ids=set(), state_deltas={(EventTypes.Member, "@user2:test"): "$event_id"}, - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( # Remove `EventTypes.Member` since there's been a change to that - # state, (persist the change to required state). That way next - # time, they request `EventTypes.Member`, we see that we haven't - # sent it before and send the new state. (if we were tracking - # that we sent any other state, we should still keep track - # that). - # - # This acts the same as the `simple_remove_type` test. It's - # possible that we could remember the specific `state_keys` that - # we have sent down before but this currently just acts the same - # as if a whole `type` was removed. Perhaps it's good that we - # "garbage collect" and forget what we've sent before for a - # given `type` when the client stops caring about a certain - # `type`. + # state, (persist the change to required state). {}, # We don't need to request anything more if they are requesting # less state now StateFilter.none(), + # Previous request did not include any explicit members, + # so there is no extra users to add to the lazy cache. + extra_users_to_add_to_lazy_cache=frozenset(), + # Explicitly remove the now invalidated @user2:test + # membership. + # + # We don't invalidate @user3:test as that membership + # hasn't changed. We continue to store the existing lazy + # members since they might be useful for future + # requests. (Alternatively, we could invalidate all + # members in the room when the client stops lazy + # loading, but we opt to keep track of them). + lazy_members_invalidated={"@user2:test"}, ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( # `EventTypes.Member` is no longer requested but since that # state hasn't changed, nothing should change (we should still # keep track that we've sent `EventTypes.Member` before). @@ -4544,13 +4575,20 @@ class RequiredStateChangesTestCase(unittest.TestCase): # We don't need to request anything more if they are requesting # less state now StateFilter.none(), + # Previous request did not include any explicit members, + # so there is no extra users to add to the lazy cache. + extra_users_to_add_to_lazy_cache=frozenset(), + # Nothing has been invalidated. + lazy_members_invalidated=frozenset(), ), ), ), ( - "state_key_retract_lazy_keep_previous_memberships_with_new_memberships", + "state_key_retract_lazy_keep_previous_explicit_memberships", """ - Test retracting the `required_state` to no longer lazy-loading room members. + Test removing explicit memberships from the `required_state` + when lazy-loading room members tracks previously sent + memberships. """, RequiredStateChangesTestParameters( previous_required_state_map={ @@ -4560,39 +4598,144 @@ class RequiredStateChangesTestCase(unittest.TestCase): "@user3:test", } }, + request_required_state_map={EventTypes.Member: {StateValues.LAZY}}, + previously_returned_lazy_user_ids=frozenset(), + request_lazy_load_user_ids={"@user3:test"}, + state_deltas={(EventTypes.Member, "@user2:test"): "$event_id"}, + expected_with_state_deltas=_RequiredStateChangesReturn( + # Since an explicit membership was removed, we record + # the new required state config and move them to lazy + # members. + {EventTypes.Member: {StateValues.LAZY}}, + # We have already sent @user3 down before. + # + # `@user3:test` is required for lazy loading, but we've + # already sent it down before (due to it being in + # `previous_required_state_map`), so we don't need to + # request it again. + StateFilter.none(), + # Remember the fact that we've sent @user3 down before, + # but not @user2 as that has been invalidated. + extra_users_to_add_to_lazy_cache={"@user3:test"}, + # Nothing to invalidate as there are no existing lazy members. + lazy_members_invalidated=frozenset(), + ), + expected_without_state_deltas=_RequiredStateChangesReturn( + # While some explicit memberships were removed, there were no + # state changes, so we don't need to persist the new required + # state config yet. + None, + # We have already sent @user3 down before. + # + # `@user3:test` is required for lazy loading, but we've + # already sent it down before (due to it being in + # `previous_required_state_map`), so we don't need to + # request it again. + StateFilter.none(), + # Remember the fact that we've sent the users down before. + extra_users_to_add_to_lazy_cache=frozenset(), + # Nothing to invalidate as there are no existing lazy members. + lazy_members_invalidated=frozenset(), + ), + ), + ), + ( + "state_key_retract_lazy_keep_previous_explicit_me_memberships", + """ + Test removing explicit $ME memberships from the `required_state` + when lazy-loading room members tracks previously sent + memberships. + """, + RequiredStateChangesTestParameters( + previous_required_state_map={ + EventTypes.Member: { + StateValues.LAZY, + StateValues.ME, + "@user2:test", + } + }, + request_required_state_map={EventTypes.Member: {StateValues.LAZY}}, + previously_returned_lazy_user_ids=frozenset(), + request_lazy_load_user_ids={"@user:test"}, + state_deltas={(EventTypes.Member, "@user2:test"): "$event_id"}, + expected_with_state_deltas=_RequiredStateChangesReturn( + # Since an explicit membership was removed, we record + # the new required state config and move them to lazy + # members. + {EventTypes.Member: {StateValues.LAZY}}, + # We have already sent @user down before. + # + # `@user:test` is required for lazy loading, but we've + # already sent it down before (due to `StateValues.ME` + # being in `previous_required_state_map`), so we don't + # need to request it again. + StateFilter.none(), + # Remember the fact that we've sent @user down before, + # but not @user2 as that has been invalidated. + extra_users_to_add_to_lazy_cache={"@user:test"}, + # Nothing to invalidate as there are no existing lazy members. + lazy_members_invalidated=frozenset(), + ), + expected_without_state_deltas=_RequiredStateChangesReturn( + # While some explicit memberships were removed, there were no + # state changes, so we don't need to persist the new required + # state config yet. + None, + # We have already sent @user down before. + # + # `@user:test` is required for lazy loading, but we've + # already sent it down before (due to `StateValues.ME` + # being in `previous_required_state_map`), so we don't + # need to request it again. + StateFilter.none(), + # No relevant state has changed and we don't persist the + # changed required_state_map, so we don't yet move the + # $ME state to the lazy cache. + extra_users_to_add_to_lazy_cache=frozenset(), + # Nothing to invalidate as there are no existing lazy members. + lazy_members_invalidated=frozenset(), + ), + ), + ), + ( + "state_key_retract_lazy_keep_previous_memberships_with_new_memberships", + """ + Test retracting the `required_state` to no longer lazy-loading room members. + """, + RequiredStateChangesTestParameters( + previous_required_state_map={EventTypes.Member: {StateValues.LAZY}}, request_required_state_map={EventTypes.Member: {"@user4:test"}}, + previously_returned_lazy_user_ids={"@user2:test", "@user3:test"}, + request_lazy_load_user_ids=frozenset(), state_deltas={(EventTypes.Member, "@user2:test"): "$event_id"}, - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( # Since "@user4:test" was added, we should persist the changed # required state config. - # + {EventTypes.Member: {"@user4:test"}}, + # We should see the new state_keys added + StateFilter.from_types([(EventTypes.Member, "@user4:test")]), + # Previous request did not include any explicit members, + # so there is no extra users to add to the lazy cache. + extra_users_to_add_to_lazy_cache=frozenset(), # Also remove "@user2:test" since that state has changed and is no # longer being requested anymore. Since something was removed, # we also should persist the changed to required state. That way next # time, they request "@user2:test", we see that we haven't sent # it before and send the new state. (we should still keep track # that we've sent specific `EventTypes.Member` before) - { - EventTypes.Member: { - "@user3:test", - "@user4:test", - } - }, - # We should see the new state_keys added - StateFilter.from_types([(EventTypes.Member, "@user4:test")]), + lazy_members_invalidated={"@user2:test"}, ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( # Since "@user4:test" was added, we should persist the changed # required state config. - { - EventTypes.Member: { - "@user2:test", - "@user3:test", - "@user4:test", - } - }, + {EventTypes.Member: {"@user4:test"}}, # We should see the new state_keys added StateFilter.from_types([(EventTypes.Member, "@user4:test")]), + # Previous request did not include any explicit members, + # so there is no extra users to add to the lazy cache. + extra_users_to_add_to_lazy_cache=frozenset(), + # We don't invalidate user2 as they haven't changed + lazy_members_invalidated=frozenset(), ), ), ), @@ -4613,7 +4756,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): # room required state config to match the request. And since we we're previously # already fetching everything, we don't have to fetch anything now that they've # narrowed. - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( { StateValues.WILDCARD: { "state_key1", @@ -4623,7 +4766,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): }, StateFilter.none(), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( { StateValues.WILDCARD: { "state_key1", @@ -4649,11 +4792,11 @@ class RequiredStateChangesTestCase(unittest.TestCase): }, state_deltas={("type1", "state_key1"): "$event_id"}, # We've added a wildcard, so we persist the change and request everything - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( {StateValues.WILDCARD: {StateValues.WILDCARD}}, StateFilter.all(), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( {StateValues.WILDCARD: {StateValues.WILDCARD}}, StateFilter.all(), ), @@ -4673,7 +4816,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): # request. And since we we're previously already fetching # everything, we don't have to fetch anything now that they've # narrowed. - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( { "type1": { "state_key1", @@ -4683,7 +4826,7 @@ class RequiredStateChangesTestCase(unittest.TestCase): }, StateFilter.none(), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( { "type1": { "state_key1", @@ -4708,11 +4851,11 @@ class RequiredStateChangesTestCase(unittest.TestCase): # update the effective room required state config to match the # request. And we need to request all of the state for that type # because we previously, only sent down a few keys. - expected_with_state_deltas=( + expected_with_state_deltas=_RequiredStateChangesReturn( {"type1": {StateValues.WILDCARD, "state_key2", "state_key3"}}, StateFilter.from_types([("type1", None)]), ), - expected_without_state_deltas=( + expected_without_state_deltas=_RequiredStateChangesReturn( { "type1": { StateValues.WILDCARD, @@ -4734,42 +4877,66 @@ def test_xxx( test_parameters: RequiredStateChangesTestParameters, ) -> None: # Without `state_deltas` - changed_required_state_map, added_state_filter = _required_state_changes( + state_changes = _required_state_changes( user_id="@user:test", prev_required_state_map=test_parameters.previous_required_state_map, request_required_state_map=test_parameters.request_required_state_map, + previously_returned_lazy_user_ids=test_parameters.previously_returned_lazy_user_ids, + request_lazy_load_user_ids=test_parameters.request_lazy_load_user_ids, state_deltas={}, ) self.assertEqual( - changed_required_state_map, - test_parameters.expected_without_state_deltas[0], + state_changes.changed_required_state_map, + test_parameters.expected_without_state_deltas.changed_required_state_map, "changed_required_state_map does not match (without state_deltas)", ) self.assertEqual( - added_state_filter, - test_parameters.expected_without_state_deltas[1], + state_changes.added_state_filter, + test_parameters.expected_without_state_deltas.added_state_filter, "added_state_filter does not match (without state_deltas)", ) + self.assertEqual( + state_changes.lazy_members_invalidated, + test_parameters.expected_without_state_deltas.lazy_members_invalidated, + "lazy_members_invalidated does not match (without state_deltas)", + ) + self.assertEqual( + state_changes.extra_users_to_add_to_lazy_cache, + test_parameters.expected_without_state_deltas.extra_users_to_add_to_lazy_cache, + "lazy_members_previously_returned does not match (without state_deltas)", + ) # With `state_deltas` - changed_required_state_map, added_state_filter = _required_state_changes( + state_changes = _required_state_changes( user_id="@user:test", prev_required_state_map=test_parameters.previous_required_state_map, request_required_state_map=test_parameters.request_required_state_map, + previously_returned_lazy_user_ids=test_parameters.previously_returned_lazy_user_ids, + request_lazy_load_user_ids=test_parameters.request_lazy_load_user_ids, state_deltas=test_parameters.state_deltas, ) self.assertEqual( - changed_required_state_map, - test_parameters.expected_with_state_deltas[0], + state_changes.changed_required_state_map, + test_parameters.expected_with_state_deltas.changed_required_state_map, "changed_required_state_map does not match (with state_deltas)", ) self.assertEqual( - added_state_filter, - test_parameters.expected_with_state_deltas[1], + state_changes.added_state_filter, + test_parameters.expected_with_state_deltas.added_state_filter, "added_state_filter does not match (with state_deltas)", ) + self.assertEqual( + state_changes.lazy_members_invalidated, + test_parameters.expected_with_state_deltas.lazy_members_invalidated, + "lazy_members_invalidated does not match (with state_deltas)", + ) + self.assertEqual( + state_changes.extra_users_to_add_to_lazy_cache, + test_parameters.expected_with_state_deltas.extra_users_to_add_to_lazy_cache, + "lazy_members_previously_returned does not match (with state_deltas)", + ) @parameterized.expand( [ @@ -4805,12 +4972,16 @@ def test_limit_retained_previous_state_keys( } # (function under test) - changed_required_state_map, added_state_filter = _required_state_changes( + state_changes = _required_state_changes( user_id="@user:test", prev_required_state_map=previous_required_state_map, request_required_state_map=request_required_state_map, + previously_returned_lazy_user_ids=frozenset(), + request_lazy_load_user_ids=frozenset(), state_deltas={}, ) + changed_required_state_map = state_changes.changed_required_state_map + assert changed_required_state_map is not None # We should only remember up to the maximum number of state keys @@ -4874,12 +5045,16 @@ def test_request_more_state_keys_than_remember_limit(self) -> None: ) # (function under test) - changed_required_state_map, added_state_filter = _required_state_changes( + state_changes = _required_state_changes( user_id="@user:test", prev_required_state_map=previous_required_state_map, request_required_state_map=request_required_state_map, + previously_returned_lazy_user_ids=frozenset(), + request_lazy_load_user_ids=frozenset(), state_deltas={}, ) + changed_required_state_map = state_changes.changed_required_state_map + assert changed_required_state_map is not None # Should include all of the requested state diff --git a/tests/rest/client/sliding_sync/test_lists_filters.py b/tests/rest/client/sliding_sync/test_lists_filters.py index 3b7b2a16d81..59facea1e0b 100644 --- a/tests/rest/client/sliding_sync/test_lists_filters.py +++ b/tests/rest/client/sliding_sync/test_lists_filters.py @@ -690,7 +690,7 @@ def test_filters_is_encrypted_with_remote_invite_room_no_stripped_state( user1_tok = self.login(user1_id, "pass") # Create a remote invite room without any `unsigned.invite_room_state` - _remote_invite_room_id = self._create_remote_invite_room_for_user( + _remote_invite_room_id, _ = self._create_remote_invite_room_for_user( user1_id, None ) @@ -760,7 +760,7 @@ def test_filters_is_encrypted_with_remote_invite_encrypted_room(self) -> None: # Create a remote invite room with some `unsigned.invite_room_state` # indicating that the room is encrypted. - remote_invite_room_id = self._create_remote_invite_room_for_user( + remote_invite_room_id, _ = self._create_remote_invite_room_for_user( user1_id, [ StrippedStateEvent( @@ -849,7 +849,7 @@ def test_filters_is_encrypted_with_remote_invite_unencrypted_room(self) -> None: # Create a remote invite room with some `unsigned.invite_room_state` # but don't set any room encryption event. - remote_invite_room_id = self._create_remote_invite_room_for_user( + remote_invite_room_id, _ = self._create_remote_invite_room_for_user( user1_id, [ StrippedStateEvent( @@ -1484,7 +1484,7 @@ def test_filters_room_types_with_remote_invite_room_no_stripped_state(self) -> N user1_tok = self.login(user1_id, "pass") # Create a remote invite room without any `unsigned.invite_room_state` - _remote_invite_room_id = self._create_remote_invite_room_for_user( + _remote_invite_room_id, _ = self._create_remote_invite_room_for_user( user1_id, None ) @@ -1554,7 +1554,7 @@ def test_filters_room_types_with_remote_invite_space(self) -> None: # Create a remote invite room with some `unsigned.invite_room_state` indicating # that it is a space room - remote_invite_room_id = self._create_remote_invite_room_for_user( + remote_invite_room_id, _ = self._create_remote_invite_room_for_user( user1_id, [ StrippedStateEvent( @@ -1637,7 +1637,7 @@ def test_filters_room_types_with_remote_invite_normal_room(self) -> None: # Create a remote invite room with some `unsigned.invite_room_state` # but the create event does not specify a room type (normal room) - remote_invite_room_id = self._create_remote_invite_room_for_user( + remote_invite_room_id, _ = self._create_remote_invite_room_for_user( user1_id, [ StrippedStateEvent( diff --git a/tests/rest/client/sliding_sync/test_rooms_required_state.py b/tests/rest/client/sliding_sync/test_rooms_required_state.py index 210280bc488..586b127f8a1 100644 --- a/tests/rest/client/sliding_sync/test_rooms_required_state.py +++ b/tests/rest/client/sliding_sync/test_rooms_required_state.py @@ -23,6 +23,7 @@ from synapse.handlers.sliding_sync import StateValues from synapse.rest.client import knock, login, room, sync from synapse.server import HomeServer +from synapse.storage.databases.main.events import DeltaState, SlidingSyncTableChanges from synapse.util.clock import Clock from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase @@ -642,11 +643,6 @@ def test_rooms_required_state_changed_membership_in_timeline_lazy_loading_room_m self._assertRequiredStateIncludes( response_body["rooms"][room_id1]["required_state"], { - # This appears because *some* membership in the room changed and the - # heroes are recalculated and is thrown in because we have it. But this - # is technically optional and not needed because we've already seen user2 - # in the last sync (and their membership hasn't changed). - state_map[(EventTypes.Member, user2_id)], # Appears because there is a message in the timeline from this user state_map[(EventTypes.Member, user4_id)], # Appears because there is a membership event in the timeline from this user @@ -841,6 +837,437 @@ def test_rooms_required_state_expand_retract_expand_lazy_loading_room_members_in exact=True, ) + def test_lazy_loading_room_members_limited_sync(self) -> None: + """Test that when using lazy loading for room members and a limited sync + missing a membership change, we include the membership change next time + said user says something. + """ + + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id1, user1_id, tok=user1_tok) + + # Send a message from each user to the room so that both memberships are sent down. + self.helper.send(room_id1, "1", tok=user1_tok) + self.helper.send(room_id1, "2", tok=user2_tok) + + # Make a first sync with lazy loading for the room members to establish + # a position + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [ + [EventTypes.Member, StateValues.LAZY], + ], + "timeline_limit": 2, + } + } + } + response_body, from_token = self.do_sync(sync_body, tok=user1_tok) + + # We should see both membership events in required_state + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + state_map[(EventTypes.Member, user1_id)], + state_map[(EventTypes.Member, user2_id)], + }, + exact=True, + ) + + # User2 changes their display name (causing a membership change) + self.helper.send_state( + room_id1, + event_type=EventTypes.Member, + state_key=user2_id, + body={ + EventContentFields.MEMBERSHIP: Membership.JOIN, + EventContentFields.MEMBERSHIP_DISPLAYNAME: "New Name", + }, + tok=user2_tok, + ) + + # Send a couple of messages to the room to push out the membership change + self.helper.send(room_id1, "3", tok=user1_tok) + self.helper.send(room_id1, "4", tok=user1_tok) + + # Make an incremental Sliding Sync request + response_body, from_token = self.do_sync( + sync_body, since=from_token, tok=user1_tok + ) + + # The membership change should *not* be included yet as user2 doesn't + # have any events in the timeline. + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1].get("required_state", []), + set(), + exact=True, + ) + + # Now user2 sends a message to the room + self.helper.send(room_id1, "5", tok=user2_tok) + + # Make another incremental Sliding Sync request + response_body, from_token = self.do_sync( + sync_body, since=from_token, tok=user1_tok + ) + + # The membership change should now be included as user2 has an event + # in the timeline. + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1].get("required_state", []), + { + state_map[(EventTypes.Member, user2_id)], + }, + exact=True, + ) + + def test_lazy_loading_room_members_across_multiple_rooms(self) -> None: + """Test that lazy loading room members are tracked per-room correctly.""" + + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create two rooms with both users in them and send a message in each + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id1, user1_id, tok=user1_tok) + self.helper.send(room_id1, "room1-msg1", tok=user2_tok) + + room_id2 = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id2, user1_id, tok=user1_tok) + self.helper.send(room_id2, "room2-msg1", tok=user2_tok) + + # Make a sync with lazy loading for the room members to establish + # a position + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [ + [EventTypes.Member, StateValues.LAZY], + ], + "timeline_limit": 1, + } + } + } + response_body, from_token = self.do_sync(sync_body, tok=user1_tok) + + # We expect to see only user2's membership in both rooms + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + state_map[(EventTypes.Member, user2_id)], + }, + exact=True, + ) + + # Send a message in room1 from user1 + self.helper.send(room_id1, "room1-msg2", tok=user1_tok) + + # Make an incremental Sliding Sync request and check that we get user1's + # membership. + response_body, from_token = self.do_sync( + sync_body, since=from_token, tok=user1_tok + ) + + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + state_map[(EventTypes.Member, user1_id)], + }, + exact=True, + ) + + # Send a message in room2 from user1 + self.helper.send(room_id2, "room2-msg2", tok=user1_tok) + + # Make an incremental Sliding Sync request and check that we get user1's + # membership. + response_body, from_token = self.do_sync( + sync_body, since=from_token, tok=user1_tok + ) + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id2) + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id2]["required_state"], + { + state_map[(EventTypes.Member, user1_id)], + }, + exact=True, + ) + + def test_lazy_loading_room_members_across_multiple_connections(self) -> None: + """Test that lazy loading room members are tracked per-connection + correctly. + + This catches bugs where if a membership got sent down one connection, + it would incorrectly assume it was sent down another connection. + """ + + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id1, user1_id, tok=user1_tok) + + self.helper.send(room_id1, "1", tok=user2_tok) + + # Make a sync with lazy loading for the room members to establish + # a position + sync_body1 = { + "conn_id": "first-connection", + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [ + [EventTypes.Member, StateValues.LAZY], + ], + "timeline_limit": 1, + } + }, + } + response_body, from_token1 = self.do_sync(sync_body1, tok=user1_tok) + + # We expect to see only user2's membership in the room + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + state_map[(EventTypes.Member, user2_id)], + }, + exact=True, + ) + + # Now make a new connection + sync_body2 = { + "conn_id": "second-connection", + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [ + [EventTypes.Member, StateValues.LAZY], + ], + "timeline_limit": 1, + } + }, + } + response_body, from_token2 = self.do_sync(sync_body2, tok=user1_tok) + + # We should see user2's membership as this is a new connection + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + state_map[(EventTypes.Member, user2_id)], + }, + exact=True, + ) + + # If we send a message from user1 and sync again on the first connection, + # we should get user1's membership + self.helper.send(room_id1, "2", tok=user1_tok) + response_body, from_token1 = self.do_sync( + sync_body1, since=from_token1, tok=user1_tok + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + state_map[(EventTypes.Member, user1_id)], + }, + exact=True, + ) + + # We sync again on the first connection to "ack" the position. This + # triggers the `sliding_sync_connection_lazy_members` to set its + # connection_position to null. + self.do_sync(sync_body1, since=from_token1, tok=user1_tok) + + # If we sync again on the second connection, we should also get user1's + # membership + response_body, _ = self.do_sync(sync_body2, since=from_token2, tok=user1_tok) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + state_map[(EventTypes.Member, user1_id)], + }, + exact=True, + ) + + def test_lazy_loading_room_members_forked_position(self) -> None: + """Test that lazy loading room members are tracked correctly when a + connection position is reused""" + + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id1, user1_id, tok=user1_tok) + + self.helper.send(room_id1, "1", tok=user2_tok) + + # Make a sync with lazy loading for the room members to establish + # a position + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [ + [EventTypes.Member, StateValues.LAZY], + ], + "timeline_limit": 1, + } + } + } + response_body, from_token = self.do_sync(sync_body, tok=user1_tok) + + # We expect to see only user2's membership in the room + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + state_map[(EventTypes.Member, user2_id)], + }, + exact=True, + ) + + # Send a message in room1 from user1 + self.helper.send(room_id1, "2", tok=user1_tok) + + # Make an incremental Sliding Sync request and check that we get user1's + # membership. + response_body, _ = self.do_sync(sync_body, since=from_token, tok=user1_tok) + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + state_map[(EventTypes.Member, user1_id)], + }, + exact=True, + ) + + # Now, reuse the original position and check we still get user1's + # membership. + response_body, _ = self.do_sync(sync_body, since=from_token, tok=user1_tok) + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + state_map[(EventTypes.Member, user1_id)], + }, + exact=True, + ) + + def test_lazy_loading_room_members_explicit_membership_removed(self) -> None: + """Test the case where we requested explicit memberships and then later + changed to lazy loading.""" + + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + room_id1 = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id1, user1_id, tok=user1_tok) + + self.helper.send(room_id1, "1", tok=user2_tok) + + # Make a sync with lazy loading for the room members to establish + # a position + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [ + [EventTypes.Member, StateValues.ME], + ], + "timeline_limit": 1, + } + } + } + response_body, from_token = self.do_sync(sync_body, tok=user1_tok) + + # We expect to see only user1's membership in the room + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + state_map[(EventTypes.Member, user1_id)], + }, + exact=True, + ) + + # Now change to lazy loading... + sync_body["lists"]["foo-list"]["required_state"] = [ + [EventTypes.Member, StateValues.LAZY], + ] + + # Send a message in room1 from user2 + self.helper.send(room_id1, "2", tok=user2_tok) + response_body, from_token = self.do_sync( + sync_body, since=from_token, tok=user1_tok + ) + + # We should see user2's membership as it's in the timeline + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + state_map[(EventTypes.Member, user2_id)], + }, + exact=True, + ) + + # Now send a message in room1 from user1 + self.helper.send(room_id1, "3", tok=user1_tok) + + response_body, _ = self.do_sync(sync_body, since=from_token, tok=user1_tok) + + # We should not see any memberships as we've already seen user1's + # membership. + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1].get("required_state", []), + [], + exact=True, + ) + def test_rooms_required_state_me(self) -> None: """ Test `rooms.required_state` correctly handles $ME. @@ -1686,3 +2113,135 @@ def test_rooms_required_state_expand_deduplicate(self) -> None: # We should not see the room name again, as we have already sent that # down. self.assertIsNone(response_body["rooms"][room_id1].get("required_state")) + + def test_lazy_loading_room_members_state_reset_non_limited_timeline(self) -> None: + """Test that when using lazy-loaded members, if a membership state is + reset to a previous state and the sync is not limited, then we send down + the state reset. + + Regression test as previously we only returned membership relevant to + the timeline and so did not tell clients about state resets for + users who did not send any timeline events. + """ + + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + room_id = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + content = self.helper.join(room_id, user1_id, tok=user1_tok) + first_event_id = content["event_id"] + + # Send a message so that the user1 membership comes down sync (because we're lazy-loading room members) + self.helper.send(room_id, "msg", tok=user1_tok) + + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [ + [EventTypes.Member, StateValues.LAZY], + ], + "timeline_limit": 1, + } + } + } + response_body, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Check that user1 is returned + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id) + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id]["required_state"], + { + state_map[(EventTypes.Member, user1_id)], + }, + exact=True, + ) + + # user1 changes their display name + content = self.helper.send_state( + room_id, + EventTypes.Member, + body={"membership": "join", "displayname": "New display name"}, + state_key=user1_id, + tok=user1_tok, + ) + second_event_id = content["event_id"] + + response_body, from_token = self.do_sync( + sync_body, since=from_token, tok=user1_tok + ) + + # We should see the updated membership state + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id) + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id]["required_state"], + { + state_map[(EventTypes.Member, user1_id)], + }, + exact=True, + ) + self.assertEqual( + response_body["rooms"][room_id]["required_state"][0]["event_id"], + second_event_id, + ) + + # Now, fake a reset the membership state to the first event + persist_event_store = self.hs.get_datastores().persist_events + assert persist_event_store is not None + + self.get_success( + persist_event_store.update_current_state( + room_id, + DeltaState( + to_insert={(EventTypes.Member, user1_id): first_event_id}, + to_delete=[], + ), + # We don't need to worry about sliding sync changes for this test + SlidingSyncTableChanges( + room_id=room_id, + joined_room_bump_stamp_to_fully_insert=None, + joined_room_updates={}, + membership_snapshot_shared_insert_values={}, + to_insert_membership_snapshots=[], + to_delete_membership_snapshots=[], + ), + ) + ) + + # Send a message from *user2* so that user1 wouldn't normally get + # synced. + self.helper.send(room_id, "msg2", tok=user2_tok) + + response_body, from_token = self.do_sync( + sync_body, since=from_token, tok=user1_tok + ) + + # This should be a non-limited sync as there is only one timeline event + # (<= `timeline_limit). This is important as we're specifically testing the non-`limited` + # timeline scenario. And for reference, we don't send down state resets + # on limited timelines when using lazy loaded memberships. + self.assertFalse( + response_body["rooms"][room_id].get("limited", False), + "Expected a non-limited timeline", + ) + + # We should see the reset membership state of user1 + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id) + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id]["required_state"], + { + state_map[(EventTypes.Member, user1_id)], + }, + ) + self.assertEqual( + response_body["rooms"][room_id]["required_state"][0]["event_id"], + first_event_id, + ) diff --git a/tests/rest/client/sliding_sync/test_sliding_sync.py b/tests/rest/client/sliding_sync/test_sliding_sync.py index bcd22d15ca6..ac8dfd37d8d 100644 --- a/tests/rest/client/sliding_sync/test_sliding_sync.py +++ b/tests/rest/client/sliding_sync/test_sliding_sync.py @@ -257,7 +257,7 @@ def _create_remote_invite_room_for_user( invitee_user_id: str, unsigned_invite_room_state: list[StrippedStateEvent] | None, invite_room_id: str | None = None, - ) -> str: + ) -> tuple[str, EventBase]: """ Create a fake invite for a remote room and persist it. @@ -323,11 +323,13 @@ def _create_remote_invite_room_for_user( context = EventContext.for_outlier(self.hs.get_storage_controllers()) persist_controller = self.hs.get_storage_controllers().persistence assert persist_controller is not None - self.get_success(persist_controller.persist_event(invite_event, context)) + persisted_event, _, _ = self.get_success( + persist_controller.persist_event(invite_event, context) + ) self._remote_invite_count += 1 - return invite_room_id + return invite_room_id, persisted_event def _bump_notifier_wait_for_events( self, @@ -763,7 +765,7 @@ def test_invited_to_forgotten_remote_room(self) -> None: user1_tok = self.login(user1_id, "pass") # Create a remote room invite (out-of-band membership) - room_id = self._create_remote_invite_room_for_user(user1_id, None) + room_id, _ = self._create_remote_invite_room_for_user(user1_id, None) # Make the Sliding Sync request sync_body = { diff --git a/tests/storage/test_sliding_sync_tables.py b/tests/storage/test_sliding_sync_tables.py index db31348a8cd..cb9be29c5d7 100644 --- a/tests/storage/test_sliding_sync_tables.py +++ b/tests/storage/test_sliding_sync_tables.py @@ -30,19 +30,23 @@ from synapse.events import EventBase, StrippedStateEvent, make_event_from_dict from synapse.events.snapshot import EventContext from synapse.rest import admin -from synapse.rest.client import login, room +from synapse.rest.client import login, room, sync from synapse.server import HomeServer from synapse.storage.databases.main.events import DeltaState from synapse.storage.databases.main.events_bg_updates import ( _resolve_stale_data_in_sliding_sync_joined_rooms_table, _resolve_stale_data_in_sliding_sync_membership_snapshots_table, ) -from synapse.types import create_requester +from synapse.types import SlidingSyncStreamToken, create_requester +from synapse.types.handlers.sliding_sync import ( + LAZY_MEMBERS_UPDATE_INTERVAL, + StateValues, +) from synapse.types.storage import _BackgroundUpdates from synapse.util.clock import Clock +from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase from tests.test_utils.event_injection import create_event -from tests.unittest import HomeserverTestCase logger = logging.getLogger(__name__) @@ -86,7 +90,7 @@ class _SlidingSyncMembershipSnapshotResult: forgotten: bool = False -class SlidingSyncTablesTestCaseBase(HomeserverTestCase): +class SlidingSyncTablesTestCaseBase(SlidingSyncBase): """ Helpers to deal with testing that the `sliding_sync_joined_rooms`/`sliding_sync_membership_snapshots` database tables are @@ -97,6 +101,7 @@ class SlidingSyncTablesTestCaseBase(HomeserverTestCase): admin.register_servlets, login.register_servlets, room.register_servlets, + sync.register_servlets, ] def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: @@ -202,78 +207,6 @@ def _get_sliding_sync_membership_snapshots( for row in rows } - _remote_invite_count: int = 0 - - def _create_remote_invite_room_for_user( - self, - invitee_user_id: str, - unsigned_invite_room_state: list[StrippedStateEvent] | None, - ) -> tuple[str, EventBase]: - """ - Create a fake invite for a remote room and persist it. - - We don't have any state for these kind of rooms and can only rely on the - stripped state included in the unsigned portion of the invite event to identify - the room. - - Args: - invitee_user_id: The person being invited - unsigned_invite_room_state: List of stripped state events to assist the - receiver in identifying the room. - - Returns: - The room ID of the remote invite room and the persisted remote invite event. - """ - invite_room_id = f"!test_room{self._remote_invite_count}:remote_server" - - invite_event_dict = { - "room_id": invite_room_id, - "sender": "@inviter:remote_server", - "state_key": invitee_user_id, - "depth": 1, - "origin_server_ts": 1, - "type": EventTypes.Member, - "content": {"membership": Membership.INVITE}, - "auth_events": [], - "prev_events": [], - } - if unsigned_invite_room_state is not None: - serialized_stripped_state_events = [] - for stripped_event in unsigned_invite_room_state: - serialized_stripped_state_events.append( - { - "type": stripped_event.type, - "state_key": stripped_event.state_key, - "sender": stripped_event.sender, - "content": stripped_event.content, - } - ) - - invite_event_dict["unsigned"] = { - "invite_room_state": serialized_stripped_state_events - } - - invite_event = make_event_from_dict( - invite_event_dict, - room_version=RoomVersions.V10, - ) - invite_event.internal_metadata.outlier = True - invite_event.internal_metadata.out_of_band_membership = True - - self.get_success( - self.store.maybe_store_room_on_outlier_membership( - room_id=invite_room_id, room_version=invite_event.room_version - ) - ) - context = EventContext.for_outlier(self.hs.get_storage_controllers()) - persisted_event, _, _ = self.get_success( - self.persist_controller.persist_event(invite_event, context) - ) - - self._remote_invite_count += 1 - - return invite_room_id, persisted_event - def _retract_remote_invite_for_user( self, user_id: str, @@ -3052,6 +2985,141 @@ def test_membership_snapshot_missing_forget( exact=True, ) + def test_lazy_loading_room_members_last_seen_ts(self) -> None: + """Test that the `last_seen_ts` column in + `sliding_sync_connection_lazy_members` is correctly kept up to date. + + We expect that it only gets updated every + `LAZY_MEMBERS_UPDATE_INTERVAL`, rather than on every sync. + """ + + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + room_id = self.helper.create_room_as(user2_id, tok=user2_tok, is_public=True) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Send a message so that user1 comes down sync. + self.helper.send(room_id, "msg", tok=user1_tok) + + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [ + [EventTypes.Member, StateValues.LAZY], + ], + "timeline_limit": 1, + } + } + } + response_body, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Check that user1 is returned + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id) + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id]["required_state"], + { + state_map[(EventTypes.Member, user1_id)], + }, + exact=True, + ) + + # Check that we have an entry in sliding_sync_connection_lazy_members + connection_pos1 = self.get_success( + SlidingSyncStreamToken.from_string(self.store, from_token) + ).connection_position + lazy_member_entries = self.get_success( + self.store.get_sliding_sync_connection_lazy_members( + connection_pos1, room_id, {user1_id} + ) + ) + self.assertIn(user1_id, lazy_member_entries) + + prev_timestamp = lazy_member_entries[user1_id] + + # If user1 sends a message then we consider it for lazy loading. We have + # previously returned it so we don't send the state down again, but it + # is still eligible for updating the timestamp. Since we last updated + # the timestamp within the last `LAZY_MEMBERS_UPDATE_INTERVAL`, we do not + # update it. + self.helper.send(room_id, "msg2", tok=user1_tok) + + response_body, from_token = self.do_sync( + sync_body, since=from_token, tok=user1_tok + ) + + # We expect the required_state map to be empty as nothing has changed. + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id) + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id].get("required_state", []), + {}, + exact=True, + ) + + connection_pos2 = self.get_success( + SlidingSyncStreamToken.from_string(self.store, from_token) + ).connection_position + + lazy_member_entries = self.get_success( + self.store.get_sliding_sync_connection_lazy_members( + connection_pos2, room_id, {user1_id} + ) + ) + + # The timestamp should be unchanged. + self.assertEqual(lazy_member_entries[user1_id], prev_timestamp) + + # Now advance the time by `LAZY_MEMBERS_UPDATE_INTERVAL` so that we + # would update the timestamp. + self.reactor.advance(LAZY_MEMBERS_UPDATE_INTERVAL.as_secs()) + + # Send a message from user2 + self.helper.send(room_id, "msg3", tok=user2_tok) + + response_body, from_token = self.do_sync( + sync_body, since=from_token, tok=user1_tok + ) + + connection_pos3 = self.get_success( + SlidingSyncStreamToken.from_string(self.store, from_token) + ).connection_position + + lazy_member_entries = self.get_success( + self.store.get_sliding_sync_connection_lazy_members( + connection_pos3, room_id, {user1_id} + ) + ) + + # The timestamp for user1 should be unchanged, as they were not sent down. + self.assertEqual(lazy_member_entries[user1_id], prev_timestamp) + + # Now if user1 sends a message, then the timestamp should be updated as + # its been over `LAZY_MEMBERS_UPDATE_INTERVAL` since we last updated it. + # (Even though we don't send the state down again). + self.helper.send(room_id, "msg4", tok=user1_tok) + + response_body, from_token = self.do_sync( + sync_body, since=from_token, tok=user1_tok + ) + connection_pos4 = self.get_success( + SlidingSyncStreamToken.from_string(self.store, from_token) + ).connection_position + + lazy_member_entries = self.get_success( + self.store.get_sliding_sync_connection_lazy_members( + connection_pos4, room_id, {user1_id} + ) + ) + # The timestamp for user1 should be updated. + self.assertGreater(lazy_member_entries[user1_id], prev_timestamp) + class SlidingSyncTablesBackgroundUpdatesTestCase(SlidingSyncTablesTestCaseBase): """ From 1f7f16477dc878152febbb74ba7c99abbd438fba Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 12 Dec 2025 11:31:55 +0000 Subject: [PATCH 063/139] Unpin Rust from 1.82.0 (#19302) --- .ci/before_build_wheel.sh | 2 +- .github/workflows/release-artifacts.yml | 2 +- changelog.d/19302.misc | 1 + pyproject.toml | 3 --- 4 files changed, 3 insertions(+), 5 deletions(-) create mode 100644 changelog.d/19302.misc diff --git a/.ci/before_build_wheel.sh b/.ci/before_build_wheel.sh index 56108dcd608..44ca97f31b4 100644 --- a/.ci/before_build_wheel.sh +++ b/.ci/before_build_wheel.sh @@ -7,4 +7,4 @@ if command -v yum &> /dev/null; then fi # Install a Rust toolchain -curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.82.0 -y --profile minimal +curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y --profile minimal diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 531680b989d..33b965d960e 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -5,7 +5,7 @@ name: Build release artifacts on: # we build on PRs and develop to (hopefully) get early warning # of things breaking (but only build one set of debs). PRs skip - # building wheels on macOS & ARM. + # building wheels on ARM. pull_request: push: branches: ["develop", "release-*"] diff --git a/changelog.d/19302.misc b/changelog.d/19302.misc new file mode 100644 index 00000000000..606ab5b52de --- /dev/null +++ b/changelog.d/19302.misc @@ -0,0 +1 @@ +Unpin the version of Rust we use to build Synapse wheels (was 1.82.0) now that MacOS support has been dropped. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 182861ca39b..a9832ccabf2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -471,9 +471,6 @@ skip = "cp3??t-* *i686* *macosx*" enable = "pypy" # We need a rust compiler. -# -# We temporarily pin Rust to 1.82.0 to work around -# https://github.com/element-hq/synapse/issues/17988 before-all = "sh .ci/before_build_wheel.sh" environment= { PATH = "$PATH:$HOME/.cargo/bin" } From 3f636386a66cbc57a6a3c3e641dfd6f1917c838e Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 12 Dec 2025 06:30:21 -0700 Subject: [PATCH 064/139] Add an Admin API endpoint for listing quarantined media (#19268) Co-authored-by: turt2live <1190097+turt2live@users.noreply.github.com> Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/19268.feature | 1 + docs/admin_api/media_admin_api.md | 27 +++++ synapse/media/media_repository.py | 2 + synapse/rest/admin/media.py | 33 ++++++ .../databases/main/media_repository.py | 10 +- synapse/storage/databases/main/room.py | 70 ++++++++++-- .../93/04_add_quarantined_ts_to_media.sql | 27 +++++ tests/rest/admin/test_media.py | 106 ++++++++++++++++++ 8 files changed, 266 insertions(+), 10 deletions(-) create mode 100644 changelog.d/19268.feature create mode 100644 synapse/storage/schema/main/delta/93/04_add_quarantined_ts_to_media.sql diff --git a/changelog.d/19268.feature b/changelog.d/19268.feature new file mode 100644 index 00000000000..cb7035fee2a --- /dev/null +++ b/changelog.d/19268.feature @@ -0,0 +1 @@ +Add an admin API for retrieving a paginated list of quarantined media. \ No newline at end of file diff --git a/docs/admin_api/media_admin_api.md b/docs/admin_api/media_admin_api.md index be72b2e3e22..a3d99cb0743 100644 --- a/docs/admin_api/media_admin_api.md +++ b/docs/admin_api/media_admin_api.md @@ -73,6 +73,33 @@ Response: } ``` +## Listing all quarantined media + +This API returns a list of all quarantined media on the server. It is paginated, and can be scoped to either local or +remote media. Note that the pagination values are also scoped to the request parameters - changing them but keeping the +same pagination values will result in unexpected results. + +Request: +```http +GET /_synapse/admin/v1/media/quarantined?from=0&limit=100&kind=local +``` + +`from` and `limit` are optional parameters, and default to `0` and `100` respectively. They are the row index and number +of rows to return - they are not timestamps. + +`kind` *MUST* either be `local` or `remote`. + +The API returns a JSON body containing MXC URIs for the quarantined media, like the following: + +```json +{ + "media": [ + "mxc://localhost/xwvutsrqponmlkjihgfedcba", + "mxc://localhost/abcdefghijklmnopqrstuvwx" + ] +} +``` + # Quarantine media Quarantining media means that it is marked as inaccessible by users. It applies diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 29c5e66ec49..e84e8423002 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -914,6 +914,7 @@ async def _download_remote_file( filesystem_id=file_id, last_access_ts=time_now_ms, quarantined_by=None, + quarantined_ts=None, authenticated=authenticated, sha256=sha256writer.hexdigest(), ) @@ -1047,6 +1048,7 @@ async def _federation_download_remote_file( filesystem_id=file_id, last_access_ts=time_now_ms, quarantined_by=None, + quarantined_ts=None, authenticated=authenticated, sha256=sha256writer.hexdigest(), ) diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py index d5346fe0d5c..4cfd9da0f92 100644 --- a/synapse/rest/admin/media.py +++ b/synapse/rest/admin/media.py @@ -293,6 +293,38 @@ async def on_GET( return HTTPStatus.OK, {"local": local_mxcs, "remote": remote_mxcs} +class ListQuarantinedMedia(RestServlet): + """Lists all quarantined media on the server.""" + + PATTERNS = admin_patterns("/media/quarantined$") + + def __init__(self, hs: "HomeServer"): + self.store = hs.get_datastores().main + self.auth = hs.get_auth() + + async def on_GET( + self, + request: SynapseRequest, + ) -> tuple[int, JsonDict]: + await assert_requester_is_admin(self.auth, request) + + start = parse_integer(request, "from", default=0) + limit = parse_integer(request, "limit", default=100) + local_or_remote = parse_string(request, "kind", required=True) + + if local_or_remote not in ["local", "remote"]: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "Query parameter `kind` must be either 'local' or 'remote'.", + ) + + mxcs = await self.store.get_quarantined_media_mxcs( + start, limit, local_or_remote == "local" + ) + + return HTTPStatus.OK, {"media": mxcs} + + class PurgeMediaCacheRestServlet(RestServlet): PATTERNS = admin_patterns("/purge_media_cache$") @@ -532,6 +564,7 @@ def register_servlets_for_media_repo(hs: "HomeServer", http_server: HttpServer) ProtectMediaByID(hs).register(http_server) UnprotectMediaByID(hs).register(http_server) ListMediaInRoom(hs).register(http_server) + ListQuarantinedMedia(hs).register(http_server) # XXX DeleteMediaByDateSize must be registered before DeleteMediaByID as # their URL routes overlap. DeleteMediaByDateSize(hs).register(http_server) diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index 50664d63e5f..c27c68fbc28 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -61,6 +61,7 @@ class LocalMedia: url_cache: str | None last_access_ts: int quarantined_by: str | None + quarantined_ts: int | None safe_from_quarantine: bool user_id: str | None authenticated: bool | None @@ -78,6 +79,7 @@ class RemoteMedia: created_ts: int last_access_ts: int quarantined_by: str | None + quarantined_ts: int | None authenticated: bool | None sha256: str | None @@ -243,6 +245,7 @@ async def get_local_media(self, media_id: str) -> LocalMedia | None: "user_id", "authenticated", "sha256", + "quarantined_ts", ), allow_none=True, desc="get_local_media", @@ -262,6 +265,7 @@ async def get_local_media(self, media_id: str) -> LocalMedia | None: user_id=row[8], authenticated=row[9], sha256=row[10], + quarantined_ts=row[11], ) async def get_local_media_by_user_paginate( @@ -319,7 +323,8 @@ def get_local_media_by_user_paginate_txn( safe_from_quarantine, user_id, authenticated, - sha256 + sha256, + quarantined_ts FROM local_media_repository WHERE user_id = ? ORDER BY {order_by_column} {order}, media_id ASC @@ -345,6 +350,7 @@ def get_local_media_by_user_paginate_txn( user_id=row[9], authenticated=row[10], sha256=row[11], + quarantined_ts=row[12], ) for row in txn ] @@ -695,6 +701,7 @@ async def get_cached_remote_media( "quarantined_by", "authenticated", "sha256", + "quarantined_ts", ), allow_none=True, desc="get_cached_remote_media", @@ -713,6 +720,7 @@ async def get_cached_remote_media( quarantined_by=row[6], authenticated=row[7], sha256=row[8], + quarantined_ts=row[9], ) async def store_cached_remote_media( diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 633df077367..182e55743a6 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -945,6 +945,50 @@ def get_retention_policy_for_room_txn( max_lifetime=max_lifetime, ) + async def get_quarantined_media_mxcs( + self, index_start: int, index_limit: int, local: bool + ) -> list[str]: + """Retrieves all the quarantined media MXC URIs starting from the given position, + ordered from oldest quarantined timestamp, then alphabetically by media ID + (including origin). + + Note that on established servers the "quarantined timestamp" may be zero due to + being introduced after the quarantine timestamp field was introduced. + + Args: + index_start: The position to start from. + index_limit: The maximum number of results to return. + local: When true, only local media will be returned. When false, only remote media will be returned. + + Returns: + The quarantined media as a list of media IDs. + """ + + def _get_quarantined_media_mxcs_txn( + txn: LoggingTransaction, + ) -> list[str]: + # We order by quarantined timestamp *and* media ID (including origin, when + # known) to ensure the ordering is stable for established servers. + if local: + sql = "SELECT '' as media_origin, media_id FROM local_media_repository WHERE quarantined_by IS NOT NULL ORDER BY quarantined_ts, media_id ASC LIMIT ? OFFSET ?" + else: + sql = "SELECT media_origin, media_id FROM remote_media_cache WHERE quarantined_by IS NOT NULL ORDER BY quarantined_ts, media_origin, media_id ASC LIMIT ? OFFSET ?" + txn.execute(sql, (index_limit, index_start)) + + mxcs = [] + + for media_origin, media_id in txn: + if local: + media_origin = self.hs.hostname + mxcs.append(f"mxc://{media_origin}/{media_id}") + + return mxcs + + return await self.db_pool.runInteraction( + "get_quarantined_media_mxcs", + _get_quarantined_media_mxcs_txn, + ) + async def get_media_mxcs_in_room(self, room_id: str) -> tuple[list[str], list[str]]: """Retrieves all the local and remote media MXC URIs in a given room @@ -952,7 +996,7 @@ async def get_media_mxcs_in_room(self, room_id: str) -> tuple[list[str], list[st room_id Returns: - The local and remote media as a lists of the media IDs. + The local and remote media as lists of the media IDs. """ def _get_media_mxcs_in_room_txn( @@ -1147,6 +1191,10 @@ def _quarantine_local_media_txn( The total number of media items quarantined """ total_media_quarantined = 0 + now_ts: int | None = self.clock.time_msec() + + if quarantined_by is None: + now_ts = None # Effectively a legacy path, update any media that was explicitly named. if media_ids: @@ -1155,13 +1203,13 @@ def _quarantine_local_media_txn( ) sql = f""" UPDATE local_media_repository - SET quarantined_by = ? + SET quarantined_by = ?, quarantined_ts = ? WHERE {sql_many_clause_sql}""" if quarantined_by is not None: sql += " AND safe_from_quarantine = FALSE" - txn.execute(sql, [quarantined_by] + sql_many_clause_args) + txn.execute(sql, [quarantined_by, now_ts] + sql_many_clause_args) # Note that a rowcount of -1 can be used to indicate no rows were affected. total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 @@ -1172,13 +1220,13 @@ def _quarantine_local_media_txn( ) sql = f""" UPDATE local_media_repository - SET quarantined_by = ? + SET quarantined_by = ?, quarantined_ts = ? WHERE {sql_many_clause_sql}""" if quarantined_by is not None: sql += " AND safe_from_quarantine = FALSE" - txn.execute(sql, [quarantined_by] + sql_many_clause_args) + txn.execute(sql, [quarantined_by, now_ts] + sql_many_clause_args) total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 return total_media_quarantined @@ -1202,6 +1250,10 @@ def _quarantine_remote_media_txn( The total number of media items quarantined """ total_media_quarantined = 0 + now_ts: int | None = self.clock.time_msec() + + if quarantined_by is None: + now_ts = None if media: sql_in_list_clause, sql_args = make_tuple_in_list_sql_clause( @@ -1211,10 +1263,10 @@ def _quarantine_remote_media_txn( ) sql = f""" UPDATE remote_media_cache - SET quarantined_by = ? + SET quarantined_by = ?, quarantined_ts = ? WHERE {sql_in_list_clause}""" - txn.execute(sql, [quarantined_by] + sql_args) + txn.execute(sql, [quarantined_by, now_ts] + sql_args) total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 total_media_quarantined = 0 @@ -1224,9 +1276,9 @@ def _quarantine_remote_media_txn( ) sql = f""" UPDATE remote_media_cache - SET quarantined_by = ? + SET quarantined_by = ?, quarantined_ts = ? WHERE {sql_many_clause_sql}""" - txn.execute(sql, [quarantined_by] + sql_many_clause_args) + txn.execute(sql, [quarantined_by, now_ts] + sql_many_clause_args) total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 return total_media_quarantined diff --git a/synapse/storage/schema/main/delta/93/04_add_quarantined_ts_to_media.sql b/synapse/storage/schema/main/delta/93/04_add_quarantined_ts_to_media.sql new file mode 100644 index 00000000000..18b76804ff6 --- /dev/null +++ b/synapse/storage/schema/main/delta/93/04_add_quarantined_ts_to_media.sql @@ -0,0 +1,27 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 Element Creations, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Add a timestamp for when the sliding sync connection position was last used, +-- only updated with a small granularity. +-- +-- This should be NOT NULL, but we need to consider existing rows. In future we +-- may want to either backfill this or delete all rows with a NULL value (and +-- then make it NOT NULL). +ALTER TABLE local_media_repository ADD COLUMN quarantined_ts BIGINT; +ALTER TABLE remote_media_cache ADD COLUMN quarantined_ts BIGINT; + +UPDATE local_media_repository SET quarantined_ts = 0 WHERE quarantined_by IS NOT NULL; +UPDATE remote_media_cache SET quarantined_ts = 0 WHERE quarantined_by IS NOT NULL; + +-- Note: We *probably* should have an index on quarantined_ts, but we're going +-- to try to defer that to a future migration after seeing the performance impact. diff --git a/tests/rest/admin/test_media.py b/tests/rest/admin/test_media.py index 8cc54cc80c2..e45cc4d2087 100644 --- a/tests/rest/admin/test_media.py +++ b/tests/rest/admin/test_media.py @@ -756,6 +756,112 @@ def _access_media( self.assertFalse(os.path.exists(local_path)) +class ListQuarantinedMediaTestCase(_AdminMediaTests): + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.server_name = hs.hostname + + @parameterized.expand(["local", "remote"]) + def test_no_auth(self, kind: str) -> None: + """ + Try to list quarantined media without authentication. + """ + + channel = self.make_request( + "GET", + "/_synapse/admin/v1/media/quarantined?kind=%s" % (kind,), + ) + + self.assertEqual(401, channel.code, msg=channel.json_body) + self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) + + @parameterized.expand(["local", "remote"]) + def test_requester_is_not_admin(self, kind: str) -> None: + """ + If the user is not a server admin, an error is returned. + """ + self.other_user = self.register_user("user", "pass") + self.other_user_token = self.login("user", "pass") + + channel = self.make_request( + "GET", + "/_synapse/admin/v1/media/quarantined?kind=%s" % (kind,), + access_token=self.other_user_token, + ) + + self.assertEqual(403, channel.code, msg=channel.json_body) + self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) + + def test_list_quarantined_media(self) -> None: + """ + Ensure we actually get results for each page. We can't really test that + remote media is quarantined, but we can test that local media is. + """ + self.admin_user = self.register_user("admin", "pass", admin=True) + self.admin_user_tok = self.login("admin", "pass") + + def _upload() -> str: + return self.helper.upload_media( + SMALL_PNG, tok=self.admin_user_tok, expect_code=200 + )["content_uri"][6:].split("/")[1] # Cut off 'mxc://' and domain + + self.media_id_1 = _upload() + self.media_id_2 = _upload() + self.media_id_3 = _upload() + + def _quarantine(media_id: str) -> None: + channel = self.make_request( + "POST", + "/_synapse/admin/v1/media/quarantine/%s/%s" + % ( + self.server_name, + media_id, + ), + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + + _quarantine(self.media_id_1) + _quarantine(self.media_id_2) + _quarantine(self.media_id_3) + + # Page 1 + channel = self.make_request( + "GET", + "/_synapse/admin/v1/media/quarantined?kind=local&from=0&limit=1", + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(1, len(channel.json_body["media"])) + + # Page 2 + channel = self.make_request( + "GET", + "/_synapse/admin/v1/media/quarantined?kind=local&from=1&limit=1", + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(1, len(channel.json_body["media"])) + + # Page 3 + channel = self.make_request( + "GET", + "/_synapse/admin/v1/media/quarantined?kind=local&from=2&limit=1", + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(1, len(channel.json_body["media"])) + + # Page 4 (no media) + channel = self.make_request( + "GET", + "/_synapse/admin/v1/media/quarantined?kind=local&from=3&limit=1", + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(0, len(channel.json_body["media"])) + + class QuarantineMediaByIDTestCase(_AdminMediaTests): def upload_media_and_return_media_id(self, data: bytes) -> str: # Upload some media into the room From 7347cc436e9632ea9b7abf484f5d5aa99755f656 Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Fri, 12 Dec 2025 14:35:46 +0100 Subject: [PATCH 065/139] Add `memberships` admin API (#19260) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/19260.feature | 1 + docs/admin_api/user_admin_api.md | 49 ++++++++ synapse/rest/admin/__init__.py | 6 +- synapse/rest/admin/users.py | 24 +++- synapse/storage/databases/main/roommember.py | 21 ++++ tests/rest/admin/test_room.py | 114 +++++++++++++++++++ 6 files changed, 212 insertions(+), 3 deletions(-) create mode 100644 changelog.d/19260.feature diff --git a/changelog.d/19260.feature b/changelog.d/19260.feature new file mode 100644 index 00000000000..19b192a009c --- /dev/null +++ b/changelog.d/19260.feature @@ -0,0 +1 @@ +Add `memberships` endpoint to the admin API. This is useful for forensics and T&S purpose. diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index 4de7e856420..5645c6dd1c1 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -505,6 +505,55 @@ with a body of: } ``` +## List room memberships of a user + +Gets a list of room memberships for a specific `user_id`. This +endpoint differs from +[`GET /_synapse/admin/v1/users//joined_rooms`](#list-joined-rooms-of-a-user) +in that it returns rooms with memberships other than "join". + +The API is: + +``` +GET /_synapse/admin/v1/users//memberships +``` + +A response body like the following is returned: + +```json + { + "memberships": { + "!DuGcnbhHGaSZQoNQR:matrix.org": "join", + "!ZtSaPCawyWtxfWiIy:matrix.org": "leave", + } + } +``` + +which is a list of room membership states for the given user. This endpoint can +be used with both local and remote users, with the caveat that the homeserver will +only be aware of the memberships for rooms one of its local users has joined. + +Remote user memberships may also be out of date if all local users have since left +a room. The homeserver will thus no longer receive membership updates about it. + +The list includes rooms that the user has since left; other membership states (knock, +invite, etc.) are also possible. + +Note that rooms will only disappear from this list if they are +[purged](./rooms.md#delete-room-api) from the homeserver. + +**Parameters** + +The following parameters should be set in the URL: + +- `user_id` - fully qualified: for example, `@user:server.com`. + +**Response** + +The following fields are returned in the JSON response body: + +- `memberships` - A map of `room_id` (string) to `membership` state (string). + ## List joined rooms of a user Gets a list of all `room_id` that a specific `user_id` is joined to and is a member of (participating in). diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index e34ebb17e62..fe3eeafd9f9 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -114,7 +114,8 @@ UserByThreePid, UserInvitesCount, UserJoinedRoomCount, - UserMembershipRestServlet, + UserJoinedRoomsRestServlet, + UserMembershipsRestServlet, UserRegisterServlet, UserReplaceMasterCrossSigningKeyRestServlet, UserRestServletV2, @@ -297,7 +298,8 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: VersionServlet(hs).register(http_server) if not auth_delegated: UserAdminServlet(hs).register(http_server) - UserMembershipRestServlet(hs).register(http_server) + UserJoinedRoomsRestServlet(hs).register(http_server) + UserMembershipsRestServlet(hs).register(http_server) if not auth_delegated: UserTokenRestServlet(hs).register(http_server) UserRestServletV2(hs).register(http_server) diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index 42e9f8043d6..406ad8f406f 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -1031,7 +1031,7 @@ async def on_PUT( return HTTPStatus.OK, {} -class UserMembershipRestServlet(RestServlet): +class UserJoinedRoomsRestServlet(RestServlet): """ Get list of joined room ID's for a user. """ @@ -1054,6 +1054,28 @@ async def on_GET( return HTTPStatus.OK, rooms_response +class UserMembershipsRestServlet(RestServlet): + """ + Get list of room memberships for a user. + """ + + PATTERNS = admin_patterns("/users/(?P[^/]*)/memberships$") + + def __init__(self, hs: "HomeServer"): + self.is_mine = hs.is_mine + self.auth = hs.get_auth() + self.store = hs.get_datastores().main + + async def on_GET( + self, request: SynapseRequest, user_id: str + ) -> tuple[int, JsonDict]: + await assert_requester_is_admin(self.auth, request) + + memberships = await self.store.get_memberships_for_user(user_id) + + return HTTPStatus.OK, {"memberships": memberships} + + class PushersRestServlet(RestServlet): """ Gets information about all pushers for a specific `user_id`. diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 9b06ab69fed..7c06080f10a 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -747,6 +747,27 @@ async def get_rooms_user_currently_banned_from( return frozenset(room_ids) + async def get_memberships_for_user(self, user_id: str) -> dict[str, str]: + """Returns a dict of room_id to membership state for a given user. + + If a remote user only returns rooms this server is currently + participating in. + """ + + rows = cast( + list[tuple[str, str]], + await self.db_pool.simple_select_list( + "current_state_events", + keyvalues={ + "type": EventTypes.Member, + "state_key": user_id, + }, + retcols=["room_id", "membership"], + desc="get_memberships_for_user", + ), + ) + return dict(rows) + @cached(max_entries=500000, iterable=True) async def get_rooms_for_user(self, user_id: str) -> frozenset[str]: """Returns a set of room_ids the user is currently joined to. diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index 1c340efa0cd..ad713b4da4d 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -2976,6 +2976,120 @@ def test_join_private_room_if_owner(self) -> None: self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(private_room_id, channel.json_body["joined_rooms"][0]) + def test_joined_rooms(self) -> None: + """ + Test joined_rooms admin endpoint. + """ + + channel = self.make_request( + "POST", + f"/_matrix/client/v3/join/{self.public_room_id}", + content={"user_id": self.second_user_id}, + access_token=self.second_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(self.public_room_id, channel.json_body["room_id"]) + + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.second_user_id}/joined_rooms", + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(self.public_room_id, channel.json_body["joined_rooms"][0]) + + def test_memberships(self) -> None: + """ + Test user memberships admin endpoint. + """ + + channel = self.make_request( + "POST", + f"/_matrix/client/v3/join/{self.public_room_id}", + content={"user_id": self.second_user_id}, + access_token=self.second_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + + other_room_id = self.helper.create_room_as( + self.admin_user, tok=self.admin_user_tok + ) + + channel = self.make_request( + "POST", + f"/_matrix/client/v3/join/{other_room_id}", + content={"user_id": self.second_user_id}, + access_token=self.second_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.second_user_id}/memberships", + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual( + { + "memberships": { + self.public_room_id: Membership.JOIN, + other_room_id: Membership.JOIN, + } + }, + channel.json_body, + ) + + channel = self.make_request( + "POST", + f"/_matrix/client/v3/rooms/{other_room_id}/leave", + content={"user_id": self.second_user_id}, + access_token=self.second_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + + invited_room_id = self.helper.create_room_as( + self.admin_user, tok=self.admin_user_tok + ) + channel = self.make_request( + "POST", + f"/_matrix/client/v3/rooms/{invited_room_id}/invite", + content={"user_id": self.second_user_id}, + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + + banned_room_id = self.helper.create_room_as( + self.admin_user, tok=self.admin_user_tok + ) + channel = self.make_request( + "POST", + f"/_matrix/client/v3/rooms/{banned_room_id}/ban", + content={"user_id": self.second_user_id}, + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/users/{self.second_user_id}/memberships", + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual( + { + "memberships": { + self.public_room_id: Membership.JOIN, + other_room_id: Membership.LEAVE, + invited_room_id: Membership.INVITE, + banned_room_id: Membership.BAN, + } + }, + channel.json_body, + ) + def test_context_as_non_admin(self) -> None: """ Test that, without being admin, one cannot use the context admin API From 048629dd13ffbcbc7603a8c729a4f5a6be20db8a Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 12 Dec 2025 13:36:34 +0000 Subject: [PATCH 066/139] minor grammar fix context: https://github.com/element-hq/synapse/pull/19260#discussion_r2614227743 --- docs/admin_api/user_admin_api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index 5645c6dd1c1..9e0a1cb70ca 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -531,7 +531,7 @@ A response body like the following is returned: which is a list of room membership states for the given user. This endpoint can be used with both local and remote users, with the caveat that the homeserver will -only be aware of the memberships for rooms one of its local users has joined. +only be aware of the memberships for rooms that one of its local users has joined. Remote user memberships may also be out of date if all local users have since left a room. The homeserver will thus no longer receive membership updates about it. From df24e0f30244b1c423f4130d64c6008be341d0b7 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Dec 2025 15:34:13 +0000 Subject: [PATCH 067/139] Fix support for older versions of zope-interface (#19274) Fixes #19269 Versions of zope-interface from RHEL, Ubuntu LTS 22 & 24 and OpenSuse don't support the new python union `X | Y` syntax for interfaces. This PR partially reverts the change over to fully use the new syntax, adds a minimum supported version of zope-interface to Synapse's dependency list, and removes the linter auto-upgrades which prefer the newer syntax. ### Pull Request Checklist * [X] Pull request is based on the develop branch * [X] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [X] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/19274.bugfix | 1 + poetry.lock | 2 +- pyproject.toml | 12 ++++++------ synapse/app/_base.py | 3 ++- synapse/app/admin_cmd.py | 4 ++-- synapse/app/generic_worker.py | 3 ++- synapse/app/homeserver.py | 4 ++-- synapse/handlers/delayed_events.py | 4 ++-- synapse/handlers/message.py | 4 ++-- synapse/handlers/user_directory.py | 4 ++-- synapse/http/client.py | 9 +++++---- synapse/http/federation/matrix_federation_agent.py | 4 ++-- synapse/http/matrixfederationclient.py | 3 ++- synapse/http/proxy.py | 4 ++-- synapse/http/proxyagent.py | 14 +++++++------- synapse/http/replicationagent.py | 3 ++- synapse/logging/_remote.py | 4 ++-- synapse/logging/handlers.py | 4 ++-- synapse/media/_base.py | 3 ++- synapse/push/emailpusher.py | 4 ++-- synapse/push/httppusher.py | 4 ++-- synapse/server.py | 5 +++-- synapse/util/file_consumer.py | 6 +++--- tests/server.py | 10 +++++----- tests/unittest.py | 3 ++- 25 files changed, 65 insertions(+), 56 deletions(-) create mode 100644 changelog.d/19274.bugfix diff --git a/changelog.d/19274.bugfix b/changelog.d/19274.bugfix new file mode 100644 index 00000000000..92aaa0fe6d1 --- /dev/null +++ b/changelog.d/19274.bugfix @@ -0,0 +1 @@ +Fix bug introduced in 1.143.0 that broke support for versions of `zope-interface` older than 6.2. diff --git a/poetry.lock b/poetry.lock index 4dacae38a47..3f8607b6874 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3542,4 +3542,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<4.0.0" -content-hash = "abbbdff591a306b56cc8890dbb2f477ac5f1a2d328baa6409e01084abc655bbf" +content-hash = "1caa5072f6304122c89377420f993a54f54587f3618ccc8094ec31642264592c" diff --git a/pyproject.toml b/pyproject.toml index a9832ccabf2..09ca2a9e77f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,7 +109,12 @@ dependencies = [ "pyrsistent>=0.18.0", # via jsonschema "requests>=2.16.0", # 2.16.0+ no longer vendors urllib3, avoiding Python 3.10+ incompatibility "urllib3>=1.26.5", # via treq; 1.26.5 fixes Python 3.10+ collections.abc compatibility - "zope-interface>=6.2", # via twisted + # 5.2 is the current version in Debian oldstable. If we don't care to support that, then 5.4 is + # the minimum version from Ubuntu 22.04 and RHEL 9. (as of 2025-12) + # When bumping this version to 6.2 or above, refer to https://github.com/element-hq/synapse/pull/19274 + # for details of Synapse improvements that may be unlocked. Particularly around the use of `|` + # syntax with zope interface types. + "zope-interface>=5.2", # via twisted ] [project.optional-dependencies] @@ -383,15 +388,10 @@ select = [ "G", # pyupgrade "UP006", - "UP007", - "UP045", ] extend-safe-fixes = [ # pyupgrade rules compatible with Python >= 3.9 "UP006", - "UP007", - # pyupgrade rules compatible with Python >= 3.10 - "UP045", # Allow ruff to automatically fix trailing spaces within a multi-line string/comment. "W293" ] diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 98d051bf04d..c64c41e9d27 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -36,6 +36,7 @@ Awaitable, Callable, NoReturn, + Optional, cast, ) from wsgiref.simple_server import WSGIServer @@ -455,7 +456,7 @@ def listen_http( root_resource: Resource, version_string: str, max_request_body_size: int, - context_factory: IOpenSSLContextFactory | None, + context_factory: Optional[IOpenSSLContextFactory], reactor: ISynapseReactor = reactor, ) -> list[Port]: """ diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index facc98164e7..0614c805dad 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -24,7 +24,7 @@ import os import sys import tempfile -from typing import Mapping, Sequence +from typing import Mapping, Optional, Sequence from twisted.internet import defer, task @@ -291,7 +291,7 @@ def load_config(argv_options: list[str]) -> tuple[HomeServerConfig, argparse.Nam def create_homeserver( config: HomeServerConfig, - reactor: ISynapseReactor | None = None, + reactor: Optional[ISynapseReactor] = None, ) -> AdminCmdServer: """ Create a homeserver instance for the Synapse admin command process. diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 9939c0fe7d7..0a4abd18393 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -21,6 +21,7 @@ # import logging import sys +from typing import Optional from twisted.web.resource import Resource @@ -335,7 +336,7 @@ def load_config(argv_options: list[str]) -> HomeServerConfig: def create_homeserver( config: HomeServerConfig, - reactor: ISynapseReactor | None = None, + reactor: Optional[ISynapseReactor] = None, ) -> GenericWorkerServer: """ Create a homeserver instance for the Synapse worker process. diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index bd2956d9e18..2b1760416b0 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -22,7 +22,7 @@ import logging import os import sys -from typing import Iterable +from typing import Iterable, Optional from twisted.internet.tcp import Port from twisted.web.resource import EncodingResourceWrapper, Resource @@ -350,7 +350,7 @@ def load_or_generate_config(argv_options: list[str]) -> HomeServerConfig: def create_homeserver( config: HomeServerConfig, - reactor: ISynapseReactor | None = None, + reactor: Optional[ISynapseReactor] = None, ) -> SynapseHomeServer: """ Create a homeserver instance for the Synapse main process. diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index cb0a4dd6b23..c58d1d42bcf 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -13,7 +13,7 @@ # import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from twisted.internet.interfaces import IDelayedCall @@ -74,7 +74,7 @@ def __init__(self, hs: "HomeServer"): cfg=self._config.ratelimiting.rc_delayed_event_mgmt, ) - self._next_delayed_event_call: IDelayedCall | None = None + self._next_delayed_event_call: Optional[IDelayedCall] = None # The current position in the current_state_delta stream self._event_pos: int | None = None diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index a6499de3a87..7808bd68cbf 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -22,7 +22,7 @@ import logging import random from http import HTTPStatus -from typing import TYPE_CHECKING, Any, Mapping, Sequence +from typing import TYPE_CHECKING, Any, Mapping, Optional, Sequence from canonicaljson import encode_canonical_json @@ -111,7 +111,7 @@ def __init__(self, hs: "HomeServer"): # The scheduled call to self._expire_event. None if no call is currently # scheduled. - self._scheduled_expiry: IDelayedCall | None = None + self._scheduled_expiry: Optional[IDelayedCall] = None if not hs.config.worker.worker_app: self.hs.run_as_background_process( diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 36b037e8e1f..e5c4de03c5a 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -21,7 +21,7 @@ import logging from http import HTTPStatus -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from twisted.internet.interfaces import IDelayedCall @@ -125,7 +125,7 @@ def __init__(self, hs: "HomeServer"): # Guard to ensure we only have one process for refreshing remote profiles self._is_refreshing_remote_profiles = False # Handle to cancel the `call_later` of `kick_off_remote_profile_refresh_process` - self._refresh_remote_profiles_call_later: IDelayedCall | None = None + self._refresh_remote_profiles_call_later: Optional[IDelayedCall] = None # Guard to ensure we only have one process for refreshing remote profiles # for the given servers. diff --git a/synapse/http/client.py b/synapse/http/client.py index f0b9201086d..05c5f13a874 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -28,6 +28,7 @@ BinaryIO, Callable, Mapping, + Optional, Protocol, ) @@ -313,7 +314,7 @@ def request( method: bytes, uri: bytes, headers: Headers | None = None, - bodyProducer: IBodyProducer | None = None, + bodyProducer: Optional[IBodyProducer] = None, ) -> defer.Deferred: h = urllib.parse.urlparse(uri.decode("ascii")) @@ -1033,7 +1034,7 @@ class BodyExceededMaxSize(Exception): class _DiscardBodyWithMaxSizeProtocol(protocol.Protocol): """A protocol which immediately errors upon receiving data.""" - transport: ITCPTransport | None = None + transport: Optional[ITCPTransport] = None def __init__(self, deferred: defer.Deferred): self.deferred = deferred @@ -1075,7 +1076,7 @@ class _MultipartParserProtocol(protocol.Protocol): Protocol to read and parse a MSC3916 multipart/mixed response """ - transport: ITCPTransport | None = None + transport: Optional[ITCPTransport] = None def __init__( self, @@ -1188,7 +1189,7 @@ def connectionLost(self, reason: Failure = connectionDone) -> None: class _ReadBodyWithMaxSizeProtocol(protocol.Protocol): """A protocol which reads body to a stream, erroring if the body exceeds a maximum size.""" - transport: ITCPTransport | None = None + transport: Optional[ITCPTransport] = None def __init__( self, stream: ByteWriteable, deferred: defer.Deferred, max_size: int | None diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index c3ba26fe039..a0167659f19 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -19,7 +19,7 @@ # import logging import urllib.parse -from typing import Any, Generator +from typing import Any, Generator, Optional from urllib.request import ( # type: ignore[attr-defined] proxy_bypass_environment, ) @@ -173,7 +173,7 @@ def request( method: bytes, uri: bytes, headers: Headers | None = None, - bodyProducer: IBodyProducer | None = None, + bodyProducer: Optional[IBodyProducer] = None, ) -> Generator[defer.Deferred, Any, IResponse]: """ Args: diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 7090960cfbb..dbd4f1e4b6a 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -33,6 +33,7 @@ Callable, Generic, Literal, + Optional, TextIO, TypeVar, cast, @@ -691,7 +692,7 @@ async def _send_request( destination_bytes, method_bytes, url_to_sign_bytes, json ) data = encode_canonical_json(json) - producer: IBodyProducer | None = QuieterFileBodyProducer( + producer: Optional[IBodyProducer] = QuieterFileBodyProducer( BytesIO(data), cooperator=self._cooperator ) else: diff --git a/synapse/http/proxy.py b/synapse/http/proxy.py index c7f5e39dd89..b3a2f84f29a 100644 --- a/synapse/http/proxy.py +++ b/synapse/http/proxy.py @@ -22,7 +22,7 @@ import json import logging import urllib.parse -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, Optional, cast from twisted.internet import protocol from twisted.internet.interfaces import ITCPTransport @@ -237,7 +237,7 @@ class _ProxyResponseBody(protocol.Protocol): request. """ - transport: ITCPTransport | None = None + transport: Optional[ITCPTransport] = None def __init__(self, request: "SynapseRequest") -> None: self._request = request diff --git a/synapse/http/proxyagent.py b/synapse/http/proxyagent.py index d315ce84758..1f8e58efbc8 100644 --- a/synapse/http/proxyagent.py +++ b/synapse/http/proxyagent.py @@ -21,7 +21,7 @@ import logging import random import re -from typing import Any, Collection, Sequence, cast +from typing import Any, Collection, Optional, Sequence, cast from urllib.parse import urlparse from urllib.request import ( # type: ignore[attr-defined] proxy_bypass_environment, @@ -119,8 +119,8 @@ def __init__( self, *, reactor: IReactorCore, - proxy_reactor: IReactorCore | None = None, - contextFactory: IPolicyForHTTPS | None = None, + proxy_reactor: Optional[IReactorCore] = None, + contextFactory: Optional[IPolicyForHTTPS] = None, connectTimeout: float | None = None, bindAddress: bytes | None = None, pool: HTTPConnectionPool | None = None, @@ -175,7 +175,7 @@ def __init__( self._policy_for_https = contextFactory self._reactor = cast(IReactorTime, reactor) - self._federation_proxy_endpoint: IStreamClientEndpoint | None = None + self._federation_proxy_endpoint: Optional[IStreamClientEndpoint] = None self._federation_proxy_credentials: ProxyCredentials | None = None if federation_proxy_locations: assert federation_proxy_credentials is not None, ( @@ -221,7 +221,7 @@ def request( method: bytes, uri: bytes, headers: Headers | None = None, - bodyProducer: IBodyProducer | None = None, + bodyProducer: Optional[IBodyProducer] = None, ) -> "defer.Deferred[IResponse]": """ Issue a request to the server indicated by the given uri. @@ -365,11 +365,11 @@ def request( def http_proxy_endpoint( proxy: bytes | None, reactor: IReactorCore, - tls_options_factory: IPolicyForHTTPS | None, + tls_options_factory: Optional[IPolicyForHTTPS], timeout: float = 30, bindAddress: bytes | str | tuple[bytes | str, int] | None = None, attemptDelay: float | None = None, -) -> tuple[IStreamClientEndpoint | None, ProxyCredentials | None]: +) -> tuple[Optional[IStreamClientEndpoint], ProxyCredentials | None]: """Parses an http proxy setting and returns an endpoint for the proxy Args: diff --git a/synapse/http/replicationagent.py b/synapse/http/replicationagent.py index 708e4c386be..3d47107cf24 100644 --- a/synapse/http/replicationagent.py +++ b/synapse/http/replicationagent.py @@ -20,6 +20,7 @@ # import logging +from typing import Optional from zope.interface import implementer @@ -149,7 +150,7 @@ def request( method: bytes, uri: bytes, headers: Headers | None = None, - bodyProducer: IBodyProducer | None = None, + bodyProducer: Optional[IBodyProducer] = None, ) -> "defer.Deferred[IResponse]": """ Issue a request to the server indicated by the given uri. diff --git a/synapse/logging/_remote.py b/synapse/logging/_remote.py index e3e0ba4beb7..72faa3c7460 100644 --- a/synapse/logging/_remote.py +++ b/synapse/logging/_remote.py @@ -25,7 +25,7 @@ from collections import deque from ipaddress import IPv4Address, IPv6Address, ip_address from math import floor -from typing import Callable +from typing import Callable, Optional import attr from zope.interface import implementer @@ -113,7 +113,7 @@ def __init__( port: int, maximum_buffer: int = 1000, level: int = logging.NOTSET, - _reactor: IReactorTime | None = None, + _reactor: Optional[IReactorTime] = None, ): super().__init__(level=level) self.host = host diff --git a/synapse/logging/handlers.py b/synapse/logging/handlers.py index 976c7075d41..984d7c22383 100644 --- a/synapse/logging/handlers.py +++ b/synapse/logging/handlers.py @@ -3,7 +3,7 @@ from logging import Handler, LogRecord from logging.handlers import MemoryHandler from threading import Thread -from typing import cast +from typing import Optional, cast from twisted.internet.interfaces import IReactorCore @@ -26,7 +26,7 @@ def __init__( target: Handler | None = None, flushOnClose: bool = True, period: float = 5.0, - reactor: IReactorCore | None = None, + reactor: Optional[IReactorCore] = None, ) -> None: """ period: the period between automatic flushes diff --git a/synapse/media/_base.py b/synapse/media/_base.py index 0fe2e5b529c..78849308766 100644 --- a/synapse/media/_base.py +++ b/synapse/media/_base.py @@ -30,6 +30,7 @@ Awaitable, BinaryIO, Generator, + Optional, ) import attr @@ -705,7 +706,7 @@ def __init__(self, hs: "HomeServer") -> None: self.file: BinaryIO | None = None self.deferred: "Deferred[None]" = Deferred() - self.consumer: interfaces.IConsumer | None = None + self.consumer: Optional[IConsumer] = None # Signals if the thread should keep reading/sending data. Set means # continue, clear means pause. diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index ce4a2102e4d..c44222f6ea1 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -20,7 +20,7 @@ # import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from twisted.internet.error import AlreadyCalled, AlreadyCancelled from twisted.internet.interfaces import IDelayedCall @@ -71,7 +71,7 @@ def __init__(self, hs: "HomeServer", pusher_config: PusherConfig, mailer: Mailer self.server_name = hs.hostname self.store = self.hs.get_datastores().main self.email = pusher_config.pushkey - self.timed_call: IDelayedCall | None = None + self.timed_call: Optional[IDelayedCall] = None self.throttle_params: dict[str, ThrottleParams] = {} self._inited = False diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 1e7e742dddd..fdfae234be7 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -21,7 +21,7 @@ import logging import random import urllib.parse -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from prometheus_client import Counter @@ -120,7 +120,7 @@ def __init__(self, hs: "HomeServer", pusher_config: PusherConfig): self.data = pusher_config.data self.backoff_delay = HttpPusher.INITIAL_BACKOFF_SEC self.failing_since = pusher_config.failing_since - self.timed_call: IDelayedCall | None = None + self.timed_call: Optional[IDelayedCall] = None self._is_processing = False self._group_unread_count_by_room = ( hs.config.push.push_group_unread_count_by_room diff --git a/synapse/server.py b/synapse/server.py index 88662c5b28e..be83a59b885 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -34,6 +34,7 @@ Any, Awaitable, Callable, + Optional, TypeVar, cast, ) @@ -320,7 +321,7 @@ def __init__( self, hostname: str, config: HomeServerConfig, - reactor: ISynapseReactor | None = None, + reactor: Optional[ISynapseReactor] = None, ): """ Args: @@ -353,7 +354,7 @@ def __init__( self._module_web_resources_consumed = False # This attribute is set by the free function `refresh_certificate`. - self.tls_server_context_factory: IOpenSSLContextFactory | None = None + self.tls_server_context_factory: Optional[IOpenSSLContextFactory] = None self._is_shutdown = False self._async_shutdown_handlers: list[ShutdownInfo] = [] diff --git a/synapse/util/file_consumer.py b/synapse/util/file_consumer.py index 8d646840843..c473c524f6f 100644 --- a/synapse/util/file_consumer.py +++ b/synapse/util/file_consumer.py @@ -19,7 +19,7 @@ # import queue -from typing import Any, BinaryIO, cast +from typing import Any, BinaryIO, Optional, Union, cast from twisted.internet import threads from twisted.internet.defer import Deferred @@ -50,7 +50,7 @@ def __init__(self, file_obj: BinaryIO, reactor: ISynapseReactor) -> None: self._reactor: ISynapseReactor = reactor # Producer we're registered with - self._producer: IPushProducer | IPullProducer | None = None + self._producer: Optional[Union[IPushProducer, IPullProducer]] = None # True if PushProducer, false if PullProducer self.streaming = False @@ -72,7 +72,7 @@ def __init__(self, file_obj: BinaryIO, reactor: ISynapseReactor) -> None: self._write_exception: Exception | None = None def registerProducer( - self, producer: IPushProducer | IPullProducer, streaming: bool + self, producer: Union[IPushProducer, IPullProducer], streaming: bool ) -> None: """Part of IConsumer interface diff --git a/tests/server.py b/tests/server.py index ce31a4162ad..d17b2478e3b 100644 --- a/tests/server.py +++ b/tests/server.py @@ -147,7 +147,7 @@ class FakeChannel: _reactor: MemoryReactorClock result: dict = attr.Factory(dict) _ip: str = "127.0.0.1" - _producer: IPullProducer | IPushProducer | None = None + _producer: Optional[Union[IPullProducer, IPushProducer]] = None resource_usage: ContextResourceUsage | None = None _request: Request | None = None @@ -248,7 +248,7 @@ def registerProducer(self, producer: IProducer, streaming: bool) -> None: # TODO This should ensure that the IProducer is an IPushProducer or # IPullProducer, unfortunately twisted.protocols.basic.FileSender does # implement those, but doesn't declare it. - self._producer = cast(IPushProducer | IPullProducer, producer) + self._producer = cast(Union[IPushProducer, IPullProducer], producer) self.producerStreaming = streaming def _produce() -> None: @@ -852,7 +852,7 @@ class FakeTransport: """Test reactor """ - _protocol: IProtocol | None = None + _protocol: Optional[IProtocol] = None """The Protocol which is producing data for this transport. Optional, but if set will get called back for connectionLost() notifications etc. """ @@ -871,7 +871,7 @@ class FakeTransport: disconnected = False connected = True buffer: bytes = b"" - producer: IPushProducer | None = None + producer: Optional[IPushProducer] = None autoflush: bool = True def getPeer(self) -> IPv4Address | IPv6Address: @@ -1073,7 +1073,7 @@ def setup_test_homeserver( cleanup_func: Callable[[Callable[[], Optional["Deferred[None]"]]], None], server_name: str = "test", config: HomeServerConfig | None = None, - reactor: ISynapseReactor | None = None, + reactor: Optional[ISynapseReactor] = None, homeserver_to_use: type[HomeServer] = TestHomeServer, db_txn_limit: int | None = None, **extra_homeserver_attributes: Any, diff --git a/tests/unittest.py b/tests/unittest.py index 7ea29364db2..6022c750d03 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -37,6 +37,7 @@ Iterable, Mapping, NoReturn, + Optional, Protocol, TypeVar, ) @@ -636,7 +637,7 @@ def setup_test_homeserver( self, server_name: str | None = None, config: JsonDict | None = None, - reactor: ISynapseReactor | None = None, + reactor: Optional[ISynapseReactor] = None, clock: Clock | None = None, **extra_homeserver_attributes: Any, ) -> HomeServer: From 466994743ab10a590fc2e70ae8bbba4677049daa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andre=20Kl=C3=A4rner?= Date: Sat, 13 Dec 2025 01:07:39 +0100 Subject: [PATCH 068/139] Document importance of `public_baseurl` for delegation and OIDC (#19270) I just stumbled across the fact that my config used delegation as recommended by the docs, and hosted Synapse on a subdomain. However my config never had `public_baseurl` set and worked without issues, until I just now tried to setup OIDC. OIDC is initialized by the client instructing to open a URL on the homeserver, and initially the correct URL is called, but Synapse does not recognize it without `public_baseurl` being set correctly. After changing this it immediately started working. So in order to prevent anybody from making the same mistake, this adds a small clarifying block in the OIDC docs. --- changelog.d/19270.doc | 1 + docs/openid.md | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 changelog.d/19270.doc diff --git a/changelog.d/19270.doc b/changelog.d/19270.doc new file mode 100644 index 00000000000..fdb7e2e51c7 --- /dev/null +++ b/changelog.d/19270.doc @@ -0,0 +1 @@ +Document the importance of `public_baseurl` when configuring OpenID Connect authentication. diff --git a/docs/openid.md b/docs/openid.md index 819f7543902..e91d375c41f 100644 --- a/docs/openid.md +++ b/docs/openid.md @@ -50,6 +50,11 @@ setting in your configuration file. See the [configuration manual](usage/configuration/config_documentation.md#oidc_providers) for some sample settings, as well as the text below for example configurations for specific providers. +For setups using [`.well-known` delegation](delegate.md), make sure +[`public_baseurl`](usage/configuration/config_documentation.md#public_baseurl) is set +appropriately. If unset, Synapse defaults to `https:///` which is used in +the OIDC callback URL. + ## OIDC Back-Channel Logout Synapse supports receiving [OpenID Connect Back-Channel Logout](https://openid.net/specs/openid-connect-backchannel-1_0.html) notifications. From 0f2b29511fd88d1dc2278f41fd6e4e2f2989fcb7 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 15 Dec 2025 10:23:33 -0700 Subject: [PATCH 069/139] Allow admins to bypass the quarantine check on media downloads (#19275) Co-authored-by: turt2live <1190097+turt2live@users.noreply.github.com> Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/19275.feature | 1 + docs/admin_api/media_admin_api.md | 14 +++ synapse/media/media_repository.py | 22 ++++- synapse/rest/client/media.py | 26 ++++- tests/rest/admin/test_admin.py | 154 +++++++++++++++++++++++++++--- 5 files changed, 197 insertions(+), 20 deletions(-) create mode 100644 changelog.d/19275.feature diff --git a/changelog.d/19275.feature b/changelog.d/19275.feature new file mode 100644 index 00000000000..5147c546cf4 --- /dev/null +++ b/changelog.d/19275.feature @@ -0,0 +1 @@ +Server admins can bypass the quarantine media check when downloading media by setting the `admin_unsafely_bypass_quarantine` query parameter to `true` on Client-Server API media download requests. \ No newline at end of file diff --git a/docs/admin_api/media_admin_api.md b/docs/admin_api/media_admin_api.md index a3d99cb0743..25481a8c552 100644 --- a/docs/admin_api/media_admin_api.md +++ b/docs/admin_api/media_admin_api.md @@ -115,6 +115,20 @@ is quarantined, Synapse will: - Quarantine any existing cached remote media. - Quarantine any future remote media. +## Downloading quarantined media + +Normally, when media is quarantined, it will return a 404 error when downloaded. +Admins can bypass this by adding `?admin_unsafely_bypass_quarantine=true` +to the [normal download URL](https://spec.matrix.org/v1.16/client-server-api/#get_matrixclientv1mediadownloadservernamemediaid). + +Bypassing the quarantine check is not recommended. Media is typically quarantined +to prevent harmful content from being served to users, which includes admins. Only +set the bypass parameter if you intentionally want to access potentially harmful +content. + +Non-admin users cannot bypass quarantine checks, even when specifying the above +query parameter. + ## Quarantining media by ID This API quarantines a single piece of local or remote media. diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index e84e8423002..cb745b96ad4 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -439,7 +439,11 @@ async def get_cached_remote_media_info( return await self.store.get_cached_remote_media(origin, media_id) async def get_local_media_info( - self, request: SynapseRequest, media_id: str, max_timeout_ms: int + self, + request: SynapseRequest, + media_id: str, + max_timeout_ms: int, + bypass_quarantine: bool = False, ) -> LocalMedia | None: """Gets the info dictionary for given local media ID. If the media has not been uploaded yet, this function will wait up to ``max_timeout_ms`` @@ -451,6 +455,7 @@ async def get_local_media_info( the file_id for local content.) max_timeout_ms: the maximum number of milliseconds to wait for the media to be uploaded. + bypass_quarantine: whether to bypass quarantine checks Returns: Either the info dictionary for the given local media ID or @@ -466,7 +471,7 @@ async def get_local_media_info( respond_404(request) return None - if media_info.quarantined_by: + if media_info.quarantined_by and not bypass_quarantine: logger.info("Media %s is quarantined", media_id) respond_404(request) return None @@ -500,6 +505,7 @@ async def get_local_media( max_timeout_ms: int, allow_authenticated: bool = True, federation: bool = False, + bypass_quarantine: bool = False, ) -> None: """Responds to requests for local media, if exists, or returns 404. @@ -513,11 +519,14 @@ async def get_local_media( media to be uploaded. allow_authenticated: whether media marked as authenticated may be served to this request federation: whether the local media being fetched is for a federation request + bypass_quarantine: whether to bypass quarantine checks Returns: Resolves once a response has successfully been written to request """ - media_info = await self.get_local_media_info(request, media_id, max_timeout_ms) + media_info = await self.get_local_media_info( + request, media_id, max_timeout_ms, bypass_quarantine=bypass_quarantine + ) if not media_info: return @@ -561,6 +570,7 @@ async def get_remote_media( ip_address: str, use_federation_endpoint: bool, allow_authenticated: bool = True, + bypass_quarantine: bool = False, ) -> None: """Respond to requests for remote media. @@ -577,6 +587,7 @@ async def get_remote_media( federation `/download` endpoint allow_authenticated: whether media marked as authenticated may be served to this request + bypass_quarantine: whether to bypass quarantine checks Returns: Resolves once a response has successfully been written to request @@ -609,6 +620,7 @@ async def get_remote_media( ip_address, use_federation_endpoint, allow_authenticated, + bypass_quarantine=bypass_quarantine, ) # Check if the media is cached on the client, if so return 304. We need @@ -697,6 +709,7 @@ async def _get_remote_media_impl( ip_address: str, use_federation_endpoint: bool, allow_authenticated: bool, + bypass_quarantine: bool = False, ) -> tuple[Responder | None, RemoteMedia]: """Looks for media in local cache, if not there then attempt to download from remote server. @@ -712,6 +725,7 @@ async def _get_remote_media_impl( ip_address: the IP address of the requester use_federation_endpoint: whether to request the remote media over the new federation /download endpoint + bypass_quarantine: whether to bypass quarantine checks Returns: A tuple of responder and the media info of the file. @@ -732,7 +746,7 @@ async def _get_remote_media_impl( file_id = media_info.filesystem_id file_info = FileInfo(server_name, file_id) - if media_info.quarantined_by: + if media_info.quarantined_by and not bypass_quarantine: logger.info("Media is quarantined") raise NotFoundError() diff --git a/synapse/rest/client/media.py b/synapse/rest/client/media.py index f145b03af45..4db3b015762 100644 --- a/synapse/rest/client/media.py +++ b/synapse/rest/client/media.py @@ -23,6 +23,7 @@ import logging import re +from synapse.api.errors import Codes, cs_error from synapse.http.server import ( HttpServer, respond_with_json, @@ -235,7 +236,23 @@ async def on_GET( # Validate the server name, raising if invalid parse_and_validate_server_name(server_name) - await self.auth.get_user_by_req(request, allow_guest=True) + requester = await self.auth.get_user_by_req(request, allow_guest=True) + is_admin = await self.auth.is_server_admin(requester) + bypass_quarantine = False + if parse_string(request, "admin_unsafely_bypass_quarantine") == "true": + if is_admin: + logger.info("Admin bypassing quarantine for media download") + bypass_quarantine = True + else: + respond_with_json( + request, + 400, + cs_error( + "Must be a server admin to bypass quarantine", + code=Codes.UNKNOWN, + ), + send_cors=True, + ) set_cors_headers(request) set_corp_headers(request) @@ -259,7 +276,11 @@ async def on_GET( if self._is_mine_server_name(server_name): await self.media_repo.get_local_media( - request, media_id, file_name, max_timeout_ms + request, + media_id, + file_name, + max_timeout_ms, + bypass_quarantine=bypass_quarantine, ) else: ip_address = request.getClientAddress().host @@ -271,6 +292,7 @@ async def on_GET( max_timeout_ms, ip_address, True, + bypass_quarantine=bypass_quarantine, ) diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py index f3740a8e35e..77d824dcd84 100644 --- a/tests/rest/admin/test_admin.py +++ b/tests/rest/admin/test_admin.py @@ -71,14 +71,43 @@ def create_resource_dict(self) -> dict[str, Resource]: return resources def _ensure_quarantined( - self, admin_user_tok: str, server_and_media_id: str + self, + user_tok: str, + server_and_media_id: str, + include_bypass_param: bool = False, ) -> None: - """Ensure a piece of media is quarantined when trying to access it.""" + """Ensure a piece of media is quarantined when trying to access it. + + The include_bypass_param flag enables the presence of the + admin_unsafely_bypass_quarantine query parameter, but still expects that the + request will fail to download the media. + """ + if include_bypass_param: + query_string = "?admin_unsafely_bypass_quarantine=true" + channel = self.make_request( + "GET", + f"/_matrix/client/v1/media/download/{server_and_media_id}{query_string}", + shorthand=False, + access_token=user_tok, + ) + + # Non-admins can't bypass, so this should fail regardless of whether the + # media is actually quarantined. + self.assertEqual( + 400, + channel.code, + msg=( + "Expected to receive a 400 when bypassing quarantined media: %s" + % server_and_media_id + ), + ) + + # Repeat the request, this time without the bypass parameter. channel = self.make_request( "GET", f"/_matrix/client/v1/media/download/{server_and_media_id}", shorthand=False, - access_token=admin_user_tok, + access_token=user_tok, ) # Should be quarantined @@ -91,6 +120,62 @@ def _ensure_quarantined( ), ) + def test_admin_can_bypass_quarantine(self) -> None: + self.register_user("admin", "pass", admin=True) + admin_user_tok = self.login("admin", "pass") + + # Upload some media + response = self.helper.upload_media(SMALL_PNG, tok=admin_user_tok) + + # Extract media ID from the response + server_name_and_media_id = response["content_uri"][6:] # Cut off 'mxc://' + server_name, media_id = server_name_and_media_id.split("/") + + # Attempt to access the media + channel = self.make_request( + "GET", + f"/_matrix/client/v1/media/download/{server_name_and_media_id}", + shorthand=False, + access_token=admin_user_tok, + ) + + # Should be successful + self.assertEqual(200, channel.code) + + # Quarantine the media + url = "/_synapse/admin/v1/media/quarantine/%s/%s" % ( + urllib.parse.quote(server_name), + urllib.parse.quote(media_id), + ) + channel = self.make_request( + "POST", + url, + access_token=admin_user_tok, + ) + self.pump(1.0) + self.assertEqual(200, channel.code, msg=channel.json_body) + + # Now access it *without* the bypass parameter - this should fail (as expected). + self._ensure_quarantined( + admin_user_tok, server_name_and_media_id, include_bypass_param=False + ) + + # Now access it *with* the bypass parameter - this should work + channel = self.make_request( + "GET", + f"/_matrix/client/v1/media/download/{server_name_and_media_id}?admin_unsafely_bypass_quarantine=true", + shorthand=False, + access_token=admin_user_tok, + ) + self.assertEqual( + 200, + channel.code, + msg=( + "Expected to receive a 200 on accessing (with bypass) quarantined media: %s" + % server_name_and_media_id + ), + ) + @parameterized.expand( [ # Attempt quarantine media APIs as non-admin @@ -154,8 +239,14 @@ def test_quarantine_media_by_id(self) -> None: self.pump(1.0) self.assertEqual(200, channel.code, msg=channel.json_body) - # Attempt to access the media - self._ensure_quarantined(admin_user_tok, server_name_and_media_id) + # Attempt to access the media (and ensure non-admins can't download it, even + # with a bypass parameter). Admins cannot download it without the bypass param. + self._ensure_quarantined( + non_admin_user_tok, server_name_and_media_id, include_bypass_param=True + ) + self._ensure_quarantined( + admin_user_tok, server_name_and_media_id, include_bypass_param=False + ) @parameterized.expand( [ @@ -214,9 +305,21 @@ def test_quarantine_all_media_in_room(self, url: str) -> None: server_and_media_id_1 = mxc_1[6:] server_and_media_id_2 = mxc_2[6:] - # Test that we cannot download any of the media anymore - self._ensure_quarantined(admin_user_tok, server_and_media_id_1) - self._ensure_quarantined(admin_user_tok, server_and_media_id_2) + # Test that we cannot download any of the media anymore, especially with the + # bypass parameter set. Admins cannot download the media without supplying the + # bypass parameter, so we check that too. + self._ensure_quarantined( + non_admin_user_tok, server_and_media_id_1, include_bypass_param=True + ) + self._ensure_quarantined( + non_admin_user_tok, server_and_media_id_2, include_bypass_param=True + ) + self._ensure_quarantined( + admin_user_tok, server_and_media_id_1, include_bypass_param=False + ) + self._ensure_quarantined( + admin_user_tok, server_and_media_id_2, include_bypass_param=False + ) def test_quarantine_all_media_by_user(self) -> None: self.register_user("user_admin", "pass", admin=True) @@ -263,10 +366,27 @@ def test_quarantine_all_media_by_user(self) -> None: channel.json_body, {"num_quarantined": 3}, "Expected 3 quarantined items" ) - # Attempt to access each piece of media - self._ensure_quarantined(admin_user_tok, server_and_media_id_1) - self._ensure_quarantined(admin_user_tok, server_and_media_id_2) - self._ensure_quarantined(admin_user_tok, server_and_media_id_3) + # Attempt to access each piece of media, ensuring that it can't be downloaded + # even with a bypass parameter. Admins should not be able to download the media + # either when not supplying the bypass parameter, so we check that too. + self._ensure_quarantined( + non_admin_user_tok, server_and_media_id_1, include_bypass_param=True + ) + self._ensure_quarantined( + non_admin_user_tok, server_and_media_id_2, include_bypass_param=True + ) + self._ensure_quarantined( + non_admin_user_tok, server_and_media_id_3, include_bypass_param=True + ) + self._ensure_quarantined( + admin_user_tok, server_and_media_id_1, include_bypass_param=False + ) + self._ensure_quarantined( + admin_user_tok, server_and_media_id_2, include_bypass_param=False + ) + self._ensure_quarantined( + admin_user_tok, server_and_media_id_3, include_bypass_param=False + ) def test_cannot_quarantine_safe_media(self) -> None: self.register_user("user_admin", "pass", admin=True) @@ -307,8 +427,14 @@ def test_cannot_quarantine_safe_media(self) -> None: ) # Attempt to access each piece of media, the first should fail, the - # second should succeed. - self._ensure_quarantined(admin_user_tok, server_and_media_id_1) + # second should succeed. We check both the non-admin user with a bypass + # parameter, and the admin user without. + self._ensure_quarantined( + non_admin_user_tok, server_and_media_id_1, include_bypass_param=True + ) + self._ensure_quarantined( + admin_user_tok, server_and_media_id_1, include_bypass_param=False + ) # Attempt to access each piece of media channel = self.make_request( From 29fd0116a5e4dbae797c02a9c731de3c92c899bc Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Tue, 16 Dec 2025 12:06:07 +0100 Subject: [PATCH 070/139] Improve proxy support for the federation_client.py dev script (#19300) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/19300.feature | 1 + scripts-dev/federation_client.py | 34 ++++++++++++++++++++++++++++---- 2 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 changelog.d/19300.feature diff --git a/changelog.d/19300.feature b/changelog.d/19300.feature new file mode 100644 index 00000000000..97e43e9b28f --- /dev/null +++ b/changelog.d/19300.feature @@ -0,0 +1 @@ +Improve proxy support for the `federation_client.py` dev script. Contributed by Denis Kasak (@dkasak). diff --git a/scripts-dev/federation_client.py b/scripts-dev/federation_client.py index 0fefc23b221..cb14f357cb2 100755 --- a/scripts-dev/federation_client.py +++ b/scripts-dev/federation_client.py @@ -145,7 +145,7 @@ def request( print("Requesting %s" % dest, file=sys.stderr) s = requests.Session() - s.mount("matrix-federation://", MatrixConnectionAdapter()) + s.mount("matrix-federation://", MatrixConnectionAdapter(verify_tls=verify_tls)) headers: dict[str, str] = { "Authorization": authorization_headers[0], @@ -267,6 +267,17 @@ def read_args_from_config(args: argparse.Namespace) -> None: class MatrixConnectionAdapter(HTTPAdapter): + """ + A Matrix federation-aware HTTP Adapter. + """ + + verify_tls: bool + """whether to verify the remote server's TLS certificate.""" + + def __init__(self, verify_tls: bool = True) -> None: + self.verify_tls = verify_tls + super().__init__() + def send( self, request: PreparedRequest, @@ -280,7 +291,7 @@ def send( assert isinstance(request.url, str) parsed = urlparse.urlsplit(request.url) server_name = parsed.netloc - well_known = self._get_well_known(parsed.netloc) + well_known = self._get_well_known(parsed.netloc, verify_tls=self.verify_tls) if well_known: server_name = well_known @@ -318,6 +329,21 @@ def get_connection_with_tls_context( print( f"Connecting to {host}:{port} with SNI {ssl_server_name}", file=sys.stderr ) + + if proxies: + scheme = parsed.scheme + if isinstance(scheme, bytes): + scheme = scheme.decode("utf-8") + + proxy_for_scheme = proxies.get(scheme) + if proxy_for_scheme: + return self.proxy_manager_for(proxy_for_scheme).connection_from_host( + host, + port=port, + scheme="https", + pool_kwargs={"server_hostname": ssl_server_name}, + ) + return self.poolmanager.connection_from_host( host, port=port, @@ -368,7 +394,7 @@ def _lookup(server_name: str) -> tuple[str, int, str]: return server_name, 8448, server_name @staticmethod - def _get_well_known(server_name: str) -> str | None: + def _get_well_known(server_name: str, verify_tls: bool = True) -> str | None: if ":" in server_name: # explicit port, or ipv6 literal. Either way, no .well-known return None @@ -379,7 +405,7 @@ def _get_well_known(server_name: str) -> str | None: print(f"fetching {uri}", file=sys.stderr) try: - resp = requests.get(uri) + resp = requests.get(uri, verify=verify_tls) if resp.status_code != 200: print("%s gave %i" % (uri, resp.status_code), file=sys.stderr) return None From 0395b71e253b0287e17c144f08d7605c10c104b0 Mon Sep 17 00:00:00 2001 From: Joshua Goins Date: Tue, 16 Dec 2025 08:02:29 -0500 Subject: [PATCH 071/139] Fix Mastodon URL previews not showing anything useful (#19231) Fixes #18444. Inside of UrlPreviewer, we need to combine two dicts (one from oEmbed, and one from OpenGraph metadata in the HTML) and in Mastodon's case they were very different. Single Page Applications (SPAs) seem to sometimes provide better information in the OpenGraph tags than the oEmbed stubs, because the oEmbed stubs are filled in with JavaScript that Synapse does not execute. This change improves previews on Mastodon and YouTube (for the same reason). Tested to not regress previews of Twitter or GitHub. --- changelog.d/19231.bugfix | 1 + synapse/media/url_previewer.py | 14 ++++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 changelog.d/19231.bugfix diff --git a/changelog.d/19231.bugfix b/changelog.d/19231.bugfix new file mode 100644 index 00000000000..580b642bb23 --- /dev/null +++ b/changelog.d/19231.bugfix @@ -0,0 +1 @@ +Fix a bug where Mastodon posts (and possibly other embeds) have the wrong description for URL previews. diff --git a/synapse/media/url_previewer.py b/synapse/media/url_previewer.py index 2c5e518918b..7782905a7ab 100644 --- a/synapse/media/url_previewer.py +++ b/synapse/media/url_previewer.py @@ -331,10 +331,16 @@ async def _do_preview(self, url: str, user: UserID, ts: int) -> bytes: # response failed or is incomplete. og_from_html = parse_html_to_open_graph(tree) - # Compile the Open Graph response by using the scraped - # information from the HTML and overlaying any information - # from the oEmbed response. - og = {**og_from_html, **og_from_oembed} + # Compile an Open Graph response by combining the oEmbed response + # and the information from the HTML, with information in the HTML + # preferred. + # + # The ordering here is intentional: certain websites (especially + # SPA JavaScript-based ones) including Mastodon and YouTube provide + # almost complete OpenGraph descriptions but only stubs for oEmbed, + # with further oEmbed information being populated with JavaScript, + # that Synapse won't execute. + og = og_from_oembed | og_from_html await self._precache_image_url(user, media_info, og) else: From 3989d22a37c9291134edeb7453f4e2a617ee57cb Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 16 Dec 2025 17:24:36 +0200 Subject: [PATCH 072/139] Implement pagination for MSC2666 (#19279) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/19279.feature | 1 + synapse/rest/client/mutual_rooms.py | 79 +++++++++++++++++++++----- tests/rest/client/test_mutual_rooms.py | 65 ++++++++++++++++++++- tests/utils.py | 2 + 4 files changed, 131 insertions(+), 16 deletions(-) create mode 100644 changelog.d/19279.feature diff --git a/changelog.d/19279.feature b/changelog.d/19279.feature new file mode 100644 index 00000000000..031e48dcebf --- /dev/null +++ b/changelog.d/19279.feature @@ -0,0 +1 @@ +Implemented pagination for the [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666) mutual rooms endpoint. Contributed by @tulir @ Beeper. diff --git a/synapse/rest/client/mutual_rooms.py b/synapse/rest/client/mutual_rooms.py index 3e5316c4b7f..a6a913db34d 100644 --- a/synapse/rest/client/mutual_rooms.py +++ b/synapse/rest/client/mutual_rooms.py @@ -19,9 +19,12 @@ # # import logging +from bisect import bisect from http import HTTPStatus from typing import TYPE_CHECKING +from unpaddedbase64 import decode_base64, encode_base64 + from synapse.api.errors import Codes, SynapseError from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_strings_from_args @@ -35,10 +38,34 @@ logger = logging.getLogger(__name__) +MUTUAL_ROOMS_BATCH_LIMIT = 100 + + +def _parse_mutual_rooms_batch_token_args(args: dict[bytes, list[bytes]]) -> str | None: + from_batches = parse_strings_from_args(args, "from") + if not from_batches: + return None + if len(from_batches) > 1: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "Duplicate from query parameter", + errcode=Codes.INVALID_PARAM, + ) + if from_batches[0]: + try: + return decode_base64(from_batches[0]).decode("utf-8") + except Exception: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "Malformed from token", + errcode=Codes.INVALID_PARAM, + ) + return None + class UserMutualRoomsServlet(RestServlet): """ - GET /uk.half-shot.msc2666/user/mutual_rooms?user_id={user_id} HTTP/1.1 + GET /uk.half-shot.msc2666/user/mutual_rooms?user_id={user_id}&from={token} HTTP/1.1 """ PATTERNS = client_patterns( @@ -56,6 +83,7 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: args: dict[bytes, list[bytes]] = request.args # type: ignore user_ids = parse_strings_from_args(args, "user_id", required=True) + from_batch = _parse_mutual_rooms_batch_token_args(args) if len(user_ids) > 1: raise SynapseError( @@ -64,29 +92,52 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: errcode=Codes.INVALID_PARAM, ) - # We don't do batching, so a batch token is illegal by default - if b"batch_token" in args: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Unknown batch_token", - errcode=Codes.INVALID_PARAM, - ) - user_id = user_ids[0] requester = await self.auth.get_user_by_req(request) if user_id == requester.user.to_string(): raise SynapseError( - HTTPStatus.UNPROCESSABLE_ENTITY, + HTTPStatus.BAD_REQUEST, "You cannot request a list of shared rooms with yourself", - errcode=Codes.INVALID_PARAM, + errcode=Codes.UNKNOWN, ) - rooms = await self.store.get_mutual_rooms_between_users( - frozenset((requester.user.to_string(), user_id)) + # Sort here instead of the database function, so that we don't expose + # clients to any unrelated changes to the sorting algorithm. + rooms = sorted( + await self.store.get_mutual_rooms_between_users( + frozenset((requester.user.to_string(), user_id)) + ) ) - return 200, {"joined": list(rooms)} + if from_batch: + # A from_batch token was provided, so cut off any rooms where the ID is + # lower than or equal to the token. This method doesn't care whether the + # provided token room still exists, nor whether it's even a real room ID. + # + # However, if rooms with a lower ID are added after the token was issued, + # they will not be included until the client makes a new request without a + # from token. This is considered acceptable, as clients generally won't + # persist these results for long periods. + rooms = rooms[bisect(rooms, from_batch) :] + + if len(rooms) <= MUTUAL_ROOMS_BATCH_LIMIT: + # We've reached the end of the list, don't return a batch token + return 200, {"joined": rooms} + + rooms = rooms[:MUTUAL_ROOMS_BATCH_LIMIT] + # We use urlsafe unpadded base64 encoding for the batch token in order to + # handle funny room IDs in old pre-v12 rooms properly. We also truncate it + # to stay within the 255-character limit of opaque tokens. + next_batch = encode_base64(rooms[-1].encode("utf-8"), urlsafe=True)[:255] + # Due to the truncation, it is technically possible to have conflicting next + # batches by creating hundreds of rooms with the same 191 character prefix + # in the room ID. In the event that some silly user does that, don't let + # them paginate further. + if next_batch == from_batch: + return 200, {"joined": rooms} + + return 200, {"joined": list(rooms), "next_batch": next_batch} def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: diff --git a/tests/rest/client/test_mutual_rooms.py b/tests/rest/client/test_mutual_rooms.py index ea063707aa3..f78c67fcd97 100644 --- a/tests/rest/client/test_mutual_rooms.py +++ b/tests/rest/client/test_mutual_rooms.py @@ -55,12 +55,16 @@ def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.store = hs.get_datastores().main + mutual_rooms.MUTUAL_ROOMS_BATCH_LIMIT = 10 - def _get_mutual_rooms(self, token: str, other_user: str) -> FakeChannel: + def _get_mutual_rooms( + self, token: str, other_user: str, since_token: str | None = None + ) -> FakeChannel: return self.make_request( "GET", "/_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms" - f"?user_id={quote(other_user)}", + f"?user_id={quote(other_user)}" + + (f"&from={quote(since_token)}" if since_token else ""), access_token=token, ) @@ -141,6 +145,52 @@ def _check_mutual_rooms_with( for room_id_id in channel.json_body["joined"]: self.assertIn(room_id_id, [room_id_one, room_id_two]) + def _create_rooms_for_pagination_test( + self, count: int + ) -> tuple[str, str, list[str]]: + u1 = self.register_user("user1", "pass") + u1_token = self.login(u1, "pass") + u2 = self.register_user("user2", "pass") + u2_token = self.login(u2, "pass") + room_ids = [] + for i in range(count): + room_id = self.helper.create_room_as(u1, is_public=i % 2 == 0, tok=u1_token) + self.helper.invite(room_id, src=u1, targ=u2, tok=u1_token) + self.helper.join(room_id, user=u2, tok=u2_token) + room_ids.append(room_id) + room_ids.sort() + return u1_token, u2, room_ids + + def test_shared_room_list_pagination_two_pages(self) -> None: + u1_token, u2, room_ids = self._create_rooms_for_pagination_test(15) + + channel = self._get_mutual_rooms(u1_token, u2) + self.assertEqual(200, channel.code, channel.result) + self.assertEqual(channel.json_body["joined"], room_ids[0:10]) + self.assertIn("next_batch", channel.json_body) + + channel = self._get_mutual_rooms(u1_token, u2, channel.json_body["next_batch"]) + self.assertEqual(200, channel.code, channel.result) + self.assertEqual(channel.json_body["joined"], room_ids[10:20]) + self.assertNotIn("next_batch", channel.json_body) + + def test_shared_room_list_pagination_one_page(self) -> None: + u1_token, u2, room_ids = self._create_rooms_for_pagination_test(10) + + channel = self._get_mutual_rooms(u1_token, u2) + self.assertEqual(200, channel.code, channel.result) + self.assertEqual(channel.json_body["joined"], room_ids) + self.assertNotIn("next_batch", channel.json_body) + + def test_shared_room_list_pagination_invalid_token(self) -> None: + u1_token, u2, room_ids = self._create_rooms_for_pagination_test(10) + + channel = self._get_mutual_rooms(u1_token, u2, "!<>##faketoken") + self.assertEqual(400, channel.code, channel.result) + self.assertEqual( + "M_INVALID_PARAM", channel.json_body["errcode"], channel.result + ) + def test_shared_room_list_after_leave(self) -> None: """ A room should no longer be considered shared if the other @@ -172,3 +222,14 @@ def test_shared_room_list_after_leave(self) -> None: channel = self._get_mutual_rooms(u2_token, u1) self.assertEqual(200, channel.code, channel.result) self.assertEqual(len(channel.json_body["joined"]), 0) + + def test_shared_room_list_nonexistent_user(self) -> None: + u1 = self.register_user("user1", "pass") + u1_token = self.login(u1, "pass") + + # Check shared rooms from user1's perspective. + # We should see the one room in common + channel = self._get_mutual_rooms(u1_token, "@meow:example.com") + self.assertEqual(200, channel.code, channel.result) + self.assertEqual(len(channel.json_body["joined"]), 0) + self.assertNotIn("next_batch", channel.json_body) diff --git a/tests/utils.py b/tests/utils.py index 4052c9a4fbc..0cf97a7e8d6 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -198,7 +198,9 @@ def default_config( "rc_invites": { "per_room": {"per_second": 10000, "burst_count": 10000}, "per_user": {"per_second": 10000, "burst_count": 10000}, + "per_issuer": {"per_second": 10000, "burst_count": 10000}, }, + "rc_room_creation": {"per_second": 10000, "burst_count": 10000}, "rc_3pid_validation": {"per_second": 10000, "burst_count": 10000}, "rc_presence": {"per_user": {"per_second": 10000, "burst_count": 10000}}, "saml2_enabled": False, From f4320b5a4926a8f5b7fb7d6899a22f7dc7236114 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Tue, 16 Dec 2025 12:42:08 -0500 Subject: [PATCH 073/139] Admin API: worker support for Query User Account (#19281) --- changelog.d/19281.feature | 1 + docs/workers.md | 5 +++++ synapse/rest/admin/__init__.py | 3 +++ synapse/rest/admin/users.py | 40 +++++++++++++++++++--------------- 4 files changed, 32 insertions(+), 17 deletions(-) create mode 100644 changelog.d/19281.feature diff --git a/changelog.d/19281.feature b/changelog.d/19281.feature new file mode 100644 index 00000000000..78d3002d907 --- /dev/null +++ b/changelog.d/19281.feature @@ -0,0 +1 @@ +Admin API: add worker support to `GET /_synapse/admin/v2/users/`. diff --git a/docs/workers.md b/docs/workers.md index 2bc8afa74f6..c2aef33e168 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -255,6 +255,8 @@ information. ^/_matrix/client/(api/v1|r0|v3|unstable)/directory/room/.*$ ^/_matrix/client/(r0|v3|unstable)/capabilities$ ^/_matrix/client/(r0|v3|unstable)/notifications$ + + # Admin API requests ^/_synapse/admin/v1/rooms/[^/]+$ # Encryption requests @@ -300,6 +302,9 @@ Additionally, the following REST endpoints can be handled for GET requests: # Presence requests ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/ + # Admin API requests + ^/_synapse/admin/v2/users/[^/]+$ + Pagination requests can also be handled, but all requests for a given room must be routed to the same instance. Additionally, care must be taken to ensure that the purge history admin API is not used while pagination requests diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index fe3eeafd9f9..b209404cd18 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -119,6 +119,7 @@ UserRegisterServlet, UserReplaceMasterCrossSigningKeyRestServlet, UserRestServletV2, + UserRestServletV2Get, UsersRestServletV2, UsersRestServletV3, UserTokenRestServlet, @@ -281,6 +282,8 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: # matrix_authentication_service integration uses the dedicated MAS API. if hs.config.experimental.msc3861.enabled: register_servlets_for_msc3861_delegation(hs, http_server) + else: + UserRestServletV2Get(hs).register(http_server) return diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index 406ad8f406f..ccd34d17d80 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -210,7 +210,7 @@ def _parse_parameter_deactivated(self, request: SynapseRequest) -> bool | None: return parse_boolean(request, "deactivated") -class UserRestServletV2(RestServlet): +class UserRestServletV2Get(RestServlet): PATTERNS = admin_patterns("/users/(?P[^/]*)$", "v2") """Get request to list user details. @@ -220,22 +220,6 @@ class UserRestServletV2(RestServlet): returns: 200 OK with user details if success otherwise an error. - - Put request to allow an administrator to add or modify a user. - This needs user to have administrator access in Synapse. - We use PUT instead of POST since we already know the id of the user - object to create. POST could be used to create guests. - - PUT /_synapse/admin/v2/users/ - { - "password": "secret", - "displayname": "User" - } - - returns: - 201 OK with new user object if user was created or - 200 OK with modified user object if user was modified - otherwise an error. """ def __init__(self, hs: "HomeServer"): @@ -267,6 +251,28 @@ async def on_GET( return HTTPStatus.OK, user_info_dict + +class UserRestServletV2(UserRestServletV2Get): + """ + Put request to allow an administrator to add or modify a user. + This needs user to have administrator access in Synapse. + We use PUT instead of POST since we already know the id of the user + object to create. POST could be used to create guests. + + Note: This inherits from `UserRestServletV2Get`, so also supports the `GET` route. + + PUT /_synapse/admin/v2/users/ + { + "password": "secret", + "displayname": "User" + } + + returns: + 201 OK with new user object if user was created or + 200 OK with modified user object if user was modified + otherwise an error. + """ + async def on_PUT( self, request: SynapseRequest, user_id: str ) -> tuple[int, JsonMapping]: From 41938d6fd2a2690a6ca5c884a787f819907d1255 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 19 Dec 2025 14:29:04 -0600 Subject: [PATCH 074/139] Log the original bind exception when encountering `Failed to listen on 0.0.0.0, continuing because listening on [::]` (#19297) **Before:** ``` WARNING - call_when_running - Failed to listen on 0.0.0.0, continuing because listening on [::] ``` **After:** ``` WARNING - call_when_running - Failed to listen on 0.0.0.0, continuing because listening on [::]. Original exception: CannotListenError: Couldn't listen on 0.0.0.0:8008: [Errno 98] Address already in use. ``` --- changelog.d/19297.misc | 1 + synapse/app/__init__.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19297.misc diff --git a/changelog.d/19297.misc b/changelog.d/19297.misc new file mode 100644 index 00000000000..aec97fd973f --- /dev/null +++ b/changelog.d/19297.misc @@ -0,0 +1 @@ +Log the original bind exception when encountering `Failed to listen on 0.0.0.0, continuing because listening on [::]`. diff --git a/synapse/app/__init__.py b/synapse/app/__init__.py index 56033f57823..fa436e874b4 100644 --- a/synapse/app/__init__.py +++ b/synapse/app/__init__.py @@ -54,7 +54,9 @@ def check_bind_error( """ if address == "0.0.0.0" and "::" in bind_addresses: logger.warning( - "Failed to listen on 0.0.0.0, continuing because listening on [::]" + "Failed to listen on 0.0.0.0, continuing because listening on [::]. Original exception: %s: %s", + type(e).__name__, + str(e), ) else: raise e From 50fabc48c322bc5c6cf08720c3779383b4c7fd52 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Dec 2025 16:04:03 +0000 Subject: [PATCH 075/139] Bump actions/checkout from 6.0.0 to 6.0.1 in the minor-and-patches group (#19319) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps the minor-and-patches group with 1 update: [actions/checkout](https://github.com/actions/checkout). Updates `actions/checkout` from 6.0.0 to 6.0.1
Release notes

Sourced from actions/checkout's releases.

v6.0.1

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v6...v6.0.1

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/checkout&package-manager=github_actions&previous-version=6.0.0&new-version=6.0.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker.yml | 2 +- .github/workflows/docs-pr.yaml | 4 +- .github/workflows/docs.yaml | 2 +- .github/workflows/fix_lint.yaml | 2 +- .github/workflows/latest_deps.yml | 10 ++--- .github/workflows/poetry_lockfile.yaml | 2 +- .github/workflows/push_complement_image.yml | 6 +-- .github/workflows/release-artifacts.yml | 8 ++-- .github/workflows/schema.yaml | 4 +- .github/workflows/tests.yml | 44 ++++++++++----------- .github/workflows/triage_labelled.yml | 2 +- .github/workflows/twisted_trunk.yml | 10 ++--- 12 files changed, 48 insertions(+), 48 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index aaf1e22d3c4..3a5b554a7fe 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -31,7 +31,7 @@ jobs: uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - name: Checkout repository - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Extract version from pyproject.toml # Note: explicitly requesting bash will mean bash is invoked with `-eo pipefail`, see diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index 4d28533a272..524739ba624 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -13,7 +13,7 @@ jobs: name: GitHub Pages runs-on: ubuntu-latest steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: # Fetch all history so that the schema_versions script works. fetch-depth: 0 @@ -50,7 +50,7 @@ jobs: name: Check links in documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Setup mdbook uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0 diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 51944b13e8c..e33add18305 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -50,7 +50,7 @@ jobs: needs: - pre steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: # Fetch all history so that the schema_versions script works. fetch-depth: 0 diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index 9daea3f3783..34f3b14e75c 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index c356ee8e3dd..0e27666ef17 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -42,7 +42,7 @@ jobs: if: needs.check_repo.outputs.should_run_workflow == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: @@ -77,7 +77,7 @@ jobs: postgres-version: "14" steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -152,7 +152,7 @@ jobs: BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }} steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -202,7 +202,7 @@ jobs: steps: - name: Check out synapse codebase - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: path: synapse @@ -234,7 +234,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/poetry_lockfile.yaml b/.github/workflows/poetry_lockfile.yaml index 5c139bf5745..29b5950ab8d 100644 --- a/.github/workflows/poetry_lockfile.yaml +++ b/.github/workflows/poetry_lockfile.yaml @@ -16,7 +16,7 @@ jobs: name: "Check locked dependencies have sdists" runs-on: ubuntu-latest steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: '3.x' diff --git a/.github/workflows/push_complement_image.yml b/.github/workflows/push_complement_image.yml index ed82482505a..b662b987540 100644 --- a/.github/workflows/push_complement_image.yml +++ b/.github/workflows/push_complement_image.yml @@ -33,17 +33,17 @@ jobs: packages: write steps: - name: Checkout specific branch (debug build) - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 if: github.event_name == 'workflow_dispatch' with: ref: ${{ inputs.branch }} - name: Checkout clean copy of develop (scheduled build) - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 if: github.event_name == 'schedule' with: ref: develop - name: Checkout clean copy of master (on-push) - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 if: github.event_name == 'push' with: ref: master diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 33b965d960e..496b9086ae4 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -27,7 +27,7 @@ jobs: name: "Calculate list of debian distros" runs-on: ubuntu-latest steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.x" @@ -55,7 +55,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: path: src @@ -125,7 +125,7 @@ jobs: os: "ubuntu-24.04-arm" steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: @@ -163,7 +163,7 @@ jobs: if: ${{ !startsWith(github.ref, 'refs/pull/') }} steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.10" diff --git a/.github/workflows/schema.yaml b/.github/workflows/schema.yaml index 0755a5f0234..356d1558071 100644 --- a/.github/workflows/schema.yaml +++ b/.github/workflows/schema.yaml @@ -14,7 +14,7 @@ jobs: name: Ensure Synapse config schema is valid runs-on: ubuntu-latest steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.x" @@ -40,7 +40,7 @@ jobs: name: Ensure generated documentation is up-to-date runs-on: ubuntu-latest steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.x" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cab6bbdefee..b5ae3c35126 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -86,7 +86,7 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: @@ -106,7 +106,7 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.x" @@ -116,7 +116,7 @@ jobs: check-lockfile: runs-on: ubuntu-latest steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.x" @@ -129,7 +129,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -151,7 +151,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -187,7 +187,7 @@ jobs: lint-crlf: runs-on: ubuntu-latest steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Check line endings run: scripts-dev/check_line_terminators.sh @@ -196,7 +196,7 @@ jobs: if: ${{ github.event_name == 'pull_request' && (github.base_ref == 'develop' || contains(github.base_ref, 'release-')) && github.event.pull_request.user.login != 'dependabot[bot]' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 @@ -214,7 +214,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -233,7 +233,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -251,7 +251,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -287,7 +287,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -307,7 +307,7 @@ jobs: needs: changes if: ${{ needs.changes.outputs.linting_readme == 'true' }} steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.x" @@ -355,7 +355,7 @@ jobs: needs: linting-done runs-on: ubuntu-latest steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.x" @@ -376,7 +376,7 @@ jobs: job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }} steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - run: sudo apt-get -qq install xmlsec1 - name: Set up PostgreSQL ${{ matrix.job.postgres-version }} if: ${{ matrix.job.postgres-version }} @@ -432,7 +432,7 @@ jobs: - changes runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -495,7 +495,7 @@ jobs: extras: ["all"] steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 # Install libs necessary for PyPy to build binary wheels for dependencies - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -545,7 +545,7 @@ jobs: job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }} steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Prepare test blacklist run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers @@ -592,7 +592,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - run: sudo apt-get -qq install xmlsec1 postgresql-client - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -636,7 +636,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Add PostgreSQL apt repository # We need a version of pg_dump that can handle the version of # PostgreSQL being tested against. The Ubuntu package repository lags @@ -691,7 +691,7 @@ jobs: steps: - name: Checkout synapse codebase - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: path: synapse @@ -727,7 +727,7 @@ jobs: - changes steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -747,7 +747,7 @@ jobs: - changes steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master diff --git a/.github/workflows/triage_labelled.yml b/.github/workflows/triage_labelled.yml index 34222b7d1bf..27ff1d80cd4 100644 --- a/.github/workflows/triage_labelled.yml +++ b/.github/workflows/triage_labelled.yml @@ -22,7 +22,7 @@ jobs: # This field is case-sensitive. TARGET_STATUS: Needs info steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: # Only clone the script file we care about, instead of the whole repo. sparse-checkout: .ci/scripts/triage_labelled_issue.sh diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 325902f131b..64486b0f05f 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -70,7 +70,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - run: sudo apt-get -qq install xmlsec1 - name: Install Rust @@ -117,7 +117,7 @@ jobs: - ${{ github.workspace }}:/src steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -175,7 +175,7 @@ jobs: steps: - name: Run actions/checkout@v4 for synapse - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: path: synapse @@ -217,7 +217,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From f79acff862aa10fcb528d0d893ccbbc2738189cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Dec 2025 16:37:16 +0000 Subject: [PATCH 076/139] Bump log from 0.4.28 to 0.4.29 in the patches group (#19318) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps the patches group with 1 update: [log](https://github.com/rust-lang/log). Updates `log` from 0.4.28 to 0.4.29
Release notes

Sourced from log's releases.

0.4.29

MSRV

This release increases log's MSRV from 1.61.0 to 1.68.0.

What's Changed

New Contributors

Full Changelog: https://github.com/rust-lang/log/compare/0.4.28...0.4.29

Changelog

Sourced from log's changelog.

[0.4.29] - 2025-12-02

What's Changed

New Contributors

Full Changelog: https://github.com/rust-lang/log/compare/0.4.28...0.4.29

Commits
  • b1e2df7 Merge pull request #719 from rust-lang/cargo/0.4.29
  • 3fe1a54 prepare for 0.4.29 release
  • 7a432d9 Merge pull request #718 from rust-lang/ci/msrv
  • 0689d56 rebump msrv to 1.68.0
  • 46b448e try drop msrv back to 1.61.0
  • 929ab38 fix up doc test feature gate
  • 957cece bump serde-dependent crates
  • bea40c8 bump msrv to 1.68.0
  • c540184 Merge pull request #716 from rust-lang/ci-smaller-matrix2
  • c971e63 Merge branch 'master' into ci-smaller-matrix2
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=log&package-manager=cargo&previous-version=0.4.28&new-version=0.4.29)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 007428a3802..892279e4f20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -705,9 +705,9 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "log" -version = "0.4.28" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "lru-slab" From 7a24fafbc376b9bffeb3277b1ad4aa950720c96c Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 29 Dec 2025 12:20:58 -0600 Subject: [PATCH 077/139] Auto-formatting `.github/workflows/tests.yml` from VSCode (#19327) --- .github/workflows/tests.yml | 124 ++++++++++++++++++------------------ changelog.d/19327.misc | 1 + 2 files changed, 62 insertions(+), 63 deletions(-) create mode 100644 changelog.d/19327.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b5ae3c35126..7923383768d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,59 +26,59 @@ jobs: linting: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting }} linting_readme: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting_readme }} steps: - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 - id: filter - # We only check on PRs - if: startsWith(github.ref, 'refs/pull/') - with: - filters: | - rust: - - 'rust/**' - - 'Cargo.toml' - - 'Cargo.lock' - - '.rustfmt.toml' - - '.github/workflows/tests.yml' - - trial: - - 'synapse/**' - - 'tests/**' - - 'rust/**' - - '.ci/scripts/calculate_jobs.py' - - 'Cargo.toml' - - 'Cargo.lock' - - 'pyproject.toml' - - 'poetry.lock' - - '.github/workflows/tests.yml' - - integration: - - 'synapse/**' - - 'rust/**' - - 'docker/**' - - 'Cargo.toml' - - 'Cargo.lock' - - 'pyproject.toml' - - 'poetry.lock' - - 'docker/**' - - '.ci/**' - - 'scripts-dev/complement.sh' - - '.github/workflows/tests.yml' - - linting: - - 'synapse/**' - - 'docker/**' - - 'tests/**' - - 'scripts-dev/**' - - 'contrib/**' - - 'synmark/**' - - 'stubs/**' - - '.ci/**' - - 'mypy.ini' - - 'pyproject.toml' - - 'poetry.lock' - - '.github/workflows/tests.yml' - - linting_readme: - - 'README.rst' + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: filter + # We only check on PRs + if: startsWith(github.ref, 'refs/pull/') + with: + filters: | + rust: + - 'rust/**' + - 'Cargo.toml' + - 'Cargo.lock' + - '.rustfmt.toml' + - '.github/workflows/tests.yml' + + trial: + - 'synapse/**' + - 'tests/**' + - 'rust/**' + - '.ci/scripts/calculate_jobs.py' + - 'Cargo.toml' + - 'Cargo.lock' + - 'pyproject.toml' + - 'poetry.lock' + - '.github/workflows/tests.yml' + + integration: + - 'synapse/**' + - 'rust/**' + - 'docker/**' + - 'Cargo.toml' + - 'Cargo.lock' + - 'pyproject.toml' + - 'poetry.lock' + - 'docker/**' + - '.ci/**' + - 'scripts-dev/complement.sh' + - '.github/workflows/tests.yml' + + linting: + - 'synapse/**' + - 'docker/**' + - 'tests/**' + - 'scripts-dev/**' + - 'contrib/**' + - 'synmark/**' + - 'stubs/**' + - '.ci/**' + - 'mypy.ini' + - 'pyproject.toml' + - 'poetry.lock' + - '.github/workflows/tests.yml' + + linting_readme: + - 'README.rst' check-sampleconfig: runs-on: ubuntu-latest @@ -219,8 +219,8 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: - components: clippy - toolchain: ${{ env.RUST_VERSION }} + components: clippy + toolchain: ${{ env.RUST_VERSION }} - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - run: cargo clippy -- -D warnings @@ -238,8 +238,8 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: - toolchain: nightly-2025-04-23 - components: clippy + toolchain: nightly-2025-04-23 + components: clippy - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - run: cargo clippy --all-features -- -D warnings @@ -349,7 +349,6 @@ jobs: lint-rustfmt lint-readme - calculate-test-jobs: if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail needs: linting-done @@ -373,7 +372,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }} + job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }} steps: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 @@ -449,12 +448,12 @@ jobs: - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: - python-version: '3.10' + python-version: "3.10" - name: Prepare old deps # Note: we install using `uv` here, not poetry or pip to allow us to test with the # minimum version of all dependencies, both those explicitly specified and those - # implicitly brought in by the explicit dependencies. + # implicitly brought in by the explicit dependencies. run: | pip install uv uv pip install --system --resolution=lowest .[all,test] @@ -605,7 +604,6 @@ jobs: PGPASSWORD: postgres PGDATABASE: postgres - portdb: if: ${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true'}} # Allow previous steps to be skipped, but not fail needs: @@ -752,7 +750,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: - toolchain: nightly-2022-12-01 + toolchain: nightly-2022-12-01 - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - run: cargo bench --no-run diff --git a/changelog.d/19327.misc b/changelog.d/19327.misc new file mode 100644 index 00000000000..d61a66907e5 --- /dev/null +++ b/changelog.d/19327.misc @@ -0,0 +1 @@ +Format `.github/workflows/tests.yml`. From bd94152e0644a00708e570b0e9abc775d9704015 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 31 Dec 2025 14:43:04 -0600 Subject: [PATCH 078/139] Stream Complement progress and format logs in a separate step after all tests are done (#19326) This way we can see what's happening as the tests run instead of nothing until the end. Also useful to split the test output from the formatting so we can take the raw test output before formatting gobbles it all up. Same thing I did in https://github.com/element-hq/synapse-rust-apps/pull/361 --- .github/workflows/tests.yml | 20 +++++++++++++++++--- changelog.d/19326.misc | 1 + 2 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 changelog.d/19326.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7923383768d..dd216fc696f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -708,14 +708,28 @@ jobs: go-version-file: complement/go.mod # use p=1 concurrency as GHA boxes are underpowered and don't like running tons of synapses at once. - - run: | + - name: Run Complement Tests + id: run_complement_tests + # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes + # are underpowered and don't like running tons of Synapse instances at once. + # -json: Output JSON format so that gotestfmt can parse it. + # + # tee /tmp/gotest.log: We tee the output to a file so that we can re-process it + # later on for better formatting with gotestfmt. But we still want the command + # to output to the terminal as it runs so we can see what's happening in + # real-time. + run: | set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | synapse/.ci/scripts/gotestfmt + COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | tee /tmp/gotest.log shell: bash env: POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} - name: Run Complement Tests + + - name: Formatted Complement test logs + # Always run this step if we attempted to run the Complement tests. + if: always() && steps.run_complement_tests.outcome != 'skipped' + run: cat /tmp/gotest.log | gotestfmt -hide "successful-downloads,empty-packages" cargo-test: if: ${{ needs.changes.outputs.rust == 'true' }} diff --git a/changelog.d/19326.misc b/changelog.d/19326.misc new file mode 100644 index 00000000000..37493c74888 --- /dev/null +++ b/changelog.d/19326.misc @@ -0,0 +1 @@ +Update CI to stream Complement progress and format logs in a separate step after all tests are done. From 9dae6cc5958e6e7300637ac22e091f295218f0d7 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 1 Jan 2026 14:00:00 -0600 Subject: [PATCH 079/139] Add a way to expose metrics from the Docker image (`SYNAPSE_ENABLE_METRICS`) (#19324) Spawning from wanting to [run a load test](https://github.com/element-hq/synapse-rust-apps/pull/397) against the Complement Docker image of Synapse and see metrics from the homeserver. ### Why not just provide your own homeserver config? Probably possible but it gets tricky when you try to use the workers variant of the Docker image (`docker/Dockerfile-workers`). The way to workaround it would probably be to `yq` edit everything in a script and change `/data/homeserver.yaml` and `/conf/workers/*.yaml` to add the `metrics` listener. And then modify `/conf/workers/shared.yaml` to add `enable_metrics: true`. Doesn't spark much joy. --- changelog.d/19324.docker | 1 + docker/Dockerfile | 8 ++- docker/README-testing.md | 2 + docker/README.md | 3 ++ docker/conf-workers/shared.yaml.j2 | 5 ++ docker/conf-workers/worker.yaml.j2 | 8 +++ docker/conf/homeserver.yaml | 9 ++++ docker/configure_workers_and_start.py | 33 ++++++++++-- docker/start.py | 57 +++++++++++++++----- docs/sample_config.yaml | 14 +++-- synapse/config/_base.py | 51 +++++++++++++++++- synapse/config/_base.pyi | 1 + synapse/config/metrics.py | 11 +++- synapse/config/server.py | 53 ++++--------------- tests/config/test_server.py | 76 +++++++++++++++++++++++---- 15 files changed, 252 insertions(+), 80 deletions(-) create mode 100644 changelog.d/19324.docker diff --git a/changelog.d/19324.docker b/changelog.d/19324.docker new file mode 100644 index 00000000000..52bf9cb7ae1 --- /dev/null +++ b/changelog.d/19324.docker @@ -0,0 +1 @@ +Add a way to expose metrics from the Docker image (`SYNAPSE_ENABLE_METRICS`). diff --git a/docker/Dockerfile b/docker/Dockerfile index 6d10dee1aa2..91116ea1c48 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -188,7 +188,13 @@ COPY --from=builder --exclude=.lock /install /usr/local COPY ./docker/start.py /start.py COPY ./docker/conf /conf -EXPOSE 8008/tcp 8009/tcp 8448/tcp +# 8008: CS Matrix API port from Synapse +# 8448: SS Matrix API port from Synapse +EXPOSE 8008/tcp 8448/tcp +# 19090: Metrics listener port for the main process (metrics must be enabled with +# SYNAPSE_ENABLE_METRICS=1). Metrics for workers are on ports starting from 19091 but +# since these are dynamic we don't expose them by default. +EXPOSE 19090/tcp ENTRYPOINT ["/start.py"] diff --git a/docker/README-testing.md b/docker/README-testing.md index db88598b8c4..009a5d612d2 100644 --- a/docker/README-testing.md +++ b/docker/README-testing.md @@ -135,3 +135,5 @@ but it does not serve TLS by default. You can configure `SYNAPSE_TLS_CERT` and `SYNAPSE_TLS_KEY` to point to a TLS certificate and key (respectively), both in PEM (textual) format. In this case, Nginx will additionally serve using HTTPS on port 8448. + + diff --git a/docker/README.md b/docker/README.md index 3438e9c441c..ed5155f541a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -75,6 +75,9 @@ The following environment variables are supported in `generate` mode: particularly tricky. * `SYNAPSE_LOG_TESTING`: if set, Synapse will log additional information useful for testing. +* `SYNAPSE_ENABLE_METRICS`: if set to `1`, the metrics listener will be enabled on the + main and worker processes. Defaults to `0` (disabled). The main process will listen on + port `19090` and workers on port `19091 + `. ## Postgres diff --git a/docker/conf-workers/shared.yaml.j2 b/docker/conf-workers/shared.yaml.j2 index 1dfc60ad110..6efbd054729 100644 --- a/docker/conf-workers/shared.yaml.j2 +++ b/docker/conf-workers/shared.yaml.j2 @@ -20,4 +20,9 @@ app_service_config_files: {%- endfor %} {%- endif %} +{# Controlled by SYNAPSE_ENABLE_METRICS #} +{% if enable_metrics %} +enable_metrics: true +{% endif %} + {{ shared_worker_config }} diff --git a/docker/conf-workers/worker.yaml.j2 b/docker/conf-workers/worker.yaml.j2 index 29ec74b4ea0..d4d2d4d4cfa 100644 --- a/docker/conf-workers/worker.yaml.j2 +++ b/docker/conf-workers/worker.yaml.j2 @@ -21,6 +21,14 @@ worker_listeners: {%- endfor %} {% endif %} +{# Controlled by SYNAPSE_ENABLE_METRICS #} +{% if metrics_port %} + - type: metrics + # Prometheus does not support Unix sockets so we don't bother with + # `SYNAPSE_USE_UNIX_SOCKET`, https://github.com/prometheus/prometheus/issues/12024 + port: {{ metrics_port }} +{% endif %} + worker_log_config: {{ worker_log_config_filepath }} {{ worker_extra_conf }} diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml index 2890990705b..e2e1338673d 100644 --- a/docker/conf/homeserver.yaml +++ b/docker/conf/homeserver.yaml @@ -53,6 +53,15 @@ listeners: - names: [federation] compress: false +{% if SYNAPSE_ENABLE_METRICS %} + - type: metrics + # The main process always uses the same port 19090 + # + # Prometheus does not support Unix sockets so we don't bother with + # `SYNAPSE_USE_UNIX_SOCKET`, https://github.com/prometheus/prometheus/issues/12024 + port: 19090 +{% endif %} + ## Database ## {% if POSTGRES_PASSWORD %} diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index e7cbd701b8e..c03f2a5e562 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -49,6 +49,10 @@ # regardless of the SYNAPSE_LOG_LEVEL setting. # * SYNAPSE_LOG_TESTING: if set, Synapse will log additional information useful # for testing. +# * SYNAPSE_USE_UNIX_SOCKET: TODO +# * `SYNAPSE_ENABLE_METRICS`: if set to `1`, the metrics listener will be enabled on the +# main and worker processes. Defaults to `0` (disabled). The main process will listen on +# port `19090` and workers on port `19091 + `. # # NOTE: According to Complement's ENTRYPOINT expectations for a homeserver image (as defined # in the project's README), this script may be run multiple times, and functionality should @@ -758,6 +762,9 @@ def generate_worker_files( # Convenience helper for if using unix sockets instead of host:port using_unix_sockets = environ.get("SYNAPSE_USE_UNIX_SOCKET", False) + + enable_metrics = environ.get("SYNAPSE_ENABLE_METRICS", "0") == "1" + # First read the original config file and extract the listeners block. Then we'll # add another listener for replication. Later we'll write out the result to the # shared config file. @@ -789,7 +796,11 @@ def generate_worker_files( # base shared worker jinja2 template. This config file will be passed to all # workers, included Synapse's main process. It is intended mainly for disabling # functionality when certain workers are spun up, and adding a replication listener. - shared_config: dict[str, Any] = {"listeners": listeners} + shared_config: dict[str, Any] = { + "listeners": listeners, + # Controls `enable_metrics: true` + "enable_metrics": enable_metrics, + } # List of dicts that describe workers. # We pass this to the Supervisor template later to generate the appropriate @@ -816,6 +827,8 @@ def generate_worker_files( # Start worker ports from this arbitrary port worker_port = 18009 + # The main process metrics port is 19090, so start workers from 19091 + worker_metrics_port = 19091 # A list of internal endpoints to healthcheck, starting with the main process # which exists even if no workers do. @@ -862,10 +875,15 @@ def generate_worker_files( {"name": worker_name, "port": str(worker_port), "config_path": config_path} ) - # Update the shared config with any worker_type specific options. The first of a - # given worker_type needs to stay assigned and not be replaced. - worker_config["shared_extra_conf"].update(shared_config) - shared_config = worker_config["shared_extra_conf"] + # Keep the `shared_config` up to date with the `shared_extra_conf` from each + # worker. + shared_config = { + **worker_config["shared_extra_conf"], + # We combine `shared_config` second to avoid overwriting existing keys + # because TODO: why? + **shared_config, + } + if using_unix_sockets: healthcheck_urls.append( f"--unix-socket /run/worker.{worker_port} http://localhost/health" @@ -891,6 +909,10 @@ def generate_worker_files( # Write out the worker's logging config file log_config_filepath = generate_worker_log_config(environ, worker_name, data_dir) + if enable_metrics: + # Enable prometheus metrics endpoint on this worker + worker_config["metrics_port"] = worker_metrics_port + # Then a worker config file convert( "/conf/worker.yaml.j2", @@ -905,6 +927,7 @@ def generate_worker_files( nginx_upstreams.setdefault(worker_type, set()).add(worker_port) worker_port += 1 + worker_metrics_port += 1 # Build the nginx location config blocks nginx_location_config = "" diff --git a/docker/start.py b/docker/start.py index c88d23695f0..19f1ab50759 100755 --- a/docker/start.py +++ b/docker/start.py @@ -31,6 +31,25 @@ def flush_buffers() -> None: sys.stderr.flush() +def strtobool(val: str) -> bool: + """Convert a string representation of truth to True or False + + True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values + are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if + 'val' is anything else. + + This is lifted from distutils.util.strtobool, with the exception that it actually + returns a bool, rather than an int. + """ + val = val.lower() + if val in ("y", "yes", "t", "true", "on", "1"): + return True + elif val in ("n", "no", "f", "false", "off", "0"): + return False + else: + raise ValueError("invalid truth value %r" % (val,)) + + def convert(src: str, dst: str, environ: Mapping[str, object]) -> None: """Generate a file from a template @@ -98,19 +117,16 @@ def generate_config_from_template( os.mkdir(config_dir) # Convert SYNAPSE_NO_TLS to boolean if exists - if "SYNAPSE_NO_TLS" in environ: - tlsanswerstring = str.lower(environ["SYNAPSE_NO_TLS"]) - if tlsanswerstring in ("true", "on", "1", "yes"): - environ["SYNAPSE_NO_TLS"] = True - else: - if tlsanswerstring in ("false", "off", "0", "no"): - environ["SYNAPSE_NO_TLS"] = False - else: - error( - 'Environment variable "SYNAPSE_NO_TLS" found but value "' - + tlsanswerstring - + '" unrecognized; exiting.' - ) + tlsanswerstring = environ.get("SYNAPSE_NO_TLS") + if tlsanswerstring is not None: + try: + environ["SYNAPSE_NO_TLS"] = strtobool(tlsanswerstring) + except ValueError: + error( + 'Environment variable "SYNAPSE_NO_TLS" found but value "' + + tlsanswerstring + + '" unrecognized; exiting.' + ) if "SYNAPSE_LOG_CONFIG" not in environ: environ["SYNAPSE_LOG_CONFIG"] = config_dir + "/log.config" @@ -164,6 +180,18 @@ def run_generate_config(environ: Mapping[str, str], ownership: str | None) -> No config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data") config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml") data_dir = environ.get("SYNAPSE_DATA_DIR", "/data") + enable_metrics_raw = environ.get("SYNAPSE_ENABLE_METRICS", "0") + + enable_metrics = False + if enable_metrics_raw is not None: + try: + enable_metrics = strtobool(enable_metrics_raw) + except ValueError: + error( + 'Environment variable "SYNAPSE_ENABLE_METRICS" found but value "' + + enable_metrics_raw + + '" unrecognized; exiting.' + ) # create a suitable log config from our template log_config_file = "%s/%s.log.config" % (config_dir, server_name) @@ -190,6 +218,9 @@ def run_generate_config(environ: Mapping[str, str], ownership: str | None) -> No "--open-private-ports", ] + if enable_metrics: + args.append("--enable-metrics") + if ownership is not None: # make sure that synapse has perms to write to the data dir. log(f"Setting ownership on {data_dir} to {ownership}") diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 0d75e6d4a1d..470e44a8edd 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -24,14 +24,18 @@ server_name: "SERVERNAME" pid_file: DATADIR/homeserver.pid listeners: - - port: 8008 + - bind_addresses: + - ::1 + - 127.0.0.1 + port: 8008 + resources: + - compress: false + names: + - client + - federation tls: false type: http x_forwarded: true - bind_addresses: ['::1', '127.0.0.1'] - resources: - - names: [client, federation] - compress: false database: name: sqlite3 args: diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 43dece4a08a..fdfbee98a11 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -44,6 +44,7 @@ import yaml from synapse.types import StrSequence +from synapse.util.stringutils import parse_and_validate_server_name from synapse.util.templates import _create_mxc_to_http_filter, _format_ts_filter logger = logging.getLogger(__name__) @@ -465,6 +466,7 @@ def generate_config( generate_secrets: bool = False, report_stats: bool | None = None, open_private_ports: bool = False, + enable_metrics: bool = False, listeners: list[dict] | None = None, tls_certificate_path: str | None = None, tls_private_key_path: str | None = None, @@ -495,9 +497,15 @@ def generate_config( open_private_ports: True to leave private ports (such as the non-TLS HTTP listener) open to the internet. + enable_metrics: True to set `enable_metrics: true` and when using the + default set of listeners, will also add the metrics listener on port 19090. + listeners: A list of descriptions of the listeners synapse should - start with each of which specifies a port (int), a list of - resources (list(str)), tls (bool) and type (str). For example: + start with each of which specifies a port (int), a list of resources + (list(str)), tls (bool) and type (str). There is a default set of + listeners when `None`. + + Example usage: [{ "port": 8448, "resources": [{"names": ["federation"]}], @@ -518,6 +526,35 @@ def generate_config( Returns: The yaml config file """ + _, bind_port = parse_and_validate_server_name(server_name) + if bind_port is not None: + unsecure_port = bind_port - 400 + else: + bind_port = 8448 + unsecure_port = 8008 + + # The default listeners + if listeners is None: + listeners = [ + { + "port": unsecure_port, + "tls": False, + "type": "http", + "x_forwarded": True, + "resources": [ + {"names": ["client", "federation"], "compress": False} + ], + } + ] + + if enable_metrics: + listeners.append( + { + "port": 19090, + "tls": False, + "type": "metrics", + } + ) conf = CONFIG_FILE_HEADER + "\n".join( dedent(conf) @@ -529,6 +566,7 @@ def generate_config( generate_secrets=generate_secrets, report_stats=report_stats, open_private_ports=open_private_ports, + enable_metrics=enable_metrics, listeners=listeners, tls_certificate_path=tls_certificate_path, tls_private_key_path=tls_private_key_path, @@ -756,6 +794,14 @@ def load_or_generate_config( " internet. Do not use this unless you know what you are doing." ), ) + generate_group.add_argument( + "--enable-metrics", + action="store_true", + help=( + "Sets `enable_metrics: true` and when using the default set of listeners, " + "will also add the metrics listener on port 19090." + ), + ) cls.invoke_all_static("add_arguments", parser) config_args = parser.parse_args(argv_options) @@ -812,6 +858,7 @@ def load_or_generate_config( report_stats=(config_args.report_stats == "yes"), generate_secrets=True, open_private_ports=config_args.open_private_ports, + enable_metrics=config_args.enable_metrics, ) os.makedirs(config_dir_path, exist_ok=True) diff --git a/synapse/config/_base.pyi b/synapse/config/_base.pyi index fe9b3333c4e..7c371d161cd 100644 --- a/synapse/config/_base.pyi +++ b/synapse/config/_base.pyi @@ -146,6 +146,7 @@ class RootConfig: generate_secrets: bool = ..., report_stats: bool | None = ..., open_private_ports: bool = ..., + enable_metrics: bool = ..., listeners: Any | None = ..., tls_certificate_path: str | None = ..., tls_private_key_path: str | None = ..., diff --git a/synapse/config/metrics.py b/synapse/config/metrics.py index 83dbee53b62..d8cddb82ed7 100644 --- a/synapse/config/metrics.py +++ b/synapse/config/metrics.py @@ -75,10 +75,19 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: ) def generate_config_section( - self, report_stats: bool | None = None, **kwargs: Any + self, + report_stats: bool | None = None, + enable_metrics: bool = False, + **kwargs: Any, ) -> str: if report_stats is not None: res = "report_stats: %s\n" % ("true" if report_stats else "false") else: res = "\n" + + # We avoid adding anything if it's `False` since that's the default (less noise + # in the default generated config) + if enable_metrics: + res += "enable_metrics: true\n" + return res diff --git a/synapse/config/server.py b/synapse/config/server.py index 495f2891599..ca94c224ea5 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -923,26 +923,21 @@ def has_tls_listener(self) -> bool: def generate_config_section( self, + *, config_dir_path: str, data_dir_path: str, server_name: str, - open_private_ports: bool, - listeners: list[dict] | None, + open_private_ports: bool = False, + listeners: list[dict] | None = None, **kwargs: Any, ) -> str: - _, bind_port = parse_and_validate_server_name(server_name) - if bind_port is not None: - unsecure_port = bind_port - 400 - else: - bind_port = 8448 - unsecure_port = 8008 - pid_file = os.path.join(data_dir_path, "homeserver.pid") - secure_listeners = [] - unsecure_listeners = [] + http_bindings = "[]" private_addresses = ["::1", "127.0.0.1"] if listeners: + secure_listeners = [] + unsecure_listeners = [] for listener in listeners: if listener["tls"]: secure_listeners.append(listener) @@ -957,43 +952,17 @@ def generate_config_section( unsecure_listeners.append(listener) - secure_http_bindings = indent( - yaml.dump(secure_listeners), " " * 10 - ).lstrip() - - unsecure_http_bindings = indent( - yaml.dump(unsecure_listeners), " " * 10 + # `lstrip` is used because the first line already has whitespace in the + # template below + http_bindings = indent( + yaml.dump(secure_listeners + unsecure_listeners), " " * 10 ).lstrip() - if not unsecure_listeners: - unsecure_http_bindings = """- port: %(unsecure_port)s - tls: false - type: http - x_forwarded: true""" % locals() - - if not open_private_ports: - unsecure_http_bindings += ( - "\n bind_addresses: ['::1', '127.0.0.1']" - ) - - unsecure_http_bindings += """ - - resources: - - names: [client, federation] - compress: false""" - - if listeners: - unsecure_http_bindings = "" - - if not secure_listeners: - secure_http_bindings = "" - return """\ server_name: "%(server_name)s" pid_file: %(pid_file)s listeners: - %(secure_http_bindings)s - %(unsecure_http_bindings)s + %(http_bindings)s """ % locals() def read_arguments(self, args: argparse.Namespace) -> None: diff --git a/tests/config/test_server.py b/tests/config/test_server.py index 5eb2540439e..d3c59ae14ca 100644 --- a/tests/config/test_server.py +++ b/tests/config/test_server.py @@ -21,6 +21,7 @@ import yaml from synapse.config._base import ConfigError, RootConfig +from synapse.config.homeserver import HomeServerConfig from synapse.config.server import ServerConfig, generate_ip_set, is_threepid_reserved from tests import unittest @@ -38,14 +39,23 @@ def test_is_threepid_reserved(self) -> None: self.assertFalse(is_threepid_reserved(config, user3)) self.assertFalse(is_threepid_reserved(config, user1_msisdn)) - def test_unsecure_listener_no_listeners_open_private_ports_false(self) -> None: + def test_default_set_of_listeners(self) -> None: + """ + Test that we get a default set of listeners from the `RootConfig` + """ conf = yaml.safe_load( - ServerConfig(RootConfig()).generate_config_section( - "CONFDIR", "/data_dir_path", "che.org", False, None + # We use `HomeServerConfig` instead of `RootConfig` as it has all of the + # `config_classes` defined. + HomeServerConfig().generate_config( + config_dir_path="CONFDIR", + data_dir_path="/data_dir_path", + server_name="che.org", + open_private_ports=False, + listeners=None, ) ) - expected_listeners = [ + expected_listeners: list[dict] = [ { "port": 8008, "tls": False, @@ -58,25 +68,61 @@ def test_unsecure_listener_no_listeners_open_private_ports_false(self) -> None: self.assertEqual(conf["listeners"], expected_listeners) - def test_unsecure_listener_no_listeners_open_private_ports_true(self) -> None: + def test_default_set_of_listeners_with_enable_metrics(self) -> None: + """ + Test that the default set of listeners from the `RootConfig` gets a metrics + listener when `enable_metrics=True`. + """ conf = yaml.safe_load( - ServerConfig(RootConfig()).generate_config_section( - "CONFDIR", "/data_dir_path", "che.org", True, None + # We use `HomeServerConfig` instead of `RootConfig` as it has all of the + # `config_classes` defined. + HomeServerConfig().generate_config( + config_dir_path="CONFDIR", + data_dir_path="/data_dir_path", + server_name="che.org", + open_private_ports=False, + enable_metrics=True, + listeners=None, ) ) - expected_listeners = [ + expected_listeners: list[dict] = [ { "port": 8008, "tls": False, "type": "http", "x_forwarded": True, + "bind_addresses": ["::1", "127.0.0.1"], "resources": [{"names": ["client", "federation"], "compress": False}], - } + }, + { + "port": 19090, + "tls": False, + "type": "metrics", + "bind_addresses": ["::1", "127.0.0.1"], + }, ] self.assertEqual(conf["listeners"], expected_listeners) + def test_unsecure_listener_no_listeners(self) -> None: + conf = yaml.safe_load( + ServerConfig(RootConfig()).generate_config_section( + config_dir_path="CONFDIR", + data_dir_path="/data_dir_path", + server_name="che.org", + open_private_ports=False, + listeners=None, + ) + ) + + # We expect `None` because we only operate with what's given to us. The default + # set of listeners comes from the logic one layer above in `RootConfig` (see + # tests above). + expected_listeners: list[dict] = [] + + self.assertEqual(conf["listeners"], expected_listeners) + def test_listeners_set_correctly_open_private_ports_false(self) -> None: listeners = [ { @@ -95,7 +141,11 @@ def test_listeners_set_correctly_open_private_ports_false(self) -> None: conf = yaml.safe_load( ServerConfig(RootConfig()).generate_config_section( - "CONFDIR", "/data_dir_path", "this.one.listens", True, listeners + config_dir_path="CONFDIR", + data_dir_path="/data_dir_path", + server_name="this.one.listens", + open_private_ports=True, + listeners=listeners, ) ) @@ -129,7 +179,11 @@ def test_listeners_set_correctly_open_private_ports_true(self) -> None: conf = yaml.safe_load( ServerConfig(RootConfig()).generate_config_section( - "CONFDIR", "/data_dir_path", "this.one.listens", True, listeners + config_dir_path="CONFDIR", + data_dir_path="/data_dir_path", + server_name="this.one.listens", + open_private_ports=True, + listeners=listeners, ) ) From 803e4b4d884b2de4b9e20dc47ffb59a983b8a4b5 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 2 Jan 2026 12:08:37 -0600 Subject: [PATCH 080/139] Make it more clear how `shared_extra_conf` is combined in our Docker configuration scripts (#19323) For reference, this PR used to include this whole `shared_config` block in the diff. But https://github.com/element-hq/synapse/pull/19324 was merged first which introduced parts of it already. Here is what this code used to look like: https://github.com/element-hq/synapse/blob/566670c363915691826b5b435c4aa7acde61b408/docker/configure_workers_and_start.py#L865-L868 --- Original context for why it was changed this way: https://github.com/matrix-org/synapse/pull/14921#discussion_r1126257933 Previously, this code made me question two things: 1. Do we actually use `worker_config["shared_extra_conf"]` in the templates? - At first glance, I couldn't see why we're updating `shared_extra_conf` here. It's not used in the `worker.yaml.j2` template so all of this seemed a bit pointless. - Turns out, updating `shared_extra_conf` itself is pointless and it's being done as a convenient place to mix the objects to get things right in `shared_config` (confusing). 1. Does it actually do anything? - Because `shared_config` starts out as an empty object, my first glance made me think we we're just updating with an empty object and then just re-assigning. But because we're in a loop, we actually accumulate the `shared_extra_conf` from each worker. I'm not sure whether I'm capturing my confusion well enough here but basically, this made me spend time trying to figure out what/why we're doing things this way and we can use a more clear pattern to accomplish the same thing. --- This change is spawning from looking at the `docker/configure_workers_and_start.py` script in order to add a metrics listener ([upcoming PR](https://github.com/element-hq/synapse/pull/19324)). --- changelog.d/19323.misc | 1 + docker/configure_workers_and_start.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/19323.misc diff --git a/changelog.d/19323.misc b/changelog.d/19323.misc new file mode 100644 index 00000000000..6699d7ea543 --- /dev/null +++ b/changelog.d/19323.misc @@ -0,0 +1 @@ +Make it more clear how `shared_extra_conf` is combined in our Docker configuration scripts. diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index c03f2a5e562..0ebee49b9d4 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -879,8 +879,8 @@ def generate_worker_files( # worker. shared_config = { **worker_config["shared_extra_conf"], - # We combine `shared_config` second to avoid overwriting existing keys - # because TODO: why? + # We combine `shared_config` second to avoid overwriting existing keys just + # for sanity sake (always use the first worker). **shared_config, } From 8f96a83d164bbead2030c2bffabd5626cbb2ce8b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 13:59:36 +0000 Subject: [PATCH 081/139] Bump actions/download-artifact from 6.0.0 to 7.0.0 (#19333) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 6.0.0 to 7.0.0.
Release notes

Sourced from actions/download-artifact's releases.

v7.0.0

v7 - What's new

[!IMPORTANT] actions/download-artifact@v7 now runs on Node.js 24 (runs.using: node24) and requires a minimum Actions Runner version of 2.327.1. If you are using self-hosted runners, ensure they are updated before upgrading.

Node.js 24

This release updates the runtime to Node.js 24. v6 had preliminary support for Node 24, however this action was by default still running on Node.js 20. Now this action by default will run on Node.js 24.

What's Changed

New Contributors

Full Changelog: https://github.com/actions/download-artifact/compare/v6.0.0...v7.0.0

Commits
  • 37930b1 Merge pull request #452 from actions/download-artifact-v7-release
  • 72582b9 doc: update readme
  • 0d2ec9d chore: release v7.0.0 for Node.js 24 support
  • fd7ae8f Merge pull request #451 from actions/fix-storage-blob
  • d484700 chore: restore minimatch.dep.yml license file
  • 03a8080 chore: remove obsolete dependency license files
  • 56fe6d9 chore: update @​actions/artifact license file to 5.0.1
  • 8e3ebc4 chore: update package-lock.json with @​actions/artifact@​5.0.1
  • 1e3c4b4 fix: update @​actions/artifact to ^5.0.0 for Node.js 24 punycode fix
  • 458627d chore: use local @​actions/artifact package for Node.js 24 testing
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/download-artifact&package-manager=github_actions&previous-version=6.0.0&new-version=7.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker.yml | 2 +- .github/workflows/release-artifacts.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3a5b554a7fe..3a038d1e581 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -95,7 +95,7 @@ jobs: - build steps: - name: Download digests - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: path: ${{ runner.temp }}/digests pattern: digests-* diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 496b9086ae4..936aae5036c 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -189,7 +189,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download all workflow run artifacts - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 - name: Build a tarball for the debs # We need to merge all the debs uploads into one folder, then compress # that. From 691e43bac93bace003ddb0c27e416764e820e6af Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 14:00:05 +0000 Subject: [PATCH 082/139] Bump actions/cache from 4.3.0 to 5.0.1 (#19332) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/cache](https://github.com/actions/cache) from 4.3.0 to 5.0.1.
Release notes

Sourced from actions/cache's releases.

v5.0.1

[!IMPORTANT] actions/cache@v5 runs on the Node.js 24 runtime and requires a minimum Actions Runner version of 2.327.1.

If you are using self-hosted runners, ensure they are updated before upgrading.

v5.0.1

What's Changed

v5.0.0

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v5...v5.0.1

v5.0.0

[!IMPORTANT] actions/cache@v5 runs on the Node.js 24 runtime and requires a minimum Actions Runner version of 2.327.1.

If you are using self-hosted runners, ensure they are updated before upgrading.

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v4.3.0...v5.0.0

Changelog

Sourced from actions/cache's changelog.

Releases

Changelog

5.0.1

  • Update @azure/storage-blob to ^12.29.1 via @actions/cache@5.0.1 #1685

5.0.0

[!IMPORTANT] actions/cache@v5 runs on the Node.js 24 runtime and requires a minimum Actions Runner version of 2.327.1. If you are using self-hosted runners, ensure they are updated before upgrading.

4.3.0

  • Bump @actions/cache to v4.1.0

4.2.4

  • Bump @actions/cache to v4.0.5

4.2.3

  • Bump @actions/cache to v4.0.3 (obfuscates SAS token in debug logs for cache entries)

4.2.2

  • Bump @actions/cache to v4.0.2

4.2.1

  • Bump @actions/cache to v4.0.1

4.2.0

TLDR; The cache backend service has been rewritten from the ground up for improved performance and reliability. actions/cache now integrates with the new cache service (v2) APIs.

The new service will gradually roll out as of February 1st, 2025. The legacy service will also be sunset on the same date. Changes in these release are fully backward compatible.

We are deprecating some versions of this action. We recommend upgrading to version v4 or v3 as soon as possible before February 1st, 2025. (Upgrade instructions below).

If you are using pinned SHAs, please use the SHAs of versions v4.2.0 or v3.4.0

If you do not upgrade, all workflow runs using any of the deprecated actions/cache will fail.

Upgrading to the recommended versions will not break your workflows.

4.1.2

... (truncated)

Commits
  • 9255dc7 Merge pull request #1686 from actions/cache-v5.0.1-release
  • 8ff5423 chore: release v5.0.1
  • 9233019 Merge pull request #1685 from salmanmkc/node24-storage-blob-fix
  • b975f2b fix: add peer property to package-lock.json for dependencies
  • d0a0e18 fix: update license files for @​actions/cache, fast-xml-parser, and strnum
  • 74de208 fix: update @​actions/cache to ^5.0.1 for Node.js 24 punycode fix
  • ac7f115 peer
  • b0f846b fix: update @​actions/cache with storage-blob fix for Node.js 24 punycode depr...
  • a783357 Merge pull request #1684 from actions/prepare-cache-v5-release
  • 3bb0d78 docs: highlight v5 runner requirement in releases
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/cache&package-manager=github_actions&previous-version=4.3.0&new-version=5.0.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release-artifacts.yml | 2 +- .github/workflows/tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 936aae5036c..5559a415a6f 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -66,7 +66,7 @@ jobs: install: true - name: Set up docker layer caching - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dd216fc696f..7f265a086bd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -174,7 +174,7 @@ jobs: # Cribbed from # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17 - name: Restore/persist mypy's cache - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 with: path: | .mypy_cache From 169d5b9590f392e758a7f4e8ce00e5f70a1b20ec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 14:15:00 +0000 Subject: [PATCH 083/139] Bump reqwest from 0.12.24 to 0.12.25 in the patches group (#19331) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps the patches group with 1 update: [reqwest](https://github.com/seanmonstar/reqwest). Updates `reqwest` from 0.12.24 to 0.12.25
Release notes

Sourced from reqwest's releases.

v0.12.25

Highlights

  • Add Error::is_upgrade() to determine if the error was from an HTTP upgrade.
  • Fix sending Proxy-Authorization if only username is configured.
  • Fix sending Proxy-Authorization to HTTPS proxies when the target is HTTP.
  • Refactor internal decompression handling to use tower-http.

What's Changed

New Contributors

Full Changelog: https://github.com/seanmonstar/reqwest/compare/v0.12.24...v0.12.25

Changelog

Sourced from reqwest's changelog.

v0.12.25

  • Add Error::is_upgrade() to determine if the error was from an HTTP upgrade.
  • Fix sending Proxy-Authorization if only username is configured.
  • Fix sending Proxy-Authorization to HTTPS proxies when the target is HTTP.
  • Refactor internal decompression handling to use tower-http.
Commits
  • f156a9f v0.12.25
  • fc1ff4f fix(proxy): forward Proxy-Authorization header to HTTPS proxies for HTTP targ...
  • b7c3712 Use decompression from tower-http (#2840)
  • 74e6f84 feat(error): add is_upgrade method to detect protocol upgrade errors (#2822)
  • c0c06b7 fix: send proxy-authorization even with empty password (#2868)
  • a2aa5a3 chore: minor improvement for docs (#2862)
  • 9c4999d docs: document WASM client (#2859)
  • a97e195 docs: document connection pooling behavior (#2851)
  • e3093ed tests: fix wasm timeout test with uncached response (#2853)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=reqwest&package-manager=cargo&previous-version=0.12.24&new-version=0.12.25)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 892279e4f20..e5ce9325df2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1024,9 +1024,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.24" +version = "0.12.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" +checksum = "3b4c14b2d9afca6a60277086b0cc6a6ae0b568f6f7916c943a8cdc79f8be240f" dependencies = [ "base64", "bytes", @@ -1468,9 +1468,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ "bitflags", "bytes", From 6b755f964bab89b6dd0f52eeafb2de2f7a4b3c73 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 14:53:58 +0000 Subject: [PATCH 084/139] Bump actions/upload-artifact from 5.0.0 to 6.0.0 (#19334) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 5.0.0 to 6.0.0.
Release notes

Sourced from actions/upload-artifact's releases.

v6.0.0

v6 - What's new

[!IMPORTANT] actions/upload-artifact@v6 now runs on Node.js 24 (runs.using: node24) and requires a minimum Actions Runner version of 2.327.1. If you are using self-hosted runners, ensure they are updated before upgrading.

Node.js 24

This release updates the runtime to Node.js 24. v5 had preliminary support for Node.js 24, however this action was by default still running on Node.js 20. Now this action by default will run on Node.js 24.

What's Changed

Full Changelog: https://github.com/actions/upload-artifact/compare/v5.0.0...v6.0.0

Commits
  • b7c566a Merge pull request #745 from actions/upload-artifact-v6-release
  • e516bc8 docs: correct description of Node.js 24 support in README
  • ddc45ed docs: update README to correct action name for Node.js 24 support
  • 615b319 chore: release v6.0.0 for Node.js 24 support
  • 017748b Merge pull request #744 from actions/fix-storage-blob
  • 38d4c79 chore: rebuild dist
  • 7d27270 chore: add missing license cache files for @​actions/core, @​actions/io, and mi...
  • 5f643d3 chore: update license files for @​actions/artifact@​5.0.1 dependencies
  • 1df1684 chore: update package-lock.json with @​actions/artifact@​5.0.1
  • b5b1a91 fix: update @​actions/artifact to ^5.0.0 for Node.js 24 punycode fix
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/upload-artifact&package-manager=github_actions&previous-version=5.0.0&new-version=6.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Devon Hudson --- .github/workflows/docker.yml | 2 +- .github/workflows/docs-pr.yaml | 2 +- .github/workflows/latest_deps.yml | 2 +- .github/workflows/release-artifacts.yml | 6 +++--- .github/workflows/tests.yml | 4 ++-- .github/workflows/twisted_trunk.yml | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3a038d1e581..301f25177a5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -75,7 +75,7 @@ jobs: touch "${{ runner.temp }}/digests/${digest#sha256:}" - name: Upload digest - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: digests-${{ matrix.suffix }} path: ${{ runner.temp }}/digests/* diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index 524739ba624..eed2acc4a9c 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -39,7 +39,7 @@ jobs: cp book/welcome_and_overview.html book/index.html - name: Upload Artifact - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: book path: book diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 0e27666ef17..9908633f8e4 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -173,7 +173,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 5559a415a6f..41a1473de35 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -101,7 +101,7 @@ jobs: echo "ARTIFACT_NAME=${DISTRO#*:}" >> "$GITHUB_OUTPUT" - name: Upload debs as artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: debs-${{ steps.artifact-name.outputs.ARTIFACT_NAME }} path: debs/* @@ -152,7 +152,7 @@ jobs: # musl: (TODO: investigate). CIBW_TEST_SKIP: pp3*-* *musl* - - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: Wheel-${{ matrix.os }} path: ./wheelhouse/*.whl @@ -173,7 +173,7 @@ jobs: - name: Build sdist run: python -m build --sdist - - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: Sdist path: dist/*.tar.gz diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7f265a086bd..f93c25c01f6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -561,7 +561,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }}) @@ -658,7 +658,7 @@ jobs: PGPASSWORD: postgres PGDATABASE: postgres - name: "Upload schema differences" - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 if: ${{ failure() && !cancelled() && steps.run_tester_script.outcome == 'failure' }} with: name: Schema dumps diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 64486b0f05f..2433632a7f9 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -147,7 +147,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) From 444bc56cda05953cb24f95f291d1d2906f3045cc Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Mon, 5 Jan 2026 20:35:11 +0100 Subject: [PATCH 085/139] Add rate limit conf to user directory endpoint (#19291) The goal is to avoid that an user could scrape the user directory too quickly. --- changelog.d/19291.misc | 1 + demo/start.sh | 6 ++++++ .../conf/workers-shared-extra.yaml.j2 | 4 ++++ .../configuration/config_documentation.md | 19 +++++++++++++++++++ schema/synapse-config.schema.yaml | 10 ++++++++++ synapse/config/ratelimiting.py | 6 ++++++ synapse/rest/client/user_directory.py | 9 +++++++++ 7 files changed, 55 insertions(+) create mode 100644 changelog.d/19291.misc diff --git a/changelog.d/19291.misc b/changelog.d/19291.misc new file mode 100644 index 00000000000..bac12b85065 --- /dev/null +++ b/changelog.d/19291.misc @@ -0,0 +1 @@ +Add a config to be able to rate limit search in the user directory. diff --git a/demo/start.sh b/demo/start.sh index e010302bf49..0b61ac99910 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -145,6 +145,12 @@ for port in 8080 8081 8082; do rc_delayed_event_mgmt: per_second: 1000 burst_count: 1000 + rc_room_creation: + per_second: 1000 + burst_count: 1000 + rc_user_directory: + per_second: 1000 + burst_count: 1000 RC ) echo "${ratelimiting}" >> "$port.config" diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index 94e74df9d11..101ff153a5a 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -102,6 +102,10 @@ rc_room_creation: per_second: 9999 burst_count: 9999 +rc_user_directory: + per_second: 9999 + burst_count: 9999 + federation_rr_transactions_per_room_per_second: 9999 allow_device_name_lookup_over_federation: true diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 7509e4d715e..badfe0a03f4 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -2041,6 +2041,25 @@ rc_room_creation: burst_count: 5.0 ``` --- +### `rc_user_directory` + +*(object)* This option allows admins to ratelimit searches in the user directory. + +_Added in Synapse 1.145.0._ + +This setting has the following sub-options: + +* `per_second` (number): Maximum number of requests a client can send per second. + +* `burst_count` (number): Maximum number of requests a client can send before being throttled. + +Default configuration: +```yaml +rc_user_directory: + per_second: 0.016 + burst_count: 200.0 +``` +--- ### `federation_rr_transactions_per_room_per_second` *(integer)* Sets outgoing federation transaction frequency for sending read-receipts, per-room. diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index bf9346995da..ca8db9c9eec 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -2274,6 +2274,16 @@ properties: examples: - per_second: 1.0 burst_count: 5.0 + rc_user_directory: + $ref: "#/$defs/rc" + description: >- + This option allows admins to ratelimit searches in the user directory. + + + _Added in Synapse 1.145.0._ + default: + per_second: 0.016 + burst_count: 200.0 federation_rr_transactions_per_room_per_second: type: integer description: >- diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py index 78d9d61d3c6..13c9c4dba03 100644 --- a/synapse/config/ratelimiting.py +++ b/synapse/config/ratelimiting.py @@ -252,3 +252,9 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: "rc_reports", defaults={"per_second": 1, "burst_count": 5}, ) + + self.rc_user_directory = RatelimitSettings.parse( + config, + "rc_user_directory", + defaults={"per_second": 0.016, "burst_count": 200}, + ) diff --git a/synapse/rest/client/user_directory.py b/synapse/rest/client/user_directory.py index 0f561c2e615..fa1342d0bfd 100644 --- a/synapse/rest/client/user_directory.py +++ b/synapse/rest/client/user_directory.py @@ -23,6 +23,7 @@ from typing import TYPE_CHECKING from synapse.api.errors import SynapseError +from synapse.api.ratelimiting import Ratelimiter from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseRequest @@ -46,6 +47,12 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.user_directory_handler = hs.get_user_directory_handler() + self._per_user_limiter = Ratelimiter( + store=hs.get_datastores().main, + clock=hs.get_clock(), + cfg=hs.config.ratelimiting.rc_user_directory, + ) + async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonMapping]: """Searches for users in directory @@ -69,6 +76,8 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonMapping]: if not self.hs.config.userdirectory.user_directory_search_enabled: return 200, {"limited": False, "results": []} + await self._per_user_limiter.ratelimit(requester) + body = parse_json_object_from_request(request) limit = int(body.get("limit", 10)) From cd252db3f56b3c9a8fac5892126f17bb11cce9af Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 6 Jan 2026 15:53:13 +0000 Subject: [PATCH 086/139] Transform events with client metadata before serialising in /event response. (#19340) Fix /event/ endpoint not transforming event with per-requester metadata Pass notif_event through filter_events_for_client \ Not aware of an actual issue here, but seems silly to bypass it Call it filter_and_transform_events_for_client to make it more obvious --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19340.bugfix | 1 + synapse/handlers/admin.py | 4 ++-- synapse/handlers/events.py | 10 ++++++---- synapse/handlers/initial_sync.py | 8 ++++---- synapse/handlers/pagination.py | 4 ++-- synapse/handlers/relations.py | 6 +++--- synapse/handlers/room.py | 4 ++-- synapse/handlers/search.py | 10 +++++----- synapse/handlers/sliding_sync/__init__.py | 4 ++-- synapse/handlers/sync.py | 6 +++--- synapse/notifier.py | 4 ++-- synapse/push/mailer.py | 7 +++---- synapse/visibility.py | 2 +- tests/rest/client/test_retention.py | 4 ++-- tests/test_visibility.py | 21 ++++++++++++--------- 15 files changed, 50 insertions(+), 45 deletions(-) create mode 100644 changelog.d/19340.bugfix diff --git a/changelog.d/19340.bugfix b/changelog.d/19340.bugfix new file mode 100644 index 00000000000..38de156aa7e --- /dev/null +++ b/changelog.d/19340.bugfix @@ -0,0 +1 @@ +Transform events with client metadata before serialising in /event response. \ No newline at end of file diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index c979752f7fd..2fb0e5814f2 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -44,7 +44,7 @@ UserInfo, create_requester, ) -from synapse.visibility import filter_events_for_client +from synapse.visibility import filter_and_transform_events_for_client if TYPE_CHECKING: from synapse.server import HomeServer @@ -251,7 +251,7 @@ async def export_user_data(self, user_id: str, writer: "ExfiltrationWriter") -> topological=last_event.depth, ) - events = await filter_events_for_client( + events = await filter_and_transform_events_for_client( self._storage_controllers, user_id, events, diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index ae17639206b..f6517def9c9 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -31,7 +31,7 @@ from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.streams.config import PaginationConfig from synapse.types import JsonDict, Requester, UserID -from synapse.visibility import filter_events_for_client +from synapse.visibility import filter_and_transform_events_for_client if TYPE_CHECKING: from synapse.server import HomeServer @@ -156,7 +156,9 @@ async def get_event( event_id: str, show_redacted: bool = False, ) -> EventBase | None: - """Retrieve a single specified event. + """Retrieve a single specified event on behalf of a user. + The event will be transformed in a user-specific and time-specific way, + e.g. having unsigned metadata added or being erased depending on who is accessing. Args: user: The local user requesting the event @@ -188,7 +190,7 @@ async def get_event( # The user is peeking if they aren't in the room already is_peeking = not is_user_in_room - filtered = await filter_events_for_client( + filtered = await filter_and_transform_events_for_client( self._storage_controllers, user.to_string(), [event], @@ -198,4 +200,4 @@ async def get_event( if not filtered: raise AuthError(403, "You don't have permission to access that event.") - return event + return filtered[0] diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index 611c4fa7b3d..1e5e98a59bd 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -49,7 +49,7 @@ from synapse.util import unwrapFirstError from synapse.util.async_helpers import concurrently_execute, gather_results from synapse.util.caches.response_cache import ResponseCache -from synapse.visibility import filter_events_for_client +from synapse.visibility import filter_and_transform_events_for_client if TYPE_CHECKING: from synapse.server import HomeServer @@ -225,7 +225,7 @@ async def handle_room(event: RoomsForUser) -> None: ) ).addErrback(unwrapFirstError) - messages = await filter_events_for_client( + messages = await filter_and_transform_events_for_client( self._storage_controllers, user_id, messages, @@ -382,7 +382,7 @@ async def _room_initial_sync_parted( room_id, limit=pagin_config.limit, end_token=stream_token ) - messages = await filter_events_for_client( + messages = await filter_and_transform_events_for_client( self._storage_controllers, requester.user.to_string(), messages, @@ -496,7 +496,7 @@ async def get_receipts() -> list[JsonMapping]: ).addErrback(unwrapFirstError) ) - messages = await filter_events_for_client( + messages = await filter_and_transform_events_for_client( self._storage_controllers, requester.user.to_string(), messages, diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 63e5dfa70c5..7b9c8290564 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -46,7 +46,7 @@ from synapse.types.state import StateFilter from synapse.util.async_helpers import ReadWriteLock from synapse.util.duration import Duration -from synapse.visibility import filter_events_for_client +from synapse.visibility import filter_and_transform_events_for_client if TYPE_CHECKING: from synapse.server import HomeServer @@ -684,7 +684,7 @@ async def get_messages( events = await event_filter.filter(events) if not use_admin_priviledge: - events = await filter_events_for_client( + events = await filter_and_transform_events_for_client( self._storage_controllers, user_id, events, diff --git a/synapse/handlers/relations.py b/synapse/handlers/relations.py index fd38ffa920f..d7d3002fbe3 100644 --- a/synapse/handlers/relations.py +++ b/synapse/handlers/relations.py @@ -40,7 +40,7 @@ from synapse.streams.config import PaginationConfig from synapse.types import JsonDict, Requester, UserID from synapse.util.async_helpers import gather_results -from synapse.visibility import filter_events_for_client +from synapse.visibility import filter_and_transform_events_for_client if TYPE_CHECKING: from synapse.server import HomeServer @@ -154,7 +154,7 @@ async def get_relations( [e.event_id for e in related_events] ) - events = await filter_events_for_client( + events = await filter_and_transform_events_for_client( self._storage_controllers, user_id, events, @@ -599,7 +599,7 @@ async def get_threads( # Limit the returned threads to those the user has participated in. events = [event for event in events if participated[event.event_id]] - events = await filter_events_for_client( + events = await filter_and_transform_events_for_client( self._storage_controllers, user_id, events, diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 1026bfd8766..e03a9123198 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -95,7 +95,7 @@ from synapse.util.duration import Duration from synapse.util.iterutils import batch_iter from synapse.util.stringutils import parse_and_validate_server_name -from synapse.visibility import filter_events_for_client +from synapse.visibility import filter_and_transform_events_for_client if TYPE_CHECKING: from synapse.server import HomeServer @@ -1919,7 +1919,7 @@ async def get_event_context( async def filter_evts(events: list[EventBase]) -> list[EventBase]: if use_admin_priviledge: return events - return await filter_events_for_client( + return await filter_and_transform_events_for_client( self._storage_controllers, user.to_string(), events, diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 20b38427a69..56c047b0e89 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -33,7 +33,7 @@ from synapse.events.utils import SerializeEventConfig from synapse.types import JsonDict, Requester, StrCollection, StreamKeyType, UserID from synapse.types.state import StateFilter -from synapse.visibility import filter_events_for_client +from synapse.visibility import filter_and_transform_events_for_client if TYPE_CHECKING: from synapse.server import HomeServer @@ -479,7 +479,7 @@ async def _search_by_rank( filtered_events = await search_filter.filter([r["event"] for r in results]) - events = await filter_events_for_client( + events = await filter_and_transform_events_for_client( self._storage_controllers, user.to_string(), filtered_events, @@ -580,7 +580,7 @@ async def _search_by_recent( filtered_events = await search_filter.filter([r["event"] for r in results]) - events = await filter_events_for_client( + events = await filter_and_transform_events_for_client( self._storage_controllers, user.to_string(), filtered_events, @@ -667,13 +667,13 @@ async def _calculate_event_contexts( len(res.events_after), ) - events_before = await filter_events_for_client( + events_before = await filter_and_transform_events_for_client( self._storage_controllers, user.to_string(), res.events_before, ) - events_after = await filter_events_for_client( + events_after = await filter_and_transform_events_for_client( self._storage_controllers, user.to_string(), res.events_after, diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index bb2e785cfa1..6feb6c292e9 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -71,7 +71,7 @@ ) from synapse.types.state import StateFilter from synapse.util.async_helpers import concurrently_execute -from synapse.visibility import filter_events_for_client +from synapse.visibility import filter_and_transform_events_for_client if TYPE_CHECKING: from synapse.server import HomeServer @@ -755,7 +755,7 @@ async def get_room_sync_data( timeline_events.reverse() # Make sure we don't expose any events that the client shouldn't see - timeline_events = await filter_events_for_client( + timeline_events = await filter_and_transform_events_for_client( self.storage_controllers, user.to_string(), timeline_events, diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 60d88274255..72e91d66ac4 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -78,7 +78,7 @@ from synapse.util.caches.lrucache import LruCache from synapse.util.caches.response_cache import ResponseCache, ResponseCacheContext from synapse.util.metrics import Measure -from synapse.visibility import filter_events_for_client +from synapse.visibility import filter_and_transform_events_for_client if TYPE_CHECKING: from synapse.server import HomeServer @@ -679,7 +679,7 @@ async def _load_filtered_recents( ) ) - recents = await filter_events_for_client( + recents = await filter_and_transform_events_for_client( self._storage_controllers, sync_config.user.to_string(), recents, @@ -789,7 +789,7 @@ async def _load_filtered_recents( ) ) - loaded_recents = await filter_events_for_client( + loaded_recents = await filter_and_transform_events_for_client( self._storage_controllers, sync_config.user.to_string(), loaded_recents, diff --git a/synapse/notifier.py b/synapse/notifier.py index d8d2db17f12..cf3923110e3 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -63,7 +63,7 @@ ) from synapse.util.duration import Duration from synapse.util.stringutils import shortstr -from synapse.visibility import filter_events_for_client +from synapse.visibility import filter_and_transform_events_for_client if TYPE_CHECKING: from synapse.server import HomeServer @@ -783,7 +783,7 @@ async def check_for_updates( ) if keyname == StreamKeyType.ROOM: - new_events = await filter_events_for_client( + new_events = await filter_and_transform_events_for_client( self._storage_controllers, user.to_string(), new_events, diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 64922074035..d18630e80ba 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -49,7 +49,7 @@ from synapse.types import StateMap, UserID from synapse.types.state import StateFilter from synapse.util.async_helpers import concurrently_execute -from synapse.visibility import filter_events_for_client +from synapse.visibility import filter_and_transform_events_for_client if TYPE_CHECKING: from synapse.server import HomeServer @@ -537,12 +537,11 @@ async def _get_notif_vars( "messages": [], } - the_events = await filter_events_for_client( + the_events = await filter_and_transform_events_for_client( self._storage_controllers, user_id, - results.events_before, + results.events_before + [notif_event], ) - the_events.append(notif_event) for event in the_events: messagevars = await self._get_message_vars(notif, event, room_state_ids) diff --git a/synapse/visibility.py b/synapse/visibility.py index bfa0db5670d..452a2d50fbb 100644 --- a/synapse/visibility.py +++ b/synapse/visibility.py @@ -75,7 +75,7 @@ @trace -async def filter_events_for_client( +async def filter_and_transform_events_for_client( storage: StorageControllers, user_id: str, events: list[EventBase], diff --git a/tests/rest/client/test_retention.py b/tests/rest/client/test_retention.py index 758d62e63b7..82a3b5b3378 100644 --- a/tests/rest/client/test_retention.py +++ b/tests/rest/client/test_retention.py @@ -28,7 +28,7 @@ from synapse.server import HomeServer from synapse.types import JsonDict, create_requester from synapse.util.clock import Clock -from synapse.visibility import filter_events_for_client +from synapse.visibility import filter_and_transform_events_for_client from tests import unittest from tests.unittest import override_config @@ -163,7 +163,7 @@ def test_visibility(self) -> None: ) self.assertEqual(2, len(events), "events retrieved from database") filtered_events = self.get_success( - filter_events_for_client( + filter_and_transform_events_for_client( storage_controllers, self.user_id, events, diff --git a/tests/test_visibility.py b/tests/test_visibility.py index 06598c29de6..b50faa2a499 100644 --- a/tests/test_visibility.py +++ b/tests/test_visibility.py @@ -31,7 +31,10 @@ from synapse.server import HomeServer from synapse.types import create_requester from synapse.util.clock import Clock -from synapse.visibility import filter_events_for_client, filter_events_for_server +from synapse.visibility import ( + filter_and_transform_events_for_client, + filter_events_for_server, +) from tests import unittest from tests.test_utils.event_injection import inject_event, inject_member_event @@ -330,7 +333,7 @@ def test_normal_operation_as_admin(self) -> None: # Do filter & assert filtered_events = self.get_success( - filter_events_for_client( + filter_and_transform_events_for_client( self.hs.get_storage_controllers(), "@admin:test", events_to_filter, @@ -369,7 +372,7 @@ def test_see_soft_failed_events(self) -> None: # Do filter & assert filtered_events = self.get_success( - filter_events_for_client( + filter_and_transform_events_for_client( self.hs.get_storage_controllers(), "@admin:test", events_to_filter, @@ -416,7 +419,7 @@ def test_see_policy_server_spammy_events(self) -> None: # Do filter & assert filtered_events = self.get_success( - filter_events_for_client( + filter_and_transform_events_for_client( self.hs.get_storage_controllers(), "@admin:test", events_to_filter, @@ -463,7 +466,7 @@ def test_see_soft_failed_and_policy_server_spammy_events(self) -> None: # Do filter & assert filtered_events = self.get_success( - filter_events_for_client( + filter_and_transform_events_for_client( self.hs.get_storage_controllers(), "@admin:test", events_to_filter, @@ -538,14 +541,14 @@ def test_joined_history_visibility(self) -> None: # accidentally serving the same event object (with the same unsigned.membership # property) to both users. joiner_filtered_events = self.get_success( - filter_events_for_client( + filter_and_transform_events_for_client( self.hs.get_storage_controllers(), "@joiner:test", events_to_filter, ) ) resident_filtered_events = self.get_success( - filter_events_for_client( + filter_and_transform_events_for_client( self.hs.get_storage_controllers(), "@resident:test", events_to_filter, @@ -641,7 +644,7 @@ def test_out_of_band_invite_rejection(self) -> None: # the invited user should be able to see both the invite and the rejection filtered_events = self.get_success( - filter_events_for_client( + filter_and_transform_events_for_client( self.hs.get_storage_controllers(), "@user:test", [invite_event, reject_event], @@ -662,7 +665,7 @@ def test_out_of_band_invite_rejection(self) -> None: # other users should see neither self.assertEqual( self.get_success( - filter_events_for_client( + filter_and_transform_events_for_client( self.hs.get_storage_controllers(), "@other:test", [invite_event, reject_event], From 39f80296c5683bfa2c06a7ade56d4c95f0e30b2a Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 6 Jan 2026 09:38:44 -0700 Subject: [PATCH 087/139] 1.145.0rc1 --- CHANGES.md | 57 +++++++++++++++++++++++++++++++ changelog.d/18402.misc | 1 - changelog.d/19187.misc | 1 - changelog.d/19206.bugfix | 1 - changelog.d/19212.misc | 1 - changelog.d/19231.bugfix | 1 - changelog.d/19232.misc | 1 - changelog.d/19234.misc | 1 - changelog.d/19253.misc | 1 - changelog.d/19254.removal | 1 - changelog.d/19258.misc | 1 - changelog.d/19260.feature | 1 - changelog.d/19262.misc | 1 - changelog.d/19267.bugfix | 1 - changelog.d/19268.feature | 1 - changelog.d/19270.doc | 1 - changelog.d/19274.bugfix | 1 - changelog.d/19275.feature | 1 - changelog.d/19278.misc | 1 - changelog.d/19279.feature | 1 - changelog.d/19281.feature | 1 - changelog.d/19289.misc | 1 - changelog.d/19291.misc | 1 - changelog.d/19297.misc | 1 - changelog.d/19300.feature | 1 - changelog.d/19302.misc | 1 - changelog.d/19323.misc | 1 - changelog.d/19324.docker | 1 - changelog.d/19326.misc | 1 - changelog.d/19327.misc | 1 - changelog.d/19340.bugfix | 1 - debian/changelog | 6 ++++ pyproject.toml | 2 +- schema/synapse-config.schema.yaml | 2 +- 34 files changed, 65 insertions(+), 32 deletions(-) delete mode 100644 changelog.d/18402.misc delete mode 100644 changelog.d/19187.misc delete mode 100644 changelog.d/19206.bugfix delete mode 100644 changelog.d/19212.misc delete mode 100644 changelog.d/19231.bugfix delete mode 100644 changelog.d/19232.misc delete mode 100644 changelog.d/19234.misc delete mode 100644 changelog.d/19253.misc delete mode 100644 changelog.d/19254.removal delete mode 100644 changelog.d/19258.misc delete mode 100644 changelog.d/19260.feature delete mode 100644 changelog.d/19262.misc delete mode 100644 changelog.d/19267.bugfix delete mode 100644 changelog.d/19268.feature delete mode 100644 changelog.d/19270.doc delete mode 100644 changelog.d/19274.bugfix delete mode 100644 changelog.d/19275.feature delete mode 100644 changelog.d/19278.misc delete mode 100644 changelog.d/19279.feature delete mode 100644 changelog.d/19281.feature delete mode 100644 changelog.d/19289.misc delete mode 100644 changelog.d/19291.misc delete mode 100644 changelog.d/19297.misc delete mode 100644 changelog.d/19300.feature delete mode 100644 changelog.d/19302.misc delete mode 100644 changelog.d/19323.misc delete mode 100644 changelog.d/19324.docker delete mode 100644 changelog.d/19326.misc delete mode 100644 changelog.d/19327.misc delete mode 100644 changelog.d/19340.bugfix diff --git a/CHANGES.md b/CHANGES.md index 83ab1849e61..1f6ed01c203 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,60 @@ +# Synapse 1.145.0rc1 (2026-01-06) + +- Ubuntu 25.04 (Plucky Puffin) will be end of life on Jan 17, 2026. Synapse will stop building packages for Ubuntu 25.04 shortly thereafter. +- Remove the "Updates to locked dependencies" section from the changelog due to lack of use and the maintenance burden. ([\#19254](https://github.com/element-hq/synapse/issues/19254)) + +## Features + +- Add `memberships` endpoint to the admin API. This is useful for forensics and T&S purpose. ([\#19260](https://github.com/element-hq/synapse/issues/19260)) +- Add an admin API for retrieving a paginated list of quarantined media. ([\#19268](https://github.com/element-hq/synapse/issues/19268)) +- Server admins can bypass the quarantine media check when downloading media by setting the `admin_unsafely_bypass_quarantine` query parameter to `true` on Client-Server API media download requests. ([\#19275](https://github.com/element-hq/synapse/issues/19275)) +- Implemented pagination for the [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666) mutual rooms endpoint. Contributed by @tulir @ Beeper. ([\#19279](https://github.com/element-hq/synapse/issues/19279)) +- Admin API: add worker support to `GET /_synapse/admin/v2/users/`. ([\#19281](https://github.com/element-hq/synapse/issues/19281)) +- Improve proxy support for the `federation_client.py` dev script. Contributed by Denis Kasak (@dkasak). ([\#19300](https://github.com/element-hq/synapse/issues/19300)) + +## Bugfixes + +- Fix sliding sync performance slow down for long lived connections. ([\#19206](https://github.com/element-hq/synapse/issues/19206)) +- Fix a bug where Mastodon posts (and possibly other embeds) have the wrong description for URL previews. ([\#19231](https://github.com/element-hq/synapse/issues/19231)) +- Fix bug where `Duration` was logged incorrectly. ([\#19267](https://github.com/element-hq/synapse/issues/19267)) +- Fix bug introduced in 1.143.0 that broke support for versions of `zope-interface` older than 6.2. ([\#19274](https://github.com/element-hq/synapse/issues/19274)) +- Transform events with client metadata before serialising in /event response. ([\#19340](https://github.com/element-hq/synapse/issues/19340)) + +## Updates to the Docker image + +- Add a way to expose metrics from the Docker image (`SYNAPSE_ENABLE_METRICS`). ([\#19324](https://github.com/element-hq/synapse/issues/19324)) + +## Improved Documentation + +- Document the importance of `public_baseurl` when configuring OpenID Connect authentication. ([\#19270](https://github.com/element-hq/synapse/issues/19270)) + +## Deprecations and Removals + +- Ubuntu 25.04 (Plucky Puffin) will be end of life on Jan 17, 2026. Synapse will stop building packages for Ubuntu 25.04 shortly thereafter. +- Remove the "Updates to locked dependencies" section from the changelog due to lack of use and the maintenance burden. ([\#19254](https://github.com/element-hq/synapse/issues/19254)) + +## Internal Changes + +- Group together dependabot update PRs to reduce the review load. ([\#18402](https://github.com/element-hq/synapse/issues/18402)) +- Fix `HomeServer.shutdown()` failing if the homeserver hasn't been setup yet. ([\#19187](https://github.com/element-hq/synapse/issues/19187)) +- Respond with useful error codes with `Content-Length` header/s are invalid. ([\#19212](https://github.com/element-hq/synapse/issues/19212)) +- Fix `HomeServer.shutdown()` failing if the homeserver failed to `start`. ([\#19232](https://github.com/element-hq/synapse/issues/19232)) +- Switch the build backend from `poetry-core` to `maturin`. ([\#19234](https://github.com/element-hq/synapse/issues/19234)) +- Raise the limit for concurrently-open non-security @dependabot PRs from 5 to 10. ([\#19253](https://github.com/element-hq/synapse/issues/19253)) +- Require 14 days to pass before pulling in general dependency updates to help mitigate upstream supply chain attacks. ([\#19258](https://github.com/element-hq/synapse/issues/19258)) +- Drop the broken netlify documentation workflow until a new one is implemented. ([\#19262](https://github.com/element-hq/synapse/issues/19262)) +- Don't include debug logs in `Clock` unless explicitly enabled. ([\#19278](https://github.com/element-hq/synapse/issues/19278)) +- Use `uv` to test olddeps to ensure all transitive dependencies use minimum versions. ([\#19289](https://github.com/element-hq/synapse/issues/19289)) +- Add a config to be able to rate limit search in the user directory. ([\#19291](https://github.com/element-hq/synapse/issues/19291)) +- Log the original bind exception when encountering `Failed to listen on 0.0.0.0, continuing because listening on [::]`. ([\#19297](https://github.com/element-hq/synapse/issues/19297)) +- Unpin the version of Rust we use to build Synapse wheels (was 1.82.0) now that MacOS support has been dropped. ([\#19302](https://github.com/element-hq/synapse/issues/19302)) +- Make it more clear how `shared_extra_conf` is combined in our Docker configuration scripts. ([\#19323](https://github.com/element-hq/synapse/issues/19323)) +- Update CI to stream Complement progress and format logs in a separate step after all tests are done. ([\#19326](https://github.com/element-hq/synapse/issues/19326)) +- Format `.github/workflows/tests.yml`. ([\#19327](https://github.com/element-hq/synapse/issues/19327)) + + + + # Synapse 1.144.0 (2025-12-09) ## Deprecation of MacOS Python wheels diff --git a/changelog.d/18402.misc b/changelog.d/18402.misc deleted file mode 100644 index 4b13652845c..00000000000 --- a/changelog.d/18402.misc +++ /dev/null @@ -1 +0,0 @@ -Group together dependabot update PRs to reduce the review load. \ No newline at end of file diff --git a/changelog.d/19187.misc b/changelog.d/19187.misc deleted file mode 100644 index d831de38c8f..00000000000 --- a/changelog.d/19187.misc +++ /dev/null @@ -1 +0,0 @@ -Fix `HomeServer.shutdown()` failing if the homeserver hasn't been setup yet. diff --git a/changelog.d/19206.bugfix b/changelog.d/19206.bugfix deleted file mode 100644 index 9cdfaa2571b..00000000000 --- a/changelog.d/19206.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix sliding sync performance slow down for long lived connections. diff --git a/changelog.d/19212.misc b/changelog.d/19212.misc deleted file mode 100644 index 83158ce2d9c..00000000000 --- a/changelog.d/19212.misc +++ /dev/null @@ -1 +0,0 @@ -Respond with useful error codes with `Content-Length` header/s are invalid. diff --git a/changelog.d/19231.bugfix b/changelog.d/19231.bugfix deleted file mode 100644 index 580b642bb23..00000000000 --- a/changelog.d/19231.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where Mastodon posts (and possibly other embeds) have the wrong description for URL previews. diff --git a/changelog.d/19232.misc b/changelog.d/19232.misc deleted file mode 100644 index 6e3e2ff6494..00000000000 --- a/changelog.d/19232.misc +++ /dev/null @@ -1 +0,0 @@ -Fix `HomeServer.shutdown()` failing if the homeserver failed to `start`. diff --git a/changelog.d/19234.misc b/changelog.d/19234.misc deleted file mode 100644 index d79bc0b19fc..00000000000 --- a/changelog.d/19234.misc +++ /dev/null @@ -1 +0,0 @@ -Switch the build backend from `poetry-core` to `maturin`. \ No newline at end of file diff --git a/changelog.d/19253.misc b/changelog.d/19253.misc deleted file mode 100644 index 1d45f936f64..00000000000 --- a/changelog.d/19253.misc +++ /dev/null @@ -1 +0,0 @@ -Raise the limit for concurrently-open non-security @dependabot PRs from 5 to 10. \ No newline at end of file diff --git a/changelog.d/19254.removal b/changelog.d/19254.removal deleted file mode 100644 index ee527cef994..00000000000 --- a/changelog.d/19254.removal +++ /dev/null @@ -1 +0,0 @@ -Remove the "Updates to locked dependencies" section from the changelog due to lack of use and the maintenance burden. \ No newline at end of file diff --git a/changelog.d/19258.misc b/changelog.d/19258.misc deleted file mode 100644 index 9155f9d20fe..00000000000 --- a/changelog.d/19258.misc +++ /dev/null @@ -1 +0,0 @@ -Require 14 days to pass before pulling in general dependency updates to help mitigate upstream supply chain attacks. \ No newline at end of file diff --git a/changelog.d/19260.feature b/changelog.d/19260.feature deleted file mode 100644 index 19b192a009c..00000000000 --- a/changelog.d/19260.feature +++ /dev/null @@ -1 +0,0 @@ -Add `memberships` endpoint to the admin API. This is useful for forensics and T&S purpose. diff --git a/changelog.d/19262.misc b/changelog.d/19262.misc deleted file mode 100644 index 31906e6623b..00000000000 --- a/changelog.d/19262.misc +++ /dev/null @@ -1 +0,0 @@ -Drop the broken netlify documentation workflow until a new one is implemented. \ No newline at end of file diff --git a/changelog.d/19267.bugfix b/changelog.d/19267.bugfix deleted file mode 100644 index 6c7ed750eca..00000000000 --- a/changelog.d/19267.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where `Duration` was logged incorrectly. diff --git a/changelog.d/19268.feature b/changelog.d/19268.feature deleted file mode 100644 index cb7035fee2a..00000000000 --- a/changelog.d/19268.feature +++ /dev/null @@ -1 +0,0 @@ -Add an admin API for retrieving a paginated list of quarantined media. \ No newline at end of file diff --git a/changelog.d/19270.doc b/changelog.d/19270.doc deleted file mode 100644 index fdb7e2e51c7..00000000000 --- a/changelog.d/19270.doc +++ /dev/null @@ -1 +0,0 @@ -Document the importance of `public_baseurl` when configuring OpenID Connect authentication. diff --git a/changelog.d/19274.bugfix b/changelog.d/19274.bugfix deleted file mode 100644 index 92aaa0fe6d1..00000000000 --- a/changelog.d/19274.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug introduced in 1.143.0 that broke support for versions of `zope-interface` older than 6.2. diff --git a/changelog.d/19275.feature b/changelog.d/19275.feature deleted file mode 100644 index 5147c546cf4..00000000000 --- a/changelog.d/19275.feature +++ /dev/null @@ -1 +0,0 @@ -Server admins can bypass the quarantine media check when downloading media by setting the `admin_unsafely_bypass_quarantine` query parameter to `true` on Client-Server API media download requests. \ No newline at end of file diff --git a/changelog.d/19278.misc b/changelog.d/19278.misc deleted file mode 100644 index d1425ff38c2..00000000000 --- a/changelog.d/19278.misc +++ /dev/null @@ -1 +0,0 @@ -Don't include debug logs in `Clock` unless explicitly enabled. diff --git a/changelog.d/19279.feature b/changelog.d/19279.feature deleted file mode 100644 index 031e48dcebf..00000000000 --- a/changelog.d/19279.feature +++ /dev/null @@ -1 +0,0 @@ -Implemented pagination for the [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666) mutual rooms endpoint. Contributed by @tulir @ Beeper. diff --git a/changelog.d/19281.feature b/changelog.d/19281.feature deleted file mode 100644 index 78d3002d907..00000000000 --- a/changelog.d/19281.feature +++ /dev/null @@ -1 +0,0 @@ -Admin API: add worker support to `GET /_synapse/admin/v2/users/`. diff --git a/changelog.d/19289.misc b/changelog.d/19289.misc deleted file mode 100644 index 4ad0dbc4300..00000000000 --- a/changelog.d/19289.misc +++ /dev/null @@ -1 +0,0 @@ -Use `uv` to test olddeps to ensure all transitive dependencies use minimum versions. diff --git a/changelog.d/19291.misc b/changelog.d/19291.misc deleted file mode 100644 index bac12b85065..00000000000 --- a/changelog.d/19291.misc +++ /dev/null @@ -1 +0,0 @@ -Add a config to be able to rate limit search in the user directory. diff --git a/changelog.d/19297.misc b/changelog.d/19297.misc deleted file mode 100644 index aec97fd973f..00000000000 --- a/changelog.d/19297.misc +++ /dev/null @@ -1 +0,0 @@ -Log the original bind exception when encountering `Failed to listen on 0.0.0.0, continuing because listening on [::]`. diff --git a/changelog.d/19300.feature b/changelog.d/19300.feature deleted file mode 100644 index 97e43e9b28f..00000000000 --- a/changelog.d/19300.feature +++ /dev/null @@ -1 +0,0 @@ -Improve proxy support for the `federation_client.py` dev script. Contributed by Denis Kasak (@dkasak). diff --git a/changelog.d/19302.misc b/changelog.d/19302.misc deleted file mode 100644 index 606ab5b52de..00000000000 --- a/changelog.d/19302.misc +++ /dev/null @@ -1 +0,0 @@ -Unpin the version of Rust we use to build Synapse wheels (was 1.82.0) now that MacOS support has been dropped. \ No newline at end of file diff --git a/changelog.d/19323.misc b/changelog.d/19323.misc deleted file mode 100644 index 6699d7ea543..00000000000 --- a/changelog.d/19323.misc +++ /dev/null @@ -1 +0,0 @@ -Make it more clear how `shared_extra_conf` is combined in our Docker configuration scripts. diff --git a/changelog.d/19324.docker b/changelog.d/19324.docker deleted file mode 100644 index 52bf9cb7ae1..00000000000 --- a/changelog.d/19324.docker +++ /dev/null @@ -1 +0,0 @@ -Add a way to expose metrics from the Docker image (`SYNAPSE_ENABLE_METRICS`). diff --git a/changelog.d/19326.misc b/changelog.d/19326.misc deleted file mode 100644 index 37493c74888..00000000000 --- a/changelog.d/19326.misc +++ /dev/null @@ -1 +0,0 @@ -Update CI to stream Complement progress and format logs in a separate step after all tests are done. diff --git a/changelog.d/19327.misc b/changelog.d/19327.misc deleted file mode 100644 index d61a66907e5..00000000000 --- a/changelog.d/19327.misc +++ /dev/null @@ -1 +0,0 @@ -Format `.github/workflows/tests.yml`. diff --git a/changelog.d/19340.bugfix b/changelog.d/19340.bugfix deleted file mode 100644 index 38de156aa7e..00000000000 --- a/changelog.d/19340.bugfix +++ /dev/null @@ -1 +0,0 @@ -Transform events with client metadata before serialising in /event response. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index 15ff7cbd9d1..3b7d0b773ec 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.145.0~rc1) stable; urgency=medium + + * New Synapse release 1.145.0rc1. + + -- Synapse Packaging team Tue, 06 Jan 2026 09:29:39 -0700 + matrix-synapse-py3 (1.144.0) stable; urgency=medium * New Synapse release 1.144.0. diff --git a/pyproject.toml b/pyproject.toml index 09ca2a9e77f..7aa5840a9df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.144.0" +version = "1.145.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index ca8db9c9eec..3b58db6938d 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/schema/synapse/v1.144/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.145/synapse-config.schema.json type: object properties: modules: From d6d1404a8ee36501b733d780d44d91dbe42cd806 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 6 Jan 2026 09:49:48 -0700 Subject: [PATCH 088/139] Add nifty titles to top level deprecations --- CHANGES.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1f6ed01c203..d075da21c2d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,12 @@ # Synapse 1.145.0rc1 (2026-01-06) -- Ubuntu 25.04 (Plucky Puffin) will be end of life on Jan 17, 2026. Synapse will stop building packages for Ubuntu 25.04 shortly thereafter. -- Remove the "Updates to locked dependencies" section from the changelog due to lack of use and the maintenance burden. ([\#19254](https://github.com/element-hq/synapse/issues/19254)) +## End of Life of Ubuntu 25.04 Plucky Puffin + +Ubuntu 25.04 (Plucky Puffin) will be end of life on Jan 17, 2026. Synapse will stop building packages for Ubuntu 25.04 shortly thereafter. + +## Updates to Locked Dependencies No Longer Included in Changelog + +The "Updates to locked dependencies" section has been removed from the changelog due to lack of use and the maintenance burden. ([\#19254](https://github.com/element-hq/synapse/issues/19254)) ## Features From 1500733f4a9f4ee5313149077547368e47182053 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 6 Jan 2026 12:30:21 -0500 Subject: [PATCH 089/139] Replace usage of deprecated `assertEquals` with `assertEqual` (#19345) --- changelog.d/19345.misc | 1 + tests/handlers/test_oauth_delegation.py | 12 ++++++------ tests/rest/admin/test_room.py | 4 ++-- tests/state/test_v21.py | 2 +- tests/storage/test_event_federation.py | 2 +- tests/types/test_init.py | 10 +++++----- 6 files changed, 16 insertions(+), 15 deletions(-) create mode 100644 changelog.d/19345.misc diff --git a/changelog.d/19345.misc b/changelog.d/19345.misc new file mode 100644 index 00000000000..3025af82c90 --- /dev/null +++ b/changelog.d/19345.misc @@ -0,0 +1 @@ +Replace usage of deprecated `assertEquals` with `assertEqual` in unit test code. \ No newline at end of file diff --git a/tests/handlers/test_oauth_delegation.py b/tests/handlers/test_oauth_delegation.py index b04aeffb923..c88f2c2d155 100644 --- a/tests/handlers/test_oauth_delegation.py +++ b/tests/handlers/test_oauth_delegation.py @@ -889,11 +889,11 @@ def test_simple_introspection(self) -> None: ) ) - self.assertEquals(requester.user.to_string(), USER_ID) - self.assertEquals(requester.device_id, DEVICE) + self.assertEqual(requester.user.to_string(), USER_ID) + self.assertEqual(requester.device_id, DEVICE) self.assertFalse(self.get_success(self._auth.is_server_admin(requester))) - self.assertEquals( + self.assertEqual( self.server.last_token_seen, "some_token", ) @@ -912,11 +912,11 @@ def test_unexpiring_token(self) -> None: ) ) - self.assertEquals(requester.user.to_string(), USER_ID) - self.assertEquals(requester.device_id, DEVICE) + self.assertEqual(requester.user.to_string(), USER_ID) + self.assertEqual(requester.device_id, DEVICE) self.assertFalse(self.get_success(self._auth.is_server_admin(requester))) - self.assertEquals( + self.assertEqual( self.server.last_token_seen, "some_token", ) diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index ad713b4da4d..4070bcaeaa6 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -3362,7 +3362,7 @@ def test_v12_room(self) -> None: pl = self.helper.get_state( room_id, EventTypes.PowerLevels, tok=self.creator_tok ) - self.assertEquals(pl["users"][self.admin_user], 100) + self.assertEqual(pl["users"][self.admin_user], 100) def test_v12_room_with_many_user_pls(self) -> None: """Test that you can be promoted to the admin user's PL in v12 rooms that contain a range of user PLs.""" @@ -3395,7 +3395,7 @@ def test_v12_room_with_many_user_pls(self) -> None: pl = self.helper.get_state( room_id, EventTypes.PowerLevels, tok=self.creator_tok ) - self.assertEquals(pl["users"][self.admin_user], 100) + self.assertEqual(pl["users"][self.admin_user], 100) class BlockRoomTestCase(unittest.HomeserverTestCase): diff --git a/tests/state/test_v21.py b/tests/state/test_v21.py index 58d800f921d..9f607176c92 100644 --- a/tests/state/test_v21.py +++ b/tests/state/test_v21.py @@ -447,7 +447,7 @@ def resolve_and_check() -> None: ) ) # no matter how many events are persisted, the overall diff should always be the same. - self.assertEquals(got_auth_diff, got_auth_diff2) + self.assertEqual(got_auth_diff, got_auth_diff2) # now we will drip feed in `events` one-by-one, persisting them then resolving with the # rest. This ensures we correctly handle mixed persisted/unpersisted events. We will finish diff --git a/tests/storage/test_event_federation.py b/tests/storage/test_event_federation.py index 508f82de4f7..59e914ca8bd 100644 --- a/tests/storage/test_event_federation.py +++ b/tests/storage/test_event_federation.py @@ -909,7 +909,7 @@ def run_test(txn: LoggingTransaction, test_case: TestCase) -> None: test_case.conflicted, test_case.additional_backwards_reachable, ) - self.assertEquals( + self.assertEqual( result.conflicted_subgraph, test_case.want_conflicted_subgraph, f"{test_case.name} : conflicted subgraph mismatch", diff --git a/tests/types/test_init.py b/tests/types/test_init.py index b7a5b93ce98..8ca9bc79732 100644 --- a/tests/types/test_init.py +++ b/tests/types/test_init.py @@ -8,13 +8,13 @@ class RoomIDTestCase(TestCase): def test_can_create_msc4291_room_ids(self) -> None: valid_msc4291_room_id = "!31hneApxJ_1o-63DmFrpeqnkFfWppnzWso1JvH3ogLM" room_id = RoomID.from_string(valid_msc4291_room_id) - self.assertEquals(RoomID.is_valid(valid_msc4291_room_id), True) - self.assertEquals( + self.assertEqual(RoomID.is_valid(valid_msc4291_room_id), True) + self.assertEqual( room_id.to_string(), valid_msc4291_room_id, ) - self.assertEquals(room_id.id, "!31hneApxJ_1o-63DmFrpeqnkFfWppnzWso1JvH3ogLM") - self.assertEquals(room_id.get_domain(), None) + self.assertEqual(room_id.id, "!31hneApxJ_1o-63DmFrpeqnkFfWppnzWso1JvH3ogLM") + self.assertEqual(room_id.get_domain(), None) def test_cannot_create_invalid_msc4291_room_ids(self) -> None: invalid_room_ids = [ @@ -47,5 +47,5 @@ def test_can_create_valid_legacy_room_ids(self) -> None: ] for room_id_str in valid_room_ids: room_id = RoomID.from_string(room_id_str) - self.assertEquals(RoomID.is_valid(room_id_str), True) + self.assertEqual(RoomID.is_valid(room_id_str), True) self.assertIsNotNone(room_id.get_domain()) From ac6463c6daa81bb6f60c2055d0fd7a49207dd521 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 6 Jan 2026 19:34:38 +0200 Subject: [PATCH 090/139] Fix media creation being ratelimited for appservices (#19335) Co-authored-by: Andrew Morgan --- changelog.d/19335.bugfix | 1 + synapse/rest/media/create_resource.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 changelog.d/19335.bugfix diff --git a/changelog.d/19335.bugfix b/changelog.d/19335.bugfix new file mode 100644 index 00000000000..d5566b97770 --- /dev/null +++ b/changelog.d/19335.bugfix @@ -0,0 +1 @@ +Fixed parallel calls to `/_matrix/media/v1/create` being ratelimited for appservices even if `rate_limited: false` was set in the registration. Contributed by @tulir @ Beeper. diff --git a/synapse/rest/media/create_resource.py b/synapse/rest/media/create_resource.py index e45df11c9fa..8bef1f4c080 100644 --- a/synapse/rest/media/create_resource.py +++ b/synapse/rest/media/create_resource.py @@ -60,15 +60,16 @@ async def on_POST(self, request: SynapseRequest) -> None: # If the create media requests for the user are over the limit, drop them. await self._create_media_rate_limiter.ratelimit(requester) - ( - reached_pending_limit, - first_expiration_ts, - ) = await self.media_repo.reached_pending_media_limit(requester.user) - if reached_pending_limit: - raise LimitExceededError( - limiter_name="max_pending_media_uploads", - retry_after_ms=first_expiration_ts - self.clock.time_msec(), - ) + if not requester.app_service or requester.app_service.is_rate_limited(): + ( + reached_pending_limit, + first_expiration_ts, + ) = await self.media_repo.reached_pending_media_limit(requester.user) + if reached_pending_limit: + raise LimitExceededError( + limiter_name="max_pending_media_uploads", + retry_after_ms=first_expiration_ts - self.clock.time_msec(), + ) content_uri, unused_expires_at = await self.media_repo.create_media_id( requester.user From 18ef7f30856db3d539e8ce97ba34fac64961fe0e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Jan 2026 19:31:09 +0000 Subject: [PATCH 091/139] Bump pynacl from 1.5.0 to 1.6.2 (#19350) Bumps [pynacl](https://github.com/pyca/pynacl) from 1.5.0 to 1.6.2.
Changelog

Sourced from pynacl's changelog.

1.6.2 (2026-01-01)

  • Updated libsodium to 1.0.20-stable (2025-12-31 build) to resolve CVE-2025-69277.

1.6.1 (2025-11-10)

  • The MAKE environment variable can now be used to specify the make binary that should be used in the build process.

1.6.0 (2025-09-11)

  • BACKWARDS INCOMPATIBLE: Removed support for Python 3.6 and 3.7.
  • Added support for the low level AEAD AES bindings.
  • Added support for crypto_core_ed25519_from_uniform.
  • Update libsodium to 1.0.20-stable (2025-08-27 build).
  • Added support for free-threaded Python 3.14.
  • Added support for Windows on ARM wheels.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=pynacl&package-manager=pip&previous-version=1.5.0&new-version=1.6.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/element-hq/synapse/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 89 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 37 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3f8607b6874..7276d005435 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,7 +31,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" files = [ {file = "authlib-1.6.5-py2.py3-none-any.whl", hash = "sha256:3e0e0507807f842b02175507bdee8957a1d5707fd4afb17c32fb43fee90b6e3a"}, {file = "authlib-1.6.5.tar.gz", hash = "sha256:6aaf9c79b7cc96c900f0b284061691c5d4e61221640a948fe690b556a6d6d10b"}, @@ -481,7 +481,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -506,7 +506,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "elementpath-4.1.5-py3-none-any.whl", hash = "sha256:2ac1a2fb31eb22bbbf817f8cf6752f844513216263f0e3892c8e79782fe4bb55"}, {file = "elementpath-4.1.5.tar.gz", hash = "sha256:c2d6dc524b29ef751ecfc416b0627668119d8812441c555d7471da41d4bacb8d"}, @@ -556,7 +556,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "hiredis-3.3.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:9937d9b69321b393fbace69f55423480f098120bc55a3316e1ca3508c4dbbd6f"}, {file = "hiredis-3.3.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:50351b77f89ba6a22aff430b993653847f36b71d444509036baa0f2d79d1ebf4"}, @@ -879,7 +879,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1017,7 +1017,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1119,7 +1119,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"url-preview\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"url-preview\"" files = [ {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e77dd455b9a16bbd2a5036a63ddbd479c19572af81b624e79ef422f929eef388"}, {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d444858b9f07cefff6455b983aea9a67f7462ba1f6cbe4a21e8bf6791bf2153"}, @@ -1405,7 +1405,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "matrix-synapse-ldap3-0.3.0.tar.gz", hash = "sha256:8bb6517173164d4b9cc44f49de411d8cebdb2e705d5dd1ea1f38733c4a009e1d"}, {file = "matrix_synapse_ldap3-0.3.0-py3-none-any.whl", hash = "sha256:8b4d701f8702551e98cc1d8c20dbed532de5613584c08d0df22de376ba99159d"}, @@ -1648,7 +1648,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1838,7 +1838,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"postgres\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"postgres\"" files = [ {file = "psycopg2-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:103e857f46bb76908768ead4e2d0ba1d1a130e7b8ed77d3ae91e8b33481813e8"}, {file = "psycopg2-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:210daed32e18f35e3140a1ebe059ac29209dd96468f2f7559aa59f75ee82a5cb"}, @@ -1856,7 +1856,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -1872,7 +1872,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2154,7 +2154,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"cache-memory\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"cache-memory\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2162,30 +2162,45 @@ files = [ [[package]] name = "pynacl" -version = "1.5.0" +version = "1.6.2" description = "Python binding to the Networking and Cryptography (NaCl) library" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" groups = ["main", "dev"] files = [ - {file = "PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1"}, - {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92"}, - {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394"}, - {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d"}, - {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858"}, - {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b"}, - {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff"}, - {file = "PyNaCl-1.5.0-cp36-abi3-win32.whl", hash = "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543"}, - {file = "PyNaCl-1.5.0-cp36-abi3-win_amd64.whl", hash = "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93"}, - {file = "PyNaCl-1.5.0.tar.gz", hash = "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba"}, + {file = "pynacl-1.6.2-cp314-cp314t-macosx_10_10_universal2.whl", hash = "sha256:622d7b07cc5c02c666795792931b50c91f3ce3c2649762efb1ef0d5684c81594"}, + {file = "pynacl-1.6.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d071c6a9a4c94d79eb665db4ce5cedc537faf74f2355e4d502591d850d3913c0"}, + {file = "pynacl-1.6.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fe9847ca47d287af41e82be1dd5e23023d3c31a951da134121ab02e42ac218c9"}, + {file = "pynacl-1.6.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:04316d1fc625d860b6c162fff704eb8426b1a8bcd3abacea11142cbd99a6b574"}, + {file = "pynacl-1.6.2-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44081faff368d6c5553ccf55322ef2819abb40e25afaec7e740f159f74813634"}, + {file = "pynacl-1.6.2-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:a9f9932d8d2811ce1a8ffa79dcbdf3970e7355b5c8eb0c1a881a57e7f7d96e88"}, + {file = "pynacl-1.6.2-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:bc4a36b28dd72fb4845e5d8f9760610588a96d5a51f01d84d8c6ff9849968c14"}, + {file = "pynacl-1.6.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3bffb6d0f6becacb6526f8f42adfb5efb26337056ee0831fb9a7044d1a964444"}, + {file = "pynacl-1.6.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2fef529ef3ee487ad8113d287a593fa26f48ee3620d92ecc6f1d09ea38e0709b"}, + {file = "pynacl-1.6.2-cp314-cp314t-win32.whl", hash = "sha256:a84bf1c20339d06dc0c85d9aea9637a24f718f375d861b2668b2f9f96fa51145"}, + {file = "pynacl-1.6.2-cp314-cp314t-win_amd64.whl", hash = "sha256:320ef68a41c87547c91a8b58903c9caa641ab01e8512ce291085b5fe2fcb7590"}, + {file = "pynacl-1.6.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d29bfe37e20e015a7d8b23cfc8bd6aa7909c92a1b8f41ee416bbb3e79ef182b2"}, + {file = "pynacl-1.6.2-cp38-abi3-macosx_10_10_universal2.whl", hash = "sha256:c949ea47e4206af7c8f604b8278093b674f7c79ed0d4719cc836902bf4517465"}, + {file = "pynacl-1.6.2-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8845c0631c0be43abdd865511c41eab235e0be69c81dc66a50911594198679b0"}, + {file = "pynacl-1.6.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:22de65bb9010a725b0dac248f353bb072969c94fa8d6b1f34b87d7953cf7bbe4"}, + {file = "pynacl-1.6.2-cp38-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:46065496ab748469cdd999246d17e301b2c24ae2fdf739132e580a0e94c94a87"}, + {file = "pynacl-1.6.2-cp38-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a66d6fb6ae7661c58995f9c6435bda2b1e68b54b598a6a10247bfcdadac996c"}, + {file = "pynacl-1.6.2-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:26bfcd00dcf2cf160f122186af731ae30ab120c18e8375684ec2670dccd28130"}, + {file = "pynacl-1.6.2-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:c8a231e36ec2cab018c4ad4358c386e36eede0319a0c41fed24f840b1dac59f6"}, + {file = "pynacl-1.6.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:68be3a09455743ff9505491220b64440ced8973fe930f270c8e07ccfa25b1f9e"}, + {file = "pynacl-1.6.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8b097553b380236d51ed11356c953bf8ce36a29a3e596e934ecabe76c985a577"}, + {file = "pynacl-1.6.2-cp38-abi3-win32.whl", hash = "sha256:5811c72b473b2f38f7e2a3dc4f8642e3a3e9b5e7317266e4ced1fba85cae41aa"}, + {file = "pynacl-1.6.2-cp38-abi3-win_amd64.whl", hash = "sha256:62985f233210dee6548c223301b6c25440852e13d59a8b81490203c3227c5ba0"}, + {file = "pynacl-1.6.2-cp38-abi3-win_arm64.whl", hash = "sha256:834a43af110f743a754448463e8fd61259cd4ab5bbedcf70f9dabad1d28a394c"}, + {file = "pynacl-1.6.2.tar.gz", hash = "sha256:018494d6d696ae03c7e656e5e74cdfd8ea1326962cc401bcf018f1ed8436811c"}, ] [package.dependencies] -cffi = ">=1.4.1" +cffi = {version = ">=2.0.0", markers = "platform_python_implementation != \"PyPy\" and python_version >= \"3.9\""} [package.extras] -docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] -tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] +docs = ["sphinx (<7)", "sphinx_rtd_theme"] +tests = ["hypothesis (>=3.27.0)", "pytest (>=7.4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] [[package]] name = "pyopenssl" @@ -2271,7 +2286,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2296,7 +2311,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, @@ -2324,7 +2339,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00"}, {file = "pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3"}, @@ -2728,7 +2743,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"sentry\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"sentry\"" files = [ {file = "sentry_sdk-2.46.0-py2.py3-none-any.whl", hash = "sha256:4eeeb60198074dff8d066ea153fa6f241fef1668c10900ea53a4200abc8da9b1"}, {file = "sentry_sdk-2.46.0.tar.gz", hash = "sha256:91821a23460725734b7741523021601593f35731808afc0bb2ba46c27b8acd91"}, @@ -2938,7 +2953,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -2954,7 +2969,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, ] @@ -3027,7 +3042,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "tornado-6.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:f81067dad2e4443b015368b24e802d0083fecada4f0a4572fdb72fc06e54a9a6"}, {file = "tornado-6.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9ac1cbe1db860b3cbb251e795c701c41d343f06a96049d6274e7c77559117e41"}, @@ -3161,7 +3176,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3407,7 +3422,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "xmlschema-2.4.0-py3-none-any.whl", hash = "sha256:dc87be0caaa61f42649899189aab2fd8e0d567f2cf548433ba7b79278d231a4a"}, {file = "xmlschema-2.4.0.tar.gz", hash = "sha256:d74cd0c10866ac609e1ef94a5a69b018ad16e39077bc6393408b40c6babee793"}, From 987b61a92b4cabab6d85abb35ca6b08a821a2760 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 6 Jan 2026 21:37:23 +0000 Subject: [PATCH 092/139] Revert "Add an Admin API endpoint for listing quarantined media (#19268)" (#19351) Fixes #19349 This reverts commit 3f636386a66cbc57a6a3c3e641dfd6f1917c838e (https://github.com/element-hq/synapse/pull/19268) as the DB migration was taking too long and blocking media access while it happened. See https://github.com/element-hq/synapse/issues/19349 for further information. ### Pull Request Checklist * [X] Pull request is based on the develop branch * [X] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [X] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Travis Ralston --- changelog.d/19268.feature | 1 - changelog.d/19351.misc | 1 + docs/admin_api/media_admin_api.md | 27 ----- synapse/media/media_repository.py | 2 - synapse/rest/admin/media.py | 33 ------ .../databases/main/media_repository.py | 10 +- synapse/storage/databases/main/room.py | 70 ++---------- .../93/04_add_quarantined_ts_to_media.sql | 27 ----- tests/rest/admin/test_media.py | 106 ------------------ 9 files changed, 11 insertions(+), 266 deletions(-) delete mode 100644 changelog.d/19268.feature create mode 100644 changelog.d/19351.misc delete mode 100644 synapse/storage/schema/main/delta/93/04_add_quarantined_ts_to_media.sql diff --git a/changelog.d/19268.feature b/changelog.d/19268.feature deleted file mode 100644 index cb7035fee2a..00000000000 --- a/changelog.d/19268.feature +++ /dev/null @@ -1 +0,0 @@ -Add an admin API for retrieving a paginated list of quarantined media. \ No newline at end of file diff --git a/changelog.d/19351.misc b/changelog.d/19351.misc new file mode 100644 index 00000000000..5e186b9a0c2 --- /dev/null +++ b/changelog.d/19351.misc @@ -0,0 +1 @@ +Revert "Add an Admin API endpoint for listing quarantined media (#19268)". diff --git a/docs/admin_api/media_admin_api.md b/docs/admin_api/media_admin_api.md index 25481a8c552..6b96eb33564 100644 --- a/docs/admin_api/media_admin_api.md +++ b/docs/admin_api/media_admin_api.md @@ -73,33 +73,6 @@ Response: } ``` -## Listing all quarantined media - -This API returns a list of all quarantined media on the server. It is paginated, and can be scoped to either local or -remote media. Note that the pagination values are also scoped to the request parameters - changing them but keeping the -same pagination values will result in unexpected results. - -Request: -```http -GET /_synapse/admin/v1/media/quarantined?from=0&limit=100&kind=local -``` - -`from` and `limit` are optional parameters, and default to `0` and `100` respectively. They are the row index and number -of rows to return - they are not timestamps. - -`kind` *MUST* either be `local` or `remote`. - -The API returns a JSON body containing MXC URIs for the quarantined media, like the following: - -```json -{ - "media": [ - "mxc://localhost/xwvutsrqponmlkjihgfedcba", - "mxc://localhost/abcdefghijklmnopqrstuvwx" - ] -} -``` - # Quarantine media Quarantining media means that it is marked as inaccessible by users. It applies diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index cb745b96ad4..8d38c1655f2 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -928,7 +928,6 @@ async def _download_remote_file( filesystem_id=file_id, last_access_ts=time_now_ms, quarantined_by=None, - quarantined_ts=None, authenticated=authenticated, sha256=sha256writer.hexdigest(), ) @@ -1062,7 +1061,6 @@ async def _federation_download_remote_file( filesystem_id=file_id, last_access_ts=time_now_ms, quarantined_by=None, - quarantined_ts=None, authenticated=authenticated, sha256=sha256writer.hexdigest(), ) diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py index 4cfd9da0f92..d5346fe0d5c 100644 --- a/synapse/rest/admin/media.py +++ b/synapse/rest/admin/media.py @@ -293,38 +293,6 @@ async def on_GET( return HTTPStatus.OK, {"local": local_mxcs, "remote": remote_mxcs} -class ListQuarantinedMedia(RestServlet): - """Lists all quarantined media on the server.""" - - PATTERNS = admin_patterns("/media/quarantined$") - - def __init__(self, hs: "HomeServer"): - self.store = hs.get_datastores().main - self.auth = hs.get_auth() - - async def on_GET( - self, - request: SynapseRequest, - ) -> tuple[int, JsonDict]: - await assert_requester_is_admin(self.auth, request) - - start = parse_integer(request, "from", default=0) - limit = parse_integer(request, "limit", default=100) - local_or_remote = parse_string(request, "kind", required=True) - - if local_or_remote not in ["local", "remote"]: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Query parameter `kind` must be either 'local' or 'remote'.", - ) - - mxcs = await self.store.get_quarantined_media_mxcs( - start, limit, local_or_remote == "local" - ) - - return HTTPStatus.OK, {"media": mxcs} - - class PurgeMediaCacheRestServlet(RestServlet): PATTERNS = admin_patterns("/purge_media_cache$") @@ -564,7 +532,6 @@ def register_servlets_for_media_repo(hs: "HomeServer", http_server: HttpServer) ProtectMediaByID(hs).register(http_server) UnprotectMediaByID(hs).register(http_server) ListMediaInRoom(hs).register(http_server) - ListQuarantinedMedia(hs).register(http_server) # XXX DeleteMediaByDateSize must be registered before DeleteMediaByID as # their URL routes overlap. DeleteMediaByDateSize(hs).register(http_server) diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index c27c68fbc28..50664d63e5f 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -61,7 +61,6 @@ class LocalMedia: url_cache: str | None last_access_ts: int quarantined_by: str | None - quarantined_ts: int | None safe_from_quarantine: bool user_id: str | None authenticated: bool | None @@ -79,7 +78,6 @@ class RemoteMedia: created_ts: int last_access_ts: int quarantined_by: str | None - quarantined_ts: int | None authenticated: bool | None sha256: str | None @@ -245,7 +243,6 @@ async def get_local_media(self, media_id: str) -> LocalMedia | None: "user_id", "authenticated", "sha256", - "quarantined_ts", ), allow_none=True, desc="get_local_media", @@ -265,7 +262,6 @@ async def get_local_media(self, media_id: str) -> LocalMedia | None: user_id=row[8], authenticated=row[9], sha256=row[10], - quarantined_ts=row[11], ) async def get_local_media_by_user_paginate( @@ -323,8 +319,7 @@ def get_local_media_by_user_paginate_txn( safe_from_quarantine, user_id, authenticated, - sha256, - quarantined_ts + sha256 FROM local_media_repository WHERE user_id = ? ORDER BY {order_by_column} {order}, media_id ASC @@ -350,7 +345,6 @@ def get_local_media_by_user_paginate_txn( user_id=row[9], authenticated=row[10], sha256=row[11], - quarantined_ts=row[12], ) for row in txn ] @@ -701,7 +695,6 @@ async def get_cached_remote_media( "quarantined_by", "authenticated", "sha256", - "quarantined_ts", ), allow_none=True, desc="get_cached_remote_media", @@ -720,7 +713,6 @@ async def get_cached_remote_media( quarantined_by=row[6], authenticated=row[7], sha256=row[8], - quarantined_ts=row[9], ) async def store_cached_remote_media( diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 182e55743a6..633df077367 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -945,50 +945,6 @@ def get_retention_policy_for_room_txn( max_lifetime=max_lifetime, ) - async def get_quarantined_media_mxcs( - self, index_start: int, index_limit: int, local: bool - ) -> list[str]: - """Retrieves all the quarantined media MXC URIs starting from the given position, - ordered from oldest quarantined timestamp, then alphabetically by media ID - (including origin). - - Note that on established servers the "quarantined timestamp" may be zero due to - being introduced after the quarantine timestamp field was introduced. - - Args: - index_start: The position to start from. - index_limit: The maximum number of results to return. - local: When true, only local media will be returned. When false, only remote media will be returned. - - Returns: - The quarantined media as a list of media IDs. - """ - - def _get_quarantined_media_mxcs_txn( - txn: LoggingTransaction, - ) -> list[str]: - # We order by quarantined timestamp *and* media ID (including origin, when - # known) to ensure the ordering is stable for established servers. - if local: - sql = "SELECT '' as media_origin, media_id FROM local_media_repository WHERE quarantined_by IS NOT NULL ORDER BY quarantined_ts, media_id ASC LIMIT ? OFFSET ?" - else: - sql = "SELECT media_origin, media_id FROM remote_media_cache WHERE quarantined_by IS NOT NULL ORDER BY quarantined_ts, media_origin, media_id ASC LIMIT ? OFFSET ?" - txn.execute(sql, (index_limit, index_start)) - - mxcs = [] - - for media_origin, media_id in txn: - if local: - media_origin = self.hs.hostname - mxcs.append(f"mxc://{media_origin}/{media_id}") - - return mxcs - - return await self.db_pool.runInteraction( - "get_quarantined_media_mxcs", - _get_quarantined_media_mxcs_txn, - ) - async def get_media_mxcs_in_room(self, room_id: str) -> tuple[list[str], list[str]]: """Retrieves all the local and remote media MXC URIs in a given room @@ -996,7 +952,7 @@ async def get_media_mxcs_in_room(self, room_id: str) -> tuple[list[str], list[st room_id Returns: - The local and remote media as lists of the media IDs. + The local and remote media as a lists of the media IDs. """ def _get_media_mxcs_in_room_txn( @@ -1191,10 +1147,6 @@ def _quarantine_local_media_txn( The total number of media items quarantined """ total_media_quarantined = 0 - now_ts: int | None = self.clock.time_msec() - - if quarantined_by is None: - now_ts = None # Effectively a legacy path, update any media that was explicitly named. if media_ids: @@ -1203,13 +1155,13 @@ def _quarantine_local_media_txn( ) sql = f""" UPDATE local_media_repository - SET quarantined_by = ?, quarantined_ts = ? + SET quarantined_by = ? WHERE {sql_many_clause_sql}""" if quarantined_by is not None: sql += " AND safe_from_quarantine = FALSE" - txn.execute(sql, [quarantined_by, now_ts] + sql_many_clause_args) + txn.execute(sql, [quarantined_by] + sql_many_clause_args) # Note that a rowcount of -1 can be used to indicate no rows were affected. total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 @@ -1220,13 +1172,13 @@ def _quarantine_local_media_txn( ) sql = f""" UPDATE local_media_repository - SET quarantined_by = ?, quarantined_ts = ? + SET quarantined_by = ? WHERE {sql_many_clause_sql}""" if quarantined_by is not None: sql += " AND safe_from_quarantine = FALSE" - txn.execute(sql, [quarantined_by, now_ts] + sql_many_clause_args) + txn.execute(sql, [quarantined_by] + sql_many_clause_args) total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 return total_media_quarantined @@ -1250,10 +1202,6 @@ def _quarantine_remote_media_txn( The total number of media items quarantined """ total_media_quarantined = 0 - now_ts: int | None = self.clock.time_msec() - - if quarantined_by is None: - now_ts = None if media: sql_in_list_clause, sql_args = make_tuple_in_list_sql_clause( @@ -1263,10 +1211,10 @@ def _quarantine_remote_media_txn( ) sql = f""" UPDATE remote_media_cache - SET quarantined_by = ?, quarantined_ts = ? + SET quarantined_by = ? WHERE {sql_in_list_clause}""" - txn.execute(sql, [quarantined_by, now_ts] + sql_args) + txn.execute(sql, [quarantined_by] + sql_args) total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 total_media_quarantined = 0 @@ -1276,9 +1224,9 @@ def _quarantine_remote_media_txn( ) sql = f""" UPDATE remote_media_cache - SET quarantined_by = ?, quarantined_ts = ? + SET quarantined_by = ? WHERE {sql_many_clause_sql}""" - txn.execute(sql, [quarantined_by, now_ts] + sql_many_clause_args) + txn.execute(sql, [quarantined_by] + sql_many_clause_args) total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 return total_media_quarantined diff --git a/synapse/storage/schema/main/delta/93/04_add_quarantined_ts_to_media.sql b/synapse/storage/schema/main/delta/93/04_add_quarantined_ts_to_media.sql deleted file mode 100644 index 18b76804ff6..00000000000 --- a/synapse/storage/schema/main/delta/93/04_add_quarantined_ts_to_media.sql +++ /dev/null @@ -1,27 +0,0 @@ --- --- This file is licensed under the Affero General Public License (AGPL) version 3. --- --- Copyright (C) 2025 Element Creations, Ltd --- --- This program is free software: you can redistribute it and/or modify --- it under the terms of the GNU Affero General Public License as --- published by the Free Software Foundation, either version 3 of the --- License, or (at your option) any later version. --- --- See the GNU Affero General Public License for more details: --- . - --- Add a timestamp for when the sliding sync connection position was last used, --- only updated with a small granularity. --- --- This should be NOT NULL, but we need to consider existing rows. In future we --- may want to either backfill this or delete all rows with a NULL value (and --- then make it NOT NULL). -ALTER TABLE local_media_repository ADD COLUMN quarantined_ts BIGINT; -ALTER TABLE remote_media_cache ADD COLUMN quarantined_ts BIGINT; - -UPDATE local_media_repository SET quarantined_ts = 0 WHERE quarantined_by IS NOT NULL; -UPDATE remote_media_cache SET quarantined_ts = 0 WHERE quarantined_by IS NOT NULL; - --- Note: We *probably* should have an index on quarantined_ts, but we're going --- to try to defer that to a future migration after seeing the performance impact. diff --git a/tests/rest/admin/test_media.py b/tests/rest/admin/test_media.py index e45cc4d2087..8cc54cc80c2 100644 --- a/tests/rest/admin/test_media.py +++ b/tests/rest/admin/test_media.py @@ -756,112 +756,6 @@ def _access_media( self.assertFalse(os.path.exists(local_path)) -class ListQuarantinedMediaTestCase(_AdminMediaTests): - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.store = hs.get_datastores().main - self.server_name = hs.hostname - - @parameterized.expand(["local", "remote"]) - def test_no_auth(self, kind: str) -> None: - """ - Try to list quarantined media without authentication. - """ - - channel = self.make_request( - "GET", - "/_synapse/admin/v1/media/quarantined?kind=%s" % (kind,), - ) - - self.assertEqual(401, channel.code, msg=channel.json_body) - self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) - - @parameterized.expand(["local", "remote"]) - def test_requester_is_not_admin(self, kind: str) -> None: - """ - If the user is not a server admin, an error is returned. - """ - self.other_user = self.register_user("user", "pass") - self.other_user_token = self.login("user", "pass") - - channel = self.make_request( - "GET", - "/_synapse/admin/v1/media/quarantined?kind=%s" % (kind,), - access_token=self.other_user_token, - ) - - self.assertEqual(403, channel.code, msg=channel.json_body) - self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) - - def test_list_quarantined_media(self) -> None: - """ - Ensure we actually get results for each page. We can't really test that - remote media is quarantined, but we can test that local media is. - """ - self.admin_user = self.register_user("admin", "pass", admin=True) - self.admin_user_tok = self.login("admin", "pass") - - def _upload() -> str: - return self.helper.upload_media( - SMALL_PNG, tok=self.admin_user_tok, expect_code=200 - )["content_uri"][6:].split("/")[1] # Cut off 'mxc://' and domain - - self.media_id_1 = _upload() - self.media_id_2 = _upload() - self.media_id_3 = _upload() - - def _quarantine(media_id: str) -> None: - channel = self.make_request( - "POST", - "/_synapse/admin/v1/media/quarantine/%s/%s" - % ( - self.server_name, - media_id, - ), - access_token=self.admin_user_tok, - ) - self.assertEqual(200, channel.code, msg=channel.json_body) - - _quarantine(self.media_id_1) - _quarantine(self.media_id_2) - _quarantine(self.media_id_3) - - # Page 1 - channel = self.make_request( - "GET", - "/_synapse/admin/v1/media/quarantined?kind=local&from=0&limit=1", - access_token=self.admin_user_tok, - ) - self.assertEqual(200, channel.code, msg=channel.json_body) - self.assertEqual(1, len(channel.json_body["media"])) - - # Page 2 - channel = self.make_request( - "GET", - "/_synapse/admin/v1/media/quarantined?kind=local&from=1&limit=1", - access_token=self.admin_user_tok, - ) - self.assertEqual(200, channel.code, msg=channel.json_body) - self.assertEqual(1, len(channel.json_body["media"])) - - # Page 3 - channel = self.make_request( - "GET", - "/_synapse/admin/v1/media/quarantined?kind=local&from=2&limit=1", - access_token=self.admin_user_tok, - ) - self.assertEqual(200, channel.code, msg=channel.json_body) - self.assertEqual(1, len(channel.json_body["media"])) - - # Page 4 (no media) - channel = self.make_request( - "GET", - "/_synapse/admin/v1/media/quarantined?kind=local&from=3&limit=1", - access_token=self.admin_user_tok, - ) - self.assertEqual(200, channel.code, msg=channel.json_body) - self.assertEqual(0, len(channel.json_body["media"])) - - class QuarantineMediaByIDTestCase(_AdminMediaTests): def upload_media_and_return_media_id(self, data: bytes) -> str: # Upload some media into the room From 6ac61e4be494cb3467348882343dd19a0a249994 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 6 Jan 2026 21:37:23 +0000 Subject: [PATCH 093/139] Revert "Add an Admin API endpoint for listing quarantined media (#19268)" (#19351) Fixes #19349 This reverts commit 3f636386a66cbc57a6a3c3e641dfd6f1917c838e (https://github.com/element-hq/synapse/pull/19268) as the DB migration was taking too long and blocking media access while it happened. See https://github.com/element-hq/synapse/issues/19349 for further information. ### Pull Request Checklist * [X] Pull request is based on the develop branch * [X] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [X] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Travis Ralston --- changelog.d/19351.misc | 1 + docs/admin_api/media_admin_api.md | 27 ----- synapse/media/media_repository.py | 2 - synapse/rest/admin/media.py | 33 ------ .../databases/main/media_repository.py | 10 +- synapse/storage/databases/main/room.py | 70 ++---------- .../93/04_add_quarantined_ts_to_media.sql | 27 ----- tests/rest/admin/test_media.py | 106 ------------------ 8 files changed, 11 insertions(+), 265 deletions(-) create mode 100644 changelog.d/19351.misc delete mode 100644 synapse/storage/schema/main/delta/93/04_add_quarantined_ts_to_media.sql diff --git a/changelog.d/19351.misc b/changelog.d/19351.misc new file mode 100644 index 00000000000..5e186b9a0c2 --- /dev/null +++ b/changelog.d/19351.misc @@ -0,0 +1 @@ +Revert "Add an Admin API endpoint for listing quarantined media (#19268)". diff --git a/docs/admin_api/media_admin_api.md b/docs/admin_api/media_admin_api.md index 25481a8c552..6b96eb33564 100644 --- a/docs/admin_api/media_admin_api.md +++ b/docs/admin_api/media_admin_api.md @@ -73,33 +73,6 @@ Response: } ``` -## Listing all quarantined media - -This API returns a list of all quarantined media on the server. It is paginated, and can be scoped to either local or -remote media. Note that the pagination values are also scoped to the request parameters - changing them but keeping the -same pagination values will result in unexpected results. - -Request: -```http -GET /_synapse/admin/v1/media/quarantined?from=0&limit=100&kind=local -``` - -`from` and `limit` are optional parameters, and default to `0` and `100` respectively. They are the row index and number -of rows to return - they are not timestamps. - -`kind` *MUST* either be `local` or `remote`. - -The API returns a JSON body containing MXC URIs for the quarantined media, like the following: - -```json -{ - "media": [ - "mxc://localhost/xwvutsrqponmlkjihgfedcba", - "mxc://localhost/abcdefghijklmnopqrstuvwx" - ] -} -``` - # Quarantine media Quarantining media means that it is marked as inaccessible by users. It applies diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index cb745b96ad4..8d38c1655f2 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -928,7 +928,6 @@ async def _download_remote_file( filesystem_id=file_id, last_access_ts=time_now_ms, quarantined_by=None, - quarantined_ts=None, authenticated=authenticated, sha256=sha256writer.hexdigest(), ) @@ -1062,7 +1061,6 @@ async def _federation_download_remote_file( filesystem_id=file_id, last_access_ts=time_now_ms, quarantined_by=None, - quarantined_ts=None, authenticated=authenticated, sha256=sha256writer.hexdigest(), ) diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py index 4cfd9da0f92..d5346fe0d5c 100644 --- a/synapse/rest/admin/media.py +++ b/synapse/rest/admin/media.py @@ -293,38 +293,6 @@ async def on_GET( return HTTPStatus.OK, {"local": local_mxcs, "remote": remote_mxcs} -class ListQuarantinedMedia(RestServlet): - """Lists all quarantined media on the server.""" - - PATTERNS = admin_patterns("/media/quarantined$") - - def __init__(self, hs: "HomeServer"): - self.store = hs.get_datastores().main - self.auth = hs.get_auth() - - async def on_GET( - self, - request: SynapseRequest, - ) -> tuple[int, JsonDict]: - await assert_requester_is_admin(self.auth, request) - - start = parse_integer(request, "from", default=0) - limit = parse_integer(request, "limit", default=100) - local_or_remote = parse_string(request, "kind", required=True) - - if local_or_remote not in ["local", "remote"]: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Query parameter `kind` must be either 'local' or 'remote'.", - ) - - mxcs = await self.store.get_quarantined_media_mxcs( - start, limit, local_or_remote == "local" - ) - - return HTTPStatus.OK, {"media": mxcs} - - class PurgeMediaCacheRestServlet(RestServlet): PATTERNS = admin_patterns("/purge_media_cache$") @@ -564,7 +532,6 @@ def register_servlets_for_media_repo(hs: "HomeServer", http_server: HttpServer) ProtectMediaByID(hs).register(http_server) UnprotectMediaByID(hs).register(http_server) ListMediaInRoom(hs).register(http_server) - ListQuarantinedMedia(hs).register(http_server) # XXX DeleteMediaByDateSize must be registered before DeleteMediaByID as # their URL routes overlap. DeleteMediaByDateSize(hs).register(http_server) diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index c27c68fbc28..50664d63e5f 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -61,7 +61,6 @@ class LocalMedia: url_cache: str | None last_access_ts: int quarantined_by: str | None - quarantined_ts: int | None safe_from_quarantine: bool user_id: str | None authenticated: bool | None @@ -79,7 +78,6 @@ class RemoteMedia: created_ts: int last_access_ts: int quarantined_by: str | None - quarantined_ts: int | None authenticated: bool | None sha256: str | None @@ -245,7 +243,6 @@ async def get_local_media(self, media_id: str) -> LocalMedia | None: "user_id", "authenticated", "sha256", - "quarantined_ts", ), allow_none=True, desc="get_local_media", @@ -265,7 +262,6 @@ async def get_local_media(self, media_id: str) -> LocalMedia | None: user_id=row[8], authenticated=row[9], sha256=row[10], - quarantined_ts=row[11], ) async def get_local_media_by_user_paginate( @@ -323,8 +319,7 @@ def get_local_media_by_user_paginate_txn( safe_from_quarantine, user_id, authenticated, - sha256, - quarantined_ts + sha256 FROM local_media_repository WHERE user_id = ? ORDER BY {order_by_column} {order}, media_id ASC @@ -350,7 +345,6 @@ def get_local_media_by_user_paginate_txn( user_id=row[9], authenticated=row[10], sha256=row[11], - quarantined_ts=row[12], ) for row in txn ] @@ -701,7 +695,6 @@ async def get_cached_remote_media( "quarantined_by", "authenticated", "sha256", - "quarantined_ts", ), allow_none=True, desc="get_cached_remote_media", @@ -720,7 +713,6 @@ async def get_cached_remote_media( quarantined_by=row[6], authenticated=row[7], sha256=row[8], - quarantined_ts=row[9], ) async def store_cached_remote_media( diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 182e55743a6..633df077367 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -945,50 +945,6 @@ def get_retention_policy_for_room_txn( max_lifetime=max_lifetime, ) - async def get_quarantined_media_mxcs( - self, index_start: int, index_limit: int, local: bool - ) -> list[str]: - """Retrieves all the quarantined media MXC URIs starting from the given position, - ordered from oldest quarantined timestamp, then alphabetically by media ID - (including origin). - - Note that on established servers the "quarantined timestamp" may be zero due to - being introduced after the quarantine timestamp field was introduced. - - Args: - index_start: The position to start from. - index_limit: The maximum number of results to return. - local: When true, only local media will be returned. When false, only remote media will be returned. - - Returns: - The quarantined media as a list of media IDs. - """ - - def _get_quarantined_media_mxcs_txn( - txn: LoggingTransaction, - ) -> list[str]: - # We order by quarantined timestamp *and* media ID (including origin, when - # known) to ensure the ordering is stable for established servers. - if local: - sql = "SELECT '' as media_origin, media_id FROM local_media_repository WHERE quarantined_by IS NOT NULL ORDER BY quarantined_ts, media_id ASC LIMIT ? OFFSET ?" - else: - sql = "SELECT media_origin, media_id FROM remote_media_cache WHERE quarantined_by IS NOT NULL ORDER BY quarantined_ts, media_origin, media_id ASC LIMIT ? OFFSET ?" - txn.execute(sql, (index_limit, index_start)) - - mxcs = [] - - for media_origin, media_id in txn: - if local: - media_origin = self.hs.hostname - mxcs.append(f"mxc://{media_origin}/{media_id}") - - return mxcs - - return await self.db_pool.runInteraction( - "get_quarantined_media_mxcs", - _get_quarantined_media_mxcs_txn, - ) - async def get_media_mxcs_in_room(self, room_id: str) -> tuple[list[str], list[str]]: """Retrieves all the local and remote media MXC URIs in a given room @@ -996,7 +952,7 @@ async def get_media_mxcs_in_room(self, room_id: str) -> tuple[list[str], list[st room_id Returns: - The local and remote media as lists of the media IDs. + The local and remote media as a lists of the media IDs. """ def _get_media_mxcs_in_room_txn( @@ -1191,10 +1147,6 @@ def _quarantine_local_media_txn( The total number of media items quarantined """ total_media_quarantined = 0 - now_ts: int | None = self.clock.time_msec() - - if quarantined_by is None: - now_ts = None # Effectively a legacy path, update any media that was explicitly named. if media_ids: @@ -1203,13 +1155,13 @@ def _quarantine_local_media_txn( ) sql = f""" UPDATE local_media_repository - SET quarantined_by = ?, quarantined_ts = ? + SET quarantined_by = ? WHERE {sql_many_clause_sql}""" if quarantined_by is not None: sql += " AND safe_from_quarantine = FALSE" - txn.execute(sql, [quarantined_by, now_ts] + sql_many_clause_args) + txn.execute(sql, [quarantined_by] + sql_many_clause_args) # Note that a rowcount of -1 can be used to indicate no rows were affected. total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 @@ -1220,13 +1172,13 @@ def _quarantine_local_media_txn( ) sql = f""" UPDATE local_media_repository - SET quarantined_by = ?, quarantined_ts = ? + SET quarantined_by = ? WHERE {sql_many_clause_sql}""" if quarantined_by is not None: sql += " AND safe_from_quarantine = FALSE" - txn.execute(sql, [quarantined_by, now_ts] + sql_many_clause_args) + txn.execute(sql, [quarantined_by] + sql_many_clause_args) total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 return total_media_quarantined @@ -1250,10 +1202,6 @@ def _quarantine_remote_media_txn( The total number of media items quarantined """ total_media_quarantined = 0 - now_ts: int | None = self.clock.time_msec() - - if quarantined_by is None: - now_ts = None if media: sql_in_list_clause, sql_args = make_tuple_in_list_sql_clause( @@ -1263,10 +1211,10 @@ def _quarantine_remote_media_txn( ) sql = f""" UPDATE remote_media_cache - SET quarantined_by = ?, quarantined_ts = ? + SET quarantined_by = ? WHERE {sql_in_list_clause}""" - txn.execute(sql, [quarantined_by, now_ts] + sql_args) + txn.execute(sql, [quarantined_by] + sql_args) total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 total_media_quarantined = 0 @@ -1276,9 +1224,9 @@ def _quarantine_remote_media_txn( ) sql = f""" UPDATE remote_media_cache - SET quarantined_by = ?, quarantined_ts = ? + SET quarantined_by = ? WHERE {sql_many_clause_sql}""" - txn.execute(sql, [quarantined_by, now_ts] + sql_many_clause_args) + txn.execute(sql, [quarantined_by] + sql_many_clause_args) total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 return total_media_quarantined diff --git a/synapse/storage/schema/main/delta/93/04_add_quarantined_ts_to_media.sql b/synapse/storage/schema/main/delta/93/04_add_quarantined_ts_to_media.sql deleted file mode 100644 index 18b76804ff6..00000000000 --- a/synapse/storage/schema/main/delta/93/04_add_quarantined_ts_to_media.sql +++ /dev/null @@ -1,27 +0,0 @@ --- --- This file is licensed under the Affero General Public License (AGPL) version 3. --- --- Copyright (C) 2025 Element Creations, Ltd --- --- This program is free software: you can redistribute it and/or modify --- it under the terms of the GNU Affero General Public License as --- published by the Free Software Foundation, either version 3 of the --- License, or (at your option) any later version. --- --- See the GNU Affero General Public License for more details: --- . - --- Add a timestamp for when the sliding sync connection position was last used, --- only updated with a small granularity. --- --- This should be NOT NULL, but we need to consider existing rows. In future we --- may want to either backfill this or delete all rows with a NULL value (and --- then make it NOT NULL). -ALTER TABLE local_media_repository ADD COLUMN quarantined_ts BIGINT; -ALTER TABLE remote_media_cache ADD COLUMN quarantined_ts BIGINT; - -UPDATE local_media_repository SET quarantined_ts = 0 WHERE quarantined_by IS NOT NULL; -UPDATE remote_media_cache SET quarantined_ts = 0 WHERE quarantined_by IS NOT NULL; - --- Note: We *probably* should have an index on quarantined_ts, but we're going --- to try to defer that to a future migration after seeing the performance impact. diff --git a/tests/rest/admin/test_media.py b/tests/rest/admin/test_media.py index e45cc4d2087..8cc54cc80c2 100644 --- a/tests/rest/admin/test_media.py +++ b/tests/rest/admin/test_media.py @@ -756,112 +756,6 @@ def _access_media( self.assertFalse(os.path.exists(local_path)) -class ListQuarantinedMediaTestCase(_AdminMediaTests): - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.store = hs.get_datastores().main - self.server_name = hs.hostname - - @parameterized.expand(["local", "remote"]) - def test_no_auth(self, kind: str) -> None: - """ - Try to list quarantined media without authentication. - """ - - channel = self.make_request( - "GET", - "/_synapse/admin/v1/media/quarantined?kind=%s" % (kind,), - ) - - self.assertEqual(401, channel.code, msg=channel.json_body) - self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) - - @parameterized.expand(["local", "remote"]) - def test_requester_is_not_admin(self, kind: str) -> None: - """ - If the user is not a server admin, an error is returned. - """ - self.other_user = self.register_user("user", "pass") - self.other_user_token = self.login("user", "pass") - - channel = self.make_request( - "GET", - "/_synapse/admin/v1/media/quarantined?kind=%s" % (kind,), - access_token=self.other_user_token, - ) - - self.assertEqual(403, channel.code, msg=channel.json_body) - self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) - - def test_list_quarantined_media(self) -> None: - """ - Ensure we actually get results for each page. We can't really test that - remote media is quarantined, but we can test that local media is. - """ - self.admin_user = self.register_user("admin", "pass", admin=True) - self.admin_user_tok = self.login("admin", "pass") - - def _upload() -> str: - return self.helper.upload_media( - SMALL_PNG, tok=self.admin_user_tok, expect_code=200 - )["content_uri"][6:].split("/")[1] # Cut off 'mxc://' and domain - - self.media_id_1 = _upload() - self.media_id_2 = _upload() - self.media_id_3 = _upload() - - def _quarantine(media_id: str) -> None: - channel = self.make_request( - "POST", - "/_synapse/admin/v1/media/quarantine/%s/%s" - % ( - self.server_name, - media_id, - ), - access_token=self.admin_user_tok, - ) - self.assertEqual(200, channel.code, msg=channel.json_body) - - _quarantine(self.media_id_1) - _quarantine(self.media_id_2) - _quarantine(self.media_id_3) - - # Page 1 - channel = self.make_request( - "GET", - "/_synapse/admin/v1/media/quarantined?kind=local&from=0&limit=1", - access_token=self.admin_user_tok, - ) - self.assertEqual(200, channel.code, msg=channel.json_body) - self.assertEqual(1, len(channel.json_body["media"])) - - # Page 2 - channel = self.make_request( - "GET", - "/_synapse/admin/v1/media/quarantined?kind=local&from=1&limit=1", - access_token=self.admin_user_tok, - ) - self.assertEqual(200, channel.code, msg=channel.json_body) - self.assertEqual(1, len(channel.json_body["media"])) - - # Page 3 - channel = self.make_request( - "GET", - "/_synapse/admin/v1/media/quarantined?kind=local&from=2&limit=1", - access_token=self.admin_user_tok, - ) - self.assertEqual(200, channel.code, msg=channel.json_body) - self.assertEqual(1, len(channel.json_body["media"])) - - # Page 4 (no media) - channel = self.make_request( - "GET", - "/_synapse/admin/v1/media/quarantined?kind=local&from=3&limit=1", - access_token=self.admin_user_tok, - ) - self.assertEqual(200, channel.code, msg=channel.json_body) - self.assertEqual(0, len(channel.json_body["media"])) - - class QuarantineMediaByIDTestCase(_AdminMediaTests): def upload_media_and_return_media_id(self, data: bytes) -> str: # Upload some media into the room From 16bc8c78ba14bfafcd21cc06489c4750b02fe56b Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 6 Jan 2026 14:49:09 -0700 Subject: [PATCH 094/139] Update changelog after reverting PR --- CHANGES.md | 1 - changelog.d/19351.misc | 1 - 2 files changed, 2 deletions(-) delete mode 100644 changelog.d/19351.misc diff --git a/CHANGES.md b/CHANGES.md index d075da21c2d..8192a0c0aca 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,7 +11,6 @@ The "Updates to locked dependencies" section has been removed from the changelog ## Features - Add `memberships` endpoint to the admin API. This is useful for forensics and T&S purpose. ([\#19260](https://github.com/element-hq/synapse/issues/19260)) -- Add an admin API for retrieving a paginated list of quarantined media. ([\#19268](https://github.com/element-hq/synapse/issues/19268)) - Server admins can bypass the quarantine media check when downloading media by setting the `admin_unsafely_bypass_quarantine` query parameter to `true` on Client-Server API media download requests. ([\#19275](https://github.com/element-hq/synapse/issues/19275)) - Implemented pagination for the [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666) mutual rooms endpoint. Contributed by @tulir @ Beeper. ([\#19279](https://github.com/element-hq/synapse/issues/19279)) - Admin API: add worker support to `GET /_synapse/admin/v2/users/`. ([\#19281](https://github.com/element-hq/synapse/issues/19281)) diff --git a/changelog.d/19351.misc b/changelog.d/19351.misc deleted file mode 100644 index 5e186b9a0c2..00000000000 --- a/changelog.d/19351.misc +++ /dev/null @@ -1 +0,0 @@ -Revert "Add an Admin API endpoint for listing quarantined media (#19268)". From a094d922c9a5b6defdff41e568ab184661ba5d5e Mon Sep 17 00:00:00 2001 From: Patrice Brend'amour Date: Wed, 7 Jan 2026 00:29:58 +0100 Subject: [PATCH 095/139] Implement synapse issue #16751: Treat local_media_directory as optional storage provider (#19204) --- changelog.d/19204.feature | 1 + .../configuration/config_documentation.md | 10 + schema/synapse-config.schema.yaml | 13 + synapse/config/repository.py | 5 + synapse/media/media_repository.py | 493 ++++++++++-------- synapse/media/media_storage.py | 205 +++++--- synapse/media/storage_provider.py | 4 +- synapse/media/thumbnailer.py | 7 - tests/federation/test_federation_media.py | 115 +++- tests/media/test_media_storage.py | 53 +- tests/replication/test_multi_media_repo.py | 94 ++++ 11 files changed, 684 insertions(+), 316 deletions(-) create mode 100644 changelog.d/19204.feature diff --git a/changelog.d/19204.feature b/changelog.d/19204.feature new file mode 100644 index 00000000000..04a2d93e323 --- /dev/null +++ b/changelog.d/19204.feature @@ -0,0 +1 @@ +Add a new config option [`enable_local_media_storage`](https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#enable_local_media_storage) which controls whether media is additionally stored locally when using configured `media_storage_providers`. Setting this to `false` allows off-site media storage without a local cache. Contributed by Patrice Brend'amour @dr.allgood. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index badfe0a03f4..98dbc63d78a 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -2111,6 +2111,16 @@ Example configuration: enable_media_repo: false ``` --- +### `enable_local_media_storage` + +*(boolean)* Enable the local on-disk media storage provider. When disabled, media is stored only in configured `media_storage_providers` and temporary files are used for processing. +**Warning:** If this option is set to `false` and no `media_storage_providers` are configured, all media requests will return 404 errors as there will be no storage backend available. Defaults to `true`. + +Example configuration: +```yaml +enable_local_media_storage: false +``` +--- ### `media_store_path` *(string)* Directory where uploaded images and attachments are stored. Defaults to `"media_store"`. diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index ca8db9c9eec..d20520bec30 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -2348,6 +2348,19 @@ properties: default: true examples: - false + enable_local_media_storage: + type: boolean + description: >- + Enable the local on-disk media storage provider. When disabled, media is + stored only in configured `media_storage_providers` and temporary files are + used for processing. + + **Warning:** If this option is set to `false` and no `media_storage_providers` + are configured, all media requests will return 404 errors as there will be + no storage backend available. + default: true + examples: + - false media_store_path: type: string description: Directory where uploaded images and attachments are stored. diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 221130b0cd1..c87442aace0 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -174,6 +174,11 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: config.get("media_store_path", "media_store") ) + # Whether to enable the local media storage provider. When disabled, + # media will only be stored in configured storage providers and temp + # files will be used for processing. + self.enable_local_media_storage = config.get("enable_local_media_storage", True) + backup_media_store_path = config.get("backup_media_store_path") synchronous_backup_media_store = config.get( diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 8d38c1655f2..3344e4c7beb 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -64,7 +64,10 @@ SHA256TransparentIOReader, SHA256TransparentIOWriter, ) -from synapse.media.storage_provider import StorageProviderWrapper +from synapse.media.storage_provider import ( + FileStorageProviderBackend, + StorageProviderWrapper, +) from synapse.media.thumbnailer import Thumbnailer, ThumbnailError from synapse.media.url_previewer import UrlPreviewer from synapse.storage.databases.main.media_repository import LocalMedia, RemoteMedia @@ -142,10 +145,23 @@ def __init__(self, hs: "HomeServer"): ) storage_providers.append(provider) + # If local media storage is enabled, create the local provider + local_provider: FileStorageProviderBackend | None = None + if hs.config.media.enable_local_media_storage and self.primary_base_path: + local_provider = FileStorageProviderBackend(hs, self.primary_base_path) + self.media_storage: MediaStorage = MediaStorage( - self.hs, self.primary_base_path, self.filepaths, storage_providers + self.hs, self.filepaths, storage_providers, local_provider ) + # Log a warning if there are no storage backends configured + if not hs.config.media.enable_local_media_storage and not storage_providers: + logger.warning( + "Local media storage is disabled and no media_storage_providers are " + "configured. All media requests will return 404 errors as there is " + "no storage backend available." + ) + self.clock.looping_call( self._start_update_recently_accessed, UPDATE_RECENTLY_ACCESSED_TS ) @@ -782,10 +798,18 @@ async def _get_remote_media_impl( except SynapseError: raise except Exception as e: - # An exception may be because we downloaded media in another - # process, so let's check if we magically have the media. - media_info = await self.store.get_cached_remote_media(server_name, media_id) - if not media_info: + # If this is a constraint violation, it means another worker + # downloaded the media first. We should fetch the existing media info. + if isinstance(e, self.store.database_engine.module.IntegrityError): + # The file has already been cleaned up in _download_remote_file + # Just fetch the existing media info + media_info = await self.store.get_cached_remote_media( + server_name, media_id + ) + if not media_info: + # This shouldn't happen, but let's raise an error if it does + raise SynapseError(500, "Failed to fetch remote media") + else: raise e file_id = media_info.filesystem_id @@ -806,6 +830,39 @@ async def _get_remote_media_impl( responder = await self.media_storage.fetch_media(file_info) return responder, media_info + async def _store_remote_media_with_cleanup( + self, + server_name: str, + media_id: str, + media_type: str, + time_now_ms: int, + upload_name: str | None, + media_length: int, + filesystem_id: str, + sha256: str, + fname: str, + ) -> None: + """Store remote media in database and clean up file on constraint violation.""" + try: + await self.store.store_cached_remote_media( + origin=server_name, + media_id=media_id, + media_type=media_type, + time_now_ms=time_now_ms, + upload_name=upload_name, + media_length=media_length, + filesystem_id=filesystem_id, + sha256=sha256, + ) + except self.store.database_engine.module.IntegrityError: + # Another worker downloaded the media first. Clean up our file. + try: + os.remove(fname) + except Exception: + pass + # Re-raise so the caller can handle it + raise + async def _download_remote_file( self, server_name: str, @@ -890,26 +947,21 @@ async def _download_remote_file( upload_name = get_filename_from_headers(headers) time_now_ms = self.clock.time_msec() - # Multiple remote media download requests can race (when using - # multiple media repos), so this may throw a violation constraint - # exception. If it does we'll delete the newly downloaded file from - # disk (as we're in the ctx manager). - # - # However: we've already called `finish()` so we may have also - # written to the storage providers. This is preferable to the - # alternative where we call `finish()` *after* this, where we could - # end up having an entry in the DB but fail to write the files to - # the storage providers. - await self.store.store_cached_remote_media( - origin=server_name, - media_id=media_id, - media_type=media_type, - time_now_ms=time_now_ms, - upload_name=upload_name, - media_length=length, - filesystem_id=file_id, - sha256=sha256writer.hexdigest(), - ) + # Multiple remote media download requests can race (when using + # multiple media repos), so this may throw a violation constraint + # exception. If it does we'll delete the newly downloaded file from + # disk. + await self._store_remote_media_with_cleanup( + server_name=server_name, + media_id=media_id, + media_type=media_type, + time_now_ms=time_now_ms, + upload_name=upload_name, + media_length=length, + filesystem_id=file_id, + sha256=sha256writer.hexdigest(), + fname=fname, + ) logger.info("Stored remote media in file %r", fname) @@ -1023,26 +1075,21 @@ async def _federation_download_remote_file( upload_name = get_filename_from_headers(headers) time_now_ms = self.clock.time_msec() - # Multiple remote media download requests can race (when using - # multiple media repos), so this may throw a violation constraint - # exception. If it does we'll delete the newly downloaded file from - # disk (as we're in the ctx manager). - # - # However: we've already called `finish()` so we may have also - # written to the storage providers. This is preferable to the - # alternative where we call `finish()` *after* this, where we could - # end up having an entry in the DB but fail to write the files to - # the storage providers. - await self.store.store_cached_remote_media( - origin=server_name, - media_id=media_id, - media_type=media_type, - time_now_ms=time_now_ms, - upload_name=upload_name, - media_length=length, - filesystem_id=file_id, - sha256=sha256writer.hexdigest(), - ) + # Multiple remote media download requests can race (when using + # multiple media repos), so this may throw a violation constraint + # exception. If it does we'll delete the newly downloaded file from + # disk. + await self._store_remote_media_with_cleanup( + server_name=server_name, + media_id=media_id, + media_type=media_type, + time_now_ms=time_now_ms, + upload_name=upload_name, + media_length=length, + filesystem_id=file_id, + sha256=sha256writer.hexdigest(), + fname=fname, + ) logger.debug("Stored remote media in file %r", fname) @@ -1115,32 +1162,31 @@ async def generate_local_exact_thumbnail( t_type: str, url_cache: bool, ) -> tuple[str, FileInfo] | None: - input_path = await self.media_storage.ensure_media_is_in_local_cache( + async with self.media_storage.ensure_media_is_in_local_cache( FileInfo(None, media_id, url_cache=url_cache) - ) - - try: - thumbnailer = Thumbnailer(input_path) - except ThumbnailError as e: - logger.warning( - "Unable to generate a thumbnail for local media %s using a method of %s and type of %s: %s", - media_id, - t_method, - t_type, - e, - ) - return None + ) as input_path: + try: + thumbnailer = Thumbnailer(input_path) + except ThumbnailError as e: + logger.warning( + "Unable to generate a thumbnail for local media %s using a method of %s and type of %s: %s", + media_id, + t_method, + t_type, + e, + ) + return None - with thumbnailer: - t_byte_source = await defer_to_thread( - self.hs.get_reactor(), - self._generate_thumbnail, - thumbnailer, - t_width, - t_height, - t_method, - t_type, - ) + with thumbnailer: + t_byte_source = await defer_to_thread( + self.hs.get_reactor(), + self._generate_thumbnail, + thumbnailer, + t_width, + t_height, + t_method, + t_type, + ) if t_byte_source: try: @@ -1191,33 +1237,32 @@ async def generate_remote_exact_thumbnail( t_method: str, t_type: str, ) -> str | None: - input_path = await self.media_storage.ensure_media_is_in_local_cache( + async with self.media_storage.ensure_media_is_in_local_cache( FileInfo(server_name, file_id) - ) - - try: - thumbnailer = Thumbnailer(input_path) - except ThumbnailError as e: - logger.warning( - "Unable to generate a thumbnail for remote media %s from %s using a method of %s and type of %s: %s", - media_id, - server_name, - t_method, - t_type, - e, - ) - return None + ) as input_path: + try: + thumbnailer = Thumbnailer(input_path) + except ThumbnailError as e: + logger.warning( + "Unable to generate a thumbnail for remote media %s from %s using a method of %s and type of %s: %s", + media_id, + server_name, + t_method, + t_type, + e, + ) + return None - with thumbnailer: - t_byte_source = await defer_to_thread( - self.hs.get_reactor(), - self._generate_thumbnail, - thumbnailer, - t_width, - t_height, - t_method, - t_type, - ) + with thumbnailer: + t_byte_source = await defer_to_thread( + self.hs.get_reactor(), + self._generate_thumbnail, + thumbnailer, + t_width, + t_height, + t_method, + t_type, + ) if t_byte_source: try: @@ -1287,151 +1332,157 @@ async def _generate_thumbnails( if not requirements: return None - input_path = await self.media_storage.ensure_media_is_in_local_cache( + async with self.media_storage.ensure_media_is_in_local_cache( FileInfo(server_name, file_id, url_cache=url_cache) - ) - - try: - thumbnailer = Thumbnailer(input_path) - except ThumbnailError as e: - logger.warning( - "Unable to generate thumbnails for remote media %s from %s of type %s: %s", - media_id, - server_name, - media_type, - e, - ) - return None - - with thumbnailer: - m_width = thumbnailer.width - m_height = thumbnailer.height - - if m_width * m_height >= self.max_image_pixels: - logger.info( - "Image too large to thumbnail %r x %r > %r", - m_width, - m_height, - self.max_image_pixels, + ) as input_path: + try: + thumbnailer = Thumbnailer(input_path) + except ThumbnailError as e: + logger.warning( + "Unable to generate thumbnails for remote media %s from %s of type %s: %s", + media_id, + server_name, + media_type, + e, ) return None - if thumbnailer.transpose_method is not None: - m_width, m_height = await defer_to_thread( - self.hs.get_reactor(), thumbnailer.transpose - ) + with thumbnailer: + m_width = thumbnailer.width + m_height = thumbnailer.height - # We deduplicate the thumbnail sizes by ignoring the cropped versions if - # they have the same dimensions of a scaled one. - thumbnails: dict[tuple[int, int, str], str] = {} - for requirement in requirements: - if requirement.method == "crop": - thumbnails.setdefault( - (requirement.width, requirement.height, requirement.media_type), - requirement.method, - ) - elif requirement.method == "scale": - t_width, t_height = thumbnailer.aspect( - requirement.width, requirement.height - ) - t_width = min(m_width, t_width) - t_height = min(m_height, t_height) - thumbnails[(t_width, t_height, requirement.media_type)] = ( - requirement.method + if m_width * m_height >= self.max_image_pixels: + logger.info( + "Image too large to thumbnail %r x %r > %r", + m_width, + m_height, + self.max_image_pixels, ) + return None - # Now we generate the thumbnails for each dimension, store it - for (t_width, t_height, t_type), t_method in thumbnails.items(): - # Generate the thumbnail - if t_method == "crop": - t_byte_source = await defer_to_thread( - self.hs.get_reactor(), - thumbnailer.crop, - t_width, - t_height, - t_type, - ) - elif t_method == "scale": - t_byte_source = await defer_to_thread( - self.hs.get_reactor(), - thumbnailer.scale, - t_width, - t_height, - t_type, + if thumbnailer.transpose_method is not None: + m_width, m_height = await defer_to_thread( + self.hs.get_reactor(), thumbnailer.transpose ) - else: - logger.error("Unrecognized method: %r", t_method) - continue - if not t_byte_source: - continue + # We deduplicate the thumbnail sizes by ignoring the cropped versions if + # they have the same dimensions of a scaled one. + thumbnails: dict[tuple[int, int, str], str] = {} + for requirement in requirements: + if requirement.method == "crop": + thumbnails.setdefault( + ( + requirement.width, + requirement.height, + requirement.media_type, + ), + requirement.method, + ) + elif requirement.method == "scale": + t_width, t_height = thumbnailer.aspect( + requirement.width, requirement.height + ) + t_width = min(m_width, t_width) + t_height = min(m_height, t_height) + thumbnails[(t_width, t_height, requirement.media_type)] = ( + requirement.method + ) - file_info = FileInfo( - server_name=server_name, - file_id=file_id, - url_cache=url_cache, - thumbnail=ThumbnailInfo( - width=t_width, - height=t_height, - method=t_method, - type=t_type, - length=t_byte_source.tell(), - ), - ) + # Now we generate the thumbnails for each dimension, store it + for (t_width, t_height, t_type), t_method in thumbnails.items(): + # Generate the thumbnail + if t_method == "crop": + t_byte_source = await defer_to_thread( + self.hs.get_reactor(), + thumbnailer.crop, + t_width, + t_height, + t_type, + ) + elif t_method == "scale": + t_byte_source = await defer_to_thread( + self.hs.get_reactor(), + thumbnailer.scale, + t_width, + t_height, + t_type, + ) + else: + logger.error("Unrecognized method: %r", t_method) + continue + + if not t_byte_source: + continue + + file_info = FileInfo( + server_name=server_name, + file_id=file_id, + url_cache=url_cache, + thumbnail=ThumbnailInfo( + width=t_width, + height=t_height, + method=t_method, + type=t_type, + length=t_byte_source.tell(), + ), + ) - async with self.media_storage.store_into_file(file_info) as (f, fname): - try: - await self.media_storage.write_to_file(t_byte_source, f) - finally: - t_byte_source.close() - - # We flush and close the file to ensure that the bytes have - # been written before getting the size. - f.flush() - f.close() - - t_len = os.path.getsize(fname) - - # Write to database - if server_name: - # Multiple remote media download requests can race (when - # using multiple media repos), so this may throw a violation - # constraint exception. If it does we'll delete the newly - # generated thumbnail from disk (as we're in the ctx - # manager). - # - # However: we've already called `finish()` so we may have - # also written to the storage providers. This is preferable - # to the alternative where we call `finish()` *after* this, - # where we could end up having an entry in the DB but fail - # to write the files to the storage providers. + async with self.media_storage.store_into_file(file_info) as ( + f, + fname, + ): try: - await self.store.store_remote_media_thumbnail( - server_name, - media_id, - file_id, - t_width, - t_height, - t_type, - t_method, - t_len, - ) - except Exception as e: - thumbnail_exists = ( - await self.store.get_remote_media_thumbnail( + await self.media_storage.write_to_file(t_byte_source, f) + finally: + t_byte_source.close() + + # We flush and close the file to ensure that the bytes have + # been written before getting the size. + f.flush() + f.close() + + t_len = os.path.getsize(fname) + + # Write to database + if server_name: + # Multiple remote media download requests can race (when + # using multiple media repos), so this may throw a violation + # constraint exception. If it does we'll delete the newly + # generated thumbnail from disk (as we're in the ctx + # manager). + # + # However: we've already called `finish()` so we may have + # also written to the storage providers. This is preferable + # to the alternative where we call `finish()` *after* this, + # where we could end up having an entry in the DB but fail + # to write the files to the storage providers. + try: + await self.store.store_remote_media_thumbnail( server_name, media_id, + file_id, t_width, t_height, t_type, + t_method, + t_len, + ) + except Exception as e: + thumbnail_exists = ( + await self.store.get_remote_media_thumbnail( + server_name, + media_id, + t_width, + t_height, + t_type, + ) ) + if not thumbnail_exists: + raise e + else: + await self.store.store_local_thumbnail( + media_id, t_width, t_height, t_type, t_method, t_len ) - if not thumbnail_exists: - raise e - else: - await self.store.store_local_thumbnail( - media_id, t_width, t_height, t_type, t_method, t_len - ) return {"width": m_width, "height": m_height} diff --git a/synapse/media/media_storage.py b/synapse/media/media_storage.py index e83869bf4d6..4c942f07f84 100644 --- a/synapse/media/media_storage.py +++ b/synapse/media/media_storage.py @@ -24,6 +24,7 @@ import logging import os import shutil +import tempfile from contextlib import closing from io import BytesIO from types import TracebackType @@ -49,13 +50,13 @@ from synapse.api.errors import NotFoundError from synapse.logging.context import defer_to_thread, run_in_background from synapse.logging.opentracing import start_active_span, trace, trace_with_opname -from synapse.media._base import ThreadedFileSender +from synapse.media.storage_provider import FileStorageProviderBackend from synapse.util.clock import Clock from synapse.util.duration import Duration from synapse.util.file_consumer import BackgroundFileConsumer from ..types import JsonDict -from ._base import FileInfo, Responder +from ._base import FileInfo, Responder, ThreadedFileSender from .filepath import MediaFilePaths if TYPE_CHECKING: @@ -150,27 +151,30 @@ def __getattr__(self, attr_name: str) -> Any: class MediaStorage: - """Responsible for storing/fetching files from local sources. + """Responsible for storing/fetching files from storage providers. Args: hs - local_media_directory: Base path where we store media on disk filepaths storage_providers: List of StorageProvider that are used to fetch and store files. + local_provider: Optional local file storage provider for caching media on disk. """ def __init__( self, hs: "HomeServer", - local_media_directory: str, filepaths: MediaFilePaths, storage_providers: Sequence["StorageProvider"], + local_provider: "FileStorageProviderBackend | None" = None, ): self.hs = hs self.reactor = hs.get_reactor() - self.local_media_directory = local_media_directory self.filepaths = filepaths self.storage_providers = storage_providers + self.local_provider = local_provider + self.local_media_directory: str | None = None + if local_provider is not None: + self.local_media_directory = local_provider.base_directory self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker self.clock = hs.get_clock() @@ -205,11 +209,11 @@ async def store_into_file( """Async Context manager used to get a file like object to write into, as described by file_info. - Actually yields a 2-tuple (file, fname,), where file is a file - like object that can be written to and fname is the absolute path of file - on disk. + Actually yields a 2-tuple (file, media_filepath,), where file is a file + like object that can be written to and media_filepath is the absolute path + of the file on disk. - fname can be used to read the contents from after upload, e.g. to + media_filepath can be used to read the contents from after upload, e.g. to generate thumbnails. Args: @@ -217,25 +221,33 @@ async def store_into_file( Example: - async with media_storage.store_into_file(info) as (f, fname,): + async with media_storage.store_into_file(info) as (f, media_filepath,): # .. write into f ... """ path = self._file_info_to_path(file_info) - fname = os.path.join(self.local_media_directory, path) + is_temp_file = False - dirname = os.path.dirname(fname) - os.makedirs(dirname, exist_ok=True) + if self.local_provider: + media_filepath = os.path.join(self.local_media_directory, path) # type: ignore[arg-type] + os.makedirs(os.path.dirname(media_filepath), exist_ok=True) - try: with start_active_span("writing to main media repo"): - with open(fname, "wb") as f: - yield f, fname + with open(media_filepath, "wb") as f: + yield f, media_filepath + else: + # No local provider, write to temp file + is_temp_file = True + with tempfile.NamedTemporaryFile(delete=False) as f: + media_filepath = f.name + yield cast(BinaryIO, f), media_filepath - with start_active_span("writing to other storage providers"): + # Spam check and store to other providers (runs for both local and temp file cases) + try: + with start_active_span("spam checking and writing to storage providers"): spam_check = ( await self._spam_checker_module_callbacks.check_media_file_for_spam( - ReadableFileWrapper(self.clock, fname), file_info + ReadableFileWrapper(self.clock, media_filepath), file_info ) ) if spam_check != self._spam_checker_module_callbacks.NOT_SPAM: @@ -251,17 +263,23 @@ async def store_into_file( with start_active_span(str(provider)): await provider.store_file(path, file_info) + # If using a temp file, delete it after uploading to storage providers + if is_temp_file: + try: + os.remove(media_filepath) + except Exception: + pass + except Exception as e: try: - os.remove(fname) + os.remove(media_filepath) except Exception: pass raise e from None async def fetch_media(self, file_info: FileInfo) -> Responder | None: - """Attempts to fetch media described by file_info from the local cache - and configured storage providers. + """Attempts to fetch media described by file_info from the configured storage providers. Args: file_info: Metadata about the media file @@ -269,6 +287,18 @@ async def fetch_media(self, file_info: FileInfo) -> Responder | None: Returns: Returns a Responder if the file was found, otherwise None. """ + # URL cache files are stored locally and should not go through storage providers + if file_info.url_cache: + path = self._file_info_to_path(file_info) + if self.local_provider: + local_path = os.path.join(self.local_media_directory, path) # type: ignore[arg-type] + if os.path.isfile(local_path): + # Import here to avoid circular import + from .media_storage import FileResponder + + return FileResponder(self.hs, open(local_path, "rb")) + return None + paths = [self._file_info_to_path(file_info)] # fallback for remote thumbnails with no method in the filename @@ -283,16 +313,18 @@ async def fetch_media(self, file_info: FileInfo) -> Responder | None: ) ) - for path in paths: - local_path = os.path.join(self.local_media_directory, path) - if os.path.exists(local_path): - logger.debug("responding with local file %s", local_path) - return FileResponder(self.hs, open(local_path, "rb")) - logger.debug("local file %s did not exist", local_path) + # Check local provider first, then other storage providers + if self.local_provider: + for path in paths: + res: Any = await self.local_provider.fetch(path, file_info) + if res: + logger.debug("Streaming %s from %s", path, self.local_provider) + return res + logger.debug("%s not found on %s", path, self.local_provider) for provider in self.storage_providers: for path in paths: - res: Any = await provider.fetch(path, file_info) + res = await provider.fetch(path, file_info) if res: logger.debug("Streaming %s from %s", path, provider) return res @@ -301,50 +333,93 @@ async def fetch_media(self, file_info: FileInfo) -> Responder | None: return None @trace - async def ensure_media_is_in_local_cache(self, file_info: FileInfo) -> str: - """Ensures that the given file is in the local cache. Attempts to - download it from storage providers if it isn't. + @contextlib.asynccontextmanager + async def ensure_media_is_in_local_cache( + self, file_info: FileInfo + ) -> AsyncIterator[str]: + """Async context manager that ensures the given file is in the local cache. + Attempts to download it from storage providers if it isn't. + + When no local provider is configured, the file is downloaded to a temporary + location and automatically cleaned up when the context manager exits. Args: file_info - Returns: + Yields: Full path to local file + + Example: + async with media_storage.ensure_media_is_in_local_cache(file_info) as path: + # use path to read the file """ path = self._file_info_to_path(file_info) - local_path = os.path.join(self.local_media_directory, path) - if os.path.exists(local_path): - return local_path - - # Fallback for paths without method names - # Should be removed in the future - if file_info.thumbnail and file_info.server_name: - legacy_path = self.filepaths.remote_media_thumbnail_rel_legacy( - server_name=file_info.server_name, - file_id=file_info.file_id, - width=file_info.thumbnail.width, - height=file_info.thumbnail.height, - content_type=file_info.thumbnail.type, - ) - legacy_local_path = os.path.join(self.local_media_directory, legacy_path) - if os.path.exists(legacy_local_path): - return legacy_local_path - - dirname = os.path.dirname(local_path) - os.makedirs(dirname, exist_ok=True) - - for provider in self.storage_providers: - res: Any = await provider.fetch(path, file_info) - if res: - with res: - consumer = BackgroundFileConsumer( - open(local_path, "wb"), self.reactor - ) - await res.write_to_consumer(consumer) - await consumer.wait() - return local_path + if self.local_provider: + local_path = os.path.join(self.local_media_directory, path) # type: ignore[arg-type] + if os.path.exists(local_path): + yield local_path + return - raise NotFoundError() + # Fallback for paths without method names + # Should be removed in the future + if file_info.thumbnail and file_info.server_name: + legacy_path = self.filepaths.remote_media_thumbnail_rel_legacy( + server_name=file_info.server_name, + file_id=file_info.file_id, + width=file_info.thumbnail.width, + height=file_info.thumbnail.height, + content_type=file_info.thumbnail.type, + ) + legacy_local_path = os.path.join( + self.local_media_directory, # type: ignore[arg-type] + legacy_path, + ) + if os.path.exists(legacy_local_path): + yield legacy_local_path + return + + os.makedirs(os.path.dirname(local_path), exist_ok=True) + + for provider in self.storage_providers: + remote_res: Any = await provider.fetch(path, file_info) + if remote_res: + with remote_res: + consumer = BackgroundFileConsumer( + open(local_path, "wb"), self.reactor + ) + await remote_res.write_to_consumer(consumer) + await consumer.wait() + yield local_path + return + + raise NotFoundError() + else: + # No local provider, download to temp file and clean up after use + for provider in self.storage_providers: + res: Any = await provider.fetch(path, file_info) + if res: + temp_path = None + try: + with tempfile.NamedTemporaryFile( + delete=False, suffix=os.path.splitext(path)[1] + ) as tmp: + temp_path = tmp.name + with res: + consumer = BackgroundFileConsumer( + open(temp_path, "wb"), self.reactor + ) + await res.write_to_consumer(consumer) + await consumer.wait() + yield temp_path + finally: + if temp_path: + try: + os.remove(temp_path) + except Exception: + pass + return + + raise NotFoundError() @trace def _file_info_to_path(self, file_info: FileInfo) -> str: diff --git a/synapse/media/storage_provider.py b/synapse/media/storage_provider.py index a87ffa08926..c5faa25b964 100644 --- a/synapse/media/storage_provider.py +++ b/synapse/media/storage_provider.py @@ -31,7 +31,6 @@ from synapse.util.async_helpers import maybe_awaitable from ._base import FileInfo, Responder -from .media_storage import FileResponder logger = logging.getLogger(__name__) @@ -178,6 +177,9 @@ async def fetch(self, path: str, file_info: FileInfo) -> Responder | None: backup_fname = os.path.join(self.base_directory, path) if os.path.isfile(backup_fname): + # Import here to avoid circular import + from .media_storage import FileResponder + return FileResponder(self.hs, open(backup_fname, "rb")) return None diff --git a/synapse/media/thumbnailer.py b/synapse/media/thumbnailer.py index fd65131c63a..cb86a0ff664 100644 --- a/synapse/media/thumbnailer.py +++ b/synapse/media/thumbnailer.py @@ -633,13 +633,6 @@ async def _select_and_respond_with_thumbnail( # width/height/method so we can just call the "generate exact" # methods. - # First let's check that we do actually have the original image - # still. This will throw a 404 if we don't. - # TODO: We should refetch the thumbnails for remote media. - await self.media_storage.ensure_media_is_in_local_cache( - FileInfo(server_name, file_id, url_cache=url_cache) - ) - if server_name: await self.media_repo.generate_remote_exact_thumbnail( server_name, diff --git a/tests/federation/test_federation_media.py b/tests/federation/test_federation_media.py index 1e849fa605c..9c4e813537e 100644 --- a/tests/federation/test_federation_media.py +++ b/tests/federation/test_federation_media.py @@ -49,6 +49,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: hs.config.media.media_store_path = self.primary_base_path + local_provider = FileStorageProviderBackend(hs, self.primary_base_path) storage_providers = [ StorageProviderWrapper( FileStorageProviderBackend(hs, self.secondary_base_path), @@ -60,7 +61,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.filepaths = MediaFilePaths(self.primary_base_path) self.media_storage = MediaStorage( - hs, self.primary_base_path, self.filepaths, storage_providers + hs, self.filepaths, storage_providers, local_provider ) self.media_repo = hs.get_media_repository() @@ -187,6 +188,115 @@ def test_federation_etag(self) -> None: self.assertNotIn("body", channel.result) +class FederationMediaTest(unittest.FederatingHomeserverTestCase): + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + self.test_dir = tempfile.mkdtemp(prefix="synapse-tests-") + self.addCleanup(shutil.rmtree, self.test_dir) + self.primary_base_path = os.path.join(self.test_dir, "primary") + self.secondary_base_path = os.path.join(self.test_dir, "secondary") + + hs.config.media.media_store_path = self.primary_base_path + + local_provider = FileStorageProviderBackend(hs, self.primary_base_path) + storage_providers = [ + StorageProviderWrapper( + FileStorageProviderBackend(hs, self.secondary_base_path), + store_local=True, + store_remote=False, + store_synchronous=True, + ) + ] + + self.filepaths = MediaFilePaths(self.primary_base_path) + self.media_storage = MediaStorage( + hs, self.filepaths, storage_providers, local_provider + ) + self.media_repo = hs.get_media_repository() + + def test_thumbnail_download_scaled(self) -> None: + content = io.BytesIO(small_png.data) + content_uri = self.get_success( + self.media_repo.create_or_update_content( + "image/png", + "test_png_thumbnail", + content, + 67, + UserID.from_string("@user_id:whatever.org"), + ) + ) + # test with an image file + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/media/thumbnail/{content_uri.media_id}?width=32&height=32&method=scale", + ) + self.pump() + self.assertEqual(200, channel.code) + + content_type = channel.headers.getRawHeaders("content-type") + assert content_type is not None + assert "multipart/mixed" in content_type[0] + assert "boundary" in content_type[0] + + # extract boundary + boundary = content_type[0].split("boundary=")[1] + # split on boundary and check that json field and expected value exist + body = channel.result.get("body") + assert body is not None + stripped_bytes = body.split(b"\r\n" + b"--" + boundary.encode("utf-8")) + found_json = any( + b"\r\nContent-Type: application/json\r\n\r\n{}" in field + for field in stripped_bytes + ) + self.assertTrue(found_json) + + # check that the png file exists and matches the expected scaled bytes + found_file = any(small_png.expected_scaled in field for field in stripped_bytes) + self.assertTrue(found_file) + + def test_thumbnail_download_cropped(self) -> None: + content = io.BytesIO(small_png.data) + content_uri = self.get_success( + self.media_repo.create_or_update_content( + "image/png", + "test_png_thumbnail", + content, + 67, + UserID.from_string("@user_id:whatever.org"), + ) + ) + # test with an image file + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/media/thumbnail/{content_uri.media_id}?width=32&height=32&method=crop", + ) + self.pump() + self.assertEqual(200, channel.code) + + content_type = channel.headers.getRawHeaders("content-type") + assert content_type is not None + assert "multipart/mixed" in content_type[0] + assert "boundary" in content_type[0] + + # extract boundary + boundary = content_type[0].split("boundary=")[1] + # split on boundary and check that json field and expected value exist + body = channel.result.get("body") + assert body is not None + stripped_bytes = body.split(b"\r\n" + b"--" + boundary.encode("utf-8")) + found_json = any( + b"\r\nContent-Type: application/json\r\n\r\n{}" in field + for field in stripped_bytes + ) + self.assertTrue(found_json) + + # check that the png file exists and matches the expected cropped bytes + found_file = any( + small_png.expected_cropped in field for field in stripped_bytes + ) + self.assertTrue(found_file) + + class FederationThumbnailTest(unittest.FederatingHomeserverTestCase): def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: super().prepare(reactor, clock, hs) @@ -197,6 +307,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: hs.config.media.media_store_path = self.primary_base_path + local_provider = FileStorageProviderBackend(hs, self.primary_base_path) storage_providers = [ StorageProviderWrapper( FileStorageProviderBackend(hs, self.secondary_base_path), @@ -208,7 +319,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.filepaths = MediaFilePaths(self.primary_base_path) self.media_storage = MediaStorage( - hs, self.primary_base_path, self.filepaths, storage_providers + hs, self.filepaths, storage_providers, local_provider ) self.media_repo = hs.get_media_repository() diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index e56354e0b3f..631718a3666 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -48,7 +48,10 @@ from synapse.media._base import FileInfo, ThumbnailInfo from synapse.media.filepath import MediaFilePaths from synapse.media.media_storage import MediaStorage, ReadableFileWrapper -from synapse.media.storage_provider import FileStorageProviderBackend +from synapse.media.storage_provider import ( + FileStorageProviderBackend, + StorageProviderWrapper, +) from synapse.media.thumbnailer import ThumbnailProvider from synapse.module_api import ModuleApi from synapse.module_api.callbacks.spamchecker_callbacks import load_legacy_spam_checkers @@ -77,11 +80,19 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: hs.config.media.media_store_path = self.primary_base_path - storage_providers = [FileStorageProviderBackend(hs, self.secondary_base_path)] + local_provider = FileStorageProviderBackend(hs, self.primary_base_path) + storage_providers = [ + StorageProviderWrapper( + FileStorageProviderBackend(hs, self.secondary_base_path), + store_local=True, + store_remote=False, + store_synchronous=True, + ), + ] self.filepaths = MediaFilePaths(self.primary_base_path) self.media_storage = MediaStorage( - hs, self.primary_base_path, self.filepaths, storage_providers + hs, self.filepaths, storage_providers, local_provider ) def test_ensure_media_is_in_local_cache(self) -> None: @@ -102,29 +113,31 @@ def test_ensure_media_is_in_local_cache(self) -> None: # to the local cache. file_info = FileInfo(None, media_id) - # This uses a real blocking threadpool so we have to wait for it to be - # actually done :/ - x = defer.ensureDeferred( - self.media_storage.ensure_media_is_in_local_cache(file_info) - ) + async def test_ensure_media() -> None: + async with self.media_storage.ensure_media_is_in_local_cache( + file_info + ) as local_path: + self.assertTrue(os.path.exists(local_path)) - # Hotloop until the threadpool does its job... - self.wait_on_thread(x) + # Asserts the file is under the expected local cache directory + self.assertEqual( + os.path.commonprefix([self.primary_base_path, local_path]), + self.primary_base_path, + ) - local_path = self.get_success(x) + with open(local_path) as f: + body = f.read() - self.assertTrue(os.path.exists(local_path)) + self.assertEqual(test_body, body) - # Asserts the file is under the expected local cache directory - self.assertEqual( - os.path.commonprefix([self.primary_base_path, local_path]), - self.primary_base_path, - ) + # This uses a real blocking threadpool so we have to wait for it to be + # actually done :/ + x = defer.ensureDeferred(test_ensure_media()) - with open(local_path) as f: - body = f.read() + # Hotloop until the threadpool does its job... + self.wait_on_thread(x) - self.assertEqual(test_body, body) + self.get_success(x) @attr.s(auto_attribs=True, slots=True, frozen=True) diff --git a/tests/replication/test_multi_media_repo.py b/tests/replication/test_multi_media_repo.py index 8dbb989850a..000dbb55940 100644 --- a/tests/replication/test_multi_media_repo.py +++ b/tests/replication/test_multi_media_repo.py @@ -206,6 +206,53 @@ def test_download_simple_file_race(self) -> None: # We expect only one new file to have been persisted. self.assertEqual(start_count + 1, self._count_remote_media()) + @override_config( + {"enable_authenticated_media": False, "enable_local_media_storage": False} + ) + def test_download_simple_file_race_no_local_storage(self) -> None: + """Test that fetching remote media works when local storage is disabled. + + This test verifies that the system handles the case where local storage + is disabled. Without storage providers configured, the media cannot be + retrieved, but the important thing is that the race condition is still + handled correctly. + """ + hs1 = self.make_worker_hs("synapse.app.generic_worker") + hs2 = self.make_worker_hs("synapse.app.generic_worker") + + start_count = self._count_remote_media() + + # Make two requests without responding to the outbound media requests. + channel1, request1 = self._get_media_req(hs1, "example.com:443", "ABC123") + channel2, request2 = self._get_media_req(hs2, "example.com:443", "ABC123") + + # Respond to the first outbound media request and check that the client + # request is successful + request1.setResponseCode(200) + request1.responseHeaders.setRawHeaders(b"Content-Type", [b"text/plain"]) + request1.write(b"Hello!") + request1.finish() + + self.pump(0.1) + + # With local storage disabled and no storage providers, + # we expect a 404 error + self.assertEqual(channel1.code, 404, channel1.result["body"]) + + # Now respond to the second with the same content. + request2.setResponseCode(200) + request2.responseHeaders.setRawHeaders(b"Content-Type", [b"text/plain"]) + request2.write(b"Hello!") + request2.finish() + + self.pump(0.1) + + # Same for the second request + self.assertEqual(channel2.code, 404, channel2.result["body"]) + + # No files should be stored locally + self.assertEqual(start_count, self._count_remote_media()) + @override_config({"enable_authenticated_media": False}) def test_download_image_race(self) -> None: """Test that fetching remote *images* from two different processes at @@ -428,6 +475,53 @@ def test_download_simple_file_race(self) -> None: # We expect only one new file to have been persisted. self.assertEqual(start_count + 1, self._count_remote_media()) + @override_config({"enable_local_media_storage": False}) + def test_download_simple_file_race_no_local_storage(self) -> None: + """Test that fetching remote media works when local storage is disabled. + + When enable_local_media_storage is False, files should only be stored in + the storage providers and not in the local filesystem. + """ + hs1 = self.make_worker_hs("synapse.app.generic_worker") + hs2 = self.make_worker_hs("synapse.app.generic_worker") + + start_count = self._count_remote_media() + + # Make two requests without responding to the outbound media requests. + channel1, request1 = self._get_media_req(hs1, "example.com:443", "ABC123") + channel2, request2 = self._get_media_req(hs2, "example.com:443", "ABC123") + + # Respond to the first outbound media request and check that the client + # request is successful + request1.setResponseCode(200) + request1.responseHeaders.setRawHeaders( + b"Content-Type", + ["multipart/mixed; boundary=6067d4698f8d40a0a794ea7d7379d53a"], + ) + request1.write(self.file_data) + request1.finish() + + self.pump(0.1) + # With local storage disabled and no storage providers, + # we expect a 404 error + self.assertEqual(channel1.code, 404, channel1.result["body"]) + + # Now respond to the second with the same content. + request2.setResponseCode(200) + request2.responseHeaders.setRawHeaders( + b"Content-Type", + ["multipart/mixed; boundary=6067d4698f8d40a0a794ea7d7379d53a"], + ) + request2.write(self.file_data) + request2.finish() + + self.pump(0.1) + + self.assertEqual(channel2.code, 404, channel2.result["body"]) + + # With local storage disabled, no files should be stored locally + self.assertEqual(start_count, self._count_remote_media()) + def test_download_image_race(self) -> None: """Test that fetching remote *images* from two different processes at the same time works. From 4dcf113bffc3de557d86fcb90bd47b12a5f287e6 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Wed, 7 Jan 2026 12:52:21 +0000 Subject: [PATCH 096/139] Support for stable `m.oauth` UIA stage for MSC4312 (#19273) --- changelog.d/19273.feature | 1 + synapse/rest/client/auth.py | 6 +++++- synapse/rest/client/keys.py | 10 ++++++++++ tests/rest/client/test_keys.py | 30 +++++++++++++++++++++++++++++- 4 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 changelog.d/19273.feature diff --git a/changelog.d/19273.feature b/changelog.d/19273.feature new file mode 100644 index 00000000000..ef849b3c37f --- /dev/null +++ b/changelog.d/19273.feature @@ -0,0 +1 @@ +Stabilise support for [MSC4312](https://github.com/matrix-org/matrix-spec-proposals/pull/4312)'s `m.oauth` User-Interactive Auth stage for resetting cross-signing identity with the OAuth 2.0 API. The old, unstable name (`org.matrix.cross_signing_reset`) is now deprecated and will be removed in a future release. \ No newline at end of file diff --git a/synapse/rest/client/auth.py b/synapse/rest/client/auth.py index 600bb51a7e7..f3254990443 100644 --- a/synapse/rest/client/auth.py +++ b/synapse/rest/client/auth.py @@ -67,7 +67,11 @@ async def on_GET(self, request: SynapseRequest, stagetype: str) -> None: if not session: raise SynapseError(400, "No session supplied") - if stagetype == "org.matrix.cross_signing_reset": + # We support the unstable (`org.matrix.cross_signing_reset`) name from MSC4312 until + # enough clients have adopted the stable name (`m.oauth`). + # Note: `org.matrix.cross_signing_reset` *is* the stable name of the *action* in the + # authorization server metadata. The unstable status only applies to the UIA stage name. + if stagetype == "m.oauth" or stagetype == "org.matrix.cross_signing_reset": if self.hs.config.mas.enabled: assert isinstance(self.auth, MasDelegatedAuth) diff --git a/synapse/rest/client/keys.py b/synapse/rest/client/keys.py index 5f488674b4a..502c5d495a9 100644 --- a/synapse/rest/client/keys.py +++ b/synapse/rest/client/keys.py @@ -560,9 +560,14 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: { "session": "dummy", "flows": [ + {"stages": ["m.oauth"]}, + # The unstable name from MSC4312 should be supported until enough clients have adopted the stable (`m.oauth`) name: {"stages": ["org.matrix.cross_signing_reset"]}, ], "params": { + "m.oauth": { + "url": url, + }, "org.matrix.cross_signing_reset": { "url": url, }, @@ -594,9 +599,14 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: { "session": "dummy", "flows": [ + {"stages": ["m.oauth"]}, + # The unstable name from MSC4312 should be supported until enough clients have adopted the stable (`m.oauth`) name: {"stages": ["org.matrix.cross_signing_reset"]}, ], "params": { + "m.oauth": { + "url": url, + }, "org.matrix.cross_signing_reset": { "url": url, }, diff --git a/tests/rest/client/test_keys.py b/tests/rest/client/test_keys.py index 817edfb8d30..9c83a284d73 100644 --- a/tests/rest/client/test_keys.py +++ b/tests/rest/client/test_keys.py @@ -353,6 +353,7 @@ class SigningKeyUploadServletTestCase(unittest.HomeserverTestCase): ] OIDC_ADMIN_TOKEN = "_oidc_admin_token" + ACCOUNT_MANAGEMENT_URL = "https://my-account.issuer" @unittest.skip_unless(HAS_AUTHLIB, "requires authlib") @override_config( @@ -362,7 +363,7 @@ class SigningKeyUploadServletTestCase(unittest.HomeserverTestCase): "msc3861": { "enabled": True, "issuer": "https://issuer", - "account_management_url": "https://my-account.issuer", + "account_management_url": ACCOUNT_MANAGEMENT_URL, "client_id": "id", "client_auth_method": "client_secret_post", "client_secret": "secret", @@ -457,6 +458,33 @@ async def mocked_get_user_by_access_token( }, ) self.assertEqual(channel.code, HTTPStatus.UNAUTHORIZED, channel.json_body) + # Ensure that the response contains the expected UIA flows from https://spec.matrix.org/v1.17/client-server-api/#oauth-authentication + self.assertIn( + {"stages": ["m.oauth"]}, + channel.json_body["flows"], + "m.oauth flow not found", + ) + self.assertSubstring( + self.ACCOUNT_MANAGEMENT_URL, + channel.json_body["params"]["m.oauth"]["url"], + "m.oauth url does not match account management URL", + ) + self.assertSubstring( + "action=org.matrix.cross_signing_reset", + channel.json_body["params"]["m.oauth"]["url"], + "m.oauth url does not include expected action", + ) + # Unstable version of the flow + self.assertIn( + {"stages": ["org.matrix.cross_signing_reset"]}, + channel.json_body["flows"], + "unstable org.matrix.cross_signing_reset flow not found", + ) + self.assertEqual( + channel.json_body["params"]["org.matrix.cross_signing_reset"]["url"], + channel.json_body["params"]["m.oauth"]["url"], + "unstable org.matrix.cross_signing_reset url does not match m.oauth url", + ) # Pretend that MAS did UIA and allowed us to replace the master key. channel = self.make_request( From 7ea78671a3903eac8de42e38287e9fcb74e93d4c Mon Sep 17 00:00:00 2001 From: Kierre Date: Wed, 7 Jan 2026 09:10:25 -0500 Subject: [PATCH 097/139] Drop support for Ubuntu 25.04 'Plucky Puffin', add support for Ubuntu 25.10 'Questing Quokka' (#19348) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/19348.misc | 1 + docs/upgrade.md | 8 ++++++++ scripts-dev/build_debian_packages.py | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19348.misc diff --git a/changelog.d/19348.misc b/changelog.d/19348.misc new file mode 100644 index 00000000000..d4255215d9f --- /dev/null +++ b/changelog.d/19348.misc @@ -0,0 +1 @@ +Drop support for Ubuntu 25.04 'Plucky Puffin', add support for Ubuntu 25.10 'Questing Quokka'. \ No newline at end of file diff --git a/docs/upgrade.md b/docs/upgrade.md index 5e7fa31580c..7c9efaccafd 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -117,6 +117,14 @@ each upgrade are complete before moving on to the next upgrade, to avoid stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). +# Upgrading to v1.146.0 + +## Drop support for Ubuntu 25.04 Plucky Puffin, and add support for 25.10 Questing Quokka + +Ubuntu 25.04 Plucky Puffin [is end-of-life as of 17 Jan +2026](https://endoflife.date/ubuntu). This release drops support for Ubuntu +25.04, and in its place adds support for Ubuntu 25.10 Questing Quokka. + # Upgrading to v1.144.0 ## Worker support for unstable MSC4140 `/restart` endpoint diff --git a/scripts-dev/build_debian_packages.py b/scripts-dev/build_debian_packages.py index d462fe6c56b..7741d4aecdf 100755 --- a/scripts-dev/build_debian_packages.py +++ b/scripts-dev/build_debian_packages.py @@ -31,7 +31,7 @@ "debian:sid", # (rolling distro, no EOL) "ubuntu:jammy", # 22.04 LTS (EOL 2027-04) (our EOL forced by Python 3.10 is 2026-10-04) "ubuntu:noble", # 24.04 LTS (EOL 2029-06) - "ubuntu:plucky", # 25.04 (EOL 2026-01) + "ubuntu:questing", # 25.10 (EOL 2026-07) "debian:trixie", # (EOL not specified yet) ) From 13dff90b5bf94ae97644c0c8f19c444d4057d86b Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 7 Jan 2026 10:08:03 -0700 Subject: [PATCH 098/139] Fix sdist include formatting for maturin --- pyproject.toml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7aa5840a9df..026f3e58709 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -415,20 +415,22 @@ line-ending = "auto" manifest-path = "rust/Cargo.toml" module-name = "synapse.synapse_rust" python-source = "." -sdist-include = [ +include = [ "AUTHORS.rst", "book.toml", - "changelog.d", + "changelog.d/**/*", "CHANGES.md", "CONTRIBUTING.md", - "demo", - "docs", + "demo/**/*", + "docs/**/*", "INSTALL.md", + "LICENSE-AGPL-3.0", + "LICENSE-COMMERCIAL", "mypy.ini", - "scripts-dev", - "synmark", + "scripts-dev/**/*", + "synmark/**/*", "sytest-blacklist", - "tests", + "tests/**/*", "UPGRADE.rst", "Cargo.toml", "Cargo.lock", @@ -436,7 +438,7 @@ sdist-include = [ "rust/build.rs", "rust/src/**", ] -sdist-exclude = ["synapse/*.so"] +exclude = ["synapse/*.so"] [build-system] # The upper bounds here are defensive, intended to prevent situations like From ecd67df49d89b5efa413ca419def33f6c255312c Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 7 Jan 2026 10:11:44 -0700 Subject: [PATCH 099/139] 1.145.0rc2 --- CHANGES.md | 8 ++++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 8192a0c0aca..74a4c3d5e89 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,11 @@ +# Synapse 1.145.0rc2 (2026-01-07) + +No significant changes since 1.145.0rc1. + +This RC fixes the source distribution packaging for uploading to PyPI. + + + # Synapse 1.145.0rc1 (2026-01-06) ## End of Life of Ubuntu 25.04 Plucky Puffin diff --git a/debian/changelog b/debian/changelog index 3b7d0b773ec..b0870c798cd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.145.0~rc2) stable; urgency=medium + + * New Synapse release 1.145.0rc2. + + -- Synapse Packaging team Wed, 07 Jan 2026 10:10:07 -0700 + matrix-synapse-py3 (1.145.0~rc1) stable; urgency=medium * New Synapse release 1.145.0rc1. diff --git a/pyproject.toml b/pyproject.toml index 026f3e58709..960d7891a88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.145.0rc1" +version = "1.145.0rc2" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 8ff1960878173c2a98e8156841b3a7635fd43627 Mon Sep 17 00:00:00 2001 From: Lukas Tautz <104827162+lukastautz@users.noreply.github.com> Date: Wed, 7 Jan 2026 19:32:16 +0100 Subject: [PATCH 100/139] Fix: use correct parameter when calling get_local_current_membership_for_user_in_room (#19353) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/19353.bugfix | 1 + synapse/handlers/message.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19353.bugfix diff --git a/changelog.d/19353.bugfix b/changelog.d/19353.bugfix new file mode 100644 index 00000000000..5dc006312c2 --- /dev/null +++ b/changelog.d/19353.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in 1.61.0 where a user's membership in a room was accidentally ignored when considering access to historical state events in rooms with the "shared" history visibility. Contributed by Lukas Tautz. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 7808bd68cbf..99ce120736d 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -322,7 +322,7 @@ async def _user_can_see_state_at_event( current_membership, _, ) = await self.store.get_local_current_membership_for_user_in_room( - user_id, event_id + user_id, room_id ) return current_membership == Membership.JOIN From 1db23023037f4610c3ee928d7495ec675ab25245 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 7 Jan 2026 13:46:03 -0500 Subject: [PATCH 101/139] Bump mdbook from 0.4.17 -> 0.5.2 and remove custom table-of-contents plugin (#19356) --- .github/workflows/docs-pr.yaml | 19 +- .github/workflows/docs.yaml | 2 +- book.toml | 4 - changelog.d/19356.misc | 1 + docs/.htmltest.yml | 5 + docs/website_files/README.md | 27 +- docs/website_files/remove-nav-buttons.css | 8 - docs/website_files/table-of-contents.css | 47 --- docs/website_files/table-of-contents.js | 148 ---------- docs/website_files/theme/index.hbs | 338 +++++++++++++--------- scripts-dev/schema_versions.py | 2 +- 11 files changed, 229 insertions(+), 372 deletions(-) create mode 100644 changelog.d/19356.misc create mode 100644 docs/.htmltest.yml delete mode 100644 docs/website_files/remove-nav-buttons.css delete mode 100644 docs/website_files/table-of-contents.css delete mode 100644 docs/website_files/table-of-contents.js diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index eed2acc4a9c..4480fd80bc1 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -21,7 +21,7 @@ jobs: - name: Setup mdbook uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0 with: - mdbook-version: '0.4.17' + mdbook-version: '0.5.2' - name: Setup python uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 @@ -55,7 +55,7 @@ jobs: - name: Setup mdbook uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0 with: - mdbook-version: '0.4.17' + mdbook-version: '0.5.2' - name: Setup htmltest run: | @@ -64,8 +64,17 @@ jobs: tar zxf htmltest_0.17.0_linux_amd64.tar.gz - name: Test links with htmltest - # Build the book with `./` as the site URL (to make checks on 404.html possible) - # Then run htmltest (without checking external links since that involves the network and is slow). run: | + # Build the book with `./` as the site URL (to make checks on 404.html possible) MDBOOK_OUTPUT__HTML__SITE_URL="./" mdbook build - ./htmltest book --skip-external + + # Delete the contents of the print.html file, as it can raise false + # positives during link checking. + # + # We empty out the file, instead of deleting it, as doing so would + # just cause htmltest to complain that links to it were invalid. + # Ideally `htmltest` would have an option to ignore specific files + # instead. + echo '' > book/print.html + + ./htmltest book --conf docs/.htmltest.yml diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index e33add18305..d64adf924c4 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -58,7 +58,7 @@ jobs: - name: Setup mdbook uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0 with: - mdbook-version: '0.4.17' + mdbook-version: '0.5.2' - name: Set version of docs run: echo 'window.SYNAPSE_VERSION = "${{ needs.pre.outputs.branch-version }}";' > ./docs/website_files/version.js diff --git a/book.toml b/book.toml index 98d749ed392..99e4d03bf41 100644 --- a/book.toml +++ b/book.toml @@ -4,7 +4,6 @@ title = "Synapse" authors = ["The Matrix.org Foundation C.I.C."] language = "en" -multilingual = false # The directory that documentation files are stored in src = "docs" @@ -31,13 +30,10 @@ site-url = "/synapse/" # Additional HTML, JS, CSS that's injected into each page of the book. # More information available in docs/website_files/README.md additional-css = [ - "docs/website_files/table-of-contents.css", - "docs/website_files/remove-nav-buttons.css", "docs/website_files/indent-section-headers.css", "docs/website_files/version-picker.css", ] additional-js = [ - "docs/website_files/table-of-contents.js", "docs/website_files/version-picker.js", "docs/website_files/version.js", ] diff --git a/changelog.d/19356.misc b/changelog.d/19356.misc new file mode 100644 index 00000000000..3b9d16a3800 --- /dev/null +++ b/changelog.d/19356.misc @@ -0,0 +1 @@ +Bump `mdbook` from 0.4.17 to 0.5.2 and remove our custom table-of-contents plugin in favour of the new default functionality. \ No newline at end of file diff --git a/docs/.htmltest.yml b/docs/.htmltest.yml new file mode 100644 index 00000000000..59d34ce0a0b --- /dev/null +++ b/docs/.htmltest.yml @@ -0,0 +1,5 @@ +# Configuration for htmltest, which we run in CI to check that links aren't broken in the built documentation. +# See all config options: https://github.com/wjdp/htmltest#wrench-configuration + +# Don't check external links, as that requires network access and is slow. +CheckExternal: false \ No newline at end of file diff --git a/docs/website_files/README.md b/docs/website_files/README.md index bc51c4865ea..b5521997d56 100644 --- a/docs/website_files/README.md +++ b/docs/website_files/README.md @@ -9,27 +9,18 @@ point to additional JS/CSS in this directory that are added on each page load. I addition, the `theme` directory contains files that overwrite their counterparts in each of the default themes included with mdbook. -Currently we use these files to generate a floating Table of Contents panel. The code for -which was partially taken from -[JorelAli/mdBook-pagetoc](https://github.com/JorelAli/mdBook-pagetoc/) -before being modified such that it scrolls with the content of the page. This is handled -by the `table-of-contents.js/css` files. The table of contents panel only appears on pages -that have more than one header, as well as only appearing on desktop-sized monitors. +Currently we use these files to make a few modifications: -We remove the navigation arrows which typically appear on the left and right side of the -screen on desktop as they interfere with the table of contents. This is handled by -the `remove-nav-buttons.css` file. +* We stylise the chapter titles in the left sidebar by indenting them + slightly so that they are more visually distinguishable from the section headers + (the bold titles). This is done through the `indent-section-headers.css` file. -Finally, we also stylise the chapter titles in the left sidebar by indenting them -slightly so that they are more visually distinguishable from the section headers -(the bold titles). This is done through the `indent-section-headers.css` file. - -In addition to these modifications, we have added a version picker to the documentation. -Users can switch between documentations for different versions of Synapse. -This functionality was implemented through the `version-picker.js` and -`version-picker.css` files. +* We add a version picker pertaining to the different documentation versions + shipped with each version of Synapse. This functionality was implemented through + the `version-picker.js` and `version-picker.css` files, and is currently the only + requirement for the custom `theme/`. More information can be found in mdbook's official documentation for [injecting page JS/CSS](https://rust-lang.github.io/mdBook/format/config.html) and -[customising the default themes](https://rust-lang.github.io/mdBook/format/theme/index.html). \ No newline at end of file +[customising the default themes](https://rust-lang.github.io/mdBook/format/theme/index.html). diff --git a/docs/website_files/remove-nav-buttons.css b/docs/website_files/remove-nav-buttons.css deleted file mode 100644 index 4b280794ea1..00000000000 --- a/docs/website_files/remove-nav-buttons.css +++ /dev/null @@ -1,8 +0,0 @@ -/* Remove the prev, next chapter buttons as they interfere with the - * table of contents. - * Note that the table of contents only appears on desktop, thus we - * only remove the desktop (wide) chapter buttons. - */ -.nav-wide-wrapper { - display: none -} \ No newline at end of file diff --git a/docs/website_files/table-of-contents.css b/docs/website_files/table-of-contents.css deleted file mode 100644 index 1b6f44b66a2..00000000000 --- a/docs/website_files/table-of-contents.css +++ /dev/null @@ -1,47 +0,0 @@ -:root { - --pagetoc-width: 250px; -} - -@media only screen and (max-width:1439px) { - .sidetoc { - display: none; - } -} - -@media only screen and (min-width:1440px) { - main { - position: relative; - margin-left: 100px !important; - margin-right: var(--pagetoc-width) !important; - } - .sidetoc { - margin-left: auto; - margin-right: auto; - left: calc(100% + (var(--content-max-width))/4 - 140px); - position: absolute; - text-align: right; - } - .pagetoc { - position: fixed; - width: var(--pagetoc-width); - overflow: auto; - right: 20px; - height: calc(100% - var(--menu-bar-height)); - } - .pagetoc a { - color: var(--fg) !important; - display: block; - padding: 5px 15px 5px 10px; - text-align: left; - text-decoration: none; - } - .pagetoc a:hover, - .pagetoc a.active { - background: var(--sidebar-bg) !important; - color: var(--sidebar-fg) !important; - } - .pagetoc .active { - background: var(--sidebar-bg); - color: var(--sidebar-fg); - } -} diff --git a/docs/website_files/table-of-contents.js b/docs/website_files/table-of-contents.js deleted file mode 100644 index 772da97fb9d..00000000000 --- a/docs/website_files/table-of-contents.js +++ /dev/null @@ -1,148 +0,0 @@ -const getPageToc = () => document.getElementsByClassName('pagetoc')[0]; - -const pageToc = getPageToc(); -const pageTocChildren = [...pageToc.children]; -const headers = [...document.getElementsByClassName('header')]; - - -// Select highlighted item in ToC when clicking an item -pageTocChildren.forEach(child => { - child.addEventHandler('click', () => { - pageTocChildren.forEach(child => { - child.classList.remove('active'); - }); - child.classList.add('active'); - }); -}); - - -/** - * Test whether a node is in the viewport - */ -function isInViewport(node) { - const rect = node.getBoundingClientRect(); - return rect.top >= 0 && rect.left >= 0 && rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) && rect.right <= (window.innerWidth || document.documentElement.clientWidth); -} - - -/** - * Set a new ToC entry. - * Clear any previously highlighted ToC items, set the new one, - * and adjust the ToC scroll position. - */ -function setTocEntry() { - let activeEntry; - const pageTocChildren = [...getPageToc().children]; - - // Calculate which header is the current one at the top of screen - headers.forEach(header => { - if (window.pageYOffset >= header.offsetTop) { - activeEntry = header; - } - }); - - // Update selected item in ToC when scrolling - pageTocChildren.forEach(child => { - if (activeEntry.href.localeCompare(child.href) === 0) { - child.classList.add('active'); - } else { - child.classList.remove('active'); - } - }); - - let tocEntryForLocation = document.querySelector(`nav a[href="${activeEntry.href}"]`); - if (tocEntryForLocation) { - const headingForLocation = document.querySelector(activeEntry.hash); - if (headingForLocation && isInViewport(headingForLocation)) { - // Update ToC scroll - const nav = getPageToc(); - const content = document.querySelector('html'); - if (content.scrollTop !== 0) { - nav.scrollTo({ - top: tocEntryForLocation.offsetTop - 100, - left: 0, - behavior: 'smooth', - }); - } else { - nav.scrollTop = 0; - } - } - } -} - - -/** - * Populate sidebar on load - */ -window.addEventListener('load', () => { - // Prevent rendering the table of contents of the "print book" page, as it - // will end up being rendered into the output (in a broken-looking way) - - // Get the name of the current page (i.e. 'print.html') - const pageNameExtension = window.location.pathname.split('/').pop(); - - // Split off the extension (as '.../print' is also a valid page name), which - // should result in 'print' - const pageName = pageNameExtension.split('.')[0]; - if (pageName === "print") { - // Don't render the table of contents on this page - return; - } - - // Only create table of contents if there is more than one header on the page - if (headers.length <= 1) { - return; - } - - // Create an entry in the page table of contents for each header in the document - headers.forEach((header, index) => { - const link = document.createElement('a'); - - // Indent shows hierarchy - let indent = '0px'; - switch (header.parentElement.tagName) { - case 'H1': - indent = '5px'; - break; - case 'H2': - indent = '20px'; - break; - case 'H3': - indent = '30px'; - break; - case 'H4': - indent = '40px'; - break; - case 'H5': - indent = '50px'; - break; - case 'H6': - indent = '60px'; - break; - default: - break; - } - - let tocEntry; - if (index == 0) { - // Create a bolded title for the first element - tocEntry = document.createElement("strong"); - tocEntry.innerHTML = header.text; - } else { - // All other elements are non-bold - tocEntry = document.createTextNode(header.text); - } - link.appendChild(tocEntry); - - link.style.paddingLeft = indent; - link.href = header.href; - pageToc.appendChild(link); - }); - setTocEntry.call(); -}); - - -// Handle active headers on scroll, if there is more than one header on the page -if (headers.length > 1) { - window.addEventListener('scroll', setTocEntry); -} diff --git a/docs/website_files/theme/index.hbs b/docs/website_files/theme/index.hbs index 9cf7521e80b..3126184c9b7 100644 --- a/docs/website_files/theme/index.hbs +++ b/docs/website_files/theme/index.hbs @@ -1,11 +1,11 @@ - + {{ title }} {{#if is_print }} - + {{/if}} {{#if base_url}} @@ -15,60 +15,78 @@ {{> head}} - - + {{#if favicon_svg}} - + {{/if}} {{#if favicon_png}} - + {{/if}} - - - + + + {{#if print_enable}} - + {{/if}} - - {{#if copy_fonts}} - - {{/if}} + - - - + + + {{#each additional_css}} - + {{/each}} {{#if mathjax_support}} - + {{/if}} + + + + + - - - +
+
+

Keyboard shortcuts

+
+

Press ← or → to navigate between chapters

+ {{#if search_enabled}} +

Press S or / to search in the book

+ {{/if}} +

Press ? to show this help

+

Press Esc to hide this help

+
+
+
+
- - + + - -