From 698a56dca2c37a972eaf64f7cdbe81b1b4278d2c Mon Sep 17 00:00:00 2001 From: Jason Little Date: Fri, 30 Jan 2026 18:59:41 -0600 Subject: [PATCH 1/6] Attempt forcing a lower case comparison to "psycopg" in complement.sh --- scripts-dev/complement.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index 7c646683de1..58086fb9936 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -210,7 +210,7 @@ main() { echo_if_github "::endgroup::" fi - + echo "Docker images built." else echo "Skipping Docker image build as requested." @@ -281,7 +281,8 @@ main() { export COMPLEMENT_SPAWN_HS_TIMEOUT_SECS=120 else export PASS_SYNAPSE_COMPLEMENT_USE_WORKERS= - if [[ "$POSTGRES" = "psycopg" ]]; then + # A handy pattern for lower-casing all letters in a variable, `${variable,,}` + if [[ "${POSTGRES,,}" = "psycopg" ]]; then export PASS_SYNAPSE_COMPLEMENT_DATABASE=psycopg elif [[ -n "$POSTGRES" ]]; then export PASS_SYNAPSE_COMPLEMENT_DATABASE=postgres @@ -319,7 +320,7 @@ main() { echo "Skipping Complement run as requested." return 0 fi - + # Run the tests! echo "Running Complement with ${test_args[@]} $@ ${test_packages[@]}" cd "$COMPLEMENT_DIR" From 088ead3f1775996aeb7e8d36061aa02d704476ea Mon Sep 17 00:00:00 2001 From: Jason Little Date: Fri, 30 Jan 2026 19:00:25 -0600 Subject: [PATCH 2/6] Add mini docstrings to the abstract postgres engine class, as well as the concrete definitions --- synapse/storage/engines/postgres.py | 7 +++++++ synapse/storage/engines/psycopg.py | 8 ++++++++ synapse/storage/engines/psycopg2.py | 6 ++++++ 3 files changed, 21 insertions(+) diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 54202cd70fa..a89654a620e 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -44,6 +44,13 @@ class PostgresEngine( BaseDatabaseEngine[ConnectionType, CursorType, IsolationLevelType], metaclass=abc.ABCMeta, ): + """ + An Abstract Base Class for use by Postgres connection libraries. + Currently supported: + * Psycopg2 -> `Psycopg2Engine` + * Psycopg -> `PsycopgEngine` + """ + isolation_level_map: Mapping[IsolationLevel, IsolationLevelType] default_isolation_level: IsolationLevelType diff --git a/synapse/storage/engines/psycopg.py b/synapse/storage/engines/psycopg.py index 18a8fcc4908..d4ff2325324 100644 --- a/synapse/storage/engines/psycopg.py +++ b/synapse/storage/engines/psycopg.py @@ -33,6 +33,14 @@ class PsycopgEngine( psycopg.Connection[tuple], psycopg.Cursor[tuple], psycopg.IsolationLevel # type: ignore[type-var] ] ): + """ + The class that defines how Synapse will interact with Postgres using the psycopg + version 3+ module. + + NOTE: While psycopg does allow for return types other than tuples, for now that is + forced in order to match with existing psycopg2 behavior. + """ + def __init__(self, database_config: Mapping[str, Any]): super().__init__(psycopg, database_config) # type: ignore[arg-type] # psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) diff --git a/synapse/storage/engines/psycopg2.py b/synapse/storage/engines/psycopg2.py index cca1d193710..ab82547c875 100644 --- a/synapse/storage/engines/psycopg2.py +++ b/synapse/storage/engines/psycopg2.py @@ -26,6 +26,12 @@ class Psycopg2Engine( PostgresEngine[psycopg2.extensions.connection, psycopg2.extensions.cursor, int] ): + """ + The class that defines how Synapse will interact with Postgres using the psycopg + version 2.x module. + + """ + def __init__(self, database_config: Mapping[str, Any]): super().__init__(psycopg2, database_config) psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) From 123aeac77d74c014048a6aa24b1777d34ab3d628 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Mon, 2 Feb 2026 06:36:32 -0600 Subject: [PATCH 3/6] I forgot that the new driver is not wired up in the complement script for use with workers yet --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2a9d3e5239d..37ed042c1f2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -689,7 +689,7 @@ jobs: - arrangement: workers database: Postgres - - arrangement: workers + - arrangement: monolith database: Psycopg steps: From 4cd79240b199c77ec0b063bd8d1abf11cb47775b Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 3 Feb 2026 10:33:08 -0600 Subject: [PATCH 4/6] Use the 'c' extra instead of the 'pure python' version for C-compiled speedups --- poetry.lock | 15 ++++++++++++++- pyproject.toml | 7 +++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 130d7aa59e0..87a4f04b7f7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1845,6 +1845,7 @@ files = [ ] [package.dependencies] +psycopg-c = {version = "3.2.10", optional = true, markers = "implementation_name != \"pypy\" and extra == \"c\""} typing-extensions = {version = ">=4.6", markers = "python_version < \"3.13\""} tzdata = {version = "*", markers = "sys_platform == \"win32\""} @@ -1856,6 +1857,18 @@ docs = ["Sphinx (>=5.0)", "furo (==2022.6.21)", "sphinx-autobuild (>=2021.3.14)" pool = ["psycopg-pool"] test = ["anyio (>=4.0)", "mypy (>=1.14)", "pproxy (>=2.7)", "pytest (>=6.2.5)", "pytest-cov (>=3.0)", "pytest-randomly (>=3.5)"] +[[package]] +name = "psycopg-c" +version = "3.2.10" +description = "PostgreSQL database adapter for Python -- C optimisation distribution" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "(extra == \"psycopg\" or extra == \"all\") and implementation_name != \"pypy\"" +files = [ + {file = "psycopg_c-3.2.10.tar.gz", hash = "sha256:30183897f5fe7ff4375b7dfcec9d44dfe8a5e009080addc1626889324a9eb1ed"}, +] + [[package]] name = "psycopg2" version = "2.9.11" @@ -3596,4 +3609,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<4.0.0" -content-hash = "ef8637fce46e54b3c82d639c1e561fd214d87de5cd955343e11cf51a0ced6d3e" +content-hash = "74a22963bbdbcc68ec6e9d107828037add1f04cd320af11f6cc997ad5f401e37" diff --git a/pyproject.toml b/pyproject.toml index a00939fcbc9..c204b636c5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -124,7 +124,10 @@ postgres = [ "psycopg2cffi>=2.8;platform_python_implementation == 'PyPy'", "psycopg2cffi-compat==1.1;platform_python_implementation == 'PyPy'", ] -psycopg = ["psycopg>=3.1"] +# v3.1 was selected here as minimum to support debian bookworm. If in the future the +# `c` extra is chosen instead, the minimum will v3.2.8 for postgres 18's libpq5 support. +# See: https://github.com/psycopg/psycopg/pull/1084 +psycopg = ["psycopg[c]>=3.1"] saml2 = [ "pysaml2>=4.5.0", @@ -179,7 +182,7 @@ all = [ "psycopg2cffi>=2.8;platform_python_implementation == 'PyPy'", "psycopg2cffi-compat==1.1;platform_python_implementation == 'PyPy'", # experimental psycopg for postgres - "psycopg>=3.1", + "psycopg[c]>=3.1", # saml2 "pysaml2>=4.5.0", # oidc and jwt From a320d8aba7471aab28bc6ff9fb2fd207b20bcafa Mon Sep 17 00:00:00 2001 From: Jason Little Date: Thu, 19 Feb 2026 05:09:53 -0600 Subject: [PATCH 5/6] Update 'c' extra to 3.2.8 as a minimum. This avoids olddebian's packaged libpq5 having Postgres 18 support which breaks previous versions of psycopg when compiled --- poetry.lock | 2 +- pyproject.toml | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 87a4f04b7f7..02d5ca473c3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3609,4 +3609,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<4.0.0" -content-hash = "74a22963bbdbcc68ec6e9d107828037add1f04cd320af11f6cc997ad5f401e37" +content-hash = "6d25a3de1875cabf4fda06042a325bc64149bcf8528a34c8669357c75a4bd0d9" diff --git a/pyproject.toml b/pyproject.toml index c204b636c5d..766b10045f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -124,10 +124,11 @@ postgres = [ "psycopg2cffi>=2.8;platform_python_implementation == 'PyPy'", "psycopg2cffi-compat==1.1;platform_python_implementation == 'PyPy'", ] -# v3.1 was selected here as minimum to support debian bookworm. If in the future the -# `c` extra is chosen instead, the minimum will v3.2.8 for postgres 18's libpq5 support. +# v3.2.8 was selected here as minimum to support the libpq5 shipped with debian +# bookworm. This libpq5 includes support for postgres 18 and psycopg will fail to build +# below this selected version. # See: https://github.com/psycopg/psycopg/pull/1084 -psycopg = ["psycopg[c]>=3.1"] +psycopg = ["psycopg[c]>=3.2.8"] saml2 = [ "pysaml2>=4.5.0", @@ -182,7 +183,7 @@ all = [ "psycopg2cffi>=2.8;platform_python_implementation == 'PyPy'", "psycopg2cffi-compat==1.1;platform_python_implementation == 'PyPy'", # experimental psycopg for postgres - "psycopg[c]>=3.1", + "psycopg[c]>=3.2.8", # saml2 "pysaml2>=4.5.0", # oidc and jwt From d94d4c0fdef91f44d4234b20ae037c49ce52fe7c Mon Sep 17 00:00:00 2001 From: Jason Little Date: Thu, 19 Feb 2026 06:57:57 -0600 Subject: [PATCH 6/6] Allow complement to be ran with WORKERS=1 and still allow the psycopg driver(in POSTGRES=psycopg) --- .github/workflows/tests.yml | 3 +++ scripts-dev/complement.sh | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 37ed042c1f2..e6d5b64a3fd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -692,6 +692,9 @@ jobs: - arrangement: monolith database: Psycopg + - arrangement: workers + database: Psycopg + steps: - name: Checkout synapse codebase uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index 58086fb9936..d1753f7937a 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -271,8 +271,12 @@ main() { export PASS_SYNAPSE_WORKER_TYPES="$WORKER_TYPES" # Workers can only use Postgres as a database. - export PASS_SYNAPSE_COMPLEMENT_DATABASE=postgres - + # A handy pattern for lower-casing all letters in a variable, `${variable,,}` + if [[ "${POSTGRES,,}" = "psycopg" ]]; then + export PASS_SYNAPSE_COMPLEMENT_DATABASE=psycopg + else + export PASS_SYNAPSE_COMPLEMENT_DATABASE=postgres + fi # And provide some more configuration to complement. # It can take quite a while to spin up a worker-mode Synapse for the first