diff --git a/.github/workflows/complement_tests.yml b/.github/workflows/complement_tests.yml new file mode 100644 index 00000000000..bf8cdd3cfcf --- /dev/null +++ b/.github/workflows/complement_tests.yml @@ -0,0 +1,182 @@ +# Re-usable workflow (https://docs.github.com/en/actions/how-tos/reuse-automations/reuse-workflows) +name: Reusable Complement testing + +on: + workflow_call: + inputs: + use_latest_deps: + type: boolean + default: false + use_twisted_trunk: + type: boolean + default: false + +# Control the permissions granted to `GITHUB_TOKEN`. +permissions: + # `actions/checkout` reads the repository (also see + # https://github.com/actions/checkout/tree/de0fac2e4500dabe0009e67214ff5f5447ce83dd/#recommended-permissions) + contents: read + +env: + RUST_VERSION: 1.87.0 + +jobs: + complement: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + include: + - arrangement: monolith + database: SQLite + + - arrangement: monolith + database: Postgres + + - arrangement: workers + database: Postgres + + - arrangement: monolith + database: Psycopg + + - arrangement: workers + database: Psycopg + + steps: + - name: Checkout synapse codebase + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: synapse + + # Log Docker system info for debugging (compare with your local environment) and + # tracking GitHub runner changes over time (can easily compare a run from last + # week with the current one in question). + - run: docker system info + shell: bash + + - name: Install Rust + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + + # We use `poetry` in `complement.sh` + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 + with: + poetry-version: "2.2.1" + # Matches the `path` where we checkout Synapse above + working-directory: "synapse" + + - name: Prepare Complement's Prerequisites + run: synapse/.ci/scripts/setup_complement_prerequisites.sh + + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + cache-dependency-path: complement/go.sum + go-version-file: complement/go.mod + + # This step is specific to the 'Twisted trunk' test run: + - name: Patch dependencies + if: ${{ inputs.use_twisted_trunk }} + run: | + set -x + DEBIAN_FRONTEND=noninteractive sudo apt-get install -yqq python3 pipx + pipx install poetry==2.2.1 + + poetry remove -n twisted + poetry add -n --extras tls git+https://github.com/twisted/twisted.git#trunk + poetry lock + working-directory: synapse + + # Run the image sanity check test first as this is the first thing we want to know + # about (are we actually testing what we expect?) and we don't want to debug + # downstream failures (wild goose chase). + - name: Sanity check Complement image + id: run_sanity_check_complement_image_test + # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes + # are underpowered and don't like running tons of Synapse instances at once. + # -json: Output JSON format so that gotestfmt can parse it. + # + # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it + # later on for better formatting with gotestfmt. But we still want the command + # to output to the terminal as it runs so we can see what's happening in + # real-time. + run: | + set -o pipefail + COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json -run 'TestSynapseVersion/Synapse_version_matches_current_git_checkout' 2>&1 | tee /tmp/gotest-sanity-check-complement.log + shell: bash + env: + POSTGRES: ${{ (matrix.database == 'Postgres' || matrix.database == 'Psycopg') && matrix.database || '' }} + WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} + + - name: Formatted sanity check Complement test logs + # Always run this step if we attempted to run the Complement tests. + if: always() && steps.run_sanity_check_complement_image_test.outcome != 'skipped' + # We do not hide successful tests in `gotestfmt` here as the list of sanity + # check tests is so short. Feel free to change this when we get more tests. + # + # Note that the `-hide` argument is interpreted by `gotestfmt`. From it, + # it derives several values under `$settings` and passes them to our + # custom `.ci/complement_package.gotpl` template to render the output. + run: cat /tmp/gotest-sanity-check-complement.log | gotestfmt -hide "successful-downloads,empty-packages" + + - name: Run Complement Tests + id: run_complement_tests + # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes + # are underpowered and don't like running tons of Synapse instances at once. + # -json: Output JSON format so that gotestfmt can parse it. + # + # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it + # later on for better formatting with gotestfmt. But we still want the command + # to output to the terminal as it runs so we can see what's happening in + # real-time. + run: | + set -o pipefail + COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | tee /tmp/gotest-complement.log + shell: bash + env: + POSTGRES: ${{ (matrix.database == 'Postgres' || matrix.database == 'Psycopg') && matrix.database || '' }} + WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} + TEST_ONLY_IGNORE_POETRY_LOCKFILE: ${{ inputs.use_latest_deps && 1 || '' }} + TEST_ONLY_SKIP_DEP_HASH_VERIFICATION: ${{ inputs.use_twisted_trunk && 1 || '' }} + + - name: Formatted Complement test logs (only failing are shown) + # Always run this step if we attempted to run the Complement tests. + if: always() && steps.run_complement_tests.outcome != 'skipped' + # Hide successful tests in order to reduce the verbosity of the otherwise very large output. + # + # Note that the `-hide` argument is interpreted by `gotestfmt`. From it, + # it derives several values under `$settings` and passes them to our + # custom `.ci/complement_package.gotpl` template to render the output. + run: cat /tmp/gotest-complement.log | gotestfmt -hide "successful-downloads,successful-tests,empty-packages" + + - name: Run in-repo Complement Tests + id: run_in_repo_complement_tests + # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes + # are underpowered and don't like running tons of Synapse instances at once. + # -json: Output JSON format so that gotestfmt can parse it. + # + # tee /tmp/gotest-in-repo-complement.log: We tee the output to a file so that we can re-process it + # later on for better formatting with gotestfmt. But we still want the command + # to output to the terminal as it runs so we can see what's happening in + # real-time. + run: | + set -o pipefail + COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json 2>&1 | tee /tmp/gotest-in-repo-complement.log + shell: bash + env: + POSTGRES: ${{ (matrix.database == 'Postgres' || matrix.database == 'Psycopg') && matrix.database || '' }} + WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} + TEST_ONLY_IGNORE_POETRY_LOCKFILE: ${{ inputs.use_latest_deps && 1 || '' }} + TEST_ONLY_SKIP_DEP_HASH_VERIFICATION: ${{ inputs.use_twisted_trunk && 1 || '' }} + + - name: Formatted in-repo Complement test logs (only failing are shown) + # Always run this step if we attempted to run the Complement tests. + if: always() && steps.run_in_repo_complement_tests.outcome != 'skipped' + # Hide successful tests in order to reduce the verbosity of the otherwise very large output. + # + # Note that the `-hide` argument is interpreted by `gotestfmt`. From it, + # it derives several values under `$settings` and passes them to our + # custom `.ci/complement_package.gotpl` template to render the output. + run: cat /tmp/gotest-in-repo-complement.log | gotestfmt -hide "successful-downloads,successful-tests,empty-packages" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 051c66ebba5..aaf3064c26d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -41,13 +41,13 @@ jobs: echo "SYNAPSE_VERSION=$(grep "^version" pyproject.toml | sed -E 's/version\s*=\s*["]([^"]*)["]/\1/')" >> $GITHUB_ENV - name: Log in to DockerHub - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -79,7 +79,7 @@ jobs: services/backend-repositories/secret/data/oci.element.io password | OCI_PASSWORD ; - name: Login to Element OCI Registry - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: oci-push.vpn.infra.element.io username: ${{ steps.import-secrets.outputs.OCI_USERNAME }} @@ -136,14 +136,14 @@ jobs: merge-multiple: true - name: Log in to DockerHub - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 if: ${{ startsWith(matrix.repository, 'docker.io') }} with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 if: ${{ startsWith(matrix.repository, 'ghcr.io') }} with: registry: ghcr.io @@ -176,7 +176,7 @@ jobs: services/backend-repositories/secret/data/oci.element.io password | OCI_PASSWORD ; - name: Login to Element OCI Registry - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: oci-push.vpn.infra.element.io username: ${{ steps.import-secrets.outputs.OCI_USERNAME }} @@ -186,7 +186,7 @@ jobs: uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Install Cosign - uses: sigstore/cosign-installer@faadad0cce49287aee09b3a48701e75088a2c6ad # v4.0.0 + uses: sigstore/cosign-installer@cad07c2e89fa2edd6e2d7bab4c1aa38e53f76003 # v4.1.1 - name: Calculate docker image tag uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 746223ef4af..d03a9295071 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -181,84 +181,11 @@ jobs: /logs/**/*.log* complement: + uses: ./.github/workflows/complement_tests.yml needs: check_repo if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'" - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - include: - - arrangement: monolith - database: SQLite - - - arrangement: monolith - database: Postgres - - - arrangement: workers - database: Postgres - - steps: - - name: Check out synapse codebase - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: synapse - - - name: Prepare Complement's Prerequisites - run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 - with: - cache-dependency-path: complement/go.sum - go-version-file: complement/go.mod - - - name: Run Complement Tests - id: run_complement_tests - # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes - # are underpowered and don't like running tons of Synapse instances at once. - # -json: Output JSON format so that gotestfmt can parse it. - # - # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it - # later on for better formatting with gotestfmt. But we still want the command - # to output to the terminal as it runs so we can see what's happening in - # real-time. - run: | - set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | tee /tmp/gotest-complement.log - shell: bash - env: - POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} - WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} - TEST_ONLY_IGNORE_POETRY_LOCKFILE: 1 - - - name: Formatted Complement test logs - # Always run this step if we attempted to run the Complement tests. - if: always() && steps.run_complement_tests.outcome != 'skipped' - run: cat /tmp/gotest-complement.log | gotestfmt -hide "successful-downloads,empty-packages" - - - name: Run in-repo Complement Tests - id: run_in_repo_complement_tests - # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes - # are underpowered and don't like running tons of Synapse instances at once. - # -json: Output JSON format so that gotestfmt can parse it. - # - # tee /tmp/gotest-in-repo-complement.log: We tee the output to a file so that we can re-process it - # later on for better formatting with gotestfmt. But we still want the command - # to output to the terminal as it runs so we can see what's happening in - # real-time. - run: | - set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json 2>&1 | tee /tmp/gotest-in-repo-complement.log - shell: bash - env: - POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} - WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} - TEST_ONLY_IGNORE_POETRY_LOCKFILE: 1 - - - name: Formatted in-repo Complement test logs - # Always run this step if we attempted to run the Complement tests. - if: always() && steps.run_in_repo_complement_tests.outcome != 'skipped' - run: cat /tmp/gotest-in-repo-complement.log | gotestfmt -hide "successful-downloads,empty-packages" + with: + use_latest_deps: true # Open an issue if the build fails, so we know about it. # Only do this if we're not experimenting with this action in a PR. diff --git a/.github/workflows/push_complement_image.yml b/.github/workflows/push_complement_image.yml index 3931dbebb32..da45bb2c035 100644 --- a/.github/workflows/push_complement_image.yml +++ b/.github/workflows/push_complement_image.yml @@ -52,7 +52,7 @@ jobs: with: poetry-version: "2.2.1" - name: Login to registry - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index c313a64250f..8c625be96e2 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -64,7 +64,7 @@ jobs: uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Set up docker layer caching - uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 96306c3d06d..e9e49b616b2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -174,7 +174,7 @@ jobs: # Cribbed from # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17 - name: Restore/persist mypy's cache - uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: | .mypy_cache @@ -670,151 +670,11 @@ jobs: schema_diff complement: + uses: ./.github/workflows/complement_tests.yml if: "${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true' }}" needs: - linting-done - changes - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - include: - - arrangement: monolith - database: SQLite - - - arrangement: monolith - database: Postgres - - - arrangement: workers - database: Postgres - - - arrangement: monolith - database: Psycopg - - - arrangement: workers - database: Psycopg - - steps: - - name: Checkout synapse codebase - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: synapse - - # Log Docker system info for debugging (compare with your local environment) and - # tracking GitHub runner changes over time (can easily compare a run from last - # week with the current one in question). - - run: docker system info - shell: bash - - - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master - with: - toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - - # We use `poetry` in `complement.sh` - - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 - with: - poetry-version: "2.2.1" - # Matches the `path` where we checkout Synapse above - working-directory: "synapse" - - - name: Prepare Complement's Prerequisites - run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 - with: - cache-dependency-path: complement/go.sum - go-version-file: complement/go.mod - - # Run the image sanity check test first as this is the first thing we want to know - # about (are we actually testing what we expect?) and we don't want to debug - # downstream failures (wild goose chase). - - name: Sanity check Complement image - id: run_sanity_check_complement_image_test - # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes - # are underpowered and don't like running tons of Synapse instances at once. - # -json: Output JSON format so that gotestfmt can parse it. - # - # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it - # later on for better formatting with gotestfmt. But we still want the command - # to output to the terminal as it runs so we can see what's happening in - # real-time. - run: | - set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json -run 'TestSynapseVersion/Synapse_version_matches_current_git_checkout' 2>&1 | tee /tmp/gotest-sanity-check-complement.log - shell: bash - env: - POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} - WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} - - - name: Formatted sanity check Complement test logs - # Always run this step if we attempted to run the Complement tests. - if: always() && steps.run_sanity_check_complement_image_test.outcome != 'skipped' - # We do not hide successful tests in `gotestfmt` here as the list of sanity - # check tests is so short. Feel free to change this when we get more tests. - # - # Note that the `-hide` argument is interpreted by `gotestfmt`. From it, - # it derives several values under `$settings` and passes them to our - # custom `.ci/complement_package.gotpl` template to render the output. - run: cat /tmp/gotest-sanity-check-complement.log | gotestfmt -hide "successful-downloads,empty-packages" - - - name: Run Complement Tests - id: run_complement_tests - # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes - # are underpowered and don't like running tons of Synapse instances at once. - # -json: Output JSON format so that gotestfmt can parse it. - # - # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it - # later on for better formatting with gotestfmt. But we still want the command - # to output to the terminal as it runs so we can see what's happening in - # real-time. - run: | - set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | tee /tmp/gotest-complement.log - shell: bash - env: - POSTGRES: ${{ (matrix.database == 'Postgres' || matrix.database == 'Psycopg') && matrix.database || '' }} - WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} - - - name: Formatted Complement test logs (only failing are shown) - # Always run this step if we attempted to run the Complement tests. - if: always() && steps.run_complement_tests.outcome != 'skipped' - # Hide successful tests in order to reduce the verbosity of the otherwise very large output. - # - # Note that the `-hide` argument is interpreted by `gotestfmt`. From it, - # it derives several values under `$settings` and passes them to our - # custom `.ci/complement_package.gotpl` template to render the output. - run: cat /tmp/gotest-complement.log | gotestfmt -hide "successful-downloads,successful-tests,empty-packages" - - - name: Run in-repo Complement Tests - id: run_in_repo_complement_tests - # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes - # are underpowered and don't like running tons of Synapse instances at once. - # -json: Output JSON format so that gotestfmt can parse it. - # - # tee /tmp/gotest-in-repo-complement.log: We tee the output to a file so that we can re-process it - # later on for better formatting with gotestfmt. But we still want the command - # to output to the terminal as it runs so we can see what's happening in - # real-time. - run: | - set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json 2>&1 | tee /tmp/gotest-in-repo-complement.log - shell: bash - env: - POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} - WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} - - - name: Formatted in-repo Complement test logs (only failing are shown) - # Always run this step if we attempted to run the Complement tests. - if: always() && steps.run_in_repo_complement_tests.outcome != 'skipped' - # Hide successful tests in order to reduce the verbosity of the otherwise very large output. - # - # Note that the `-hide` argument is interpreted by `gotestfmt`. From it, - # it derives several values under `$settings` and passes them to our - # custom `.ci/complement_package.gotpl` template to render the output. - run: cat /tmp/gotest-in-repo-complement.log | gotestfmt -hide "successful-downloads,successful-tests,empty-packages" cargo-test: if: ${{ needs.changes.outputs.rust == 'true' }} diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index be73e5283b0..d9d61152fbd 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -154,96 +154,11 @@ jobs: /logs/**/*.log* complement: + uses: ./.github/workflows/complement_tests.yml needs: check_repo if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'" - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - include: - - arrangement: monolith - database: SQLite - - - arrangement: monolith - database: Postgres - - - arrangement: workers - database: Postgres - - steps: - - name: Run actions/checkout@v4 for synapse - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: synapse - - - name: Prepare Complement's Prerequisites - run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 - with: - cache-dependency-path: complement/go.sum - go-version-file: complement/go.mod - - # This step is specific to the 'Twisted trunk' test run: - - name: Patch dependencies - run: | - set -x - DEBIAN_FRONTEND=noninteractive sudo apt-get install -yqq python3 pipx - pipx install poetry==2.1.1 - - poetry remove -n twisted - poetry add -n --extras tls git+https://github.com/twisted/twisted.git#trunk - poetry lock - working-directory: synapse - - - name: Run Complement Tests - id: run_complement_tests - # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes - # are underpowered and don't like running tons of Synapse instances at once. - # -json: Output JSON format so that gotestfmt can parse it. - # - # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it - # later on for better formatting with gotestfmt. But we still want the command - # to output to the terminal as it runs so we can see what's happening in - # real-time. - run: | - set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | tee /tmp/gotest-complement.log - shell: bash - env: - POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} - WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} - TEST_ONLY_SKIP_DEP_HASH_VERIFICATION: 1 - - - name: Formatted Complement test logs - # Always run this step if we attempted to run the Complement tests. - if: always() && steps.run_complement_tests.outcome != 'skipped' - run: cat /tmp/gotest-complement.log | gotestfmt -hide "successful-downloads,empty-packages" - - - name: Run in-repo Complement Tests - id: run_in_repo_complement_tests - # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes - # are underpowered and don't like running tons of Synapse instances at once. - # -json: Output JSON format so that gotestfmt can parse it. - # - # tee /tmp/gotest-in-repo-complement.log: We tee the output to a file so that we can re-process it - # later on for better formatting with gotestfmt. But we still want the command - # to output to the terminal as it runs so we can see what's happening in - # real-time. - run: | - set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json 2>&1 | tee /tmp/gotest-in-repo-complement.log - shell: bash - env: - POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} - WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} - TEST_ONLY_SKIP_DEP_HASH_VERIFICATION: 1 - - - name: Formatted in-repo Complement test logs - # Always run this step if we attempted to run the Complement tests. - if: always() && steps.run_in_repo_complement_tests.outcome != 'skipped' - run: cat /tmp/gotest-in-repo-complement.log | gotestfmt -hide "successful-downloads,empty-packages" + with: + use_twisted_trunk: true # open an issue if the build fails, so we know about it. open-issue: diff --git a/CHANGES.md b/CHANGES.md index bcf66142f9d..384042db2fa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,59 @@ +# Synapse 1.152.0rc1 (2026-04-22) + +For deployments using workers, please note that this version introduces a new `quarantined_media_changes` stream writer, which may require configuration changes. +Please see the [the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/develop/docs/upgrade.md#upgrading-to-v11520) for details. + +Without configuring this new stream writer, only the main process will be able to handle the `/media/quarantine` admin API endpoints for quarantining media. + + +## Features + +- Add a ["Listing quarantined media changes" Admin API](https://element-hq.github.io/synapse/latest/admin_api/media_admin_api.html#listing-quarantined-media-changes) for retrieving a paginated record of when media became (un)quarantined. ([\#19558](https://github.com/element-hq/synapse/issues/19558), [\#19677](https://github.com/element-hq/synapse/issues/19677)) +- Advertise [MSC4445](https://github.com/matrix-org/matrix-spec-proposals/pull/4445) sync timeline order in `unstable_features`. ([\#19642](https://github.com/element-hq/synapse/issues/19642)) +- Report the Rust compiler version used in the Prometheus metrics. Contributed by Noah Markert. ([\#19643](https://github.com/element-hq/synapse/issues/19643)) +- Passthrough 'article' and 'profile' OpenGraph metadata on URL preview requests. ([\#19659](https://github.com/element-hq/synapse/issues/19659)) +- Add a way to re-sign local events with a new signing key. ([\#19668](https://github.com/element-hq/synapse/issues/19668)) +- Support [MSC4450: Identity Provider selection for User-Interactive Authentication with Legacy Single Sign-On](https://github.com/matrix-org/matrix-spec-proposals/pull/4450). ([\#19693](https://github.com/element-hq/synapse/issues/19693)) +- Add a ["Listing quarantined media changes" Admin API](https://element-hq.github.io/synapse/latest/admin_api/media_admin_api.html#listing-quarantined-media-changes) for retrieving a paginated record of when media became (un)quarantined. ([\#19694](https://github.com/element-hq/synapse/issues/19694)) +- Add experimental support for [MSC4242](https://github.com/matrix-org/matrix-spec-proposals/pull/4242): State DAGs. Excludes federation support. ([\#19424](https://github.com/element-hq/synapse/issues/19424)) +- Adds [Admin API](https://element-hq.github.io/synapse/latest/usage/administration/admin_api/index.html) endpoints to + list, fetch and delete user reports. ([\#19657](https://github.com/element-hq/synapse/issues/19657)) +- Reduce database disk space usage by pruning old rows from `device_lists_changes_in_room`. ([\#19473](https://github.com/element-hq/synapse/issues/19473), [\#19709](https://github.com/element-hq/synapse/issues/19709)) + +## Bugfixes + +- Reject `device_keys: null` in the request to [`POST /_matrix/client/v3/keys/upload`](https://spec.matrix.org/v1.16/client-server-api/#post_matrixclientv3keysupload), as per the spec. This was temporarily allowed as a workaround for misbehaving clients. ([\#19637](https://github.com/element-hq/synapse/issues/19637)) +- Fix database migrations failing on platforms where SQLite is configured with `SQLITE_DBCONFIG_DEFENSIVE` by default, such as macOS. ([\#19690](https://github.com/element-hq/synapse/issues/19690)) +- Fix a bug introduced in v1.145 where a non-admin could bypass admin checks for downloading remote quarantined media. This relied on the media already being previously present on the homeserver. ([\#19639](https://github.com/element-hq/synapse/issues/19639)) + +## Improved Documentation + +- Include a workaround for running the unit tests with SQLite under recent versions of MacOS. ([\#19615](https://github.com/element-hq/synapse/issues/19615)) +- Fix Docker image link typo in worker docs. ([\#19645](https://github.com/element-hq/synapse/issues/19645)) +- Update developer stream docs for creating a new stream to point out `_setup_sequence(...)` in `portdb`. ([\#19675](https://github.com/element-hq/synapse/issues/19675)) +- Update the developer stream docs for creating a new stream to highlight places that require documentation updates. ([\#19696](https://github.com/element-hq/synapse/issues/19696)) + +## Internal Changes + +- Update CI to use re-usable Complement GitHub CI workflow. ([\#19533](https://github.com/element-hq/synapse/issues/19533)) +- Fix docstring for `limit` argument in `_maybe_backfill_inner(...)`. ([\#19630](https://github.com/element-hq/synapse/issues/19630)) +- Document context for why increase timeout for policy server requests. ([\#19633](https://github.com/element-hq/synapse/issues/19633)) +- Run lint script to format Complement tests introduced in [#19509](https://github.com/element-hq/synapse/pull/19509). ([\#19636](https://github.com/element-hq/synapse/issues/19636)) +- Small simplifications to the events class. ([\#19680](https://github.com/element-hq/synapse/issues/19680), [\#19712](https://github.com/element-hq/synapse/issues/19712)) +- Introduce `spam_checker_spammy` internal event metadata. ([\#19453](https://github.com/element-hq/synapse/issues/19453)) +- Add a `FilteredEvent` class that saves us copying events. ([\#19640](https://github.com/element-hq/synapse/issues/19640)) +- Convert `EventInternalMetadata` to use `Arc>`. ([\#19669](https://github.com/element-hq/synapse/issues/19669)) + + +# Synapse 1.151.0 (2026-04-07) + +## Bugfixes + +- Fix `KNOWN_ROOM_VERSIONS.__contains__` raising `TypeError` for non-string keys, which could cause `/sync` to fail for rooms with a `NULL` room version in the database. Bug introduced in [#19589](https://github.com/element-hq/synapse/pull/19589) as part of v1.151.0rc1. ([\#19649](https://github.com/element-hq/synapse/issues/19649)) + + + + # Synapse 1.151.0rc1 (2026-03-31) ## Features diff --git a/Cargo.lock b/Cargo.lock index e1747182e3b..c7f5e0ddbf4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1125,9 +1125,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.10" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "ring", "rustls-pki-types", diff --git a/changelog.d/18475.feature b/changelog.d/18475.feature new file mode 100644 index 00000000000..06c13db43e5 --- /dev/null +++ b/changelog.d/18475.feature @@ -0,0 +1 @@ +Make ACLs apply to EDUs per [MSC4163](https://github.com/matrix-org/matrix-spec-proposals/pull/4163). diff --git a/changelog.d/19398.bugfix b/changelog.d/19398.bugfix new file mode 100644 index 00000000000..07679b31aec --- /dev/null +++ b/changelog.d/19398.bugfix @@ -0,0 +1 @@ +Allow user requested erasure to succeed even if Synapse has disabled profile changes. Contributed by Famedly. diff --git a/changelog.d/19615.doc b/changelog.d/19615.doc deleted file mode 100644 index 9e28ac83127..00000000000 --- a/changelog.d/19615.doc +++ /dev/null @@ -1 +0,0 @@ -Include a workaround for running the unit tests with SQLite under recent versions of MacOS. diff --git a/changelog.d/19633.misc b/changelog.d/19633.misc deleted file mode 100644 index 37efd5309fb..00000000000 --- a/changelog.d/19633.misc +++ /dev/null @@ -1 +0,0 @@ -Document context for why increase timeout for policy server requests. diff --git a/changelog.d/19636.misc b/changelog.d/19636.misc deleted file mode 100644 index 9ea0f046038..00000000000 --- a/changelog.d/19636.misc +++ /dev/null @@ -1 +0,0 @@ -Run lint script to format Complement tests introduced in [#19509](https://github.com/element-hq/synapse/pull/19509). diff --git a/changelog.d/19643.feature b/changelog.d/19643.feature deleted file mode 100644 index 0851623fbfb..00000000000 --- a/changelog.d/19643.feature +++ /dev/null @@ -1 +0,0 @@ -Report the Rust compiler version used in the Prometheus metrics. Contributed by Noah Markert. \ No newline at end of file diff --git a/changelog.d/19711.doc b/changelog.d/19711.doc new file mode 100644 index 00000000000..d00ee6a908e --- /dev/null +++ b/changelog.d/19711.doc @@ -0,0 +1 @@ +Add warning about known problems when configuring `use_frozen_dicts`. diff --git a/changelog.d/19714.bugfix b/changelog.d/19714.bugfix new file mode 100644 index 00000000000..6aba7b21a61 --- /dev/null +++ b/changelog.d/19714.bugfix @@ -0,0 +1 @@ +Have SSS return a new response immediately if a room subscription have changed and produced a new response. diff --git a/changelog.d/19720.feature b/changelog.d/19720.feature new file mode 100644 index 00000000000..97a1d35de5f --- /dev/null +++ b/changelog.d/19720.feature @@ -0,0 +1 @@ +Stabilize MSC3266, removing the experimental config flag `msc3266_enabled`. Add support for stable room summary endpoints. Contributed by @dasha-uwu. diff --git a/changelog.d/19722.feature b/changelog.d/19722.feature new file mode 100644 index 00000000000..30104b7e74d --- /dev/null +++ b/changelog.d/19722.feature @@ -0,0 +1 @@ +Partial [MSC4311](https://github.com/matrix-org/matrix-spec-proposals/pull/4311) implementation: `m.room.create` is now a required part of stripped `invite_state`/`knock_state` . Contributed by @FrenchGithubUser @Famedly. diff --git a/changelog.d/19727.bugfix b/changelog.d/19727.bugfix new file mode 100644 index 00000000000..a535bd6aa4b --- /dev/null +++ b/changelog.d/19727.bugfix @@ -0,0 +1 @@ +Fix a bug where when upgrading a room to v12 the power level event in the old room got mutated to remove the user upgrading the room's power. diff --git a/complement/go.mod b/complement/go.mod index 716aabdf321..d1ccf6b4962 100644 --- a/complement/go.mod +++ b/complement/go.mod @@ -7,6 +7,7 @@ toolchain go1.24.4 require ( github.com/matrix-org/complement v0.0.0-20251120181401-44111a2a8a9d github.com/matrix-org/gomatrixserverlib v0.0.0-20250813150445-9f5070a65744 + github.com/tidwall/gjson v1.18.0 ) require ( @@ -22,6 +23,7 @@ require ( require ( github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -40,16 +42,15 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect - go.opentelemetry.io/otel v1.36.0 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 // indirect - go.opentelemetry.io/otel/metric v1.36.0 // indirect - go.opentelemetry.io/otel/trace v1.36.0 // indirect + go.opentelemetry.io/otel/metric v1.41.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect go.opentelemetry.io/proto/otlp v1.7.0 // indirect golang.org/x/crypto v0.45.0 // indirect golang.org/x/sys v0.38.0 // indirect diff --git a/complement/go.sum b/complement/go.sum index 79a35aa14c6..f82487882b3 100644 --- a/complement/go.sum +++ b/complement/go.sum @@ -4,6 +4,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= @@ -81,8 +83,8 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -95,24 +97,24 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= -go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9HsPl42NIXFIFSUSSs0fiqra0= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0/go.mod h1:90PoxvaEB5n6AOdZvi+yWJQoE95U8Dhhw2bSyRqnTD0= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.34.0 h1:BEj3SPM81McUZHYjRS5pEgNgnmzGJ5tRpU5krWnV8Bs= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.34.0/go.mod h1:9cKLGBDzI/F3NoHLQGm4ZrYdIHsvGt6ej6hUowxY0J4= -go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= -go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= -go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= -go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/contrib/grafana/synapse.json b/contrib/grafana/synapse.json index ceacc10369f..a67d1aba0b7 100644 --- a/contrib/grafana/synapse.json +++ b/contrib/grafana/synapse.json @@ -6809,6 +6809,155 @@ ], "title": "Stale extremity dropping", "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "For a given percentage P, the number X where P% of events were persisted to rooms with X state DAG forward extremities or fewer.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 50 + }, + "id": 181, + "options": { + "alertThreshold": true + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.5, rate(synapse_storage_msc4242_state_dag_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "50%", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.75, rate(synapse_storage_msc4242_state_dag_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "75%", + "refId": "B" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.90, rate(synapse_storage_msc4242_state_dag_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "90%", + "refId": "C" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, rate(synapse_storage_msc4242_state_dag_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "99%", + "refId": "D" + } + ], + "title": "Events persisted, by number of state DAG forward extremities in room (quantiles)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Colour reflects the number of events persisted to rooms with the given number of state DAG forward extremities, or fewer.", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 50 + }, + "id": 127, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 1, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#5794F2", + "min": 0, + "mode": "opacity", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "short" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(synapse_storage_msc4242_state_dag_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0)", + "format": "heatmap", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Events persisted, by number of state DAG forward extremities in room (heatmap)", + "type": "heatmap" } ], "title": "Extremities", @@ -7711,4 +7860,4 @@ "uid": "000000012", "version": 1, "weekStart": "" -} +} \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index 36fa0e51402..e9a8e200e1e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +matrix-synapse-py3 (1.152.0~rc1) stable; urgency=medium + + * New Synapse release 1.152.0rc1. + + -- Synapse Packaging team Wed, 22 Apr 2026 12:03:58 +0100 + +matrix-synapse-py3 (1.151.0) stable; urgency=medium + + * New synapse release 1.151.0. + + -- Synapse Packaging team Tue, 07 Apr 2026 12:15:10 +0000 + matrix-synapse-py3 (1.151.0~rc1) stable; urgency=medium * New synapse release 1.151.0rc1. diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index 9fd7fc954a3..e829292aca9 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -123,8 +123,6 @@ experimental_features: msc3874_enabled: true # no UIA for x-signing upload for the first time msc3967_enabled: true - # Expose a room summary for public rooms - msc3266_enabled: true # Send to-device messages to application services msc2409_to_device_messages_enabled: true # Allow application services to masquerade devices diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 1b8d4f9989e..26c8556eff4 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -119,7 +119,7 @@ }, "media_repository": { "app": "synapse.app.generic_worker", - "listener_resources": ["media", "client"], + "listener_resources": ["media", "client", "replication"], "endpoint_patterns": [ "^/_matrix/media/", "^/_synapse/admin/v1/purge_media_cache$", diff --git a/docs/admin_api/media_admin_api.md b/docs/admin_api/media_admin_api.md index 6b96eb33564..750be85bbe9 100644 --- a/docs/admin_api/media_admin_api.md +++ b/docs/admin_api/media_admin_api.md @@ -247,6 +247,42 @@ Response: {} ``` +## Listing quarantined media changes + +When media is quarantined or unquarantined, a change record is created in the +database. This API returns those change records in the order they were created. + +**Note**: This API should be considered *best-effort* and expected to have missing or +duplicate records. Currently, this only captures any media explicitly (un)quarantined by +the media quarantine admin API, and the other cases are tracked by +https://github.com/element-hq/synapse/issues/19672. Historical media uploaded before +Synapse 1.152.0 is backfilled in a background update on a best-effort basis. + +Each page has a maximum of 100 records. The first page has the oldest records, +paginating forwards with each `next_batch` value. + +Request: + +``` +GET /_synapse/admin/v1/media/quarantine_changes?from=2 +``` + +Where `from` is the `next_batch` value from a previous request. It is optional +and defaults to the first page (the value `0`). + +Response: + +```json +{ + "next_batch": 4, + "changes": [ + { "origin": "example.org", "media_id": "abcdefg12345...", "quarantined": true }, + { "origin": "example.org", "media_id": "abcdefg12345...", "quarantined": false }, + { "origin": "another.example.org", "media_id": "abcdefg12345...", "quarantined": true } + ] +} +``` + # Delete local media This API deletes the *local* media from the disk of your own server. This includes any local thumbnails and copies of media downloaded from diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md index 7e0dd4059d6..ab506b5f05a 100644 --- a/docs/development/contributing_guide.md +++ b/docs/development/contributing_guide.md @@ -309,21 +309,6 @@ works without further arguments). Your Postgres account needs to be able to create databases; see the postgres docs for [`ALTER ROLE`](https://www.postgresql.org/docs/current/sql-alterrole.html). -### Running the tests on MacOS - -To run the unit tests with SQLite on MacOS you will need to swap out the default MacOS -python interpreter with the homebrew version. This is due to the default using a -locked down version of Python and sqlite3. After installing python from homebrew: - -``` - poetry env remove --all - poetry env use /opt/homebrew/bin/python3.13 - poetry install - poetry run trial tests -``` - -This example uses python 3.13, but choose whichever version you want. - ## Run the integration tests ([Sytest](https://github.com/matrix-org/sytest)). The integration tests are a more comprehensive suite of tests. They diff --git a/docs/development/synapse_architecture/streams.md b/docs/development/synapse_architecture/streams.md index 364f90af75d..e7ab79091e9 100644 --- a/docs/development/synapse_architecture/streams.md +++ b/docs/development/synapse_architecture/streams.md @@ -158,6 +158,8 @@ These rough notes and links may help you to create a new stream and add all the necessary registration and event handling. **Create your stream:** +- Create a Postgres-specific database delta file to [add a new `SEQUENCE`](https://github.com/element-hq/synapse/blob/35b55e962aa0bed3b2da5a3c12e3783ddf7604ca/synapse/storage/schema/main/delta/93/01_sticky_events_seq.sql.postgres#L14-L18) (this will be referenced by the `MultiWriterIdGenerator` below). +- Update `synapse/_scripts/synapse_port_db.py` so it knows about your new `SEQUENCE`: [add a new `_setup_sequence(...)`](https://github.com/element-hq/synapse/blob/35b55e962aa0bed3b2da5a3c12e3783ddf7604ca/synapse/_scripts/synapse_port_db.py#L883C24-L888) - [create a stream class and stream row class](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/replication/tcp/streams/_base.py#L728) - will need an [ID generator](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/storage/databases/main/thread_subscriptions.py#L75) - may need [writer configuration](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/config/workers.py#L177), if there isn't already an obvious source of configuration for which workers should be designated as writers to your new stream. @@ -177,6 +179,13 @@ necessary registration and event handling. - don't forget the super call - add local-only [invalidations to your writer transactions](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/storage/databases/main/thread_subscriptions.py#L201) +**Update docs:** +- Update the [*Stream + writers*](https://github.com/element-hq/synapse/blob/develop/docs/workers.md#stream-writers) + section in the worker docs with a new section for the stream +- If this stream can only be handled by specific workers, add a new section to the + [upgrade notes](https://github.com/element-hq/synapse/blob/develop/docs/upgrade.md). + **For streams to be used in sync:** - add a new field to [`StreamToken`](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/types/__init__.py#L1003) - add a new [`StreamKeyType`](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/types/__init__.py#L999) diff --git a/docs/upgrade.md b/docs/upgrade.md index aeae82c114a..44ab342d5a2 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -117,6 +117,19 @@ each upgrade are complete before moving on to the next upgrade, to avoid stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). +# Upgrading to v1.152.0 + +## Workers which quarantine media must be stream writers + +A new [`quarantined_media_changes` stream writer](./workers.md#the-quarantined_media_changes-stream) is +introduced. Existing deployments which route the `/quarantine_media` endpoints to a +worker (instead of the main process) *must* also add those workers to the +`quarantined_media_changes` stream writer list. Quarantining media will not work without +this. + +If your deployment does not use workers, or instead uses the main process for +quarantining media, you do not need to make any changes to your configuration. + # Upgrading to v1.150.0 ## Removal of the `systemd` pip extra diff --git a/docs/usage/administration/admin_api/background_updates.md b/docs/usage/administration/admin_api/background_updates.md index 7b75ee55870..7a78b8964b4 100644 --- a/docs/usage/administration/admin_api/background_updates.md +++ b/docs/usage/administration/admin_api/background_updates.md @@ -107,3 +107,6 @@ The following JSON body parameters are available: - `job_name` - A string which job to run. Valid values are: - `populate_stats_process_rooms` - Recalculate the stats for all rooms. - `regenerate_directory` - Recalculate the [user directory](../../../user_directory.md) if it is stale or out of sync. + - `event_resign` - Re-sign all locally-sent events with the current signing key. This is useful after rotating the server's signing key to ensure all historical events are signed with the new key. Optional additional parameters: + - `old_key` - Only re-sign events whose signature verifies against this key. Format: `"ed25519:key_id base64_public_key"` (e.g. `"ed25519:my_old_key XGX0JRS2Af3be3knz2fBiRbApjm2Dh61gXDJA8kcJNI"`). + - `before_ts` - Only re-sign events with a `received_ts` less than this value (milliseconds since the epoch). diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index e468fda53ad..b14795eb562 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -194,7 +194,11 @@ user_agent_suffix: ' (I''m a teapot; Linux x86_64)' --- ### `use_frozen_dicts` -*(boolean)* Determines whether we should freeze the internal dict object in `FrozenEvent`. Freezing prevents bugs where we accidentally share e.g. signature dicts. However, freezing a dict is expensive. Defaults to `false`. +*(boolean)* Determines whether we should freeze the internal dict object in `FrozenEvent`. Freezing prevents bugs where we accidentally share e.g. signature dicts. However, freezing a dict is expensive. + +> ⚠️ **Warning** – This option is known to introduce a new class of [comparison bugs](https://github.com/element-hq/synapse/issues/18117) in Synapse. + +Defaults to `false`. Example configuration: ```yaml diff --git a/docs/workers.md b/docs/workers.md index c2aef33e168..8d3aad19c66 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -60,10 +60,10 @@ virtualenv, these can be installed with: pip install "matrix-synapse[redis]" ``` -Note that these dependencies are included when synapse is installed with `pip -install matrix-synapse[all]`. They are also included in the debian packages from -`packages.matrix.org` and in the docker images at -https://hub.docker.com/r/ectorim/synapse/. +Note that these dependencies are included when Synapse is installed with `pip install +matrix-synapse[all]`. They are also included in the [Debian +packages](setup/installation.md#debianubuntu) and in the [Docker +images](setup/installation.md#docker-images-and-ansible-playbooks). To make effective use of the workers, you will need to configure an HTTP reverse-proxy such as nginx or haproxy, which will direct incoming requests to @@ -576,6 +576,14 @@ configured as stream writer for the `device_lists` stream: ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/device_signing/upload$ ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/signatures/upload$ +##### The `quarantined_media_changes` stream + +The `quarantined_media_changes` stream supports multiple writers. The following endpoints +can be handled by any worker, but should be routed directly to one of the workers +configured as stream writer for the `quarantined_media_changes` stream: + + ^/_synapse/admin/v1/quarantine_media/.*$ + #### Restrict outbound federation traffic to a specific set of workers The diff --git a/poetry.lock b/poetry.lock index 1b7986e0808..8701b305e7b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -14,27 +14,27 @@ files = [ [[package]] name = "attrs" -version = "25.4.0" +version = "26.1.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373"}, - {file = "attrs-25.4.0.tar.gz", hash = "sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11"}, + {file = "attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309"}, + {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, ] [[package]] name = "authlib" -version = "1.6.9" +version = "1.6.11" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = true python-versions = ">=3.9" groups = ["main"] markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" files = [ - {file = "authlib-1.6.9-py2.py3-none-any.whl", hash = "sha256:f08b4c14e08f0861dc18a32357b33fbcfd2ea86cfe3fe149484b4d764c4a0ac3"}, - {file = "authlib-1.6.9.tar.gz", hash = "sha256:d8f2421e7e5980cc1ddb4e32d3f5fa659cfaf60d8eaf3281ebed192e4ab74f04"}, + {file = "authlib-1.6.11-py2.py3-none-any.whl", hash = "sha256:c8687a9a26451c51a34a06fa17bb97cb15bba46a6a626755e2d7f50da8bff3e3"}, + {file = "authlib-1.6.11.tar.gz", hash = "sha256:64db35b9b01aeccb4715a6c9a6613a06f2bd7be2ab9d2eb89edd1dfc7580a38f"}, ] [package.dependencies] @@ -453,61 +453,61 @@ files = [ [[package]] name = "cryptography" -version = "46.0.6" +version = "46.0.7" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = "!=3.9.0,!=3.9.1,>=3.8" groups = ["main", "dev"] files = [ - {file = "cryptography-46.0.6-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:64235194bad039a10bb6d2d930ab3323baaec67e2ce36215fd0952fad0930ca8"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:26031f1e5ca62fcb9d1fcb34b2b60b390d1aacaa15dc8b895a9ed00968b97b30"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9a693028b9cbe51b5a1136232ee8f2bc242e4e19d456ded3fa7c86e43c713b4a"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:67177e8a9f421aa2d3a170c3e56eca4e0128883cf52a071a7cbf53297f18b175"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:d9528b535a6c4f8ff37847144b8986a9a143585f0540fbcb1a98115b543aa463"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:22259338084d6ae497a19bae5d4c66b7ca1387d3264d1c2c0e72d9e9b6a77b97"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:760997a4b950ff00d418398ad73fbc91aa2894b5c1db7ccb45b4f68b42a63b3c"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:3dfa6567f2e9e4c5dceb8ccb5a708158a2a871052fa75c8b78cb0977063f1507"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:cdcd3edcbc5d55757e5f5f3d330dd00007ae463a7e7aa5bf132d1f22a4b62b19"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:d4e4aadb7fc1f88687f47ca20bb7227981b03afaae69287029da08096853b738"}, - {file = "cryptography-46.0.6-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2b417edbe8877cda9022dde3a008e2deb50be9c407eef034aeeb3a8b11d9db3c"}, - {file = "cryptography-46.0.6-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:380343e0653b1c9d7e1f55b52aaa2dbb2fdf2730088d48c43ca1c7c0abb7cc2f"}, - {file = "cryptography-46.0.6-cp311-abi3-win32.whl", hash = "sha256:bcb87663e1f7b075e48c3be3ecb5f0b46c8fc50b50a97cf264e7f60242dca3f2"}, - {file = "cryptography-46.0.6-cp311-abi3-win_amd64.whl", hash = "sha256:6739d56300662c468fddb0e5e291f9b4d084bead381667b9e654c7dd81705124"}, - {file = "cryptography-46.0.6-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:2ef9e69886cbb137c2aef9772c2e7138dc581fad4fcbcf13cc181eb5a3ab6275"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7f417f034f91dcec1cb6c5c35b07cdbb2ef262557f701b4ecd803ee8cefed4f4"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d24c13369e856b94892a89ddf70b332e0b70ad4a5c43cf3e9cb71d6d7ffa1f7b"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:aad75154a7ac9039936d50cf431719a2f8d4ed3d3c277ac03f3339ded1a5e707"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:3c21d92ed15e9cfc6eb64c1f5a0326db22ca9c2566ca46d845119b45b4400361"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:4668298aef7cddeaf5c6ecc244c2302a2b8e40f384255505c22875eebb47888b"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:8ce35b77aaf02f3b59c90b2c8a05c73bac12cea5b4e8f3fbece1f5fddea5f0ca"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:c89eb37fae9216985d8734c1afd172ba4927f5a05cfd9bf0e4863c6d5465b013"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:ed418c37d095aeddf5336898a132fba01091f0ac5844e3e8018506f014b6d2c4"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:69cf0056d6947edc6e6760e5f17afe4bea06b56a9ac8a06de9d2bd6b532d4f3a"}, - {file = "cryptography-46.0.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8e7304c4f4e9490e11efe56af6713983460ee0780f16c63f219984dab3af9d2d"}, - {file = "cryptography-46.0.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b928a3ca837c77a10e81a814a693f2295200adb3352395fad024559b7be7a736"}, - {file = "cryptography-46.0.6-cp314-cp314t-win32.whl", hash = "sha256:97c8115b27e19e592a05c45d0dd89c57f81f841cc9880e353e0d3bf25b2139ed"}, - {file = "cryptography-46.0.6-cp314-cp314t-win_amd64.whl", hash = "sha256:c797e2517cb7880f8297e2c0f43bb910e91381339336f75d2c1c2cbf811b70b4"}, - {file = "cryptography-46.0.6-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:12cae594e9473bca1a7aceb90536060643128bb274fcea0fc459ab90f7d1ae7a"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:639301950939d844a9e1c4464d7e07f902fe9a7f6b215bb0d4f28584729935d8"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ed3775295fb91f70b4027aeba878d79b3e55c0b3e97eaa4de71f8f23a9f2eb77"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:8927ccfbe967c7df312ade694f987e7e9e22b2425976ddbf28271d7e58845290"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:b12c6b1e1651e42ab5de8b1e00dc3b6354fdfd778e7fa60541ddacc27cd21410"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:063b67749f338ca9c5a0b7fe438a52c25f9526b851e24e6c9310e7195aad3b4d"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:02fad249cb0e090b574e30b276a3da6a149e04ee2f049725b1f69e7b8351ec70"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e6142674f2a9291463e5e150090b95a8519b2fb6e6aaec8917dd8d094ce750d"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:456b3215172aeefb9284550b162801d62f5f264a081049a3e94307fe20792cfa"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:341359d6c9e68834e204ceaf25936dffeafea3829ab80e9503860dcc4f4dac58"}, - {file = "cryptography-46.0.6-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9a9c42a2723999a710445bc0d974e345c32adfd8d2fac6d8a251fa829ad31cfb"}, - {file = "cryptography-46.0.6-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6617f67b1606dfd9fe4dbfa354a9508d4a6d37afe30306fe6c101b7ce3274b72"}, - {file = "cryptography-46.0.6-cp38-abi3-win32.whl", hash = "sha256:7f6690b6c55e9c5332c0b59b9c8a3fb232ebf059094c17f9019a51e9827df91c"}, - {file = "cryptography-46.0.6-cp38-abi3-win_amd64.whl", hash = "sha256:79e865c642cfc5c0b3eb12af83c35c5aeff4fa5c672dc28c43721c2c9fdd2f0f"}, - {file = "cryptography-46.0.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:2ea0f37e9a9cf0df2952893ad145fd9627d326a59daec9b0802480fa3bcd2ead"}, - {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a3e84d5ec9ba01f8fd03802b2147ba77f0c8f2617b2aff254cedd551844209c8"}, - {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:12f0fa16cc247b13c43d56d7b35287ff1569b5b1f4c5e87e92cc4fcc00cd10c0"}, - {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:50575a76e2951fe7dbd1f56d181f8c5ceeeb075e9ff88e7ad997d2f42af06e7b"}, - {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:90e5f0a7b3be5f40c3a0a0eafb32c681d8d2c181fc2a1bdabe9b3f611d9f6b1a"}, - {file = "cryptography-46.0.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6728c49e3b2c180ef26f8e9f0a883a2c585638db64cf265b49c9ba10652d430e"}, - {file = "cryptography-46.0.6.tar.gz", hash = "sha256:27550628a518c5c6c903d84f637fbecf287f6cb9ced3804838a1295dc1fd0759"}, + {file = "cryptography-46.0.7-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:ea42cbe97209df307fdc3b155f1b6fa2577c0defa8f1f7d3be7d31d189108ad4"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b36a4695e29fe69215d75960b22577197aca3f7a25b9cf9d165dcfe9d80bc325"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5ad9ef796328c5e3c4ceed237a183f5d41d21150f972455a9d926593a1dcb308"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:73510b83623e080a2c35c62c15298096e2a5dc8d51c3b4e1740211839d0dea77"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cbd5fb06b62bd0721e1170273d3f4d5a277044c47ca27ee257025146c34cbdd1"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:420b1e4109cc95f0e5700eed79908cef9268265c773d3a66f7af1eef53d409ef"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:24402210aa54baae71d99441d15bb5a1919c195398a87b563df84468160a65de"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:8a469028a86f12eb7d2fe97162d0634026d92a21f3ae0ac87ed1c4a447886c83"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9694078c5d44c157ef3162e3bf3946510b857df5a3955458381d1c7cfc143ddb"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:42a1e5f98abb6391717978baf9f90dc28a743b7d9be7f0751a6f56a75d14065b"}, + {file = "cryptography-46.0.7-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:91bbcb08347344f810cbe49065914fe048949648f6bd5c2519f34619142bbe85"}, + {file = "cryptography-46.0.7-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5d1c02a14ceb9148cc7816249f64f623fbfee39e8c03b3650d842ad3f34d637e"}, + {file = "cryptography-46.0.7-cp311-abi3-win32.whl", hash = "sha256:d23c8ca48e44ee015cd0a54aeccdf9f09004eba9fc96f38c911011d9ff1bd457"}, + {file = "cryptography-46.0.7-cp311-abi3-win_amd64.whl", hash = "sha256:397655da831414d165029da9bc483bed2fe0e75dde6a1523ec2fe63f3c46046b"}, + {file = "cryptography-46.0.7-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:d151173275e1728cf7839aaa80c34fe550c04ddb27b34f48c232193df8db5842"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:db0f493b9181c7820c8134437eb8b0b4792085d37dbb24da050476ccb664e59c"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ebd6daf519b9f189f85c479427bbd6e9c9037862cf8fe89ee35503bd209ed902"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b7b412817be92117ec5ed95f880defe9cf18a832e8cafacf0a22337dc1981b4d"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:fbfd0e5f273877695cb93baf14b185f4878128b250cc9f8e617ea0c025dfb022"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:ffca7aa1d00cf7d6469b988c581598f2259e46215e0140af408966a24cf086ce"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:60627cf07e0d9274338521205899337c5d18249db56865f943cbe753aa96f40f"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:80406c3065e2c55d7f49a9550fe0c49b3f12e5bfff5dedb727e319e1afb9bf99"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:c5b1ccd1239f48b7151a65bc6dd54bcfcc15e028c8ac126d3fada09db0e07ef1"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:d5f7520159cd9c2154eb61eb67548ca05c5774d39e9c2c4339fd793fe7d097b2"}, + {file = "cryptography-46.0.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fcd8eac50d9138c1d7fc53a653ba60a2bee81a505f9f8850b6b2888555a45d0e"}, + {file = "cryptography-46.0.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:65814c60f8cc400c63131584e3e1fad01235edba2614b61fbfbfa954082db0ee"}, + {file = "cryptography-46.0.7-cp314-cp314t-win32.whl", hash = "sha256:fdd1736fed309b4300346f88f74cd120c27c56852c3838cab416e7a166f67298"}, + {file = "cryptography-46.0.7-cp314-cp314t-win_amd64.whl", hash = "sha256:e06acf3c99be55aa3b516397fe42f5855597f430add9c17fa46bf2e0fb34c9bb"}, + {file = "cryptography-46.0.7-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:462ad5cb1c148a22b2e3bcc5ad52504dff325d17daf5df8d88c17dda1f75f2a4"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:84d4cced91f0f159a7ddacad249cc077e63195c36aac40b4150e7a57e84fffe7"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:128c5edfe5e5938b86b03941e94fac9ee793a94452ad1365c9fc3f4f62216832"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5e51be372b26ef4ba3de3c167cd3d1022934bc838ae9eaad7e644986d2a3d163"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cdf1a610ef82abb396451862739e3fc93b071c844399e15b90726ef7470eeaf2"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1d25aee46d0c6f1a501adcddb2d2fee4b979381346a78558ed13e50aa8a59067"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:cdfbe22376065ffcf8be74dc9a909f032df19bc58a699456a21712d6e5eabfd0"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:abad9dac36cbf55de6eb49badd4016806b3165d396f64925bf2999bcb67837ba"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:935ce7e3cfdb53e3536119a542b839bb94ec1ad081013e9ab9b7cfd478b05006"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:35719dc79d4730d30f1c2b6474bd6acda36ae2dfae1e3c16f2051f215df33ce0"}, + {file = "cryptography-46.0.7-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7bbc6ccf49d05ac8f7d7b5e2e2c33830d4fe2061def88210a126d130d7f71a85"}, + {file = "cryptography-46.0.7-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a1529d614f44b863a7b480c6d000fe93b59acee9c82ffa027cfadc77521a9f5e"}, + {file = "cryptography-46.0.7-cp38-abi3-win32.whl", hash = "sha256:f247c8c1a1fb45e12586afbb436ef21ff1e80670b2861a90353d9b025583d246"}, + {file = "cryptography-46.0.7-cp38-abi3-win_amd64.whl", hash = "sha256:506c4ff91eff4f82bdac7633318a526b1d1309fc07ca76a3ad182cb5b686d6d3"}, + {file = "cryptography-46.0.7-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fc9ab8856ae6cf7c9358430e49b368f3108f050031442eaeb6b9d87e4dcf4e4f"}, + {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d3b99c535a9de0adced13d159c5a9cf65c325601aa30f4be08afd680643e9c15"}, + {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d02c738dacda7dc2a74d1b2b3177042009d5cab7c7079db74afc19e56ca1b455"}, + {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:04959522f938493042d595a736e7dbdff6eb6cc2339c11465b3ff89343b65f65"}, + {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:3986ac1dee6def53797289999eabe84798ad7817f3e97779b5061a95b0ee4968"}, + {file = "cryptography-46.0.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:258514877e15963bd43b558917bc9f54cf7cf866c38aa576ebf47a77ddbc43a4"}, + {file = "cryptography-46.0.7.tar.gz", hash = "sha256:e4cfd68c5f3e0bfdad0d38e023239b96a2fe84146481852dffbcca442c245aa5"}, ] [package.dependencies] @@ -521,7 +521,7 @@ nox = ["nox[uv] (>=2024.4.15)"] pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.14)", "ruff (>=0.11.11)"] sdist = ["build (>=1.0.0)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi (>=2024)", "cryptography-vectors (==46.0.6)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] +test = ["certifi (>=2024)", "cryptography-vectors (==46.0.7)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] test-randomorder = ["pytest-randomly"] [[package]] @@ -601,118 +601,118 @@ test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3. [[package]] name = "hiredis" -version = "3.3.0" +version = "3.3.1" description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] markers = "extra == \"redis\" or extra == \"all\"" files = [ - {file = "hiredis-3.3.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:9937d9b69321b393fbace69f55423480f098120bc55a3316e1ca3508c4dbbd6f"}, - {file = "hiredis-3.3.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:50351b77f89ba6a22aff430b993653847f36b71d444509036baa0f2d79d1ebf4"}, - {file = "hiredis-3.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1d00bce25c813eec45a2f524249f58daf51d38c9d3347f6f643ae53826fc735a"}, - {file = "hiredis-3.3.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ef840d9f142556ed384180ed8cdf14ff875fcae55c980cbe5cec7adca2ef4d8"}, - {file = "hiredis-3.3.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:88bc79d7e9b94d17ed1bd8b7f2815ed0eada376ed5f48751044e5e4d179aa2f2"}, - {file = "hiredis-3.3.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7165c7363e59b258e1875c51f35c0b2b9901e6c691037b487d8a0ace2c137ed2"}, - {file = "hiredis-3.3.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8c3be446f0c38fbe6863a7cf4522c9a463df6e64bee87c4402e9f6d7d2e7f869"}, - {file = "hiredis-3.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:96f9a27643279853b91a1fb94a88b559e55fdecec86f1fcd5f2561492be52e47"}, - {file = "hiredis-3.3.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:0a5eebb170de1b415c78ae5ca3aee17cff8b885df93c2055d54320e789d838f4"}, - {file = "hiredis-3.3.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:200678547ac3966bac3e38df188211fdc13d5f21509c23267e7def411710e112"}, - {file = "hiredis-3.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:dd9d78c5363a858f9dc5e698e5e1e402b83c00226cba294f977a92c53092b549"}, - {file = "hiredis-3.3.0-cp310-cp310-win32.whl", hash = "sha256:a0d31ff178b913137a7a08c7377e93805914755a15c3585e203d0d74496456c0"}, - {file = "hiredis-3.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:7b41833c8f0d4c7fbfaa867c8ed9a4e4aaa71d7c54e4806ed62da2d5cd27b40d"}, - {file = "hiredis-3.3.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:63ee6c1ae6a2462a2439eb93c38ab0315cd5f4b6d769c6a34903058ba538b5d6"}, - {file = "hiredis-3.3.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:31eda3526e2065268a8f97fbe3d0e9a64ad26f1d89309e953c80885c511ea2ae"}, - {file = "hiredis-3.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a26bae1b61b7bcafe3d0d0c7d012fb66ab3c95f2121dbea336df67e344e39089"}, - {file = "hiredis-3.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b9546079f7fd5c50fbff9c791710049b32eebe7f9b94debec1e8b9f4c048cba2"}, - {file = "hiredis-3.3.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ae327fc13b1157b694d53f92d50920c0051e30b0c245f980a7036e299d039ab4"}, - {file = "hiredis-3.3.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4016e50a8be5740a59c5af5252e5ad16c395021a999ad24c6604f0d9faf4d346"}, - {file = "hiredis-3.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c17b473f273465a3d2168a57a5b43846165105ac217d5652a005e14068589ddc"}, - {file = "hiredis-3.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9ecd9b09b11bd0b8af87d29c3f5da628d2bdc2a6c23d2dd264d2da082bd4bf32"}, - {file = "hiredis-3.3.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:00fb04eac208cd575d14f246e74a468561081ce235937ab17d77cde73aefc66c"}, - {file = "hiredis-3.3.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:60814a7d0b718adf3bfe2c32c6878b0e00d6ae290ad8e47f60d7bba3941234a6"}, - {file = "hiredis-3.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:fcbd1a15e935aa323b5b2534b38419511b7909b4b8ee548e42b59090a1b37bb1"}, - {file = "hiredis-3.3.0-cp311-cp311-win32.whl", hash = "sha256:73679607c5a19f4bcfc9cf6eb54480bcd26617b68708ac8b1079da9721be5449"}, - {file = "hiredis-3.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:30a4df3d48f32538de50648d44146231dde5ad7f84f8f08818820f426840ae97"}, - {file = "hiredis-3.3.0-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:5b8e1d6a2277ec5b82af5dce11534d3ed5dffeb131fd9b210bc1940643b39b5f"}, - {file = "hiredis-3.3.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c4981de4d335f996822419e8a8b3b87367fcef67dc5fb74d3bff4df9f6f17783"}, - {file = "hiredis-3.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1706480a683e328ae9ba5d704629dee2298e75016aa0207e7067b9c40cecc271"}, - {file = "hiredis-3.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a95cef9989736ac313639f8f545b76b60b797e44e65834aabbb54e4fad8d6c8"}, - {file = "hiredis-3.3.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ca2802934557ccc28a954414c245ba7ad904718e9712cb67c05152cf6b9dd0a3"}, - {file = "hiredis-3.3.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fe730716775f61e76d75810a38ee4c349d3af3896450f1525f5a4034cf8f2ed7"}, - {file = "hiredis-3.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:749faa69b1ce1f741f5eaf743435ac261a9262e2d2d66089192477e7708a9abc"}, - {file = "hiredis-3.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:95c9427f2ac3f1dd016a3da4e1161fa9d82f221346c8f3fdd6f3f77d4e28946c"}, - {file = "hiredis-3.3.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c863ee44fe7bff25e41f3a5105c936a63938b76299b802d758f40994ab340071"}, - {file = "hiredis-3.3.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:2213c7eb8ad5267434891f3241c7776e3bafd92b5933fc57d53d4456247dc542"}, - {file = "hiredis-3.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a172bae3e2837d74530cd60b06b141005075db1b814d966755977c69bd882ce8"}, - {file = "hiredis-3.3.0-cp312-cp312-win32.whl", hash = "sha256:cb91363b9fd6d41c80df9795e12fffbaf5c399819e6ae8120f414dedce6de068"}, - {file = "hiredis-3.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:04ec150e95eea3de9ff8bac754978aa17b8bf30a86d4ab2689862020945396b0"}, - {file = "hiredis-3.3.0-cp313-cp313-macosx_10_15_universal2.whl", hash = "sha256:b7048b4ec0d5dddc8ddd03da603de0c4b43ef2540bf6e4c54f47d23e3480a4fa"}, - {file = "hiredis-3.3.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:e5f86ce5a779319c15567b79e0be806e8e92c18bb2ea9153e136312fafa4b7d6"}, - {file = "hiredis-3.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fbdb97a942e66016fff034df48a7a184e2b7dc69f14c4acd20772e156f20d04b"}, - {file = "hiredis-3.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0fb4bea72fe45ff13e93ddd1352b43ff0749f9866263b5cca759a4c960c776f"}, - {file = "hiredis-3.3.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85b9baf98050e8f43c2826ab46aaf775090d608217baf7af7882596aef74e7f9"}, - {file = "hiredis-3.3.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:69079fb0f0ebb61ba63340b9c4bce9388ad016092ca157e5772eb2818209d930"}, - {file = "hiredis-3.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c17f77b79031ea4b0967d30255d2ae6e7df0603ee2426ad3274067f406938236"}, - {file = "hiredis-3.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:45d14f745fc177bc05fc24bdf20e2b515e9a068d3d4cce90a0fb78d04c9c9d9a"}, - {file = "hiredis-3.3.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:ba063fdf1eff6377a0c409609cbe890389aefddfec109c2d20fcc19cfdafe9da"}, - {file = "hiredis-3.3.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:1799cc66353ad066bfdd410135c951959da9f16bcb757c845aab2f21fc4ef099"}, - {file = "hiredis-3.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2cbf71a121996ffac82436b6153290815b746afb010cac19b3290a1644381b07"}, - {file = "hiredis-3.3.0-cp313-cp313-win32.whl", hash = "sha256:a7cbbc6026bf03659f0b25e94bbf6e64f6c8c22f7b4bc52fe569d041de274194"}, - {file = "hiredis-3.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:a8def89dd19d4e2e4482b7412d453dec4a5898954d9a210d7d05f60576cedef6"}, - {file = "hiredis-3.3.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c135bda87211f7af9e2fd4e046ab433c576cd17b69e639a0f5bb2eed5e0e71a9"}, - {file = "hiredis-3.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2f855c678230aed6fc29b962ce1cc67e5858a785ef3a3fd6b15dece0487a2e60"}, - {file = "hiredis-3.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4059c78a930cbb33c391452ccce75b137d6f89e2eebf6273d75dafc5c2143c03"}, - {file = "hiredis-3.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:334a3f1d14c253bb092e187736c3384203bd486b244e726319bbb3f7dffa4a20"}, - {file = "hiredis-3.3.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fd137b147235447b3d067ec952c5b9b95ca54b71837e1b38dbb2ec03b89f24fc"}, - {file = "hiredis-3.3.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8f88f4f2aceb73329ece86a1cb0794fdbc8e6d614cb5ca2d1023c9b7eb432db8"}, - {file = "hiredis-3.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:550f4d1538822fc75ebf8cf63adc396b23d4958bdbbad424521f2c0e3dfcb169"}, - {file = "hiredis-3.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:54b14211fbd5930fc696f6fcd1f1f364c660970d61af065a80e48a1fa5464dd6"}, - {file = "hiredis-3.3.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c9e96f63dbc489fc86f69951e9f83dadb9582271f64f6822c47dcffa6fac7e4a"}, - {file = "hiredis-3.3.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:106e99885d46684d62ab3ec1d6b01573cc0e0083ac295b11aaa56870b536c7ec"}, - {file = "hiredis-3.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:087e2ef3206361281b1a658b5b4263572b6ba99465253e827796964208680459"}, - {file = "hiredis-3.3.0-cp314-cp314-win32.whl", hash = "sha256:80638ebeab1cefda9420e9fedc7920e1ec7b4f0513a6b23d58c9d13c882f8065"}, - {file = "hiredis-3.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:a68aaf9ba024f4e28cf23df9196ff4e897bd7085872f3a30644dca07fa787816"}, - {file = "hiredis-3.3.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:f7f80442a32ce51ee5d89aeb5a84ee56189a0e0e875f1a57bbf8d462555ae48f"}, - {file = "hiredis-3.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a1a67530da714954ed50579f4fe1ab0ddbac9c43643b1721c2cb226a50dde263"}, - {file = "hiredis-3.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:616868352e47ab355559adca30f4f3859f9db895b4e7bc71e2323409a2add751"}, - {file = "hiredis-3.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e799b79f3150083e9702fc37e6243c0bd47a443d6eae3f3077b0b3f510d6a145"}, - {file = "hiredis-3.3.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9ef1dfb0d2c92c3701655e2927e6bbe10c499aba632c7ea57b6392516df3864b"}, - {file = "hiredis-3.3.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c290da6bc2a57e854c7da9956cd65013483ede935677e84560da3b848f253596"}, - {file = "hiredis-3.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fd8c438d9e1728f0085bf9b3c9484d19ec31f41002311464e75b69550c32ffa8"}, - {file = "hiredis-3.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1bbc6b8a88bbe331e3ebf6685452cebca6dfe6d38a6d4efc5651d7e363ba28bd"}, - {file = "hiredis-3.3.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:55d8c18fe9a05496c5c04e6eccc695169d89bf358dff964bcad95696958ec05f"}, - {file = "hiredis-3.3.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4ddc79afa76b805d364e202a754666cb3c4d9c85153cbfed522871ff55827838"}, - {file = "hiredis-3.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8e8a4b8540581dcd1b2b25827a54cfd538e0afeaa1a0e3ca87ad7126965981cc"}, - {file = "hiredis-3.3.0-cp314-cp314t-win32.whl", hash = "sha256:298593bb08487753b3afe6dc38bac2532e9bac8dcee8d992ef9977d539cc6776"}, - {file = "hiredis-3.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b442b6ab038a6f3b5109874d2514c4edf389d8d8b553f10f12654548808683bc"}, - {file = "hiredis-3.3.0-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:114c0b9f1b5fad99edae38e747018aead358a4f4e9720cc1876495d78cdb8276"}, - {file = "hiredis-3.3.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:c6d91a5e6904ed7eca21d74b041e03f2ad598dd08a6065b06a776974fe5d003c"}, - {file = "hiredis-3.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:76374faa075e996c895cbe106ba923852a9f8146f2aa59eba22111c5e5ec6316"}, - {file = "hiredis-3.3.0-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50a54397bd104c2e2f5b7696bbdab8ba2973d3075e4deb932adb025b8863de91"}, - {file = "hiredis-3.3.0-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:15edee02cc9cc06e07e2bcfae07e283e640cc1aeedd08b4c6934bf1a0113c607"}, - {file = "hiredis-3.3.0-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ff3179a57745d0f8d71fa8bf3ea3944d3f557dcfa4431304497987fecad381dd"}, - {file = "hiredis-3.3.0-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bdb7cd9e1e73db78f145a09bb837732790d0912eb963dee5768631faf2ece162"}, - {file = "hiredis-3.3.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:4d3b4e0d4445faf9041c52a98cb5d2b65c4fcaebb2aa02efa7c6517c4917f7e8"}, - {file = "hiredis-3.3.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:ffea6c407cff532c7599d3ec9e8502c2c865753cebab044f3dfce9afbf71a8df"}, - {file = "hiredis-3.3.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:bcd745a28e1b3216e42680d91e142a42569dfad68a6f40535080c47b0356c796"}, - {file = "hiredis-3.3.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:4c18a97ea55d1a58f5c3adfe236b3e7cccedc6735cbd36ab1c786c52fd823667"}, - {file = "hiredis-3.3.0-cp38-cp38-win32.whl", hash = "sha256:77eacd969e3c6ff50c2b078c27d2a773c652248a5d81af5765a8663478d0bc02"}, - {file = "hiredis-3.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:161a4a595a53475587aef8dc549d0527962879b0c5d62f7947b44ba7e5084b76"}, - {file = "hiredis-3.3.0-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:1203697a7ebadc7cf873acc189df9e44fcb377b636e6660471707ac8d5bcba68"}, - {file = "hiredis-3.3.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:9a7ea2344d277317160da4911f885bcf7dfd8381b830d76b442f7775b41544b3"}, - {file = "hiredis-3.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9bd7c9a089cf4e4f4b5a61f412c76293449bac6b0bf92bb49a3892850bd5c899"}, - {file = "hiredis-3.3.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:294de11e3995128c784534e327d1f9382b88dc5407356465df7934c710e8392d"}, - {file = "hiredis-3.3.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4a3aab895358368f81f9546a7cd192b6fb427f785cb1a8853cf9db38df01e9ca"}, - {file = "hiredis-3.3.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:eaf8418e33e23d6d7ef0128eff4c06ab3040d40b9bbc8a24d6265d751a472596"}, - {file = "hiredis-3.3.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41aea51949142bad4e40badb0396392d7f4394791e4097a0951ab75bcc58ff84"}, - {file = "hiredis-3.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1f9a5f84a8bd29ac5b9953b27e8ba5508396afeabf1d165611a1e31fbd90a0e1"}, - {file = "hiredis-3.3.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:a5f9fde56550ebbe962f437a4c982b0856d03aea7fab09e30fa6c0f9be992b40"}, - {file = "hiredis-3.3.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:c567aab02612d91f3e747fc492100ae894515194f85d6fb6bb68958c0e718721"}, - {file = "hiredis-3.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:ca97c5e6f9e9b9f0aed61b70fed2d594ce2f7472905077d2d10b307c50a41008"}, - {file = "hiredis-3.3.0-cp39-cp39-win32.whl", hash = "sha256:776dc5769d5eb05e969216de095377ff61c802414a74bd3c24a4ca8526c897ab"}, - {file = "hiredis-3.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:538a9f5fbb3a8a4ef0c3abd309cccb90cd2ba9976fcc2b44193af9507d005b48"}, - {file = "hiredis-3.3.0.tar.gz", hash = "sha256:105596aad9249634361815c574351f1bd50455dc23b537c2940066c4a9dea685"}, + {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:f525734382a47f9828c9d6a1501522c78d5935466d8e2be1a41ba40ca5bb922b"}, + {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:6e2e1024f0a021777740cb7c633a0efb2c4a4bc570f508223a8dcbcf79f99ef9"}, + {file = "hiredis-3.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c1d68c6980d4690a4550bd3db6c03146f7be68ef5d08d38bb1fb68b3e9c32fe3"}, + {file = "hiredis-3.3.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0caf3fc8af0767794b335753781c3fa35f2a3e975c098edbc8f733d35d6a95e4"}, + {file = "hiredis-3.3.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:81a1669b6631976b1dc9d3d58ed1ab3333e9f52feb91a2a1fb8241101ac3b665"}, + {file = "hiredis-3.3.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c8139e9011117822391c5bcfd674c5948fb1e4b8cb9adf6f13d9890859ee3a1a"}, + {file = "hiredis-3.3.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:042e57de8a2cae91e3e7c0af32960ea2c5107b2f27f68a740295861e68780a8a"}, + {file = "hiredis-3.3.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:65f6ac06a9f0c32c254660ec6a9329d81d589e8f5d0a9837a941d5424a6be1ef"}, + {file = "hiredis-3.3.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:002fc0201b9af1cc8960e27cdc501ad1f8cdd6dbadb2091c6ddbd4e5ace6cb77"}, + {file = "hiredis-3.3.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:9ebae74ce2b977c2fcb22d6a10aa0acb730022406977b2bcb6ddd6788f5c414a"}, + {file = "hiredis-3.3.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8a52b24cd710690c4a7e191c7e300136ad2ecb3c68ffe7e95b598e76de166e5e"}, + {file = "hiredis-3.3.1-cp310-cp310-win32.whl", hash = "sha256:1ebc307a87b099d0877dbd2bdc0bae427258e7ec67f60a951e89027f8dc2568f"}, + {file = "hiredis-3.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:62cc62284541bb2a86c898c7d5e8388661cade91c184cb862095ed547e80588f"}, + {file = "hiredis-3.3.1-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:26f899cde0279e4b7d370716ff80320601c2bd93cdf3e774a42bdd44f65b41f8"}, + {file = "hiredis-3.3.1-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:a2f049c3f3c83e886cd1f53958e2a1ebb369be626bef9e50d8b24d79864f1df6"}, + {file = "hiredis-3.3.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5f316cf2d0558f5027aab19dde7d7e4901c26c21fa95367bc37784e8f547bbf2"}, + {file = "hiredis-3.3.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:03baa381964b8df356d19ec4e3a6ae656044249a87b0def257fe1e08dbaf6094"}, + {file = "hiredis-3.3.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304481241e081bc26f0778b2c2b99f9c43917e4e724a016dcc9439b7ab12c726"}, + {file = "hiredis-3.3.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8597c35c9e82f65fd5897c4a2188c65d7daf10607b102960137b23d261cd957b"}, + {file = "hiredis-3.3.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad940dc2db545dc978cb41cb9a683e2ff328f3ef581230b9ca40ff6c3d01d542"}, + {file = "hiredis-3.3.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:156be6a0c736ee145cfe0fb155d0e96cec8d4872cf8b4f76ad6a2ee6ab391d0a"}, + {file = "hiredis-3.3.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:583de2f16528e66081cbdfe510d8488c2de73039dc00aada7d22bd49d73a4a94"}, + {file = "hiredis-3.3.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:c24c1460486b6b36083252c2db21a814becf8495ccd0e76b7286623e37239b63"}, + {file = "hiredis-3.3.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a58a58cef0d911b1717154179a9ff47852249c536ea5966bde4370b6b20638ff"}, + {file = "hiredis-3.3.1-cp311-cp311-win32.whl", hash = "sha256:e0db44cf81e4d7b94f3776b9f89111f74ed6bbdbfd42a22bc4a5ce0644d3e060"}, + {file = "hiredis-3.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:1f7bceb03a1b934872ffe3942eaeed7c7e09096e67b53f095b81f39c7a819113"}, + {file = "hiredis-3.3.1-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:60543f3b068b16a86e99ed96b7fdae71cdc1d8abdfe9b3f82032a555e52ece7e"}, + {file = "hiredis-3.3.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:2611bfaaadc5e8d43fb7967f9bbf1110c8beaa83aee2f2d812c76f11cfb56c6a"}, + {file = "hiredis-3.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8e3754ce60e1b11b0afad9a053481ff184d2ee24bea47099107156d1b84a84aa"}, + {file = "hiredis-3.3.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e89dabf436ee79b358fd970dcbed6333a36d91db73f27069ca24a02fb138a404"}, + {file = "hiredis-3.3.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4f7e242eab698ad0be5a4b2ec616fa856569c57455cc67c625fd567726290e5f"}, + {file = "hiredis-3.3.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53148a4e21057541b6d8e493b2ea1b500037ddf34433c391970036f3cbce00e3"}, + {file = "hiredis-3.3.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25132902d3eff38781e0d54f27a0942ec849e3c07dbdce83c4d92b7e43c8dce"}, + {file = "hiredis-3.3.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3fb6573efa15a29c12c0c0f7170b14e7c1347fe4bb39b6a15b779f46015cc929"}, + {file = "hiredis-3.3.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:487658e1db83c1ee9fbbac6a43039ea76957767a5987ffb16b590613f9e68297"}, + {file = "hiredis-3.3.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a1d190790ee39b8b7adeeb10fc4090dc4859eb4e75ed27bd8108710eef18f358"}, + {file = "hiredis-3.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a42c7becd4c9ec4ab5769c754eb61112777bdc6e1c1525e2077389e193b5f5aa"}, + {file = "hiredis-3.3.1-cp312-cp312-win32.whl", hash = "sha256:17ec8b524055a88b80d76c177dbbbe475a25c17c5bf4b67bdbdbd0629bcae838"}, + {file = "hiredis-3.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:0fac4af8515e6cca74fc701169ae4dc9a71a90e9319c9d21006ec9454b43aa2f"}, + {file = "hiredis-3.3.1-cp313-cp313-macosx_10_15_universal2.whl", hash = "sha256:afe3c3863f16704fb5d7c2c6ff56aaf9e054f6d269f7b4c9074c5476178d1aba"}, + {file = "hiredis-3.3.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:f19ee7dc1ef8a6497570d91fa4057ba910ad98297a50b8c44ff37589f7c89d17"}, + {file = "hiredis-3.3.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:09f5e510f637f2c72d2a79fb3ad05f7b6211e057e367ca5c4f97bb3d8c9d71f4"}, + {file = "hiredis-3.3.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b46e96b50dad03495447860510daebd2c96fd44ed25ba8ccb03e9f89eaa9d34"}, + {file = "hiredis-3.3.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b4fe7f38aa8956fcc1cea270e62601e0e11066aff78e384be70fd283d30293b6"}, + {file = "hiredis-3.3.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2b96da7e365d6488d2a75266a662cbe3cc14b28c23dd9b0c9aa04b5bc5c20192"}, + {file = "hiredis-3.3.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:52d5641027d6731bc7b5e7d126a5158a99784a9f8c6de3d97ca89aca4969e9f8"}, + {file = "hiredis-3.3.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:eddeb9a153795cf6e615f9f3cef66a1d573ff3b6ee16df2b10d1d1c2f2baeaa8"}, + {file = "hiredis-3.3.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:011a9071c3df4885cac7f58a2623feac6c8e2ad30e6ba93c55195af05ce61ff5"}, + {file = "hiredis-3.3.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:264ee7e9cb6c30dc78da4ecf71d74cf14ca122817c665d838eda8b4384bce1b0"}, + {file = "hiredis-3.3.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d1434d0bcc1b3ef048bae53f26456405c08aeed9827e65b24094f5f3a6793f1"}, + {file = "hiredis-3.3.1-cp313-cp313-win32.whl", hash = "sha256:f915a34fb742e23d0d61573349aa45d6f74037fde9d58a9f340435eff8d62736"}, + {file = "hiredis-3.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:d8e56e0d1fe607bfff422633f313aec9191c3859ab99d11ff097e3e6e068000c"}, + {file = "hiredis-3.3.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:439f9a5cc8f9519ce208a24cdebfa0440fef26aa682a40ba2c92acb10a53f5e0"}, + {file = "hiredis-3.3.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3724f0e58c6ff76fd683429945491de71324ab1bc0ad943a8d68cb0932d24075"}, + {file = "hiredis-3.3.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:29fe35e3c6fe03204e75c86514f452591957a1e06b05d86e10d795455b71c355"}, + {file = "hiredis-3.3.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d42f3a13290f89191568fc113d95a3d2c8759cdd8c3672f021d8b7436f909e75"}, + {file = "hiredis-3.3.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2afc675b831f7552da41116fffffca4340f387dc03f56d6ec0c7895ab0b59a10"}, + {file = "hiredis-3.3.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4106201cd052d9eabe3cb7b5a24b0fe37307792bda4fcb3cf6ddd72f697828e8"}, + {file = "hiredis-3.3.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8887bf0f31e4b550bd988c8863b527b6587d200653e9375cd91eea2b944b7424"}, + {file = "hiredis-3.3.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1ac7697365dbe45109273b34227fee6826b276ead9a4a007e0877e1d3f0fcf21"}, + {file = "hiredis-3.3.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2b6da6e07359107c653a809b3cff2d9ccaeedbafe33c6f16434aef6f53ce4a2b"}, + {file = "hiredis-3.3.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:ce334915f5d31048f76a42c607bf26687cf045eb1bc852b7340f09729c6a64fc"}, + {file = "hiredis-3.3.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee11fd431f83d8a5b29d370b9d79a814d3218d30113bdcd44657e9bdf715fc92"}, + {file = "hiredis-3.3.1-cp314-cp314-win32.whl", hash = "sha256:e0356561b4a97c83b9ee3de657a41b8d1a1781226853adaf47b550bb988fda6f"}, + {file = "hiredis-3.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:80aba5f85d6227faee628ae28d1c3b69c661806a0636548ac56c68782606454f"}, + {file = "hiredis-3.3.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:907f7b5501a534030738f0f27459a612d2266fd0507b007bb8f3e6de08167920"}, + {file = "hiredis-3.3.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:de94b409f49eb6a588ebdd5872e826caec417cd77c17af0fb94f2128427f1a2a"}, + {file = "hiredis-3.3.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:79cd03e7ff550c17758a7520bf437c156d3d4c8bb74214deeafa69cda49c85a4"}, + {file = "hiredis-3.3.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6ffa7ba2e2da1f806f3181b9730b3e87ba9dbfec884806725d4584055ba3faa6"}, + {file = "hiredis-3.3.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ee37fe8cf081b72dea72f96a0ee604f492ec02252eb77dc26ff6eec3f997b580"}, + {file = "hiredis-3.3.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9bfdeff778d3f7ff449ca5922ab773899e7d31e26a576028b06a5e9cf0ed8c34"}, + {file = "hiredis-3.3.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:027ce4fabfeff5af5b9869d5524770877f9061d118bc36b85703ae3faf5aad8e"}, + {file = "hiredis-3.3.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:dcea8c3f53674ae68e44b12e853b844a1d315250ca6677b11ec0c06aff85e86c"}, + {file = "hiredis-3.3.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0b5ff2f643f4b452b0597b7fe6aa35d398cb31d8806801acfafb1558610ea2aa"}, + {file = "hiredis-3.3.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:3586c8a5f56d34b9dddaaa9e76905f31933cac267251006adf86ec0eef7d0400"}, + {file = "hiredis-3.3.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a110d19881ca78a88583d3b07231e7c6864864f5f1f3491b638863ea45fa8708"}, + {file = "hiredis-3.3.1-cp314-cp314t-win32.whl", hash = "sha256:98fd5b39410e9d69e10e90d0330e35650becaa5dd2548f509b9598f1f3c6124d"}, + {file = "hiredis-3.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:ab1f646ff531d70bfd25f01e60708dfa3d105eb458b7dedd9fe9a443039fd809"}, + {file = "hiredis-3.3.1-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:a3af4e9f277d6b8acd369dc44a723a055752fca9d045094383af39f90a3e3729"}, + {file = "hiredis-3.3.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:526db52e5234a9463520e960a509d6c1bd5128d1ab1b569cbf459fe39189e8ab"}, + {file = "hiredis-3.3.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:90d6b9f2652303aefd2c5a26a5e14cb74a3a63d10faa642c08d790e99442a088"}, + {file = "hiredis-3.3.1-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4479e36d263251dba8ab8ea81adf07e7f1163603c7102c5de1e130b83b4fad3b"}, + {file = "hiredis-3.3.1-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2390ad81c03d93ef1d5afd18ffcf5935de827f1a2b96b2c829437968bdabccb"}, + {file = "hiredis-3.3.1-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:65c05b79cb8366c123357b354a16f9fc3f7187159422f143638d1c26b7240ed4"}, + {file = "hiredis-3.3.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09d41a3a965f7c261223d516ebda607aee4d8440dd7637f01af9a4c05872f0c4"}, + {file = "hiredis-3.3.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:113e098e4a6b3cc5500e05e7cb1548ba9e83de5fe755941b11f6020a76e6c03a"}, + {file = "hiredis-3.3.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:e31e92b61d56244047ad600812e16f7587a6172f74810fd919ff993af12b9149"}, + {file = "hiredis-3.3.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:77c5d2bebbc9d06691abb512a31d0f54e1562af0b872891463a67a949b5278ef"}, + {file = "hiredis-3.3.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:137c14905ea6f2933967200bc7b2a0c8ec9387888b273fd0004f25b994fd0343"}, + {file = "hiredis-3.3.1-cp38-cp38-win32.whl", hash = "sha256:f2f94355affd51088f57f8674b0e294704c3c7c3d7d3b1545310f5b135d4843b"}, + {file = "hiredis-3.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:b1e3b9f4bf9a4120510ba77a77b2fb674893cd6795653545152bb11a79eecfcb"}, + {file = "hiredis-3.3.1-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:743b85bd6902856cac457ddd8cd7dd48c89c47d641b6016ff5e4d015bfbd4799"}, + {file = "hiredis-3.3.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:b37df4b10cb15dedfc203f69312d8eedd617b941c21df58c13af59496c53ad0f"}, + {file = "hiredis-3.3.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8650158217b469d8b6087f490929211b0493a9121154c4efaafd1dec9e19319e"}, + {file = "hiredis-3.3.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c74bd9926954e7e575f9cd9890f63defd90cd8f812dfbf8e1efb72acc9355456"}, + {file = "hiredis-3.3.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f1c1b2e8f00b71e6214234d313f655a3a27cd4384b054126ce04073c1d47045"}, + {file = "hiredis-3.3.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:01cf82a514bc4fd145b99333c28523e61b7a9ad051a245804323ebf4e7b1c6a6"}, + {file = "hiredis-3.3.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:db46baf157feefd88724e6a7f145fe996a5990a8604ed9292b45d563360e513b"}, + {file = "hiredis-3.3.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5e55d90b431b0c6b64ae5a624208d4aea318566d31872e595ee723c0f5b9a79f"}, + {file = "hiredis-3.3.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:40ae8a7041fcb328a6bc7202d8c4e6e0d38d434b2e3880b1ee8ed754f17cd836"}, + {file = "hiredis-3.3.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:d14229beaa76e66c3a25f9477d973336441ca820df853679a98796256813316f"}, + {file = "hiredis-3.3.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b3df9447f9209f9aa0434ca74050e9509670c1ad99398fe5807abb90e5f3a014"}, + {file = "hiredis-3.3.1-cp39-cp39-win32.whl", hash = "sha256:48ff424f8aa36aacd9fdaa68efeb27d2e8771f293af4305bdb15d92194ca6631"}, + {file = "hiredis-3.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:318f772dd321404075d406825266e574ee0f4751be1831424c2ebd5722609398"}, + {file = "hiredis-3.3.1.tar.gz", hash = "sha256:da6f0302360e99d32bc2869772692797ebadd536e1b826d0103c72ba49d38698"}, ] [[package]] @@ -2310,14 +2310,14 @@ typing-extensions = ">=4.14.1" [[package]] name = "pygithub" -version = "2.8.1" +version = "2.9.0" description = "Use the full Github API v3" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "pygithub-2.8.1-py3-none-any.whl", hash = "sha256:23a0a5bca93baef082e03411bf0ce27204c32be8bfa7abc92fe4a3e132936df0"}, - {file = "pygithub-2.8.1.tar.gz", hash = "sha256:341b7c78521cb07324ff670afd1baa2bf5c286f8d9fd302c1798ba594a5400c9"}, + {file = "pygithub-2.9.0-py3-none-any.whl", hash = "sha256:5e2b260ce327bffce9b00f447b65953ef7078ffe93e5a5425624a3075483927c"}, + {file = "pygithub-2.9.0.tar.gz", hash = "sha256:a26abda1222febba31238682634cad11d8b966137ed6cc3c5e445b29a11cb0a4"}, ] [package.dependencies] @@ -2554,14 +2554,14 @@ six = ">=1.5" [[package]] name = "python-multipart" -version = "0.0.22" +version = "0.0.26" description = "A streaming multipart parser for Python" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "python_multipart-0.0.22-py3-none-any.whl", hash = "sha256:2b2cd894c83d21bf49d702499531c7bafd057d730c201782048f7945d82de155"}, - {file = "python_multipart-0.0.22.tar.gz", hash = "sha256:7340bef99a7e0032613f56dc36027b959fd3b30a787ed62d310e951f7c3a3a58"}, + {file = "python_multipart-0.0.26-py3-none-any.whl", hash = "sha256:c0b169f8c4484c13b0dcf2ef0ec3a4adb255c4b7d18d8e420477d2b1dd03f185"}, + {file = "python_multipart-0.0.26.tar.gz", hash = "sha256:08fadc45918cd615e26846437f50c5d6d23304da32c341f289a617127b081f17"}, ] [[package]] @@ -2971,15 +2971,15 @@ doc = ["Sphinx", "sphinx-rtd-theme"] [[package]] name = "sentry-sdk" -version = "2.54.0" +version = "2.57.0" description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] markers = "extra == \"sentry\" or extra == \"all\"" files = [ - {file = "sentry_sdk-2.54.0-py2.py3-none-any.whl", hash = "sha256:fd74e0e281dcda63afff095d23ebcd6e97006102cdc8e78a29f19ecdf796a0de"}, - {file = "sentry_sdk-2.54.0.tar.gz", hash = "sha256:2620c2575128d009b11b20f7feb81e4e4e8ae08ec1d36cbc845705060b45cc1b"}, + {file = "sentry_sdk-2.57.0-py2.py3-none-any.whl", hash = "sha256:812c8bf5ff3d2f0e89c82f5ce80ab3a6423e102729c4706af7413fd1eb480585"}, + {file = "sentry_sdk-2.57.0.tar.gz", hash = "sha256:4be8d1e71c32fb27f79c577a337ac8912137bba4bcbc64a4ec1da4d6d8dc5199"}, ] [package.dependencies] @@ -2990,6 +2990,7 @@ urllib3 = ">=1.26.11" aiohttp = ["aiohttp (>=3.5)"] anthropic = ["anthropic (>=0.16)"] arq = ["arq (>=0.23)"] +asyncio = ["httpcore[asyncio] (==1.*)"] asyncpg = ["asyncpg (>=0.23)"] beam = ["apache-beam (>=2.12)"] bottle = ["bottle (>=0.12.13)"] @@ -3010,7 +3011,7 @@ huggingface-hub = ["huggingface_hub (>=0.22)"] langchain = ["langchain (>=0.0.210)"] langgraph = ["langgraph (>=0.6.6)"] launchdarkly = ["launchdarkly-server-sdk (>=9.8.0)"] -litellm = ["litellm (>=1.77.5)"] +litellm = ["litellm (>=1.77.5,!=1.82.7,!=1.82.8)"] litestar = ["litestar (>=2.0.0)"] loguru = ["loguru (>=0.5)"] mcp = ["mcp (>=1.15.0)"] @@ -3153,20 +3154,20 @@ files = [ [[package]] name = "sqlglot" -version = "29.0.1" +version = "30.2.1" description = "An easily customizable SQL parser and transpiler" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "sqlglot-29.0.1-py3-none-any.whl", hash = "sha256:06a473ea6c2b3632ac67bd38e687a6860265bf4156e66b54adeda15d07f00c65"}, - {file = "sqlglot-29.0.1.tar.gz", hash = "sha256:0010b4f77fb996c8d25dd4b16f3654e6da163ff1866ceabc70b24e791c203048"}, + {file = "sqlglot-30.2.1-py3-none-any.whl", hash = "sha256:f23d9ee9427ef9d20df15f9b0ffa57d9eb45e52b012219a349d1e6b50ed926d1"}, + {file = "sqlglot-30.2.1.tar.gz", hash = "sha256:ef4a67cc6f66a8043085eb8ea95fa9541c1625dffa9145ad4e9815a7ba60a199"}, ] [package.extras] -c = ["sqlglotc"] -dev = ["duckdb (>=0.6)", "mypy", "pandas", "pandas-stubs", "pdoc", "pre-commit", "pyperf", "python-dateutil", "pytz", "ruff (==0.7.2)", "types-python-dateutil", "types-pytz", "typing_extensions"] -rs = ["sqlglotrs (==0.13.0)"] +c = ["sqlglotc (==30.2.1)"] +dev = ["duckdb (>=0.6)", "pandas", "pandas-stubs", "pdoc", "pre-commit", "pyperf", "python-dateutil", "pytz", "ruff (==0.15.6)", "setuptools_scm", "sqlglot-mypy (>=1.19.1.post1)", "types-python-dateutil", "types-pytz", "typing_extensions"] +rs = ["sqlglotc (==30.2.1)", "sqlglotrs (==0.13.0)"] [[package]] name = "threadloop" @@ -3807,4 +3808,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<4.0.0" -content-hash = "515c310623717d6da6b6ab8ffc9aac9195ffdd8719826b371714431baea0b475" +content-hash = "9c51ee27ca9e648ca63c62eb89cd53ea88c0c8c87d52c9a4dd123efb56be6883" diff --git a/pyproject.toml b/pyproject.toml index c2d771b3818..14f68e6b8c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.151.0rc1" +version = "1.152.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ @@ -29,7 +29,10 @@ dependencies = [ # We require 2.0.0 for immutabledict support. "canonicaljson>=2.0.0,<3.0.0", # we use the type definitions added in signedjson 1.1. - "signedjson>=1.1.0,<2.0.0", + # 1.1.0 erroneously removed decode_verify_key_base64 (reintroduced in 1.1.1). + # 1.1.1 is mispackaged (importlib-metadata dependency without minimum version bound) + # 1.1.2, 1.1.3 and 1.1.4 were all released on the same day, so no good reason to use the older version. + "signedjson>=1.1.4,<2.0.0", # validating SSL certs for IP addresses requires service_identity 18.1. "service-identity>=18.1.0", # Twisted 18.9 introduces some logger improvements that the structured @@ -63,7 +66,7 @@ dependencies = [ "prometheus-client>=0.6.0", # we use `order`, which arrived in attrs 19.2.0. # Note: 21.1.0 broke `/sync`, see https://github.com/matrix-org/synapse/issues/9936 - "attrs>=19.2.0,!=21.1.0", + "attrs>=26.1.0,!=21.1.0", "netaddr>=0.7.18", # Jinja 2.x is incompatible with MarkupSafe>=2.1. To ensure that admins do not # end up with a broken installation, with recent MarkupSafe but old Jinja, we @@ -139,7 +142,7 @@ saml2 = [ "defusedxml>=0.7.1", # via pysaml2 "pytz>=2018.3", # via pysaml2 ] -oidc = ["authlib>=0.15.1"] +oidc = ["authlib>=1.6.11"] url-preview = ["lxml>=4.6.3"] sentry = ["sentry-sdk>=0.7.2"] opentracing = [ @@ -183,7 +186,7 @@ all = [ # saml2 "pysaml2>=4.5.0", # oidc and jwt - "authlib>=0.15.1", + "authlib>=1.6.11", # url-preview "lxml>=4.6.3", # sentry diff --git a/rust/src/events/internal_metadata.rs b/rust/src/events/internal_metadata.rs index a53c1e771b6..6fd3d06b00b 100644 --- a/rust/src/events/internal_metadata.rs +++ b/rust/src/events/internal_metadata.rs @@ -32,12 +32,16 @@ //! attributes, but for small number of keys is actually faster than using a //! hash or btree map. -use std::{num::NonZeroI64, ops::Deref}; +use std::{ + num::NonZeroI64, + ops::Deref, + sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}, +}; use anyhow::Context; use log::warn; use pyo3::{ - exceptions::PyAttributeError, + exceptions::{PyAttributeError, PyRuntimeError}, pybacked::PyBackedStr, pyclass, pymethods, types::{PyAnyMethods, PyDict, PyDictMethods, PyString}, @@ -55,11 +59,13 @@ enum EventInternalMetadataData { SoftFailed(bool), ProactivelySend(bool), PolicyServerSpammy(bool), + SpamCheckerSpammy(bool), Redacted(bool), TxnId(Box), DelayId(Box), TokenId(i64), DeviceId(Box), + CalculatedAuthEventIDs(Vec), // MSC4242: State DAGs } impl EventInternalMetadataData { @@ -105,6 +111,13 @@ impl EventInternalMetadataData { .to_owned() .into_any(), ), + EventInternalMetadataData::SpamCheckerSpammy(o) => ( + pyo3::intern!(py, "spam_checker_spammy"), + o.into_pyobject(py) + .unwrap_infallible() + .to_owned() + .into_any(), + ), EventInternalMetadataData::Redacted(o) => ( pyo3::intern!(py, "redacted"), o.into_pyobject(py) @@ -128,6 +141,10 @@ impl EventInternalMetadataData { pyo3::intern!(py, "device_id"), o.into_pyobject(py).unwrap_infallible().into_any(), ), + EventInternalMetadataData::CalculatedAuthEventIDs(o) => ( + pyo3::intern!(py, "calculated_auth_event_ids"), + o.into_pyobject(py).unwrap().into_any(), + ), } } @@ -173,6 +190,11 @@ impl EventInternalMetadataData { .extract() .with_context(|| format!("'{key_str}' has invalid type"))?, ), + "spam_checker_spammy" => EventInternalMetadataData::SpamCheckerSpammy( + value + .extract() + .with_context(|| format!("'{key_str}' has invalid type"))?, + ), "redacted" => EventInternalMetadataData::Redacted( value .extract() @@ -201,6 +223,11 @@ impl EventInternalMetadataData { .map(String::into_boxed_str) .with_context(|| format!("'{key_str}' has invalid type"))?, ), + "calculated_auth_event_ids" => EventInternalMetadataData::CalculatedAuthEventIDs( + value + .extract() + .with_context(|| format!("'{key_str}' has invalid type"))?, + ), _ => return Ok(None), }; @@ -222,19 +249,6 @@ macro_rules! get_property_opt { }; } -/// Helper macro to find the given field in internal metadata, raising an -/// attribute error if not found. -macro_rules! get_property { - ($self:expr, $name:ident) => { - get_property_opt!($self, $name).ok_or_else(|| { - PyAttributeError::new_err(format!( - "'EventInternalMetadata' has no attribute '{}'", - stringify!($name), - )) - }) - }; -} - /// Helper macro to set the give field. macro_rules! set_property { ($self:expr, $name:ident, $obj:expr) => { @@ -249,75 +263,28 @@ macro_rules! set_property { }; } -#[pyclass] +/// The inner data for `EventInternalMetadata`, containing all fields and +/// business logic with pure Rust types. #[derive(Clone)] -pub struct EventInternalMetadata { +struct EventInternalMetadataInner { /// The fields of internal metadata. This functions as a mapping. data: Vec, /// The stream ordering of this event. None, until it has been persisted. - #[pyo3(get, set)] - stream_ordering: Option, - #[pyo3(get, set)] - instance_name: Option, + pub stream_ordering: Option, + pub instance_name: Option, /// The event ID of the redaction event, if this event has been redacted. /// This is set dynamically at load time and is not persisted to the database. - #[pyo3(get, set)] - redacted_by: Option, + pub redacted_by: Option, /// whether this event is an outlier (ie, whether we have the state at that /// point in the DAG) - #[pyo3(get, set)] - outlier: bool, + pub outlier: bool, } -#[pymethods] -impl EventInternalMetadata { - #[new] - fn new(dict: &Bound<'_, PyDict>) -> PyResult { - let mut data = Vec::with_capacity(dict.len()); - - for (key, value) in dict.iter() { - match EventInternalMetadataData::from_python_pair(&key, &value) { - Ok(Some(entry)) => data.push(entry), - Ok(None) => {} - Err(err) => { - warn!("Ignoring internal metadata field '{key}', as failed to convert to Rust due to {err}") - } - } - } - - data.shrink_to_fit(); - - Ok(EventInternalMetadata { - data, - stream_ordering: None, - instance_name: None, - redacted_by: None, - outlier: false, - }) - } - - fn copy(&self) -> Self { - self.clone() - } - - /// Get a dict holding the data stored in the `internal_metadata` column in the database. - /// - /// Note that `outlier` and `stream_ordering` are stored in separate columns so are not returned here. - fn get_dict(&self, py: Python<'_>) -> PyResult> { - let dict = PyDict::new(py); - - for entry in &self.data { - let (key, value) = entry.to_python_pair(py); - dict.set_item(key, value)?; - } - - Ok(dict.into()) - } - - fn is_outlier(&self) -> bool { +impl EventInternalMetadataInner { + pub fn is_outlier(&self) -> bool { self.outlier } @@ -338,7 +305,7 @@ impl EventInternalMetadata { /// /// (Added in synapse 0.99.0, so may be unreliable for events received /// before that) - fn is_out_of_band_membership(&self) -> bool { + pub fn is_out_of_band_membership(&self) -> bool { get_property_opt!(self, OutOfBandMembership) .copied() .unwrap_or(false) @@ -350,9 +317,8 @@ impl EventInternalMetadata { /// /// returns a str with the name of the server this event is sent on behalf /// of. - fn get_send_on_behalf_of(&self) -> Option<&str> { - let s = get_property_opt!(self, SendOnBehalfOf); - s.map(|a| a.deref()) + pub fn get_send_on_behalf_of(&self) -> Option<&str> { + get_property_opt!(self, SendOnBehalfOf).map(|a| a.deref()) } /// Whether the redaction event needs to be rechecked when fetching @@ -363,7 +329,7 @@ impl EventInternalMetadata { /// /// If the sender of the redaction event is allowed to redact any event /// due to auth rules, then this will always return false. - fn need_to_check_redaction(&self) -> bool { + pub fn need_to_check_redaction(&self) -> bool { get_property_opt!(self, RecheckRedaction) .copied() .unwrap_or(false) @@ -376,7 +342,7 @@ impl EventInternalMetadata { /// clients. /// 2. They should not be added to the forward extremities (and therefore /// not to current state). - fn is_soft_failed(&self) -> bool { + pub fn is_soft_failed(&self) -> bool { get_property_opt!(self, SoftFailed) .copied() .unwrap_or(false) @@ -386,7 +352,7 @@ impl EventInternalMetadata { /// /// This is used for sending dummy events internally. Servers and clients /// can still explicitly fetch the event. - fn should_proactively_send(&self) -> bool { + pub fn should_proactively_send(&self) -> bool { get_property_opt!(self, ProactivelySend) .copied() .unwrap_or(true) @@ -396,129 +362,431 @@ impl EventInternalMetadata { /// /// This is used for efficiently checking whether an event has been marked /// as redacted without needing to make another database call. - fn is_redacted(&self) -> bool { + pub fn is_redacted(&self) -> bool { get_property_opt!(self, Redacted).copied().unwrap_or(false) } /// Whether this event can trigger a push notification - fn is_notifiable(&self) -> bool { + pub fn is_notifiable(&self) -> bool { !self.outlier || self.is_out_of_band_membership() } - // ** The following are the getters and setters of the various properties ** + pub fn get_out_of_band_membership(&self) -> Option { + get_property_opt!(self, OutOfBandMembership).copied() + } + + pub fn get_recheck_redaction(&self) -> Option { + get_property_opt!(self, RecheckRedaction).copied() + } + + pub fn get_soft_failed(&self) -> Option { + get_property_opt!(self, SoftFailed).copied() + } + + pub fn get_proactively_send(&self) -> Option { + get_property_opt!(self, ProactivelySend).copied() + } + + pub fn get_policy_server_spammy(&self) -> bool { + get_property_opt!(self, PolicyServerSpammy) + .copied() + .unwrap_or(false) + } + + pub fn get_redacted(&self) -> Option { + get_property_opt!(self, Redacted).copied() + } + + pub fn get_txn_id(&self) -> Option<&str> { + get_property_opt!(self, TxnId).map(|s| s.deref()) + } + + pub fn get_delay_id(&self) -> Option<&str> { + get_property_opt!(self, DelayId).map(|s| s.deref()) + } + + pub fn get_calculated_auth_event_ids(&self) -> Option<&Vec> { + get_property_opt!(self, CalculatedAuthEventIDs) + } + + pub fn get_token_id(&self) -> Option { + get_property_opt!(self, TokenId).copied() + } + + pub fn get_device_id(&self) -> Option<&str> { + get_property_opt!(self, DeviceId).map(|s| s.deref()) + } + + pub fn set_out_of_band_membership(&mut self, obj: bool) { + set_property!(self, OutOfBandMembership, obj); + } + + pub fn set_send_on_behalf_of(&mut self, obj: String) { + set_property!(self, SendOnBehalfOf, obj.into_boxed_str()); + } + + pub fn set_recheck_redaction(&mut self, obj: bool) { + set_property!(self, RecheckRedaction, obj); + } + + pub fn set_soft_failed(&mut self, obj: bool) { + set_property!(self, SoftFailed, obj); + } + + pub fn set_proactively_send(&mut self, obj: bool) { + set_property!(self, ProactivelySend, obj); + } + + pub fn set_policy_server_spammy(&mut self, obj: bool) { + set_property!(self, PolicyServerSpammy, obj); + } + + fn get_spam_checker_spammy(&self) -> bool { + get_property_opt!(self, SpamCheckerSpammy) + .copied() + .unwrap_or(false) + } + + fn set_spam_checker_spammy(&mut self, obj: bool) { + set_property!(self, SpamCheckerSpammy, obj); + } + + pub fn set_redacted(&mut self, obj: bool) { + set_property!(self, Redacted, obj); + } + + pub fn set_txn_id(&mut self, obj: String) { + set_property!(self, TxnId, obj.into_boxed_str()); + } + + pub fn set_delay_id(&mut self, obj: String) { + set_property!(self, DelayId, obj.into_boxed_str()); + } + + pub fn set_token_id(&mut self, obj: i64) { + set_property!(self, TokenId, obj); + } + + pub fn set_device_id(&mut self, obj: String) { + set_property!(self, DeviceId, obj.into_boxed_str()); + } + + pub fn set_calculated_auth_event_ids(&mut self, obj: Vec) { + set_property!(self, CalculatedAuthEventIDs, obj); + } +} + +#[pyclass(frozen)] +#[derive(Clone)] +pub struct EventInternalMetadata { + inner: Arc>, +} + +impl EventInternalMetadata { + fn read_inner(&self) -> PyResult> { + self.inner + .read() + .map_err(|_| PyRuntimeError::new_err("EventInternalMetadata lock poisoned")) + } + + /// Get a write lock on the inner data. + /// + /// Note that callers should be careful not to panic while holding the write + /// lock, as this will poison the lock.q + fn write_inner(&self) -> PyResult> { + self.inner + .write() + .map_err(|_| PyRuntimeError::new_err("EventInternalMetadata lock poisoned")) + } +} + +/// Helper to convert `None` to an `AttributeError` for a property getter. +fn attr_err(val: Option, name: &str) -> PyResult { + val.ok_or_else(|| { + PyAttributeError::new_err(format!("'EventInternalMetadata' has no attribute '{name}'",)) + }) +} + +#[pymethods] +impl EventInternalMetadata { + #[new] + fn new(dict: &Bound<'_, PyDict>) -> PyResult { + let mut data = Vec::with_capacity(dict.len()); + + for (key, value) in dict.iter() { + match EventInternalMetadataData::from_python_pair(&key, &value) { + Ok(Some(entry)) => data.push(entry), + Ok(None) => {} + Err(err) => { + warn!("Ignoring internal metadata field '{key}', as failed to convert to Rust due to {err}") + } + } + } + + data.shrink_to_fit(); + + Ok(EventInternalMetadata { + inner: Arc::new(RwLock::new(EventInternalMetadataInner { + data, + stream_ordering: None, + instance_name: None, + redacted_by: None, + outlier: false, + })), + }) + } + + fn copy(&self) -> PyResult { + let guard = self.read_inner()?; + Ok(EventInternalMetadata { + inner: Arc::new(RwLock::new(guard.clone())), + }) + } + + /// Get a dict holding the data stored in the `internal_metadata` column in + /// the database. + /// + /// Note that `outlier` and `stream_ordering` are stored in separate columns + /// so are not returned here. + fn get_dict(&self, py: Python<'_>) -> PyResult> { + let guard = self.read_inner()?; + let dict = PyDict::new(py); + + for entry in &guard.data { + let (key, value) = entry.to_python_pair(py); + dict.set_item(key, value)?; + } + + Ok(dict.into()) + } + + fn is_outlier(&self) -> PyResult { + Ok(self.read_inner()?.is_outlier()) + } + + fn is_out_of_band_membership(&self) -> PyResult { + Ok(self.read_inner()?.is_out_of_band_membership()) + } + + fn get_send_on_behalf_of(&self) -> PyResult> { + Ok(self + .read_inner()? + .get_send_on_behalf_of() + .map(|s| s.to_owned())) + } + + fn need_to_check_redaction(&self) -> PyResult { + Ok(self.read_inner()?.need_to_check_redaction()) + } + + fn is_soft_failed(&self) -> PyResult { + Ok(self.read_inner()?.is_soft_failed()) + } + + fn should_proactively_send(&self) -> PyResult { + Ok(self.read_inner()?.should_proactively_send()) + } + + fn is_redacted(&self) -> PyResult { + Ok(self.read_inner()?.is_redacted()) + } + + fn is_notifiable(&self) -> PyResult { + Ok(self.read_inner()?.is_notifiable()) + } + + #[getter] + fn get_stream_ordering(&self) -> PyResult> { + Ok(self.read_inner()?.stream_ordering) + } + #[setter] + fn set_stream_ordering(&self, val: Option) -> PyResult<()> { + self.write_inner()?.stream_ordering = val; + Ok(()) + } + + #[getter] + fn get_instance_name(&self) -> PyResult> { + Ok(self.read_inner()?.instance_name.clone()) + } + #[setter] + fn set_instance_name(&self, val: Option) -> PyResult<()> { + self.write_inner()?.instance_name = val; + Ok(()) + } + + #[getter] + fn get_redacted_by(&self) -> PyResult> { + Ok(self.read_inner()?.redacted_by.clone()) + } + #[setter] + fn set_redacted_by(&self, val: Option) -> PyResult<()> { + self.write_inner()?.redacted_by = val; + Ok(()) + } + + #[getter] + fn get_outlier(&self) -> PyResult { + Ok(self.read_inner()?.outlier) + } + #[setter] + fn set_outlier(&self, val: bool) -> PyResult<()> { + self.write_inner()?.outlier = val; + Ok(()) + } #[getter] fn get_out_of_band_membership(&self) -> PyResult { - let bool = get_property!(self, OutOfBandMembership)?; - Ok(*bool) + attr_err( + self.read_inner()?.get_out_of_band_membership(), + "out_of_band_membership", + ) } #[setter] - fn set_out_of_band_membership(&mut self, obj: bool) { - set_property!(self, OutOfBandMembership, obj); + fn set_out_of_band_membership(&self, obj: bool) -> PyResult<()> { + self.write_inner()?.set_out_of_band_membership(obj); + Ok(()) } #[getter(send_on_behalf_of)] - fn getter_send_on_behalf_of(&self) -> PyResult<&str> { - let s = get_property!(self, SendOnBehalfOf)?; - Ok(s) + fn getter_send_on_behalf_of(&self) -> PyResult { + let guard = self.read_inner()?; + attr_err( + guard.get_send_on_behalf_of().map(|s| s.to_owned()), + "send_on_behalf_of", + ) } #[setter] - fn set_send_on_behalf_of(&mut self, obj: String) { - set_property!(self, SendOnBehalfOf, obj.into_boxed_str()); + fn set_send_on_behalf_of(&self, obj: String) -> PyResult<()> { + self.write_inner()?.set_send_on_behalf_of(obj); + Ok(()) } #[getter] fn get_recheck_redaction(&self) -> PyResult { - let bool = get_property!(self, RecheckRedaction)?; - Ok(*bool) + attr_err( + self.read_inner()?.get_recheck_redaction(), + "recheck_redaction", + ) } #[setter] - fn set_recheck_redaction(&mut self, obj: bool) { - set_property!(self, RecheckRedaction, obj); + fn set_recheck_redaction(&self, obj: bool) -> PyResult<()> { + self.write_inner()?.set_recheck_redaction(obj); + Ok(()) } #[getter] fn get_soft_failed(&self) -> PyResult { - let bool = get_property!(self, SoftFailed)?; - Ok(*bool) + attr_err(self.read_inner()?.get_soft_failed(), "soft_failed") } #[setter] - fn set_soft_failed(&mut self, obj: bool) { - set_property!(self, SoftFailed, obj); + fn set_soft_failed(&self, obj: bool) -> PyResult<()> { + self.write_inner()?.set_soft_failed(obj); + Ok(()) } #[getter] fn get_proactively_send(&self) -> PyResult { - let bool = get_property!(self, ProactivelySend)?; - Ok(*bool) + attr_err( + self.read_inner()?.get_proactively_send(), + "proactively_send", + ) } #[setter] - fn set_proactively_send(&mut self, obj: bool) { - set_property!(self, ProactivelySend, obj); + fn set_proactively_send(&self, obj: bool) -> PyResult<()> { + self.write_inner()?.set_proactively_send(obj); + Ok(()) } #[getter] fn get_policy_server_spammy(&self) -> PyResult { - Ok(get_property_opt!(self, PolicyServerSpammy) - .copied() - .unwrap_or(false)) + Ok(self.read_inner()?.get_policy_server_spammy()) } #[setter] - fn set_policy_server_spammy(&mut self, obj: bool) { - set_property!(self, PolicyServerSpammy, obj); + fn set_policy_server_spammy(&self, obj: bool) -> PyResult<()> { + self.write_inner()?.set_policy_server_spammy(obj); + Ok(()) + } + + #[getter] + fn get_spam_checker_spammy(&self) -> PyResult { + Ok(self.read_inner()?.get_spam_checker_spammy()) + } + #[setter] + fn set_spam_checker_spammy(&self, obj: bool) -> PyResult<()> { + self.write_inner()?.set_spam_checker_spammy(obj); + Ok(()) } #[getter] fn get_redacted(&self) -> PyResult { - let bool = get_property!(self, Redacted)?; - Ok(*bool) + attr_err(self.read_inner()?.get_redacted(), "redacted") } #[setter] - fn set_redacted(&mut self, obj: bool) { - set_property!(self, Redacted, obj); + fn set_redacted(&self, obj: bool) -> PyResult<()> { + self.write_inner()?.set_redacted(obj); + Ok(()) } /// The transaction ID, if it was set when the event was created. #[getter] - fn get_txn_id(&self) -> PyResult<&str> { - let s = get_property!(self, TxnId)?; - Ok(s) + fn get_txn_id(&self) -> PyResult { + let guard = self.read_inner()?; + attr_err(guard.get_txn_id().map(|s| s.to_owned()), "txn_id") } #[setter] - fn set_txn_id(&mut self, obj: String) { - set_property!(self, TxnId, obj.into_boxed_str()); + fn set_txn_id(&self, obj: String) -> PyResult<()> { + self.write_inner()?.set_txn_id(obj); + Ok(()) + } + + /// The calculated auth event IDs, if it was set when the event was created. + #[getter] + fn get_calculated_auth_event_ids(&self) -> PyResult> { + let guard = self.read_inner()?; + attr_err( + guard.get_calculated_auth_event_ids().cloned(), + "calculated_auth_event_ids", + ) + } + #[setter] + fn set_calculated_auth_event_ids(&self, obj: Vec) -> PyResult<()> { + self.write_inner()?.set_calculated_auth_event_ids(obj); + Ok(()) } /// The delay ID, set only if the event was a delayed event. #[getter] - fn get_delay_id(&self) -> PyResult<&str> { - let s = get_property!(self, DelayId)?; - Ok(s) + fn get_delay_id(&self) -> PyResult { + let guard = self.read_inner()?; + attr_err(guard.get_delay_id().map(|s| s.to_owned()), "delay_id") } #[setter] - fn set_delay_id(&mut self, obj: String) { - set_property!(self, DelayId, obj.into_boxed_str()); + fn set_delay_id(&self, obj: String) -> PyResult<()> { + self.write_inner()?.set_delay_id(obj); + Ok(()) } /// The access token ID of the user who sent this event, if any. #[getter] fn get_token_id(&self) -> PyResult { - let r = get_property!(self, TokenId)?; - Ok(*r) + attr_err(self.read_inner()?.get_token_id(), "token_id") } #[setter] - fn set_token_id(&mut self, obj: i64) { - set_property!(self, TokenId, obj); + fn set_token_id(&self, obj: i64) -> PyResult<()> { + self.write_inner()?.set_token_id(obj); + Ok(()) } /// The device ID of the user who sent this event, if any. #[getter] - fn get_device_id(&self) -> PyResult<&str> { - let s = get_property!(self, DeviceId)?; - Ok(s) + fn get_device_id(&self) -> PyResult { + let guard = self.read_inner()?; + attr_err(guard.get_device_id().map(|s| s.to_owned()), "device_id") } #[setter] - fn set_device_id(&mut self, obj: String) { - set_property!(self, DeviceId, obj.into_boxed_str()); + fn set_device_id(&self, obj: String) -> PyResult<()> { + self.write_inner()?.set_device_id(obj); + Ok(()) } } diff --git a/rust/src/room_versions.rs b/rust/src/room_versions.rs index c0b304e18c9..dbc962174dd 100644 --- a/rust/src/room_versions.rs +++ b/rust/src/room_versions.rs @@ -47,6 +47,9 @@ impl EventFormatVersions { /// MSC4291 room IDs as hashes: introduced for room HydraV11 #[classattr] const ROOM_V11_HYDRA_PLUS: i32 = 4; + /// MSC4242 state DAGs: adds prev_state_events, removes auth_events + #[classattr] + const ROOM_VMSC4242: i32 = 5; } /// Enum to identify the state resolution algorithms. @@ -146,6 +149,14 @@ pub struct RoomVersion { /// /// In these room versions, we are stricter with event size validation. pub strict_event_byte_limits_room_versions: bool, + /// MSC4242: State DAGs. Creates events with prev_state_events instead of auth_events and derives + /// state from it. Events are always processed in causal order without any gaps in the DAG + /// (prev_state_events are always known), guaranteeing that processed events have a path to the + /// create event. This is an emergent property of state DAGs as asserting that there is a path + /// to the create event every time we insert an event would be prohibitively expensive. + /// This is similar to how doubly-linked lists can potentially not refer to previous items correctly + /// without verifying the list's integrity, but doing it on every insert is too expensive. + pub msc4242_state_dags: bool, } const ROOM_VERSION_V1: RoomVersion = RoomVersion { @@ -170,6 +181,7 @@ const ROOM_VERSION_V1: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; const ROOM_VERSION_V2: RoomVersion = RoomVersion { @@ -194,6 +206,7 @@ const ROOM_VERSION_V2: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; const ROOM_VERSION_V3: RoomVersion = RoomVersion { @@ -218,6 +231,7 @@ const ROOM_VERSION_V3: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; const ROOM_VERSION_V4: RoomVersion = RoomVersion { @@ -242,6 +256,7 @@ const ROOM_VERSION_V4: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; const ROOM_VERSION_V5: RoomVersion = RoomVersion { @@ -266,6 +281,7 @@ const ROOM_VERSION_V5: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; const ROOM_VERSION_V6: RoomVersion = RoomVersion { @@ -290,6 +306,7 @@ const ROOM_VERSION_V6: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; const ROOM_VERSION_V7: RoomVersion = RoomVersion { @@ -314,6 +331,7 @@ const ROOM_VERSION_V7: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; const ROOM_VERSION_V8: RoomVersion = RoomVersion { @@ -338,6 +356,7 @@ const ROOM_VERSION_V8: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; const ROOM_VERSION_V9: RoomVersion = RoomVersion { @@ -362,6 +381,7 @@ const ROOM_VERSION_V9: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; const ROOM_VERSION_V10: RoomVersion = RoomVersion { @@ -386,6 +406,7 @@ const ROOM_VERSION_V10: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; /// MSC3389 (Redaction changes for events with a relation) based on room version "10". @@ -411,6 +432,7 @@ const ROOM_VERSION_MSC3389V10: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: true, + msc4242_state_dags: false, }; /// MSC1767 (Extensible Events) based on room version "10". @@ -436,6 +458,7 @@ const ROOM_VERSION_MSC1767V10: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; /// MSC3757 (Restricting who can overwrite a state event) based on room version "10". @@ -461,6 +484,7 @@ const ROOM_VERSION_MSC3757V10: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; const ROOM_VERSION_V11: RoomVersion = RoomVersion { @@ -485,6 +509,7 @@ const ROOM_VERSION_V11: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: true, // Changed from v10 + msc4242_state_dags: false, }; /// MSC3757 (Restricting who can overwrite a state event) based on room version "11". @@ -510,6 +535,7 @@ const ROOM_VERSION_MSC3757V11: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: true, + msc4242_state_dags: false, }; const ROOM_VERSION_HYDRA_V11: RoomVersion = RoomVersion { @@ -534,6 +560,7 @@ const ROOM_VERSION_HYDRA_V11: RoomVersion = RoomVersion { msc4289_creator_power_enabled: true, // Changed from v11 msc4291_room_ids_as_hashes: true, // Changed from v11 strict_event_byte_limits_room_versions: true, + msc4242_state_dags: false, }; const ROOM_VERSION_V12: RoomVersion = RoomVersion { @@ -558,6 +585,32 @@ const ROOM_VERSION_V12: RoomVersion = RoomVersion { msc4289_creator_power_enabled: true, // Changed from v11 msc4291_room_ids_as_hashes: true, // Changed from v11 strict_event_byte_limits_room_versions: true, + msc4242_state_dags: false, +}; + +const ROOM_VERSION_MSC4242V12: RoomVersion = RoomVersion { + identifier: "org.matrix.msc4242.12", + disposition: RoomDisposition::UNSTABLE, + event_format: EventFormatVersions::ROOM_VMSC4242, + state_res: StateResolutionVersions::V2_1, + enforce_key_validity: true, + special_case_aliases_auth: false, + strict_canonicaljson: true, + limit_notifications_power_levels: true, + implicit_room_creator: true, + updated_redaction_rules: true, + restricted_join_rule: true, + restricted_join_rule_fix: true, + knock_join_rule: true, + msc3389_relation_redactions: false, + knock_restricted_join_rule: true, + enforce_int_power_levels: true, + msc3931_push_features: &[], + msc3757_enabled: false, + msc4289_creator_power_enabled: true, + msc4291_room_ids_as_hashes: true, + strict_event_byte_limits_room_versions: true, + msc4242_state_dags: true, }; /// Helper class for managing the known room versions, and providing dict-like @@ -600,7 +653,12 @@ impl KnownRoomVersionsMapping { Ok(()) } - fn __getitem__(&self, key: &str) -> PyResult { + fn __getitem__(&self, key: &Bound<'_, PyAny>) -> PyResult { + // We need to accept anything as the key, but we know that only strings + // are valid keys, so if it's not a string we just raise a KeyError. + let Ok(key) = key.extract::<&str>() else { + return Err(PyKeyError::new_err(key.to_string())); + }; let versions = self.versions.read().unwrap(); versions .iter() @@ -609,9 +667,14 @@ impl KnownRoomVersionsMapping { .ok_or_else(|| PyKeyError::new_err(key.to_string())) } - fn __contains__(&self, key: &str) -> PyResult { + fn __contains__(&self, key: &Bound<'_, PyAny>) -> bool { + // We need to accept anything as the key, but we know that only strings + // are valid keys, so if it's not a string we just return false. + let Ok(key) = key.extract::<&str>() else { + return false; + }; let versions = self.versions.read().unwrap(); - Ok(versions.iter().any(|v| v.identifier == key)) + versions.iter().any(|v| v.identifier == key) } fn keys(&self) -> PyResult> { @@ -790,6 +853,10 @@ impl RoomVersions { fn V12(py: Python<'_>) -> PyResult> { ROOM_VERSION_V12.into_py_any(py) } + #[classattr] + fn MSC4242v12(py: Python<'_>) -> PyResult> { + ROOM_VERSION_MSC4242V12.into_py_any(py) + } } /// Called when registering modules with python. @@ -804,11 +871,12 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> child_module.add_class::()?; // Build KNOWN_EVENT_FORMAT_VERSIONS as a frozenset - let known_ef: [i32; 4] = [ + let known_ef: [i32; 5] = [ EventFormatVersions::ROOM_V1_V2, EventFormatVersions::ROOM_V3, EventFormatVersions::ROOM_V4_PLUS, EventFormatVersions::ROOM_V11_HYDRA_PLUS, + EventFormatVersions::ROOM_VMSC4242, ]; let known_event_format_versions = PyFrozenSet::new(py, known_ef)?; child_module.add("KNOWN_EVENT_FORMAT_VERSIONS", known_event_format_versions)?; diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index f16aac81b2c..6bb7932fb1d 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/schema/synapse/v1.151/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.152/synapse-config.schema.json type: object properties: modules: @@ -94,6 +94,10 @@ properties: Determines whether we should freeze the internal dict object in `FrozenEvent`. Freezing prevents bugs where we accidentally share e.g. signature dicts. However, freezing a dict is expensive. + + + > ⚠️ **Warning** – This option is known to introduce a new class of [comparison + bugs](https://github.com/element-hq/synapse/issues/18117) in Synapse. default: false examples: - true diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 6ccba4d728d..eb5ac2cb1b0 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -136,6 +136,7 @@ "users": ["shadow_banned", "approved", "locked", "suspended"], "un_partial_stated_event_stream": ["rejection_status_changed"], "users_who_share_rooms": ["share_private"], + "quarantined_media_changes": ["quarantined"], } @@ -912,6 +913,10 @@ def alter_table(txn: LoggingTransaction) -> None: await self._setup_autoincrement_sequence( "state_groups_pending_deletion", "sequence_number" ) + await self._setup_sequence( + "quarantined_media_id_seq", + [("quarantined_media_changes", "stream_id")], + ) # Step 3. Get tables. self.progress.set_state("Fetching tables") diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index 0614c805dad..67189e91e72 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -34,6 +34,7 @@ from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging from synapse.events import EventBase +from synapse.events.utils import FilteredEvent from synapse.handlers.admin import ExfiltrationWriter from synapse.server import HomeServer from synapse.storage.database import DatabasePool, LoggingDatabaseConnection @@ -150,14 +151,14 @@ def __init__(self, user_id: str, directory: str | None = None): if list(os.listdir(self.base_directory)): raise Exception("Directory must be empty") - def write_events(self, room_id: str, events: list[EventBase]) -> None: + def write_events(self, room_id: str, filtered_events: list[FilteredEvent]) -> None: room_directory = os.path.join(self.base_directory, "rooms", room_id) os.makedirs(room_directory, exist_ok=True) events_file = os.path.join(room_directory, "events") with open(events_file, "a") as f: - for event in events: - json.dump(event.get_pdu_json(), fp=f) + for filtered_event in filtered_events: + json.dump(filtered_event.event.get_pdu_json(), fp=f) def write_state( self, room_id: str, event_id: str, state: StateMap[EventBase] @@ -175,7 +176,7 @@ def write_state( def write_invite( self, room_id: str, event: EventBase, state: StateMap[EventBase] ) -> None: - self.write_events(room_id, [event]) + self.write_events(room_id, [FilteredEvent.state(event)]) # We write the invite state somewhere else as they aren't full events # and are only a subset of the state at the event. @@ -191,7 +192,7 @@ def write_invite( def write_knock( self, room_id: str, event: EventBase, state: StateMap[EventBase] ) -> None: - self.write_events(room_id, [event]) + self.write_events(room_id, [FilteredEvent.state(event)]) # We write the knock state somewhere else as they aren't full events # and are only a subset of the state at the event. diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index d4e9d50b966..66c962e17db 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -40,7 +40,7 @@ TransactionUnusedFallbackKeys, ) from synapse.events import EventBase -from synapse.events.utils import SerializeEventConfig +from synapse.events.utils import FilteredEvent, SerializeEventConfig from synapse.http.client import SimpleHttpClient, is_unknown_endpoint from synapse.logging import opentracing from synapse.metrics import SERVER_NAME_LABEL @@ -545,7 +545,7 @@ async def _serialize( ) -> list[JsonDict]: time_now = self.clock.time_msec() return await self._event_serializer.serialize_events( - list(events), + [FilteredEvent(event=e, membership=None) for e in events], time_now, config=SerializeEventConfig( as_client_event=True, diff --git a/synapse/config/api.py b/synapse/config/api.py index 03b92249a96..b5979a089d2 100644 --- a/synapse/config/api.py +++ b/synapse/config/api.py @@ -48,6 +48,9 @@ def _get_prejoin_state_entries( self, config: JsonDict ) -> Iterable[tuple[str, str | None]]: """Get the event types and state keys to include in the prejoin state.""" + # MSC4311: the create event must always be included in invite/knock state. + yield EventTypes.Create, "" + room_prejoin_state_config = config.get("room_prejoin_state") or {} # backwards-compatibility support for room_invite_state_types diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index e77412a7974..f7c452bc73b 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -386,9 +386,6 @@ def read_config( # MSC3814 (dehydrated devices with SSSS) self.msc3814_enabled: bool = experimental.get("msc3814_enabled", False) - # MSC3266 (room summary api) - self.msc3266_enabled: bool = experimental.get("msc3266_enabled", False) - # MSC2409 (this setting only relates to optionally sending to-device messages). # Presence, typing and read receipt EDUs are already sent to application services that # have opted in to receive them. If enabled, this adds to-device messages to that list. @@ -479,6 +476,12 @@ def read_config( # Enable room version (and thus applicable push rules from MSC3931/3932) KNOWN_ROOM_VERSIONS.add_room_version(RoomVersions.MSC1767v10) + # MSC4242: State DAGs + self.msc4242_enabled: bool = experimental.get("msc4242_enabled", False) + if self.msc4242_enabled: + # Enable the room version + KNOWN_ROOM_VERSIONS.add_room_version(RoomVersions.MSC4242v12) + # MSC3391: Removing account data. self.msc3391_enabled = experimental.get("msc3391_enabled", False) @@ -604,3 +607,9 @@ def read_config( # Note that sticky events persisted before this feature is enabled will not be # considered sticky by the local homeserver. self.msc4354_enabled: bool = experimental.get("msc4354_enabled", False) + + # MSC4450: Identity Provider selection for User-Interactive Authentication + # with Legacy Single Sign-On (`m.login.sso`) + # Tracked in: https://github.com/element-hq/synapse/issues/19691 + # Note that this is only applicable to legacy auth, not MAS integration (OAuth 2.0). + self.msc4450_enabled: bool = experimental.get("msc4450_enabled", False) diff --git a/synapse/config/workers.py b/synapse/config/workers.py index 996be88cb26..fb7378bfc81 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -142,6 +142,8 @@ class WriterLocations: push_rules: The instances that write to the push stream. Currently can only be a single instance. device_lists: The instances that write to the device list stream. + quarantined_media_changes: The instances that write to the quarantined media + changes stream. """ events: list[str] = attr.ib( @@ -180,6 +182,10 @@ class WriterLocations: default=["master"], converter=_instance_to_list_converter, ) + quarantined_media_changes: list[str] = attr.ib( + default=[MAIN_PROCESS_INSTANCE_NAME], + converter=_instance_to_list_converter, + ) @attr.s(auto_attribs=True) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index d13d5d04c3b..d789c06a9cf 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -27,7 +27,7 @@ from canonicaljson import encode_canonical_json from signedjson.sign import sign_json -from signedjson.types import SigningKey +from signedjson.types import SigningKey, VerifyKey from unpaddedbase64 import decode_base64, encode_base64 from synapse.api.errors import Codes, SynapseError @@ -35,7 +35,7 @@ from synapse.events import EventBase from synapse.events.utils import prune_event, prune_event_dict from synapse.logging.opentracing import trace -from synapse.types import JsonDict +from synapse.types import JsonDict, UserID logger = logging.getLogger(__name__) @@ -192,3 +192,54 @@ def add_hashes_and_signatures( event_dict["signatures"] = compute_event_signature( room_version, event_dict, signature_name=signature_name, signing_key=signing_key ) + + +def resign_event( + ev: EventBase, + server_name: str, + signing_key: SigningKey, + time_now: int | None = None, +) -> JsonDict: + """Re-sign the provided event with the given signing key. Any existing signatures on the event + for this server_name are removed. + + If there has been no signature for this event by this server_name, the event is still re-signed. + If there have been signatures on this event by this server_name, the event is not re-checked for + validity. As such, only events that have valid signatures should be passed into this function + e.g. from the event_json table in the database. + """ + event_dict = ev.get_pdu_json(time_now=time_now) + event_dict["signatures"].pop( + server_name, None + ) # remove existing signatures for this server_name + event_dict["signatures"].update( + compute_event_signature( + ev.room_version, + event_dict, + server_name, + signing_key, + ) + ) + return event_dict + + +def event_needs_resigning( + ev: EventBase, server_name: str, verify_key: VerifyKey +) -> bool: + """Check if this event needs re-signing. + + This returns True if all of the following are True: + - the event `sender` domain matches the `server_name` provided. + - the event has not been already signed with this `verify_key`. + """ + sender = UserID.from_string(ev.sender) + if sender.domain != server_name: + return False + want_key_id = verify_key.alg + ":" + verify_key.version + signed_with_current_key_id = ev.signatures.get(server_name, {}).get( + want_key_id, None + ) + if signed_with_current_key_id: + return False + + return True diff --git a/synapse/event_auth.py b/synapse/event_auth.py index f39f55b472d..ca528ae2358 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -61,7 +61,7 @@ EventFormatVersions, RoomVersion, ) -from synapse.events import is_creator +from synapse.events import FrozenEventVMSC4242, is_creator from synapse.state import CREATE_KEY from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.types import ( @@ -186,6 +186,70 @@ async def check_state_independent_auth_rules( # 1.5 Otherwise, allow return + # State DAGs 2. Considering the event's prev_state_events: + if event.room_version.msc4242_state_dags: + prev_state_events_ids = set(cast(FrozenEventVMSC4242, event).prev_state_events) + # Fetch all of the `prev_state_events` + prev_state_events = {} + # Try to load the `prev_state_events` from `batched_auth_events` initially as + # that can save us a database hit. + if batched_auth_events is not None: + prev_state_events = { + event_id: value + for event_id in prev_state_events_ids + if (value := batched_auth_events.get(event_id)) is not None + } + # Fetch the rest of the `prev_state_events` + missing_prev_state_events_ids = prev_state_events_ids - set( + prev_state_events.keys() + ) + fetched_prev_state_events = await store.get_events( + missing_prev_state_events_ids, + redact_behaviour=EventRedactBehaviour.as_is, + allow_rejected=True, + ) + prev_state_events.update(fetched_prev_state_events) + if len(prev_state_events) != len(prev_state_events_ids): + # we should have all the `prev_state_events` by now, so if we do not, that suggests + # a Synapse programming error + known_prev_state_event_ids = set(prev_state_events) + raise AssertionError( + f"Event {event.event_id} has unknown prev_state_events " + + f"({len(prev_state_events)}/{len(prev_state_events_ids)} known)" + + f"{prev_state_events_ids - known_prev_state_event_ids} missing " + + f"out of {prev_state_events_ids}" + ) + for prev_state_event in prev_state_events.values(): + # 2.1 If there are entries which do not belong in the same room, reject. + if prev_state_event.room_id != event.room_id: + raise AuthError( + 403, + "During auth for event %s in room %s, found event %s in prev_state_events " + "which belongs to a different room %s" + % ( + event.event_id, + event.room_id, + prev_state_event.event_id, + prev_state_event.room_id, + ), + ) + # 2.2 If there are entries which do not have a state_key, reject. + if not prev_state_event.is_state(): + raise AuthError( + 403, + f"During auth for event {event.event_id} in room {event.room_id}, event has a " + + f"prev_state_event which is not state: {prev_state_event.event_id}", + ) + # 2.3 If there are entries which were themselves rejected under the checks performed on + # receipt of a PDU, reject. + if prev_state_event.rejected_reason is not None: + raise AuthError( + 403, + f"During auth for event {event.event_id} in room {event.room_id}, event has a " + + f"prev_state_event which is rejected ({prev_state_event.rejected_reason}): " + + f"{prev_state_event.event_id}", + ) + # 2. Reject if event has auth_events that: ... auth_events: ChainMap[str, EventBase] = ChainMap() if batched_auth_events: @@ -358,7 +422,7 @@ def check_state_dependent_auth_rules( # a user is allowed to issue invites. Fixes # https://github.com/vector-im/vector-web/issues/1208 hopefully if event.type == EventTypes.ThirdPartyInvite: - user_level = get_user_power_level(event.user_id, auth_dict) + user_level = get_user_power_level(event.sender, auth_dict) invite_level = get_named_level(auth_dict, "invite", 0) if user_level < invite_level: @@ -409,8 +473,8 @@ def _check_size_limits(event: "EventBase") -> None: # Codepoint size check: Synapse always enforced these limits, so apply # them strictly. - if len(event.user_id) > 255: - raise EventSizeError("'user_id' too large", unpersistable=True) + if len(event.sender) > 255: + raise EventSizeError("'sender' too large", unpersistable=True) if len(event.room_id) > 255: raise EventSizeError("'room_id' too large", unpersistable=True) if event.is_state() and len(event.state_key) > 255: @@ -423,8 +487,8 @@ def _check_size_limits(event: "EventBase") -> None: strict_byte_limits = event.room_version.strict_event_byte_limits_room_versions # Byte size check: if these fail, then be lenient to avoid breaking rooms. - if len(event.user_id.encode("utf-8")) > 255: - raise EventSizeError("'user_id' too large", unpersistable=strict_byte_limits) + if len(event.sender.encode("utf-8")) > 255: + raise EventSizeError("'sender' too large", unpersistable=strict_byte_limits) if len(event.room_id.encode("utf-8")) > 255: raise EventSizeError("'room_id' too large", unpersistable=strict_byte_limits) if event.is_state() and len(event.state_key.encode("utf-8")) > 255: @@ -450,6 +514,12 @@ def _check_create(event: "EventBase") -> None: if event.prev_event_ids(): raise AuthError(403, "Create event has prev events") + # State DAGs 1.2 If it has any prev_state_events, reject. + if event.room_version.msc4242_state_dags: + assert isinstance(event, FrozenEventVMSC4242) + if len(event.prev_state_events) > 0: + raise AuthError(403, "Create event has prev state events") + if event.room_version.msc4291_room_ids_as_hashes: # 1.2 If the create event has a room_id, reject if "room_id" in event: @@ -544,7 +614,7 @@ def _is_membership_change_allowed( raise AuthError(403, "This room has been marked as unfederatable.") # get info about the caller - key = (EventTypes.Member, event.user_id) + key = (EventTypes.Member, event.sender) caller = auth_events.get(key) caller_in_room = caller and caller.membership == Membership.JOIN @@ -569,7 +639,7 @@ def _is_membership_change_allowed( else: join_rule = JoinRules.INVITE - user_level = get_user_power_level(event.user_id, auth_events) + user_level = get_user_power_level(event.sender, auth_events) target_level = get_user_power_level(target_user_id, auth_events) invite_level = get_named_level(auth_events, "invite", 0) @@ -587,7 +657,7 @@ def _is_membership_change_allowed( "membership": membership, "join_rule": join_rule, "target_user_id": target_user_id, - "event.user_id": event.user_id, + "event.sender": event.sender, }, ) @@ -607,14 +677,14 @@ def _is_membership_change_allowed( if ( (caller_invited or caller_knocked) and Membership.LEAVE == membership - and target_user_id == event.user_id + and target_user_id == event.sender ): return if not caller_in_room: # caller isn't joined raise UnstableSpecAuthError( 403, - "%s not in room %s." % (event.user_id, event.room_id), + "%s not in room %s." % (event.sender, event.room_id), errcode=Codes.NOT_JOINED, ) @@ -645,7 +715,7 @@ def _is_membership_change_allowed( # * They are already joined (it's a NOOP). # * The room is public. # * The room is restricted and the user meets the allows rules. - if event.user_id != target_user_id: + if event.sender != target_user_id: raise AuthError(403, "Cannot force another user to join.") elif target_banned: raise AuthError(403, "You are banned from this room") @@ -705,7 +775,7 @@ def _is_membership_change_allowed( "You cannot unban user %s." % (target_user_id,), errcode=Codes.INSUFFICIENT_POWER, ) - elif target_user_id != event.user_id: + elif target_user_id != event.sender: kick_level = get_named_level(auth_events, "kick", 50) if user_level < kick_level or user_level <= target_level: @@ -733,7 +803,7 @@ def _is_membership_change_allowed( or join_rule != JoinRules.KNOCK_RESTRICTED ): raise AuthError(403, "You don't have permission to knock") - elif target_user_id != event.user_id: + elif target_user_id != event.sender: raise AuthError(403, "You cannot knock for other users") elif target_in_room: raise UnstableSpecAuthError( @@ -752,10 +822,10 @@ def _is_membership_change_allowed( def _check_event_sender_in_room( event: "EventBase", auth_events: StateMap["EventBase"] ) -> None: - key = (EventTypes.Member, event.user_id) + key = (EventTypes.Member, event.sender) member_event = auth_events.get(key) - _check_joined_room(member_event, event.user_id, event.room_id) + _check_joined_room(member_event, event.sender, event.room_id) def _check_joined_room( @@ -809,7 +879,7 @@ def _can_send_event(event: "EventBase", auth_events: StateMap["EventBase"]) -> b power_levels_event = get_power_level_event(auth_events) send_level = get_send_level(event.type, state_key, power_levels_event) - user_level = get_user_power_level(event.user_id, auth_events) + user_level = get_user_power_level(event.sender, auth_events) if user_level < send_level: raise UnstableSpecAuthError( @@ -822,7 +892,7 @@ def _can_send_event(event: "EventBase", auth_events: StateMap["EventBase"]) -> b if ( state_key is not None and state_key.startswith("@") - and state_key != event.user_id + and state_key != event.sender ): if event.room_version.msc3757_enabled: try: @@ -841,7 +911,7 @@ def _can_send_event(event: "EventBase", auth_events: StateMap["EventBase"]) -> b ) if ( # sender is owner of the state key - state_key_user_id == event.user_id + state_key_user_id == event.sender # sender has higher PL than the owner of the state key or user_level > get_user_power_level(state_key_user_id, auth_events) ): @@ -868,7 +938,7 @@ def check_redaction( AuthError if the event sender is definitely not allowed to redact the target event. """ - user_level = get_user_power_level(event.user_id, auth_events) + user_level = get_user_power_level(event.sender, auth_events) redact_level = get_named_level(auth_events, "redact", 50) @@ -966,7 +1036,7 @@ def _check_power_levels( if not current_state: return - user_level = get_user_power_level(event.user_id, auth_events) + user_level = get_user_power_level(event.sender, auth_events) # Check other levels: levels_to_check: list[tuple[str, str | None]] = [ @@ -1020,7 +1090,7 @@ def _check_power_levels( if new_level == old_level: continue - if dir == "users" and level_to_check != event.user_id: + if dir == "users" and level_to_check != event.sender: if old_level == user_level: raise AuthError( 403, @@ -1137,7 +1207,7 @@ def _verify_third_party_invite( if invite_event.sender != event.sender: return False - if event.user_id != invite_event.user_id: + if event.sender != invite_event.sender: return False if signed["mxid"] != event.state_key: diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index e6162997dd7..ac9b31cfcf3 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -34,6 +34,7 @@ ) import attr +from typing_extensions import deprecated from unpaddedbase64 import encode_base64 from synapse.api.constants import ( @@ -44,10 +45,7 @@ ) from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions from synapse.synapse_rust.events import EventInternalMetadata -from synapse.types import ( - JsonDict, - StrCollection, -) +from synapse.types import JsonDict, StateKey, StrCollection from synapse.util.caches import intern_dict from synapse.util.duration import Duration from synapse.util.frozenutils import freeze @@ -65,6 +63,10 @@ NOTE: This is overridden by the configuration by the Synapse worker apps, but for the sake of tests, it is set here because it cannot be configured on the homeserver object itself. + +FIXME: Because of how this option works (changing the underlying types), it causes +subtle downstream bugs that makes type comparisons brittle, tracked by +https://github.com/element-hq/synapse/issues/18117 """ T = TypeVar("T") @@ -221,6 +223,8 @@ def __init__( # get_state_key() (and a check for None). state_key: DictProperty[str] = DictProperty("state_key") type: DictProperty[str] = DictProperty("type") + + # This is a deprecated property, use `sender` instead. Only used by modules. user_id: DictProperty[str] = DictProperty("sender") @property @@ -288,9 +292,6 @@ def get_templated_pdu_json(self) -> JsonDict: return template_json - def __getitem__(self, field: str) -> Any | None: - return self._dict[field] - def __contains__(self, field: str) -> bool: return field in self._dict @@ -367,6 +368,11 @@ def __repr__(self) -> str: ">" ) + # Using `__getitem__` is deprecated. Only used by modules. + @deprecated("Use attribute access instead") + def __getitem__(self, field: str) -> Any | None: + return self._dict[field] + class FrozenEvent(EventBase): format_version = EventFormatVersions.ROOM_V1_V2 # All events of this type are V1 @@ -579,9 +585,60 @@ def auth_event_ids(self) -> StrCollection: return [*self._dict["auth_events"], create_event_id] +class FrozenEventVMSC4242(FrozenEventV4): + """FrozenEventVMSC4242, which differs from FrozenEventV4 only in the addition of prev_state_events""" + + format_version = EventFormatVersions.ROOM_VMSC4242 + prev_state_events: DictProperty[list[str]] = DictProperty("prev_state_events") + + def __init__( + self, + event_dict: JsonDict, + room_version: RoomVersion, + internal_metadata_dict: JsonDict | None = None, + rejected_reason: str | None = None, + ): + # Similar to how we assert event_id isn't in V2+ events, we do the same with auth_events. + # We don't expect `auth_events` in the wire format because we calculate it from prev_state_events. + assert "auth_events" not in event_dict + super().__init__( + event_dict=event_dict, + room_version=room_version, + internal_metadata_dict=internal_metadata_dict, + rejected_reason=rejected_reason, + ) + + def auth_event_ids(self) -> StrCollection: + """Returns the list of _calculated_ auth event IDs. + + Returns: + The list of event IDs of this event's auth events + """ + # Catches cases where we accidentally call auth_event_ids() prior to calculating what they + # actually are. The exception being the m.room.create event which has no auth events. + if self.type != EventTypes.Create: + assert len(self.internal_metadata.calculated_auth_event_ids) > 0 + return self.internal_metadata.calculated_auth_event_ids + + def __repr__(self) -> str: + rejection = f"REJECTED={self.rejected_reason}, " if self.rejected_reason else "" + + return ( + f"<{self.__class__.__name__} " + f"{rejection}" + f"event_id={self.event_id}, " + f"type={self.get('type')}, " + f"state_key={self.get('state_key')}, " + f"prev_events={self.get('prev_events')}, " + f"prev_state_events={self.get('prev_state_events')}, " + f"outlier={self.internal_metadata.is_outlier()}" + ">" + ) + + def _event_type_from_format_version( format_version: int, -) -> type[FrozenEvent | FrozenEventV2 | FrozenEventV3]: +) -> type[FrozenEvent | FrozenEventV2 | FrozenEventV3 | FrozenEventVMSC4242]: """Returns the python type to use to construct an Event object for the given event format version. @@ -598,6 +655,8 @@ def _event_type_from_format_version( return FrozenEventV2 elif format_version == EventFormatVersions.ROOM_V4_PLUS: return FrozenEventV3 + elif format_version == EventFormatVersions.ROOM_VMSC4242: + return FrozenEventVMSC4242 elif format_version == EventFormatVersions.ROOM_V11_HYDRA_PLUS: return FrozenEventV4 else: @@ -659,6 +718,24 @@ def relation_from_event(event: EventBase) -> _EventRelation | None: return _EventRelation(parent_id, rel_type, aggregation_key) +def event_exists_in_state_dag( + event: Union["EventBase", "EventBuilder", "EventMetadata", "StateKey"], +) -> bool: + """Given an event, returns true if this event should form part of the state DAG. + Only valid for room versions which use a state DAG (MSC4242).""" + state_key = None + if isinstance(event, EventMetadata): + state_key = event.state_key + elif isinstance(event, tuple): # StateKey + # can't use StateKey else you get: + # "Subscripted generics cannot be used with class and instance checks" + state_key = event[1] + else: + state_key = event.state_key if event.is_state() else None + + return state_key is not None + + def is_creator(create: EventBase, user_id: str) -> bool: """ Return true if the provided user ID is the room creator. @@ -693,3 +770,13 @@ class StrippedStateEvent: state_key: str sender: str content: dict[str, Any] + + +@attr.s(slots=True, frozen=True, auto_attribs=True) +class EventMetadata: + """Returned by `get_metadata_for_events`""" + + room_id: str + event_type: str + state_key: str | None + rejection_reason: str | None diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 2cd1bf6106f..78eb98e1e59 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -132,6 +132,7 @@ async def build( prev_event_ids: list[str], auth_event_ids: list[str] | None, depth: int | None = None, + prev_state_events: list[str] | None = None, ) -> EventBase: """Transform into a fully signed and hashed event @@ -143,10 +144,51 @@ async def build( depth: Override the depth used to order the event in the DAG. Should normally be set to None, which will cause the depth to be calculated based on the prev_events. - + prev_state_events: The event IDs to use as prev_state_events. + Only applicable on MSC4242 state DAG rooms. If this is supplied, auth_event_ids + must not be specified unless this event is part of a batch such that the builder + will be unable to compute the auth_event_ids due to the events not being persisted + yet. Returns: The signed and hashed event. """ + # If the caller specifies this, make sure the room version supports it. + if prev_state_events: + assert self.room_version.msc4242_state_dags + if self.room_version.msc4242_state_dags: + assert prev_state_events is not None + if self.room_id: + state_ids = await self._state.compute_state_after_events( + self.room_id, + prev_state_events, + state_filter=StateFilter.from_types( + auth_types_for_event(self.room_version, self) + ), + await_full_state=False, + ) + # When we create rooms we only insert the create+member events, and batch the rest. + # Therefore, we may not have state_ids from compute_state_after_events as the + # prev_state_events are unknown. If this happens, the caller provides the auth events + # to use instead. + calculated_auth_event_ids: list[ + str + ] = [] # assume it's the create event which has [] + if len(state_ids) == 0 and len(prev_state_events) > 0: + # it's a batched event, so we should have been provided the auth_events + assert auth_event_ids and len(auth_event_ids) > 0 + calculated_auth_event_ids = auth_event_ids + else: + calculated_auth_event_ids = ( + self._event_auth_handler.compute_auth_events(self, state_ids) + ) + else: + # event is a state DAG event and is the create event (room_id is not provided), + # therefore there are no auth_events. + calculated_auth_event_ids = [] + assert self.type == EventTypes.Create and self.state_key == "" + self.internal_metadata.calculated_auth_event_ids = calculated_auth_event_ids + auth_event_ids = calculated_auth_event_ids + # Create events always have empty auth_events. if self.type == EventTypes.Create and self.is_state() and self.state_key == "": auth_event_ids = [] @@ -155,6 +197,8 @@ async def build( if auth_event_ids is None: # Every non-create event must have a room ID assert self.room_id is not None + # this block must not be hit for MSC4242 rooms as it resolves state with prev_events + assert not self.room_version.msc4242_state_dags state_ids = await self._state.compute_state_after_events( self.room_id, prev_event_ids, @@ -231,7 +275,6 @@ async def build( # rejected by other servers (and so that they can be persisted in # the db) depth = min(depth, MAX_DEPTH) - event_dict: dict[str, Any] = { "auth_events": auth_events, "prev_events": prev_events, @@ -241,8 +284,6 @@ async def build( "unsigned": self.unsigned, "depth": depth, } - if self.room_id is not None: - event_dict["room_id"] = self.room_id if self.room_version.msc4291_room_ids_as_hashes: # In MSC4291: the create event has no room ID as the create event ID /is/ the room ID. @@ -262,6 +303,14 @@ async def build( auth_event_ids.remove(create_event_id) event_dict["auth_events"] = auth_event_ids + if self.room_version.msc4242_state_dags: + # Auth events are removed entirely on state DAG rooms + event_dict.pop("auth_events") + assert prev_state_events is not None + event_dict["prev_state_events"] = prev_state_events + if self.room_id is not None: + event_dict["room_id"] = self.room_id + if self.is_state(): event_dict["state_key"] = self._state_key diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 76ebac8b17c..f038fb5578d 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -41,6 +41,7 @@ MAX_PDU_SIZE, EventContentFields, EventTypes, + EventUnsignedContentFields, RelationTypes, ) from synapse.api.errors import Codes, SynapseError @@ -155,6 +156,10 @@ def prune_event_dict(room_version: RoomVersion, event_dict: JsonDict) -> JsonDic # Earlier room versions from had additional allowed keys. if not room_version.updated_redaction_rules: allowed_keys.extend(["prev_state", "membership", "origin"]) + # Custom room versions add new allowed keys and remove others + if room_version.msc4242_state_dags: + allowed_keys.extend(["prev_state_events"]) + allowed_keys.remove("auth_events") event_type = event_dict["type"] @@ -416,6 +421,50 @@ def format_event_for_client_v2_without_room_id(d: JsonDict) -> JsonDict: return d +@attr.s(slots=True, frozen=True, auto_attribs=True) +class FilteredEvent: + """An event annotated with per-user data for client serialization. + + Produced by filter_and_transform_events_for_client. Carries the user's + membership at the time of the event so serialization can inject it into + unsigned.membership (MSC4115) without cloning the underlying event. + """ + + event: "EventBase" + """The event to be serialized.""" + + membership: str | None + """The user whose requesting the event's membership at the time of the + event was sent. + + This is None if we didn't compute the membership. In Synapse this happens a) + when returning state events to state endpoints, or b) when the event is + returned to an admin. + + According to the spec we don't have to include the membership for any events + if we don't want to, especially if its expensive to compute. In practice + clients really only care about events in the room timeline so that in + encrypted room they can determine if they should be able to decrypt the + event or not. + """ + + @classmethod + def state(cls, event: "EventBase") -> "FilteredEvent": + """Wrap a state event with no per-user membership annotation. + + The event must be a state event (i.e. have a state_key). + """ + assert event.is_state(), ( + f"FilteredEvent.state() called with non-state event {event.event_id}" + ) + return cls(event=event, membership=None) + + @classmethod + def admin_override(cls, event: "EventBase") -> "FilteredEvent": + """Wrap an event that bypasses visibility filtering due to admin privileges.""" + return cls(event=event, membership=None) + + @attr.s(slots=True, frozen=True, auto_attribs=True) class SerializeEventConfig: as_client_event: bool = True @@ -435,6 +484,9 @@ class SerializeEventConfig: # only server admins can see through other configuration. For example, # whether an event was soft failed by the server. include_admin_metadata: bool = False + # Whether MSC4354 (sticky events) is enabled. When True, the sticky TTL + # will be computed and included in the unsigned section of sticky events. + msc4354_enabled: bool = False @only_event_fields.validator def _validate_only_event_fields( @@ -461,6 +513,7 @@ def _serialize_event( time_now_ms: int, *, config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG, + membership: str | None = None, ) -> JsonDict: """Serialize event for clients @@ -468,6 +521,8 @@ def _serialize_event( e time_now_ms config: Event serialization config + membership: The requesting user's membership at the time of the event, + to be injected into unsigned.membership (MSC4115). Returns: The serialized event dictionary. @@ -564,6 +619,23 @@ def _serialize_event( if e.internal_metadata.policy_server_spammy: d["unsigned"]["io.element.synapse.policy_server_spammy"] = True + if config.msc4354_enabled: + sticky_duration = e.sticky_duration() + if sticky_duration: + expires_at = ( + # min() ensures that the origin server can't lie about the time and + # send the event 'in the future', as that would allow them to exceed + # the 1 hour limit on stickiness duration. + min(e.origin_server_ts, time_now_ms) + sticky_duration.as_millis() + ) + if expires_at > time_now_ms: + d["unsigned"][EventUnsignedContentFields.STICKY_TTL] = ( + expires_at - time_now_ms + ) + + if membership is not None: + d["unsigned"][EventUnsignedContentFields.MEMBERSHIP] = membership + return d @@ -577,13 +649,15 @@ class EventClientSerializer: def __init__(self, hs: "HomeServer") -> None: self._store = hs.get_datastores().main self._auth = hs.get_auth() + self._config = hs.config + self._clock = hs.get_clock() self._add_extra_fields_to_unsigned_client_event_callbacks: list[ ADD_EXTRA_FIELDS_TO_UNSIGNED_CLIENT_EVENT_CALLBACK ] = [] async def serialize_event( self, - event: JsonDict | EventBase, + event: JsonDict | FilteredEvent, time_now: int, *, config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG, @@ -605,7 +679,7 @@ async def serialize_event( The serialized event """ # To handle the case of presence events and the like - if not isinstance(event, EventBase): + if not isinstance(event, FilteredEvent): return event # Force-enable server admin metadata because the only time an event with @@ -617,11 +691,16 @@ async def serialize_event( ): config = make_config_for_admin(config) - serialized_event = _serialize_event(event, time_now, config=config) + if self._config.experimental.msc4354_enabled: + config = attr.evolve(config, msc4354_enabled=True) + + serialized_event = _serialize_event( + event.event, time_now, config=config, membership=event.membership + ) # If the event was redacted, fetch the redaction event from the database # and include it in the serialized event's unsigned section. - redacted_by: str | None = event.internal_metadata.redacted_by + redacted_by: str | None = event.event.internal_metadata.redacted_by if redacted_by is not None: serialized_event.setdefault("unsigned", {})["redacted_by"] = redacted_by if redaction_map is not None: @@ -648,7 +727,7 @@ async def serialize_event( new_unsigned = {} for callback in self._add_extra_fields_to_unsigned_client_event_callbacks: - u = await callback(event) + u = await callback(event.event) new_unsigned.update(u) if new_unsigned: @@ -666,9 +745,9 @@ async def serialize_event( # Check if there are any bundled aggregations to include with the event. if bundle_aggregations: - if event.event_id in bundle_aggregations: + if event.event.event_id in bundle_aggregations: await self._inject_bundled_aggregations( - event, + event.event, time_now, config, bundle_aggregations, @@ -720,7 +799,7 @@ async def _inject_bundled_aggregations( # `sender` of the edit; however MSC3925 proposes extending it to the whole # of the edit, which is what we do here. serialized_aggregations[RelationTypes.REPLACE] = await self.serialize_event( - event_aggregations.replace, + FilteredEvent(event=event_aggregations.replace, membership=None), time_now, config=config, ) @@ -730,7 +809,7 @@ async def _inject_bundled_aggregations( thread = event_aggregations.thread serialized_latest_event = await self.serialize_event( - thread.latest_event, + FilteredEvent(event=thread.latest_event, membership=None), time_now, config=config, bundle_aggregations=bundled_aggregations, @@ -755,7 +834,7 @@ async def _inject_bundled_aggregations( @trace async def serialize_events( self, - events: Collection[JsonDict | EventBase], + events: Collection[JsonDict | FilteredEvent], time_now: int, *, config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG, @@ -780,11 +859,13 @@ async def serialize_events( ) # Batch-fetch all redaction events in one go rather than one per event. - redaction_ids = { - e.internal_metadata.redacted_by - for e in events - if isinstance(e, EventBase) and e.internal_metadata.redacted_by is not None - } + redaction_ids: set[str] = set() + for e in events: + base = e.event if isinstance(e, FilteredEvent) else e + if isinstance(base, EventBase): + redacted_by = base.internal_metadata.redacted_by + if redacted_by is not None: + redaction_ids.add(redacted_by) redaction_map = ( await self._store.get_events(redaction_ids) if redaction_ids else {} ) diff --git a/synapse/events/validator.py b/synapse/events/validator.py index b27f8a942af..ff22b2287f2 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -63,14 +63,17 @@ def validate_new(self, event: EventBase, config: HomeServerConfig) -> None: if event.format_version == EventFormatVersions.ROOM_V1_V2: EventID.from_string(event.event_id) - required = [ + required = { "auth_events", "content", "hashes", "prev_events", "sender", "type", - ] + } + if event.room_version.msc4242_state_dags: + required.remove("auth_events") + required.add("prev_state_events") for k in required: if k not in event: diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 04ba5b86db0..fe0710a0bf4 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -177,7 +177,9 @@ async def _check_sigs_and_hash( # Note: we don't redact the event so admins can inspect the event after the # fact. Other processes may redact the event, but that won't be applied to # the database copy of the event until the server's config requires it. - return pdu + # + # We also *don't* return early here as we would still like to evaluate + # `spam_checker_spammy`, for completeness. spam_check = await self._spam_checker_module_callbacks.check_event_for_spam(pdu) @@ -194,6 +196,8 @@ async def _check_sigs_and_hash( # using the event in prev_events). redacted_event = prune_event(pdu) redacted_event.internal_metadata.soft_failed = True + # Mark this as spam so we don't re-evaluate soft-failure status. + redacted_event.internal_metadata.spam_checker_spammy = True return redacted_event return pdu diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 55151ca549c..78a1900c731 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -1108,6 +1108,11 @@ async def send_join( SynapseError: if the chosen remote server returns a 300/400 code, or no servers successfully handle the request. """ + # See related restriction in /createRoom requests in handlers/room.py + if room_version.msc4242_state_dags: + raise UnsupportedRoomVersionError( + "Homeserver does not support this room version over federation" + ) async def send_request(destination: str) -> SendJoinResult: response = await self._do_send_join( diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index eff6d637894..1bbe1444223 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -20,6 +20,7 @@ # [This file includes modifications made by New Vector Limited] # # +import copy import logging import random from typing import ( @@ -568,9 +569,58 @@ async def _process_edu(edu_dict: JsonDict) -> None: origin=origin, destination=self.server_name, edu_type=edu_dict["edu_type"], - content=edu_dict["content"], + # Make a deep-copy as we mutate the content down below + content=copy.deepcopy(edu_dict["content"]), ) + try: + # Server ACL's apply to `EduTypes.TYPING` per MSC4163: + # + # > For typing notifications (m.typing), the room_id field inside + # > content should be checked, with the typing notification ignored if + # > the origin of the request is a server which is forbidden by the + # > room's ACL. Ignoring the typing notification means that the EDU + # > MUST be dropped upon receipt. + if edu.edu_type == EduTypes.TYPING: + origin_host, _ = parse_server_name(origin) + room_id = edu.content["room_id"] + try: + await self.check_server_matches_acl(origin_host, room_id) + except AuthError: + logger.warning( + "Ignoring typing EDU for room %s from banned server because of ACL's", + room_id, + ) + return + + # Server ACL's apply to `EduTypes.RECEIPT` per MSC4163: + # + # > For read receipts (m.receipt), all receipts inside a room_id + # > inside content should be ignored if the origin of the request is + # > forbidden by the room's ACL. + if edu.edu_type == EduTypes.RECEIPT: + origin_host, _ = parse_server_name(origin) + to_remove_room_ids = set() + for room_id in edu.content.keys(): + try: + await self.check_server_matches_acl(origin_host, room_id) + except AuthError: + to_remove_room_ids.add(room_id) + + if to_remove_room_ids: + logger.warning( + "Ignoring receipts in EDU for rooms %s from banned server %s because of ACL's", + to_remove_room_ids, + origin_host, + ) + + for room_id in to_remove_room_ids: + edu.content.pop(room_id) + + if not edu.content: + # If we've removed all the rooms, we can just ignore the whole EDU + return + await self.registry.on_edu(edu.edu_type, origin, edu.content) except Exception: # If there was an error handling the EDU, we must reject the diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index 2fb0e5814f2..51a752472f1 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -32,7 +32,8 @@ from synapse.api.constants import Direction, EventTypes, Membership from synapse.api.errors import SynapseError -from synapse.events import EventBase +from synapse.events import EventBase, FrozenEventVMSC4242 +from synapse.events.utils import FilteredEvent from synapse.types import ( JsonMapping, Requester, @@ -251,32 +252,40 @@ async def export_user_data(self, user_id: str, writer: "ExfiltrationWriter") -> topological=last_event.depth, ) - events = await filter_and_transform_events_for_client( + filtered_events = await filter_and_transform_events_for_client( self._storage_controllers, user_id, events, ) - writer.write_events(room_id, events) + writer.write_events(room_id, filtered_events) # Update the extremity tracking dicts - for event in events: + for filtered_event in filtered_events: # Check if we have any prev events that haven't been # processed yet, and add those to the appropriate dicts. - unseen_events = set(event.prev_event_ids()) - written_events + unseen_events = ( + set(filtered_event.event.prev_event_ids()) - written_events + ) if unseen_events: - event_to_unseen_prevs[event.event_id] = unseen_events + event_to_unseen_prevs[filtered_event.event.event_id] = ( + unseen_events + ) for unseen in unseen_events: unseen_to_child_events.setdefault(unseen, set()).add( - event.event_id + filtered_event.event.event_id ) # Now check if this event is an unseen prev event, if so # then we remove this event from the appropriate dicts. - for child_id in unseen_to_child_events.pop(event.event_id, []): - event_to_unseen_prevs[child_id].discard(event.event_id) + for child_id in unseen_to_child_events.pop( + filtered_event.event.event_id, [] + ): + event_to_unseen_prevs[child_id].discard( + filtered_event.event.event_id + ) - written_events.add(event.event_id) + written_events.add(filtered_event.event.event_id) logger.info( "Written %d events in room %s", len(written_events), room_id @@ -485,9 +494,16 @@ async def _redact_all_events( event_dict["redacts"] = event.event_id try: + prev_state_events = None + if room_version.msc4242_state_dags: + assert isinstance(event, FrozenEventVMSC4242) + prev_state_events = event.prev_state_events + assert prev_state_events is not None, ( + "Parent event of redaction has no `prev_state_events` which should be impossible as `prev_state_events` is a required field in MSC4242 rooms" + ) # set the prev event to the offending message to allow for redactions # to be processed in the case where the user has been kicked/banned before - # redactions are requested + # redactions are requested. ( redaction, _, @@ -496,6 +512,7 @@ async def _redact_all_events( event_dict, prev_event_ids=[event.event_id], ratelimit=False, + prev_state_events=prev_state_events, ) except Exception as ex: logger.info( @@ -511,7 +528,7 @@ class ExfiltrationWriter(metaclass=abc.ABCMeta): """Interface used to specify how to write exported data.""" @abc.abstractmethod - def write_events(self, room_id: str, events: list[EventBase]) -> None: + def write_events(self, room_id: str, events: list[FilteredEvent]) -> None: """Write a batch of events for a room.""" raise NotImplementedError() diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index b5c0cbdba23..1c8e108ea8a 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -1738,13 +1738,17 @@ def _do_validate_hash(checked_hash: bytes) -> bool: else: return False - async def start_sso_ui_auth(self, request: SynapseRequest, session_id: str) -> str: + async def start_sso_ui_auth( + self, request: SynapseRequest, session_id: str, preferred_idp_id: str | None + ) -> str: """ Get the HTML for the SSO redirect confirmation page. Args: request: The incoming HTTP request session_id: The user interactive authentication session ID. + preferred_idp_id: The ID of the identity provider to use. If `None` one will + be picked unpredictably from those the user has already signed in with. Returns: The HTML to render. @@ -1768,15 +1772,26 @@ async def start_sso_ui_auth(self, request: SynapseRequest, session_id: str) -> s # it not being offered. raise SynapseError(400, "User has no SSO identities") - # for now, just pick one - idp_id, sso_auth_provider = next(iter(idps.items())) - if len(idps) > 0: - logger.warning( - "User %r has previously logged in with multiple SSO IdPs; arbitrarily " - "picking %r", - user_id_to_verify, - idp_id, - ) + if preferred_idp_id is not None: + # Use the idp specified by the client. + sso_auth_provider = idps.get(preferred_idp_id) + if sso_auth_provider is None: + raise SynapseError( + 400, + f"Unknown preferred Identity Provider: '{preferred_idp_id}'", + errcode=Codes.INVALID_PARAM, + ) + else: + idp_id, sso_auth_provider = next(iter(idps.items())) + if len(idps) > 0: + # We arbitrarily picked an IdP from multiple potential + # candidates. This may be undesirable for the user. + logger.warning( + "User %r has previously logged in with multiple SSO IdPs; arbitrarily " + "picking %r", + user_id_to_verify, + idp_id, + ) redirect_url = await sso_auth_provider.handle_redirect_request( request, None, session_id diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 9a371651fb0..22254666487 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -58,6 +58,7 @@ DeviceListUpdates, JsonDict, JsonMapping, + MultiWriterStreamToken, ScheduledTask, StrCollection, StreamKeyType, @@ -1193,7 +1194,16 @@ async def handle_room_un_partial_stated(self, room_id: str) -> None: changes = await self.store.get_device_list_changes_in_room( room_id, device_lists_stream_id ) - local_changes = {(u, d) for u, d in changes if self.hs.is_mine_id(u)} + if changes is not None: + local_changes = {(u, d) for u, d in changes if self.hs.is_mine_id(u)} + else: + # The `device_lists_stream_id` is too old, so we need to fall back + # to looking for changes for all local users. + local_users = await self.store.get_local_users_in_room(room_id) + local_changes = await self.store.get_device_changes_for_users( + MultiWriterStreamToken(stream=device_lists_stream_id), local_users + ) + if not local_changes: return diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index f6517def9c9..2518716bc70 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -25,8 +25,7 @@ from synapse.api.constants import EduTypes, EventTypes, Membership, PresenceState from synapse.api.errors import AuthError, SynapseError -from synapse.events import EventBase -from synapse.events.utils import SerializeEventConfig +from synapse.events.utils import FilteredEvent, SerializeEventConfig from synapse.handlers.presence import format_user_presence_state from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.streams.config import PaginationConfig @@ -102,19 +101,19 @@ async def get_stream( # joined room, we need to send down presence for those users. to_add: list[JsonDict] = [] for event in events: - if not isinstance(event, EventBase): + if not isinstance(event, FilteredEvent): continue - if event.type == EventTypes.Member: - if event.membership != Membership.JOIN: + if event.event.type == EventTypes.Member: + if event.event.membership != Membership.JOIN: continue # Send down presence. - if event.state_key == requester.user.to_string(): + if event.event.state_key == requester.user.to_string(): # Send down presence for everyone in the room. users: Iterable[str] = await self.store.get_users_in_room( - event.room_id + event.event.room_id ) else: - users = [event.state_key] + users = [event.event.state_key] states = await presence_handler.get_states(users) to_add.extend( @@ -155,7 +154,7 @@ async def get_event( room_id: str | None, event_id: str, show_redacted: bool = False, - ) -> EventBase | None: + ) -> FilteredEvent | None: """Retrieve a single specified event on behalf of a user. The event will be transformed in a user-specific and time-specific way, e.g. having unsigned metadata added or being erased depending on who is accessing. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 14805ac80f7..b3444dd2ef0 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -242,7 +242,9 @@ async def _maybe_backfill_inner( Args: room_id: The room to backfill in. current_depth: The depth to check at for any upcoming backfill points. - limit: The max number of events to request from the remote federated server. + limit: The number of events that the pagination request will + return. This is used as part of the heuristic to decide if we + should back paginate. processing_start_time: The time when `maybe_backfill` started processing. Only used for timing. If `None`, no timing observation will be made. @@ -1188,7 +1190,7 @@ async def _make_and_verify_event( # We should assert some things. # FIXME: Do this in a nicer way assert event.type == EventTypes.Member - assert event.user_id == user_id + assert event.sender == user_id assert event.state_key == user_id assert event.room_id == room_id return origin, event, room_version diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index 1e5e98a59bd..591a0aefd33 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -30,7 +30,7 @@ Membership, ) from synapse.api.errors import SynapseError -from synapse.events.utils import SerializeEventConfig +from synapse.events.utils import FilteredEvent, SerializeEventConfig from synapse.events.validator import EventValidator from synapse.handlers.presence import format_user_presence_state from synapse.handlers.receipts import ReceiptEventSource @@ -186,7 +186,7 @@ async def handle_room(event: RoomsForUser) -> None: invite_event = await self.store.get_event(event.event_id) d["invite"] = await self._event_serializer.serialize_event( - invite_event, + FilteredEvent.state(event=invite_event), time_now, config=serializer_options, ) @@ -225,7 +225,7 @@ async def handle_room(event: RoomsForUser) -> None: ) ).addErrback(unwrapFirstError) - messages = await filter_and_transform_events_for_client( + filtered_messages = await filter_and_transform_events_for_client( self._storage_controllers, user_id, messages, @@ -240,7 +240,7 @@ async def handle_room(event: RoomsForUser) -> None: d["messages"] = { "chunk": ( await self._event_serializer.serialize_events( - messages, + filtered_messages, time_now=time_now, config=serializer_options, ) @@ -250,7 +250,7 @@ async def handle_room(event: RoomsForUser) -> None: } d["state"] = await self._event_serializer.serialize_events( - current_state.values(), + [FilteredEvent.state(e) for e in current_state.values()], time_now=time_now, config=serializer_options, ) @@ -382,7 +382,9 @@ async def _room_initial_sync_parted( room_id, limit=pagin_config.limit, end_token=stream_token ) - messages = await filter_and_transform_events_for_client( + filtered_messages: list[ + FilteredEvent + ] = await filter_and_transform_events_for_client( self._storage_controllers, requester.user.to_string(), messages, @@ -402,7 +404,7 @@ async def _room_initial_sync_parted( "chunk": ( # Don't bundle aggregations as this is a deprecated API. await self._event_serializer.serialize_events( - messages, time_now, config=serialize_options + filtered_messages, time_now, config=serialize_options ) ), "start": await start_token.to_string(self.store), @@ -411,7 +413,9 @@ async def _room_initial_sync_parted( "state": ( # Don't bundle aggregations as this is a deprecated API. await self._event_serializer.serialize_events( - room_state.values(), time_now, config=serialize_options + [FilteredEvent.state(e) for e in room_state.values()], + time_now, + config=serialize_options, ) ), "presence": [], @@ -435,7 +439,7 @@ async def _room_initial_sync_joined( serialize_options = SerializeEventConfig(requester=requester) # Don't bundle aggregations as this is a deprecated API. state = await self._event_serializer.serialize_events( - current_state.values(), + [FilteredEvent.state(e) for e in current_state.values()], time_now, config=serialize_options, ) @@ -456,9 +460,7 @@ async def get_presence() -> list[JsonDict]: if not self.hs.config.server.presence_enabled: return [] - states = await presence_handler.get_states( - [m.user_id for m in room_members] - ) + states = await presence_handler.get_states([m.sender for m in room_members]) return [ { @@ -496,7 +498,9 @@ async def get_receipts() -> list[JsonMapping]: ).addErrback(unwrapFirstError) ) - messages = await filter_and_transform_events_for_client( + filtered_messages: list[ + FilteredEvent + ] = await filter_and_transform_events_for_client( self._storage_controllers, requester.user.to_string(), messages, @@ -512,7 +516,7 @@ async def get_receipts() -> list[JsonMapping]: "chunk": ( # Don't bundle aggregations as this is a deprecated API. await self._event_serializer.serialize_events( - messages, time_now, config=serialize_options + filtered_messages, time_now, config=serialize_options ) ), "start": await start_token.to_string(self.store), diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index eb016225159..4032c7eca97 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -53,7 +53,7 @@ from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.api.urls import ConsentURIBuilder from synapse.event_auth import validate_event_for_room_version -from synapse.events import EventBase, relation_from_event +from synapse.events import EventBase, FrozenEventVMSC4242, relation_from_event from synapse.events.builder import EventBuilder from synapse.events.snapshot import ( EventContext, @@ -61,7 +61,11 @@ UnpersistedEventContext, UnpersistedEventContextBase, ) -from synapse.events.utils import SerializeEventConfig, maybe_upsert_event_field +from synapse.events.utils import ( + FilteredEvent, + SerializeEventConfig, + maybe_upsert_event_field, +) from synapse.events.validator import EventValidator from synapse.handlers.directory import DirectoryHandler from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME @@ -261,7 +265,7 @@ async def get_state_events( room_state = room_state_events[membership_event_id] events = await self._event_serializer.serialize_events( - room_state.values(), + [FilteredEvent.state(e) for e in room_state.values()], self.clock.time_msec(), config=SerializeEventConfig(requester=requester), ) @@ -585,6 +589,7 @@ async def create_event( state_map: StateMap[str] | None = None, for_batch: bool = False, current_state_group: int | None = None, + prev_state_events: list[str] | None = None, delay_id: str | None = None, ) -> tuple[EventBase, UnpersistedEventContextBase]: """ @@ -640,6 +645,10 @@ async def create_event( current_state_group: the current state group, used only for creating events for batch persisting + prev_state_events: + The state event IDs which represent the current forward extremities of the state DAG. + Only applicable on room versions which use a state DAG (MSC4242). + delay_id: The delay ID of this event, if it was a delayed event. Raises: @@ -744,6 +753,7 @@ async def create_event( state_map=state_map, for_batch=for_batch, current_state_group=current_state_group, + prev_state_events=prev_state_events, ) # In an ideal world we wouldn't need the second part of this condition. However, @@ -895,7 +905,7 @@ async def deduplicate_state_event( if not prev_event: return None - if prev_event and event.user_id == prev_event.user_id: + if prev_event and event.sender == prev_event.sender: prev_content = encode_canonical_json(prev_event.content) next_content = encode_canonical_json(event.content) if prev_content == next_content: @@ -972,6 +982,7 @@ async def create_and_send_nonmember_event( ignore_shadow_ban: bool = False, outlier: bool = False, depth: int | None = None, + prev_state_events: list[str] | None = None, delay_id: str | None = None, ) -> tuple[EventBase, int]: """ @@ -1001,6 +1012,9 @@ async def create_and_send_nonmember_event( depth: Override the depth used to order the event in the DAG. Should normally be set to None, which will cause the depth to be calculated based on the prev_events. + prev_state_events: + The state event IDs which represent the current forward extremities of the state DAG. + Only applicable on room versions which use a state DAG (MSC4242). delay_id: The delay ID of this event, if it was a delayed event. Returns: @@ -1098,6 +1112,7 @@ async def create_and_send_nonmember_event( ignore_shadow_ban=ignore_shadow_ban, outlier=outlier, depth=depth, + prev_state_events=prev_state_events, delay_id=delay_id, ) @@ -1112,6 +1127,7 @@ async def _create_and_send_nonmember_event_locked( ignore_shadow_ban: bool = False, outlier: bool = False, depth: int | None = None, + prev_state_events: list[str] | None = None, delay_id: str | None = None, ) -> tuple[EventBase, int]: room_id = event_dict["room_id"] @@ -1141,6 +1157,7 @@ async def _create_and_send_nonmember_event_locked( state_event_ids=state_event_ids, outlier=outlier, depth=depth, + prev_state_events=prev_state_events, delay_id=delay_id, ) context = await unpersisted_context.persist(event) @@ -1236,6 +1253,7 @@ async def create_new_client_event( state_map: StateMap[str] | None = None, for_batch: bool = False, current_state_group: int | None = None, + prev_state_events: list[str] | None = None, ) -> tuple[EventBase, UnpersistedEventContextBase]: """Create a new event for a local client. If bool for_batch is true, will create an event using the prev_event_ids, and will create an event context for @@ -1277,9 +1295,30 @@ async def create_new_client_event( current_state_group: the current state group, used only for creating events for batch persisting + prev_state_events: + The state event IDs which represent the current forward extremities of the state DAG. + Only applicable on room versions which use a state DAG (MSC4242). + If unset, populates them from the current state dag forward extremities. + Returns: Tuple of created event, UnpersistedEventContext """ + if builder.room_version.msc4242_state_dags: + assert auth_event_ids is None + # (kegan) I can't find any call-site which uses this. We can't risk letting in + # untrusted input, so for now assert that we aren't told about any state. + assert state_event_ids is None + + if builder.room_id: + if prev_state_events is None: + prev_state_events = list( + await self.store.get_state_dag_extremities(builder.room_id) + ) + else: + # create event doesn't need prev_state_events to be fetched, but it must be non-None. + assert builder.type == EventTypes.Create and builder.state_key == "" + prev_state_events = [] + # Strip down the state_event_ids to only what we need to auth the event. # For example, we don't need extra m.room.member that don't match event.sender if state_event_ids is not None: @@ -1353,7 +1392,10 @@ async def create_new_client_event( assert state_map is not None auth_ids = self._event_auth_handler.compute_auth_events(builder, state_map) event = await builder.build( - prev_event_ids=prev_event_ids, auth_event_ids=auth_ids, depth=depth + prev_event_ids=prev_event_ids, + auth_event_ids=auth_ids, + depth=depth, + prev_state_events=prev_state_events, ) context: UnpersistedEventContextBase = ( @@ -1370,6 +1412,7 @@ async def create_new_client_event( prev_event_ids=prev_event_ids, auth_event_ids=auth_event_ids, depth=depth, + prev_state_events=prev_state_events, ) # Pass on the outlier property from the builder to the event @@ -1541,7 +1584,7 @@ async def handle_new_client_event( EventTypes.Message, EventTypes.Encrypted, ]: - await self.store.set_room_participation(event.user_id, event.room_id) + await self.store.set_room_participation(event.sender, event.room_id) if event.internal_metadata.is_out_of_band_membership(): # the only sort of out-of-band-membership events we expect to see here are @@ -1559,6 +1602,20 @@ async def handle_new_client_event( auth_event = event_id_to_event.get(event_id) if auth_event: batched_auth_events[event_id] = auth_event + if event.room_version.msc4242_state_dags: + assert isinstance(event, FrozenEventVMSC4242) + # State DAG rooms will check that the prev_state_events are not rejected. + # To do that, we need to make sure we pass in the prev_state_events as + # batched_auth_events, else we will fail the event due to the + # prev_state_events not existing in the database. + for prev_state_event_id in event.prev_state_events: + prev_state_event = event_id_to_event.get( + prev_state_event_id + ) + if prev_state_event: + batched_auth_events[prev_state_event_id] = ( + prev_state_event + ) await self._event_auth_handler.check_auth_rules_from_context( event, batched_auth_events ) @@ -1813,7 +1870,10 @@ async def cache_joined_hosts_for_events( # set for a while, so that the expiry time is reset. state_entry = await self.state.resolve_state_groups_for_events( - event.room_id, event_ids=event.prev_event_ids() + event.room_id, + event_ids=event.prev_state_events + if isinstance(event, FrozenEventVMSC4242) + else event.prev_event_ids(), ) if state_entry.state_group: @@ -2098,7 +2158,7 @@ async def persist_and_notify_client_events( "Could not find event %s" % (event.redacts,) ) - if event.user_id != original_event.user_id: + if event.sender != original_event.sender: raise AuthError( 403, "You don't have permission to redact events" ) @@ -2356,9 +2416,16 @@ async def _rebuild_event_after_third_party_rules( # case. prev_event_ids = await self.store.get_prev_events_for_room(builder.room_id) + prev_state_events = None + if original_event.room_version.msc4242_state_dags: + prev_state_events = list( + await self.store.get_state_dag_extremities(builder.room_id) + ) + event = await builder.build( prev_event_ids=prev_event_ids, auth_event_ids=None, + prev_state_events=prev_state_events, ) # we rebuild the event context, to be on the safe side. If nothing else, diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 7b9c8290564..8cbe4b63c88 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -29,6 +29,7 @@ from synapse.api.errors import SynapseError from synapse.api.filtering import Filter from synapse.events import EventBase +from synapse.events.utils import FilteredEvent from synapse.handlers.relations import BundledAggregations from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.logging.opentracing import trace @@ -79,7 +80,7 @@ class GetMessagesResult: Everything needed to serialize a `/messages` response. """ - messages_chunk: list[EventBase] + messages_chunk: list[FilteredEvent] """ A list of room events. @@ -684,16 +685,18 @@ async def get_messages( events = await event_filter.filter(events) if not use_admin_priviledge: - events = await filter_and_transform_events_for_client( + filtered_events = await filter_and_transform_events_for_client( self._storage_controllers, user_id, events, is_peeking=(member_event_id is None), ) + else: + filtered_events = [FilteredEvent.admin_override(e) for e in events] # if after the filter applied there are no more events # return immediately - but there might be more in next_token batch - if not events: + if not filtered_events: return GetMessagesResult( messages_chunk=[], bundled_aggregations={}, @@ -703,16 +706,16 @@ async def get_messages( ) state = None - if event_filter and event_filter.lazy_load_members and len(events) > 0: + if event_filter and event_filter.lazy_load_members and len(filtered_events) > 0: # TODO: remove redundant members # FIXME: we also care about invite targets etc. state_filter = StateFilter.from_types( - (EventTypes.Member, event.sender) for event in events + (EventTypes.Member, event.event.sender) for event in filtered_events ) state_ids = await self._state_storage_controller.get_state_ids_for_event( - events[0].event_id, state_filter=state_filter + filtered_events[0].event.event_id, state_filter=state_filter ) if state_ids: @@ -720,11 +723,11 @@ async def get_messages( state = list(state_dict.values()) aggregations = await self._relations_handler.get_bundled_aggregations( - events, user_id + filtered_events, user_id ) return GetMessagesResult( - messages_chunk=events, + messages_chunk=filtered_events, bundled_aggregations=aggregations, state=state, start_token=from_token, diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index d123bcdd367..c3886795b66 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -406,28 +406,6 @@ async def delete_profile_upon_deactivation( # have it. raise AuthError(400, "Cannot remove another user's profile") - if not by_admin: - current_profile = await self.store.get_profileinfo(target_user) - if not self.hs.config.registration.enable_set_displayname: - if current_profile.display_name: - # SUSPICIOUS: It seems strange to block deactivation on this, - # though this is preserving previous behaviour. - raise SynapseError( - 400, - "Changing display name is disabled on this server", - Codes.FORBIDDEN, - ) - - if not self.hs.config.registration.enable_set_avatar_url: - if current_profile.avatar_url: - # SUSPICIOUS: It seems strange to block deactivation on this, - # though this is preserving previous behaviour. - raise SynapseError( - 400, - "Changing avatar is disabled on this server", - Codes.FORBIDDEN, - ) - await self.store.delete_profile(target_user) await self._third_party_rules.on_profile_update( diff --git a/synapse/handlers/relations.py b/synapse/handlers/relations.py index d7d3002fbe3..ee4f8d672ee 100644 --- a/synapse/handlers/relations.py +++ b/synapse/handlers/relations.py @@ -33,7 +33,7 @@ from synapse.api.constants import Direction, EventTypes, RelationTypes from synapse.api.errors import SynapseError from synapse.events import EventBase, relation_from_event -from synapse.events.utils import SerializeEventConfig +from synapse.events.utils import FilteredEvent, SerializeEventConfig from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.logging.opentracing import trace from synapse.storage.databases.main.relations import ThreadsNextBatch, _RelatedEvent @@ -139,7 +139,7 @@ async def get_relations( # not passing them in here we should get a better cache hit rate). related_events, next_token = await self._main_store.get_relations_for_event( event_id=event_id, - event=event, + event=event.event, room_id=room_id, relation_type=relation_type, event_type=event_type, @@ -154,7 +154,9 @@ async def get_relations( [e.event_id for e in related_events] ) - events = await filter_and_transform_events_for_client( + filtered_events: list[ + FilteredEvent + ] = await filter_and_transform_events_for_client( self._storage_controllers, user_id, events, @@ -164,14 +166,14 @@ async def get_relations( # The relations returned for the requested event do include their # bundled aggregations. aggregations = await self.get_bundled_aggregations( - events, requester.user.to_string() + filtered_events, requester.user.to_string() ) now = self._clock.time_msec() serialize_options = SerializeEventConfig(requester=requester) return_value: JsonDict = { "chunk": await self._event_serializer.serialize_events( - events, + filtered_events, now, bundle_aggregations=aggregations, config=serialize_options, @@ -389,7 +391,7 @@ async def _get_threads_for_events( potential_events, _ = await self._main_store.get_relations_for_event( room_id, event_id, - event, + event.event, RelationTypes.THREAD, direction=Direction.FORWARDS, ) @@ -417,7 +419,7 @@ async def _get_threads_for_events( potential_events[-1].event_id, ) continue - latest_thread_event = event + latest_thread_event = event.event results[event_id] = _ThreadAggregation( latest_event=latest_thread_event, @@ -432,12 +434,12 @@ async def _get_threads_for_events( @trace async def get_bundled_aggregations( - self, events: Iterable[EventBase], user_id: str + self, filtered_events: Iterable[FilteredEvent], user_id: str ) -> dict[str, BundledAggregations]: """Generate bundled aggregations for events. Args: - events: The iterable of events to calculate bundled aggregations for. + filtered_events: The iterable of filtered events to calculate bundled aggregations for. user_id: The user requesting the bundled aggregations. Returns: @@ -453,7 +455,9 @@ async def get_bundled_aggregations( events_by_id = {} # A map of event ID to the relation in that event, if there is one. relations_by_id: dict[str, str] = {} - for event in events: + for filtered_event in filtered_events: + event = filtered_event.event + # State events do not get bundled aggregations. if event.is_state(): continue @@ -599,7 +603,9 @@ async def get_threads( # Limit the returned threads to those the user has participated in. events = [event for event in events if participated[event.event_id]] - events = await filter_and_transform_events_for_client( + filtered_events: list[ + FilteredEvent + ] = await filter_and_transform_events_for_client( self._storage_controllers, user_id, events, @@ -607,12 +613,12 @@ async def get_threads( ) aggregations = await self.get_bundled_aggregations( - events, requester.user.to_string() + filtered_events, requester.user.to_string() ) now = self._clock.time_msec() serialized_events = await self._event_serializer.serialize_events( - events, now, bundle_aggregations=aggregations + filtered_events, now, bundle_aggregations=aggregations ) return_value: JsonDict = {"chunk": serialized_events} diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 1c3489a00e3..6a5c76c6676 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -27,6 +27,7 @@ import random import string from collections import OrderedDict +from collections.abc import Mapping from http import HTTPStatus from typing import ( TYPE_CHECKING, @@ -65,9 +66,13 @@ from synapse.api.ratelimiting import Ratelimiter from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion from synapse.event_auth import validate_event_for_room_version -from synapse.events import EventBase +from synapse.events import EventBase, event_exists_in_state_dag from synapse.events.snapshot import UnpersistedEventContext -from synapse.events.utils import copy_and_fixup_power_levels_contents +from synapse.events.utils import ( + FilteredEvent, + PowerLevelsContent, + copy_and_fixup_power_levels_contents, +) from synapse.handlers.relations import BundledAggregations from synapse.rest.admin._base import assert_user_is_admin from synapse.streams import EventSource @@ -109,9 +114,9 @@ @attr.s(slots=True, frozen=True, auto_attribs=True) class EventContext: - events_before: list[EventBase] - event: EventBase - events_after: list[EventBase] + events_before: list[FilteredEvent] + event: FilteredEvent + events_after: list[FilteredEvent] state: list[EventBase] aggregations: dict[str, BundledAggregations] start: str @@ -500,10 +505,12 @@ async def _update_upgraded_room_pls( except AuthError as e: logger.warning("Unable to update PLs in old room: %s", e) + power_levels_content: JsonMapping = old_room_pl_state.content + new_room_version = await self.store.get_room_version(new_room_id) if new_room_version.msc4289_creator_power_enabled: - self._remove_creators_from_pl_users_map( - old_room_pl_state.content.get("users", {}), + power_levels_content = self._copy_and_remove_creators_from_pl_users_map( + power_levels_content, requester.user.to_string(), additional_creators, ) @@ -515,9 +522,7 @@ async def _update_upgraded_room_pls( "state_key": "", "room_id": new_room_id, "sender": requester.user.to_string(), - "content": copy_and_fixup_power_levels_contents( - old_room_pl_state.content - ), + "content": copy_and_fixup_power_levels_contents(power_levels_content), }, ratelimit=False, ) @@ -686,11 +691,12 @@ async def clone_existing_room( if new_room_version.msc4289_creator_power_enabled: # the creator(s) cannot be in the users map - self._remove_creators_from_pl_users_map( - user_power_levels, + fixed_power_levels = self._copy_and_remove_creators_from_pl_users_map( + power_levels, user_id, additional_creators, ) + initial_state[(EventTypes.PowerLevels, "")] = fixed_power_levels # We construct a subset of what the body of a call to /createRoom would look like # for passing to the spam checker. We don't include a preset here, as we expect the @@ -1237,6 +1243,10 @@ async def create_room( creation_content = config.get("creation_content", {}) # override any attempt to set room versions via the creation_content creation_content["room_version"] = room_version.identifier + # We do not currently support federating state DAG rooms. + # See related restriction in /send_join requests in federation_client.py. + if room_version.msc4242_state_dags: + creation_content[EventContentFields.FEDERATE] = False # trusted private chats have the invited users marked as additional creators if ( @@ -1486,6 +1496,11 @@ async def _send_events_for_new_room( # the most recently created event prev_event: list[str] = [] + # This should be the most recently created state event as we create each event + prev_state_events: list[str] | None = ( + [] if room_version.msc4242_state_dags else None + ) + # a map of event types, state keys -> event_ids. We collect these mappings this as events are # created (but not persisted to the db) to determine state for future created events # (as this info can't be pulled from the db) @@ -1512,6 +1527,7 @@ async def create_event( """ nonlocal depth nonlocal prev_event + nonlocal prev_state_events # Create the event dictionary. event_dict = {"type": etype, "content": content} @@ -1525,6 +1541,7 @@ async def create_event( creator, event_dict, prev_event_ids=prev_event, + prev_state_events=prev_state_events, depth=depth, # Take a copy to ensure each event gets a unique copy of # state_map since it is modified below. @@ -1535,7 +1552,8 @@ async def create_event( depth += 1 prev_event = [new_event.event_id] state_map[(new_event.type, new_event.state_key)] = new_event.event_id - + if room_version.msc4242_state_dags and event_exists_in_state_dag(new_event): + prev_state_events = [new_event.event_id] return new_event, new_unpersisted_context preset_config, config = self._room_preset_config(room_config) @@ -1568,6 +1586,8 @@ async def create_event( ignore_shadow_ban=True, ) last_sent_event_id = ev.event_id + if room_version.msc4242_state_dags: + prev_state_events = [ev.event_id] member_event_id, _ = await self.room_member_handler.update_membership( creator, @@ -1579,8 +1599,11 @@ async def create_event( new_room=True, prev_event_ids=[last_sent_event_id], depth=depth, + prev_state_events=prev_state_events, ) prev_event = [member_event_id] + if room_version.msc4242_state_dags: + prev_state_events = [member_event_id] # update the depth and state map here as the membership event has been created # through a different code path @@ -1812,12 +1835,19 @@ def _room_preset_config(self, room_config: JsonDict) -> tuple[str, dict]: ) return preset_name, preset_config - def _remove_creators_from_pl_users_map( + def _copy_and_remove_creators_from_pl_users_map( self, - users_map: dict[str, int], + power_levels_content: PowerLevelsContent, creator: str, additional_creators: list[str] | None, - ) -> None: + ) -> PowerLevelsContent: + users_map = power_levels_content.get("users", {}) + if not users_map: + return power_levels_content + + assert isinstance(users_map, Mapping) + users_map = dict(users_map) + creators = [creator] if additional_creators: creators.extend(additional_creators) @@ -1825,6 +1855,9 @@ def _remove_creators_from_pl_users_map( # the creator(s) cannot be in the users map users_map.pop(creator, None) + power_levels_content = {**power_levels_content, "users": users_map} + return power_levels_content + def _generate_room_id(self) -> str: """Generates a random room ID. @@ -1916,9 +1949,9 @@ async def get_event_context( # The user is peeking if they aren't in the room already is_peeking = not is_user_in_room - async def filter_evts(events: list[EventBase]) -> list[EventBase]: + async def filter_evts(events: list[EventBase]) -> list[FilteredEvent]: if use_admin_priviledge: - return events + return [FilteredEvent.admin_override(e) for e in events] return await filter_and_transform_events_for_client( self._storage_controllers, user.to_string(), @@ -1946,31 +1979,33 @@ async def filter_evts(events: list[EventBase]) -> list[EventBase]: events_before = await event_filter.filter(events_before) events_after = await event_filter.filter(events_after) - events_before = await filter_evts(events_before) - events_after = await filter_evts(events_after) + filtered_events_before = await filter_evts(events_before) + filtered_events_after = await filter_evts(events_after) # filter_evts can return a pruned event in case the user is allowed to see that # there's something there but not see the content, so use the event that's in # `filtered` rather than the event we retrieved from the datastore. - event = filtered[0] + filtered_event = filtered[0] # Fetch the aggregations. aggregations = await self._relations_handler.get_bundled_aggregations( - itertools.chain(events_before, (event,), events_after), + itertools.chain( + filtered_events_before, (filtered_event,), filtered_events_after + ), user.to_string(), ) - if events_after: - last_event_id = events_after[-1].event_id + if filtered_events_after: + last_event_id = filtered_events_after[-1].event.event_id else: last_event_id = event_id if event_filter and event_filter.lazy_load_members: state_filter = StateFilter.from_lazy_load_member_list( - ev.sender + ev.event.sender for ev in itertools.chain( - events_before, - (event,), - events_after, + filtered_events_before, + (filtered_event,), + filtered_events_after, ) ) else: @@ -1993,9 +2028,9 @@ async def filter_evts(events: list[EventBase]) -> list[EventBase]: token = StreamToken.START return EventContext( - events_before=events_before, - event=event, - events_after=events_after, + events_before=filtered_events_before, + event=filtered_event, + events_after=filtered_events_after, state=state_events, aggregations=aggregations, start=await token.copy_and_replace( diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index b2e678e90e9..236c8ca03c7 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -36,9 +36,11 @@ from synapse.api.errors import ( AuthError, Codes, + NotFoundError, PartialStateConflictError, ShadowBanError, SynapseError, + UnsupportedRoomVersionError, ) from synapse.api.ratelimiting import Ratelimiter from synapse.event_auth import get_named_level, get_power_level_event @@ -408,6 +410,7 @@ async def _local_membership_update( require_consent: bool = True, outlier: bool = False, origin_server_ts: int | None = None, + prev_state_events: list[str] | None = None, delay_id: str | None = None, ) -> tuple[str, int]: """ @@ -494,6 +497,7 @@ async def _local_membership_update( depth=depth, require_consent=require_consent, outlier=outlier, + prev_state_events=prev_state_events, delay_id=delay_id, ) context = await unpersisted_context.persist(event) @@ -590,6 +594,7 @@ async def update_membership( state_event_ids: list[str] | None = None, depth: int | None = None, origin_server_ts: int | None = None, + prev_state_events: list[str] | None = None, delay_id: str | None = None, ) -> tuple[str, int]: """Update a user's membership in a room. @@ -684,6 +689,7 @@ async def update_membership( state_event_ids=state_event_ids, depth=depth, origin_server_ts=origin_server_ts, + prev_state_events=prev_state_events, delay_id=delay_id, ) @@ -707,6 +713,7 @@ async def update_membership_locked( state_event_ids: list[str] | None = None, depth: int | None = None, origin_server_ts: int | None = None, + prev_state_events: list[str] | None = None, delay_id: str | None = None, ) -> tuple[str, int]: """Helper for update_membership. @@ -951,10 +958,21 @@ async def update_membership_locked( require_consent=require_consent, outlier=outlier, origin_server_ts=origin_server_ts, + prev_state_events=prev_state_events, delay_id=delay_id, ) - latest_event_ids = await self.store.get_prev_events_for_room(room_id) + is_state_dags = False + try: + room_version = await self.store.get_room_version(room_id) + is_state_dags = room_version.msc4242_state_dags + except (NotFoundError, UnsupportedRoomVersionError): + pass + + if is_state_dags: + latest_event_ids = list(await self.store.get_state_dag_extremities(room_id)) + else: + latest_event_ids = await self.store.get_prev_events_for_room(room_id) is_partial_state_room = await self.store.is_partial_state_room(room_id) partial_state_before_join = await self.state_handler.compute_state_after_events( @@ -1165,6 +1183,8 @@ async def update_membership_locked( # see: https://github.com/matrix-org/synapse/issues/7139 if len(latest_event_ids) == 0: latest_event_ids = [invite.event_id] + if invite.room_version.msc4242_state_dags: + prev_state_events = [invite.event_id] # or perhaps this is a remote room that a local user has knocked on elif current_membership_type == Membership.KNOCK: @@ -1210,6 +1230,7 @@ async def update_membership_locked( require_consent=require_consent, outlier=outlier, origin_server_ts=origin_server_ts, + prev_state_events=prev_state_events, delay_id=delay_id, ) @@ -2108,10 +2129,21 @@ async def _generate_local_out_of_band_leave( # # the prev_events consist solely of the previous membership event. prev_event_ids = [previous_membership_event.event_id] - auth_event_ids = ( - list(previous_membership_event.auth_event_ids()) + prev_event_ids - ) + auth_event_ids = None + # Authorise the leave by referencing the previous membership + prev_state_event_ids = None + if previous_membership_event.room_version.msc4242_state_dags: + prev_state_event_ids = [ + previous_membership_event.event_id, + ] + else: + auth_event_ids = ( + list(previous_membership_event.auth_event_ids()) + prev_event_ids + ) + # State DAG rooms should not have auth events specified + # Normal rooms should not have prev state event IDs specified + assert not (prev_state_event_ids is not None and auth_event_ids is not None) # Try several times, it could fail with PartialStateConflictError # in handle_new_client_event, cf comment in except block. max_retries = 5 @@ -2127,6 +2159,7 @@ async def _generate_local_out_of_band_leave( prev_event_ids=prev_event_ids, auth_event_ids=auth_event_ids, outlier=True, + prev_state_events=prev_state_event_ids, ) context = await unpersisted_context.persist(event) event.internal_metadata.out_of_band_membership = True diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 56c047b0e89..30e072d011e 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -29,8 +29,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.errors import NotFoundError, SynapseError from synapse.api.filtering import Filter -from synapse.events import EventBase -from synapse.events.utils import SerializeEventConfig +from synapse.events.utils import FilteredEvent, SerializeEventConfig from synapse.types import JsonDict, Requester, StrCollection, StreamKeyType, UserID from synapse.types.state import StateFilter from synapse.visibility import filter_and_transform_events_for_client @@ -48,7 +47,7 @@ class _SearchResult: # A mapping of event ID to the rank of that event. rank_map: dict[str, int] # A list of the resulting events. - allowed_events: list[EventBase] + allowed_events: list[FilteredEvent] # A map of room ID to results. room_groups: dict[str, JsonDict] # A set of event IDs to highlight. @@ -355,12 +354,12 @@ async def _search( state_results = {} if include_state: - for room_id in {e.room_id for e in search_result.allowed_events}: + for room_id in {e.event.room_id for e in search_result.allowed_events}: state = await self._storage_controllers.state.get_current_state(room_id) state_results[room_id] = list(state.values()) aggregations = await self._relations_handler.get_bundled_aggregations( - # Generate an iterable of EventBase for all the events that will be + # Generate an iterable of FilteredEvent for all the events that will be # returned, including contextual events. itertools.chain( # The events_before and events_after for each context. @@ -396,14 +395,14 @@ async def _search( results = [ { - "rank": search_result.rank_map[e.event_id], + "rank": search_result.rank_map[e.event.event_id], "result": await self._event_serializer.serialize_event( e, time_now, bundle_aggregations=aggregations, config=serialize_options, ), - "context": contexts.get(e.event_id, {}), + "context": contexts.get(e.event.event_id, {}), } for e in search_result.allowed_events ] @@ -417,7 +416,9 @@ async def _search( if state_results: rooms_cat_res["state"] = { room_id: await self._event_serializer.serialize_events( - state_events, time_now, config=serialize_options + [FilteredEvent.state(e) for e in state_events], + time_now, + config=serialize_options, ) for room_id, state_events in state_results.items() } @@ -485,19 +486,19 @@ async def _search_by_rank( filtered_events, ) - events.sort(key=lambda e: -rank_map[e.event_id]) + events.sort(key=lambda e: -rank_map[e.event.event_id]) allowed_events = events[: search_filter.limit] for e in allowed_events: rm = room_groups.setdefault( - e.room_id, {"results": [], "order": rank_map[e.event_id]} + e.event.room_id, {"results": [], "order": rank_map[e.event.event_id]} ) - rm["results"].append(e.event_id) + rm["results"].append(e.event.event_id) s = sender_group.setdefault( - e.sender, {"results": [], "order": rank_map[e.event_id]} + e.event.sender, {"results": [], "order": rank_map[e.event.event_id]} ) - s["results"].append(e.event_id) + s["results"].append(e.event.event_id) return ( _SearchResult( @@ -549,7 +550,7 @@ async def _search_by_recent( highlights = set() - room_events: list[EventBase] = [] + room_events: list[FilteredEvent] = [] i = 0 pagination_token = batch_token @@ -595,11 +596,11 @@ async def _search_by_recent( pagination_token = results[-1]["pagination_token"] for event in room_events: - group = room_groups.setdefault(event.room_id, {"results": []}) - group["results"].append(event.event_id) + group = room_groups.setdefault(event.event.room_id, {"results": []}) + group["results"].append(event.event.event_id) if room_events and len(room_events) >= search_filter.limit: - last_event_id = room_events[-1].event_id + last_event_id = room_events[-1].event.event_id pagination_token = results_map[last_event_id]["pagination_token"] # We want to respect the given batch group and group keys so @@ -632,7 +633,7 @@ async def _search_by_recent( async def _calculate_event_contexts( self, user: UserID, - allowed_events: list[EventBase], + allowed_events: list[FilteredEvent], before_limit: int, after_limit: int, include_profile: bool, @@ -658,7 +659,7 @@ async def _calculate_event_contexts( contexts = {} for event in allowed_events: res = await self.store.get_events_around( - event.room_id, event.event_id, before_limit, after_limit + event.event.room_id, event.event.event_id, before_limit, after_limit ) logger.info( @@ -692,14 +693,14 @@ async def _calculate_event_contexts( if include_profile: senders = { - ev.sender + ev.event.sender for ev in itertools.chain(events_before, [event], events_after) } if events_after: - last_event_id = events_after[-1].event_id + last_event_id = events_after[-1].event.event_id else: - last_event_id = event.event_id + last_event_id = event.event.event_id state_filter = StateFilter.from_types( [(EventTypes.Member, sender) for sender in senders] @@ -718,6 +719,6 @@ async def _calculate_event_contexts( if s.type == EventTypes.Member and s.state_key in senders } - contexts[event.event_id] = context + contexts[event.event.event_id] = context return contexts diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index 6feb6c292e9..a3443b300cc 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -23,7 +23,7 @@ from synapse.api.constants import Direction, EventTypes, Membership from synapse.events import EventBase -from synapse.events.utils import strip_event +from synapse.events.utils import FilteredEvent, strip_event from synapse.handlers.relations import BundledAggregations from synapse.handlers.sliding_sync.extensions import SlidingSyncExtensionHandler from synapse.handlers.sliding_sync.room_lists import ( @@ -167,34 +167,38 @@ async def wait_for_sync_for_user( timeout_ms -= after_wait_ts - before_wait_ts timeout_ms = max(timeout_ms, 0) - # We're going to respond immediately if the timeout is 0 or if this is an - # initial sync (without a `from_token`) so we can avoid calling - # `notifier.wait_for_events()`. - if timeout_ms == 0 or from_token is None: - now_token = self.event_sources.get_current_token() - result = await self.current_sync_for_user( + # Compute a response immediately. We always need to do this before + # waiting for new data (unlike in /v3/sync), as the request config might + # have changed (e.g. new room subscriptions, etc). + now_token = self.event_sources.get_current_token() + result = await self.current_sync_for_user( + sync_config, + from_token=from_token, + to_token=now_token, + ) + + # Return immediately if we have a result, the timeout is 0, or this is + # an initial sync. + if result or timeout_ms == 0 or from_token is None: + return result, did_wait + + # Otherwise, we wait for something to happen and report it to the user. + async def current_sync_callback( + before_token: StreamToken, after_token: StreamToken + ) -> SlidingSyncResult: + return await self.current_sync_for_user( sync_config, from_token=from_token, - to_token=now_token, + to_token=after_token, ) - else: - # Otherwise, we wait for something to happen and report it to the user. - async def current_sync_callback( - before_token: StreamToken, after_token: StreamToken - ) -> SlidingSyncResult: - return await self.current_sync_for_user( - sync_config, - from_token=from_token, - to_token=after_token, - ) - result = await self.notifier.wait_for_events( - sync_config.user.to_string(), - timeout_ms, - current_sync_callback, - from_token=from_token.stream_token, - ) - did_wait = True + result = await self.notifier.wait_for_events( + sync_config.user.to_string(), + timeout_ms, + current_sync_callback, + from_token=now_token, + ) + did_wait = True return result, did_wait @@ -679,7 +683,7 @@ async def get_room_sync_data( # membership. Currently, we have to make all of these optional because # `invite`/`knock` rooms only have `stripped_state`. See # https://github.com/matrix-org/matrix-spec-proposals/pull/3575#discussion_r1653045932 - timeline_events: list[EventBase] = [] + timeline_events: list[FilteredEvent] = [] bundled_aggregations: dict[str, BundledAggregations] | None = None limited: bool | None = None prev_batch_token: StreamToken | None = None @@ -739,7 +743,7 @@ async def get_room_sync_data( # Use `stream_ordering` for updates else paginate_room_events_by_stream_ordering ) - timeline_events, new_room_key, limited = await pagination_method( + raw_timeline_events, new_room_key, limited = await pagination_method( room_id=room_id, # The bounds are reversed so we can paginate backwards # (from newer to older events) starting at to_bound. @@ -752,13 +756,13 @@ async def get_room_sync_data( # We want to return the events in ascending order (the last event is the # most recent). - timeline_events.reverse() + raw_timeline_events.reverse() # Make sure we don't expose any events that the client shouldn't see timeline_events = await filter_and_transform_events_for_client( self.storage_controllers, user.to_string(), - timeline_events, + raw_timeline_events, is_peeking=room_membership_for_user_at_to_token.membership != Membership.JOIN, filter_send_to_client=True, @@ -778,12 +782,17 @@ async def get_room_sync_data( if from_token is not None: for timeline_event in reversed(timeline_events): # This fields should be present for all persisted events - assert timeline_event.internal_metadata.stream_ordering is not None - assert timeline_event.internal_metadata.instance_name is not None + assert ( + timeline_event.event.internal_metadata.stream_ordering + is not None + ) + assert ( + timeline_event.event.internal_metadata.instance_name is not None + ) persisted_position = PersistedEventPosition( - instance_name=timeline_event.internal_metadata.instance_name, - stream=timeline_event.internal_metadata.stream_ordering, + instance_name=timeline_event.event.internal_metadata.instance_name, + stream=timeline_event.event.internal_metadata.stream_ordering, ) if persisted_position.persisted_after( from_token.stream_token.room_key @@ -1061,13 +1070,13 @@ async def get_room_sync_data( if timeline_events is not None: for timeline_event in timeline_events: # Anyone who sent a message is relevant - timeline_membership.add(timeline_event.sender) + timeline_membership.add(timeline_event.event.sender) # We also care about invite, ban, kick, targets, # etc. - if timeline_event.type == EventTypes.Member: + if timeline_event.event.type == EventTypes.Member: timeline_membership.add( - timeline_event.state_key + timeline_event.event.state_key ) # The client needs to know the membership of everyone in @@ -1480,7 +1489,7 @@ async def _get_bump_stamp( self, room_id: str, to_token: StreamToken, - timeline: list[EventBase], + timeline: list[FilteredEvent], check_outside_timeline: bool, ) -> int | None: """Get a bump stamp for the room, if we have a bump event and it has @@ -1500,8 +1509,8 @@ async def _get_bump_stamp( # those matches. We iterate backwards and take the stream ordering # of the first event that matches the bump event types. for timeline_event in reversed(timeline): - if timeline_event.type in SLIDING_SYNC_DEFAULT_BUMP_EVENT_TYPES: - new_bump_stamp = timeline_event.internal_metadata.stream_ordering + if timeline_event.event.type in SLIDING_SYNC_DEFAULT_BUMP_EVENT_TYPES: + new_bump_stamp = timeline_event.event.internal_metadata.stream_ordering # All persisted events have a stream ordering assert new_bump_stamp is not None diff --git a/synapse/handlers/sliding_sync/extensions.py b/synapse/handlers/sliding_sync/extensions.py index 9b7a01df142..4a324e9661c 100644 --- a/synapse/handlers/sliding_sync/extensions.py +++ b/synapse/handlers/sliding_sync/extensions.py @@ -761,7 +761,7 @@ async def handle_previously_room(room_id: str) -> None: # in the timeline to avoid bloating and blowing up the sync response # as the number of users in the room increases. (this behavior is part of the spec) initial_rooms_and_event_ids = [ - (room_id, event.event_id) + (room_id, event.event.event_id) for room_id in initial_rooms if room_id in actual_room_response_map for event in actual_room_response_map[room_id].timeline_events diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index 8969d915836..216ef3b0710 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -852,11 +852,15 @@ async def _filter_relevant_rooms_to_send( previous_connection_state.room_configs.get(room_id) ) if prev_room_sync_config is not None: - # Always include rooms whose timeline limit has increased. - # (see the "XXX: Odd behavior" described below) + # Always include rooms whose effective config has + # expanded. This covers timeline-limit increases and + # required-state additions introduced by room + # subscriptions overriding list-derived params. if ( - prev_room_sync_config.timeline_limit - < room_config.timeline_limit + prev_room_sync_config.combine_room_sync_config( + room_config + ) + != prev_room_sync_config ): rooms_should_send.add(room_id) continue diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index c8ef5e2aa6c..c88f703ae98 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -43,6 +43,7 @@ from synapse.api.presence import UserPresenceState from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.events import EventBase +from synapse.events.utils import FilteredEvent from synapse.handlers.relations import BundledAggregations from synapse.logging import issue9533_logger from synapse.logging.context import current_context @@ -123,7 +124,7 @@ class SyncConfig: @attr.s(slots=True, frozen=True, auto_attribs=True) class TimelineBatch: prev_batch: StreamToken - events: Sequence[EventBase] + events: Sequence[FilteredEvent] limited: bool # A mapping of event ID to the bundled aggregations for the above events. # This is only calculated if limited is true. @@ -148,7 +149,7 @@ class JoinedSyncResult: state: StateMap[EventBase] ephemeral: list[JsonDict] account_data: list[JsonDict] - sticky: list[EventBase] + sticky: list[FilteredEvent] unread_notifications: JsonDict unread_thread_notifications: JsonDict summary: JsonDict | None @@ -699,6 +700,7 @@ async def _load_filtered_recents( log_kv({"limited": limited}) + filtered_recents: list[FilteredEvent] if potential_recents: recents = await sync_config.filter_collection.filter_room_timeline( potential_recents @@ -725,29 +727,32 @@ async def _load_filtered_recents( ) ) - recents = await filter_and_transform_events_for_client( + filtered_recents = await filter_and_transform_events_for_client( self._storage_controllers, sync_config.user.to_string(), recents, always_include_ids=current_state_ids, ) - log_kv({"recents_after_visibility_filtering": len(recents)}) + log_kv({"recents_after_visibility_filtering": len(filtered_recents)}) else: - recents = [] + filtered_recents = [] if not limited or block_all_timeline: prev_batch_token = upto_token - if recents: - assert recents[0].internal_metadata.stream_ordering + if filtered_recents: + assert filtered_recents[0].event.internal_metadata.stream_ordering room_key = RoomStreamToken( - stream=recents[0].internal_metadata.stream_ordering - 1 + stream=filtered_recents[ + 0 + ].event.internal_metadata.stream_ordering + - 1 ) prev_batch_token = upto_token.copy_and_replace( StreamKeyType.ROOM, room_key ) return TimelineBatch( - events=recents, prev_batch=prev_batch_token, limited=False + events=filtered_recents, prev_batch=prev_batch_token, limited=False ) filtering_factor = 2 @@ -764,7 +769,7 @@ async def _load_filtered_recents( elif since_token and not newly_joined_room: since_key = since_token.room_key - while limited and len(recents) < timeline_limit and max_repeat: + while limited and len(filtered_recents) < timeline_limit and max_repeat: # For initial `/sync`, we want to view a historical section of the # timeline; to fetch events by `topological_ordering` (best # representation of the room DAG as others were seeing it at the time). @@ -835,26 +840,35 @@ async def _load_filtered_recents( ) ) - loaded_recents = await filter_and_transform_events_for_client( + loaded_filtered_recents: list[ + FilteredEvent + ] = await filter_and_transform_events_for_client( self._storage_controllers, sync_config.user.to_string(), loaded_recents, always_include_ids=current_state_ids, ) - log_kv({"loaded_recents_after_client_filtering": len(loaded_recents)}) + log_kv( + { + "loaded_recents_after_client_filtering": len( + loaded_filtered_recents + ) + } + ) - loaded_recents.extend(recents) - recents = loaded_recents + loaded_filtered_recents.extend(filtered_recents) + filtered_recents = loaded_filtered_recents max_repeat -= 1 - if len(recents) > timeline_limit: + if len(filtered_recents) > timeline_limit: limited = True - recents = recents[-timeline_limit:] - assert recents[0].internal_metadata.stream_ordering + filtered_recents = filtered_recents[-timeline_limit:] + assert filtered_recents[0].event.internal_metadata.stream_ordering room_key = RoomStreamToken( - stream=recents[0].internal_metadata.stream_ordering - 1 + stream=filtered_recents[0].event.internal_metadata.stream_ordering + - 1 ) prev_batch_token = upto_token.copy_and_replace(StreamKeyType.ROOM, room_key) @@ -865,12 +879,12 @@ async def _load_filtered_recents( if limited or newly_joined_room: bundled_aggregations = ( await self._relations_handler.get_bundled_aggregations( - recents, sync_config.user.to_string() + filtered_recents, sync_config.user.to_string() ) ) return TimelineBatch( - events=recents, + events=filtered_recents, prev_batch=prev_batch_token, # Also mark as limited if this is a new room or there has been a gap # (to force client to paginate the gap). @@ -976,8 +990,8 @@ async def compute_summary( # ...or ones which are in the timeline... for ev in batch.events: - if ev.type == EventTypes.Member: - existing_members.add(ev.state_key) + if ev.event.type == EventTypes.Member: + existing_members.add(ev.event.state_key) # ...and then ensure any missing ones get included in state. missing_hero_event_ids = [ @@ -1084,32 +1098,34 @@ async def compute_state_delta( first_event_by_sender_map = {} for event in batch.events: # Build the map from user IDs to the first timeline event they sent. - if event.sender not in first_event_by_sender_map: - first_event_by_sender_map[event.sender] = event + if event.event.sender not in first_event_by_sender_map: + first_event_by_sender_map[event.event.sender] = event.event # When using `state_after`, there is no special treatment with # regards to state also being in the `timeline`. Always fetch # relevant membership regardless of whether the state event is in # the `timeline`. if sync_config.use_state_after: - members_to_fetch.add(event.sender) + members_to_fetch.add(event.event.sender) # For `state`, the client is supposed to do a flawed re-construction # of state over time by starting with the given `state` and layering # on state from the `timeline` as you go (flawed because state # resolution). In this case, we only need their membership in # `state` when their membership isn't already in the `timeline`. - elif (EventTypes.Member, event.sender) not in timeline_state: - members_to_fetch.add(event.sender) + elif (EventTypes.Member, event.event.sender) not in timeline_state: + members_to_fetch.add(event.event.sender) # FIXME: we also care about invite targets etc. - if event.is_state(): - timeline_state[(event.type, event.state_key)] = event.event_id + if event.event.is_state(): + timeline_state[(event.event.type, event.event.state_key)] = ( + event.event.event_id + ) else: timeline_state = { - (event.type, event.state_key): event.event_id + (event.event.type, event.event.state_key): event.event.event_id for event in batch.events - if event.is_state() + if event.event.is_state() } # Now calculate the state to return in the sync response for the room. @@ -1340,7 +1356,7 @@ async def _compute_state_delta_for_full_sync( # timeline, but that is good enough here. state_at_timeline_start = ( await self._state_storage_controller.get_state_ids_for_event( - batch.events[0].event_id, + batch.events[0].event.event_id, state_filter=state_filter, await_full_state=await_full_state, ) @@ -1470,10 +1486,10 @@ async def _compute_state_delta_for_incremental_sync( prev_event_id = last_event_id_prev_batch for e in batch.events: - if e.prev_event_ids() != [prev_event_id]: + if e.event.prev_event_ids() != [prev_event_id]: is_linear_timeline = False break - prev_event_id = e.event_id + prev_event_id = e.event.event_id if is_linear_timeline and not batch.limited: state_ids: StateMap[str] = {} @@ -1487,7 +1503,7 @@ async def _compute_state_delta_for_incremental_sync( state_ids = ( await self._state_storage_controller.get_state_ids_for_event( - batch.events[0].event_id, + batch.events[0].event.event_id, # we only want members! state_filter=StateFilter.from_types( (EventTypes.Member, member) @@ -1501,7 +1517,7 @@ async def _compute_state_delta_for_incremental_sync( if batch: state_at_timeline_start = ( await self._state_storage_controller.get_state_ids_for_event( - batch.events[0].event_id, + batch.events[0].event.event_id, state_filter=state_filter, await_full_state=await_full_state, ) @@ -2854,7 +2870,7 @@ async def _generate_room_entry( # if there are membership changes in the timeline, or # if membership has changed during a gappy sync, or # if this is an initial sync. - any(ev.type == EventTypes.Member for ev in batch.events) + any(ev.event.type == EventTypes.Member for ev in batch.events) or ( # XXX: this may include false positives in the form of LL # members which have snuck into state @@ -2870,7 +2886,7 @@ async def _generate_room_entry( if room_builder.rtype == "joined": unread_notifications: dict[str, int] = {} - sticky_events: list[EventBase] = [] + sticky_events: list[FilteredEvent] = [] if sticky_event_ids: # As per MSC4354: # Remove sticky events that are already in the timeline, else we will needlessly duplicate @@ -2880,7 +2896,7 @@ async def _generate_room_entry( # This is particularly important given the risk of sticky events spam since # anyone can send sticky events, so halving the bandwidth on average for each sticky # event is helpful. - timeline_event_id_set = {ev.event_id for ev in batch.events} + timeline_event_id_set = {ev.event.event_id for ev in batch.events} # Must preserve sticky event stream order sticky_event_ids = [ e for e in sticky_event_ids if e not in timeline_event_id_set @@ -3144,7 +3160,8 @@ def calculate_user_changes(self) -> tuple[AbstractSet[str], AbstractSet[str]]: if self.since_token: for joined_sync in self.joined: it = itertools.chain( - joined_sync.state.values(), joined_sync.timeline.events + joined_sync.state.values(), + (e.event for e in joined_sync.timeline.events), ) for event in it: if event.type == EventTypes.Member: diff --git a/synapse/handlers/thread_subscriptions.py b/synapse/handlers/thread_subscriptions.py index 539672c7fe1..29cb045d001 100644 --- a/synapse/handlers/thread_subscriptions.py +++ b/synapse/handlers/thread_subscriptions.py @@ -53,7 +53,7 @@ async def get_thread_subscription_settings( raise NotFoundError("No such thread root") return await self.store.get_subscription_for_thread( - user_id.to_string(), event.room_id, thread_root_event_id + user_id.to_string(), event.event.room_id, thread_root_event_id ) async def subscribe_user_to_thread( @@ -103,7 +103,7 @@ async def subscribe_user_to_thread( ) if autosub_cause_event is None: raise NotFoundError("Automatic subscription event not found") - relation = relation_from_event(autosub_cause_event) + relation = relation_from_event(autosub_cause_event.event) if ( relation is None or relation.rel_type != RelationTypes.THREAD @@ -115,7 +115,9 @@ async def subscribe_user_to_thread( errcode=Codes.MSC4306_NOT_IN_THREAD, ) - automatic_event_orderings = EventOrderings.from_event(autosub_cause_event) + automatic_event_orderings = EventOrderings.from_event( + autosub_cause_event.event + ) else: automatic_event_orderings = None @@ -174,7 +176,7 @@ async def unsubscribe_user_from_thread( outcome = await self.store.unsubscribe_user_from_thread( user_id.to_string(), - event.room_id, + event.event.room_id, thread_root_event_id, ) diff --git a/synapse/media/preview_html.py b/synapse/media/preview_html.py index 22ad581f829..4f9315f4c91 100644 --- a/synapse/media/preview_html.py +++ b/synapse/media/preview_html.py @@ -278,17 +278,15 @@ def parse_html_to_open_graph(tree: "etree._Element") -> dict[str, str | None]: # "og:video:height" : "720", # "og:video:secure_url": "https://www.youtube.com/v/LXDBoHyjmtw?version=3", - og = _get_meta_tags(tree, "property", "og") + ogRoot = _get_meta_tags(tree, "property", "og") - # TODO: Search for properties specific to the different Open Graph types, - # such as article: meta tags, e.g.: - # - # "article:publisher" : "https://www.facebook.com/thethudonline" /> - # "article:author" content="https://www.facebook.com/thethudonline" /> - # "article:tag" content="baby" /> - # "article:section" content="Breaking News" /> - # "article:published_time" content="2016-03-31T19:58:24+00:00" /> - # "article:modified_time" content="2016-04-01T18:31:53+00:00" /> + # https://ogp.me/#type_article + ogArticle = _get_meta_tags(tree, "property", "article") + # https://ogp.me/#type_profile + ogProfile = _get_meta_tags(tree, "property", "profile") + + # Merge as-is + og = ogRoot | ogArticle | ogProfile # Search for Twitter Card (twitter:) meta tags, e.g.: # diff --git a/synapse/notifier.py b/synapse/notifier.py index 93d438def71..f1cec74462f 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -41,6 +41,7 @@ from synapse.api.constants import EduTypes, EventTypes, HistoryVisibility, Membership from synapse.api.errors import AuthError from synapse.events import EventBase +from synapse.events.utils import FilteredEvent from synapse.handlers.presence import format_user_presence_state from synapse.logging import issue9533_logger from synapse.logging.context import PreserveLoggingContext @@ -210,7 +211,7 @@ def new_listener(self, token: StreamToken) -> "Deferred[StreamToken]": @attr.s(slots=True, frozen=True, auto_attribs=True) class EventStreamResult: - events: list[JsonDict | EventBase] + events: list[JsonDict | FilteredEvent] start_token: StreamToken end_token: StreamToken @@ -765,7 +766,7 @@ async def check_for_updates( # The events fetched from each source are a JsonDict, EventBase, or # UserPresenceState, but see below for UserPresenceState being # converted to JsonDict. - events: list[JsonDict | EventBase] = [] + events: list[JsonDict | FilteredEvent] = [] end_token = from_token for keyname, source in self.event_sources.sources.get_sources(): diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index fdfae234be7..ca63a99e3e9 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -501,7 +501,7 @@ async def dispatch_push_event( "event_id": event.event_id, "room_id": event.room_id, "type": event.type, - "sender": event.user_id, + "sender": event.sender, "prio": priority, } if not self.disable_badge_count: diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index d18630e80ba..1ebbc6d4f39 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -543,8 +543,10 @@ async def _get_notif_vars( results.events_before + [notif_event], ) - for event in the_events: - messagevars = await self._get_message_vars(notif, event, room_state_ids) + for filtered_event in the_events: + messagevars = await self._get_message_vars( + notif, filtered_event.event, room_state_ids + ) if messagevars is not None: ret["messages"].append(messagevars) diff --git a/synapse/replication/tcp/streams/__init__.py b/synapse/replication/tcp/streams/__init__.py index 067847617fa..e41573cf689 100644 --- a/synapse/replication/tcp/streams/__init__.py +++ b/synapse/replication/tcp/streams/__init__.py @@ -39,6 +39,7 @@ PresenceStream, PushersStream, PushRulesStream, + QuarantinedMediaStream, ReceiptsStream, StickyEventsStream, Stream, @@ -73,6 +74,7 @@ ThreadSubscriptionsStream, UnPartialStatedRoomStream, UnPartialStatedEventStream, + QuarantinedMediaStream, ) } @@ -96,4 +98,5 @@ "ThreadSubscriptionsStream", "UnPartialStatedRoomStream", "UnPartialStatedEventStream", + "QuarantinedMediaStream", ] diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index 1ea6b4fa857..a73f767add2 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -808,3 +808,50 @@ async def _update_function( return [], to_token, False return rows, rows[-1][0], len(updates) == limit + + +@attr.s(slots=True, auto_attribs=True) +class QuarantinedMediaStreamRow: + """Row for QuarantinedMediaStream""" + + # We store the origin and media_id as media is scoped to the origin and are uniquely + # identified by (origin, media_id). + + origin: str + media_id: str + quarantined: bool + + +class QuarantinedMediaStream(_StreamFromIdGen): + """Stream to track changes to (un)quarantined media.""" + + NAME = "quarantined_media" + ROW_TYPE = QuarantinedMediaStreamRow + + def __init__(self, hs: "HomeServer"): + self.store = hs.get_datastores().main + super().__init__( + hs.get_instance_name(), + self._update_function, + self.store._quarantined_media_changes_id_gen, + ) + + async def _update_function( + self, instance_name: str, from_token: Token, to_token: Token, limit: int + ) -> StreamUpdateResult: + updates = await self.store.get_quarantined_media_changes( + from_id=from_token, to_id=to_token, limit=limit + ) + rows = [ + ( + update.stream_id, + # Args to `QuarantinedMediaStreamRow` + (update.origin, update.media_id, update.quarantined), + ) + for update in updates + ] + + if not rows: + return [], to_token, False + + return rows, rows[-1][0], len(updates) == limit diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index b209404cd18..0774b6ed405 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -96,6 +96,10 @@ LargestRoomsStatistics, UserMediaStatisticsRestServlet, ) +from synapse.rest.admin.user_reports import ( + UserReportDetailRestServlet, + UserReportsRestServlet, +) from synapse.rest.admin.username_available import UsernameAvailableRestServlet from synapse.rest.admin.users import ( AccountDataRestServlet, @@ -312,6 +316,8 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: LargestRoomsStatistics(hs).register(http_server) EventReportDetailRestServlet(hs).register(http_server) EventReportsRestServlet(hs).register(http_server) + UserReportsRestServlet(hs).register(http_server) + UserReportDetailRestServlet(hs).register(http_server) AccountDataRestServlet(hs).register(http_server) PushersRestServlet(hs).register(http_server) MakeRoomAdminRestServlet(hs).register(http_server) diff --git a/synapse/rest/admin/background_updates.py b/synapse/rest/admin/background_updates.py index 96190c416d4..e693a0afd3a 100644 --- a/synapse/rest/admin/background_updates.py +++ b/synapse/rest/admin/background_updates.py @@ -18,6 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # +import json import logging from http import HTTPStatus from typing import TYPE_CHECKING @@ -150,6 +151,24 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: "populate_user_directory_process_users", ), ] + elif job_name == "event_resign": + old_key = body.get("old_key") + if old_key is not None and not isinstance(old_key, str): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "'old_key' must be a string", + ) + before_ts = body.get("before_ts") + if before_ts is not None and not isinstance(before_ts, int): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "'before_ts' must be an integer", + ) + progress = { + "old_key": old_key, + "before_ts": before_ts, + } + jobs = [("event_resign", json.dumps(progress), "")] else: raise SynapseError(HTTPStatus.BAD_REQUEST, "Invalid job_name") diff --git a/synapse/rest/admin/events.py b/synapse/rest/admin/events.py index 8da7a67820a..1c311b04713 100644 --- a/synapse/rest/admin/events.py +++ b/synapse/rest/admin/events.py @@ -3,6 +3,7 @@ from synapse.api.errors import NotFoundError from synapse.events.utils import ( + FilteredEvent, SerializeEventConfig, format_event_raw, ) @@ -66,7 +67,9 @@ async def on_GET( ) res = { "event": await self._event_serializer.serialize_event( - event, self._clock.time_msec(), config=config + FilteredEvent.admin_override(event), + self._clock.time_msec(), + config=config, ) } diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py index d5346fe0d5c..1633cca884a 100644 --- a/synapse/rest/admin/media.py +++ b/synapse/rest/admin/media.py @@ -230,6 +230,80 @@ async def on_POST( return HTTPStatus.OK, {} +class ListQuarantineChanges(RestServlet): + """Lists the quarantine changes to media. + + Uses the pagination format described by https://spec.matrix.org/v1.18/appendices/#pagination + """ + + PATTERNS = admin_patterns("/media/quarantine_changes$") + + def __init__(self, hs: "HomeServer"): + self.store = hs.get_datastores().main + self.auth = hs.get_auth() + self.server_name = hs.hostname + self.replication = hs.get_replication_data_handler() + + async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: + await assert_requester_is_admin(self.auth, request) + + from_id = parse_integer(request, "from", default=0) + limit = 100 # arbitrary; not enough to cause problems (hopefully) + + # Validate the `from` token + max_id = await self.store.get_max_allocated_quarantined_media_stream_id() + if from_id > max_id: + # The caller is trying to get future data, which we don't allow because + # we know it's an invalid state that should never happen. We could + # wait until we reach the token but we might as well not waste our + # resources on that which is why `wait_for_quarantined_media_stream_id(...)` + # has assertions around this. + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "The `from` token is considered invalid because it includes stream positions " + "greater than the furthest persisted position across all of the workers." + "This indicates either a Synapse programming error (as we should never hand out " + "invalid future tokens) or a fabricated `from` token. If you've modified the token, " + "you can try paginating from the beginning again.", + errcode=Codes.INVALID_PARAM, + ) + + # We need to wait to ensure that our current worker is actually caught up with + # the stream position, otherwise we might not return what we think we're returning. + if not await self.store.wait_for_quarantined_media_stream_id(from_id): + raise SynapseError( + HTTPStatus.INTERNAL_SERVER_ERROR, + "Timed out while waiting for the worker serving this request to catch up to the given " + "`from` stream position. Assuming this is a valid `from` token, this indicates an issue " + "with Synapse or the worker deployment lagging behind the replication stream. Please try " + "the request again later.", + errcode=Codes.UNKNOWN, + ) + + to_id = await self.store.get_current_quarantined_media_stream_id() + changes = await self.store.get_quarantined_media_changes( + from_id=from_id, + to_id=to_id, + limit=limit, + ) + + serialized_changes = [ + { + "origin": c.origin if c.origin is not None else self.server_name, + "media_id": c.media_id, + "quarantined": c.quarantined, + } + for c in changes + ] + + # We know the last record will have the highest stream ID, so use that one. If + # there aren't any records, just return the `to_id` value because it'll be the + # furthest stream position possible. + next_batch = changes[-1].stream_id if len(changes) > 0 else to_id + + return HTTPStatus.OK, {"next_batch": next_batch, "changes": serialized_changes} + + class ProtectMediaByID(RestServlet): """Protect local media from being quarantined.""" @@ -529,6 +603,7 @@ def register_servlets_for_media_repo(hs: "HomeServer", http_server: HttpServer) QuarantineMediaByID(hs).register(http_server) UnquarantineMediaByID(hs).register(http_server) QuarantineMediaByUser(hs).register(http_server) + ListQuarantineChanges(hs).register(http_server) ProtectMediaByID(hs).register(http_server) UnprotectMediaByID(hs).register(http_server) ListMediaInRoom(hs).register(http_server) diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index a886859ffab..61511b93601 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -29,6 +29,7 @@ from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError from synapse.api.filtering import Filter from synapse.events.utils import ( + FilteredEvent, SerializeEventConfig, ) from synapse.handlers.pagination import ( @@ -529,7 +530,9 @@ async def on_GET( ) events = await self.store.get_events(event_ids.values()) now = self.clock.time_msec() - room_state = await self._event_serializer.serialize_events(events.values(), now) + room_state = await self._event_serializer.serialize_events( + [FilteredEvent.state(e) for e in events.values()], now + ) ret = {"state": room_state} return HTTPStatus.OK, ret @@ -897,7 +900,8 @@ async def on_GET( bundle_aggregations=event_context.aggregations, ), "state": await self._event_serializer.serialize_events( - event_context.state, time_now + [FilteredEvent.state(e) for e in event_context.state], + time_now, ), "start": event_context.start, "end": event_context.end, diff --git a/synapse/rest/admin/user_reports.py b/synapse/rest/admin/user_reports.py new file mode 100644 index 00000000000..119dc865172 --- /dev/null +++ b/synapse/rest/admin/user_reports.py @@ -0,0 +1,173 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + + +import logging +from http import HTTPStatus +from typing import TYPE_CHECKING + +from synapse.api.constants import Direction +from synapse.api.errors import Codes, NotFoundError, SynapseError +from synapse.http.servlet import RestServlet, parse_enum, parse_integer, parse_string +from synapse.http.site import SynapseRequest +from synapse.rest.admin._base import admin_patterns, assert_requester_is_admin +from synapse.types import JsonDict + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class UserReportsRestServlet(RestServlet): + """ + List all reported users that are known to the homeserver. Results are returned + in a dictionary containing report information. Supports pagination. + The requester must have administrator access in Synapse. + + GET /_synapse/admin/v1/user_reports + returns: + 200 OK with list of reports if success otherwise an error. + + Args: + The parameters `from` and `limit` are required only for pagination. + By default, a `limit` of 100 is used. + The parameter `dir` can be used to define the order of results. + The `user_id` query parameter filters by the user ID of the reporter of the target user. + The `target_user_id` query parameter filters by user id of the target user. + Returns: + A list of user reprots and an integer representing the total number of user + reports that exist given this query + """ + + PATTERNS = admin_patterns("/user_reports$") + + def __init__(self, hs: "HomeServer"): + self._auth = hs.get_auth() + self._store = hs.get_datastores().main + + async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: + await assert_requester_is_admin(self._auth, request) + + start = parse_integer(request, "from", default=0) + limit = parse_integer(request, "limit", default=100) + direction = parse_enum(request, "dir", Direction, Direction.BACKWARDS) + user_id = parse_string(request, "user_id") + target_user_id = parse_string(request, "target_user_id") + + if start < 0: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "The start parameter must be a positive integer.", + errcode=Codes.INVALID_PARAM, + ) + + if limit < 0: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "The limit parameter must be a positive integer.", + errcode=Codes.INVALID_PARAM, + ) + + user_reports, total = await self._store.get_user_reports_paginate( + start, limit, direction, user_id, target_user_id + ) + ret = {"user_reports": user_reports, "total": total} + if (start + limit) < total: + ret["next_token"] = start + len(user_reports) + + return HTTPStatus.OK, ret + + +class UserReportDetailRestServlet(RestServlet): + """ + Get a specific user report that is known to the homeserver. Results are returned + in a dictionary containing report information. + The requester must have administrator access in Synapse. + + GET /_synapse/admin/v1/user_reports/ + returns: + 200 OK with details report if success otherwise an error. + + Args: + The parameter `report_id` is the ID of the user report in the database. + Returns: + JSON blob of information about the user report + """ + + PATTERNS = admin_patterns("/user_reports/(?P[^/]*)$") + + def __init__(self, hs: "HomeServer"): + self._auth = hs.get_auth() + self._store = hs.get_datastores().main + + async def on_GET( + self, request: SynapseRequest, report_id: str + ) -> tuple[int, JsonDict]: + await assert_requester_is_admin(self._auth, request) + + message = ( + "The report_id parameter must be a string representing a positive integer." + ) + try: + resolved_report_id = int(report_id) + except ValueError: + raise SynapseError( + HTTPStatus.BAD_REQUEST, message, errcode=Codes.INVALID_PARAM + ) + + if resolved_report_id < 0: + raise SynapseError( + HTTPStatus.BAD_REQUEST, message, errcode=Codes.INVALID_PARAM + ) + + ret = await self._store.get_user_report(resolved_report_id) + if not ret: + raise NotFoundError("User report not found") + + id, received_ts, target_user_id, user_id, reason = ret + response = { + "id": id, + "received_ts": received_ts, + "target_user_id": target_user_id, + "user_id": user_id, + "reason": reason, + } + + return HTTPStatus.OK, response + + async def on_DELETE( + self, request: SynapseRequest, report_id: str + ) -> tuple[int, JsonDict]: + await assert_requester_is_admin(self._auth, request) + + message = ( + "The report_id parameter must be a string representing a positive integer." + ) + try: + resolved_report_id = int(report_id) + except ValueError: + raise SynapseError( + HTTPStatus.BAD_REQUEST, message, errcode=Codes.INVALID_PARAM + ) + + if resolved_report_id < 0: + raise SynapseError( + HTTPStatus.BAD_REQUEST, message, errcode=Codes.INVALID_PARAM + ) + + if await self._store.delete_user_report(resolved_report_id): + return HTTPStatus.OK, {} + + raise NotFoundError("User report not found") diff --git a/synapse/rest/client/auth.py b/synapse/rest/client/auth.py index f3254990443..566c9c98c50 100644 --- a/synapse/rest/client/auth.py +++ b/synapse/rest/client/auth.py @@ -20,13 +20,13 @@ # import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from twisted.web.server import Request from synapse.api.auth.mas import MasDelegatedAuth from synapse.api.constants import LoginType -from synapse.api.errors import LoginError, SynapseError +from synapse.api.errors import Codes, LoginError, SynapseError from synapse.api.urls import CLIENT_API_PREFIX from synapse.http.server import HttpServer, respond_with_html, respond_with_redirect from synapse.http.servlet import RestServlet, parse_string @@ -61,12 +61,27 @@ def __init__(self, hs: "HomeServer"): hs.config.registration.registration_token_template ) self.success_template = hs.config.registration.fallback_success_template + self._msc4450_enabled = hs.config.experimental.msc4450_enabled async def on_GET(self, request: SynapseRequest, stagetype: str) -> None: session = parse_string(request, "session") if not session: raise SynapseError(400, "No session supplied") + # MSC4450 query parameter which allows clients to specify the Identity Provider + # they wish to use for legacy SSO during User-Interactive Authentication. + idp_id: Optional[str] = None + + if self._msc4450_enabled: + idp_id = parse_string(request, "io.element.idp_id") + + if idp_id is not None and stagetype != LoginType.SSO: + raise SynapseError( + 400, + Codes.INVALID_PARAM, + "idp_id can only be specified for the `m.login.sso` auth type", + ) + # We support the unstable (`org.matrix.cross_signing_reset`) name from MSC4312 until # enough clients have adopted the stable name (`m.oauth`). # Note: `org.matrix.cross_signing_reset` *is* the stable name of the *action* in the @@ -118,7 +133,7 @@ async def on_GET(self, request: SynapseRequest, stagetype: str) -> None: elif stagetype == LoginType.SSO: # Display a confirmation page which prompts the user to # re-authenticate with their SSO provider. - html = await self.auth_handler.start_sso_ui_auth(request, session) + html = await self.auth_handler.start_sso_ui_auth(request, session, idp_id) elif stagetype == LoginType.REGISTRATION_TOKEN: html = self.registration_token_template.render( diff --git a/synapse/rest/client/keys.py b/synapse/rest/client/keys.py index 89b68331f2e..2c65a55ea17 100644 --- a/synapse/rest/client/keys.py +++ b/synapse/rest/client/keys.py @@ -159,6 +159,23 @@ class KeyObject(RequestBodyModel): device_keys: DeviceKeys | None = None """Identity keys for the device. May be absent if no new identity keys are required.""" + @field_validator("device_keys", mode="before") + @classmethod + def validate_device_keys_not_null(cls, v: Any) -> Any: + """Reject explicit `null` for `device_keys` while still allowing + the field to be omitted (in which case the default `None` is used). + + The spec says `device_keys` may be omitted, but when present it + must be a `DeviceKeys` object — not `null`. + + Pydantic's experimental `Missing` sentinel would be a cleaner way + to express this, but it's not stable yet: + https://docs.pydantic.dev/latest/concepts/experimental/#missing-sentinel + """ + if v is None: + raise ValueError("device_keys must not be null") + return v + fallback_keys: Mapping[StrictStr, StrictStr | KeyObject] | None = None """ The public key which should be used if the device's one-time keys are @@ -241,7 +258,7 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: 400, "To upload keys, you must pass device_id when authenticating" ) - if "device_keys" in body and isinstance(body["device_keys"], dict): + if "device_keys" in body: # Validate the provided `user_id` and `device_id` fields in # `device_keys` match that of the requesting user. We can't do # this directly in the pydantic model as we don't have access @@ -249,13 +266,13 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: # # TODO: We could use ValidationInfo when we switch to Pydantic v2. # https://docs.pydantic.dev/latest/concepts/validators/#validation-info - if body["device_keys"].get("user_id") != user_id: + if body["device_keys"]["user_id"] != user_id: raise SynapseError( code=HTTPStatus.BAD_REQUEST, errcode=Codes.BAD_JSON, msg="Provided `user_id` in `device_keys` does not match that of the authenticated user", ) - if body["device_keys"].get("device_id") != device_id: + if body["device_keys"]["device_id"] != device_id: raise SynapseError( code=HTTPStatus.BAD_REQUEST, errcode=Codes.BAD_JSON, diff --git a/synapse/rest/client/media.py b/synapse/rest/client/media.py index 4db3b015762..15f58acb955 100644 --- a/synapse/rest/client/media.py +++ b/synapse/rest/client/media.py @@ -253,6 +253,7 @@ async def on_GET( ), send_cors=True, ) + return set_cors_headers(request) set_corp_headers(request) diff --git a/synapse/rest/client/notifications.py b/synapse/rest/client/notifications.py index 2420e9fffbe..f80a43b2978 100644 --- a/synapse/rest/client/notifications.py +++ b/synapse/rest/client/notifications.py @@ -24,6 +24,7 @@ from synapse.api.constants import ReceiptTypes from synapse.events.utils import ( + FilteredEvent, SerializeEventConfig, format_event_for_client_v2_without_room_id, ) @@ -111,7 +112,7 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: "ts": pa.received_ts, "event": ( await self._event_serializer.serialize_event( - notif_events[pa.event_id], + FilteredEvent(event=notif_events[pa.event_id], membership=None), now, config=serialize_options, ) diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 65d9c130efc..48ef42c7d6f 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -53,6 +53,7 @@ from synapse.api.filtering import Filter from synapse.events.utils import ( EventClientSerializer, + FilteredEvent, SerializeEventConfig, format_event_for_client_v2, ) @@ -286,7 +287,7 @@ async def on_GET( if format == "event": event = await self._event_serializer.serialize_event( - data, + FilteredEvent.state(data), self.clock.time_msec(), config=SerializeEventConfig( event_format=format_event_for_client_v2, @@ -866,7 +867,9 @@ async def encode_messages_response( serialized_result[ "state" ] = await serialize_deps.event_serializer.serialize_events( - get_messages_result.state, time_now, config=serialize_options + [FilteredEvent.state(e) for e in get_messages_result.state], + time_now, + config=serialize_options, ) return serialized_result @@ -1172,7 +1175,7 @@ async def on_GET( config=serializer_options, ), "state": await self._event_serializer.serialize_events( - event_context.state, + [FilteredEvent.state(e) for e in event_context.state], time_now, config=serializer_options, ), @@ -1715,16 +1718,18 @@ async def on_GET( class RoomSummaryRestServlet(ResolveRoomIdMixin, RestServlet): PATTERNS = ( - # deprecated endpoint, to be removed + # deprecated unstable endpoint, to be removed re.compile( "^/_matrix/client/unstable/im.nheko.summary" "/rooms/(?P[^/]*)/summary$" ), - # recommended endpoint + # recommended unstable endpoint re.compile( "^/_matrix/client/unstable/im.nheko.summary" "/summary/(?P[^/]*)$" ), + # stable endpoint + re.compile("^/_matrix/client/v1/room_summary/(?P[^/]*)$"), ) CATEGORY = "Client API requests" @@ -1772,8 +1777,7 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: RoomTypingRestServlet(hs).register(http_server) RoomEventContextServlet(hs).register(http_server) RoomHierarchyRestServlet(hs).register(http_server) - if hs.config.experimental.msc3266_enabled: - RoomSummaryRestServlet(hs).register(http_server) + RoomSummaryRestServlet(hs).register(http_server) RoomEventServlet(hs).register(http_server) JoinedRoomsRestServlet(hs).register(http_server) RoomAliasListServlet(hs).register(http_server) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 710d097eab0..c3cf0dc3c4d 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -18,7 +18,6 @@ # [This file includes modifications made by New Vector Limited] # # -import itertools import logging from collections import defaultdict from typing import TYPE_CHECKING, Any, Mapping @@ -31,6 +30,7 @@ from synapse.api.presence import UserPresenceState from synapse.api.ratelimiting import Ratelimiter from synapse.events.utils import ( + FilteredEvent, SerializeEventConfig, format_event_for_client_v2_without_room_id, format_event_raw, @@ -448,7 +448,9 @@ async def encode_invited( invited = {} for room in rooms: invite = await self._event_serializer.serialize_event( - room.invite, time_now, config=serialize_options + FilteredEvent.state(event=room.invite), + time_now, + config=serialize_options, ) unsigned = dict(invite.get("unsigned", {})) invite["unsigned"] = unsigned @@ -484,7 +486,9 @@ async def encode_knocked( knocked = {} for room in rooms: knock = await self._event_serializer.serialize_event( - room.knock, time_now, config=serialize_options + FilteredEvent.state(event=room.knock), + time_now, + config=serialize_options, ) # Extract the `unsigned` key from the knock event. @@ -574,7 +578,7 @@ async def encode_room( state_events = state_dict.values() - for event in itertools.chain(state_events, timeline_events): + for event in state_events: # We've had bug reports that events were coming down under the # wrong room. if event.room_id != room.room_id: @@ -584,9 +588,21 @@ async def encode_room( room.room_id, event.room_id, ) + for filtered_event in timeline_events: + # We've had bug reports that events were coming down under the + # wrong room. + if filtered_event.event.room_id != room.room_id: + logger.warning( + "Event %r is under room %r instead of %r", + filtered_event.event.event_id, + room.room_id, + filtered_event.event.room_id, + ) serialized_state = await self._event_serializer.serialize_events( - state_events, time_now, config=serialize_options + [FilteredEvent.state(e) for e in state_events], + time_now, + config=serialize_options, ) serialized_timeline = await self._event_serializer.serialize_events( timeline_events, @@ -974,7 +990,7 @@ async def encode_rooms( ): serialized_required_state = ( await self.event_serializer.serialize_events( - room_result.required_state, + [FilteredEvent.state(e) for e in room_result.required_state], time_now, config=serialize_options, ) diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 7bf4b12e8b7..bb1711f2cf2 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -206,6 +206,8 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: "org.matrix.msc4354": self.config.experimental.msc4354_enabled, # MSC4380: Invite blocking "org.matrix.msc4380.stable": True, + # MSC4445: Sync timeline order + "org.matrix.msc4445.initial_sync_timeline_topological_ordering": True, }, }, ) diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index a92233c863e..2f0e3f2c3e5 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -37,7 +37,7 @@ from synapse.api.constants import EventTypes from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, StateResolutionVersions -from synapse.events import EventBase +from synapse.events import EventBase, FrozenEventVMSC4242 from synapse.events.snapshot import ( EventContext, UnpersistedEventContext, @@ -239,31 +239,6 @@ async def compute_state_after_events( ) return await ret.get_state(self._state_storage_controller, state_filter) - async def get_current_user_ids_in_room( - self, room_id: str, latest_event_ids: StrCollection - ) -> set[str]: - """ - Get the users IDs who are currently in a room. - - Note: This is much slower than using the equivalent method - `DataStore.get_users_in_room` or `DataStore.get_users_in_room_with_profiles`, - so this should only be used when wanting the users at a particular point - in the room. - - Args: - room_id: The ID of the room. - latest_event_ids: Precomputed list of latest event IDs. Will be computed if None. - Returns: - Set of user IDs in the room. - """ - - assert latest_event_ids is not None - - logger.debug("calling resolve_state_groups from get_current_user_ids_in_room") - entry = await self.resolve_state_groups_for_events(room_id, latest_event_ids) - state = await entry.get_state(self._state_storage_controller, StateFilter.all()) - return await self.store.get_joined_user_ids_from_state(room_id, state) - async def get_hosts_in_room_at_events( self, room_id: str, event_ids: StrCollection ) -> frozenset[str]: @@ -303,7 +278,8 @@ async def calculate_context_info( membership events. `False` if `state_ids_before_event` is the full state. `None` when `state_ids_before_event` is not provided. In this case, the - flag will be calculated based on `event`'s prev events. + flag will be calculated based on `event`'s `prev_events` or `prev_state_events` + for state DAG rooms. state_group_before_event: the current state group at the time of event, if known Returns: @@ -337,7 +313,11 @@ async def calculate_context_info( # (This is slightly racy - the prev-events might get fixed up before we use # their states - but I don't think that really matters; it just means we # might redundantly recalculate the state for this event later.) - prev_event_ids = event.prev_event_ids() + prev_event_ids = frozenset( + event.prev_state_events + if isinstance(event, FrozenEventVMSC4242) + else event.prev_event_ids() + ) incomplete_prev_events = await self.store.get_partial_state_events( prev_event_ids ) @@ -355,7 +335,7 @@ async def calculate_context_info( entry = await self.resolve_state_groups_for_events( event.room_id, - event.prev_event_ids(), + prev_event_ids, await_full_state=False, ) diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index 2948227807f..7cc6a39639a 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -35,6 +35,7 @@ Generic, Iterable, TypeVar, + cast, ) import attr @@ -43,7 +44,9 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership -from synapse.events import EventBase +from synapse.api.errors import SynapseError +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS +from synapse.events import EventBase, FrozenEventVMSC4242, event_exists_in_state_dag from synapse.events.snapshot import EventContext, EventPersistencePair from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable @@ -68,6 +71,7 @@ from synapse.types.state import StateFilter from synapse.util.async_helpers import ObservableDeferred, yieldable_gather_results from synapse.util.metrics import Measure +from synapse.util.stringutils import shortstr if TYPE_CHECKING: from synapse.server import HomeServer @@ -111,6 +115,14 @@ buckets=(0, 1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf"), ) +# The number of forward extremities for each new event. +msc4242_state_dag_forward_extremities_counter = Histogram( + "synapse_storage_msc4242_state_dag_forward_extremities_persisted", + "Number of forward extremities for each new event in the state DAG", + labelnames=[SERVER_NAME_LABEL], + buckets=(1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf"), +) + state_resolutions_during_persistence = Counter( "synapse_storage_events_state_resolutions_during_persistence", "Number of times we had to do state res to calculate new current state", @@ -529,7 +541,15 @@ async def _calculate_current_state(self, room_id: str) -> StateMap[str]: Returns: map from (type, state_key) to event id for the current state in the room """ - latest_event_ids = await self.main_store.get_latest_event_ids_in_room(room_id) + room_version = await self.main_store.get_room_version_id(room_id) + room_version_obj = KNOWN_ROOM_VERSIONS[room_version] + if room_version_obj.msc4242_state_dags: + latest_event_ids = await self.main_store.get_state_dag_extremities(room_id) + else: + latest_event_ids = await self.main_store.get_latest_event_ids_in_room( + room_id + ) + state_groups = set( ( await self.main_store._get_state_group_for_events(latest_event_ids) @@ -551,7 +571,6 @@ async def _calculate_current_state(self, room_id: str) -> StateMap[str]: # Avoid a circular import. from synapse.state import StateResolutionStore - room_version = await self.main_store.get_room_version_id(room_id) res = await self._state_resolution_handler.resolve_state_groups( room_id, room_version, @@ -615,28 +634,52 @@ async def _persist_event_batch( for x in range(0, len(events_and_contexts), 100) ] + # Get the room version for the first event. This room version is the same for all events + # as events_and_contexts is all for one room. + assert len(events_and_contexts) > 0 + room_version = events_and_contexts[0][0].room_version + for chunk in chunks: # We can't easily parallelize these since different chunks # might contain the same event. :( new_forward_extremities = None state_delta_for_room = None + new_state_dag_extrems = None if not backfilled: - with Measure( - self._clock, - name="_calculate_state_and_extrem", - server_name=self.server_name, - ): - # Work out the new "current state" for the room. - # We do this by working out what the new extremities are and then - # calculating the state from that. - ( - new_forward_extremities, - state_delta_for_room, - ) = await self._calculate_new_forward_extremities_and_state_delta( - room_id, chunk - ) + if room_version.msc4242_state_dags: + with Measure( + self._clock, + name="_process_state_dag_forward_extremities_and_state_delta", + server_name=self.server_name, + ): + assert all( + isinstance(ev, FrozenEventVMSC4242) for ev, _ in chunk + ) + ( + new_forward_extremities, # for prev_events + state_delta_for_room, # for state groups + new_state_dag_extrems, # for prev_state_events + ) = await self._process_state_dag_forward_extremities_and_state_delta( + room_id, + cast(list[tuple[FrozenEventVMSC4242, EventContext]], chunk), + ) + else: + with Measure( + self._clock, + name="_calculate_state_and_extrem", + server_name=self.server_name, + ): + # Work out the new "current state" for the room. + # We do this by working out what the new extremities are and then + # calculating the state from that. + ( + new_forward_extremities, + state_delta_for_room, + ) = await self._calculate_new_forward_extremities_and_state_delta( + room_id, chunk + ) with Measure( self._clock, @@ -666,6 +709,7 @@ async def _persist_event_batch( use_negative_stream_ordering=backfilled, inhibit_local_membership_updates=backfilled, new_event_links=new_event_links, + new_state_dag_forward_extremities=new_state_dag_extrems, ) return replaced_events @@ -793,6 +837,216 @@ async def _calculate_new_forward_extremities_and_state_delta( return (new_forward_extremities, delta) + async def _process_state_dag_forward_extremities_and_state_delta( + self, + room_id: str, + event_contexts: list[tuple[FrozenEventVMSC4242, EventContext]], + ) -> tuple[set[str] | None, DeltaState | None, set[str] | None]: + """Process the forwards extremities for state DAG rooms. + Returns: + - the new room dag extremities which should be written when these events are persisted. + - the state delta for the room, if applicable. + - the new state dag extremities which should be written when these events are persisted. + + NB: this does not write them because if it did, new events may see them _before_ the events + get persisted, causing failures in retrieving state groups. + """ + # Update forward extremities + # ...for the state DAG + existing_state_dag_fwd_extrems = ( + await self.main_store.get_state_dag_extremities(room_id) + ) + new_state_dag_fwd_extrems = await self._calculate_new_state_dag_extremities( + room_id, + existing_state_dag_fwd_extrems, + event_contexts, + ) + # ...and the room DAG + existing_room_dag_fwd_extrems = ( + await self.main_store.get_latest_event_ids_in_room(room_id) + ) + new_room_dag_fwd_extrems = await self._calculate_new_extremities( + room_id, + cast(list[EventPersistencePair], event_contexts), + existing_room_dag_fwd_extrems, + ) + assert new_room_dag_fwd_extrems, ( + f"No room dag forward extremities left in room {room_id}!" + ) + + # See if we need to calculate a state delta + if new_state_dag_fwd_extrems == existing_state_dag_fwd_extrems: + # No change in state extremities, so no new state to calculate + return new_room_dag_fwd_extrems, None, new_state_dag_fwd_extrems + + with Measure( + self._clock, + name="persist_events.state_dag.get_new_state_after_events", + server_name=self.server_name, + ): + (current_state, delta_ids, _) = await self._get_new_state_after_events( + room_id, + cast(list[EventPersistencePair], event_contexts), + existing_state_dag_fwd_extrems, + new_state_dag_fwd_extrems, + # do not prune forward extremities in the state DAG + # else we lose eventual delivery + should_prune=False, + ) + + # Following logic cargoculted from _calculate_new_forward_extremities_and_state_delta + # If either are not None then there has been a change, + # and we need to work out the delta (or use that + # given) + delta = None + if delta_ids is not None: + # If there is a delta we know that we've + # only added or replaced state, never + # removed keys entirely. + delta = DeltaState([], delta_ids) + elif current_state is not None: + with Measure( + self._clock, + name="persist_events.calculate_state_delta", + server_name=self.server_name, + ): + delta = await self._calculate_state_delta(room_id, current_state) + + if delta: + # If we have a change of state then lets check + # whether we're actually still a member of the room, + # or if our last user left. If we're no longer in + # the room then we delete the current state and + # extremities. + is_still_joined = await self._is_server_still_joined( + room_id, + cast(list[EventPersistencePair], event_contexts), + delta, + ) + if not is_still_joined: + logger.info("Server no longer in room %s", room_id) + delta.no_longer_in_room = True + + return new_room_dag_fwd_extrems, delta, new_state_dag_fwd_extrems + + async def _calculate_new_state_dag_extremities( + self, + room_id: str, + existing_fwd_extrems: frozenset[str], + event_contexts: list[tuple[FrozenEventVMSC4242, EventContext]], + ) -> set[str]: + """Calculate the new state dag forward extremities. Modifies existing_fwd_extrems. + + Assumes that event_contexts are only state events which should be in the state DAG. + + Raises: + SynapseError: if the new events include unknown prev_state_events + AssertionError: if there are no state DAG forward extremities remaining in the room + """ + # Events are always processed in causal order without any gaps in the DAG + # (prev_state_events are always known), guaranteeing that processed events have a path to the + # create event. This is an emergent property of state DAGs as asserting that there is a path + # to the create event every time we insert an event would be prohibitively expensive. + # This is similar to how doubly-linked lists can potentially not refer to previous items correctly + # without verifying the list's integrity, but doing it on every insert is too expensive. + + # filter out events which don't belong in the state dag. + new_state_events_contexts = [ + (e, ctx) for e, ctx in event_contexts if event_exists_in_state_dag(e) + ] + if len(new_state_events_contexts) == 0: + # if there are no state events being persisted, then the fwd extremities of the state dag + # do not change. + return set(existing_fwd_extrems) + + # This logic is very similar to _calculate_new_extremities with a few key differences: + # - We do not "Remove any events which are prev_events of any existing events." because the + # state DAG mandates that events are processed in causal order, so there MUST NOT be any + # existing, processed events which have the to-be-persisted events as prev_state_events. + # - We don't care if they are an "outlier" in the main room dag, so long as they AREN'T + # an outlier on the state dag, which this function checks, so we don't check outlier-ness. + # - We allow *soft-failed* events to become forward extremities, as per the MSC. We do not + # allow *rejected* events to become forward extremities though. + + rejected_events = [ev for ev, ctx in new_state_events_contexts if ctx.rejected] + new_state_events = [ + ev for ev, ctx in new_state_events_contexts if not ctx.rejected + ] + # We want to check that we are not missing any prev_state_events. + # To do this, we include rejected events in this check because other events may point to them. + # If we didn't include them, we might incorrectly say we are missing events when we are not. + all_new_state_events = set(rejected_events + new_state_events) + + # First, verify that we know all prev_state_events. If we fail this check then we don't have + # a complete DAG and that is bad, so bail out. + + # Start with them all missing. + missing_prev_state_events = { + e_id for event in all_new_state_events for e_id in event.prev_state_events + } + + # remove prev events which appear in all_events + missing_prev_state_events.difference_update( + event.event_id for event in all_new_state_events + ) + # the rest of these events should be present in the DB. Some of them may be forward extremities, + # some may not be, that's ok. + seen_events = await self.main_store.have_seen_events( + room_id, + missing_prev_state_events, + ) + missing_prev_state_events.difference_update(seen_events) + + if len(missing_prev_state_events) > 0: + logger.error( + "_calculate_new_state_dag_extremities: missing the following prev_state_events in room %s : %s", + room_id, + missing_prev_state_events, + ) + logger.error( + "_calculate_new_state_dag_extremities: was handling %s", + shortstr([ev.event_id for ev in all_new_state_events]), + ) + raise SynapseError( + code=500, + msg=f"missing {len(missing_prev_state_events)} prev_state_events in room {room_id}", + ) + + # Now calculate the forward extremities. + + # start with the existing forward extremities + result = set(existing_fwd_extrems) + + # add all the new events to the list + result.update(event.event_id for event in new_state_events) + + # Now remove all events which are prev_state_events of any of the new events + result.difference_update( + e_id for event in new_state_events for e_id in event.prev_state_events + ) + + # Finally handle the case where the new events have rejected/soft-failed `prev_state_events`. + # If they do we need to remove them and their `prev_state_events`, + # otherwise we end up with dangling extremities. + # Specifically, this handles the case where (F=fwd extrem, SF=soft-failed, N=new event) + # F <-- SF <-- SF <-- N + # where we want to remove F as a forward extremity and replace with N. + existing_prevs = await self.persist_events_store._get_prevs_before_rejected( + (e_id for event in new_state_events for e_id in event.prev_state_events), + include_soft_failed=False, + ) + result.difference_update(existing_prevs) + + # We only update metrics for events that change forward extremities + if result != existing_fwd_extrems: + msc4242_state_dag_forward_extremities_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).observe(len(result)) + + # There should always be at least one forward extremity. + assert result, f"No state dag forward extremities left in room {room_id}!" + return result + async def _calculate_new_extremities( self, room_id: str, @@ -859,6 +1113,7 @@ async def _get_new_state_after_events( events_context: list[EventPersistencePair], old_latest_event_ids: AbstractSet[str], new_latest_event_ids: set[str], + should_prune: bool = True, ) -> tuple[StateMap[str] | None, StateMap[str] | None, set[str]]: """Calculate the current state dict after adding some new events to a room @@ -873,9 +1128,15 @@ async def _get_new_state_after_events( old_latest_event_ids: the old forward extremities for the room. - new_latest_event_ids : + new_latest_event_ids: the new forward extremities for the room. + should_prune: + if true, attempt to prune the forward extremities. + Pruning means we will not communicate some new events to other servers, + which can compromise eventual delivery, so graphs which are fully synchronised + e.g. state DAGs should not prune. + Returns: Returns a tuple of two state maps and a set of new forward extremities. @@ -1015,7 +1276,7 @@ async def _get_new_state_after_events( # If the returned state matches the state group of one of the new # forward extremities then we check if we are able to prune some state # extremities. - if res.state_group and res.state_group in new_state_groups: + if should_prune and res.state_group and res.state_group in new_state_groups: new_latest_event_ids = await self._prune_extremities( room_id, new_latest_event_ids, diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index e9ecf464111..8670d68f38c 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -79,6 +79,19 @@ BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES = "remove_dup_outbound_pokes" +# Background update name for adding an index on +# `device_lists_changes_in_room.inserted_ts`. +BG_UPDATE_ADD_INSERTED_TS_INDEX = "device_lists_changes_in_room_inserted_ts_idx" + + +# Prunes entries out of the `device_lists_changes_in_room` table that are more +# than this old. +PRUNE_DEVICE_LISTS_CHANGES_IN_ROOM_AGE = Duration(days=30) + +# The number of rows to delete at once when pruning old entries out of the +# `device_lists_changes_in_room` table. +PRUNE_DEVICE_LISTS_BATCH_SIZE = 1000 + class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): _device_list_id_gen: MultiWriterIdGenerator @@ -194,6 +207,10 @@ def __init__( self.clock.looping_call( self._prune_old_outbound_device_pokes, Duration(hours=1) ) + self.clock.looping_call( + self._prune_device_lists_changes_in_room, + Duration(hours=1), + ) def process_replication_rows( self, stream_name: str, instance_name: str, token: int, rows: Iterable[Any] @@ -1143,6 +1160,35 @@ async def get_users_whose_devices_changed( The set of user_ids whose devices have changed since `from_key` (exclusive) until `to_key` (inclusive). """ + return { + user_id + for user_id, _ in await self.get_device_changes_for_users( + from_key, user_ids, to_key + ) + } + + @cancellable + async def get_device_changes_for_users( + self, + from_key: MultiWriterStreamToken, + user_ids: Collection[str], + to_key: MultiWriterStreamToken | None = None, + ) -> set[tuple[str, str]]: + """Get set of user/device ID tuple whose devices have changed since `from_key` that + are in the given list of user_ids. + + Args: + from_key: The minimum device lists stream token to query device list changes for, + exclusive. + user_ids: If provided, only check if these users have changed their device lists. + Otherwise changes from all users are returned. + to_key: The maximum device lists stream token to query device list changes for, + inclusive. If None then no upper limit is applied. + + Returns: + The set of user/device ID tuples whose devices have changed since `from_key` + (exclusive) until `to_key` (inclusive). + """ # Get set of users who *may* have changed. Users not in the returned # list have definitely not changed. user_ids_to_check = self._device_list_stream_cache.get_entities_changed( @@ -1156,18 +1202,18 @@ async def get_users_whose_devices_changed( if to_key is None: to_key = self.get_device_stream_token() - def _get_users_whose_devices_changed_txn( + def get_device_changes_for_users_txn( txn: LoggingTransaction, from_key: MultiWriterStreamToken, to_key: MultiWriterStreamToken, - ) -> set[str]: + ) -> set[tuple[str, str]]: sql = """ - SELECT user_id, stream_id, instance_name + SELECT user_id, device_id, stream_id, instance_name FROM device_lists_stream WHERE ? < stream_id AND stream_id <= ? AND %s """ - changes: set[str] = set() + changes: set[tuple[str, str]] = set() # Query device changes with a batch of users at a time for chunk in batch_iter(user_ids_to_check, 100): @@ -1179,8 +1225,8 @@ def _get_users_whose_devices_changed_txn( [from_key.stream, to_key.get_max_stream_pos()] + args, ) changes.update( - user_id - for (user_id, stream_id, instance_name) in txn + (user_id, device_id) + for (user_id, device_id, stream_id, instance_name) in txn if MultiWriterStreamToken.is_stream_position_in_range( low=from_key, high=to_key, @@ -1192,8 +1238,8 @@ def _get_users_whose_devices_changed_txn( return changes return await self.db_pool.runInteraction( - "get_users_whose_devices_changed", - _get_users_whose_devices_changed_txn, + "get_device_changes_for_users", + get_device_changes_for_users_txn, from_key, to_key, ) @@ -1699,17 +1745,22 @@ def get_devices_not_accessed_since_txn( return devices - @cached() - async def _get_min_device_lists_changes_in_room(self) -> int: - """Returns the minimum stream ID that we have entries for - `device_lists_changes_in_room` + def _get_max_pruned_device_lists_changes_in_room_txn( + self, txn: LoggingTransaction + ) -> int: + """Returns the maximum stream ID that has been pruned from + `device_lists_changes_in_room`. + + Any queries for stream IDs less than this value cannot be answered + completely, as the data has been deleted. """ - return await self.db_pool.simple_select_one_onecol( - table="device_lists_changes_in_room", + return self.db_pool.simple_select_one_onecol_txn( + txn, + table="device_lists_changes_in_room_max_pruned_stream_id", keyvalues={}, - retcol="COALESCE(MIN(stream_id), 0)", - desc="get_min_device_lists_changes_in_room", + retcol="stream_id", + allow_none=False, ) @cancellable @@ -1728,55 +1779,54 @@ async def get_device_list_changes_in_rooms( if not room_ids: return set() - min_stream_id = await self._get_min_device_lists_changes_in_room() - - # Return early if there are no rows to process in device_lists_changes_in_room - if min_stream_id > from_token.stream: - return None - changed_room_ids = self._device_list_room_stream_cache.get_entities_changed( room_ids, from_token.stream ) if not changed_room_ids: return set() - sql = """ - SELECT user_id, stream_id, instance_name - FROM device_lists_changes_in_room - WHERE {clause} AND stream_id > ? AND stream_id <= ? - """ - def _get_device_list_changes_in_rooms_txn( txn: LoggingTransaction, - chunk: list[str], - ) -> set[str]: - clause, args = make_in_list_sql_clause( - self.database_engine, "room_id", chunk + ) -> set[str] | None: + # Check if the from_token is too old (i.e. data has been pruned). + max_pruned_stream_id = ( + self._get_max_pruned_device_lists_changes_in_room_txn(txn) ) - args.append(from_token.stream) - args.append(to_token.get_max_stream_pos()) - - txn.execute(sql.format(clause=clause), args) - return { - user_id - for (user_id, stream_id, instance_name) in txn - if MultiWriterStreamToken.is_stream_position_in_range( - low=from_token, - high=to_token, - instance_name=instance_name, - pos=stream_id, + if max_pruned_stream_id > from_token.stream: + return None + + changes: set[str] = set() + + for chunk in batch_iter(changed_room_ids, 1000): + clause, args = make_in_list_sql_clause( + self.database_engine, "room_id", chunk ) - } - - changes = set() - for chunk in batch_iter(changed_room_ids, 1000): - changes |= await self.db_pool.runInteraction( - "get_device_list_changes_in_rooms", - _get_device_list_changes_in_rooms_txn, - chunk, - ) + args.append(from_token.stream) + args.append(to_token.get_max_stream_pos()) - return changes + sql = f""" + SELECT user_id, stream_id, instance_name + FROM device_lists_changes_in_room + WHERE {clause} AND stream_id > ? AND stream_id <= ? + """ + txn.execute(sql, args) + changes.update( + user_id + for (user_id, stream_id, instance_name) in txn + if MultiWriterStreamToken.is_stream_position_in_range( + low=from_token, + high=to_token, + instance_name=instance_name, + pos=stream_id, + ) + ) + + return changes + + return await self.db_pool.runInteraction( + "get_device_list_changes_in_rooms", + _get_device_list_changes_in_rooms_txn, + ) async def get_all_device_list_changes(self, from_id: int, to_id: int) -> set[str]: """Return the set of rooms where devices have changed since the given @@ -1785,46 +1835,66 @@ async def get_all_device_list_changes(self, from_id: int, to_id: int) -> set[str Will raise an exception if the given stream ID is too old. """ - min_stream_id = await self._get_min_device_lists_changes_in_room() - - if min_stream_id > from_id: - raise Exception("stream ID is too old") - - sql = """ - SELECT DISTINCT room_id FROM device_lists_changes_in_room - WHERE stream_id > ? AND stream_id <= ? - """ - def _get_all_device_list_changes_txn( txn: LoggingTransaction, - ) -> set[str]: + ) -> set[str] | None: + # Check if the from_token is too old (i.e. data has been pruned). + max_pruned_stream_id = ( + self._get_max_pruned_device_lists_changes_in_room_txn(txn) + ) + if max_pruned_stream_id > from_id: + logger.warning( + "Given stream ID is too old %d < %d", + from_id, + max_pruned_stream_id, + ) + return None + + sql = """ + SELECT DISTINCT room_id FROM device_lists_changes_in_room + WHERE stream_id > ? AND stream_id <= ? + """ + txn.execute(sql, (from_id, to_id)) return {room_id for (room_id,) in txn} - return await self.db_pool.runInteraction( + room_ids = await self.db_pool.runInteraction( "get_all_device_list_changes", _get_all_device_list_changes_txn, ) + if room_ids is None: + raise Exception(f"Given stream ID is too old {from_id}") + + return room_ids + async def get_device_list_changes_in_room( self, room_id: str, min_stream_id: int - ) -> Collection[tuple[str, str]]: + ) -> Collection[tuple[str, str]] | None: """Get all device list changes that happened in the room since the given stream ID. Returns: Collection of user ID/device ID tuples of all devices that have - changed - """ - - sql = """ - SELECT DISTINCT user_id, device_id FROM device_lists_changes_in_room - WHERE room_id = ? AND stream_id > ? + changed, or None if the given stream ID is too old and so a complete + list cannot be calculated. """ def get_device_list_changes_in_room_txn( txn: LoggingTransaction, - ) -> Collection[tuple[str, str]]: + ) -> Collection[tuple[str, str]] | None: + # Check if the from_token is too old (i.e. data has been pruned). + max_pruned_stream_id = ( + self._get_max_pruned_device_lists_changes_in_room_txn(txn) + ) + if max_pruned_stream_id > min_stream_id: + return None + + sql = """ + SELECT DISTINCT user_id, device_id FROM device_lists_changes_in_room + WHERE room_id = ? AND stream_id > ? + """ + txn.execute(sql, (room_id, min_stream_id)) return cast(Collection[tuple[str, str]], txn.fetchall()) @@ -2160,6 +2230,8 @@ def _add_device_outbound_room_poke_txn( encoded_context = json_encoder.encode(context) + now = self.clock.time_msec() + # The `device_lists_changes_in_room.stream_id` column matches the # corresponding `stream_id` of the update in the `device_lists_stream` # table, i.e. all rows persisted for the same device update will have @@ -2175,6 +2247,7 @@ def _add_device_outbound_room_poke_txn( "instance_name", "converted_to_destinations", "opentracing_context", + "inserted_ts", ), values=[ ( @@ -2186,6 +2259,7 @@ def _add_device_outbound_room_poke_txn( # We only need to calculate outbound pokes for local users not self.hs.is_mine_id(user_id), encoded_context, + now, ) for room_id in room_ids for device_id, stream_id in zip(device_ids, stream_ids) @@ -2401,6 +2475,154 @@ async def set_device_change_last_converted_pos( desc="set_device_change_last_converted_pos", ) + @wrap_as_background_process("prune_device_lists_changes_in_room") + async def _prune_device_lists_changes_in_room(self) -> None: + """Delete old entries out of the `device_lists_changes_in_room`, so that + the table doesn't grow indefinitely. + """ + + # Let's only do this pruning if the index on inserted_ts has been + # created, otherwise this query will be very inefficient. + has_index_been_created = ( + await self.db_pool.updates.has_completed_background_update( + BG_UPDATE_ADD_INSERTED_TS_INDEX + ) + ) + if not has_index_been_created: + return + + prune_before_ts = ( + self.clock.time_msec() - PRUNE_DEVICE_LISTS_CHANGES_IN_ROOM_AGE.as_millis() + ) + + # Get stream ID corresponding to the prune_before_ts timestamp. We can + # delete all rows with a stream ID less than or equal to this, as they + # will be older than the cutoff. + # + # Some rows will have a NULL inserted_ts (due to being inserted before + # the column was added), but we can assume that the timestamp will + # monotonically increase with stream ID, so we can safely ignore those + # rows when calculating the cutoff stream ID. This means that we may end + # up keeping some rows with a non-NULL inserted_ts that are older than + # the cutoff, but that's better than accidentally deleting rows that are + # newer than the cutoff. + cutoff_sql = """ + SELECT stream_id FROM device_lists_changes_in_room + WHERE inserted_ts <= ? AND inserted_ts IS NOT NULL + ORDER BY inserted_ts DESC + LIMIT 1 + """ + + def get_prune_before_stream_id_txn(txn: LoggingTransaction) -> int | None: + txn.execute(cutoff_sql, (prune_before_ts,)) + row = txn.fetchone() + return row[0] if row else None + + prune_before_stream_id = await self.db_pool.runInteraction( + "prune_device_lists_changes_in_room_get_stream_id", + get_prune_before_stream_id_txn, + ) + + if prune_before_stream_id is None: + return + + # Get the max stream ID in the table so we avoid deleting it. We need + # to keep the latest row so that we can calculate the maximum stream ID + # used. + max_stream_id = await self.db_pool.simple_select_one_onecol( + table="device_lists_changes_in_room", + keyvalues={}, + retcol="MAX(stream_id)", + desc="prune_device_lists_changes_in_room_get_max_stream_id", + ) + if prune_before_stream_id >= max_stream_id: + prune_before_stream_id = max_stream_id - 1 + + logger.debug( + "Pruning device_lists_changes_in_room before stream ID %d (timestamp %d)", + prune_before_stream_id, + prune_before_ts, + ) + + # Now delete all rows with stream_id less than the + # prune_before_stream_id. + # + # We also delete in batches to avoid massive churn when initially + # clearing out all the old entries. + # + # We set a minimum stream ID so that when we delete in batches the + # database doesn't have to scan through all the (dead) tuples that were just + # deleted to find the next batch to delete. + + # The minimum stream ID to delete in the next batch, c.f. comment above. + # We default to 0 here as that is less than all possible stream IDs. + min_stream_id = 0 + + def prune_device_lists_changes_in_room_txn(txn: LoggingTransaction) -> int: + nonlocal min_stream_id + + delete_sql = """ + DELETE FROM device_lists_changes_in_room + WHERE stream_id IN ( + SELECT stream_id FROM device_lists_changes_in_room + WHERE ? < stream_id AND stream_id <= ? + ORDER BY stream_id ASC + LIMIT ? + ) + RETURNING stream_id + """ + txn.execute( + delete_sql, + (min_stream_id, prune_before_stream_id, PRUNE_DEVICE_LISTS_BATCH_SIZE), + ) + + # We can't use rowcount as that is incorrect on SQLite when using + # RETURNING. + num_deleted = 0 + for row in txn: + num_deleted += 1 + min_stream_id = max(min_stream_id, row[0]) + + if num_deleted: + # Update the max pruned stream ID tracking table so that the + # safety check knows data up to this point has been deleted. + self.db_pool.simple_update_one_txn( + txn, + table="device_lists_changes_in_room_max_pruned_stream_id", + keyvalues={}, + updatevalues={"stream_id": min_stream_id}, + ) + + return num_deleted + + progress_num_rows_deleted = 0 + while True: + batch_deleted = await self.db_pool.runInteraction( + "prune_device_lists_changes_in_room", + prune_device_lists_changes_in_room_txn, + ) + + finished = batch_deleted < PRUNE_DEVICE_LISTS_BATCH_SIZE + + progress_num_rows_deleted += batch_deleted + + # Periodically report progress in the logs. We do this either when + # we've deleted a significant number of rows or when we've finished + # deleting all rows in this round. + if finished or progress_num_rows_deleted > 10000: + logger.info( + "Pruned %d rows from device_lists_changes_in_room", + progress_num_rows_deleted, + ) + progress_num_rows_deleted = 0 + + if finished: + break + + # Sleep for a short time to avoid hammering the database too much if + # there are a lot of rows to delete. + await self.clock.sleep(Duration(milliseconds=100)) + class DeviceBackgroundUpdateStore(SQLBaseStore): _instance_name: str @@ -2459,6 +2681,15 @@ def __init__( columns=["room_id", "stream_id"], ) + # Add indexes to speed up pruning of device_lists_changes_in_room + self.db_pool.updates.register_background_index_update( + BG_UPDATE_ADD_INSERTED_TS_INDEX, + index_name="device_lists_changes_in_room_inserted_ts_idx", + table="device_lists_changes_in_room", + columns=["inserted_ts"], + where_clause="inserted_ts IS NOT NULL", + ) + async def _drop_device_list_streams_non_unique_indexes( self, progress: JsonDict, batch_size: int ) -> int: diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index cc7083b605d..415926eb0a3 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -1493,6 +1493,15 @@ async def get_latest_event_ids_in_room(self, room_id: str) -> frozenset[str]: ) return frozenset(event_ids) + async def get_state_dag_extremities(self, room_id: str) -> frozenset[str]: + event_ids = await self.db_pool.simple_select_onecol( + table="msc4242_state_dag_forward_extremities", + keyvalues={"room_id": room_id}, + retcol="event_id", + desc="get_state_dag_extremities", + ) + return frozenset(event_ids) + async def get_min_depth(self, room_id: str) -> int | None: """For the given room, get the minimum depth we have seen for it.""" return await self.db_pool.runInteraction( diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 941a5f9f3a3..12c918eca64 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -48,7 +48,9 @@ from synapse.api.room_versions import RoomVersions from synapse.events import ( EventBase, + FrozenEventVMSC4242, StrippedStateEvent, + event_exists_in_state_dag, is_creator, relation_from_event, ) @@ -295,6 +297,7 @@ async def _persist_events_and_state_updates( new_event_links: dict[str, NewEventChainLinks], use_negative_stream_ordering: bool = False, inhibit_local_membership_updates: bool = False, + new_state_dag_forward_extremities: set[str] | None = None, ) -> None: """Persist a set of events alongside updates to the current state and forward extremities tables. @@ -315,6 +318,8 @@ async def _persist_events_and_state_updates( from being updated by these events. This should be set to True for backfilled events because backfilled events in the past do not affect the current local state. + new_state_dag_forward_extremities: A set of event IDs that are the new forward + extremities for the state DAG for this room. MSC4242 only. Returns: Resolves when the events have been persisted @@ -379,6 +384,7 @@ async def _persist_events_and_state_updates( new_forward_extremities=new_forward_extremities, new_event_links=new_event_links, sliding_sync_table_changes=sliding_sync_table_changes, + new_state_dag_forward_extremities=new_state_dag_forward_extremities, ) persist_event_counter.labels(**{SERVER_NAME_LABEL: self.server_name}).inc( len(events_and_contexts) @@ -962,8 +968,10 @@ def _get_events_which_are_prevs_txn( return results - async def _get_prevs_before_rejected(self, event_ids: Iterable[str]) -> set[str]: - """Get soft-failed ancestors to remove from the extremities. + async def _get_prevs_before_rejected( + self, event_ids: Iterable[str], include_soft_failed: bool = True + ) -> set[str]: + """Get soft-failed/rejected ancestors to remove from the extremities. Given a set of events, find all those that have been soft-failed or rejected. Returns those soft failed/rejected events and their prev @@ -976,7 +984,8 @@ async def _get_prevs_before_rejected(self, event_ids: Iterable[str]) -> set[str] Args: event_ids: Events to find prev events for. Note that these must have already been persisted. - + include_soft_failed: Soft-failed events are included in the search. If false, only + rejected events are included. Returns: The previous events. """ @@ -1016,7 +1025,7 @@ def _get_prevs_before_rejected_txn( continue soft_failed = db_to_json(metadata).get("soft_failed") - if soft_failed or rejected: + if (include_soft_failed and soft_failed) or rejected: to_recursively_check.append(prev_event_id) existing_prevs.add(prev_event_id) @@ -1038,6 +1047,7 @@ def _persist_events_txn( new_forward_extremities: set[str] | None, new_event_links: dict[str, NewEventChainLinks], sliding_sync_table_changes: SlidingSyncTableChanges | None, + new_state_dag_forward_extremities: set[str] | None = None, ) -> None: """Insert some number of room events into the necessary database tables. @@ -1146,6 +1156,11 @@ def _persist_events_txn( max_stream_order=max_stream_order, ) + if new_state_dag_forward_extremities: + self._set_state_dag_extremities_txn( + txn, room_id, new_state_dag_forward_extremities + ) + self._persist_transaction_ids_txn(txn, events_and_contexts) # Insert into event_to_state_groups. @@ -2475,6 +2490,29 @@ def _update_forward_extremities_txn( ], ) + def _set_state_dag_extremities_txn( + self, txn: LoggingTransaction, room_id: str, new_extrems: Collection[str] + ) -> None: + self.db_pool.simple_delete_txn( + txn, + table="msc4242_state_dag_forward_extremities", + keyvalues={ + "room_id": room_id, + }, + ) + self.db_pool.simple_insert_many_txn( + txn, + table="msc4242_state_dag_forward_extremities", + keys=("room_id", "event_id"), + values=[ + ( + room_id, + event_id, + ) + for event_id in new_extrems + ], + ) + @classmethod def _filter_events_and_contexts_for_duplicates( cls, events_and_contexts: list[EventPersistencePair] @@ -2859,6 +2897,12 @@ def _update_metadata_tables_txn( self._handle_event_relations(txn, event) + if event.room_version.msc4242_state_dags and event_exists_in_state_dag( + event + ): + assert isinstance(event, FrozenEventVMSC4242) + self._store_state_dag_edges(txn, event) + # Store the labels for this event. labels = event.content.get(EventContentFields.LABELS) if labels: @@ -2935,6 +2979,36 @@ def local_prefill() -> None: txn.async_call_after(external_prefill) txn.call_after(local_prefill) + def _store_state_dag_edges( + self, txn: LoggingTransaction, event: FrozenEventVMSC4242 + ) -> None: + # the create event has no edge but we still need to persist it as get_state_dag just + # yanks all rows in this table. It's a bit gross to store NULL as the prev_state_event_id + # though. + if len(event.prev_state_events) == 0 and event.type == EventTypes.Create: + self.db_pool.simple_insert_txn( + txn, + table="msc4242_state_dag_edges", + values={ + "room_id": event.room_id, + "event_id": event.event_id, + "prev_state_event_id": None, + }, + ) + return + assert len(event.prev_state_events) > 0 + self.db_pool.simple_upsert_many_txn( + txn, + table="msc4242_state_dag_edges", + key_names=["room_id", "event_id", "prev_state_event_id"], + key_values=[ + (event.room_id, event.event_id, prev_state_event) + for prev_state_event in event.prev_state_events + ], + value_names=(), + value_values=(), + ) + def _store_redaction(self, txn: LoggingTransaction, event: EventBase) -> None: assert event.redacts is not None self.db_pool.simple_upsert_txn( @@ -3028,7 +3102,7 @@ def _store_room_members_txn( event.event_id, event.internal_metadata.stream_ordering, event.state_key, - event.user_id, + event.sender, event.room_id, event.membership, non_null_str_or_none(event.content.get("displayname")), @@ -3456,7 +3530,13 @@ def _store_event_state_mappings_txn( """ state_groups = {} for event, context in events_and_contexts: - if event.internal_metadata.is_outlier(): + # state dag rooms allow outliers to have state, as `/get_missing_events` state dag events are nominally + # outliers (not present in the timeline) but do need state persisted so we can calculate + # what the auth_events are for the event. + if ( + not event.room_version.msc4242_state_dags + and event.internal_metadata.is_outlier() + ): # double-check that we don't have any events that claim to be outliers # *and* have partial state (which is meaningless: we should have no # state at all for an outlier) diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index 934cd157caf..d2623f0760e 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -23,6 +23,8 @@ from typing import TYPE_CHECKING, cast import attr +from signedjson.key import decode_verify_key_base64, get_verify_key +from signedjson.sign import SignatureVerifyException, verify_signed_json from synapse.api.constants import ( MAX_DEPTH, @@ -31,7 +33,12 @@ RelationTypes, ) from synapse.api.room_versions import KNOWN_ROOM_VERSIONS +from synapse.crypto.event_signing import ( + event_needs_resigning, + resign_event, +) from synapse.events import EventBase, make_event_from_dict +from synapse.events.utils import prune_event_dict from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause from synapse.storage.database import ( DatabasePool, @@ -39,6 +46,7 @@ LoggingTransaction, make_tuple_comparison_clause, ) +from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.databases.main.events import ( SLIDING_SYNC_RELEVANT_STATE_SET, PersistEventsStore, @@ -48,6 +56,7 @@ ) from synapse.storage.databases.main.events_worker import ( DatabaseCorruptionError, + EventRedactBehaviour, InvalidEventError, ) from synapse.storage.databases.main.state_deltas import StateDeltasStore @@ -112,7 +121,9 @@ class _JoinedRoomStreamOrderingUpdate: most_recent_bump_stamp: int | None -class EventsBackgroundUpdatesStore(StreamWorkerStore, StateDeltasStore, SQLBaseStore): +class EventsBackgroundUpdatesStore( + StreamWorkerStore, StateDeltasStore, CacheInvalidationWorkerStore, SQLBaseStore +): def __init__( self, database: DatabasePool, @@ -346,6 +357,11 @@ def __init__( _BackgroundUpdates.FIXUP_MAX_DEPTH_CAP, self.fixup_max_depth_cap_bg_update ) + self.db_pool.updates.register_background_update_handler( + _BackgroundUpdates.EVENT_RESIGN, + self._resign_events, + ) + # We want this to run on the main database at startup before we start processing # events. # @@ -1370,7 +1386,7 @@ def _event_arbitrary_relations_txn(txn: LoggingTransaction) -> int: ) # Iterate the parent IDs and invalidate caches. - self._invalidate_cache_and_stream_bulk( # type: ignore[attr-defined] + self._invalidate_cache_and_stream_bulk( txn, self.get_relations_for_event, # type: ignore[attr-defined] { @@ -1381,7 +1397,7 @@ def _event_arbitrary_relations_txn(txn: LoggingTransaction) -> int: for r in relations_to_insert }, ) - self._invalidate_cache_and_stream_bulk( # type: ignore[attr-defined] + self._invalidate_cache_and_stream_bulk( txn, self.get_thread_summary, # type: ignore[attr-defined] {(r[1],) for r in relations_to_insert}, @@ -2713,6 +2729,177 @@ def redo_max_depth_bg_update_txn(txn: LoggingTransaction) -> tuple[bool, int]: return num_rooms + async def _resign_events(self, progress: dict, batch_size: int) -> int: + """Retroactively re-sign events signed with a different key than the + current signing key. + + Optional progress parameters: + old_key: If set, only re-sign events whose signature can be + verified with this key. Format: "algorithm:key_id base64key" + (e.g. "ed25519:my_old_key XGX0JRS2Af3be3k..."). + before_ts: If set, only re-sign events with a received_ts less + than this value (milliseconds since epoch). + """ + + # Read optional filter parameters from progress. These are set once + # when the job is created and preserved across batches. + old_key_str: str | None = progress.get("old_key") + before_ts: int | None = progress.get("before_ts") + + # Parse the old verify key if provided. + old_verify_key = None + if old_key_str is not None: + parts = old_key_str.split(" ", 1) + if len(parts) == 2: + key_id, key_base64 = parts + alg, _, version = key_id.partition(":") + old_verify_key = decode_verify_key_base64(alg, version, key_base64) + else: + raise ValueError( + f"Invalid old_key format: expected 'algorithm:version base64key', got {old_key_str!r}" + ) + + # Load the next set of candidate events to re-sign. + # Returns the event IDs and the highest stream position for those events. + # If no event IDs are returned, this signals the background update is complete. + def _fetch_next_events_txn( + txn: LoggingTransaction, + ) -> tuple[list[str], int]: + # Start from the minimum 32-bit integer to ensure we cover events + # with negative stream orderings (e.g. from backfill). + last_stream_pos: int = progress.get("last_stream_pos", -(1 << 31)) + + sql = """ + SELECT event_id, stream_ordering FROM events + WHERE stream_ordering > ? AND sender LIKE ? + """ + args: list[object] = [ + last_stream_pos, + f"%:{self.hs.hostname}", + ] + + if before_ts is not None: + sql += " AND received_ts < ?" + args.append(before_ts) + + sql += " ORDER BY stream_ordering ASC LIMIT ?" + args.append(batch_size) + + txn.execute(sql, args) + event_rows: list[tuple[str, int]] = txn.fetchall() + if not event_rows: + return [], last_stream_pos + + last_stream_pos = event_rows[-1][1] + return [row[0] for row in event_rows], last_stream_pos + + next_event_ids, max_stream_pos = await self.db_pool.runInteraction( + "_resign_events._fetch_next_events", + _fetch_next_events_txn, + ) + logger.debug( + "Resign[num_checking=%d,sp=%d]", len(next_event_ids), max_stream_pos + ) + + if not next_event_ids: + await self.db_pool.updates._end_background_update( + _BackgroundUpdates.EVENT_RESIGN + ) + return 0 + + next_events = await self.get_events_as_list( + next_event_ids, + redact_behaviour=EventRedactBehaviour.as_is, + ) + current_verify_key = get_verify_key(self.hs.signing_key) + + # Re-sign any events that need it. + # A list of event IDs and their newly signed event dicts. + resigned_events: list[tuple[str, JsonDict]] = [] + for event in next_events: + if not event_needs_resigning(event, self.hs.hostname, current_verify_key): + continue + + # If old_key is set, only re-sign events whose signature verifies + # with the provided old key. + if old_verify_key is not None: + old_key_id = f"{old_verify_key.alg}:{old_verify_key.version}" + server_sigs = event.signatures.get(self.hs.hostname, {}) + if old_key_id not in server_sigs: + # Event wasn't signed with this key ID at all, skip. + continue + + # Verify the signature is genuinely from this key. We prune + # first since signatures are computed over the redacted form. + pruned = prune_event_dict(event.room_version, event.get_pdu_json()) + try: + verify_signed_json(pruned, self.hs.hostname, old_verify_key) + except SignatureVerifyException: + # In this case, the key ID was right but the signature doesn't match + # the public key we had. We definitely need to log about this. + logger.warning( + "Event %s has a signature for key %s that does not " + "verify — skipping", + event.event_id, + old_key_id, + ) + continue + + event_dict = resign_event(event, self.hs.hostname, self.hs.signing_key) + resigned_events.append((event.event_id, event_dict)) + + # Atomically write the new stream pos progress with the new signatures, + # else we may update the pos and crash before writing the new + # signatures, thus not re-signing at all! + def _write_events_txn( + txn: LoggingTransaction, + events_to_write: list[tuple[str, JsonDict]], + max_stream_pos: int, + ) -> None: + if events_to_write: + self.db_pool.simple_update_many_txn( + txn, + "event_json", + key_names=["event_id"], + key_values=[[event_id] for event_id, _ in events_to_write], + value_names=["json"], + value_values=[ + [json_encoder.encode(event_dict)] + for _, event_dict in events_to_write + ], + ) + # Always update the progress even if we re-sign nothing. + self.db_pool.updates._background_update_progress_txn( + txn, + _BackgroundUpdates.EVENT_RESIGN, + progress={ + "last_stream_pos": max_stream_pos, + "old_key": old_key_str, + "before_ts": before_ts, + }, + ) + + # Invalidate the event cache for re-signed events so that other + # workers also pick up the new signatures. + for event_id, _ in events_to_write: + self.invalidate_get_event_cache_after_txn(txn, event_id) + self._send_invalidation_to_replication( + txn, "_get_event_cache", (event_id,) + ) + + await self.db_pool.runInteraction( + "_resign_events._write_events_txn", + _write_events_txn, + resigned_events, + max_stream_pos, + ) + + logger.info("Re-signed %d events", len(resigned_events)) + + # Even if we don't re-sign them, we need to let the background updater + # know we're still churning through the events. + return len(next_event_ids) + def _resolve_stale_data_in_sliding_sync_tables( txn: LoggingTransaction, diff --git a/synapse/storage/databases/main/purge_events.py b/synapse/storage/databases/main/purge_events.py index d55ea5cf7d4..fe8079c2010 100644 --- a/synapse/storage/databases/main/purge_events.py +++ b/synapse/storage/databases/main/purge_events.py @@ -71,6 +71,10 @@ # so must be deleted first. "sliding_sync_joined_rooms", "sliding_sync_membership_snapshots", + # Note: msc4242_state_dag_forward_extremities/edges have a foreign key to the `events` table + # so must be deleted first. + "msc4242_state_dag_forward_extremities", + "msc4242_state_dag_edges", "events", "federation_inbound_events_staging", "receipts_graph", diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 7ac88e4c2aa..a0c42082f0e 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -21,6 +21,7 @@ # import logging +from dataclasses import dataclass from enum import Enum from typing import ( TYPE_CHECKING, @@ -44,6 +45,7 @@ from synapse.api.room_versions import RoomVersion, RoomVersions from synapse.config.homeserver import HomeServerConfig from synapse.events import EventBase +from synapse.replication.tcp.streams._base import QuarantinedMediaStream from synapse.replication.tcp.streams.partial_state import UnPartialStatedRoomStream from synapse.storage._base import ( db_to_json, @@ -58,8 +60,14 @@ from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.types import Cursor from synapse.storage.util.id_generators import IdGenerator, MultiWriterIdGenerator -from synapse.types import JsonDict, RetentionPolicy, StrCollection, ThirdPartyInstanceID +from synapse.types import ( + JsonDict, + RetentionPolicy, + StrCollection, + ThirdPartyInstanceID, +) from synapse.util.caches.descriptors import cached, cachedList +from synapse.util.duration import Duration from synapse.util.json import json_encoder from synapse.util.stringutils import MXC_REGEX @@ -102,6 +110,14 @@ class RoomStats(LargestRoomStats): public: bool +@dataclass(frozen=True) +class QuarantinedMediaUpdate: + stream_id: int # for the quarantined_media_changes stream + origin: str + media_id: str + quarantined: bool + + class RoomSortOrder(Enum): """ Enum to define the sorting method used when returning rooms with get_rooms_paginate @@ -162,11 +178,169 @@ def __init__( writers=["master"], ) + self._can_write_quarantined_media_changes = ( + self._instance_name in hs.config.worker.writers.quarantined_media_changes + ) + + self._quarantined_media_changes_id_gen: MultiWriterIdGenerator = ( + MultiWriterIdGenerator( + db_conn=db_conn, + db=database, + notifier=hs.get_replication_notifier(), + stream_name=QuarantinedMediaStream.NAME, + server_name=self.server_name, + instance_name=self._instance_name, + tables=[("quarantined_media_changes", "instance_name", "stream_id")], + sequence_name="quarantined_media_id_seq", + writers=hs.config.worker.writers.quarantined_media_changes, + ) + ) + + # Register a background update to flag already-quarantined media in the quarantine + # media changes table. This is to populate the API endpoint which consumes the + # table with initial data that callers expect (namely, a list of currently + # quarantined media). + self.db_pool.updates.register_background_update_handler( + _BackgroundUpdates.FLAG_EXISTING_QUARANTINED_MEDIA, + self._flag_existing_quarantined_media, + ) + + async def _flag_existing_quarantined_media( + self, progress: JsonDict, batch_size: int + ) -> int: + """Background update function to flag existing already-quarantined media in + the new `quarantine_media_changes` table. + + This only flags quarantined media as the API which reads the table is only + concerned with *changes* to the quarantined state - media does not start as + quarantined, so if it's already quarantined then it has changed state at + some point. Media which isn't quarantined has not changed state (as far as + this function can tell). + + We don't know when the media was originally quarantined, so this inserts as if + the media was quarantined now and are processed in an arbitrary order. + + Further, due to lack of timestamp or history, media which was quarantined then + unquarantined will not be picked up by this background task. + + Function signature is as per `register_background_update_handler` requirements. + + Args: + progress: The progress dictionary from the background update. + batch_size: The number of rows to process in each batch. + + Returns: + The number of rows inserted. + """ + # Note: we actually process 2x the batch_size per batch because we use it twice: + # once for the local_media_repository table, and once for the remote_media_cache + # table. This is fine though - we're still doing the work in batches. + + # We track the progress for the local and remote tables separately just in case + # there are media ID collisions that would make a mess of `ORDER BY media_id + # LIMIT ?`. If there are collisions towards the end of the returned set, the LIMIT + # might cut off some rows and make a future call with `WHERE media_id > ?` miss + # them too. By tracking each table separately, we avoid this kind of issue. + last_local_media_id = progress.get("last_local_media_id", "") + last_remote_media_id = progress.get("last_remote_media_id", "") + last_remote_origin = progress.get("last_remote_origin", "") + + # The `ORDER BY` here would normally miss records if the admin (un)quarantined a + # record, but that doesn't affect the background update because we also insert + # into the stream table upon quarantine status changing. Worst case is the admin + # newly quarantines some media, adding a row to the stream table, then we run + # over it again in the background update, adding a second row. Duplicate rows are + # non-issues for us. + # + # Another similar issue is if Synapse is downgraded partway through the background + # update then has more media quarantined. If Synapse is later upgraded again, the + # media that was quarantined while downgraded will only be imported if the media + # IDs are ordered higher than the last processed media ID. Media that has a lower + # ID will be skipped by the background update. + # + # Note: Already-quarantined media is indicated by the `quarantined_by` field being + # non-null. We only want quarantined media, per docstring above. + # + # This background update is considered *best effort* for the reasons above. Data + # might be missing or duplicated, and that's just going to have to be okay. This + # is further reinforced by not all changes being captured by the table anyway. + # See https://github.com/element-hq/synapse/issues/19672 for more details. + def flag_quarantined(txn: LoggingTransaction) -> int: + # It doesn't matter which order we do these in, as long as we do both of them. + txn.execute( + """ + SELECT NULL AS media_origin, media_id + FROM local_media_repository + WHERE quarantined_by IS NOT NULL + AND media_id > ? + ORDER BY media_id + LIMIT ? + """, + (last_local_media_id, batch_size), + ) + local_media_result = cast(list[tuple[str | None, str]], txn.fetchall()) + if len(local_media_result) > 0: + self._insert_quarantine_changes_txn(txn, local_media_result, True) + + # We use a >= ? on the media origin to avoid missing records when media IDs + # collide between origins (the table's unique constraint is on `(media_origin, media_id)`). + # Filtering by `(media_origin, media_id)` also makes sure we're using an index. + txn.execute( + """ + SELECT media_origin, media_id + FROM remote_media_cache + WHERE quarantined_by IS NOT NULL + AND media_origin >= ? AND media_id > ? + ORDER BY media_origin, media_id + LIMIT ? + """, + (last_remote_origin, last_remote_media_id, batch_size), + ) + remote_media_result = cast(list[tuple[str | None, str]], txn.fetchall()) + if len(remote_media_result) > 0: + self._insert_quarantine_changes_txn(txn, remote_media_result, True) + + self.db_pool.updates._background_update_progress_txn( + txn, + _BackgroundUpdates.FLAG_EXISTING_QUARANTINED_MEDIA, + { + "last_local_media_id": local_media_result[-1][1] + if len(local_media_result) > 0 + else last_local_media_id, + "last_remote_media_id": remote_media_result[-1][1] + if len(remote_media_result) > 0 + else last_remote_media_id, + "last_remote_origin": remote_media_result[-1][0] + if len(remote_media_result) > 0 + else last_remote_origin, + }, + ) + + return len(local_media_result) + len(remote_media_result) + + logger.info( + "Flagging existing quarantined media with local offset %s and remote offset %s/%s", + last_local_media_id, + last_remote_origin, + last_remote_media_id, + ) + num_flagged = await self.db_pool.runInteraction( + "_flag_existing_quarantined_media.flag_quarantined", + flag_quarantined, + ) + if num_flagged <= 0: # probably never negative, but why trust computers? + await self.db_pool.updates._end_background_update( + _BackgroundUpdates.FLAG_EXISTING_QUARANTINED_MEDIA + ) + return num_flagged + def process_replication_position( self, stream_name: str, instance_name: str, token: int ) -> None: if stream_name == UnPartialStatedRoomStream.NAME: self._un_partial_stated_rooms_stream_id_gen.advance(instance_name, token) + elif stream_name == QuarantinedMediaStream.NAME: + self._quarantined_media_changes_id_gen.advance(instance_name, token) return super().process_replication_position(stream_name, instance_name, token) async def store_room( @@ -1128,6 +1302,188 @@ def _get_media_ids_by_user_txn( return local_media_ids + async def get_current_quarantined_media_stream_id(self) -> int: + """Gets the position of the quarantined media changes stream. + + Returns: + int - the current stream ID + """ + return self._quarantined_media_changes_id_gen.get_current_token() + + async def get_max_allocated_quarantined_media_stream_id(self) -> int: + """Gets the maximum allocated position of the quarantined media changes stream. + + Returns: + int - the maximum stream ID + """ + return await self._quarantined_media_changes_id_gen.get_max_allocated_token() + + async def wait_for_quarantined_media_stream_id(self, target_id: int) -> bool: + """Waits until the quarantined media changes stream reaches the given stream ID. + + See https://github.com/element-hq/synapse/pull/19644 for more details. + + TODO: Replace function and call sites with https://github.com/element-hq/synapse/pull/19644 + + Args: + target_id: The stream ID to wait for. + + Returns: + True when caught up to the target stream ID. + False when timing out while waiting. + """ + # We ideally would use something like `wait_for_stream_position` in the meantime, + # but that short circuits if the instance name matches the current instance name. + # Doing so means that if *another* writer is actually leading the to_id, then we'll + # assume that we're caught up when we aren't. + # + # NOTE: Because this is implemented to wait for stream positions by integer ID, + # we're technically waiting for *all* workers to catch up rather than just waiting + # for *our* worker to catch up. This is okay for now because the quarantined media + # stream should be pretty fast to update, and if it's not then the only thing we're + # affecting is an admin API that probably has a tool automatically retrying requests + # anyway. https://github.com/element-hq/synapse/pull/19644 does the waiting properly + # so this should be replaced by that (or similar). + + # Get the minimum shared position/ID across all workers + current_id = self._quarantined_media_changes_id_gen.get_current_token() + if current_id >= target_id: + return True # nothing to wait for: we're already caught up. + + # "This should never happen". Tokens we hand out via the API should exist. If they + # don't, then we're in a bad state and need to explode. + max_persisted_position = ( + await self._quarantined_media_changes_id_gen.get_max_allocated_token() + ) + assert max_persisted_position >= target_id, ( + f"Unable to wait for invalid future token (token={target_id} has positions " + f"ahead of our max persisted position={max_persisted_position})" + ) + + # Start waiting until we've caught up to the `stream_token` + start = self.clock.time_msec() + logged = False + while True: + # Like above, get the minimum shared ID across all workers + current_id = self._quarantined_media_changes_id_gen.get_current_token() + if current_id >= target_id: + return True + + now = self.clock.time_msec() + + # Timed out + if now - start > 10_000: + return False + + if not logged: + logger.info( + "Waiting for current token to reach %s; currently at %s", + target_id, + current_id, + ) + logged = True + + # TODO: be better + await self.clock.sleep(Duration(milliseconds=500)) + + async def get_quarantined_media_changes( + self, *, from_id: int, to_id: int, limit: int + ) -> list[QuarantinedMediaUpdate]: + """ + Get updates to quarantined media in stream ordering since `from_id`. + + Paginating forwards: from_id < x <= to_id, (ascending order) + + Args: + from_id: The starting stream ID (exclusive) + to_id: The ending stream ID (inclusive) + limit: The maximum number of rows to return + + Returns: + List of `QuarantinedMediaUpdate` update rows in stream ordering (ascending order). + + Raises: + SynapseError: If waiting for `to_id` took too long. + """ + if to_id < from_id: + # the to_id is behind the from_id, which means no results + return [] + + return await self.db_pool.runInteraction( + "get_quarantined_media_changes", + self._get_quarantined_media_changes_txn, + from_id, + to_id, + limit, + ) + + def _get_quarantined_media_changes_txn( + self, txn: LoggingTransaction, from_id: int, to_id: int, limit: int + ) -> list[QuarantinedMediaUpdate]: + txn.execute( + """ + SELECT stream_id, origin, media_id, quarantined + FROM quarantined_media_changes + WHERE ? < stream_id AND stream_id <= ? + ORDER BY stream_id ASC + LIMIT ? + """, + (from_id, to_id, limit), + ) + return [ + QuarantinedMediaUpdate( + stream_id=stream_id, + origin=origin, + media_id=media_id, + quarantined=quarantined, + ) + for stream_id, origin, media_id, quarantined in txn + ] + + def _insert_quarantine_changes_txn( + self, + txn: LoggingTransaction, + origins_and_media_ids: list[tuple[str | None, str]], + quarantined: bool, + ) -> None: + """Records media being (un)quarantined in the stream. + + Args: + txn (cursor) + origins_and_media_ids: The [origin, media_id] tuples to record. The origin + may be None if the media is local. + quarantined: Whether the media is being quarantined or unquarantined. + """ + assert self._can_write_quarantined_media_changes + medias_with_stream_ids = zip( + origins_and_media_ids, + self._quarantined_media_changes_id_gen.get_next_mult_txn( + txn, len(origins_and_media_ids) + ), + strict=True, + ) + self.db_pool.simple_insert_many_txn( + txn, + "quarantined_media_changes", + keys=( + "instance_name", + "stream_id", + "origin", + "media_id", + "quarantined", + ), + values=[ + ( + self._instance_name, + stream_id, + origin, + media_id, + quarantined, + ) + for (origin, media_id), stream_id in medias_with_stream_ids + ], + ) + def _quarantine_local_media_txn( self, txn: LoggingTransaction, @@ -1161,9 +1517,23 @@ def _quarantine_local_media_txn( if quarantined_by is not None: sql += " AND safe_from_quarantine = FALSE" + sql += " RETURNING media_id" + txn.execute(sql, [quarantined_by] + sql_many_clause_args) - # Note that a rowcount of -1 can be used to indicate no rows were affected. - total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 + media_ids_affected = txn.fetchall() + # We use `len(media_ids_affected)` here and below because both queries may + # affect fewer or more rows than the `media_ids` input. For example, if the + # media_ids point to already-quarantined media, then nothing was updated. + # Similarly, the below query might find more media than the `media_ids` + # because it's searching for hashes instead. + total_media_quarantined += len(media_ids_affected) + if len(media_ids_affected) > 0: + # Flag media that was newly (un)quarantined in the changes table. + self._insert_quarantine_changes_txn( + txn, + [(None, media_id) for (media_id,) in media_ids_affected], + quarantined_by is not None, + ) # Update any media that was identified via hash. if hashes: @@ -1178,8 +1548,17 @@ def _quarantine_local_media_txn( if quarantined_by is not None: sql += " AND safe_from_quarantine = FALSE" + sql += " RETURNING media_id" + txn.execute(sql, [quarantined_by] + sql_many_clause_args) - total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 + media_ids_affected = txn.fetchall() + total_media_quarantined += len(media_ids_affected) + if len(media_ids_affected) > 0: + self._insert_quarantine_changes_txn( + txn, + [(None, media_id) for (media_id,) in media_ids_affected], + quarantined_by is not None, + ) return total_media_quarantined @@ -1212,10 +1591,18 @@ def _quarantine_remote_media_txn( sql = f""" UPDATE remote_media_cache SET quarantined_by = ? - WHERE {sql_in_list_clause}""" + WHERE {sql_in_list_clause} + RETURNING media_origin, media_id""" txn.execute(sql, [quarantined_by] + sql_args) - total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 + media_ids_affected = cast(list[tuple[str | None, str]], txn.fetchall()) + total_media_quarantined += len(media_ids_affected) + if len(media_ids_affected) > 0: + self._insert_quarantine_changes_txn( + txn, + media_ids_affected, + quarantined_by is not None, + ) if hashes: sql_many_clause_sql, sql_many_clause_args = make_in_list_sql_clause( @@ -1224,9 +1611,17 @@ def _quarantine_remote_media_txn( sql = f""" UPDATE remote_media_cache SET quarantined_by = ? - WHERE {sql_many_clause_sql}""" + WHERE {sql_many_clause_sql} + RETURNING media_origin, media_id""" txn.execute(sql, [quarantined_by] + sql_many_clause_args) - total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 + media_ids_affected = cast(list[tuple[str | None, str]], txn.fetchall()) + total_media_quarantined += len(media_ids_affected) + if len(media_ids_affected) > 0: + self._insert_quarantine_changes_txn( + txn, + media_ids_affected, + quarantined_by is not None, + ) return total_media_quarantined @@ -1882,6 +2277,132 @@ async def delete_event_report(self, report_id: int) -> bool: return True + async def get_user_report( + self, report_id: int + ) -> tuple[int, int, str, str, str] | None: + """Retrieve a user report + + Args: + report_id: ID of user report in database + Returns: + JSON dict of information from a user report or None if the + report does not exist. + """ + + return await self.db_pool.simple_select_one( + table="user_reports", + keyvalues={"id": report_id}, + retcols=("id", "received_ts", "target_user_id", "user_id", "reason"), + allow_none=True, + desc="get_user_report", + ) + + async def get_user_reports_paginate( + self, + start: int, + limit: int, + direction: Direction = Direction.BACKWARDS, + user_id: str | None = None, + target_user_id: str | None = None, + ) -> tuple[list[JsonDict], int]: + """Retrieve a paginated list of user reports + + Args: + start: event offset to begin the query from + limit: number of rows to retrieve + direction: Whether to fetch the most recent first (backwards) or the + oldest first (forwards) + user_id: search for user_id of the reporter. Ignored if user_id is None + target_user_id: search for user_id of the target. Ignored if target_user_id is None + Returns: + Tuple of: + json list of user reports + total number of user reports matching the filter criteria + """ + + def _get_user_reports_paginate_txn( + txn: LoggingTransaction, + ) -> tuple[list[dict[str, Any]], int]: + filters = [] + args: list[object] = [] + + if user_id: + filters.append("user_id LIKE ?") + args.extend(["%" + user_id + "%"]) + if target_user_id: + filters.append("target_user_id LIKE ?") + args.extend(["%" + target_user_id + "%"]) + + if direction == Direction.BACKWARDS: + order = "DESC" + else: + order = "ASC" + + where_clause = "WHERE " + " AND ".join(filters) if len(filters) > 0 else "" + + sql = f""" + SELECT COUNT(*) as total_user_reports + FROM user_reports {where_clause} + """ + txn.execute(sql, args) + count = cast(tuple[int], txn.fetchone())[0] + + sql = f""" + SELECT + id, + received_ts, + target_user_id, + user_id, + reason + FROM user_reports + {where_clause} + ORDER BY received_ts {order} + LIMIT ? + OFFSET ? + """ + + args += [limit, start] + txn.execute(sql, args) + + user_reports = [] + for row in txn: + user_reports.append( + { + "id": row[0], + "received_ts": row[1], + "target_user_id": row[2], + "user_id": row[3], + "reason": row[4], + } + ) + + return user_reports, count + + return await self.db_pool.runInteraction( + "get_user_reports_paginate", _get_user_reports_paginate_txn + ) + + async def delete_user_report(self, report_id: int) -> bool: + """Remove a user report from database. + + Args: + report_id: Report to delete + + Returns: + Whether the report was successfully deleted or not. + """ + try: + await self.db_pool.simple_delete_one( + table="user_reports", + keyvalues={"id": report_id}, + desc="delete_user_report", + ) + except StoreError: + # Deletion failed because report does not exist + return False + + return True + async def set_room_is_public(self, room_id: str, is_public: bool) -> None: await self.db_pool.simple_update_one( table="rooms", @@ -2001,6 +2522,7 @@ class _BackgroundUpdates: REPLACE_ROOM_DEPTH_MIN_DEPTH = "replace_room_depth_min_depth" POPULATE_ROOMS_CREATOR_COLUMN = "populate_rooms_creator_column" ADD_ROOM_TYPE_COLUMN = "add_room_type_column" + FLAG_EXISTING_QUARANTINED_MEDIA = "flag_existing_quarantined_media" _REPLACE_ROOM_DEPTH_SQL_COMMANDS = ( diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index cfde107b486..87523e6f180 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -38,7 +38,7 @@ from synapse.api.constants import EventContentFields, EventTypes, Membership from synapse.api.errors import NotFoundError, UnsupportedRoomVersionError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion -from synapse.events import EventBase +from synapse.events import EventBase, EventMetadata from synapse.events.snapshot import EventContext from synapse.logging.opentracing import trace from synapse.replication.tcp.streams import UnPartialStatedEventStream @@ -78,16 +78,6 @@ class Sentinel: ROOM_UNKNOWN_SENTINEL = Sentinel() -@attr.s(slots=True, frozen=True, auto_attribs=True) -class EventMetadata: - """Returned by `get_metadata_for_events`""" - - room_id: str - event_type: str - state_key: str | None - rejection_reason: str | None - - def _retrieve_and_check_room_version(room_id: str, room_version_id: str) -> RoomVersion: v = KNOWN_ROOM_VERSIONS.get(room_version_id) if not v: diff --git a/synapse/storage/databases/main/sticky_events.py b/synapse/storage/databases/main/sticky_events.py index 38b84443df6..eee6b924159 100644 --- a/synapse/storage/databases/main/sticky_events.py +++ b/synapse/storage/databases/main/sticky_events.py @@ -302,10 +302,14 @@ def insert_sticky_events_txn( Skips inserting events: - if they are considered spammy by the policy server; (unsure if correct, track: https://github.com/matrix-org/matrix-spec-proposals/pull/4354#discussion_r2727593350) + - if they are considered spammy by a Synapse spam checker module; - if they are rejected; - if they are outliers (they should be reconsidered for insertion when de-outliered); or - if they are not sticky (e.g. if the stickiness expired). + Note: Soft-failed sticky events ARE inserted, as their soft-failed status + could be re-evaluated later. + Skipping the insertion of these types of 'invalid' events is useful for performance reasons because they would fill up the table yet we wouldn't show them to clients anyway. @@ -321,7 +325,12 @@ def insert_sticky_events_txn( sticky_events: list[tuple[EventBase, int]] = [] for ev in events: # MSC: Note: policy servers and other similar antispam techniques still apply to these events. - if ev.internal_metadata.policy_server_spammy: + # We don't filter out soft-failed events altogether (in case they get re-evaluated later), + # so filter out `spam_checker_spammy` events specifically as we don't want to re-evaluate _those_ later. + if ( + ev.internal_metadata.policy_server_spammy + or ev.internal_metadata.spam_checker_spammy + ): continue # We shouldn't be passed rejected events, but if we do, we filter them out too. if ev.rejected_reason is not None: @@ -332,7 +341,7 @@ def insert_sticky_events_txn( sticky_duration = ev.sticky_duration() if sticky_duration is None: continue - # Calculate the end time as start_time + effecitve sticky duration + # Calculate the end time as start_time + effective sticky duration expires_at = min(ev.origin_server_ts, now_ms) + sticky_duration.as_millis() # Filter out already expired sticky events if expires_at <= now_ms: diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py index a205c2e8f89..e7138b49f4b 100644 --- a/synapse/storage/engines/sqlite.py +++ b/synapse/storage/engines/sqlite.py @@ -21,6 +21,7 @@ import platform import sqlite3 import struct +import sys import threading from typing import TYPE_CHECKING, Any, Mapping @@ -93,6 +94,20 @@ def on_new_connection(self, db_conn: "LoggingDatabaseConnection") -> None: # We need to import here to avoid an import loop. from synapse.storage.prepare_database import prepare_database + if sys.version_info >= (3, 12): + # Opportunistically disable the SQLITE_DBCONFIG_DEFENSIVE + # flag on the database, as some of our database migrations + # alter the schema and this is forbidden in defensive mode. + # + # This is only known to be necessary on macOS, though SQLite can + # in theory be configured to be defensive by default on any + # platform. + # + # The constant for this is only exposed in the sqlite3 module + # on Python >= 3.12. + assert isinstance(db_conn.conn, sqlite3.Connection) + db_conn.conn.setconfig(sqlite3.SQLITE_DBCONFIG_DEFENSIVE, False) + if self._is_in_memory: # In memory databases need to be rebuilt each time. Ideally we'd # reuse the same connection as we do when starting up, but that diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index e3095a9d0d0..1afc6d0b2a6 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -174,6 +174,7 @@ Changes in SCHEMA_VERSION = 94 - Add `recheck` column (boolean, default true) to the `redactions` table. + - MSC4242: Add state DAG tables. """ diff --git a/synapse/storage/schema/main/delta/94/03_device_lists_room_timestamp.sql b/synapse/storage/schema/main/delta/94/03_device_lists_room_timestamp.sql new file mode 100644 index 00000000000..b0ae1eaaf6f --- /dev/null +++ b/synapse/storage/schema/main/delta/94/03_device_lists_room_timestamp.sql @@ -0,0 +1,18 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 Element Creations, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +ALTER TABLE device_lists_changes_in_room ADD COLUMN inserted_ts BIGINT; + +-- Add a background update to add index +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9403, 'device_lists_changes_in_room_inserted_ts_idx', '{}'); diff --git a/synapse/storage/schema/main/delta/94/03_quarantined_media_tracking.sql b/synapse/storage/schema/main/delta/94/03_quarantined_media_tracking.sql new file mode 100644 index 00000000000..0eb74d85d49 --- /dev/null +++ b/synapse/storage/schema/main/delta/94/03_quarantined_media_tracking.sql @@ -0,0 +1,46 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Represents a stream of when media is quarantined and unquarantined. Note that it's possible for duplicate rows to +-- exist in this table. When the background update which backfills this table is running, it orders quarantined media +-- by `media_id`. This table is also populated when an admin newly quarantines a piece of media. If the admin happens +-- to newly quarantine media that hasn't yet been processed by the background update, both the admin's API call and the +-- background update will add a row to this table. If the background update has already progressed past the media's ID, +-- then only the admin's API call will add a row to this table. +-- +-- Note also that this table might not be inserted with all possible cases of media being quarantined. For example, if +-- media is quarantined by hash upon upload or URL preview, it might not show up here. See https://github.com/element-hq/synapse/issues/19672 +-- for more details. +-- +-- Overall, this table is very much intended to be *best effort* for media that was quarantined before the table existed. +CREATE TABLE quarantined_media_changes ( + -- Position in the quarantined media stream + stream_id INTEGER NOT NULL PRIMARY KEY, + + -- Name of the worker sending this (makes us compatible with multiple writers) + instance_name TEXT NOT NULL, + + -- Media origin. NULL if local media. + -- We store the origin and media_id as media is scoped to the origin and are uniquely identified by (origin, media_id). + origin TEXT NULL, + + -- Media ID at the origin. + media_id TEXT NOT NULL, + + -- True if quarantined at this position, false otherwise. + quarantined BOOLEAN NOT NULL +); + +-- Start the background update to populate existing quarantined media in the table. See update handler for more details. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9305, 'flag_existing_quarantined_media', '{}'); diff --git a/synapse/storage/schema/main/delta/94/03_quarantined_media_tracking_seq.sql.postgres b/synapse/storage/schema/main/delta/94/03_quarantined_media_tracking_seq.sql.postgres new file mode 100644 index 00000000000..85f50ba8e79 --- /dev/null +++ b/synapse/storage/schema/main/delta/94/03_quarantined_media_tracking_seq.sql.postgres @@ -0,0 +1,18 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +CREATE SEQUENCE quarantined_media_id_seq; +-- Synapse streams start at 2, because the default position is 1 +-- so any item inserted at position 1 is ignored. +-- We have to use nextval not START WITH 2, see https://github.com/element-hq/synapse/issues/18712 +SELECT nextval('quarantined_media_id_seq'); diff --git a/synapse/storage/schema/main/delta/94/03_state_dag_fwd_extrems.sql b/synapse/storage/schema/main/delta/94/03_state_dag_fwd_extrems.sql new file mode 100644 index 00000000000..bc5c738ba53 --- /dev/null +++ b/synapse/storage/schema/main/delta/94/03_state_dag_fwd_extrems.sql @@ -0,0 +1,38 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +CREATE TABLE IF NOT EXISTS msc4242_state_dag_forward_extremities( + -- we always expect the room to exist. If it gets removed, delete fwd extremities. + room_id TEXT NOT NULL REFERENCES rooms(room_id) ON DELETE CASCADE, + event_id TEXT NOT NULL REFERENCES events(event_id) ON DELETE CASCADE, + -- it doesn't make sense to reference the same event multiple times, and this uniqueness + -- index is also used to delete events once they are no longer forward extremities. + UNIQUE (event_id) +); +-- When creating events, we want to select all forward extremities for a room which this index helps with. +CREATE INDEX msc4242_state_dag_room ON msc4242_state_dag_forward_extremities(room_id); + + +CREATE TABLE IF NOT EXISTS msc4242_state_dag_edges( + -- Deleting the room deletes the state DAG. + room_id TEXT NOT NULL REFERENCES rooms(room_id) ON DELETE CASCADE, + -- the event IDs being referenced must exist (hence REFERENCES) and we do not want to accidentally delete + -- the event and create a hole in the state DAG. It is not possible for a state + -- DAG room to function with an holey DAG, so these events _cannot_ be purged. To purge them, the + -- entire room would need to be deleted. + event_id TEXT NOT NULL REFERENCES events(event_id), + -- one of the `prev_state_events` for this event ID. We must have it since we must have the entire state DAG. + -- can be NULL for the create event. + prev_state_event_id TEXT REFERENCES events(event_id) +); +CREATE UNIQUE INDEX msc4242_state_dag_edges_key ON msc4242_state_dag_edges(room_id, event_id, prev_state_event_id); diff --git a/synapse/storage/schema/main/delta/94/04_device_lists_changes_max_pruned.sql b/synapse/storage/schema/main/delta/94/04_device_lists_changes_max_pruned.sql new file mode 100644 index 00000000000..73836841daf --- /dev/null +++ b/synapse/storage/schema/main/delta/94/04_device_lists_changes_max_pruned.sql @@ -0,0 +1,34 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + + +-- Tracks the maximum stream_id that has been deleted (pruned) from the +-- device_lists_changes_in_room table. This is used to determine whether it's +-- safe to read from that table for a given stream_id — if the requested +-- stream_id is < the value here, the data has been pruned and the table cannot +-- provide a complete answer. +-- +-- We need a separate table, rather than looking at the minimum stream_id in the +-- device_lists_changes_in_room table, because not all valid stream IDs will +-- have entries in the table. This could lead to situations where the minimum +-- stream ID was potentially much more recent than when we actually pruned. This +-- would cause us to incorrectly think that the table was not safe to read from, +-- when in fact it was. +CREATE TABLE IF NOT EXISTS device_lists_changes_in_room_max_pruned_stream_id ( + Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, + stream_id BIGINT NOT NULL +); + +-- We assume that nothing has been deleted from the device_lists_changes_in_room +-- table, so we can set the initial value to 0. +INSERT INTO device_lists_changes_in_room_max_pruned_stream_id (stream_id) VALUES (0); diff --git a/synapse/synapse_rust/events.pyi b/synapse/synapse_rust/events.pyi index e6753bbbad2..fe0ca044208 100644 --- a/synapse/synapse_rust/events.pyi +++ b/synapse/synapse_rust/events.pyi @@ -38,6 +38,19 @@ class EventInternalMetadata: policy_server_spammy: bool """whether the policy server indicated that this event is spammy""" + spam_checker_spammy: bool + """Whether a spam checker module indicated that this event is spammy + + Note that spam checkers also cause the event to be marked as soft-failed. + + This flags exists for two reasons: + 1. as debugging information + 2. to prevent the soft-failed re-evaluation of spammy events + (the re-evaluation behaviour originates from MSC4354 Sticky Events) + + Note that historical spammy events won't have this flag. + """ + txn_id: str """The transaction ID, if it was set when the event was created.""" delay_id: str @@ -47,6 +60,9 @@ class EventInternalMetadata: device_id: str """The device ID of the user who sent this event, if any.""" + # MSC4242 state dags + calculated_auth_event_ids: list[str] + def get_dict(self) -> JsonDict: ... def is_outlier(self) -> bool: ... def copy(self) -> "EventInternalMetadata": ... diff --git a/synapse/synapse_rust/room_versions.pyi b/synapse/synapse_rust/room_versions.pyi index 909e3a1c26f..9bbb538f185 100644 --- a/synapse/synapse_rust/room_versions.pyi +++ b/synapse/synapse_rust/room_versions.pyi @@ -31,6 +31,8 @@ class EventFormatVersions: """MSC1884-style format: introduced for room v4""" ROOM_V11_HYDRA_PLUS: int """MSC4291 room IDs as hashes: introduced for room HydraV11""" + ROOM_VMSC4242: int + """MSC4242 state DAGs: adds prev_state_events, removes auth_events""" KNOWN_EVENT_FORMAT_VERSIONS: frozenset[int] @@ -113,6 +115,14 @@ class RoomVersion: rather than in codepoints. If true, this room version uses stricter event size validation.""" + msc4242_state_dags: bool + """MSC4242: State DAGs. Creates events with prev_state_events instead of auth_events and derives + state from it. Events are always processed in causal order without any gaps in the DAG + (prev_state_events are always known), guaranteeing that processed events have a path to the + create event. This is an emergent property of state DAGs as asserting that there is a path + to the create event every time we insert an event would be prohibitively expensive. + This is similar to how doubly-linked lists can potentially not refer to previous items correctly + without verifying the list's integrity, but doing it on every insert is too expensive.""" class RoomVersions: V1: RoomVersion @@ -132,6 +142,7 @@ class RoomVersions: MSC3757v11: RoomVersion HydraV11: RoomVersion V12: RoomVersion + MSC4242v12: RoomVersion class KnownRoomVersionsMapping(Mapping[str, RoomVersion]): def add_room_version(self, room_version: RoomVersion) -> None: ... diff --git a/synapse/types/handlers/sliding_sync.py b/synapse/types/handlers/sliding_sync.py index 694b3e1645e..1a84bf1ff8f 100644 --- a/synapse/types/handlers/sliding_sync.py +++ b/synapse/types/handlers/sliding_sync.py @@ -34,6 +34,7 @@ from synapse.api.constants import EventTypes from synapse.events import EventBase +from synapse.events.utils import FilteredEvent from synapse.types import ( DeviceListUpdates, JsonDict, @@ -185,7 +186,7 @@ class StrippedHero: # Should be empty for invite/knock rooms with `stripped_state` required_state: list[EventBase] # Should be empty for invite/knock rooms with `stripped_state` - timeline_events: list[EventBase] + timeline_events: list[FilteredEvent] bundled_aggregations: dict[str, "BundledAggregations"] | None # Optional because it's only relevant to invite/knock rooms stripped_state: list[JsonDict] diff --git a/synapse/types/storage/__init__.py b/synapse/types/storage/__init__.py index 992c36cabad..6b857aeb0ec 100644 --- a/synapse/types/storage/__init__.py +++ b/synapse/types/storage/__init__.py @@ -66,3 +66,5 @@ class _BackgroundUpdates: FIXUP_MAX_DEPTH_CAP = "fixup_max_depth_cap" REDACTIONS_RECHECK_BG_UPDATE = "redactions_recheck" + + EVENT_RESIGN = "event_resign" diff --git a/synapse/visibility.py b/synapse/visibility.py index 5ba2a14a24a..fc3e9dfa498 100644 --- a/synapse/visibility.py +++ b/synapse/visibility.py @@ -31,14 +31,13 @@ from synapse.api.constants import ( EventTypes, - EventUnsignedContentFields, HistoryVisibility, JoinRules, Membership, ) from synapse.events import EventBase from synapse.events.snapshot import EventContext -from synapse.events.utils import clone_event, prune_event +from synapse.events.utils import FilteredEvent, prune_event from synapse.logging.opentracing import trace from synapse.storage.controllers import StorageControllers from synapse.storage.databases.main import DataStore @@ -82,7 +81,7 @@ async def filter_and_transform_events_for_client( is_peeking: bool = False, always_include_ids: frozenset[str] = frozenset(), filter_send_to_client: bool = True, -) -> list[EventBase]: +) -> list[FilteredEvent]: """ Check which events a user is allowed to see. If the user can see the event but its sender asked for their data to be erased, prune the content of the event. @@ -102,8 +101,8 @@ async def filter_and_transform_events_for_client( also be called to check whether a user can see the state at a given point. Returns: - The filtered events. The `unsigned` data is annotated with the membership state - of `user_id` at each event. + The filtered events, wrapped in FilteredEvent with the requesting user's + membership at each event annotated for use during serialization (MSC4115). """ # Filter out events that have been soft failed so that we don't relay them # to clients, unless they're a server admin and want that to happen. @@ -176,7 +175,7 @@ async def filter_and_transform_events_for_client( room_id ] = await storage.main.get_retention_policy_for_room(room_id) - def allowed(event: EventBase) -> EventBase | None: + def allowed(event: EventBase) -> FilteredEvent | None: state_after_event = event_id_to_state.get(event.event_id) filtered = _check_client_allowed_to_see_event( user_id=user_id, @@ -233,28 +232,9 @@ def allowed(event: EventBase) -> EventBase | None: else Membership.LEAVE ) - # Copy the event before updating the unsigned data: this shouldn't be persisted - # to the cache! - cloned = clone_event(filtered) - cloned.unsigned[EventUnsignedContentFields.MEMBERSHIP] = user_membership - if storage.main.config.experimental.msc4354_enabled: - sticky_duration = cloned.sticky_duration() - if sticky_duration: - now_ms = storage.main.clock.time_msec() - expires_at = ( - # min() ensures that the origin server can't lie about the time and - # send the event 'in the future', as that would allow them to exceed - # the 1 hour limit on stickiness duration. - min(cloned.origin_server_ts, now_ms) + sticky_duration.as_millis() - ) - if expires_at > now_ms: - cloned.unsigned[EventUnsignedContentFields.STICKY_TTL] = ( - expires_at - now_ms - ) - - return cloned + return FilteredEvent(event=filtered, membership=user_membership) - # Check each event: gives an iterable of None or (a modified) EventBase. + # Check each event: gives an iterable of None or a FilteredEvent. filtered_events = map(allowed, events) # Turn it into a list and remove None entries before returning. diff --git a/tests/config/test_api.py b/tests/config/test_api.py index e6cc3e21ed3..d4fa398a47e 100644 --- a/tests/config/test_api.py +++ b/tests/config/test_api.py @@ -5,7 +5,6 @@ from synapse.config import ConfigError from synapse.config._base import RootConfig from synapse.config.api import ApiConfig -from synapse.types.state import StateFilter DEFAULT_PREJOIN_STATE_PAIRS = { ("m.room.join_rules", ""), @@ -38,7 +37,11 @@ def test_disable_default_event_types(self) -> None: disable_default_event_types: true """ ) - self.assertEqual(config.room_prejoin_state, StateFilter.none()) + # MSC4311: m.room.create is always included even when defaults are disabled + self.assertEqual( + set(config.room_prejoin_state.concrete_types()), + {("m.room.create", "")}, + ) def test_event_without_state_key(self) -> None: config = self.read_config( @@ -50,7 +53,11 @@ def test_event_without_state_key(self) -> None: """ ) self.assertEqual(config.room_prejoin_state.wildcard_types(), ["foo"]) - self.assertEqual(config.room_prejoin_state.concrete_types(), []) + # MSC4311: m.room.create is always included + self.assertEqual( + set(config.room_prejoin_state.concrete_types()), + {("m.room.create", "")}, + ) def test_event_with_specific_state_key(self) -> None: config = self.read_config( @@ -62,9 +69,10 @@ def test_event_with_specific_state_key(self) -> None: """ ) self.assertFalse(config.room_prejoin_state.has_wildcards()) + # MSC4311: m.room.create is always included self.assertEqual( set(config.room_prejoin_state.concrete_types()), - {("foo", "bar")}, + {("foo", "bar"), ("m.room.create", "")}, ) def test_repeated_event_with_specific_state_key(self) -> None: @@ -78,9 +86,10 @@ def test_repeated_event_with_specific_state_key(self) -> None: """ ) self.assertFalse(config.room_prejoin_state.has_wildcards()) + # MSC4311: m.room.create is always included self.assertEqual( set(config.room_prejoin_state.concrete_types()), - {("foo", "bar"), ("foo", "baz")}, + {("foo", "bar"), ("foo", "baz"), ("m.room.create", "")}, ) def test_no_specific_state_key_overrides_specific_state_key(self) -> None: @@ -94,7 +103,11 @@ def test_no_specific_state_key_overrides_specific_state_key(self) -> None: """ ) self.assertEqual(config.room_prejoin_state.wildcard_types(), ["foo"]) - self.assertEqual(config.room_prejoin_state.concrete_types(), []) + # MSC4311: m.room.create is always included + self.assertEqual( + set(config.room_prejoin_state.concrete_types()), + {("m.room.create", "")}, + ) config = self.read_config( """ @@ -106,7 +119,11 @@ def test_no_specific_state_key_overrides_specific_state_key(self) -> None: """ ) self.assertEqual(config.room_prejoin_state.wildcard_types(), ["foo"]) - self.assertEqual(config.room_prejoin_state.concrete_types(), []) + # MSC4311: m.room.create is always included + self.assertEqual( + set(config.room_prejoin_state.concrete_types()), + {("m.room.create", "")}, + ) def test_bad_event_type_entry_raises(self) -> None: with self.assertRaises(ConfigError): diff --git a/tests/crypto/test_event_signing.py b/tests/crypto/test_event_signing.py index 9cdc1604da2..334ff64bc21 100644 --- a/tests/crypto/test_event_signing.py +++ b/tests/crypto/test_event_signing.py @@ -19,12 +19,23 @@ # # -from signedjson.key import decode_signing_key_base64 +from typing import TypedDict + +from signedjson.key import ( + decode_signing_key_base64, + generate_signing_key, + get_verify_key, +) from signedjson.types import SigningKey from synapse.api.room_versions import RoomVersions -from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import make_event_from_dict +from synapse.crypto.event_signing import ( + add_hashes_and_signatures, + event_needs_resigning, + resign_event, +) +from synapse.events import EventBase, make_event_from_dict +from synapse.types import JsonDict from tests import unittest @@ -107,3 +118,121 @@ def test_sign_message(self) -> None: "Ay4aj2b5oJ1k8INYZ9n3KnszCflM0emwcmQQ7vxpbdc" "Sv9bkJxIZdWX1IJllcZLq89+D3sSabE+vqPtZs9akDw", ) + + +class EventResigningTestCase(unittest.TestCase): + def setUp(self) -> None: + self.signing_key: SigningKey = decode_signing_key_base64( + KEY_ALG, KEY_VER, SIGNING_KEY_SEED + ) + + def test_resign(self) -> None: + event_dict: JsonDict = { + "content": {"body": "Here is the message content"}, + "event_id": "$fffff:" + HOSTNAME, + "origin_server_ts": 1000000, + "type": "m.room.message", + "room_id": "!r:" + HOSTNAME, + "sender": "@u:" + HOSTNAME, + "signatures": {}, + "unsigned": {"age_ts": 1000000}, + } + add_hashes_and_signatures( + RoomVersions.V1, event_dict, HOSTNAME, self.signing_key + ) + event = make_event_from_dict(event_dict) + self.assertIn(HOSTNAME, event.signatures) + self.assertIn(KEY_NAME, event.signatures[HOSTNAME]) + signature = event.signatures[HOSTNAME][KEY_NAME] + + # Re-sign with a different key + signing_key_2: SigningKey = generate_signing_key("2") + key_name_2 = "ed25519:2" + + resigned_event = resign_event(event, HOSTNAME, signing_key_2) + self.assertIn(HOSTNAME, resigned_event["signatures"]) + self.assertIn(key_name_2, resigned_event["signatures"][HOSTNAME]) + self.assertEqual( + len(resigned_event["signatures"][HOSTNAME]), 1 + ) # the previous signature was removed. + self.assertNotEqual( + signature, resigned_event["signatures"][HOSTNAME][key_name_2] + ) # different signatures + + # Repeat but with an event without any signatures. + event_dict = { + "content": {"body": "Here is the message content"}, + "event_id": "$fffff:" + HOSTNAME, + "origin_server_ts": 1000000, + "type": "m.room.message", + "room_id": "!r:" + HOSTNAME, + "sender": "@u:" + HOSTNAME, + "signatures": {}, + "unsigned": {"age_ts": 1000000}, + } + event = make_event_from_dict(event_dict) + resigned_event = resign_event(event, HOSTNAME, signing_key_2) + self.assertIn(HOSTNAME, resigned_event["signatures"]) + self.assertIn(key_name_2, resigned_event["signatures"][HOSTNAME]) + self.assertEqual(len(resigned_event["signatures"][HOSTNAME]), 1) + + def test_event_needs_resigning(self) -> None: + event_that_needs_resigning_dict: JsonDict = { + "content": {"body": "Here is the message content"}, + "event_id": "$fffff:" + HOSTNAME, + "origin_server_ts": 1000000, + "type": "m.room.message", + "room_id": "!r:" + HOSTNAME, + "sender": "@u:" + HOSTNAME, + "unsigned": {"age_ts": 1000000}, + } + internal_metadata: JsonDict = {} + event_that_needs_resigning = make_event_from_dict( + event_that_needs_resigning_dict, + RoomVersions.V1, + internal_metadata, + ) + self.assertEqual( + event_needs_resigning( + event_that_needs_resigning, HOSTNAME, get_verify_key(self.signing_key) + ), + True, + ) + + class TestCase(TypedDict): + name: str + event: EventBase + + events_that_dont_need_resigning: list[TestCase] = [ + { + "name": "sender domain isn't ours", + "event": make_event_from_dict( + {**event_that_needs_resigning_dict, "sender": "@u:somewhereelse"}, + RoomVersions.V1, + internal_metadata, + ), + }, + { + "name": "already signed with this key", + "event": make_event_from_dict( + { + **event_that_needs_resigning_dict, + "signatures": { + HOSTNAME: { + KEY_NAME: "thisisntchecked", + }, + }, + }, + RoomVersions.V1, + internal_metadata, + ), + }, + ] + for test_case in events_that_dont_need_resigning: + self.assertEqual( + event_needs_resigning( + test_case["event"], HOSTNAME, get_verify_key(self.signing_key) + ), + False, + test_case["name"], + ) diff --git a/tests/events/test_auto_accept_invites.py b/tests/events/test_auto_accept_invites.py index 72ade457589..e0ebdf0bcac 100644 --- a/tests/events/test_auto_accept_invites.py +++ b/tests/events/test_auto_accept_invites.py @@ -380,7 +380,7 @@ async def test_ignore_invite_for_missing_user(self) -> None: join_updates, _ = sync_join(self, inviting_user_id) # Assert that the last event in the room was not a member event for the target user. self.assertEqual( - join_updates[0].timeline.events[-1].content["membership"], "invite" + join_updates[0].timeline.events[-1].event.content["membership"], "invite" ) @override_config( @@ -423,7 +423,7 @@ async def test_ignore_invite_for_deactivated_user(self) -> None: join_updates, b = sync_join(self, inviting_user_id) # Assert that the last event in the room was not a member event for the target user. self.assertEqual( - join_updates[0].timeline.events[-1].content["membership"], "invite" + join_updates[0].timeline.events[-1].event.content["membership"], "invite" ) @override_config( @@ -466,7 +466,7 @@ async def test_ignore_invite_for_suspended_user(self) -> None: join_updates, b = sync_join(self, inviting_user_id) # Assert that the last event in the room was not a member event for the target user. self.assertEqual( - join_updates[0].timeline.events[-1].content["membership"], "invite" + join_updates[0].timeline.events[-1].event.content["membership"], "invite" ) @override_config( @@ -509,7 +509,7 @@ async def test_ignore_invite_for_locked_user(self) -> None: join_updates, b = sync_join(self, inviting_user_id) # Assert that the last event in the room was not a member event for the target user. self.assertEqual( - join_updates[0].timeline.events[-1].content["membership"], "invite" + join_updates[0].timeline.events[-1].event.content["membership"], "invite" ) diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index af44b5dec1b..12ef42866d2 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -28,6 +28,7 @@ from synapse.api.room_versions import RoomVersions from synapse.events import EventBase, make_event_from_dict from synapse.events.utils import ( + FilteredEvent, PowerLevelsContent, SerializeEventConfig, _split_field, @@ -655,7 +656,7 @@ def serialize( ) -> JsonDict: return self.get_success( self._event_serializer.serialize_event( - ev, + FilteredEvent(event=ev, membership=None), 1479807801915, config=SerializeEventConfig( only_event_fields=fields, diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index 275e5dfa1d8..a40e0b06807 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -432,7 +432,14 @@ def _test_get_extremities_common(self, room_version: str) -> None: self.assertEqual(channel.json_body["error"], "Server is banned from room") self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") - @parameterized.expand([(k,) for k in KNOWN_ROOM_VERSIONS.keys()]) + # FIXME: Exclude MSC4242 room versions whilst it lacks federation support + @parameterized.expand( + [ + (k,) + for k in KNOWN_ROOM_VERSIONS.keys() + if k != RoomVersions.MSC4242v12.identifier + ] + ) @override_config( {"use_frozen_dicts": True, "experimental_features": {"msc4370_enabled": True}} ) @@ -440,7 +447,14 @@ def test_get_extremities_with_frozen_dicts(self, room_version: str) -> None: """Test GET /extremities with USE_FROZEN_DICTS=True""" self._test_get_extremities_common(room_version) - @parameterized.expand([(k,) for k in KNOWN_ROOM_VERSIONS.keys()]) + # FIXME: Exclude MSC4242 room versions whilst it lacks federation support + @parameterized.expand( + [ + (k,) + for k in KNOWN_ROOM_VERSIONS.keys() + if k != RoomVersions.MSC4242v12.identifier + ] + ) @override_config( {"use_frozen_dicts": False, "experimental_features": {"msc4370_enabled": True}} ) @@ -573,12 +587,18 @@ def _test_send_join_common(self, room_version: str) -> None: @override_config({"use_frozen_dicts": True}) def test_send_join_with_frozen_dicts(self, room_version: str) -> None: """Test send_join with USE_FROZEN_DICTS=True""" + if room_version == RoomVersions.MSC4242v12.identifier: + # TODO: This room version doesn't work over federation in this PR. + return self._test_send_join_common(room_version) @parameterized.expand([(k,) for k in KNOWN_ROOM_VERSIONS.keys()]) @override_config({"use_frozen_dicts": False}) def test_send_join_without_frozen_dicts(self, room_version: str) -> None: """Test send_join with USE_FROZEN_DICTS=False""" + if room_version == RoomVersions.MSC4242v12.identifier: + # TODO: This room version doesn't work over federation in this PR. + return self._test_send_join_common(room_version) def test_send_join_partial_state(self) -> None: diff --git a/tests/handlers/test_admin.py b/tests/handlers/test_admin.py index 49bd3ba3f4c..a368363d7e3 100644 --- a/tests/handlers/test_admin.py +++ b/tests/handlers/test_admin.py @@ -81,7 +81,8 @@ def test_single_public_joined_room(self) -> None: # Check that the right number of events were written counter = Counter( - (event.type, getattr(event, "state_key", None)) for event in written_events + (event.event.type, getattr(event.event, "state_key", None)) + for event in written_events ) self.assertEqual(counter[(EventTypes.Message, None)], 2) self.assertEqual(counter[(EventTypes.Member, self.user1)], 1) @@ -119,7 +120,8 @@ def test_single_private_joined_room(self) -> None: # Check that the right number of events were written counter = Counter( - (event.type, getattr(event, "state_key", None)) for event in written_events + (event.event.type, getattr(event.event, "state_key", None)) + for event in written_events ) self.assertEqual(counter[(EventTypes.Message, None)], 1) self.assertEqual(counter[(EventTypes.Member, self.user1)], 1) @@ -151,7 +153,8 @@ def test_single_left_room(self) -> None: # Check that the right number of events were written counter = Counter( - (event.type, getattr(event, "state_key", None)) for event in written_events + (event.event.type, getattr(event.event, "state_key", None)) + for event in written_events ) self.assertEqual(counter[(EventTypes.Message, None)], 2) self.assertEqual(counter[(EventTypes.Member, self.user1)], 1) @@ -192,7 +195,8 @@ def test_single_left_rejoined_private_room(self) -> None: # Check that the right number of events were written counter = Counter( - (event.type, getattr(event, "state_key", None)) for event in written_events + (event.event.type, getattr(event.event, "state_key", None)) + for event in written_events ) self.assertEqual(counter[(EventTypes.Message, None)], 2) self.assertEqual(counter[(EventTypes.Member, self.user1)], 1) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 6336edb108a..7f67d0ca5e4 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -568,8 +568,8 @@ def test_match_interesting_room_members( # notify us because the interesting user is joined to the room where the # message was sent. self.assertEqual(service, interested_appservice) - self.assertEqual(events[0]["type"], "m.room.message") - self.assertEqual(events[0]["sender"], alice) + self.assertEqual(events[0].type, "m.room.message") + self.assertEqual(events[0].sender, alice) else: self.send_mock.assert_not_called() @@ -628,8 +628,8 @@ def test_application_services_receive_events_sent_by_interesting_local_user( # Events sent from an interesting local user should also be picked up as # interesting to the appservice. self.assertEqual(service, interested_appservice) - self.assertEqual(events[0]["type"], "m.room.message") - self.assertEqual(events[0]["sender"], alice) + self.assertEqual(events[0].type, "m.room.message") + self.assertEqual(events[0].sender, alice) def test_sending_read_receipt_batches_to_application_services(self) -> None: """Tests that a large batch of read receipts are sent correctly to diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index f99e3cd4a28..9e44b1dc1e0 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -21,20 +21,42 @@ # from unittest import mock +from unittest.mock import AsyncMock, Mock, patch +import signedjson.key +from parameterized import parameterized +from signedjson.types import SigningKey + +from twisted.internet import defer from twisted.internet.defer import ensureDeferred from twisted.internet.testing import MemoryReactor -from synapse.api.constants import RoomEncryptionAlgorithms +from synapse.api.constants import EventTypes, JoinRules, RoomEncryptionAlgorithms from synapse.api.errors import NotFoundError, SynapseError +from synapse.api.room_versions import RoomVersions from synapse.appservice import ApplicationService +from synapse.crypto.event_signing import add_hashes_and_signatures +from synapse.events import EventBase, FrozenEventV3 +from synapse.federation.federation_client import SendJoinResult +from synapse.federation.transport.client import ( + StateRequestResponse, + TransportLayerClient, +) +from synapse.federation.units import Transaction from synapse.handlers.device import MAX_DEVICE_DISPLAY_NAME_LEN, DeviceWriterHandler from synapse.rest import admin from synapse.rest.client import devices, login, register from synapse.server import HomeServer from synapse.storage.databases.main.appservice import _make_exclusive_regex -from synapse.types import JsonDict, UserID, create_requester +from synapse.types import ( + JsonDict, + StateMap, + UserID, + create_requester, + get_domain_from_id, +) from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.task_scheduler import TaskScheduler from tests import unittest @@ -581,3 +603,334 @@ def test_dehydrate_v2_and_fetch_events(self) -> None: self.assertTrue(len(res["next_batch"]) > 1) self.assertEqual(len(res["events"]), 1) self.assertEqual(res["events"][0]["content"]["body"], "foo") + + +@patch("synapse.crypto.keyring.Keyring.process_request", AsyncMock(return_value=None)) +class DeviceUnPartialStateTestCase(unittest.HomeserverTestCase): + """Tests that local device list changes during partial state are sent to + remote servers when the room un-partials.""" + + servlets = [ + admin.register_servlets, + login.register_servlets, + ] + + # The two remote servers to fake + REMOTE1_SERVER_NAME = "remote1" + REMOTE1_SERVER_SIGNATURE_KEY = signedjson.key.generate_signing_key("test") + REMOTE1_USER = f"@user:{REMOTE1_SERVER_NAME}" + + REMOTE2_SERVER_NAME = "remote2" + REMOTE2_SERVER_SIGNATURE_KEY = signedjson.key.generate_signing_key("test") + REMOTE2_USER = f"@user:{REMOTE2_SERVER_NAME}" + + def default_config(self) -> JsonDict: + config = super().default_config() + # Enable federation so that get_device_updates_by_remote works. + config["federation_sender_instances"] = ["master"] + return config + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + # Mock the federation transport client to prevent actual network calls. + self.federation_transport_client = AsyncMock(TransportLayerClient) + + self.federation_transport_client.send_transaction.return_value = {} + + hs = self.setup_test_homeserver( + federation_transport_client=self.federation_transport_client, + ) + handler = hs.get_device_handler() + assert isinstance(handler, DeviceWriterHandler) + self.device_handler = handler + self.store = hs.get_datastores().main + + return hs + + def _build_public_room(self) -> StateMap[EventBase]: + """Build a public room DAG that has REMOTE1 in it""" + + room_id = f"!room:{self.REMOTE1_SERVER_NAME}" + room_version = RoomVersions.V10 + + events: list[EventBase] = [] + + # First we make the create event + create_event_dict: JsonDict = { + "auth_events": [], + "content": { + "creator": self.REMOTE1_USER, + "room_version": room_version.identifier, + }, + "depth": 0, + "origin_server_ts": 0, + "prev_events": [], + "room_id": room_id, + "sender": self.REMOTE1_USER, + "state_key": "", + "type": EventTypes.Create, + } + + add_hashes_and_signatures( + room_version, + create_event_dict, + self.REMOTE1_SERVER_NAME, + self.REMOTE1_SERVER_SIGNATURE_KEY, + ) + + create_event = FrozenEventV3(create_event_dict, room_version, {}, None) + events.append(create_event) + + room_version = self.hs.config.server.default_room_version + join_event_dict: JsonDict = { + "auth_events": [ + create_event.event_id, + ], + "content": {"membership": "join"}, + "depth": 1, + "origin_server_ts": 100, + "prev_events": [create_event.event_id], + "sender": self.REMOTE1_USER, + "state_key": self.REMOTE1_USER, + "room_id": room_id, + "type": EventTypes.Member, + } + add_hashes_and_signatures( + room_version, + join_event_dict, + self.hs.hostname, + self.hs.signing_key, + ) + join_event = FrozenEventV3(join_event_dict, room_version, {}, None) + events.append(join_event) + + # Then set the join rules to public + join_rules_event_dict: JsonDict = { + "auth_events": [create_event.event_id, join_event.event_id], + "content": {"join_rule": JoinRules.PUBLIC}, + "depth": 2, + "origin_server_ts": 200, + "prev_events": [join_event.event_id], + "room_id": room_id, + "sender": self.REMOTE1_USER, + "state_key": "", + "type": EventTypes.JoinRules, + } + + add_hashes_and_signatures( + room_version, + join_rules_event_dict, + self.REMOTE1_SERVER_NAME, + self.REMOTE1_SERVER_SIGNATURE_KEY, + ) + join_rules_event = FrozenEventV3(join_rules_event_dict, room_version, {}, None) + events.append(join_rules_event) + + return {(event.type, event.state_key): event for event in events} + + def _build_signed_join_event( + self, + room_id: str, + user: str, + signing_key: SigningKey, + state: StateMap[EventBase], + ) -> FrozenEventV3: + """Build a join event for the local user, signed by the local server.""" + + latest_event = max(state.values(), key=lambda e: e.depth) + + room_version = self.hs.config.server.default_room_version + join_event_dict: JsonDict = { + "auth_events": [ + state[(EventTypes.Create, "")].event_id, + state[(EventTypes.JoinRules, "")].event_id, + ], + "content": {"membership": "join"}, + "depth": latest_event.depth + 1, + "origin_server_ts": latest_event.origin_server_ts + 100, + "prev_events": [latest_event.event_id], + "sender": user, + "state_key": user, + "room_id": room_id, + "type": EventTypes.Member, + } + add_hashes_and_signatures( + room_version, + join_event_dict, + get_domain_from_id(user), + signing_key, + ) + return FrozenEventV3(join_event_dict, room_version, {}, None) + + @parameterized.expand([("not_pruned", False), ("pruned", True)]) + @patch( + "synapse.storage.databases.main.devices.PRUNE_DEVICE_LISTS_CHANGES_IN_ROOM_AGE", + Duration(minutes=1), + ) + def test_local_device_changes_sent_to_new_servers_on_un_partial_state( + self, _test_suffix: str, prune_device_lists_change_in_room: bool + ) -> None: + """When a room un-partials, local device list changes made during the + partial state period should be sent to remote servers that were NOT + known at the time of the partial join. + + We do this by creating a room with one remote server, partialling + joining it, then receiving a join event from a second remote server. The + second remote server should receive a device list update EDU for any + local device changes that happened during the partial state period. + + We parameterize this test over whether during the unpartial process we + prune the `device_list_changes_in_room` table, to check that the + unpartial process correctly handles the case. + """ + + local_user = self.register_user("alice", "password") + self.login("alice", "password") + + # Build the remote room's state events. + room_state = self._build_public_room() + + # Before joining, we mock out the federation endpoints that are used + # during the unpartial process, so that we can control when the + # unpartial process completes. + get_room_state_ids_deferred: defer.Deferred[JsonDict] = defer.Deferred() + get_room_state_deferred: defer.Deferred[StateRequestResponse] = defer.Deferred() + self.federation_transport_client.get_room_state_ids = Mock( + side_effect=[get_room_state_ids_deferred] + ) + self.federation_transport_client.get_room_state = Mock( + side_effect=[get_room_state_deferred] + ) + + # Now make the local server partially join the room. + room_id = room_state[(EventTypes.Create, "")].room_id + room_version = room_state[(EventTypes.Create, "")].room_version + + local_join_event = self._build_signed_join_event( + room_id, local_user, self.hs.signing_key, room_state + ) + + # Mock the federation client endpoints for the partial join. + mock_make_membership_event = AsyncMock( + return_value=(self.REMOTE1_SERVER_NAME, local_join_event, room_version) + ) + mock_send_join = AsyncMock( + return_value=SendJoinResult( + local_join_event, + self.REMOTE1_SERVER_NAME, + state=list(room_state.values()), + auth_chain=list(room_state.values()), + partial_state=True, + # Only REMOTE1_SERVER_NAME is known at join time. + servers_in_room={self.REMOTE1_SERVER_NAME}, + ) + ) + + fed_handler = self.hs.get_federation_handler() + fed_client = self.hs.get_federation_client() + with ( + patch.object( + fed_client, "make_membership_event", mock_make_membership_event + ), + patch.object(fed_client, "send_join", mock_send_join), + ): + self.get_success( + fed_handler.do_invite_join( + [self.REMOTE1_SERVER_NAME], room_id, local_user, {} + ) + ) + + # The room should now be in partial state. + self.assertTrue(self.get_success(self.store.is_partial_state_room(room_id))) + + # A local device change happens while the room is in partial state. + self.get_success( + self.store.add_device_change_to_streams( + local_user, ["NEW_DEVICE"], [room_id] + ) + ) + + if prune_device_lists_change_in_room: + # Add a device change for another room, as we won't prune the most + # recent change. + self.get_success( + self.store.add_device_change_to_streams( + "@other:user", ["device1"], ["!some:room"] + ) + ) + + # Now prune the device list changes for the room. This simulates the + # case where the unpartial process prunes the + # `device_list_changes_in_room` table before processing the device + # list changes. + self.reactor.advance(120) # Advance past the pruning threshold + self.get_success(self.store._prune_device_lists_changes_in_room()) + + # Assert we actually pruned the device list changes for the room. + room_ids = self.get_success( + self.store.db_pool.simple_select_onecol( + table="device_lists_changes_in_room", + keyvalues={}, + retcol="room_id", + ) + ) + self.assertCountEqual(room_ids, ["!some:room"]) + + # Join the second server + new_state = dict(room_state) + new_state[(EventTypes.Member, local_user)] = local_join_event + join_event_2 = self._build_signed_join_event( + room_id, + self.REMOTE2_USER, + self.REMOTE2_SERVER_SIGNATURE_KEY, + new_state, + ) + + self.get_success( + self.hs.get_federation_event_handler().on_receive_pdu( + self.REMOTE2_SERVER_NAME, join_event_2 + ) + ) + + # Some EDUs may get sent out immediately, such as presence updates. + # However, we only care about the device list update EDU sent by the + # unpartialling process. Let's wait a few seconds and reset the mock. + self.reactor.advance(5) + self.federation_transport_client.send_transaction.reset_mock() + + # We now unblock the unpartial processs by returning the room state and + # state ids. This should trigger the device list update to be sent to + # REMOTE2_SERVER_NAME. + self.federation_transport_client.get_room_state_ids.assert_called_once_with( + self.REMOTE1_SERVER_NAME, + room_id, + event_id=local_join_event.prev_event_ids()[0], + ) + + get_room_state_ids_deferred.callback( + { + "pdu_ids": [event.event_id for event in room_state.values()], + "auth_event_ids": [], + } + ) + get_room_state_deferred.callback( + StateRequestResponse( + state=list(room_state.values()), + auth_events=[], + ) + ) + + # The device list EDU isn't necessarily sent out immediately + self.reactor.advance(30) + + # Check that only one transaction was sent, and that it contains the + # device list update EDU for the new device to REMOTE2_SERVER_NAME. + self.federation_transport_client.send_transaction.assert_called_once() + args, _ = self.federation_transport_client.send_transaction.call_args + transaction: Transaction = args[0] + + self.assertEqual(transaction.destination, self.REMOTE2_SERVER_NAME) + self.assertEqual(len(transaction.edus), 1) + + edu = transaction.edus[0] + self.assertEqual(edu["edu_type"], "m.device_list_update") + self.assertEqual(edu["content"]["device_id"], "NEW_DEVICE") diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 7085531548c..e4a41cf1ae4 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -19,7 +19,7 @@ # # import logging -from typing import Collection, cast +from typing import Collection from unittest import TestCase from unittest.mock import AsyncMock, Mock, patch @@ -140,7 +140,7 @@ def test_rejected_message_event_state(self) -> None: "content": {}, "room_id": room_id, "sender": "@yetanotheruser:" + OTHER_SERVER, - "depth": cast(int, join_event["depth"]) + 1, + "depth": join_event.depth + 1, "prev_events": [join_event.event_id], "auth_events": [], "origin_server_ts": self.clock.time_msec(), @@ -192,7 +192,7 @@ def test_rejected_state_event_state(self) -> None: "content": {}, "room_id": room_id, "sender": "@yetanotheruser:" + OTHER_SERVER, - "depth": cast(int, join_event["depth"]) + 1, + "depth": join_event.depth + 1, "prev_events": [join_event.event_id], "auth_events": [], "origin_server_ts": self.clock.time_msec(), diff --git a/tests/handlers/test_federation_event.py b/tests/handlers/test_federation_event.py index 3d856b93462..1aaa86e2e87 100644 --- a/tests/handlers/test_federation_event.py +++ b/tests/handlers/test_federation_event.py @@ -1121,7 +1121,9 @@ async def get_event( return {"pdus": [missing_event.get_pdu_json()]} async def get_room_state_ids( - destination: str, room_id: str, event_id: str + destination: str, + room_id: str, + event_id: str, ) -> JsonDict: self.assertEqual(destination, self.OTHER_SERVER_NAME) self.assertEqual(event_id, missing_event.event_id) @@ -1131,7 +1133,10 @@ async def get_room_state_ids( } async def get_room_state( - room_version: RoomVersion, destination: str, room_id: str, event_id: str + room_version: RoomVersion, + destination: str, + room_id: str, + event_id: str, ) -> StateRequestResponse: self.assertEqual(destination, self.OTHER_SERVER_NAME) self.assertEqual(event_id, missing_event.event_id) diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 2e521a86b60..5152e8fc536 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -38,6 +38,7 @@ from synapse.util.task_scheduler import TaskStatus from tests import unittest +from tests.unittest import override_config class ProfileTestCase(unittest.HomeserverTestCase): @@ -314,9 +315,8 @@ async def potentially_slow_update_membership( membership[state_tuple].content["displayname"], "Frank Jr." ) + @override_config({"enable_set_displayname": False}) def test_set_my_name_if_disabled(self) -> None: - self.hs.config.registration.enable_set_displayname = False - # Setting displayname for the first time is allowed self.get_success(self.store.set_profile_displayname(self.frank, "Frank")) @@ -435,9 +435,8 @@ def test_set_my_avatar(self) -> None: (self.get_success(self.store.get_profile_avatar_url(self.frank))), ) + @override_config({"enable_set_avatar_url": False}) def test_set_my_avatar_if_disabled(self) -> None: - self.hs.config.registration.enable_set_avatar_url = False - # Setting displayname for the first time is allowed self.get_success( self.store.set_profile_avatar_url(self.frank, "http://my.server/me.png") diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index 18ec2ca6b6d..b9dee1c9547 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -307,7 +307,7 @@ def test_ban_wins_race_with_join(self) -> None: self.assertEqual(len(alice_sync_result.joined), 1) self.assertEqual(alice_sync_result.joined[0].room_id, room_id) last_room_creation_event_id = ( - alice_sync_result.joined[0].timeline.events[-1].event_id + alice_sync_result.joined[0].timeline.events[-1].event.event_id ) # Eve, a ne'er-do-well, registers. @@ -402,7 +402,7 @@ def test_state_includes_changes_on_forks(self) -> None: ) ) last_room_creation_event_id = ( - initial_sync_result.joined[0].timeline.events[-1].event_id + initial_sync_result.joined[0].timeline.events[-1].event.event_id ) # Send a state event, and a regular event, both using the same prev ID @@ -437,7 +437,7 @@ def test_state_includes_changes_on_forks(self) -> None: self.assertEqual(room_sync.room_id, room_id) self.assertTrue(room_sync.timeline.limited) self.assertEqual( - [e.event_id for e in room_sync.timeline.events], + [e.event.event_id for e in room_sync.timeline.events], [e3_event, e4_event], ) self.assertEqual( @@ -476,7 +476,7 @@ def test_state_includes_changes_on_forks_when_events_excluded(self) -> None: ) ) last_room_creation_event_id = ( - initial_sync_result.joined[0].timeline.events[-1].event_id + initial_sync_result.joined[0].timeline.events[-1].event.event_id ) # Send a state event, and a regular event, both using the same prev ID @@ -521,7 +521,7 @@ def test_state_includes_changes_on_forks_when_events_excluded(self) -> None: self.assertEqual(room_sync.room_id, room_id) self.assertTrue(room_sync.timeline.limited) self.assertEqual( - [e.event_id for e in room_sync.timeline.events], + [e.event.event_id for e in room_sync.timeline.events], [e3_event], ) self.assertEqual( @@ -563,7 +563,7 @@ def test_state_includes_changes_on_long_lived_forks(self) -> None: ) ) last_room_creation_event_id = ( - initial_sync_result.joined[0].timeline.events[-1].event_id + initial_sync_result.joined[0].timeline.events[-1].event.event_id ) # Send a state event, and a regular event, both using the same prev ID @@ -593,7 +593,7 @@ def test_state_includes_changes_on_long_lived_forks(self) -> None: self.assertEqual(room_sync.room_id, room_id) self.assertTrue(room_sync.timeline.limited) self.assertEqual( - [e.event_id for e in room_sync.timeline.events], + [e.event.event_id for e in room_sync.timeline.events], [e3_event], ) @@ -632,7 +632,7 @@ def test_state_includes_changes_on_long_lived_forks(self) -> None: self.assertEqual(room_sync.room_id, room_id) self.assertFalse(room_sync.timeline.limited) self.assertEqual( - [e.event_id for e in room_sync.timeline.events], + [e.event.event_id for e in room_sync.timeline.events], [e4_event], ) @@ -701,7 +701,7 @@ def test_state_includes_changes_on_ungappy_syncs(self) -> None: ) ) last_room_creation_event_id = ( - initial_sync_result.joined[0].timeline.events[-1].event_id + initial_sync_result.joined[0].timeline.events[-1].event.event_id ) # Send a state event, and a regular event, both using the same prev ID @@ -728,7 +728,7 @@ def test_state_includes_changes_on_ungappy_syncs(self) -> None: room_sync = initial_sync_result.joined[0] self.assertEqual(room_sync.room_id, room_id) self.assertEqual( - [e.event_id for e in room_sync.timeline.events], + [e.event.event_id for e in room_sync.timeline.events], [e3_event], ) if self.use_state_after: @@ -757,7 +757,7 @@ def test_state_includes_changes_on_ungappy_syncs(self) -> None: self.assertEqual(room_sync.room_id, room_id) self.assertFalse(room_sync.timeline.limited) self.assertEqual( - [e.event_id for e in room_sync.timeline.events], + [e.event.event_id for e in room_sync.timeline.events], [e4_event, e5_event], ) @@ -855,7 +855,7 @@ def test_archived_rooms_do_not_include_state_after_leave( # The last three events in the timeline should be those leading up to the # leave self.assertEqual( - [e.event_id for e in sync_room_result.timeline.events[-3:]], + [e.event.event_id for e in sync_room_result.timeline.events[-3:]], [before_message_event, before_state_event, leave_event], ) @@ -947,7 +947,7 @@ async def _check_sigs_and_hash_for_pulled_events_and_fetch( ) event_ids = [] for event in sync_result.joined[0].timeline.events: - event_ids.append(event.event_id) + event_ids.append(event.event.event_id) self.assertNotIn(call_event.event_id, event_ids) # it will come down in a private room, though @@ -995,7 +995,7 @@ async def _check_sigs_and_hash_for_pulled_events_and_fetch( ) priv_event_ids = [] for event in private_sync_result.joined[0].timeline.events: - priv_event_ids.append(event.event_id) + priv_event_ids.append(event.event.event_id) self.assertIn(private_call_event.event_id, priv_event_ids) diff --git a/tests/media/test_html_preview.py b/tests/media/test_html_preview.py index d3f1e8833a7..ddcbbee8976 100644 --- a/tests/media/test_html_preview.py +++ b/tests/media/test_html_preview.py @@ -433,6 +433,28 @@ def test_twitter_tag(self) -> None: }, ) + def test_extended_opengraph(self) -> None: + """Ensure we pull in profile and article data from opengraph.""" + html = b""" + + + + + + """ + tree = decode_body(html, "http://example.com/test.html") + assert tree is not None + og = parse_html_to_open_graph(tree) + self.assertEqual( + og, + { + "og:title": None, + "og:description": "My description", + "profile:username": "myname", + "article:published_time": "2026-04-07T10:07:37Z", + }, + ) + def test_nested_nodes(self) -> None: """A body with some nested nodes. Tests that we iterate over children in the right order (and don't reverse the order of the text).""" diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py index 12c8942bc80..f1b20a12ecb 100644 --- a/tests/module_api/test_api.py +++ b/tests/module_api/test_api.py @@ -828,6 +828,24 @@ async def _on_logged_out_mock( # Ensure the pushers were deleted after the callback. self.assertEqual(len(self.hs.get_pusherpool().pushers[user_id].values()), 0) + def test_event_deprecated_methods(self) -> None: + """Test that deprecated methods on events are still functional.""" + user_id = self.register_user("user", "password") + tok = self.login("user", "password") + + room_id = self.helper.create_room_as(tok=tok) + + state = self.get_success( + self.hs.get_storage_controllers().state.get_current_state(room_id) + ) + create_event = state[(EventTypes.Create, "")] + + # `.user_id` is a deprecated alias for `.sender`. + self.assertEqual(create_event.user_id, user_id) + + # The event supports looking up keys via `__getitem__` although deprecated + self.assertEqual(create_event["room_id"], room_id) + class ModuleApiWorkerTestCase(BaseModuleApiTestCase, BaseMultiWorkerStreamTestCase): """For testing ModuleApi functionality in a multi-worker setup""" diff --git a/tests/module_api/test_spamchecker.py b/tests/module_api/test_spamchecker.py index 42ef969ce01..572f7bdbcfa 100644 --- a/tests/module_api/test_spamchecker.py +++ b/tests/module_api/test_spamchecker.py @@ -12,22 +12,33 @@ # . # # +from http import HTTPStatus from typing import Literal from twisted.internet.testing import MemoryReactor -from synapse.api.constants import EventContentFields, EventTypes +from synapse.api.constants import ( + EventContentFields, + EventTypes, + Membership, +) +from synapse.api.room_versions import RoomVersions from synapse.config.server import DEFAULT_ROOM_VERSION +from synapse.events import make_event_from_dict +from synapse.module_api import EventBase from synapse.rest import admin, login, room, room_upgrade_rest_servlet from synapse.server import HomeServer from synapse.types import Codes, JsonDict from synapse.util.clock import Clock +from tests import unittest from tests.server import FakeChannel from tests.unittest import HomeserverTestCase class SpamCheckerTestCase(HomeserverTestCase): + """Tests for the spam checker module API.""" + servlets = [ room.register_servlets, admin.register_servlets, @@ -284,3 +295,178 @@ async def user_may_send_state_event( self.assertEqual(channel.code, 403) self.assertEqual(channel.json_body["errcode"], Codes.FORBIDDEN) + + +class FederatedEventSpamCheckMetadataTestCase(unittest.FederatingHomeserverTestCase): + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + self._module_api = hs.get_module_api() + self._store = hs.get_datastores().main + self._storage_controllers = hs.get_storage_controllers() + self._federation_event_handler = hs.get_federation_event_handler() + self._federation_server = hs.get_federation_server() + self._state_handler = hs.get_state_handler() + self._persistence_controller = hs.get_storage_controllers().persistence + + # Create a room + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + self.room_id = self.helper.create_room_as( + user1_id, + tok=user1_tok, + is_public=True, + room_version=RoomVersions.V10.identifier, + ) + + # Prepare a join for the 'remote' user + state_map = self.get_success( + self._storage_controllers.state.get_current_state(self.room_id) + ) + forward_extremity_event_ids = self.get_success( + self.hs.get_datastores().main.get_latest_event_ids_in_room(self.room_id) + ) + self.remote_user_id = f"@remoteuser:{self.OTHER_SERVER_NAME}" + self.remote_user_join_event = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": self.remote_user_id, + "state_key": self.remote_user_id, + "depth": 1000, + "origin_server_ts": 1, + "type": EventTypes.Member, + "content": {"membership": Membership.JOIN}, + "auth_events": [ + state_map[(EventTypes.Create, "")].event_id, + state_map[(EventTypes.JoinRules, "")].event_id, + ], + "prev_events": list(forward_extremity_event_ids), + } + ), + room_version=RoomVersions.V10, + ) + + # Send the join + self.get_success( + self._federation_event_handler.on_receive_pdu( + self.OTHER_SERVER_NAME, self.remote_user_join_event + ) + ) + + # Check the join made it to the 'local' view of the room + self.helper.get_event( + room_id=self.room_id, + event_id=self.remote_user_join_event.event_id, + tok=user1_tok, + expect_code=HTTPStatus.OK, + ) + + def test_federated_events_with_spam_checker_metadata(self) -> None: + """ + Simulates receiving spammy and non-spammy events over federation, + then checks their `spam_checker_spammy` flag is set properly. + """ + + async def check_event_for_spam(event: EventBase) -> Literal["NOT_SPAM"] | Codes: + if event.type == EventTypes.Message: + if "ham" not in event.content["body"]: + return Codes.FORBIDDEN + return "NOT_SPAM" + + # Register a spam checker callback that only allows messages with 'ham' + self._module_api.register_spam_checker_callbacks( + check_event_for_spam=check_event_for_spam + ) + + # Prepare a spammy and a non-spammy event. + forward_extremity_event_ids = self.get_success( + self._store.get_latest_event_ids_in_room(self.room_id) + ) + state_map = self.get_success( + self._storage_controllers.state.get_current_state(self.room_id) + ) + spammy_event = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": self.remote_user_id, + "depth": 2000, + "origin_server_ts": 2, + "type": EventTypes.Message, + "content": {"body": "this is spam", "msgtype": "m.text"}, + "auth_events": [ + state_map[(EventTypes.Create, "")].event_id, + state_map[(EventTypes.JoinRules, "")].event_id, + state_map[(EventTypes.Member, self.remote_user_id)].event_id, + ], + "prev_events": list(forward_extremity_event_ids), + } + ), + room_version=RoomVersions.V10, + ) + non_spammy_event = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": self.remote_user_id, + "depth": 2000, + "origin_server_ts": 2, + "type": EventTypes.Message, + "content": {"body": "delicious ham", "msgtype": "m.text"}, + "auth_events": [ + state_map[(EventTypes.Create, "")].event_id, + state_map[(EventTypes.JoinRules, "")].event_id, + state_map[(EventTypes.Member, self.remote_user_id)].event_id, + ], + "prev_events": list(forward_extremity_event_ids), + } + ), + room_version=RoomVersions.V10, + ) + + # Receive these events over federation + # We need to let the federation server have them because it will + # invoke `_check_sigs_and_hash` which invokes the spam checker. + self.get_success( + self._federation_server._handle_received_pdu( + self.OTHER_SERVER_NAME, spammy_event + ) + ) + self.get_success( + self._federation_server._handle_received_pdu( + self.OTHER_SERVER_NAME, non_spammy_event + ) + ) + + # Retrieve the events from the database + retrieved_spammy_event = self.get_success( + self._store.get_event(spammy_event.event_id, allow_rejected=True) + ) + retrieved_non_spammy_event = self.get_success( + self._store.get_event(non_spammy_event.event_id, allow_rejected=True) + ) + + # Assert the spammy flags (and soft-failed flags, for good measure) are set properly + self.assertTrue( + retrieved_spammy_event.internal_metadata.spam_checker_spammy, + "Spammy inbound event should be marked as spam_checker_spammy!", + ) + self.assertTrue( + retrieved_spammy_event.internal_metadata.is_soft_failed(), + "Spammy inbound event should be soft-failed.", + ) + + self.assertFalse( + retrieved_non_spammy_event.internal_metadata.spam_checker_spammy, + "Non-spammy inbound event should not be marked as spam_checker_spammy!", + ) + self.assertFalse( + retrieved_non_spammy_event.internal_metadata.is_soft_failed(), + "Non-spammy inbound event should not be soft-failed.", + ) diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py index 77d824dcd84..b77a72ec4af 100644 --- a/tests/rest/admin/test_admin.py +++ b/tests/rest/admin/test_admin.py @@ -18,12 +18,15 @@ # [This file includes modifications made by New Vector Limited] # # +from __future__ import annotations import urllib.parse -from typing import cast +from typing import Any, cast +from unittest.mock import Mock from parameterized import parameterized +from twisted.internet.defer import Deferred from twisted.internet.testing import MemoryReactor from twisted.web.resource import Resource @@ -70,6 +73,24 @@ def create_resource_dict(self) -> dict[str, Resource]: resources["/_matrix/media"] = self.hs.get_media_repository_resource() return resources + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + self.fetches: list[tuple[tuple[Any, ...], dict[str, Any]]] = [] + + # A remote fetch of media that was not intentional. + # Used to check that remote media fetches do NOT happen. + def unexpected_remote_fetch(*args: Any, **kwargs: Any) -> Deferred[Any]: + self.fetches.append((args, kwargs)) + return Deferred() + + client = Mock() + client.federation_get_file = unexpected_remote_fetch + client.get_file = unexpected_remote_fetch + + return self.setup_test_homeserver( + clock=clock, + federation_http_client=client, + ) + def _ensure_quarantined( self, user_tok: str, @@ -176,6 +197,28 @@ def test_admin_can_bypass_quarantine(self) -> None: ), ) + def test_non_admin_bypass_does_not_fetch_remote_media(self) -> None: + self.register_user("nonadmin", "pass", admin=False) + non_admin_user_tok = self.login("nonadmin", "pass") + + channel = self.make_request( + "GET", + "/_matrix/client/v1/media/download/example.com/remote_media" + "?admin_unsafely_bypass_quarantine=true", + shorthand=False, + access_token=non_admin_user_tok, + await_result=False, + ) + self.pump() + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual( + channel.json_body["error"], + "Must be a server admin to bypass quarantine", + ) + # Check that a remote fetch attempt did not occur. + self.assertEqual(self.fetches, []) + @parameterized.expand( [ # Attempt quarantine media APIs as non-admin diff --git a/tests/rest/admin/test_media.py b/tests/rest/admin/test_media.py index 8cc54cc80c2..6469d305b24 100644 --- a/tests/rest/admin/test_media.py +++ b/tests/rest/admin/test_media.py @@ -756,6 +756,133 @@ def _access_media( self.assertFalse(os.path.exists(local_path)) +class ListQuarantinedMediaChangesTestCase(_AdminMediaTests): + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.server_name = hs.hostname + self.admin_user = self.register_user("admin", "pass", admin=True) + self.admin_user_tok = self.login("admin", "pass") + + def test_no_auth(self) -> None: + """ + Try to list quarantined media changes without authentication. + """ + + channel = self.make_request( + "GET", + "/_synapse/admin/v1/media/quarantine_changes", + ) + + self.assertEqual(401, channel.code, msg=channel.json_body) + self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) + + def test_requester_is_not_admin(self) -> None: + """ + If the user is not a server admin, an error is returned. + """ + self.other_user = self.register_user("user", "pass") + self.other_user_token = self.login("user", "pass") + + channel = self.make_request( + "GET", + "/_synapse/admin/v1/media/quarantine_changes", + access_token=self.other_user_token, + ) + + self.assertEqual(403, channel.code, msg=channel.json_body) + self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) + + def _quarantine_local_media(self, media_id: str, admin_user_tok: str) -> None: + channel = self.make_request( + "POST", + "/_synapse/admin/v1/media/quarantine/%s/%s" + % ( + self.server_name, + media_id, + ), + access_token=admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + + def _local_upload(self, admin_user_tok: str) -> str: + response = self.helper.upload_media( + SMALL_PNG, tok=admin_user_tok, expect_code=200 + ) + origin_and_media_id = response["content_uri"][6:] # Cut off 'mxc://' + _origin, media_id = origin_and_media_id.split("/") + return media_id + + def test_list_quarantined_media(self) -> None: + """ + Ensure we actually get results for each page and that pagination is seamless. + """ + # Upload 105 media objects to test multiple pages + self.media_ids = [self._local_upload(self.admin_user_tok) for _ in range(105)] + + # No changes before quarantine + channel = self.make_request( + "GET", + "/_synapse/admin/v1/media/quarantine_changes", + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(0, len(channel.json_body["changes"])) + + # We expect to continue from the current stream position because we have no changes + self.assertEqual(1, channel.json_body["next_batch"]) + + # Quarantine by hash should kick in to get the other 104 media objects + self._quarantine_local_media(self.media_ids[0], self.admin_user_tok) + + # Page 1 (implied ?from=0) + channel = self.make_request( + "GET", + "/_synapse/admin/v1/media/quarantine_changes", + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(100, len(channel.json_body["changes"])) + self.assertEqual(101, channel.json_body["next_batch"]) + for row in channel.json_body["changes"]: + self.assertIn( + row["media_id"], + self.media_ids[0:100], + ) + self.assertEqual(row["origin"], self.server_name) + self.assertEqual(row["quarantined"], True) + + # Page 2 (explicit ?from, using next_batch) + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/media/quarantine_changes?from={channel.json_body['next_batch']}", + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(5, len(channel.json_body["changes"])) + self.assertEqual(106, channel.json_body["next_batch"]) + for row in channel.json_body["changes"]: + self.assertIn( + row["media_id"], + self.media_ids[100:], + ) + self.assertEqual(row["origin"], self.server_name) + self.assertEqual(row["quarantined"], True) + + def test_list_quarantined_media_bounds_high(self) -> None: + """ + Ensure out of bounds (token stream position greater than our furthest persisted + position) requests with high `from` values are met with an appropriate error. + """ + # Page that's very much out of range + channel = self.make_request( + "GET", + "/_synapse/admin/v1/media/quarantine_changes?from=900000", + access_token=self.admin_user_tok, + ) + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + + class QuarantineMediaByIDTestCase(_AdminMediaTests): def upload_media_and_return_media_id(self, data: bytes) -> str: # Upload some media into the room diff --git a/tests/rest/admin/test_user_reports.py b/tests/rest/admin/test_user_reports.py new file mode 100644 index 00000000000..94ae242d862 --- /dev/null +++ b/tests/rest/admin/test_user_reports.py @@ -0,0 +1,644 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + + +from twisted.internet.testing import MemoryReactor + +import synapse.rest.admin +from synapse.api.errors import Codes +from synapse.rest.client import login, reporting, room +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util.clock import Clock + +from tests import unittest + + +class UserReportsTestCase(unittest.HomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + room.register_servlets, + reporting.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.admin_user = self.register_user("admin", "pass", admin=True) + self.admin_user_tok = self.login("admin", "pass") + + self.users = {} + for i in range(10): + self.users[i] = self.register_user(f"user{i}", "pass") + + # users 1 and 2 report all other users + reporter_1_tok = self.login(self.users[0], "pass") + reporter_2_tok = self.login(self.users[1], "pass") + for num, user in self.users.items(): + if num <= 1: + continue + if num % 2 == 0: + self._report_user(user, reporter_1_tok) + else: + self._report_user(user, reporter_2_tok) + + self.url = "/_synapse/admin/v1/user_reports" + + def test_no_auth(self) -> None: + """ + Try to get a user report without authentication. + """ + channel = self.make_request("GET", self.url, {}) + + self.assertEqual(401, channel.code, msg=channel.json_body) + self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) + + def test_requester_is_no_admin(self) -> None: + """ + If the user is not a server admin, an error 403 is returned. + """ + rando_tok = self.login(self.users[4], "pass") + channel = self.make_request( + "GET", + self.url, + access_token=rando_tok, + ) + + self.assertEqual(403, channel.code, msg=channel.json_body) + self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) + + def test_default_success(self) -> None: + """ + Testing list of reported users + """ + + channel = self.make_request( + "GET", + self.url, + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 8) + self.assertEqual(len(channel.json_body["user_reports"]), 8) + self.assertNotIn("next_token", channel.json_body) + self._check_fields(channel.json_body["user_reports"]) + + def test_limit(self) -> None: + """ + Testing list of reported users with limit + """ + + channel = self.make_request( + "GET", + self.url + "?limit=5", + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 8) + self.assertEqual(len(channel.json_body["user_reports"]), 5) + self.assertEqual(channel.json_body["next_token"], 5) + self._check_fields(channel.json_body["user_reports"]) + + def test_from(self) -> None: + """ + Testing list of reported users with a defined starting point (from) + """ + + channel = self.make_request( + "GET", + self.url + "?from=5", + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 8) + self.assertEqual(len(channel.json_body["user_reports"]), 3) + self.assertNotIn("next_token", channel.json_body) + self._check_fields(channel.json_body["user_reports"]) + + def test_limit_and_from(self) -> None: + """ + Testing list of reported users with a defined starting point and limit + """ + + channel = self.make_request( + "GET", + self.url + "?from=2&limit=5", + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 8) + self.assertEqual(channel.json_body["next_token"], 7) + self.assertEqual(len(channel.json_body["user_reports"]), 5) + self._check_fields(channel.json_body["user_reports"]) + + def test_filter_by_target_user_id(self) -> None: + """ + Testing list of reported users with a filter of target_user_id + """ + + channel = self.make_request( + "GET", + self.url + "?target_user_id=%s" % self.users[3], + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 1) + self.assertEqual(len(channel.json_body["user_reports"]), 1) + self.assertNotIn("next_token", channel.json_body) + self._check_fields(channel.json_body["user_reports"]) + + for report in channel.json_body["user_reports"]: + self.assertEqual(report["target_user_id"], self.users[3]) + + def test_filter_user(self) -> None: + """ + Testing list of reported users with a filter of reporting user + """ + + channel = self.make_request( + "GET", + self.url + "?user_id=%s" % self.users[0], + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 4) + self.assertEqual(len(channel.json_body["user_reports"]), 4) + self.assertNotIn("next_token", channel.json_body) + self._check_fields(channel.json_body["user_reports"]) + + for report in channel.json_body["user_reports"]: + self.assertEqual(report["user_id"], self.users[0]) + + def test_filter_user_and_target_user(self) -> None: + """ + Testing list of reported users with a filter of reporting user and target_user + """ + + channel = self.make_request( + "GET", + self.url + "?user_id=%s&target_user_id=%s" % (self.users[1], self.users[7]), + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 1) + self.assertEqual(len(channel.json_body["user_reports"]), 1) + self.assertNotIn("next_token", channel.json_body) + self._check_fields(channel.json_body["user_reports"]) + + for report in channel.json_body["user_reports"]: + self.assertEqual(report["user_id"], self.users[1]) + self.assertEqual(report["target_user_id"], self.users[7]) + + def test_valid_search_order(self) -> None: + """ + Testing search order. Order by timestamps. + """ + + # fetch the most recent first, largest timestamp + channel = self.make_request( + "GET", + self.url + "?dir=b", + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 8) + self.assertEqual(len(channel.json_body["user_reports"]), 8) + report = 1 + while report < len(channel.json_body["user_reports"]): + self.assertGreaterEqual( + channel.json_body["user_reports"][report - 1]["received_ts"], + channel.json_body["user_reports"][report]["received_ts"], + ) + report += 1 + + # fetch the oldest first, smallest timestamp + channel = self.make_request( + "GET", + self.url + "?dir=f", + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 8) + self.assertEqual(len(channel.json_body["user_reports"]), 8) + report = 1 + while report < len(channel.json_body["user_reports"]): + self.assertLessEqual( + channel.json_body["user_reports"][report - 1]["received_ts"], + channel.json_body["user_reports"][report]["received_ts"], + ) + report += 1 + + def test_invalid_search_order(self) -> None: + """ + Testing that a invalid search order returns a 400 + """ + + channel = self.make_request( + "GET", + self.url + "?dir=bar", + access_token=self.admin_user_tok, + ) + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + self.assertEqual( + "Query parameter 'dir' must be one of ['b', 'f']", + channel.json_body["error"], + ) + + def test_limit_is_negative(self) -> None: + """ + Testing that a negative limit parameter returns a 400 + """ + + channel = self.make_request( + "GET", + self.url + "?limit=-5", + access_token=self.admin_user_tok, + ) + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + + def test_from_is_negative(self) -> None: + """ + Testing that a negative from parameter returns a 400 + """ + + channel = self.make_request( + "GET", + self.url + "?from=-5", + access_token=self.admin_user_tok, + ) + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + + def test_next_token(self) -> None: + """ + Testing that `next_token` appears at the right place + """ + + # `next_token` does not appear + # Number of results is the number of entries + channel = self.make_request( + "GET", + self.url + "?limit=8", + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 8) + self.assertEqual(len(channel.json_body["user_reports"]), 8) + self.assertNotIn("next_token", channel.json_body) + + # `next_token` does not appear + # Number of max results is larger than the number of entries + channel = self.make_request( + "GET", + self.url + "?limit=10", + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 8) + self.assertEqual(len(channel.json_body["user_reports"]), 8) + self.assertNotIn("next_token", channel.json_body) + + # `next_token` does appear + # Number of max results is smaller than the number of entries + channel = self.make_request( + "GET", + self.url + "?limit=6", + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 8) + self.assertEqual(len(channel.json_body["user_reports"]), 6) + self.assertEqual(channel.json_body["next_token"], 6) + + # Check + # Set `from` to value of `next_token` for request remaining entries + # `next_token` does not appear + channel = self.make_request( + "GET", + self.url + "?from=6", + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 8) + self.assertEqual(len(channel.json_body["user_reports"]), 2) + self.assertNotIn("next_token", channel.json_body) + + def _report_user(self, target_user: str, reporter_tok: str) -> None: + """Report a user""" + channel = self.make_request( + "POST", + "_matrix/client/v3/users/%s/report" % (target_user), + {"reason": "stone-cold bummer"}, + access_token=reporter_tok, + shorthand=False, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + + def _check_fields(self, content: list[JsonDict]) -> None: + """Checks that all attributes are present in a user report""" + for c in content: + self.assertIn("id", c) + self.assertIn("received_ts", c) + self.assertIn("user_id", c) + self.assertIn("target_user_id", c) + self.assertIn("reason", c) + + +class UserReportDetailTestCase(unittest.HomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + room.register_servlets, + reporting.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.admin_user = self.register_user("admin", "pass", admin=True) + self.admin_user_tok = self.login("admin", "pass") + + self.bad_user = self.register_user("user", "pass") + self.bad_user_tok = self.login("user", "pass") + + channel = self.make_request( + "POST", + "_matrix/client/v3/users/%s/report" % (self.bad_user), + {"reason": "stone-cold bummer"}, + access_token=self.admin_user_tok, + shorthand=False, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + + # first created user report gets `id`=2 + self.url = "/_synapse/admin/v1/user_reports/2" + + def test_no_auth(self) -> None: + """ + Try to get user report without authentication. + """ + channel = self.make_request("GET", self.url, {}) + + self.assertEqual(401, channel.code, msg=channel.json_body) + self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) + + def test_requester_is_no_admin(self) -> None: + """ + If the user is not a server admin, an error 403 is returned. + """ + + channel = self.make_request( + "GET", + self.url, + access_token=self.bad_user_tok, + ) + + self.assertEqual(403, channel.code, msg=channel.json_body) + self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) + + def test_default_success(self) -> None: + """ + Testing get a reported user + """ + + channel = self.make_request( + "GET", + self.url, + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self._check_fields(channel.json_body) + + def test_invalid_report_id(self) -> None: + """ + Testing that an invalid `report_id` returns a 400. + """ + + # `report_id` is negative + channel = self.make_request( + "GET", + "/_synapse/admin/v1/user_reports/-123", + access_token=self.admin_user_tok, + ) + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + self.assertEqual( + "The report_id parameter must be a string representing a positive integer.", + channel.json_body["error"], + ) + + # `report_id` is a non-numerical string + channel = self.make_request( + "GET", + "/_synapse/admin/v1/user_reports/abcdef", + access_token=self.admin_user_tok, + ) + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + self.assertEqual( + "The report_id parameter must be a string representing a positive integer.", + channel.json_body["error"], + ) + + # `report_id` is undefined + channel = self.make_request( + "GET", + "/_synapse/admin/v1/user_reports/", + access_token=self.admin_user_tok, + ) + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + self.assertEqual( + "The report_id parameter must be a string representing a positive integer.", + channel.json_body["error"], + ) + + def test_report_id_not_found(self) -> None: + """ + Testing that a not existing `report_id` returns a 404. + """ + + channel = self.make_request( + "GET", + "/_synapse/admin/v1/user_reports/123", + access_token=self.admin_user_tok, + ) + + self.assertEqual(404, channel.code, msg=channel.json_body) + self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) + self.assertEqual("User report not found", channel.json_body["error"]) + + def _check_fields(self, content: JsonDict) -> None: + """Checks that all attributes are present in a user report""" + self.assertIn("id", content) + self.assertIn("received_ts", content) + self.assertIn("target_user_id", content) + self.assertIn("user_id", content) + self.assertIn("reason", content) + + +class DeleteUserReportTestCase(unittest.HomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self._store = hs.get_datastores().main + + self.admin_user = self.register_user("admin", "pass", admin=True) + self.admin_user_tok = self.login("admin", "pass") + + self.bad_user = self.register_user("user", "pass") + self.other_user_tok = self.login("user", "pass") + + # create report + self.get_success( + self._store.add_user_report( + self.bad_user, + self.admin_user, + "super bummer", + self.clock.time_msec(), + ) + ) + + self.url = "/_synapse/admin/v1/user_reports/2" + + def test_no_auth(self) -> None: + """ + Try to delete user report without authentication. + """ + channel = self.make_request("DELETE", self.url) + + self.assertEqual(401, channel.code, msg=channel.json_body) + self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) + + def test_requester_is_no_admin(self) -> None: + """ + If the user is not a server admin, an error 403 is returned. + """ + + channel = self.make_request( + "DELETE", + self.url, + access_token=self.other_user_tok, + ) + + self.assertEqual(403, channel.code, msg=channel.json_body) + self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) + + def test_delete_success(self) -> None: + """ + Testing delete a report. + """ + + channel = self.make_request( + "DELETE", + self.url, + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual({}, channel.json_body) + + channel = self.make_request( + "GET", + self.url, + access_token=self.admin_user_tok, + ) + + # check that report was deleted + self.assertEqual(404, channel.code, msg=channel.json_body) + self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) + + def test_invalid_report_id(self) -> None: + """ + Testing that an invalid `report_id` returns a 400. + """ + + # `report_id` is negative + channel = self.make_request( + "DELETE", + "/_synapse/admin/v1/user_reports/-123", + access_token=self.admin_user_tok, + ) + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + self.assertEqual( + "The report_id parameter must be a string representing a positive integer.", + channel.json_body["error"], + ) + + # `report_id` is a non-numerical string + channel = self.make_request( + "DELETE", + "/_synapse/admin/v1/user_reports/abcdef", + access_token=self.admin_user_tok, + ) + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + self.assertEqual( + "The report_id parameter must be a string representing a positive integer.", + channel.json_body["error"], + ) + + # `report_id` is undefined + channel = self.make_request( + "DELETE", + "/_synapse/admin/v1/user_reports/", + access_token=self.admin_user_tok, + ) + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + self.assertEqual( + "The report_id parameter must be a string representing a positive integer.", + channel.json_body["error"], + ) + + def test_report_id_not_found(self) -> None: + """ + Testing that a not existing `report_id` returns a 404. + """ + + channel = self.make_request( + "DELETE", + "/_synapse/admin/v1/user_reports/123", + access_token=self.admin_user_tok, + ) + + self.assertEqual(404, channel.code, msg=channel.json_body) + self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) + self.assertEqual("User report not found", channel.json_body["error"]) diff --git a/tests/rest/client/sliding_sync/test_room_subscriptions.py b/tests/rest/client/sliding_sync/test_room_subscriptions.py index 811478f1ba7..d970af367d3 100644 --- a/tests/rest/client/sliding_sync/test_room_subscriptions.py +++ b/tests/rest/client/sliding_sync/test_room_subscriptions.py @@ -22,6 +22,7 @@ from synapse.api.constants import EventTypes, HistoryVisibility from synapse.rest.client import login, room, sync from synapse.server import HomeServer +from synapse.types import JsonDict from synapse.util.clock import Clock from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase @@ -126,6 +127,124 @@ def test_room_subscriptions_with_join_membership(self) -> None: response_body["rooms"][room_id1], ) + def test_room_subscription_required_state_expansion_returns_immediately( + self, + ) -> None: + """ + Test that adding a room subscription with stronger params than the list causes an + incremental long-poll to return immediately, even without new stream activity. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + room_id1 = self.helper.create_room_as(user1_id, tok=user1_tok) + + sync_body: JsonDict = { + "lists": { + "foo-list": { + "ranges": [[0, 0]], + "required_state": [], + "timeline_limit": 0, + } + }, + "conn_id": "conn_id", + } + _, from_token = self.do_sync(sync_body, tok=user1_tok) + + sync_body["room_subscriptions"] = { + room_id1: { + "required_state": [ + [EventTypes.Create, ""], + ], + "timeline_limit": 0, + } + } + + channel = self.make_request( + "POST", + self.sync_endpoint + f"?timeout=10000&pos={from_token}", + content=sync_body, + access_token=user1_tok, + await_result=False, + ) + channel.await_result(timeout_ms=3000) + self.assertEqual(channel.code, 200, channel.json_body) + + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + + room_response = channel.json_body["rooms"][room_id1] + self.assertNotIn("initial", room_response) + self._assertRequiredStateIncludes( + room_response["required_state"], + { + state_map[(EventTypes.Create, "")], + }, + exact=True, + ) + + def test_room_subscription_required_state_change_returns_immediately(self) -> None: + """ + Test that expanding an existing room subscription's required state causes an + incremental long-poll to return immediately, even without new stream activity. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + room_id1 = self.helper.create_room_as( + user1_id, tok=user1_tok, extra_content={"name": "Foo"} + ) + + sync_body: JsonDict = { + "room_subscriptions": { + room_id1: { + "required_state": [ + [EventTypes.Create, ""], + ], + "timeline_limit": 0, + } + }, + "conn_id": "conn_id", + } + response_body, from_token = self.do_sync(sync_body, tok=user1_tok) + + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + state_map[(EventTypes.Create, "")], + }, + exact=True, + ) + + sync_body["room_subscriptions"][room_id1]["required_state"] = [ + [EventTypes.Create, ""], + [EventTypes.Name, ""], + ] + + channel = self.make_request( + "POST", + self.sync_endpoint + f"?timeout=10000&pos={from_token}", + content=sync_body, + access_token=user1_tok, + await_result=False, + ) + channel.await_result(timeout_ms=3000) + self.assertEqual(channel.code, 200, channel.json_body) + + room_response = channel.json_body["rooms"][room_id1] + self.assertNotIn("initial", room_response) + self._assertRequiredStateIncludes( + room_response["required_state"], + { + state_map[(EventTypes.Name, "")], + }, + exact=True, + ) + def test_room_subscriptions_with_leave_membership(self) -> None: """ Test `room_subscriptions` with a leave room should give us timeline and state diff --git a/tests/rest/client/sliding_sync/test_rooms_meta.py b/tests/rest/client/sliding_sync/test_rooms_meta.py index 9e3f8aaf943..4ff07b4d261 100644 --- a/tests/rest/client/sliding_sync/test_rooms_meta.py +++ b/tests/rest/client/sliding_sync/test_rooms_meta.py @@ -12,6 +12,7 @@ # . # import logging +from typing import Any from parameterized import parameterized, parameterized_class @@ -966,7 +967,7 @@ def test_rooms_bump_stamp_backfill(self) -> None: creator = "@user:other" room_id = "!foo:other" room_version = RoomVersions.V10 - shared_kwargs = { + shared_kwargs: dict[str, Any] = { "room_id": room_id, "room_version": room_version.identifier, } diff --git a/tests/rest/client/test_account.py b/tests/rest/client/test_account.py index ffa96c7840a..42102230f00 100644 --- a/tests/rest/client/test_account.py +++ b/tests/rest/client/test_account.py @@ -31,14 +31,14 @@ import synapse.rest.admin from synapse.api.constants import LoginType, Membership -from synapse.api.errors import Codes, HttpResponseException +from synapse.api.errors import Codes, HttpResponseException, SynapseError from synapse.appservice import ApplicationService from synapse.rest import admin from synapse.rest.client import account, login, register, room from synapse.rest.synapse.client.password_reset import PasswordResetSubmitTokenResource from synapse.server import HomeServer from synapse.storage._base import db_to_json -from synapse.types import JsonDict, UserID +from synapse.types import JsonDict, UserID, create_requester from synapse.util.clock import Clock from tests import unittest @@ -500,6 +500,123 @@ def test_deactivate_account(self) -> None: channel = self.make_request("GET", "account/whoami", access_token=tok) self.assertEqual(channel.code, 401) + def test_deactivate_erase_account(self) -> None: + """ + Test that a user account can be signaled for erasure on the Matrix spec endpoint + for client access, `/account/deactivate` and that profile data is erased as part + of the process + """ + mxid = self.register_user("kermit", "test") + user_id = UserID.from_string(mxid) + tok = self.login("kermit", "test") + + profile_handler = self.hs.get_profile_handler() + + # Set some profile data that can be checked for after the user is erased + self.get_success( + profile_handler.set_displayname( + user_id, create_requester(user_id), "Kermit the Frog" + ) + ) + self.get_success( + profile_handler.set_avatar_url( + user_id, create_requester(user_id), "http://test/Kermit.jpg" + ) + ) + # Verify it is set + self.assertEqual( + self.get_success(profile_handler.get_displayname(user_id)), + "Kermit the Frog", + ) + self.assertEqual( + self.get_success(profile_handler.get_avatar_url(user_id)), + "http://test/Kermit.jpg", + ) + + # Deactivate! + self.deactivate(mxid, tok, erase=True) + + store = self.hs.get_datastores().main + + # Check that the user has been marked as deactivated. + self.assertTrue(self.get_success(store.get_user_deactivated_status(mxid))) + + # On deactivation with 'erase', the entire database row is erased. Both of these + # should raise a 404(Not Found) SynapseError + display_name_failure = self.get_failure( + profile_handler.get_displayname(user_id), SynapseError + ) + assert display_name_failure.value.code == HTTPStatus.NOT_FOUND + + avatar_url_failure = self.get_failure( + profile_handler.get_avatar_url(user_id), SynapseError + ) + assert avatar_url_failure.value.code == HTTPStatus.NOT_FOUND + + # Check that this access token has been invalidated. + channel = self.make_request("GET", "account/whoami", access_token=tok) + self.assertEqual(channel.code, 401) + + @override_config({"enable_set_displayname": False, "enable_set_avatar_url": False}) + def test_deactivate_erase_account_with_disabled_profile_changes(self) -> None: + """ + Test that deactivating the user with the 'erase' option will remove existing + profile data, even with the Synapse configuration to forbid profile changes + """ + mxid = self.register_user("kermit", "test") + user_id = UserID.from_string(mxid) + tok = self.login("kermit", "test") + + profile_handler = self.hs.get_profile_handler() + + # Can not use the profile handler to set a display name when it is disabled. Use + # the database directly + store = self.hs.get_datastores().main + self.get_success(store.set_profile_displayname(user_id, "Kermit the Frog")) + self.get_success( + store.set_profile_avatar_url(user_id, "http://test/Kermit.jpg") + ) + + # Verify it is set + self.assertEqual( + (self.get_success(store.get_profile_displayname(user_id))), + "Kermit the Frog", + ) + self.assertEqual( + self.get_success(profile_handler.get_displayname(user_id)), + "Kermit the Frog", + ) + self.assertEqual( + (self.get_success(store.get_profile_avatar_url(user_id))), + "http://test/Kermit.jpg", + ) + self.assertEqual( + self.get_success(profile_handler.get_avatar_url(user_id)), + "http://test/Kermit.jpg", + ) + + # Deactivate! + self.deactivate(mxid, tok, erase=True) + + # Check that the user has been marked as deactivated. + self.assertTrue(self.get_success(store.get_user_deactivated_status(mxid))) + + # On deactivation with 'erase', the entire database row is erased. Both of these + # should raise a 404(Not Found) SynapseError + display_name_failure = self.get_failure( + profile_handler.get_displayname(user_id), SynapseError + ) + assert display_name_failure.value.code == HTTPStatus.NOT_FOUND + + avatar_url_failure = self.get_failure( + profile_handler.get_avatar_url(user_id), SynapseError + ) + assert avatar_url_failure.value.code == HTTPStatus.NOT_FOUND + + # Check that this access token has been invalidated. + channel = self.make_request("GET", "account/whoami", access_token=tok) + self.assertEqual(channel.code, 401) + def test_pending_invites(self) -> None: """Tests that deactivating a user rejects every pending invite for them.""" store = self.hs.get_datastores().main @@ -698,14 +815,23 @@ def test_background_update_deletes_deactivated_users_server_side_backup_keys( ) self.assertEqual(len(res2), 4) - def deactivate(self, user_id: str, tok: str) -> None: + def deactivate(self, user_id: str, tok: str, erase: bool = False) -> None: + """ + Helper to deactivate a user using the /account/deactivate endpoint, optionally + with erasure + + Args: + user_id: the string formatted mxid(not a UserID) + tok: the user's access token + erase: bool of if this should be a full erasure request + """ request_data = { "auth": { "type": "m.login.password", "user": user_id, "password": "test", }, - "erase": False, + "erase": erase, } channel = self.make_request( "POST", "account/deactivate", request_data, access_token=tok diff --git a/tests/rest/client/test_auth.py b/tests/rest/client/test_auth.py index ffaf0e5a321..c266937a65a 100644 --- a/tests/rest/client/test_auth.py +++ b/tests/rest/client/test_auth.py @@ -43,6 +43,62 @@ from tests.server import FakeChannel from tests.unittest import override_config, skip_unless +TEST_MULTIPLE_OIDC_IDP_ID1 = "friendface" +TEST_MULTIPLE_OIDC_IDP_ID2 = "potatocloud" +TEST_MULTIPLE_OIDC_IDP_ID3 = "svnswitch" + +TEST_MULTIPLE_OIDC_ISSUER1 = "https://friendface.invalid/" +TEST_MULTIPLE_OIDC_ISSUER2 = "https://potato.invalid/" +TEST_MULTIPLE_OIDC_ISSUER3 = "https://subversionswitch.invalid/" + +TEST_MULTIPLE_OIDC_PROVIDERS = [ + { + "idp_id": TEST_MULTIPLE_OIDC_IDP_ID1, + "idp_name": "FriendFace", + "idp_icon": "mxc://example.invalid/friendface", + "idp_brand": "friendface", + "issuer": TEST_MULTIPLE_OIDC_ISSUER1, + "client_id": "id-for-friendface", + "client_secret": "secret-for-friendface", + "scopes": ["openid"], + "discover": False, + "authorization_endpoint": f"{TEST_MULTIPLE_OIDC_ISSUER1}oidc/auth", + "token_endpoint": f"{TEST_MULTIPLE_OIDC_ISSUER1}oidc/token", + "jwks_uri": f"{TEST_MULTIPLE_OIDC_ISSUER1}oidc/jwks", + }, + { + "idp_id": TEST_MULTIPLE_OIDC_IDP_ID2, + "idp_name": "Potato Cloud", + "idp_icon": "mxc://example.invalid/potatocloud", + "idp_brand": "potato", + "issuer": TEST_MULTIPLE_OIDC_ISSUER2, + "client_id": "id-for-potato", + "client_secret": "secret-for-potato", + "scopes": ["openid"], + "discover": False, + "authorization_endpoint": f"{TEST_MULTIPLE_OIDC_ISSUER2}oidc/auth", + "token_endpoint": f"{TEST_MULTIPLE_OIDC_ISSUER2}oidc/token", + "jwks_uri": f"{TEST_MULTIPLE_OIDC_ISSUER2}oidc/jwks", + }, + { + "idp_id": TEST_MULTIPLE_OIDC_IDP_ID3, + "idp_name": "SubversionSwitch", + "idp_icon": "mxc://example.invalid/svnswitch", + "idp_brand": "svnswitch", + "issuer": TEST_MULTIPLE_OIDC_ISSUER3, + "client_id": "id-for-svnswitch", + "client_secret": "secret-for-svnswitch", + "scopes": ["openid"], + "discover": False, + "authorization_endpoint": f"{TEST_MULTIPLE_OIDC_ISSUER3}oidc/auth", + "token_endpoint": f"{TEST_MULTIPLE_OIDC_ISSUER3}oidc/token", + "jwks_uri": f"{TEST_MULTIPLE_OIDC_ISSUER3}oidc/jwks", + }, +] +""" +`oidc_providers` config example for multiple OIDC providers. +""" + class DummyRecaptchaChecker(UserInteractiveAuthChecker): def __init__(self, hs: HomeServer) -> None: @@ -506,6 +562,90 @@ def test_ui_auth_via_sso(self) -> None: body={"auth": {"session": session_id}}, ) + @skip_unless(HAS_OIDC, "requires OIDC") + @override_config( + { + "oidc_providers": TEST_MULTIPLE_OIDC_PROVIDERS, + "experimental_features": {"msc4450_enabled": True}, + } + ) + def test_msc4450_select_idp_id(self) -> None: + """ + Test for MSC4450: Identity Provider selection for + User-Interactive Authentication with Legacy Single Sign-On. + + We configure 3 OIDC providers and then check that we can select + which one to redirect to for User-Interactive Authentication. + """ + + # Attach the user to the OIDC providers manually + for idp_id in ( + TEST_MULTIPLE_OIDC_IDP_ID1, + TEST_MULTIPLE_OIDC_IDP_ID2, + ): + self.get_success( + self.hs.get_datastores().main.record_user_external_id( + # `oidc-` is a magic prefix needed for OIDC providers + f"oidc-{idp_id}", + # arbitrary/opaque provider-specific external ID, doesn't really matter + "some.ext.user.id", + self.user, + ) + ) + + # Start a user-interactive authentication session by + # calling the device deletion API + channel = self.delete_device( + self.user_tok, self.device_id, HTTPStatus.UNAUTHORIZED + ) + auth_session = channel.json_body["session"] + flows = channel.json_body["flows"] + self.assertCountEqual( + flows, [{"stages": ["m.login.sso"]}, {"stages": ["m.login.password"]}] + ) + + # Try to start the User-Interactive Auth against both identity providers. + # Make sure we get the identity provider that we ask for. + # + # We need to try against both to be sure that Synapse doesn't just conveniently happen to + # arbitrarily select the identity provider we test. + for idp_id, provider_config in ( + ( + TEST_MULTIPLE_OIDC_IDP_ID1, + TEST_MULTIPLE_OIDC_PROVIDERS[0], + ), + ( + TEST_MULTIPLE_OIDC_IDP_ID2, + TEST_MULTIPLE_OIDC_PROVIDERS[1], + ), + ): + endpoint = f"/_matrix/client/v3/auth/m.login.sso/fallback/web?session={auth_session}&io.element.idp_id=oidc-{idp_id}" + channel = self.make_request("GET", endpoint) + self.assertEqual( + channel.code, + HTTPStatus.OK, + f"Failed to use the {endpoint} endpoint as part of the UIA flow for idp_id={idp_id} : response_body={channel.text_body}", + ) + + # This 'Continue with ...' text is templated by `synapse/res/templates/sso_auth_confirm.html` + self.assertIn( + f"Continue with {provider_config['idp_name']}", + channel.text_body, + ) + + self.assertIn(provider_config["authorization_endpoint"], channel.text_body) + + # Test that we can't use the 3rd OIDC provider as we're not + # registered with it + channel = self.make_request( + "GET", + f"/_matrix/client/v3/auth/m.login.sso/fallback/web?session={auth_session}&io.element.idp_id=oidc-{TEST_MULTIPLE_OIDC_IDP_ID3}", + ) + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.json_body) + self.assertEqual( + channel.json_body["errcode"], Codes.INVALID_PARAM, channel.json_body + ) + @skip_unless(HAS_OIDC, "requires OIDC") @override_config({"oidc_config": TEST_OIDC_CONFIG}) def test_does_not_offer_password_for_sso_user(self) -> None: diff --git a/tests/rest/client/test_keys.py b/tests/rest/client/test_keys.py index 9c83a284d73..fc21d83ef2e 100644 --- a/tests/rest/client/test_keys.py +++ b/tests/rest/client/test_keys.py @@ -160,9 +160,12 @@ def test_upload_keys_fails_on_invalid_user_id_or_device_id(self) -> None: channel.result, ) - def test_upload_keys_succeeds_when_fields_are_explicitly_set_to_null(self) -> None: + def test_upload_keys_rejects_device_keys_set_to_null(self) -> None: """ - This is a regression test for https://github.com/element-hq/synapse/pull/19023. + Test that sending `device_keys: null` is rejected per spec. + The spec says `device_keys` may be omitted, but not set to `null`. + + See https://github.com/element-hq/synapse/issues/19030. """ device_id = "DEVICE_ID" self.register_user("alice", "wonderland") @@ -173,12 +176,10 @@ def test_upload_keys_succeeds_when_fields_are_explicitly_set_to_null(self) -> No "/_matrix/client/v3/keys/upload", { "device_keys": None, - "one_time_keys": None, - "fallback_keys": None, }, alice_token, ) - self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) class KeyQueryTestCase(unittest.HomeserverTestCase): diff --git a/tests/rest/client/test_retention.py b/tests/rest/client/test_retention.py index 82a3b5b3378..1f8d9154ca6 100644 --- a/tests/rest/client/test_retention.py +++ b/tests/rest/client/test_retention.py @@ -23,6 +23,7 @@ from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes +from synapse.events.utils import FilteredEvent from synapse.rest import admin from synapse.rest.client import login, room from synapse.server import HomeServer @@ -173,7 +174,9 @@ def test_visibility(self) -> None: # We should only get one event back. self.assertEqual(len(filtered_events), 1, filtered_events) # That event should be the second, not outdated event. - self.assertEqual(filtered_events[0].event_id, valid_event_id, filtered_events) + self.assertEqual( + filtered_events[0].event.event_id, valid_event_id, filtered_events + ) def _test_retention_event_purged(self, room_id: str, increment: float) -> None: """Run the following test scenario to test the message retention policy support: @@ -253,7 +256,11 @@ def get_event(self, event_id: str, expect_none: bool = False) -> JsonDict: assert event is not None time_now = self.clock.time_msec() - serialized = self.get_success(self.serializer.serialize_event(event, time_now)) + serialized = self.get_success( + self.serializer.serialize_event( + FilteredEvent(event=event, membership=None), time_now + ) + ) return serialized diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index e6ada1adb27..74a8678ae99 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -29,6 +29,7 @@ from synapse.api.constants import ( EventContentFields, EventTypes, + JoinRules, ReceiptTypes, RelationTypes, ) @@ -394,6 +395,69 @@ def test_knock_room_state(self) -> None: ) +class SyncCreateEventInPrejoinStateTestCase(unittest.HomeserverTestCase): + """MSC4311: Tests that m.room.create is present in invite_state and knock_state""" + + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + room.register_servlets, + sync.register_servlets, + knock.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + return config + + def test_create_event_present_in_invite_state(self) -> None: + """m.room.create must appear in invite_state.""" + inviter = self.register_user("inviter", "pass") + inviter_tok = self.login("inviter", "pass") + invitee = self.register_user("invitee", "pass") + invitee_tok = self.login("invitee", "pass") + + room_id = self.helper.create_room_as(inviter, tok=inviter_tok) + self.helper.invite(room=room_id, src=inviter, targ=invitee, tok=inviter_tok) + + channel = self.make_request("GET", "/sync", access_token=invitee_tok) + self.assertEqual(channel.code, 200, channel.json_body) + + invite_state_events = channel.json_body["rooms"]["invite"][room_id][ + "invite_state" + ]["events"] + event_types = {stripped_event["type"] for stripped_event in invite_state_events} + self.assertIn(EventTypes.Create, event_types) + + def test_create_event_present_in_knock_state(self) -> None: + """m.room.create must appear in knock_state.""" + host = self.register_user("host", "pass") + host_tok = self.login("host", "pass") + knocker = self.register_user("knocker", "pass") + knocker_tok = self.login("knocker", "pass") + + room_id = self.helper.create_room_as( + host, is_public=False, room_version="7", tok=host_tok + ) + self.helper.send_state( + room_id, + EventTypes.JoinRules, + {"join_rule": JoinRules.KNOCK}, + tok=host_tok, + ) + + self.helper.knock(room_id, knocker, tok=knocker_tok) + + channel = self.make_request("GET", "/sync", access_token=knocker_tok) + self.assertEqual(channel.code, 200, channel.json_body) + + knock_state_events = channel.json_body["rooms"]["knock"][room_id][ + "knock_state" + ]["events"] + event_types = {stripped_event["type"] for stripped_event in knock_state_events} + self.assertIn(EventTypes.Create, event_types) + + class UnreadMessagesTestCase(unittest.HomeserverTestCase): servlets = [ synapse.rest.admin.register_servlets, diff --git a/tests/rest/client/test_upgrade_room.py b/tests/rest/client/test_upgrade_room.py index ee264929090..6cb85c94c42 100644 --- a/tests/rest/client/test_upgrade_room.py +++ b/tests/rest/client/test_upgrade_room.py @@ -23,6 +23,7 @@ from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventContentFields, EventTypes, Membership, RoomTypes +from synapse.api.room_versions import RoomVersions from synapse.config.server import DEFAULT_ROOM_VERSION from synapse.rest import admin from synapse.rest.client import login, room, room_upgrade_rest_servlet @@ -58,6 +59,7 @@ def _upgrade_room( token: str | None = None, room_id: str | None = None, expire_cache: bool = True, + new_version: str = DEFAULT_ROOM_VERSION, ) -> FakeChannel: if expire_cache: # We don't want a cached response. @@ -70,7 +72,7 @@ def _upgrade_room( "POST", f"/_matrix/client/r0/rooms/{room_id}/upgrade", # This will upgrade a room to the same version, but that's fine. - content={"new_version": DEFAULT_ROOM_VERSION}, + content={"new_version": new_version}, access_token=token or self.creator_token, ) @@ -431,3 +433,51 @@ def test_bans(self) -> None: tok=self.creator_token, ) self.assertEqual(content[EventContentFields.MEMBERSHIP], Membership.BAN) + + def test_creator_removed_from_powerlevels_v12(self) -> None: + """ + Test that the creator is removed from the power levels users map when + upgrading to a room version with MSC4289. + """ + # Create a room on room version 11, which doesn't have MSC4289. + room_id = self.helper.create_room_as( + self.creator, tok=self.creator_token, room_version="11" + ) + self.helper.join(room_id, self.other, tok=self.other_token) + + # Retrieve the room's current power levels. + old_power_level_event = self.get_success( + self.hs.get_storage_controllers().state.get_current_state_event( + room_id, "m.room.power_levels", "" + ) + ) + assert old_power_level_event is not None + + # The creator should be in the users map with power level 100. + self.assertEqual(old_power_level_event.content["users"][self.creator], 100) + + # Upgrade the room to version 12, which has MSC4289. + channel = self._upgrade_room( + room_id=room_id, new_version=RoomVersions.V12.identifier + ) + self.assertEqual(200, channel.code, channel.result) + + # Extract the new room ID. + new_room_id = channel.json_body["replacement_room"] + + # Fetch the new room's power level event. + new_power_levels = self.helper.get_state( + new_room_id, + "m.room.power_levels", + tok=self.creator_token, + ) + + # The creator should no longer be in the users map. + self.assertNotIn(self.creator, new_power_levels["users"]) + + # The creator should still be in the old power levels event with power + # level 100. + # + # This is a regression test where previously Synapse would accidentally + # mutate the old power levels event. + self.assertEqual(old_power_level_event.content["users"][self.creator], 100) diff --git a/tests/storage/test_devices.py b/tests/storage/test_devices.py index 1d1979e19ff..b153c749808 100644 --- a/tests/storage/test_devices.py +++ b/tests/storage/test_devices.py @@ -19,15 +19,21 @@ # # +import itertools from typing import Collection +from unittest.mock import patch from twisted.internet.testing import MemoryReactor import synapse.api.errors from synapse.api.constants import EduTypes from synapse.server import HomeServer +from synapse.storage.databases.main.devices import ( + PRUNE_DEVICE_LISTS_CHANGES_IN_ROOM_AGE, +) from synapse.types import JsonDict from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests.unittest import HomeserverTestCase @@ -351,3 +357,101 @@ def test_update_unknown_device(self) -> None: synapse.api.errors.StoreError, ) self.assertEqual(404, exc.value.code) + + @patch("synapse.storage.databases.main.devices.PRUNE_DEVICE_LISTS_BATCH_SIZE", 5) + def test_prune_old_device_lists_changes_in_room(self) -> None: + """Test that old entries in the `device_lists_changes_in_room` table are pruned properly.""" + + # Pretend the user is in a few rooms. + room_ids = [f"!room{i}:test" for i in range(20)] + + # Create a generator for device IDs so we can easily create many unique + # device IDs without having to keep track of the count ourselves. + device_id_gen = (f"device_id{i}" for i in itertools.count()) + + def get_devices_in_room_status() -> tuple[int, str]: + """Helper function to get the count of entries in + `device_lists_changes_in_room` and the minimum device_id.""" + return self.get_success( + self.store.db_pool.simple_select_one( + table="device_lists_changes_in_room", + keyvalues={}, + retcols=("COUNT(*)", "MIN(device_id)"), + ) + ) + + # First we add some initial entries to the `device_lists_changes_in_room`. + self.get_success( + self.store.add_device_change_to_streams( + user_id="@user_id:test", + device_ids=[next(device_id_gen) for _ in range(10)], + room_ids=room_ids, + ) + ) + + # Advance the reactor a while, but not long enough to trigger pruning. + self.reactor.advance(Duration(hours=1).as_secs()) + + # The `device_lists_changes_in_room` table should now have 10 * + # len(room_ids) entries, and the minimum device_id should be + # `device_id0`. + count, min_device_id = get_devices_in_room_status() + self.assertEqual(count, 10 * len(room_ids)) + self.assertEqual(min_device_id, "device_id0") + + # Record the max pruned stream ID before pruning, so we can check + # that this correctly updates after pruning. + starting_max_pruned_id = self.get_success( + self.store.db_pool.runInteraction( + "get_max_pruned_device_lists_changes_in_room", + self.store._get_max_pruned_device_lists_changes_in_room_txn, + ) + ) + + # Now we add some more entries. + self.get_success( + self.store.add_device_change_to_streams( + user_id="@user_id:test", + device_ids=[next(device_id_gen) for _ in range(10)], + room_ids=room_ids, + ) + ) + + # Advance the reactor a while more, so that the first batch of entries is + # now old enough to be pruned. + self.reactor.advance( + (PRUNE_DEVICE_LISTS_CHANGES_IN_ROOM_AGE - Duration(minutes=30)).as_secs() + ) + + # Advance repeatedly a bit so that the pruning process can run to completion. + for _ in range(10): + self.reactor.advance(Duration(milliseconds=110).as_secs()) + + # Check that the old entries have been pruned, and the new entries are still there. + count, min_device_id = get_devices_in_room_status() + self.assertEqual(count, 10 * len(room_ids)) + self.assertEqual(min_device_id, "device_id10") + + # We should always keep the most recent entries, even if they are old enough to be pruned. + self.reactor.advance( + (PRUNE_DEVICE_LISTS_CHANGES_IN_ROOM_AGE + Duration(minutes=30)).as_secs() + ) + + # Advance repeatedly a bit so that the pruning process can run to completion. + for _ in range(10): + self.reactor.advance(Duration(milliseconds=110).as_secs()) + + count, min_device_id = get_devices_in_room_status() + # We should always keep the most recent entries so that we can + # calculate the maximum stream ID used. + self.assertEqual(count, len(room_ids)) + self.assertEqual(min_device_id, "device_id19") + + # Check that the max pruned stream ID has been advanced after pruning. + max_pruned_id = self.get_success( + self.store.db_pool.runInteraction( + "get_max_pruned_device_lists_changes_in_room", + self.store._get_max_pruned_device_lists_changes_in_room_txn, + ) + ) + self.assertGreater(max_pruned_id, starting_max_pruned_id) diff --git a/tests/storage/test_events_bg_updates.py b/tests/storage/test_events_bg_updates.py index a5b53de77f2..aceacec98e0 100644 --- a/tests/storage/test_events_bg_updates.py +++ b/tests/storage/test_events_bg_updates.py @@ -14,17 +14,23 @@ # +import json + +import signedjson.key from canonicaljson import encode_canonical_json from twisted.internet.testing import MemoryReactor from synapse.api.constants import MAX_DEPTH from synapse.api.room_versions import RoomVersion, RoomVersions +from synapse.rest import admin +from synapse.rest.client import login, room from synapse.server import HomeServer +from synapse.storage.background_updates import BackgroundUpdater from synapse.types.storage import _BackgroundUpdates from synapse.util.clock import Clock -from tests.unittest import HomeserverTestCase +from tests.unittest import HomeserverTestCase, override_config class TestFixupMaxDepthCapBgUpdate(HomeserverTestCase): @@ -287,3 +293,163 @@ def test_batching(self) -> None: self.assertTrue(self._get_recheck("$redact5:test")) self.assertFalse(self._get_recheck("$redact6:test")) self.assertTrue(self._get_recheck("$redact7:test")) + + +class TestResignEventsBgUpdate(HomeserverTestCase): + """Test the background update that re-signs events.""" + + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + self.updates: BackgroundUpdater = self.hs.get_datastores().main.db_pool.updates + self.store = self.hs.get_datastores().main + self.db_pool = self.store.db_pool + + self.room_id = "!testroom:example.com" + + @override_config({"caches": {"global_factor": 1}, "event_cache_size": "999"}) + def test_events_are_resigned_after_bg_update_runs(self) -> None: + """Test that the background update correctly re-signs existing events with the + new key""" + + # Ensure all background updates have finished running + self.wait_for_background_updates() + + # Set up a room with a local and remote user in it. + self.register_user("user", "pass") + token = self.login("user", "pass") + + # Create new room + room_id = self.helper.create_room_as( + "user", room_version=RoomVersions.V12.identifier, tok=token + ) + + # Send a message + body = self.helper.send(room_id, body="Test", tok=token) + + old_event = self.get_success(self.store.get_event(body["event_id"])) + old_key_id = f"{self.hs.signing_key.alg}:{self.hs.signing_key.version}" + + # Ensure the message event is in the cache so that we test the cache is + # invalidated properly + res = self.store._get_event_cache.get_local((old_event.event_id,)) + self.assertEqual(res.event, old_event, "Event not cached as expected.") # type: ignore + + # Ensure message event is signed with original signing key + self.assertIn( + old_key_id, old_event.signatures[self.hs.config.server.server_name] + ) + + # Generate a new signing key + self.hs.signing_key = signedjson.key.generate_signing_key("new-test-key") + + # Reinsert the background update as it was already run at the start of + # the test. + self.get_success( + self.db_pool.simple_insert( + table="background_updates", + values={ + "update_name": "event_resign", + "progress_json": "{}", + }, + ) + ) + self.updates.start_doing_background_updates() + # Ensure the background updates have finished running + self.wait_for_background_updates() + + # Get the event from the database again + new_event = self.get_success(self.store.get_event(body["event_id"])) + new_key_id = f"{self.hs.signing_key.alg}:{self.hs.signing_key.version}" + + # Ensure message event is signed with new signing key, and not with the original + # signing key + self.assertNotIn( + old_key_id, new_event.signatures[self.hs.config.server.server_name] + ) + self.assertIn( + new_key_id, new_event.signatures[self.hs.config.server.server_name] + ) + + @override_config({"caches": {"global_factor": 1}, "event_cache_size": "999"}) + def test_old_key_filter(self) -> None: + """Test that old_key parameter causes only events whose signature + verifies against the provided key to be re-signed.""" + + self.wait_for_background_updates() + + self.register_user("user2", "pass") + token = self.login("user2", "pass") + + room_id = self.helper.create_room_as( + "user2", room_version=RoomVersions.V12.identifier, tok=token + ) + body = self.helper.send(room_id, body="Test old_key", tok=token) + + old_signing_key = self.hs.signing_key + old_key_id = f"{old_signing_key.alg}:{old_signing_key.version}" + old_verify_key = signedjson.key.get_verify_key(old_signing_key) + old_key_param = ( + f"{old_verify_key.alg}:{old_verify_key.version} " + f"{signedjson.key.encode_verify_key_base64(old_verify_key)}" + ) + + # Generate a new signing key + self.hs.signing_key = signedjson.key.generate_signing_key("new-test-key-2") + + # Generate a different key but reuse the same key ID/version, to + # ensure we're filtering on the actual public key, not just the ID. + wrong_key = signedjson.key.generate_signing_key(old_signing_key.version) + wrong_verify_key = signedjson.key.get_verify_key(wrong_key) + wrong_key_param = ( + f"{old_verify_key.alg}:{old_verify_key.version} " + f"{signedjson.key.encode_verify_key_base64(wrong_verify_key)}" + ) + + # Insert BG update with old_key filter pointing to a WRONG key + self.get_success( + self.db_pool.simple_insert( + table="background_updates", + values={ + "update_name": "event_resign", + "progress_json": json.dumps({"old_key": wrong_key_param}), + }, + ) + ) + self.updates.start_doing_background_updates() + self.wait_for_background_updates() + + # Event should NOT have been re-signed (wrong key) + event_after = self.get_success(self.store.get_event(body["event_id"])) + self.assertIn( + old_key_id, event_after.signatures[self.hs.config.server.server_name] + ) + + # Now insert BG update with the CORRECT old key + self.get_success( + self.db_pool.simple_insert( + table="background_updates", + values={ + "update_name": "event_resign", + "progress_json": json.dumps({"old_key": old_key_param}), + }, + ) + ) + self.updates.start_doing_background_updates() + self.wait_for_background_updates() + + # Event should now be re-signed + new_event = self.get_success(self.store.get_event(body["event_id"])) + new_key_id = f"{self.hs.signing_key.alg}:{self.hs.signing_key.version}" + self.assertNotIn( + old_key_id, new_event.signatures[self.hs.config.server.server_name] + ) + self.assertIn( + new_key_id, new_event.signatures[self.hs.config.server.server_name] + ) diff --git a/tests/storage/test_msc4242_state_dag.py b/tests/storage/test_msc4242_state_dag.py new file mode 100644 index 00000000000..8775e5c8eb7 --- /dev/null +++ b/tests/storage/test_msc4242_state_dag.py @@ -0,0 +1,371 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from typing import Iterable +from unittest.mock import Mock + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.api.constants import EventTypes +from synapse.api.errors import SynapseError +from synapse.api.room_versions import RoomVersions +from synapse.events import FrozenEventVMSC4242, make_event_from_dict +from synapse.events.snapshot import EventContext +from synapse.rest.client import room +from synapse.server import HomeServer +from synapse.util.clock import Clock + +from tests.unittest import HomeserverTestCase, override_config + + +class MSC4242StateDagsTests(HomeserverTestCase): + user_id = "@user1:server" + servlets = [room.register_servlets] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + hs = self.setup_test_homeserver("server") + return hs + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.room_id = self.helper.create_room_as( + self.user_id, + room_version=RoomVersions.MSC4242v12.identifier, + ) + + self.store = hs.get_datastores().main + self._storage_controllers = self.hs.get_storage_controllers() + + def _get_prev_state_events(self, event_id: str) -> list[str]: + ev = self.helper.get_event(self.room_id, event_id) + prev_state_events: list[str] | None = ev.get("prev_state_events", None) + assert prev_state_events is not None + return prev_state_events + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_forward_extremities_are_calculated(self) -> None: + """ + Check that forward extremities are set as prev_state_events and that they don't change + for non-state events. + """ + # they don't change for messages + first_event_id = self.helper.send(self.room_id, body="test1")["event_id"] + first_prev_state_events = self._get_prev_state_events(first_event_id) + assert len(first_prev_state_events) == 1 + second_id = self.helper.send(self.room_id, body="test2")["event_id"] + second_prev_state_events = self._get_prev_state_events(second_id) + assert len(second_prev_state_events) == 1 + self.assertIncludes( + set(first_prev_state_events), set(second_prev_state_events), exact=True + ) + + # send an auth event, which should change the prev_state_events on *subsequent* events + join_rule_state_event_id = self.helper.send_state( + self.room_id, + EventTypes.JoinRules, + { + "join_rule": "knock", + }, + tok="nope", + )["event_id"] + join_rule_prev_state_event_ids = self._get_prev_state_events( + join_rule_state_event_id + ) + self.assertIncludes( + set(second_prev_state_events), + set(join_rule_prev_state_event_ids), + exact=True, + ) + + # prev_state_events should always point to the join rule now + third_event_id = self.helper.send(self.room_id, body="test3")["event_id"] + third_prev_state_events = self._get_prev_state_events(third_event_id) + self.assertIncludes( + set(third_prev_state_events), {join_rule_state_event_id}, exact=True + ) + # and non-auth state should also update prev_state_events + name_state_event_id = self.helper.send_state( + self.room_id, + EventTypes.Name, + { + "name": "State DAGs!", + }, + tok="nope", + )["event_id"] + name_prev_state_event_ids = self._get_prev_state_events(name_state_event_id) + self.assertIncludes( + set(name_prev_state_event_ids), {join_rule_state_event_id}, exact=True + ) + fourth_event_id = self.helper.send(self.room_id, body="test4")["event_id"] + fourth_prev_state_events = self._get_prev_state_events(fourth_event_id) + self.assertIncludes( + set(fourth_prev_state_events), {name_state_event_id}, exact=True + ) + + +class MSC4242EventPersistenceStateDagsStoreTestCase(HomeserverTestCase): + servlets = [ + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + persistence = hs.get_storage_controllers().persistence + assert persistence is not None + self.persistence = persistence + self.room_id = "!foo:bar" + self.seen_event_ids: set[str] = set() + self.persistence.main_store = Mock(spec=["have_seen_events"]) + self.persistence.main_store.have_seen_events.side_effect = ( + self._have_seen_events + ) + self.rejected_event_ids_and_their_prevs: set[str] = set() + self.persistence.persist_events_store = Mock( + spec=["_get_prevs_before_rejected"] + ) + self.persistence.persist_events_store._get_prevs_before_rejected.side_effect = ( + self._get_prevs_before_rejected + ) + + async def _have_seen_events( + self, room_id: str, event_ids: Iterable[str] + ) -> set[str]: + unknown_events = set(event_ids) + return self.seen_event_ids.intersection(unknown_events) + + async def _get_prevs_before_rejected( + self, event_ids: Iterable[str], include_soft_failed: bool = True + ) -> set[str]: + return self.rejected_event_ids_and_their_prevs + + def _make_event( + self, + id: str, + prev_state_events: list[str], + rejected: bool = False, + ) -> tuple[FrozenEventVMSC4242, EventContext]: + ev = make_event_from_dict( + { + "prev_state_events": prev_state_events, + "content": { + "membership": "join", + }, + "sender": "@unimportant:info", + "state_key": "@unimportant:info", + "type": "m.room.member", + "room_id": self.room_id, + }, + room_version=RoomVersions.MSC4242v12, + ) + assert isinstance(ev, FrozenEventVMSC4242) + ev._event_id = id + ctx = Mock() + ctx.rejected = rejected + return ev, ctx + + def _test( + self, + current_fwds: list[str], + new_events: list[tuple[FrozenEventVMSC4242, EventContext]], + want_new_extrems: set[str], + want_raises: bool = False, + ) -> None: + """ + Tests the logic of _calculate_new_state_dag_extremities. + + Tests that the new extremities calculated as a result of processing current_fwds and new_events + matches want_new_extrems or raises if want_raises is True. + """ + coroutine = self.persistence._calculate_new_state_dag_extremities( + self.room_id, + frozenset(current_fwds), + new_events, + ) + if want_raises: + f = self.get_failure(coroutine, SynapseError) + assert f is not None + return + + new_extrems = set(self.get_success(coroutine)) + self.assertIncludes( + new_extrems, + set(want_new_extrems), + exact=True, + message=f"want_new_extrems={want_new_extrems} got={new_extrems}", + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_calculate_new_state_dag_extremities_simple(self) -> None: + # Simple linear chain + self._test( + current_fwds=[], + new_events=[ + self._make_event("$1", []), + self._make_event("$2", ["$1"]), + self._make_event("$3", ["$2"]), + self._make_event("$4", ["$3"]), + ], + want_new_extrems={"$4"}, + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_calculate_new_state_dag_extremities_fork(self) -> None: + # Simple fork so we end up with two forward extrems + self._test( + current_fwds=[], + new_events=[ + self._make_event("$1", []), + self._make_event("$2", ["$1"]), + self._make_event("$3", ["$2"]), + self._make_event("$4", ["$2"]), + ], + want_new_extrems={"$3", "$4"}, + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_calculate_new_state_dag_extremities_merge(self) -> None: + # Simple fork so we end up with two forward extrems + self._test( + current_fwds=[], + new_events=[ + self._make_event("$1", []), + self._make_event("$2", ["$1"]), + self._make_event("$3", ["$1"]), + self._make_event("$4", ["$2", "$3"]), + ], + want_new_extrems={"$4"}, + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_calculate_new_state_dag_extremities_fork_on_existing(self) -> None: + # Fork where we are adding to older events + self.seen_event_ids = {"$1", "$2", "$3"} + self._test( + current_fwds=["$3"], + new_events=[ + self._make_event("$4", ["$3"]), # append to the forward extrem + self._make_event("$5", ["$1"]), # append to the root + ], + want_new_extrems={"$4", "$5"}, + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_calculate_new_state_dag_extremities_merge_on_existing(self) -> None: + # Merge where we are merging to older events + self.seen_event_ids = {"$1", "$2", "$3"} + self._test( + current_fwds=["$3"], + new_events=[ + self._make_event("$4", ["$3", "$2"]), + ], + want_new_extrems={"$4"}, + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_calculate_new_state_dag_extremities_merge_on_not_current(self) -> None: + # Merge where we are merging to older events + self.seen_event_ids = {"$1", "$2", "$3"} + self._test( + current_fwds=["$3"], + new_events=[ + self._make_event("$4", ["$1", "$2"]), + ], + want_new_extrems={"$3", "$4"}, + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_calculate_new_state_dag_extremities_append_with_rejected(self) -> None: + # rejected events cannot be forward extremities + self.seen_event_ids = {"$1", "$2", "$3"} + self._test( + current_fwds=["$3"], + new_events=[ + self._make_event("$4", ["$3"], rejected=True), + ], + want_new_extrems={"$3"}, + ) + + self._test( + current_fwds=["$3"], + new_events=[ + self._make_event("$4", ["$3"], rejected=True), + self._make_event("$5", ["$4"], rejected=True), + ], + want_new_extrems={"$3"}, + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_calculate_new_state_dag_extremities_append_with_rejected_in_chain( + self, + ) -> None: + # rejected events cannot be forward extremities, but events that come after them can. + # this shouldn't cause multiple forward extremities. + self.seen_event_ids = {"$1", "$2", "$3"} + self.rejected_event_ids_and_their_prevs = {"$4", "$3"} + self._test( + current_fwds=["$3"], + new_events=[ + self._make_event("$4", ["$3"], rejected=True), + self._make_event("$5", ["$4"]), + ], + want_new_extrems={"$5"}, + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_calculate_new_state_dag_extremities_missing_prevs_raises(self) -> None: + self._test( + current_fwds=[], + new_events=[ + self._make_event("$1", []), + self._make_event("$2", ["$1"]), + self._make_event("$3", ["$unknown"]), + self._make_event("$4", ["$3"]), + ], + want_new_extrems={"$4"}, + want_raises=True, + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_calculate_new_state_dag_extremities_complex(self) -> None: + """ + 1 + | \ + 2 4 + | + 3 + + Exists already, then becomes... + + 1______ + | \\ | + 2 4 5R + | | | + 3--7 6R + | \\ / \ + 10R 8 9 + + """ + # Merge where we are merging to older events + self.seen_event_ids = {"$1", "$2", "$3", "$4"} + self.rejected_event_ids_and_their_prevs = {"$1", "$5", "$6", "$3", "$10"} + self._test( + current_fwds=["$3", "$4"], + new_events=[ + self._make_event("$5", ["$1"], rejected=True), + self._make_event("$6", ["$5"], rejected=True), + self._make_event("$7", ["$4", "$3"]), + self._make_event("$8", ["$6", "$7"]), + self._make_event("$9", ["$6"]), + self._make_event("$10", ["$3"], rejected=True), + ], + want_new_extrems={"$8", "$9"}, + ) diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index c82ccf1600b..c3462457060 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -152,7 +152,7 @@ def test_redact(self) -> None: self.assertObjectHasAttributes( { "type": EventTypes.Message, - "user_id": self.u_alice.to_string(), + "sender": self.u_alice.to_string(), "content": {"body": "t", "msgtype": "message"}, }, event, @@ -173,7 +173,7 @@ def test_redact(self) -> None: self.assertObjectHasAttributes( { "type": EventTypes.Message, - "user_id": self.u_alice.to_string(), + "sender": self.u_alice.to_string(), "content": {}, }, event, @@ -191,7 +191,7 @@ def test_redact_join(self) -> None: self.assertObjectHasAttributes( { "type": EventTypes.Member, - "user_id": self.u_bob.to_string(), + "sender": self.u_bob.to_string(), "content": {"membership": Membership.JOIN, "blue": "red"}, }, event, @@ -212,7 +212,7 @@ def test_redact_join(self) -> None: self.assertObjectHasAttributes( { "type": EventTypes.Member, - "user_id": self.u_bob.to_string(), + "sender": self.u_bob.to_string(), "content": {"membership": Membership.JOIN}, }, event, @@ -232,6 +232,7 @@ async def build( prev_event_ids: list[str], auth_event_ids: list[str] | None, depth: int | None = None, + prev_state_events: list[str] | None = None, ) -> EventBase: built_event = await self._base_builder.build( prev_event_ids=prev_event_ids, auth_event_ids=auth_event_ids @@ -328,7 +329,7 @@ def test_redact_censor(self) -> None: self.assertObjectHasAttributes( { "type": EventTypes.Message, - "user_id": self.u_alice.to_string(), + "sender": self.u_alice.to_string(), "content": {"body": "t", "msgtype": "message"}, }, event, @@ -347,7 +348,7 @@ def test_redact_censor(self) -> None: self.assertObjectHasAttributes( { "type": EventTypes.Message, - "user_id": self.u_alice.to_string(), + "sender": self.u_alice.to_string(), "content": {}, }, event, diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index f8c5260fa2e..b4fb5d56288 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -23,9 +23,12 @@ from synapse.api.room_versions import RoomVersions from synapse.server import HomeServer +from synapse.storage.databases.main.room import _BackgroundUpdates +from synapse.storage.types import Cursor from synapse.types import RoomAlias, RoomID, UserID from synapse.util.clock import Clock +from tests.rest.admin.test_media import _AdminMediaTests from tests.unittest import HomeserverTestCase @@ -67,3 +70,89 @@ def test_get_room_with_stats_unknown_room(self) -> None: self.assertIsNone( self.get_success(self.store.get_room_with_stats("!uknown:test")) ) + + +class FlagExistingQuarantinedMediaBackgroundUpdatesTestCase(_AdminMediaTests): + """ + Test the `flag_existing_quarantined_media` background update. + """ + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + + def test_populates_quarantined_only(self) -> None: + self.register_user("admin", "pass", admin=True) + admin_user_tok = self.login("admin", "pass") + + # Upload two distinct media items so we can quarantine one. If they shared content, + # then the quarantine-by-hash code would hit both. + _unaffected_media_upload_response = self.helper.upload_media( + b"first content", tok=admin_user_tok, expect_code=200 + ) + # Upload the media we're going to quarantine + media_upload_response = self.helper.upload_media( + b"second content", tok=admin_user_tok, expect_code=200 + ) + # Extract media ID from the response + quarantined_media_origin_and_media_id = media_upload_response["content_uri"][ + 6: + ] # cut off 'mxc://' + quarantined_media_origin, quarantined_media_id = ( + quarantined_media_origin_and_media_id.split("/") + ) + + # Ideally we'd also upload remote media to ensure that `remote_media_cache` gets picked up, but that's + # a little tricky to set up in the test here. We hope that local and remote media + # are treated similarly during the background update. + + # Quarantine the media like an admin would. Because the quarantine API also inserts + # a record into the database for us, we'll clear out the `quarantined_media_changes` + # table before running the background update. This will simulate already-quarantined + # media being in the database prior to the background update. + channel = self.make_request( + "POST", + "/_synapse/admin/v1/media/quarantine/%s/%s" + % ( + quarantined_media_origin, + quarantined_media_id, + ), + access_token=admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + + # Do that table clear we mentioned above + def _wipe_table(txn: Cursor) -> None: + txn.execute("DELETE FROM quarantined_media_changes") + + self.get_success( + self.store.db_pool.runInteraction( + "test_populates_quarantined_only._wipe_table", _wipe_table + ) + ) + + # Insert and run the background update + self.get_success( + self.store.db_pool.simple_insert( + table="background_updates", + values={ + "update_name": _BackgroundUpdates.FLAG_EXISTING_QUARANTINED_MEDIA, + "progress_json": "{}", + }, + ) + ) + self.store.db_pool.updates._all_done = False + self.wait_for_background_updates() + + # Check that the changes table is now populated, and has exactly 1 quarantined + # media object in it (the one we quarantined). + changes: list[tuple[str | None, str, bool]] = self.get_success( + self.store.db_pool.simple_select_list( + "quarantined_media_changes", + None, + retcols=("origin", "media_id", "quarantined"), + ) + ) + self.assertEqual(len(changes), 1) + self.assertEqual(changes[0][0], None) # origin (local media) + self.assertEqual(changes[0][1], quarantined_media_id) # media_id + self.assertEqual(changes[0][2], True) # quarantined flag diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index f8d64e8ce68..711c9ba4ac6 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -20,7 +20,7 @@ # # import logging -from typing import cast +from typing import Any, cast from twisted.internet.testing import MemoryReactor @@ -238,7 +238,7 @@ def test_join_locally_forgotten_room(self) -> None: creator = "@user:other" room_id = "!foo:other" room_version = RoomVersions.V10 - shared_kwargs = { + shared_kwargs: dict[str, Any] = { "room_id": room_id, "room_version": room_version.identifier, } diff --git a/tests/storage/test_sliding_sync_tables.py b/tests/storage/test_sliding_sync_tables.py index f5bbd496631..39226ff9bef 100644 --- a/tests/storage/test_sliding_sync_tables.py +++ b/tests/storage/test_sliding_sync_tables.py @@ -18,7 +18,7 @@ # # import logging -from typing import cast +from typing import Any, cast import attr from parameterized import parameterized @@ -873,7 +873,7 @@ def test_joined_room_bump_stamp_backfill(self) -> None: creator = "@user:other" room_id = "!foo:other" room_version = RoomVersions.V10 - shared_kwargs = { + shared_kwargs: dict[str, Any] = { "room_id": room_id, "room_version": room_version.identifier, } diff --git a/tests/storage/test_sticky_events.py b/tests/storage/test_sticky_events.py index 60243cb2f40..e77b362f528 100644 --- a/tests/storage/test_sticky_events.py +++ b/tests/storage/test_sticky_events.py @@ -30,6 +30,7 @@ from synapse.util.duration import Duration from tests import unittest +from tests.test_utils.event_injection import inject_event from tests.utils import USE_POSTGRES_FOR_TESTS @@ -276,3 +277,154 @@ def test_outlier_events_not_in_table(self) -> None: ) self.assertEqual(len(updates), 1) self.assertEqual(updates[0].event_id, event_non_outlier.event_id) + + def test_soft_failed_events_are_tracked(self) -> None: + """ + Tests that sticky events marked as soft_failed ARE inserted + into the sticky_events table, as their soft-failed status can be re-evaluated later, + as per MSC4354. + """ + user_id = self.register_user("testuser", "pass") + token = self.login(user_id, "pass") + room_id = self.helper.create_room_as(user_id, tok=token) + + start_id = self.store.get_max_sticky_events_stream_id() + + # Create and persist a sticky event that is soft-failed + soft_failed_sticky_event = self.get_success( + inject_event( + self.hs, + room_id=room_id, + sender=user_id, + type=EventTypes.Message, + content={"body": "spam checker spammy message", "msgtype": "m.text"}, + internal_metadata={"soft_failed": True}, + # Corresponds to StickyEvent.EVENT_FIELD_NAME + msc4354_sticky=StickyEventField( + duration_ms=Duration(minutes=1).as_millis() + ), + ) + ) + + end_id = self.store.get_max_sticky_events_stream_id() + + updates = self.get_success( + self.store.get_updated_sticky_events( + from_id=start_id, to_id=end_id, limit=10 + ) + ) + + self.assertEqual(len(updates), 1) + self.assertEqual(updates[0].event_id, soft_failed_sticky_event.event_id) + + def test_policy_server_spammy_events_are_not_tracked(self) -> None: + """ + Tests that sticky events marked as policy_server_spammy are NOT inserted + into the sticky_events table, as they are exempt from the soft-failed + re-evaluation logic. + """ + user_id = self.register_user("testuser", "pass") + token = self.login(user_id, "pass") + room_id = self.helper.create_room_as(user_id, tok=token) + + start_id = self.store.get_max_sticky_events_stream_id() + + # Create and persist a sticky event that is marked policy_server_spammy + # N.B. policy_server_spammy events are always soft-failed too + _spammy_sticky_event = self.get_success( + inject_event( + self.hs, + room_id=room_id, + sender=user_id, + type=EventTypes.Message, + content={"body": "spam checker spammy message", "msgtype": "m.text"}, + internal_metadata={"soft_failed": True, "policy_server_spammy": True}, + # Corresponds to StickyEvent.EVENT_FIELD_NAME + msc4354_sticky=StickyEventField( + duration_ms=Duration(minutes=1).as_millis() + ), + ) + ) + + # Also insert a valid sticky event as a canary for the test setup + valid_sticky_event = self.get_success( + inject_event( + self.hs, + room_id=room_id, + sender=user_id, + type=EventTypes.Message, + content={"body": "normal sticky", "msgtype": "m.text"}, + # Corresponds to StickyEvent.EVENT_FIELD_NAME + msc4354_sticky=StickyEventField( + duration_ms=Duration(minutes=1).as_millis() + ), + ) + ) + + end_id = self.store.get_max_sticky_events_stream_id() + + # Verify only the regular event was inserted + updates = self.get_success( + self.store.get_updated_sticky_events( + from_id=start_id, to_id=end_id, limit=10 + ) + ) + + self.assertEqual(len(updates), 1) + self.assertEqual(updates[0].event_id, valid_sticky_event.event_id) + + def test_spam_checker_spammy_events_are_not_tracked(self) -> None: + """ + Tests that sticky events marked as spam_checker_spammy are NOT inserted + into the sticky_events table, as they are exempt from the soft-failed + re-evaluation logic. + """ + user_id = self.register_user("testuser", "pass") + token = self.login(user_id, "pass") + room_id = self.helper.create_room_as(user_id, tok=token) + + start_id = self.store.get_max_sticky_events_stream_id() + + # Create and persist a sticky event that is marked spam_checker_spammy + # N.B. spam_checker_spammy events are always soft-failed too + _spammy_sticky_event = self.get_success( + inject_event( + self.hs, + room_id=room_id, + sender=user_id, + type=EventTypes.Message, + content={"body": "spam checker spammy message", "msgtype": "m.text"}, + internal_metadata={"soft_failed": True, "spam_checker_spammy": True}, + # Corresponds to StickyEvent.EVENT_FIELD_NAME + msc4354_sticky=StickyEventField( + duration_ms=Duration(minutes=1).as_millis() + ), + ) + ) + + # Also insert a valid sticky event as a canary for the test setup + valid_sticky_event = self.get_success( + inject_event( + self.hs, + room_id=room_id, + sender=user_id, + type=EventTypes.Message, + content={"body": "normal sticky", "msgtype": "m.text"}, + # Corresponds to StickyEvent.EVENT_FIELD_NAME + msc4354_sticky=StickyEventField( + duration_ms=Duration(minutes=1).as_millis() + ), + ) + ) + + end_id = self.store.get_max_sticky_events_stream_id() + + # Verify only the valid sticky event was inserted + updates = self.get_success( + self.store.get_updated_sticky_events( + from_id=start_id, to_id=end_id, limit=10 + ) + ) + + self.assertEqual(len(updates), 1) + self.assertEqual(updates[0].event_id, valid_sticky_event.event_id) diff --git a/tests/synapse_rust/test_room_versions.py b/tests/synapse_rust/test_room_versions.py new file mode 100644 index 00000000000..aff1ab2783e --- /dev/null +++ b/tests/synapse_rust/test_room_versions.py @@ -0,0 +1,45 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS + +from tests import unittest + + +class KnownRoomVersionsMappingTestCase(unittest.TestCase): + """Tests for the Rust-backed KNOWN_ROOM_VERSIONS mapping.""" + + def test_contains_none(self) -> None: + """Test that `None in KNOWN_ROOM_VERSIONS` returns False + rather than raising a TypeError, matching Python dict semantics.""" + self.assertFalse(None in KNOWN_ROOM_VERSIONS) + + def test_contains_non_string(self) -> None: + """Test that non-string keys return False from __contains__.""" + self.assertFalse(42 in KNOWN_ROOM_VERSIONS) # type: ignore[comparison-overlap] + self.assertFalse(3.14 in KNOWN_ROOM_VERSIONS) # type: ignore[comparison-overlap] + self.assertFalse([] in KNOWN_ROOM_VERSIONS) # type: ignore[comparison-overlap] + + def test_contains_known_version(self) -> None: + self.assertTrue("1" in KNOWN_ROOM_VERSIONS) + + def test_contains_unknown_version(self) -> None: + self.assertFalse("unknown" in KNOWN_ROOM_VERSIONS) + + def test_getitem_non_string_raises_key_error(self) -> None: + """Test that KNOWN_ROOM_VERSIONS[non_string] raises KeyError, + not TypeError, matching Python dict semantics.""" + with self.assertRaises(KeyError): + KNOWN_ROOM_VERSIONS[42] # type: ignore[index] + with self.assertRaises(KeyError): + KNOWN_ROOM_VERSIONS[None] # type: ignore[index] diff --git a/tests/test_event_auth.py b/tests/test_event_auth.py index 934a2fd3071..9258f0d4dc1 100644 --- a/tests/test_event_auth.py +++ b/tests/test_event_auth.py @@ -20,15 +20,16 @@ # import unittest +from collections import namedtuple from typing import Any, Collection, Iterable from parameterized import parameterized from synapse import event_auth -from synapse.api.constants import EventContentFields +from synapse.api.constants import EventContentFields, RejectedReason from synapse.api.errors import AuthError, SynapseError from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase, event_exists_in_state_dag, make_event_from_dict from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.types import JsonDict, get_domain_from_id @@ -374,6 +375,195 @@ def test_msc2432_alias_event(self) -> None: auth_events, ) + def test_msc4242_state_dag_rules(self) -> None: + """Tests additional rules in place for state DAG rooms. + + 1. m.room.create => if it has any prev_state_events, reject. + 2. Considering the event's prev_state_events: + i. If there are entries which do not belong in the same room, reject. + ii. If there are entries which do not have a state_key, reject. + iii. If there are entries which were themselves rejected under the checks performed on receipt of a PDU, reject. + """ + creator = "@creator:example.com" + room_version = RoomVersions.MSC4242v12 + + create_event = make_event_from_dict( + { + "type": "m.room.create", + "sender": creator, + "state_key": "", + "content": {"creator": creator}, + "prev_events": [], + "prev_state_events": [], + }, + room_version, + ) + create_event_2 = make_event_from_dict( + { + "type": "m.room.create", + "sender": creator, + "state_key": "", + "content": {"creator": creator, "another": "room"}, + "prev_events": [], + "prev_state_events": [], + }, + room_version, + ) + room_id = create_event.room_id + another_room_id = create_event_2.room_id + join_event = make_event_from_dict( + { + "room_id": room_id, + "type": "m.room.member", + "sender": creator, + "state_key": creator, + "content": {"membership": "join"}, + "prev_events": [create_event.event_id], + "prev_state_events": [create_event.event_id], + }, + room_version, + {"calculated_auth_event_ids": [create_event.event_id]}, + ) + event_in_another_room = make_event_from_dict( + { + "room_id": another_room_id, + "type": "m.room.join_rules", + "sender": creator, + "state_key": "", + "content": {"join_rule": "public"}, + "prev_events": [join_event.event_id], + "prev_state_events": [join_event.event_id], + }, + room_version, + {"calculated_auth_event_ids": [create_event.event_id, join_event.event_id]}, + ) + msg_event = make_event_from_dict( + { + "room_id": room_id, + "type": "m.room.message", + "sender": creator, + "content": {"msgtype": "m.text", "body": "I am a message"}, + "prev_events": [join_event.event_id], + "prev_state_events": [join_event.event_id], + }, + room_version, + {"calculated_auth_event_ids": [create_event.event_id, join_event.event_id]}, + ) + rejected_event = make_event_from_dict( + { + "room_id": room_id, + "type": "m.room.name", + "sender": creator, + "state_key": "", + "content": {"name": "REJECTED"}, + "prev_events": [join_event.event_id], + "prev_state_events": [join_event.event_id], + }, + room_version, + {"calculated_auth_event_ids": [create_event.event_id, join_event.event_id]}, + rejected_reason=RejectedReason.AUTH_ERROR, + ) + RejectingTestCase = namedtuple( + "RejectingTestCase", "name events_in_store test_event" + ) + rejecting_test_cases = [ + RejectingTestCase( + name="create event has prev_state_events", + events_in_store=[], + test_event=make_event_from_dict( + { + "type": "m.room.create", + "sender": creator, + "state_key": "", + "content": {"creator": creator}, + "prev_events": [], + "prev_state_events": [create_event.event_id], + }, + room_version, + {}, + ), + ), + RejectingTestCase( + name="prev_state_event belongs in a different room", + events_in_store=[create_event, join_event, event_in_another_room], + test_event=make_event_from_dict( + { + "room_id": room_id, + "type": "m.room.name", + "sender": creator, + "state_key": "", + "content": {"name": "prev_state_event is in another room"}, + "prev_events": [join_event.event_id], + "prev_state_events": [event_in_another_room.event_id], + }, + room_version, + { + "calculated_auth_event_ids": [ + create_event.event_id, + join_event.event_id, + ] + }, + ), + ), + RejectingTestCase( + name="prev_state_event is a message event", + events_in_store=[create_event, join_event, msg_event], + test_event=make_event_from_dict( + { + "room_id": room_id, + "type": "m.room.name", + "sender": creator, + "state_key": "", + "content": {"name": "prev state event is a message"}, + "prev_events": [msg_event.event_id], + "prev_state_events": [msg_event.event_id], + }, + room_version, + { + "calculated_auth_event_ids": [ + create_event.event_id, + join_event.event_id, + ] + }, + ), + ), + RejectingTestCase( + name="prev_state_event was rejected", + events_in_store=[create_event, join_event, rejected_event], + test_event=make_event_from_dict( + { + "room_id": room_id, + "type": "m.room.name", + "sender": creator, + "state_key": "", + "content": {"name": "prev state event was rejected"}, + "prev_events": [join_event.event_id], + "prev_state_events": [rejected_event.event_id], + }, + room_version, + { + "calculated_auth_event_ids": [ + create_event.event_id, + join_event.event_id, + ] + }, + ), + ), + ] + + for test_case in rejecting_test_cases: + event_store = _StubEventSourceStore() + event_store.add_events(test_case.events_in_store) + + with self.assertRaises( + AuthError, msg=f"test case {test_case.name} was not rejected" + ): + get_awaitable_result( + event_auth.check_state_independent_auth_rules( + event_store, test_case.test_event + ) + ) + @parameterized.expand([(RoomVersions.V1, True), (RoomVersions.V6, False)]) def test_notifications( self, room_version: RoomVersion, allow_modification: bool @@ -769,6 +959,105 @@ def create_event(pl_event_content: dict[str, Any]) -> EventBase: with self.assertRaises(SynapseError): event_auth._check_power_levels(event.room_version, event, {}) + def test_event_exists_in_state_dag(self) -> None: + events_that_exist_in_state_dag = [ + { + "type": "m.room.create", + "state_key": "", + "content": {}, + }, + { + "type": "m.room.join_rules", + "state_key": "", + "content": {}, + }, + { + "type": "m.room.power_levels", + "state_key": "", + "content": {}, + }, + { + "type": "m.room.server_acl", + "state_key": "", + "content": {}, + }, + { + "type": "m.room.member", + "state_key": "@alice:somewhere", + "content": {}, + }, + { + "type": "m.room.third_party_invite", + "state_key": "flibble", + "content": {}, + }, + { + "type": "m.room.create", + "state_key": " ", + "content": {}, + }, + { + "type": "m.room.join_rules", + "state_key": " ", + "content": {}, + }, + { + "type": "m.room.power_levels", + "state_key": " ", + "content": {}, + }, + { + "type": "m.room.name", + "state_key": "", + "content": {}, + }, + { + "type": "m.room.member", + "state_key": "", + "content": {}, + }, + { + "type": "m.room.member", + "state_key": "hello_world", + "content": {}, + }, + ] + events_that_dont_exist_in_state_dag = [ + { + "type": "m.room.message", + "content": {}, + }, + { + "type": "m.room.create", + "content": {}, + }, + { + "type": "m.room.join_rules", + "content": {}, + }, + { + "type": "m.room.power_levels", + "content": {}, + }, + ] + + def check_events(events: list[dict], should_exist: bool) -> None: + for ev in events: + base = { + "room_id": TEST_ROOM_ID, + "sender": "@test:test.com", + "signatures": {"test.com": {"ed25519:0": "some9signature"}}, + } + base.update(ev) + event = make_event_from_dict(base, RoomVersions.V10) + got = event_exists_in_state_dag(event) + self.assertEqual( + got, should_exist, f"{ev} should_exist={should_exist} but got {got}" + ) + + check_events(events_that_exist_in_state_dag, should_exist=True) + check_events(events_that_dont_exist_in_state_dag, should_exist=False) + # helpers for making events TEST_DOMAIN = "example.com" diff --git a/tests/test_utils/event_injection.py b/tests/test_utils/event_injection.py index a90fc5884d8..36f6baa1239 100644 --- a/tests/test_utils/event_injection.py +++ b/tests/test_utils/event_injection.py @@ -18,7 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # -from typing import Any +from typing import Any, Mapping import synapse.server from synapse.api.constants import EventTypes @@ -63,6 +63,8 @@ async def inject_event( hs: synapse.server.HomeServer, room_version: str | None = None, prev_event_ids: list[str] | None = None, + *, + internal_metadata: Mapping[str, Any] | None = None, **kwargs: Any, ) -> EventBase: """Inject a generic event into a room @@ -72,9 +74,12 @@ async def inject_event( room_version: the version of the room we're inserting into. if not specified, will be looked up prev_event_ids: prev_events for the event. If not specified, will be looked up + internal_metadata: Dict representing the event's internal metadata; see `EventBase.internal_metadata` kwargs: fields for the event to be created """ - event, context = await create_event(hs, room_version, prev_event_ids, **kwargs) + event, context = await create_event( + hs, room_version, prev_event_ids, internal_metadata=internal_metadata, **kwargs + ) persistence = hs.get_storage_controllers().persistence assert persistence is not None @@ -88,8 +93,11 @@ async def create_event( hs: synapse.server.HomeServer, room_version: str | None = None, prev_event_ids: list[str] | None = None, + *, + internal_metadata: Mapping[str, Any] | None = None, **kwargs: Any, ) -> tuple[EventBase, EventContext]: + internal_metadata = internal_metadata or {} if room_version is None: room_version = await hs.get_datastores().main.get_room_version_id( kwargs["room_id"] @@ -106,11 +114,13 @@ async def create_event( ) # Copy over writable internal_metadata, if set - # Dev note: This isn't everything that's writable. `for k,v` doesn't work here :( - if kwargs.get("internal_metadata", {}).get("soft_failed", False): - event.internal_metadata.soft_failed = True - if kwargs.get("internal_metadata", {}).get("policy_server_spammy", False): - event.internal_metadata.policy_server_spammy = True + if internal_metadata: + for key, value in internal_metadata.items(): + # Note: this calls the relevant `#[setter]` function in the (Rust) event class' + # internal metadata struct. + # Will reject unknown keys with exceptions. + # This is desirable for our test suite anyway. + setattr(event.internal_metadata, key, value) context = await unpersisted_context.persist(event) diff --git a/tests/test_visibility.py b/tests/test_visibility.py index b50faa2a499..9a5efbdd399 100644 --- a/tests/test_visibility.py +++ b/tests/test_visibility.py @@ -22,7 +22,7 @@ from twisted.test.proto_helpers import MemoryReactor -from synapse.api.constants import AccountDataTypes, EventUnsignedContentFields +from synapse.api.constants import AccountDataTypes from synapse.api.room_versions import RoomVersions from synapse.events import EventBase, make_event_from_dict from synapse.events.snapshot import EventContext @@ -341,7 +341,7 @@ def test_normal_operation_as_admin(self) -> None: ) self.assertEqual( [e.event_id for e in [self.regular_event]], - [e.event_id for e in filtered_events], + [e.event.event_id for e in filtered_events], ) def test_see_soft_failed_events(self) -> None: @@ -380,7 +380,7 @@ def test_see_soft_failed_events(self) -> None: ) self.assertEqual( [e.event_id for e in [self.regular_event, self.soft_failed_event]], - [e.event_id for e in filtered_events], + [e.event.event_id for e in filtered_events], ) def test_see_policy_server_spammy_events(self) -> None: @@ -427,7 +427,7 @@ def test_see_policy_server_spammy_events(self) -> None: ) self.assertEqual( [e.event_id for e in [self.regular_event, self.spammy_event]], - [e.event_id for e in filtered_events], + [e.event.event_id for e in filtered_events], ) def test_see_soft_failed_and_policy_server_spammy_events(self) -> None: @@ -477,7 +477,7 @@ def test_see_soft_failed_and_policy_server_spammy_events(self) -> None: e.event_id for e in [self.regular_event, self.soft_failed_event, self.spammy_event] ], - [e.event_id for e in filtered_events], + [e.event.event_id for e in filtered_events], ) @@ -559,14 +559,11 @@ def test_joined_history_visibility(self) -> None: # and messages sent between the two, but not before or after. self.assertEqual( [e.event_id for e in [join_event, during_event, leave_event]], - [e.event_id for e in joiner_filtered_events], + [e.event.event_id for e in joiner_filtered_events], ) self.assertEqual( ["join", "join", "leave"], - [ - e.unsigned[EventUnsignedContentFields.MEMBERSHIP] - for e in joiner_filtered_events - ], + [e.membership for e in joiner_filtered_events], ) # The resident user should see all the events. @@ -581,14 +578,11 @@ def test_joined_history_visibility(self) -> None: after_event, ] ], - [e.event_id for e in resident_filtered_events], + [e.event.event_id for e in resident_filtered_events], ) self.assertEqual( ["join", "join", "join", "join", "join"], - [ - e.unsigned[EventUnsignedContentFields.MEMBERSHIP] - for e in resident_filtered_events - ], + [e.membership for e in resident_filtered_events], ) @@ -651,15 +645,12 @@ def test_out_of_band_invite_rejection(self) -> None: ) ) self.assertEqual( - [e.event_id for e in filtered_events], + [e.event.event_id for e in filtered_events], [e.event_id for e in [invite_event, reject_event]], ) self.assertEqual( ["invite", "leave"], - [ - e.unsigned[EventUnsignedContentFields.MEMBERSHIP] - for e in filtered_events - ], + [e.membership for e in filtered_events], ) # other users should see neither