From fa52c62a895e2c02ce9c9c9eaa061e4a559577ff Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 28 Apr 2026 11:47:16 +0100 Subject: [PATCH 01/92] 1.152.0 --- CHANGES.md | 7 ++++++- debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 384042db2fa..c3ba1242eed 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,10 +1,15 @@ -# Synapse 1.152.0rc1 (2026-04-22) +# Synapse 1.152.0 (2026-04-28) + +No significant changes since 1.152.0rc1. + +## Configuration changes needed for deployments using workers For deployments using workers, please note that this version introduces a new `quarantined_media_changes` stream writer, which may require configuration changes. Please see the [the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/develop/docs/upgrade.md#upgrading-to-v11520) for details. Without configuring this new stream writer, only the main process will be able to handle the `/media/quarantine` admin API endpoints for quarantining media. +# Synapse 1.152.0rc1 (2026-04-22) ## Features diff --git a/debian/changelog b/debian/changelog index e9a8e200e1e..ff9dfe3e13d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.152.0) stable; urgency=medium + + * New Synapse release 1.152.0. + + -- Synapse Packaging team Tue, 28 Apr 2026 11:45:01 +0100 + matrix-synapse-py3 (1.152.0~rc1) stable; urgency=medium * New Synapse release 1.152.0rc1. diff --git a/pyproject.toml b/pyproject.toml index 44b8296438c..9509609ff3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.152.0rc1" +version = "1.152.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 16863c87d56508ed163b3f6da9ba463e902b7e63 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 28 Apr 2026 13:45:53 +0100 Subject: [PATCH 02/92] Changelog tweaks --- CHANGES.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c3ba1242eed..3425bcca8c5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,13 +13,12 @@ Without configuring this new stream writer, only the main process will be able t ## Features -- Add a ["Listing quarantined media changes" Admin API](https://element-hq.github.io/synapse/latest/admin_api/media_admin_api.html#listing-quarantined-media-changes) for retrieving a paginated record of when media became (un)quarantined. ([\#19558](https://github.com/element-hq/synapse/issues/19558), [\#19677](https://github.com/element-hq/synapse/issues/19677)) +- Add a ["Listing quarantined media changes" Admin API](https://element-hq.github.io/synapse/latest/admin_api/media_admin_api.html#listing-quarantined-media-changes) for retrieving a paginated record of when media became (un)quarantined. ([\#19558](https://github.com/element-hq/synapse/issues/19558), [\#19677](https://github.com/element-hq/synapse/issues/19677), [\#19694](https://github.com/element-hq/synapse/issues/19694)) - Advertise [MSC4445](https://github.com/matrix-org/matrix-spec-proposals/pull/4445) sync timeline order in `unstable_features`. ([\#19642](https://github.com/element-hq/synapse/issues/19642)) - Report the Rust compiler version used in the Prometheus metrics. Contributed by Noah Markert. ([\#19643](https://github.com/element-hq/synapse/issues/19643)) - Passthrough 'article' and 'profile' OpenGraph metadata on URL preview requests. ([\#19659](https://github.com/element-hq/synapse/issues/19659)) - Add a way to re-sign local events with a new signing key. ([\#19668](https://github.com/element-hq/synapse/issues/19668)) - Support [MSC4450: Identity Provider selection for User-Interactive Authentication with Legacy Single Sign-On](https://github.com/matrix-org/matrix-spec-proposals/pull/4450). ([\#19693](https://github.com/element-hq/synapse/issues/19693)) -- Add a ["Listing quarantined media changes" Admin API](https://element-hq.github.io/synapse/latest/admin_api/media_admin_api.html#listing-quarantined-media-changes) for retrieving a paginated record of when media became (un)quarantined. ([\#19694](https://github.com/element-hq/synapse/issues/19694)) - Add experimental support for [MSC4242](https://github.com/matrix-org/matrix-spec-proposals/pull/4242): State DAGs. Excludes federation support. ([\#19424](https://github.com/element-hq/synapse/issues/19424)) - Adds [Admin API](https://element-hq.github.io/synapse/latest/usage/administration/admin_api/index.html) endpoints to list, fetch and delete user reports. ([\#19657](https://github.com/element-hq/synapse/issues/19657)) @@ -35,7 +34,7 @@ Without configuring this new stream writer, only the main process will be able t - Include a workaround for running the unit tests with SQLite under recent versions of MacOS. ([\#19615](https://github.com/element-hq/synapse/issues/19615)) - Fix Docker image link typo in worker docs. ([\#19645](https://github.com/element-hq/synapse/issues/19645)) -- Update developer stream docs for creating a new stream to point out `_setup_sequence(...)` in `portdb`. ([\#19675](https://github.com/element-hq/synapse/issues/19675)) +- Update the developer stream docs for creating a new stream to point out `_setup_sequence(...)` in `portdb`. ([\#19675](https://github.com/element-hq/synapse/issues/19675)) - Update the developer stream docs for creating a new stream to highlight places that require documentation updates. ([\#19696](https://github.com/element-hq/synapse/issues/19696)) ## Internal Changes From 76b4fdceed0739d83ac79588416dc88f25d8d14e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2026 17:46:03 +0100 Subject: [PATCH 03/92] Add a canonical JSON impl (#19739) This comes from https://github.com/erikjohnston/rust-signed-json/blob/main/src/json.rs. We need to be able to serialise canonical JSON in Rust to be able to calculate event IDs once we port the event class to Rust. We could instead make the above a properly published crate, but feels easier to pull it into Synapse utils. --- Cargo.lock | 16 + changelog.d/19739.misc | 1 + rust/Cargo.toml | 3 +- rust/src/canonical_json.rs | 841 +++++++++++++++++++++++++++++++++++++ rust/src/lib.rs | 1 + 5 files changed, 861 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19739.misc create mode 100644 rust/src/canonical_json.rs diff --git a/Cargo.lock b/Cargo.lock index c7f5e0ddbf4..832d5129fe2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -164,6 +164,12 @@ dependencies = [ "syn", ] +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "equivalent" version = "1.0.2" @@ -662,6 +668,15 @@ dependencies = [ "serde", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -1336,6 +1351,7 @@ dependencies = [ "http", "http-body-util", "icu_segmenter", + "itertools", "lazy_static", "log", "mime", diff --git a/changelog.d/19739.misc b/changelog.d/19739.misc new file mode 100644 index 00000000000..24562b24fcd --- /dev/null +++ b/changelog.d/19739.misc @@ -0,0 +1 @@ +Add a Rust canonical JSON serializer. diff --git a/rust/Cargo.toml b/rust/Cargo.toml index bca2f6ed70f..e6b378a092f 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -44,7 +44,7 @@ pythonize = "0.27.0" regex = "1.6.0" sha2 = "0.10.8" serde = { version = "1.0.144", features = ["derive"] } -serde_json = "1.0.85" +serde_json = { version = "1.0.85", features = ["raw_value"] } ulid = "1.1.2" icu_segmenter = "2.0.0" reqwest = { version = "0.12.15", default-features = false, features = [ @@ -56,6 +56,7 @@ http-body-util = "0.1.3" futures = "0.3.31" tokio = { version = "1.44.2", features = ["rt", "rt-multi-thread"] } once_cell = "1.18.0" +itertools = "0.14.0" [features] extension-module = ["pyo3/extension-module"] diff --git a/rust/src/canonical_json.rs b/rust/src/canonical_json.rs new file mode 100644 index 00000000000..ff1fcd3ee41 --- /dev/null +++ b/rust/src/canonical_json.rs @@ -0,0 +1,841 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + * Originally licensed under the Apache License, Version 2.0: + * . + * + * [This file includes modifications made by Element Creations Ltd] + */ + +//! Serialize a Rust data structure into canonical JSON data. +//! +//! See the [Canonical +//! JSON](https://matrix.org/docs/spec/appendices#canonical-json) docs for more +//! information. + +use std::{ + collections::BTreeMap, + convert::TryFrom, + io::{self, Write}, +}; + +use serde::ser::SerializeMap; +use serde::{ + ser::{Error as _, SerializeStruct}, + Serialize, +}; +use serde_json::{ + ser::{Formatter, Serializer}, + value::RawValue, + Value, +}; + +/// The minimum integer that can be used in canonical JSON. +pub const MIN_VALID_INTEGER: i64 = -(2i64.pow(53)) + 1; + +/// The maximum integer that can be used in canonical JSON. +pub const MAX_VALID_INTEGER: i64 = (2i64.pow(53)) - 1; + +/// Options to control how strict JSON canonicalization is. +#[derive(Clone, Debug)] +pub struct CanonicalizationOptions { + /// Configure the serializer to strictly enforce the canonical JSON allowable number range. + /// Allows JSON for room versions v5 or less when `false`. + enforce_int_range: bool, +} + +impl CanonicalizationOptions { + /// Creates an instance of [CanonicalizationOptions] with permissive JSON enforcement settings. + pub fn relaxed() -> Self { + Self { + enforce_int_range: false, + } + } + + /// Creates an instance of [CanonicalizationOptions] with strict JSON enforcement settings. + pub fn strict() -> Self { + Self { + enforce_int_range: true, + } + } +} + +/// Serialize the given data structure as a canonical JSON byte vector. +/// +/// See the [Canonical +/// JSON](https://matrix.org/docs/spec/appendices#canonical-json) docs for more +/// information. +/// +/// Note: serializing [`RawValue`] is not supported, as it may contain JSON that +/// is not canonical. +/// +/// # Errors +/// +/// Serialization can fail if `T`'s implementation of `Serialize` decides to +/// fail, if `T` contains a map with non-string keys, or if `T` contains numbers +/// that are not integers in the range `[-2**53 + 1, 2**53 - 1]`. +pub fn to_vec_canonical( + value: &T, + options: CanonicalizationOptions, +) -> Result, serde_json::Error> +where + T: Serialize + ?Sized, +{ + let mut vec = Vec::new(); + let mut ser = CanonicalSerializer::new(&mut vec, options); + value.serialize(&mut ser)?; + + Ok(vec) +} + +/// Serialize the given data structure as a canonical JSON string. +/// +/// See the [Canonical +/// JSON](https://matrix.org/docs/spec/appendices#canonical-json) docs for more +/// information. +/// +/// Note: serializing [`RawValue`] is not supported, as it may contain JSON that +/// is not canonical. +/// +/// # Errors +/// +/// Serialization can fail if `T`'s implementation of `Serialize` decides to +/// fail, if `T` contains a map with non-string keys, or if `T` contains numbers +/// that are not integers in the range `[-2**53 + 1, 2**53 - 1]`. +pub fn to_string_canonical( + value: &T, + options: CanonicalizationOptions, +) -> Result +where + T: Serialize + ?Sized, +{ + let vec = to_vec_canonical(value, options)?; + + // We'll always get valid UTF-8 out + let json_string = String::from_utf8(vec).expect("valid utf8"); + + Ok(json_string) +} + +/// A helper function that asserts that an integer is in the valid range. +fn assert_integer_in_range(v: I) -> Result<(), serde_json::Error> +where + i64: TryFrom, +{ + let res = i64::try_from(v); + match res { + Ok(MIN_VALID_INTEGER..=MAX_VALID_INTEGER) => Ok(()), + Ok(_) | Err(_) => Err(serde_json::Error::custom("integer out of range")), + } +} + +/// A JSON formatter that ensures all strings are encoded as per the [Canonical +/// JSON](https://matrix.org/docs/spec/appendices#canonical-json) spec. +pub struct CanonicalFormatter; + +impl Formatter for CanonicalFormatter { + fn write_string_fragment(&mut self, writer: &mut W, fragment: &str) -> io::Result<()> + where + W: ?Sized + io::Write, + { + // `fragment` only contains characters that are not escaped, and don't + // need to be escaped, so they can be written directly to the writer. + writer.write_all(fragment.as_bytes()) + } + + fn write_char_escape( + &mut self, + writer: &mut W, + char_escape: serde_json::ser::CharEscape, + ) -> io::Result<()> + where + W: ?Sized + io::Write, + { + use serde_json::ser::CharEscape::*; + + let s = match char_escape { + Quote => b"\\\"" as &[u8], + ReverseSolidus => b"\\\\", + Solidus => b"/", // Note: this doesn't need to be escaped (and appears unused in serde_json). + Backspace => b"\\b", + FormFeed => b"\\f", + LineFeed => b"\\n", + CarriageReturn => b"\\r", + Tab => b"\\t", + AsciiControl(byte) => { + static HEX_DIGITS: [u8; 16] = *b"0123456789abcdef"; + let bytes = &[ + b'\\', + b'u', + b'0', + b'0', + HEX_DIGITS[(byte >> 4) as usize], + HEX_DIGITS[(byte & 0xF) as usize], + ]; + return writer.write_all(bytes); + } + }; + + writer.write_all(s) + } +} + +/// A JSON serializer that outputs [Canonical +/// JSON](https://matrix.org/docs/spec/appendices#canonical-json). +pub struct CanonicalSerializer { + inner: Serializer, + options: CanonicalizationOptions, +} + +impl CanonicalSerializer +where + W: Write, +{ + /// Create a new serializer that writes the canonical JSON bytes to the + /// given writer. + pub fn new(writer: W, options: CanonicalizationOptions) -> Self { + Self { + inner: Serializer::with_formatter(writer, CanonicalFormatter), + options, + } + } +} + +// We implement the serializer by proxying all calls to the standard +// `serde_json` serializer, except where we a) buffer up maps and structs so that we can +// sort them, and b) ensure that all numbers are integers in the valid range. +impl<'a, W> serde::Serializer for &'a mut CanonicalSerializer +where + W: Write, +{ + type Ok = <&'a mut Serializer as serde::Serializer>::Ok; + + type Error = <&'a mut Serializer as serde::Serializer>::Error; + + type SerializeSeq = + <&'a mut Serializer as serde::Serializer>::SerializeSeq; + + type SerializeTuple = + <&'a mut Serializer as serde::Serializer>::SerializeTuple; + + type SerializeTupleStruct = + <&'a mut Serializer as serde::Serializer>::SerializeTupleStruct; + + type SerializeTupleVariant = + <&'a mut Serializer as serde::Serializer>::SerializeTupleVariant; + + type SerializeMap = CanonicalSerializeMap<'a, W>; + + type SerializeStruct = CanonicalSerializeMap<'a, W>; + + type SerializeStructVariant = + <&'a mut Serializer as serde::Serializer>::SerializeStructVariant; + + fn serialize_bool(self, v: bool) -> Result { + self.inner.serialize_bool(v) + } + + fn serialize_i8(self, v: i8) -> Result { + assert_integer_in_range(v)?; + + self.inner.serialize_i8(v) + } + + fn serialize_i16(self, v: i16) -> Result { + assert_integer_in_range(v)?; + + self.inner.serialize_i16(v) + } + + fn serialize_i32(self, v: i32) -> Result { + assert_integer_in_range(v)?; + + self.inner.serialize_i32(v) + } + + fn serialize_i64(self, v: i64) -> Result { + if self.options.enforce_int_range { + assert_integer_in_range(v)?; + } + + self.inner.serialize_i64(v) + } + + fn serialize_i128(self, v: i128) -> Result { + if self.options.enforce_int_range { + assert_integer_in_range(v)?; + } + + self.inner.serialize_i128(v) + } + + fn serialize_u8(self, v: u8) -> Result { + assert_integer_in_range(v)?; + + self.inner.serialize_u8(v) + } + + fn serialize_u16(self, v: u16) -> Result { + self.inner.serialize_u16(v) + } + + fn serialize_u32(self, v: u32) -> Result { + assert_integer_in_range(v)?; + + self.inner.serialize_u32(v) + } + + fn serialize_u64(self, v: u64) -> Result { + if self.options.enforce_int_range { + assert_integer_in_range(v)?; + } + + self.inner.serialize_u64(v) + } + + fn serialize_u128(self, v: u128) -> Result { + if self.options.enforce_int_range { + assert_integer_in_range(v)?; + } + + self.inner.serialize_u128(v) + } + + fn serialize_f32(self, _: f32) -> Result { + Err(serde_json::Error::custom( + "non-integer numbers are not allowed", + )) + } + + fn serialize_f64(self, _: f64) -> Result { + Err(serde_json::Error::custom( + "non-integer numbers are not allowed", + )) + } + + fn serialize_char(self, v: char) -> Result { + self.inner.serialize_char(v) + } + + fn serialize_str(self, v: &str) -> Result { + self.inner.serialize_str(v) + } + + fn serialize_bytes(self, v: &[u8]) -> Result { + self.inner.serialize_bytes(v) + } + + fn serialize_none(self) -> Result { + self.inner.serialize_none() + } + + fn serialize_some(self, value: &T) -> Result + where + T: serde::Serialize + ?Sized, + { + self.inner.serialize_some(value) + } + + fn serialize_unit(self) -> Result { + self.inner.serialize_unit() + } + + fn serialize_unit_struct(self, name: &'static str) -> Result { + self.inner.serialize_unit_struct(name) + } + + fn serialize_unit_variant( + self, + name: &'static str, + variant_index: u32, + variant: &'static str, + ) -> Result { + self.inner + .serialize_unit_variant(name, variant_index, variant) + } + + fn serialize_newtype_struct( + self, + name: &'static str, + value: &T, + ) -> Result + where + T: serde::Serialize + ?Sized, + { + self.inner.serialize_newtype_struct(name, value) + } + + fn serialize_newtype_variant( + self, + name: &'static str, + variant_index: u32, + variant: &'static str, + value: &T, + ) -> Result + where + T: serde::Serialize + ?Sized, + { + self.inner + .serialize_newtype_variant(name, variant_index, variant, value) + } + + fn serialize_seq(self, len: Option) -> Result { + self.inner.serialize_seq(len) + } + + fn serialize_tuple(self, len: usize) -> Result { + self.inner.serialize_tuple(len) + } + + fn serialize_tuple_struct( + self, + name: &'static str, + len: usize, + ) -> Result { + self.inner.serialize_tuple_struct(name, len) + } + + fn serialize_tuple_variant( + self, + name: &'static str, + variant_index: u32, + variant: &'static str, + len: usize, + ) -> Result { + self.inner + .serialize_tuple_variant(name, variant_index, variant, len) + } + + fn serialize_map(self, _len: Option) -> Result { + Ok(CanonicalSerializeMap::new( + &mut self.inner, + self.options.clone(), + )) + } + + fn serialize_struct( + self, + name: &'static str, + _len: usize, + ) -> Result { + // We want to disallow `RawValue` as we don't know if its contents is + // canonical JSON. + // + // Note: the `name` here comes from `serde_json::raw::TOKEN`, which + // unfortunately isn't exported by the crate. + if name == "$serde_json::private::RawValue" { + return Err(Self::Error::custom("`RawValue` is not supported")); + } + Ok(CanonicalSerializeMap::new( + &mut self.inner, + self.options.clone(), + )) + } + + fn serialize_struct_variant( + self, + name: &'static str, + variant_index: u32, + variant: &'static str, + len: usize, + ) -> Result { + self.inner + .serialize_struct_variant(name, variant_index, variant, len) + } + + fn collect_str(self, value: &T) -> Result + where + T: std::fmt::Display + ?Sized, + { + self.inner.collect_str(value) + } +} + +/// A helper type for [`CanonicalSerializer`] that serializes JSON maps in +/// lexicographic order. +#[doc(hidden)] +pub struct CanonicalSerializeMap<'a, W> { + // We buffer up the key and serialized value for each field we see. + // The BTreeMap will then serialize in lexicographic order. + map: BTreeMap>, + // A key which we're still waiting for a value for + last_key: Option, + // The serializer to use to write the sorted map too. + ser: &'a mut Serializer, + options: CanonicalizationOptions, +} + +impl<'a, W> CanonicalSerializeMap<'a, W> { + fn new( + ser: &'a mut Serializer, + options: CanonicalizationOptions, + ) -> Self { + Self { + map: BTreeMap::new(), + last_key: None, + ser, + options, + } + } +} + +impl<'a, W> SerializeMap for CanonicalSerializeMap<'a, W> +where + W: Write, +{ + type Ok = (); + + type Error = serde_json::Error; + + fn serialize_key(&mut self, key: &T) -> Result<(), Self::Error> + where + T: serde::Serialize + ?Sized, + { + if self.last_key.is_some() { + // This can only happen if `serialize_key` is called multiple times + // in a row without a `serialize_value` call in between. This + // violates the contract of `SerializeMap`. + return Err(Self::Error::custom( + "serialize_key called multiple times in a row without serialize_value", + )); + } + + // Parse the `key` into a string. + let key_string = if let Value::String(str) = serde_json::to_value(key)? { + str + } else { + return Err(Self::Error::custom("key must be a string")); + }; + + self.last_key = Some(key_string); + + Ok(()) + } + + fn serialize_value(&mut self, value: &T) -> Result<(), Self::Error> + where + T: serde::Serialize + ?Sized, + { + let key_string = if let Some(key_string) = self.last_key.take() { + key_string + } else { + // `serde` should ensure that for every `serialize_key` there is a + // `serialize_field` call, so `last_key` should never be None here. + unreachable!() + }; + + // We serialize the value canonically, then store it as a `RawValue` in + // the buffer map. + let value_string = to_string_canonical(value, self.options.clone())?; + + self.map + .insert(key_string, RawValue::from_string(value_string)?); + + Ok(()) + } + + fn end(self) -> Result { + // No more entries in the map being serialized, so we can now serialize + // our buffered map (which will be serialized in the correct order as + // its a BTreeMap). + self.map.serialize(self.ser)?; + + Ok(()) + } +} + +impl<'a, W> SerializeStruct for CanonicalSerializeMap<'a, W> +where + W: Write, +{ + type Ok = (); + + type Error = serde_json::Error; + + fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error> + where + T: Serialize + ?Sized, + { + let key_string = key.to_string(); + + // We serialize the value canonically, then store it as a `RawValue` in + // the buffer map. + let value_string = to_string_canonical(value, self.options.clone())?; + + self.map + .insert(key_string, RawValue::from_string(value_string)?); + + Ok(()) + } + + fn end(self) -> Result { + self.map.serialize(self.ser)?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use itertools::Itertools; + use serde::Serializer; + use serde_json::json; + + use super::*; + + #[test] + fn empty() { + let test = json!({}); + + let json_string = to_string_canonical(&test, CanonicalizationOptions::strict()).unwrap(); + + assert_eq!(json_string, r#"{}"#); + } + + #[test] + fn order_struct_fields() { + #[derive(Serialize)] + struct Test { + b: u8, + a: u8, + } + + let test = Test { b: 1, a: 2 }; + + let json_string = to_string_canonical(&test, CanonicalizationOptions::strict()).unwrap(); + + assert_eq!(json_string, r#"{"a":2,"b":1}"#); + } + + #[test] + fn strings() { + let test = json!({ + "a": "\u{1F37B}", + "b": "\n", + "c": "\x01", + }); + + let json_string = to_string_canonical(&test, CanonicalizationOptions::strict()).unwrap(); + + assert_eq!(json_string, r#"{"a":"🍻","b":"\n","c":"\u0001"}"#); + } + + #[test] + fn escapes() { + let mut buffer; + let mut char_buffer = [0u8; 4]; + + // Ensure that we encode every UTF-8 character correctly + for c in '\0'..='\u{10FFFF}' { + // Serialize the character and strip out the quotes to make comparison easier. + let json_string = to_string_canonical(&c, CanonicalizationOptions::strict()).unwrap(); + let unquoted_json_string = &json_string[1..json_string.len() - 1]; + + let expected = match c { + // Some control characters have specific escape codes. + '\x08' => r"\b", + '\x09' => r"\t", + '\x0A' => r"\n", + '\x0C' => r"\f", + '\x0D' => r"\r", + '\x22' => r#"\""#, + '\x5C' => r"\\", + // Otherwise any character less than \x1F gets escaped as + // `\u00xx` + '\0'..='\x1F' => { + buffer = format!(r"\u00{:02x}", c as u32); + &buffer + } + // And everything else doesn't get escaped + _ => c.encode_utf8(&mut char_buffer), + }; + + // The serialized character will be wrapped in quotes. + assert_eq!(unquoted_json_string, expected); + } + } + + #[test] + fn nested_map() { + let test = json!({ + "a": {"b": 1} + }); + + let json_string = to_string_canonical(&test, CanonicalizationOptions::strict()).unwrap(); + + assert_eq!(json_string, r#"{"a":{"b":1}}"#); + } + + #[test] + fn floats() { + assert!(to_string_canonical(&100.0f32, CanonicalizationOptions::strict()).is_err()); + assert!(to_string_canonical(&100.0f64, CanonicalizationOptions::strict()).is_err()); + } + + #[test] + fn integers() { + assert_eq!( + to_string_canonical(&100u8, CanonicalizationOptions::strict()).unwrap(), + "100" + ); + assert_eq!( + to_string_canonical(&100u16, CanonicalizationOptions::strict()).unwrap(), + "100" + ); + assert_eq!( + to_string_canonical(&100u32, CanonicalizationOptions::strict()).unwrap(), + "100" + ); + assert_eq!( + to_string_canonical(&100u64, CanonicalizationOptions::strict()).unwrap(), + "100" + ); + assert_eq!( + to_string_canonical(&100u128, CanonicalizationOptions::strict()).unwrap(), + "100" + ); + + assert_eq!( + to_string_canonical(&100i8, CanonicalizationOptions::strict()).unwrap(), + "100" + ); + assert_eq!( + to_string_canonical(&100i16, CanonicalizationOptions::strict()).unwrap(), + "100" + ); + assert_eq!( + to_string_canonical(&100i32, CanonicalizationOptions::strict()).unwrap(), + "100" + ); + assert_eq!( + to_string_canonical(&100i64, CanonicalizationOptions::strict()).unwrap(), + "100" + ); + assert_eq!( + to_string_canonical(&100i128, CanonicalizationOptions::strict()).unwrap(), + "100" + ); + + assert!(to_string_canonical(&2u64.pow(60), CanonicalizationOptions::strict()).is_err()); + assert!(to_string_canonical(&2u128.pow(60), CanonicalizationOptions::strict()).is_err()); + + assert!(to_string_canonical(&2i64.pow(60), CanonicalizationOptions::strict()).is_err()); + assert!(to_string_canonical(&2i128.pow(60), CanonicalizationOptions::strict()).is_err()); + assert!(to_string_canonical(&-(2i64.pow(60)), CanonicalizationOptions::strict()).is_err()); + assert!(to_string_canonical(&-(2i128.pow(60)), CanonicalizationOptions::strict()).is_err()); + } + + #[test] + fn backwards_compatibility() { + assert_eq!( + to_string_canonical(&u64::MAX, CanonicalizationOptions::relaxed()).unwrap(), + format!("{}", u64::MAX) + ); + assert_eq!( + to_string_canonical(&u128::MAX, CanonicalizationOptions::relaxed()).unwrap(), + format!("{}", u128::MAX) + ); + assert_eq!( + to_string_canonical(&i128::MAX, CanonicalizationOptions::relaxed()).unwrap(), + format!("{}", i128::MAX) + ); + assert_eq!( + to_string_canonical(&-i128::MAX, CanonicalizationOptions::relaxed()).unwrap(), + format!("{}", -i128::MAX) + ); + } + + #[test] + fn hashmap_order() { + let mut test = HashMap::new(); + test.insert("e", 1); + test.insert("d", 1); + test.insert("c", 1); + test.insert("b", 1); + test.insert("a", 1); + test.insert("AA", 1); + + let json_string = to_string_canonical(&test, CanonicalizationOptions::strict()).unwrap(); + + assert_eq!(json_string, r#"{"AA":1,"a":1,"b":1,"c":1,"d":1,"e":1}"#); + } + + #[test] + fn raw_value() { + let raw_value = RawValue::from_string("{}".to_string()).unwrap(); + + assert!(to_string_canonical(&raw_value, CanonicalizationOptions::strict()).is_err()); + } + + #[test] + fn map_with_duplicate_keys() { + let mut output = Vec::new(); + let mut serializer = + CanonicalSerializer::new(&mut output, CanonicalizationOptions::strict()); + let mut map_serializer = serializer.serialize_map(None).unwrap(); + + map_serializer.serialize_entry("a", &1).unwrap(); + map_serializer.serialize_entry("a", &2).unwrap(); + + // Also try with different representations of the same key (e.g. `\t` and `\u{0009}`). + map_serializer.serialize_entry("\t", &2).unwrap(); + map_serializer.serialize_entry("\u{0009}", &2).unwrap(); + + SerializeMap::end(map_serializer).unwrap(); + + assert_eq!(String::from_utf8(output).unwrap(), r#"{"\t":2,"a":2}"#); + } + + #[test] + fn map_with_out_of_order_keys() { + let mut output = Vec::new(); + let mut serializer = + CanonicalSerializer::new(&mut output, CanonicalizationOptions::strict()); + let mut map_serializer = serializer.serialize_map(None).unwrap(); + + // An ordered list of keys to insert, and the expected way they should be serialized. + let ascii_order = [ + ('\0', r"\u0000"), + ('\t', r"\t"), + (' ', r" "), + ('!', r"!"), + ('"', r#"\""#), + ('&', r"&"), + ('A', r"A"), + ('\\', r"\\"), + ('a', r"a"), + ('🍻', r"🍻"), + ]; + + // Double check that the keys are in the expected order. + assert!(ascii_order.is_sorted_by_key(|(c, _)| u32::from(*c))); + + // Serialize the keys in the reverse order. + for (c, _) in ascii_order.iter().rev() { + map_serializer.serialize_entry(c.into(), &1).unwrap(); + } + SerializeMap::end(map_serializer).unwrap(); + + // The expected JSON should have the keys in the correct order, and the + // correct escaping. + let expected_json_inner = ascii_order + .iter() + .map(|(_, escaped)| format!(r#""{escaped}":1"#)) + .join(","); + let expected_json = r"{".to_owned() + &expected_json_inner + r"}"; + + assert_eq!(String::from_utf8(output).unwrap(), expected_json); + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 3b049a51b76..cc89862e4e7 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -5,6 +5,7 @@ use pyo3::prelude::*; use pyo3_log::ResetHandle; pub mod acl; +pub mod canonical_json; pub mod duration; pub mod errors; pub mod events; From ed3cafdb73e55306a428853f4b5e4f42bb293488 Mon Sep 17 00:00:00 2001 From: Oleg Girko Date: Wed, 29 Apr 2026 18:03:58 +0100 Subject: [PATCH 04/92] Partially revert "Bump authlib from 1.6.9 to 1.6.11 (#19703)" (#19742) The original commit should only have changed the lockfile. This reverts commit bdb1cf7416b46a637b3dae323cb05b4d94fafc82 (from https://github.com/element-hq/synapse/pull/19703). --------- Co-authored-by: Olivier 'reivilibre --- changelog.d/19742.bugfix | 1 + poetry.lock | 2 +- pyproject.toml | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 changelog.d/19742.bugfix diff --git a/changelog.d/19742.bugfix b/changelog.d/19742.bugfix new file mode 100644 index 00000000000..342769b65b7 --- /dev/null +++ b/changelog.d/19742.bugfix @@ -0,0 +1 @@ +Fix packaging for Fedora and EPEL caused by unnecessary bumping `authlib` minimum version requirement in `pyproject.toml` file. Contributed by Oleg Girko. diff --git a/poetry.lock b/poetry.lock index fbfada01fba..deceadea04b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3756,4 +3756,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<4.0.0" -content-hash = "8c52685a47cc4affa09d79a044e0aed4c15131a5581e2c4f641ffb5e538eec13" +content-hash = "d97bee07fec0f4048d964aa7127a50813920bce77b00e5191aa1815f83922c85" diff --git a/pyproject.toml b/pyproject.toml index 685a4a3a340..cd94c1aa105 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -137,7 +137,7 @@ saml2 = [ "defusedxml>=0.7.1", # via pysaml2 "pytz>=2018.3", # via pysaml2 ] -oidc = ["authlib>=1.6.11"] +oidc = ["authlib>=0.15.1"] url-preview = ["lxml>=4.6.3"] sentry = ["sentry-sdk>=0.7.2"] opentracing = [ @@ -179,7 +179,7 @@ all = [ # saml2 "pysaml2>=4.5.0", # oidc and jwt - "authlib>=1.6.11", + "authlib>=0.15.1", # url-preview "lxml>=4.6.3", # sentry From c376cdd2eeb448bdce39e058385f5f1bcc43701b Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 29 Apr 2026 17:17:53 +0000 Subject: [PATCH 05/92] Configure Dependabot to only update Python dependencies in the lockfile. (#19743) See: - https://github.com/element-hq/synapse/pull/19742 - https://github.com/element-hq/synapse/pull/19686 (etc) Documentation https://docs.github.com/en/code-security/reference/supply-chain-security/dependabot-options-reference#versioning-strategy-- We were considering `lockfile-only` but it sounds like `increase-if-necessary` would increase the upper bound for us, if we had one. Let's try it. --------- Signed-off-by: Olivier 'reivilibre --- .github/dependabot.yml | 1 + changelog.d/19743.misc | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/19743.misc diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 38920ead7ab..dc5bcaed143 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,6 +7,7 @@ updates: package-ecosystem: "pip" directory: "/" open-pull-requests-limit: 10 + versioning-strategy: "increase-if-necessary" schedule: interval: "weekly" # Group patch updates to packages together into a single PR, as they rarely diff --git a/changelog.d/19743.misc b/changelog.d/19743.misc new file mode 100644 index 00000000000..35c4841386e --- /dev/null +++ b/changelog.d/19743.misc @@ -0,0 +1 @@ +Configure Dependabot to only update Python dependencies in the lockfile, unless widening upper bounds. From 8fc23aa665fb5cee13660030e337f85271b470c3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Apr 2026 20:16:11 +0100 Subject: [PATCH 06/92] Bump pillow from 12.1.1 to 12.2.0 (#19686) --- poetry.lock | 235 ++++++++++++++++++++++++++-------------------------- 1 file changed, 117 insertions(+), 118 deletions(-) diff --git a/poetry.lock b/poetry.lock index deceadea04b..368ee3c408f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,7 +31,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" files = [ {file = "authlib-1.6.11-py2.py3-none-any.whl", hash = "sha256:c8687a9a26451c51a34a06fa17bb97cb15bba46a6a626755e2d7f50da8bff3e3"}, {file = "authlib-1.6.11.tar.gz", hash = "sha256:64db35b9b01aeccb4715a6c9a6613a06f2bd7be2ab9d2eb89edd1dfc7580a38f"}, @@ -62,7 +62,7 @@ description = "Backport of CPython tarfile module" optional = false python-versions = ">=3.8" groups = ["dev"] -markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" files = [ {file = "backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34"}, {file = "backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"}, @@ -531,7 +531,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -556,7 +556,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "elementpath-4.8.0-py3-none-any.whl", hash = "sha256:5393191f84969bcf8033b05ec4593ef940e58622ea13cefe60ecefbbf09d58d9"}, {file = "elementpath-4.8.0.tar.gz", hash = "sha256:5822a2560d99e2633d95f78694c7ff9646adaa187db520da200a8e9479dc46ae"}, @@ -606,7 +606,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:f525734382a47f9828c9d6a1501522c78d5935466d8e2be1a41ba40ca5bb922b"}, {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:6e2e1024f0a021777740cb7c633a0efb2c4a4bc570f508223a8dcbcf79f99ef9"}, @@ -889,7 +889,7 @@ description = "Read metadata from Python packages" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" files = [ {file = "importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151"}, {file = "importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb"}, @@ -930,7 +930,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1122,7 +1122,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1239,7 +1239,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"url-preview\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"url-preview\"" files = [ {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e77dd455b9a16bbd2a5036a63ddbd479c19572af81b624e79ef422f929eef388"}, {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d444858b9f07cefff6455b983aea9a67f7462ba1f6cbe4a21e8bf6791bf2153"}, @@ -1553,7 +1553,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "matrix_synapse_ldap3-0.4.0-py3-none-any.whl", hash = "sha256:bf080037230d2af5fd3639cb87266de65c1cad7a68ea206278c5b4bf9c1a17f3"}, {file = "matrix_synapse_ldap3-0.4.0.tar.gz", hash = "sha256:cff52ba780170de5e6e8af42863d2648ee23f3bf0a9fea6db52372f9fc00be2b"}, @@ -1834,7 +1834,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1901,103 +1901,103 @@ files = [ [[package]] name = "pillow" -version = "12.1.1" +version = "12.2.0" description = "Python Imaging Library (fork)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "pillow-12.1.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1f1625b72740fdda5d77b4def688eb8fd6490975d06b909fd19f13f391e077e0"}, - {file = "pillow-12.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:178aa072084bd88ec759052feca8e56cbb14a60b39322b99a049e58090479713"}, - {file = "pillow-12.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b66e95d05ba806247aaa1561f080abc7975daf715c30780ff92a20e4ec546e1b"}, - {file = "pillow-12.1.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:89c7e895002bbe49cdc5426150377cbbc04767d7547ed145473f496dfa40408b"}, - {file = "pillow-12.1.1-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a5cbdcddad0af3da87cb16b60d23648bc3b51967eb07223e9fed77a82b457c4"}, - {file = "pillow-12.1.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9f51079765661884a486727f0729d29054242f74b46186026582b4e4769918e4"}, - {file = "pillow-12.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:99c1506ea77c11531d75e3a412832a13a71c7ebc8192ab9e4b2e355555920e3e"}, - {file = "pillow-12.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:36341d06738a9f66c8287cf8b876d24b18db9bd8740fa0672c74e259ad408cff"}, - {file = "pillow-12.1.1-cp310-cp310-win32.whl", hash = "sha256:6c52f062424c523d6c4db85518774cc3d50f5539dd6eed32b8f6229b26f24d40"}, - {file = "pillow-12.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:c6008de247150668a705a6338156efb92334113421ceecf7438a12c9a12dab23"}, - {file = "pillow-12.1.1-cp310-cp310-win_arm64.whl", hash = "sha256:1a9b0ee305220b392e1124a764ee4265bd063e54a751a6b62eff69992f457fa9"}, - {file = "pillow-12.1.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:e879bb6cd5c73848ef3b2b48b8af9ff08c5b71ecda8048b7dd22d8a33f60be32"}, - {file = "pillow-12.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:365b10bb9417dd4498c0e3b128018c4a624dc11c7b97d8cc54effe3b096f4c38"}, - {file = "pillow-12.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d4ce8e329c93845720cd2014659ca67eac35f6433fd3050393d85f3ecef0dad5"}, - {file = "pillow-12.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc354a04072b765eccf2204f588a7a532c9511e8b9c7f900e1b64e3e33487090"}, - {file = "pillow-12.1.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7e7976bf1910a8116b523b9f9f58bf410f3e8aa330cd9a2bb2953f9266ab49af"}, - {file = "pillow-12.1.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:597bd9c8419bc7c6af5604e55847789b69123bbe25d65cc6ad3012b4f3c98d8b"}, - {file = "pillow-12.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2c1fc0f2ca5f96a3c8407e41cca26a16e46b21060fe6d5b099d2cb01412222f5"}, - {file = "pillow-12.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:578510d88c6229d735855e1f278aa305270438d36a05031dfaae5067cc8eb04d"}, - {file = "pillow-12.1.1-cp311-cp311-win32.whl", hash = "sha256:7311c0a0dcadb89b36b7025dfd8326ecfa36964e29913074d47382706e516a7c"}, - {file = "pillow-12.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:fbfa2a7c10cc2623f412753cddf391c7f971c52ca40a3f65dc5039b2939e8563"}, - {file = "pillow-12.1.1-cp311-cp311-win_arm64.whl", hash = "sha256:b81b5e3511211631b3f672a595e3221252c90af017e399056d0faabb9538aa80"}, - {file = "pillow-12.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ab323b787d6e18b3d91a72fc99b1a2c28651e4358749842b8f8dfacd28ef2052"}, - {file = "pillow-12.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:adebb5bee0f0af4909c30db0d890c773d1a92ffe83da908e2e9e720f8edf3984"}, - {file = "pillow-12.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bb66b7cc26f50977108790e2456b7921e773f23db5630261102233eb355a3b79"}, - {file = "pillow-12.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aee2810642b2898bb187ced9b349e95d2a7272930796e022efaf12e99dccd293"}, - {file = "pillow-12.1.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a0b1cd6232e2b618adcc54d9882e4e662a089d5768cd188f7c245b4c8c44a397"}, - {file = "pillow-12.1.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7aac39bcf8d4770d089588a2e1dd111cbaa42df5a94be3114222057d68336bd0"}, - {file = "pillow-12.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ab174cd7d29a62dd139c44bf74b698039328f45cb03b4596c43473a46656b2f3"}, - {file = "pillow-12.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:339ffdcb7cbeaa08221cd401d517d4b1fe7a9ed5d400e4a8039719238620ca35"}, - {file = "pillow-12.1.1-cp312-cp312-win32.whl", hash = "sha256:5d1f9575a12bed9e9eedd9a4972834b08c97a352bd17955ccdebfeca5913fa0a"}, - {file = "pillow-12.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:21329ec8c96c6e979cd0dfd29406c40c1d52521a90544463057d2aaa937d66a6"}, - {file = "pillow-12.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:af9a332e572978f0218686636610555ae3defd1633597be015ed50289a03c523"}, - {file = "pillow-12.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d242e8ac078781f1de88bf823d70c1a9b3c7950a44cdf4b7c012e22ccbcd8e4e"}, - {file = "pillow-12.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:02f84dfad02693676692746df05b89cf25597560db2857363a208e393429f5e9"}, - {file = "pillow-12.1.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:e65498daf4b583091ccbb2556c7000abf0f3349fcd57ef7adc9a84a394ed29f6"}, - {file = "pillow-12.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6c6db3b84c87d48d0088943bf33440e0c42370b99b1c2a7989216f7b42eede60"}, - {file = "pillow-12.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8b7e5304e34942bf62e15184219a7b5ad4ff7f3bb5cca4d984f37df1a0e1aee2"}, - {file = "pillow-12.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:18e5bddd742a44b7e6b1e773ab5db102bd7a94c32555ba656e76d319d19c3850"}, - {file = "pillow-12.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc44ef1f3de4f45b50ccf9136999d71abb99dca7706bc75d222ed350b9fd2289"}, - {file = "pillow-12.1.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a8eb7ed8d4198bccbd07058416eeec51686b498e784eda166395a23eb99138e"}, - {file = "pillow-12.1.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47b94983da0c642de92ced1702c5b6c292a84bd3a8e1d1702ff923f183594717"}, - {file = "pillow-12.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:518a48c2aab7ce596d3bf79d0e275661b846e86e4d0e7dec34712c30fe07f02a"}, - {file = "pillow-12.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a550ae29b95c6dc13cf69e2c9dc5747f814c54eeb2e32d683e5e93af56caa029"}, - {file = "pillow-12.1.1-cp313-cp313-win32.whl", hash = "sha256:a003d7422449f6d1e3a34e3dd4110c22148336918ddbfc6a32581cd54b2e0b2b"}, - {file = "pillow-12.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:344cf1e3dab3be4b1fa08e449323d98a2a3f819ad20f4b22e77a0ede31f0faa1"}, - {file = "pillow-12.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:5c0dd1636633e7e6a0afe7bf6a51a14992b7f8e60de5789018ebbdfae55b040a"}, - {file = "pillow-12.1.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0330d233c1a0ead844fc097a7d16c0abff4c12e856c0b325f231820fee1f39da"}, - {file = "pillow-12.1.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5dae5f21afb91322f2ff791895ddd8889e5e947ff59f71b46041c8ce6db790bc"}, - {file = "pillow-12.1.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2e0c664be47252947d870ac0d327fea7e63985a08794758aa8af5b6cb6ec0c9c"}, - {file = "pillow-12.1.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:691ab2ac363b8217f7d31b3497108fb1f50faab2f75dfb03284ec2f217e87bf8"}, - {file = "pillow-12.1.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9e8064fb1cc019296958595f6db671fba95209e3ceb0c4734c9baf97de04b20"}, - {file = "pillow-12.1.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:472a8d7ded663e6162dafdf20015c486a7009483ca671cece7a9279b512fcb13"}, - {file = "pillow-12.1.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:89b54027a766529136a06cfebeecb3a04900397a3590fd252160b888479517bf"}, - {file = "pillow-12.1.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:86172b0831b82ce4f7877f280055892b31179e1576aa00d0df3bb1bbf8c3e524"}, - {file = "pillow-12.1.1-cp313-cp313t-win32.whl", hash = "sha256:44ce27545b6efcf0fdbdceb31c9a5bdea9333e664cda58a7e674bb74608b3986"}, - {file = "pillow-12.1.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a285e3eb7a5a45a2ff504e31f4a8d1b12ef62e84e5411c6804a42197c1cf586c"}, - {file = "pillow-12.1.1-cp313-cp313t-win_arm64.whl", hash = "sha256:cc7d296b5ea4d29e6570dabeaed58d31c3fea35a633a69679fb03d7664f43fb3"}, - {file = "pillow-12.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:417423db963cb4be8bac3fc1204fe61610f6abeed1580a7a2cbb2fbda20f12af"}, - {file = "pillow-12.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:b957b71c6b2387610f556a7eb0828afbe40b4a98036fc0d2acfa5a44a0c2036f"}, - {file = "pillow-12.1.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:097690ba1f2efdeb165a20469d59d8bb03c55fb6621eb2041a060ae8ea3e9642"}, - {file = "pillow-12.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2815a87ab27848db0321fb78c7f0b2c8649dee134b7f2b80c6a45c6831d75ccd"}, - {file = "pillow-12.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f7ed2c6543bad5a7d5530eb9e78c53132f93dfa44a28492db88b41cdab885202"}, - {file = "pillow-12.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:652a2c9ccfb556235b2b501a3a7cf3742148cd22e04b5625c5fe057ea3e3191f"}, - {file = "pillow-12.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d6e4571eedf43af33d0fc233a382a76e849badbccdf1ac438841308652a08e1f"}, - {file = "pillow-12.1.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b574c51cf7d5d62e9be37ba446224b59a2da26dc4c1bb2ecbe936a4fb1a7cb7f"}, - {file = "pillow-12.1.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a37691702ed687799de29a518d63d4682d9016932db66d4e90c345831b02fb4e"}, - {file = "pillow-12.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f95c00d5d6700b2b890479664a06e754974848afaae5e21beb4d83c106923fd0"}, - {file = "pillow-12.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:559b38da23606e68681337ad74622c4dbba02254fc9cb4488a305dd5975c7eeb"}, - {file = "pillow-12.1.1-cp314-cp314-win32.whl", hash = "sha256:03edcc34d688572014ff223c125a3f77fb08091e4607e7745002fc214070b35f"}, - {file = "pillow-12.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:50480dcd74fa63b8e78235957d302d98d98d82ccbfac4c7e12108ba9ecbdba15"}, - {file = "pillow-12.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:5cb1785d97b0c3d1d1a16bc1d710c4a0049daefc4935f3a8f31f827f4d3d2e7f"}, - {file = "pillow-12.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:1f90cff8aa76835cba5769f0b3121a22bd4eb9e6884cfe338216e557a9a548b8"}, - {file = "pillow-12.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1f1be78ce9466a7ee64bfda57bdba0f7cc499d9794d518b854816c41bf0aa4e9"}, - {file = "pillow-12.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:42fc1f4677106188ad9a55562bbade416f8b55456f522430fadab3cef7cd4e60"}, - {file = "pillow-12.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:98edb152429ab62a1818039744d8fbb3ccab98a7c29fc3d5fcef158f3f1f68b7"}, - {file = "pillow-12.1.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d470ab1178551dd17fdba0fef463359c41aaa613cdcd7ff8373f54be629f9f8f"}, - {file = "pillow-12.1.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6408a7b064595afcab0a49393a413732a35788f2a5092fdc6266952ed67de586"}, - {file = "pillow-12.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5d8c41325b382c07799a3682c1c258469ea2ff97103c53717b7893862d0c98ce"}, - {file = "pillow-12.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c7697918b5be27424e9ce568193efd13d925c4481dd364e43f5dff72d33e10f8"}, - {file = "pillow-12.1.1-cp314-cp314t-win32.whl", hash = "sha256:d2912fd8114fc5545aa3a4b5576512f64c55a03f3ebcca4c10194d593d43ea36"}, - {file = "pillow-12.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:4ceb838d4bd9dab43e06c363cab2eebf63846d6a4aeaea283bbdfd8f1a8ed58b"}, - {file = "pillow-12.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:7b03048319bfc6170e93bd60728a1af51d3dd7704935feb228c4d4faab35d334"}, - {file = "pillow-12.1.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:600fd103672b925fe62ed08e0d874ea34d692474df6f4bf7ebe148b30f89f39f"}, - {file = "pillow-12.1.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:665e1b916b043cef294bc54d47bf02d87e13f769bc4bc5fa225a24b3a6c5aca9"}, - {file = "pillow-12.1.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:495c302af3aad1ca67420ddd5c7bd480c8867ad173528767d906428057a11f0e"}, - {file = "pillow-12.1.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8fd420ef0c52c88b5a035a0886f367748c72147b2b8f384c9d12656678dfdfa9"}, - {file = "pillow-12.1.1-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f975aa7ef9684ce7e2c18a3aa8f8e2106ce1e46b94ab713d156b2898811651d3"}, - {file = "pillow-12.1.1-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8089c852a56c2966cf18835db62d9b34fef7ba74c726ad943928d494fa7f4735"}, - {file = "pillow-12.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:cb9bb857b2d057c6dfc72ac5f3b44836924ba15721882ef103cecb40d002d80e"}, - {file = "pillow-12.1.1.tar.gz", hash = "sha256:9ad8fa5937ab05218e2b6a4cff30295ad35afd2f83ac592e68c0d871bb0fdbc4"}, + {file = "pillow-12.2.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:a4e8f36e677d3336f35089648c8955c51c6d386a13cf6ee9c189c5f5bd713a9f"}, + {file = "pillow-12.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e589959f10d9824d39b350472b92f0ce3b443c0a3442ebf41c40cb8361c5b97"}, + {file = "pillow-12.2.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a52edc8bfff4429aaabdf4d9ee0daadbbf8562364f940937b941f87a4290f5ff"}, + {file = "pillow-12.2.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:975385f4776fafde056abb318f612ef6285b10a1f12b8570f3647ad0d74b48ec"}, + {file = "pillow-12.2.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd9c0c7a0c681a347b3194c500cb1e6ca9cab053ea4d82a5cf45b6b754560136"}, + {file = "pillow-12.2.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:88d387ff40b3ff7c274947ed3125dedf5262ec6919d83946753b5f3d7c67ea4c"}, + {file = "pillow-12.2.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:51c4167c34b0d8ba05b547a3bb23578d0ba17b80a5593f93bd8ecb123dd336a3"}, + {file = "pillow-12.2.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:34c0d99ecccea270c04882cb3b86e7b57296079c9a4aff88cb3b33563d95afaa"}, + {file = "pillow-12.2.0-cp310-cp310-win32.whl", hash = "sha256:b85f66ae9eb53e860a873b858b789217ba505e5e405a24b85c0464822fe88032"}, + {file = "pillow-12.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:673aa32138f3e7531ccdbca7b3901dba9b70940a19ccecc6a37c77d5fdeb05b5"}, + {file = "pillow-12.2.0-cp310-cp310-win_arm64.whl", hash = "sha256:3e080565d8d7c671db5802eedfb438e5565ffa40115216eabb8cd52d0ecce024"}, + {file = "pillow-12.2.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:8be29e59487a79f173507c30ddf57e733a357f67881430449bb32614075a40ab"}, + {file = "pillow-12.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:71cde9a1e1551df7d34a25462fc60325e8a11a82cc2e2f54578e5e9a1e153d65"}, + {file = "pillow-12.2.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f490f9368b6fc026f021db16d7ec2fbf7d89e2edb42e8ec09d2c60505f5729c7"}, + {file = "pillow-12.2.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8bd7903a5f2a4545f6fd5935c90058b89d30045568985a71c79f5fd6edf9b91e"}, + {file = "pillow-12.2.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3997232e10d2920a68d25191392e3a4487d8183039e1c74c2297f00ed1c50705"}, + {file = "pillow-12.2.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e74473c875d78b8e9d5da2a70f7099549f9eb37ded4e2f6a463e60125bccd176"}, + {file = "pillow-12.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:56a3f9c60a13133a98ecff6197af34d7824de9b7b38c3654861a725c970c197b"}, + {file = "pillow-12.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:90e6f81de50ad6b534cab6e5aef77ff6e37722b2f5d908686f4a5c9eba17a909"}, + {file = "pillow-12.2.0-cp311-cp311-win32.whl", hash = "sha256:8c984051042858021a54926eb597d6ee3012393ce9c181814115df4c60b9a808"}, + {file = "pillow-12.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:6e6b2a0c538fc200b38ff9eb6628228b77908c319a005815f2dde585a0664b60"}, + {file = "pillow-12.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:9a8a34cc89c67a65ea7437ce257cea81a9dad65b29805f3ecee8c8fe8ff25ffe"}, + {file = "pillow-12.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d192a155bbcec180f8564f693e6fd9bccff5a7af9b32e2e4bf8c9c69dbad6b5"}, + {file = "pillow-12.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3f40b3c5a968281fd507d519e444c35f0ff171237f4fdde090dd60699458421"}, + {file = "pillow-12.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:03e7e372d5240cc23e9f07deca4d775c0817bffc641b01e9c3af208dbd300987"}, + {file = "pillow-12.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b86024e52a1b269467a802258c25521e6d742349d760728092e1bc2d135b4d76"}, + {file = "pillow-12.2.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7371b48c4fa448d20d2714c9a1f775a81155050d383333e0a6c15b1123dda005"}, + {file = "pillow-12.2.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62f5409336adb0663b7caa0da5c7d9e7bdbaae9ce761d34669420c2a801b2780"}, + {file = "pillow-12.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:01afa7cf67f74f09523699b4e88c73fb55c13346d212a59a2db1f86b0a63e8c5"}, + {file = "pillow-12.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5"}, + {file = "pillow-12.2.0-cp312-cp312-win32.whl", hash = "sha256:58f62cc0f00fd29e64b29f4fd923ffdb3859c9f9e6105bfc37ba1d08994e8940"}, + {file = "pillow-12.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f84204dee22a783350679a0333981df803dac21a0190d706a50475e361c93f5"}, + {file = "pillow-12.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:af73337013e0b3b46f175e79492d96845b16126ddf79c438d7ea7ff27783a414"}, + {file = "pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:8297651f5b5679c19968abefd6bb84d95fe30ef712eb1b2d9b2d31ca61267f4c"}, + {file = "pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:50d8520da2a6ce0af445fa6d648c4273c3eeefbc32d7ce049f22e8b5c3daecc2"}, + {file = "pillow-12.2.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:766cef22385fa1091258ad7e6216792b156dc16d8d3fa607e7545b2b72061f1c"}, + {file = "pillow-12.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5d2fd0fa6b5d9d1de415060363433f28da8b1526c1c129020435e186794b3795"}, + {file = "pillow-12.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:56b25336f502b6ed02e889f4ece894a72612fe885889a6e8c4c80239ff6e5f5f"}, + {file = "pillow-12.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f1c943e96e85df3d3478f7b691f229887e143f81fedab9b20205349ab04d73ed"}, + {file = "pillow-12.2.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:03f6fab9219220f041c74aeaa2939ff0062bd5c364ba9ce037197f4c6d498cd9"}, + {file = "pillow-12.2.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cdfebd752ec52bf5bb4e35d9c64b40826bc5b40a13df7c3cda20a2c03a0f5ed"}, + {file = "pillow-12.2.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eedf4b74eda2b5a4b2b2fb4c006d6295df3bf29e459e198c90ea48e130dc75c3"}, + {file = "pillow-12.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:00a2865911330191c0b818c59103b58a5e697cae67042366970a6b6f1b20b7f9"}, + {file = "pillow-12.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1e1757442ed87f4912397c6d35a0db6a7b52592156014706f17658ff58bbf795"}, + {file = "pillow-12.2.0-cp313-cp313-win32.whl", hash = "sha256:144748b3af2d1b358d41286056d0003f47cb339b8c43a9ea42f5fea4d8c66b6e"}, + {file = "pillow-12.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:390ede346628ccc626e5730107cde16c42d3836b89662a115a921f28440e6a3b"}, + {file = "pillow-12.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:8023abc91fba39036dbce14a7d6535632f99c0b857807cbbbf21ecc9f4717f06"}, + {file = "pillow-12.2.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:042db20a421b9bafecc4b84a8b6e444686bd9d836c7fd24542db3e7df7baad9b"}, + {file = "pillow-12.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:dd025009355c926a84a612fecf58bb315a3f6814b17ead51a8e48d3823d9087f"}, + {file = "pillow-12.2.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88ddbc66737e277852913bd1e07c150cc7bb124539f94c4e2df5344494e0a612"}, + {file = "pillow-12.2.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d362d1878f00c142b7e1a16e6e5e780f02be8195123f164edf7eddd911eefe7c"}, + {file = "pillow-12.2.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c727a6d53cb0018aadd8018c2b938376af27914a68a492f59dfcaca650d5eea"}, + {file = "pillow-12.2.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd8c21c98c5cc60653bcb311bef2ce0401642b7ce9d09e03a7da87c878289d4"}, + {file = "pillow-12.2.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9f08483a632889536b8139663db60f6724bfcb443c96f1b18855860d7d5c0fd4"}, + {file = "pillow-12.2.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dac8d77255a37e81a2efcbd1fc05f1c15ee82200e6c240d7e127e25e365c39ea"}, + {file = "pillow-12.2.0-cp313-cp313t-win32.whl", hash = "sha256:ee3120ae9dff32f121610bb08e4313be87e03efeadfc6c0d18f89127e24d0c24"}, + {file = "pillow-12.2.0-cp313-cp313t-win_amd64.whl", hash = "sha256:325ca0528c6788d2a6c3d40e3568639398137346c3d6e66bb61db96b96511c98"}, + {file = "pillow-12.2.0-cp313-cp313t-win_arm64.whl", hash = "sha256:2e5a76d03a6c6dcef67edabda7a52494afa4035021a79c8558e14af25313d453"}, + {file = "pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3adc9215e8be0448ed6e814966ecf3d9952f0ea40eb14e89a102b87f450660d8"}, + {file = "pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:6a9adfc6d24b10f89588096364cc726174118c62130c817c2837c60cf08a392b"}, + {file = "pillow-12.2.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6a6e67ea2e6feda684ed370f9a1c52e7a243631c025ba42149a2cc5934dec295"}, + {file = "pillow-12.2.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2bb4a8d594eacdfc59d9e5ad972aa8afdd48d584ffd5f13a937a664c3e7db0ed"}, + {file = "pillow-12.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:80b2da48193b2f33ed0c32c38140f9d3186583ce7d516526d462645fd98660ae"}, + {file = "pillow-12.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22db17c68434de69d8ecfc2fe821569195c0c373b25cccb9cbdacf2c6e53c601"}, + {file = "pillow-12.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7b14cc0106cd9aecda615dd6903840a058b4700fcb817687d0ee4fc8b6e389be"}, + {file = "pillow-12.2.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cbeb542b2ebc6fcdacabf8aca8c1a97c9b3ad3927d46b8723f9d4f033288a0f"}, + {file = "pillow-12.2.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bfd07bc812fbd20395212969e41931001fd59eb55a60658b0e5710872e95286"}, + {file = "pillow-12.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9aba9a17b623ef750a4d11b742cbafffeb48a869821252b30ee21b5e91392c50"}, + {file = "pillow-12.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:deede7c263feb25dba4e82ea23058a235dcc2fe1f6021025dc71f2b618e26104"}, + {file = "pillow-12.2.0-cp314-cp314-win32.whl", hash = "sha256:632ff19b2778e43162304d50da0181ce24ac5bb8180122cbe1bf4673428328c7"}, + {file = "pillow-12.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:4e6c62e9d237e9b65fac06857d511e90d8461a32adcc1b9065ea0c0fa3a28150"}, + {file = "pillow-12.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:b1c1fbd8a5a1af3412a0810d060a78b5136ec0836c8a4ef9aa11807f2a22f4e1"}, + {file = "pillow-12.2.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:57850958fe9c751670e49b2cecf6294acc99e562531f4bd317fa5ddee2068463"}, + {file = "pillow-12.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d5d38f1411c0ed9f97bcb49b7bd59b6b7c314e0e27420e34d99d844b9ce3b6f3"}, + {file = "pillow-12.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c0a9f29ca8e79f09de89293f82fc9b0270bb4af1d58bc98f540cc4aedf03166"}, + {file = "pillow-12.2.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1610dd6c61621ae1cf811bef44d77e149ce3f7b95afe66a4512f8c59f25d9ebe"}, + {file = "pillow-12.2.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a34329707af4f73cf1782a36cd2289c0368880654a2c11f027bcee9052d35dd"}, + {file = "pillow-12.2.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e9c4f5b3c546fa3458a29ab22646c1c6c787ea8f5ef51300e5a60300736905e"}, + {file = "pillow-12.2.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06"}, + {file = "pillow-12.2.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f278f034eb75b4e8a13a54a876cc4a5ab39173d2cdd93a638e1b467fc545ac43"}, + {file = "pillow-12.2.0-cp314-cp314t-win32.whl", hash = "sha256:6bb77b2dcb06b20f9f4b4a8454caa581cd4dd0643a08bacf821216a16d9c8354"}, + {file = "pillow-12.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6562ace0d3fb5f20ed7290f1f929cae41b25ae29528f2af1722966a0a02e2aa1"}, + {file = "pillow-12.2.0-cp314-cp314t-win_arm64.whl", hash = "sha256:aa88ccfe4e32d362816319ed727a004423aab09c5cea43c01a4b435643fa34eb"}, + {file = "pillow-12.2.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0538bd5e05efec03ae613fd89c4ce0368ecd2ba239cc25b9f9be7ed426b0af1f"}, + {file = "pillow-12.2.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:394167b21da716608eac917c60aa9b969421b5dcbbe02ae7f013e7b85811c69d"}, + {file = "pillow-12.2.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5d04bfa02cc2d23b497d1e90a0f927070043f6cbf303e738300532379a4b4e0f"}, + {file = "pillow-12.2.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0c838a5125cee37e68edec915651521191cef1e6aa336b855f495766e77a366e"}, + {file = "pillow-12.2.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a6c9fa44005fa37a91ebfc95d081e8079757d2e904b27103f4f5fa6f0bf78c0"}, + {file = "pillow-12.2.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:25373b66e0dd5905ed63fa3cae13c82fbddf3079f2c8bf15c6fb6a35586324c1"}, + {file = "pillow-12.2.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:bfa9c230d2fe991bed5318a5f119bd6780cda2915cca595393649fc118ab895e"}, + {file = "pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5"}, ] [package.extras] @@ -2032,7 +2032,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"postgres\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"postgres\"" files = [ {file = "psycopg2-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:103e857f46bb76908768ead4e2d0ba1d1a130e7b8ed77d3ae91e8b33481813e8"}, {file = "psycopg2-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:210daed32e18f35e3140a1ebe059ac29209dd96468f2f7559aa59f75ee82a5cb"}, @@ -2050,7 +2050,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -2066,7 +2066,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2348,7 +2348,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"cache-memory\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"cache-memory\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2480,7 +2480,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2505,7 +2505,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -2533,7 +2533,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pytz-2026.1.post1-py2.py3-none-any.whl", hash = "sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a"}, {file = "pytz-2026.1.post1.tar.gz", hash = "sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1"}, @@ -2938,7 +2938,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"sentry\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"sentry\"" files = [ {file = "sentry_sdk-2.57.0-py2.py3-none-any.whl", hash = "sha256:812c8bf5ff3d2f0e89c82f5ce80ab3a6423e102729c4706af7413fd1eb480585"}, {file = "sentry_sdk-2.57.0.tar.gz", hash = "sha256:4be8d1e71c32fb27f79c577a337ac8912137bba4bcbc64a4ec1da4d6d8dc5199"}, @@ -3138,7 +3138,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -3154,7 +3154,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "thrift-0.22.0.tar.gz", hash = "sha256:42e8276afbd5f54fe1d364858b6877bc5e5a4a5ed69f6a005b94ca4918fe1466"}, ] @@ -3220,7 +3220,6 @@ files = [ {file = "tomli-2.4.0-py3-none-any.whl", hash = "sha256:1f776e7d669ebceb01dee46484485f43a4048746235e683bcdffacdf1fb4785a"}, {file = "tomli-2.4.0.tar.gz", hash = "sha256:aa89c3f6c277dd275d8e243ad24f3b5e701491a860d5121f2cdd399fbb31fc9c"}, ] -markers = {main = "python_version < \"3.14\""} [[package]] name = "tornado" @@ -3229,7 +3228,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:487dc9cc380e29f58c7ab88f9e27cdeef04b2140862e5076a66fb6bb68bb1bfa"}, {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:65a7f1d46d4bb41df1ac99f5fcb685fb25c7e61613742d5108b010975a9a6521"}, @@ -3361,7 +3360,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3622,7 +3621,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "xmlschema-2.5.1-py3-none-any.whl", hash = "sha256:ec2b2a15c8896c1fcd14dcee34ca30032b99456c3c43ce793fdb9dca2fb4b869"}, {file = "xmlschema-2.5.1.tar.gz", hash = "sha256:4f7497de6c8b6dc2c28ad7b9ed6e21d186f4afe248a5bea4f54eedab4da44083"}, @@ -3643,7 +3642,7 @@ description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" files = [ {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, From 2e7019ebc8df9b3439ca8fbda37a175a41c77d3e Mon Sep 17 00:00:00 2001 From: Noah Markert Date: Thu, 30 Apr 2026 14:37:40 +0200 Subject: [PATCH 07/92] Expose tombstone status in room details (#19737) Exposes `tombstoned` and `replacement_room` in room details on admin API endpoint `GET /_synapse/admin/v1/rooms/`. Resolves #18347 --- changelog.d/19737.feature | 1 + docs/admin_api/rooms.md | 7 ++++++- synapse/rest/admin/rooms.py | 10 ++++++++++ tests/rest/admin/test_room.py | 4 ++++ 4 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19737.feature diff --git a/changelog.d/19737.feature b/changelog.d/19737.feature new file mode 100644 index 00000000000..13bf2405df3 --- /dev/null +++ b/changelog.d/19737.feature @@ -0,0 +1 @@ +Exposes `tombstoned` and `replacement_room` in room details on admin API endpoint `GET /_synapse/admin/v1/rooms/`. Contributed by Noah Markert. diff --git a/docs/admin_api/rooms.md b/docs/admin_api/rooms.md index 11e787c236c..c7544033e8c 100644 --- a/docs/admin_api/rooms.md +++ b/docs/admin_api/rooms.md @@ -308,6 +308,9 @@ The following fields are possible in the JSON response body: If the room does not define a type, the value will be `null`. * `forgotten` - Whether all local users have [forgotten](https://spec.matrix.org/latest/client-server-api/#leaving-rooms) the room. +* `tombstoned` - Whether the room has been tombstoned (permanently closed). +* `replacement_room` - The room ID of the new room that users should join instead, if this room was tombstoned. Will be + `null` if the room has not been tombstoned, or if it was tombstoned without designating a successor room. The API is: @@ -337,7 +340,9 @@ A response body like the following is returned: "history_visibility": "shared", "state_events": 93534, "room_type": "m.space", - "forgotten": false + "forgotten": false, + "tombstoned": false, + "replacement_room": null } ``` diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index 61511b93601..3783211a927 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -367,6 +367,7 @@ def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.room_shutdown_handler = hs.get_room_shutdown_handler() self.pagination_handler = hs.get_pagination_handler() + self._storage_controllers = hs.get_storage_controllers() async def on_GET( self, request: SynapseRequest, room_id: str @@ -383,6 +384,15 @@ async def on_GET( members ) result["forgotten"] = await self.store.is_locally_forgotten_room(room_id) + tombstone_event = await self._storage_controllers.state.get_current_state_event( + room_id, + EventTypes.Tombstone, + "", + ) + result["tombstoned"] = tombstone_event is not None + result["replacement_room"] = ( + tombstone_event.content.get("replacement_room") if tombstone_event else None + ) return HTTPStatus.OK, result diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index b32665eb73b..507cf10c5de 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -2311,10 +2311,14 @@ def test_single_room(self) -> None: self.assertIn("state_events", channel.json_body) self.assertIn("room_type", channel.json_body) self.assertIn("forgotten", channel.json_body) + self.assertIn("tombstoned", channel.json_body) + self.assertIn("replacement_room", channel.json_body) self.assertEqual(room_id_1, channel.json_body["room_id"]) self.assertIs(True, channel.json_body["federatable"]) self.assertIs(True, channel.json_body["public"]) + self.assertIs(False, channel.json_body["tombstoned"]) + self.assertIs(None, channel.json_body["replacement_room"]) def test_single_room_devices(self) -> None: """Test that `joined_local_devices` can be requested correctly""" From b8d7324373e2f8c2010877f5399da302d234dc22 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 May 2026 09:22:27 +0000 Subject: [PATCH 08/92] Bump the minor-and-patches group across 1 directory with 3 updates (#19736) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker.yml | 4 ++-- .github/workflows/docs-pr.yaml | 2 +- .github/workflows/latest_deps.yml | 2 +- .github/workflows/release-artifacts.yml | 8 ++++---- .github/workflows/tests.yml | 6 +++--- .github/workflows/twisted_trunk.yml | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index aaf3064c26d..c451e493011 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -87,7 +87,7 @@ jobs: - name: Build and push by digest id: build - uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 with: push: true labels: | @@ -108,7 +108,7 @@ jobs: touch "${{ runner.temp }}/digests/${digest#sha256:}" - name: Upload digest - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: digests-${{ matrix.suffix }} path: ${{ runner.temp }}/digests/* diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index cba12937436..b2161146db8 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -39,7 +39,7 @@ jobs: cp book/welcome_and_overview.html book/index.html - name: Upload Artifact - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: book path: book diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index d03a9295071..fa3f31e2c9a 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -172,7 +172,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 8c625be96e2..41a7cb56114 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -64,7 +64,7 @@ jobs: uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Set up docker layer caching - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} @@ -99,7 +99,7 @@ jobs: echo "ARTIFACT_NAME=${DISTRO#*:}" >> "$GITHUB_OUTPUT" - name: Upload debs as artifacts - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: debs-${{ steps.artifact-name.outputs.ARTIFACT_NAME }} path: debs/* @@ -150,7 +150,7 @@ jobs: # musl: (TODO: investigate). CIBW_TEST_SKIP: pp3*-* *musl* - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: Wheel-${{ matrix.os }} path: ./wheelhouse/*.whl @@ -171,7 +171,7 @@ jobs: - name: Build sdist run: python -m build --sdist - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: Sdist path: dist/*.tar.gz diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2d548a38832..c1277e3d5f5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -174,7 +174,7 @@ jobs: # Cribbed from # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17 - name: Restore/persist mypy's cache - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: | .mypy_cache @@ -561,7 +561,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }}) @@ -658,7 +658,7 @@ jobs: PGPASSWORD: postgres PGDATABASE: postgres - name: "Upload schema differences" - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: ${{ failure() && !cancelled() && steps.run_tester_script.outcome == 'failure' }} with: name: Schema dumps diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index d9d61152fbd..8d26e73e375 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -145,7 +145,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) From 697ef33dcbce376efd8941973e687d7908f1b64a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 May 2026 09:38:32 +0000 Subject: [PATCH 09/92] Bump gitpython from 3.1.46 to 3.1.47 (#19731) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 368ee3c408f..d66b175b0ab 100644 --- a/poetry.lock +++ b/poetry.lock @@ -582,21 +582,21 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.46" +version = "3.1.47" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "gitpython-3.1.46-py3-none-any.whl", hash = "sha256:79812ed143d9d25b6d176a10bb511de0f9c67b1fa641d82097b0ab90398a2058"}, - {file = "gitpython-3.1.46.tar.gz", hash = "sha256:400124c7d0ef4ea03f7310ac2fbf7151e09ff97f2a3288d64a440c584a29c37f"}, + {file = "gitpython-3.1.47-py3-none-any.whl", hash = "sha256:489f590edfd6d20571b2c0e72c6a6ac6915ee8b8cd04572330e3842207a78905"}, + {file = "gitpython-3.1.47.tar.gz", hash = "sha256:dba27f922bd2b42cb54c87a8ab3cb6beb6bf07f3d564e21ac848913a05a8a3cd"}, ] [package.dependencies] gitdb = ">=4.0.1,<5" [package.extras] -doc = ["sphinx (>=7.1.2,<7.2)", "sphinx-autodoc-typehints", "sphinx_rtd_theme"] +doc = ["sphinx (>=7.4.7,<8)", "sphinx-autodoc-typehints", "sphinx_rtd_theme"] test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3.8\"", "mypy (==1.18.2) ; python_version >= \"3.9\"", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions ; python_version < \"3.11\""] [[package]] diff --git a/pyproject.toml b/pyproject.toml index cd94c1aa105..f4c94acc1c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -292,7 +292,7 @@ dev = [ # The following are used by the release script "click>=8.1.3", # GitPython was == 3.1.14; bumped to 3.1.20, the first release with type hints. - "GitPython>=3.1.20", + "GitPython>=3.1.47", "markdown-it-py>=3.0.0", "pygithub>=1.59", # The following are executed as commands by the release script. From 6100f6e4f7fb0c72f1ae2802683ebc811c0e3a77 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 1 May 2026 11:42:00 -0500 Subject: [PATCH 10/92] Backfill from nearby points past pagination token (#19611) The juicy details and explanation are in the diff itself. Split out from https://github.com/element-hq/synapse/pull/18873 in order to fix paginating from [MSC3871](https://github.com/matrix-org/matrix-spec-proposals/pull/3871) gap tokens actually backfilling history. To be clear, this is a good change to make outside of the [MSC3871](https://github.com/matrix-org/matrix-spec-proposals/pull/3871) use case. For example (as the new Complement test shows), fixes a problem where if you try to paginate `/messages` from tokens returned by `/context`, we could fail to backfill anything new and hide away history. Also fixes https://github.com/matrix-org/complement/pull/853 --- changelog.d/19611.bugfix | 1 + complement/tests/room_messages_test.go | 394 ++++++++++++++++++ synapse/handlers/federation.py | 20 +- .../databases/main/event_federation.py | 89 ++-- tests/handlers/test_federation.py | 117 ------ 5 files changed, 476 insertions(+), 145 deletions(-) create mode 100644 changelog.d/19611.bugfix create mode 100644 complement/tests/room_messages_test.go diff --git a/changelog.d/19611.bugfix b/changelog.d/19611.bugfix new file mode 100644 index 00000000000..4952fd00db5 --- /dev/null +++ b/changelog.d/19611.bugfix @@ -0,0 +1 @@ +Fix Synapse not backfilling new history when attempting to use a pagination token near a backward extremity. diff --git a/complement/tests/room_messages_test.go b/complement/tests/room_messages_test.go new file mode 100644 index 00000000000..d491d1db857 --- /dev/null +++ b/complement/tests/room_messages_test.go @@ -0,0 +1,394 @@ +// This file is licensed under the Affero General Public License (AGPL) version 3. +// +// Copyright (C) 2026 Element Creations Ltd +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// See the GNU Affero General Public License for more details: +// . + +package synapse_tests + +import ( + "encoding/json" + "fmt" + "net/url" + "slices" + "strings" + "testing" + + "github.com/matrix-org/complement" + "github.com/matrix-org/complement/b" + "github.com/matrix-org/complement/client" + "github.com/matrix-org/complement/helpers" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/tidwall/gjson" +) + +func TestMessagesOverFederation(t *testing.T) { + deployment := complement.Deploy(t, 2) + defer deployment.Destroy(t) + + alice := deployment.Register(t, "hs1", helpers.RegistrationOpts{ + LocalpartSuffix: "alice", + }) + bob := deployment.Register(t, "hs2", helpers.RegistrationOpts{ + LocalpartSuffix: "bob", + }) + + // The typical convention to find backfill points is from the backward extremities in + // the DAG. Backward extremities are the oldest events we know of in the room but we + // only know of them because some other event referenced them by prev_event and aren't + // known to the homeserver yet (meaning we don't know their depth specifically). So we + // can only do approximate depth comparisons (use the depth of the known events + // they're connected to). And we don't know if those backward extremities point to a + // long chain/fork of history that could stretch back far enough to be visible. + // + // This means a naive homeserver implementation that looks for backward extremities <= + // depth of the `/messages?dir=b&from=xxx` token may overlook a backfill point that could + // reveal more history in the window the user is currently paginating in. + // + // This could be a near miss as this test is specifically stressing or a more deep miss + // as the backward extremity could reveal an entire fork of history that stretches + // back far enough to be visible. + // + // In Synapse, we consider "nearby" as anything within range of the `limit` specified + // in `/messages?dir=b&from=xxx&limit=xxx`. + // + // This test lives in our in-repo Complement tests for Synapse because the Matrix spec + // doesn't have any rules for how a homeserver should backfill. Practically speaking, + // homeservers that don't do anything for this problem will just hide messages from + // clients. This underscores the fact why it's necessary for homeservers to indicate that + // there is a gap (using MSC3871) at the very least. + // + // -------------------------------------------------- + // + // Even with MSC3871 gaps, the tested behavior here is necessary as the gap prev/next + // tokens point before/after the event (remember: tokens are positions between + // events), so if you use `/messages?dir=b&from=`, we can't + // rely on naive depth comparison. MSC3871 Complement tests will also exercise this. + // Example: + // + // t0 t1 t2 t3 t4 + // [A] <--- [B] <--- [C] <--- [bob join 4] + // + // When Bob calls `/messages?dir=b&backfill=false`, he sees a gap (`{ event_id: "bob + // join 4", prev_pagination_token: "t3", next_pagination_token: "t4" }`) and tries to + // fill it in with `/messages?dir=b&from=t3&limit=10&backfill=true`. To find backfill + // points, Synapse will compare `t3` with the backward extremity at an approximate + // depth of 4. Which is why we take `t3`, add the `limit=10` and then do the + // comparison (find any backfill points with an approximate depth <= 13). + t.Run("Backfill from nearby backward extremities past token", func(t *testing.T) { + // Alice creates the room + roomID := alice.MustCreateRoom(t, map[string]interface{}{ + // The `public_chat` preset includes `history_visibility: "shared"` ("Previous + // events are always accessible to newly joined members. All events in the + // room are accessible, even those sent when the member was not a part of the + // room."), which is what we want to test. + "preset": "public_chat", + }) + + // Keep track of the order + eventIDs := make([]string, 0) + // Map from event_id to event info + eventMap := make(map[string]EventInfo) + + // Send some message history into the room + numberOfMessagesToSend := 3 + messageDrafts := make([]MessageDraft, 0, numberOfMessagesToSend) + for i := 0; i < numberOfMessagesToSend; i++ { + messageDrafts = append( + messageDrafts, + MessageDraft{alice, fmt.Sprintf("message history %d", i+1)}, + ) + } + sendAndTrackMessages(t, roomID, messageDrafts, &eventIDs, &eventMap) + + // Bob joins the room + bob.MustJoinRoom(t, roomID, []spec.ServerName{ + deployment.GetFullyQualifiedHomeserverName(t, "hs1"), + }) + bobJoinEventID := getStateID(t, bob, roomID, "m.room.member", bob.UserID) + + // Make it easy to cross-reference the events being talked about in the logs + for eventIndex, eventID := range eventIDs { + t.Logf("Message %d -> event_id=%s", eventIndex, eventID) + } + + // Use a `/context` request to get a pagination token just before Bob's join event + // (remember: tokens are positions between events) + // + // Usually a client would just use `/messages?dir=b` to start getting history + // after joining but this is valid as well. To illustrate a more real example of + // this, someone can use `/timestamp_to_event` to jump back in history and + // `/context` to start paginating history. + contextRes := bob.MustDo( + t, + "GET", + []string{"_matrix", "client", "v3", "rooms", roomID, "context", bobJoinEventID}, + client.WithContentType("application/json"), + client.WithQueries(url.Values{ + "limit": []string{"0"}, + }), + ) + contextResResBody := client.ParseJSON(t, contextRes) + // > `start`: A token that can be used to paginate backwards with. + // > - https://spec.matrix.org/v1.17/client-server-api/#get_matrixclientv3roomsroomidcontexteventid + paginationToken := client.GetJSONFieldStr(t, contextResResBody, "start") + + // Paginate backwards from the join event + messagesRes := bob.MustDo( + t, + "GET", + []string{"_matrix", "client", "v3", "rooms", roomID, "messages"}, + client.WithContentType("application/json"), + client.WithQueries(url.Values{ + "dir": []string{"b"}, + "limit": []string{"100"}, + "from": []string{paginationToken}, + }), + ) + messagesResBody := client.ParseJSON(t, messagesRes) + + // Since `dir=b`, these will be in reverse chronological order + actualEventIDsFromRequest := extractEventIDsFromMessagesResponse(t, messagesResBody) + + // Put them in chronological order to match the expected list + chronologicalActualEventIds := slices.Clone(actualEventIDsFromRequest) + slices.Reverse(chronologicalActualEventIds) + + // Assert timeline order + assertEventsInOrder(t, chronologicalActualEventIds, eventIDs) + }) + + // TODO: Backfill test to make sure we backfill from forks when viewing history (see + // docstring above). + // + // 1. Alice (hs1, engineered homeserver) creates a room with events A, B + // 1. Bob (hs2) joins the room + // 1. Bob leaves the room + // 1. Alice creates a fork from A with some history (1, 2, 3) and connects it back with a new event C + // 1. Bob joins back + // 1. Bob paginates `/messages?dir=b&from=` + // 1. Ensure Bob sees events: B, 2, 1, A + // + // 1 <--- 2 <----- 3 + // / \ + // A <------- B ▲ <--- C <-- D + // | + // Paginate backwards from this point + // t.Run("Backfill from nearby backward extremities past token (fork)", func(t *testing.T) { +} + +// These utilities match what we're using in the Complement repo (see +// `matrix-org/complement` -> `tests/csapi/room_messages_test.go`) + +type MessageDraft struct { + Sender *client.CSAPI + Message string +} + +type EventInfo struct { + MessageDraft MessageDraft + EventID string +} + +func sendMessageDrafts( + t *testing.T, + roomID string, + messageDrafts []MessageDraft, +) []string { + t.Helper() + + eventIDs := make([]string, len(messageDrafts)) + for messageDraftIndex, messageDraft := range messageDrafts { + eventID := messageDraft.Sender.SendEventSynced(t, roomID, b.Event{ + Type: "m.room.message", + Content: map[string]interface{}{ + "msgtype": "m.text", + "body": messageDraft.Message, + }, + }) + eventIDs[messageDraftIndex] = eventID + } + + return eventIDs +} + +// sendAndTrackMessages sends the given message drafts to the room, keeping track of the +// new events in the list of `eventIDs` and `eventMap`. Returns the list of new event +// IDs that were sent. +func sendAndTrackMessages( + t *testing.T, + roomID string, + messageDrafts []MessageDraft, + eventIDs *[]string, + eventMap *map[string]EventInfo, +) []string { + t.Helper() + + newEventIDs := sendMessageDrafts(t, roomID, messageDrafts) + + *eventIDs = append(*eventIDs, newEventIDs...) + for i, eventID := range newEventIDs { + (*eventMap)[eventID] = EventInfo{ + MessageDraft: messageDrafts[i], + EventID: eventID, + } + } + + return newEventIDs +} + +// extractEventIDsFromMessagesResponse extracts the event IDs from the given +// `/messages` response body. +func extractEventIDsFromMessagesResponse( + t *testing.T, + messagesResBody json.RawMessage, +) []string { + t.Helper() + + wantKey := "chunk" + keyRes := gjson.GetBytes(messagesResBody, wantKey) + if !keyRes.Exists() { + t.Fatalf("extractEventIDsFromMessagesResponse: missing key '%s'", wantKey) + } + if !keyRes.IsArray() { + t.Fatalf( + "extractEventIDsFromMessagesResponse: key '%s' is not an array (was %s)", + wantKey, + keyRes.Type, + ) + } + + var eventIDs []string + actualEvents := keyRes.Array() + for _, event := range actualEvents { + eventIDs = append(eventIDs, event.Get("event_id").Str) + } + + return eventIDs +} + +func filterEventIDs(t *testing.T, actualEventIDs []string, expectedEventIDs []string) []string { + t.Helper() + + relevantActualEventIDs := make([]string, 0, len(expectedEventIDs)) + for _, eventID := range actualEventIDs { + if slices.Contains(expectedEventIDs, eventID) { + relevantActualEventIDs = append(relevantActualEventIDs, eventID) + } + } + + return relevantActualEventIDs +} + +// assertEventsInOrder asserts all `actualEventIDs` are present and in order according +// to `expectedEventIDs`. Other unrelated events can be in between. +func assertEventsInOrder(t *testing.T, actualEventIDs []string, expectedEventIDs []string) { + t.Helper() + + relevantActualEventIDs := filterEventIDs(t, actualEventIDs, expectedEventIDs) + + if len(relevantActualEventIDs) != len(expectedEventIDs) { + t.Fatalf( + "expected %d events in timeline (got %d relevant events filtered down from %d events)\n%s", + len(expectedEventIDs), + len(relevantActualEventIDs), + len(actualEventIDs), + generateEventOrderDiffString(relevantActualEventIDs, expectedEventIDs), + ) + } + + for i, eventID := range relevantActualEventIDs { + if eventID != expectedEventIDs[i] { + t.Fatalf( + "expected event ID %s (got %s) at index %d\n%s", + expectedEventIDs[i], + eventID, + i, + generateEventOrderDiffString(relevantActualEventIDs, expectedEventIDs), + ) + } + } +} + +func generateEventOrderDiffString(actualEventIDs []string, expectedEventIDs []string) string { + expectedLines := make([]string, len(expectedEventIDs)) + for i, expectedEventID := range expectedEventIDs { + isExpectedInActual := slices.Contains(actualEventIDs, expectedEventID) + isMissingIndicatorString := " " + if !isExpectedInActual { + isMissingIndicatorString = "?" + } + + expectedLines[i] = fmt.Sprintf("%2d: %s %s", i, isMissingIndicatorString, expectedEventID) + } + expectedDiffString := strings.Join(expectedLines, "\n") + + actualLines := make([]string, len(actualEventIDs)) + for actualEventIndex, actualEventID := range actualEventIDs { + isActualInExpected := slices.Contains(expectedEventIDs, actualEventID) + isActualInExpectedIndicatorString := " " + if isActualInExpected { + isActualInExpectedIndicatorString = "+" + } + + expectedIndex := slices.Index(expectedEventIDs, actualEventID) + expectedIndexString := "" + if actualEventIndex != expectedIndex { + expectedDirectionString := "⬆️" + if expectedIndex > actualEventIndex { + expectedDirectionString = "⬇️" + } + + expectedIndexString = fmt.Sprintf( + " (expected index %d %s)", + expectedIndex, + expectedDirectionString, + ) + } + + actualLines[actualEventIndex] = fmt.Sprintf("%2d: %s %s%s", + actualEventIndex, isActualInExpectedIndicatorString, actualEventID, expectedIndexString, + ) + } + actualDiffString := strings.Join(actualLines, "\n") + + return fmt.Sprintf( + "Actual events ('+' = found expected items):\n%s\nExpected events ('?' = missing expected items):\n%s", + actualDiffString, + expectedDiffString, + ) +} + +func getStateID( + t *testing.T, + c *client.CSAPI, + roomID string, + stateType string, + stateKey string, +) string { + t.Helper() + + stateRes := c.MustDo(t, "GET", []string{"_matrix", "client", "v3", "rooms", roomID, "state"}) + stateResBody := client.ParseJSON(t, stateRes) + eventJSON := gjson.ParseBytes(stateResBody) + if !eventJSON.IsArray() { + t.Fatalf("expected array of state events but found %s", eventJSON.Type) + } + + events := eventJSON.Array() + + for _, event := range events { + if event.Get("type").Str == stateType && event.Get("state_key").Str == stateKey { + return event.Get("event_id").Str + } + } + + t.Fatalf("Unable to find state event for (%s, %s). Room state: %s", stateType, stateKey, events) + return "" +} diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b3444dd2ef0..166a02d7c7e 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -105,6 +105,12 @@ ) +NUMBER_OF_EVENTS_TO_BACKFILL = 100 +""" +The number of events we try to backfill from other servers in a single request. +""" + + # TODO: We can refactor this away now that there is only one backfill point again class _BackfillPointType(Enum): # a regular backwards extremity (ie, an event which we don't yet have, but which @@ -255,7 +261,9 @@ async def _maybe_backfill_inner( _BackfillPoint(event_id, depth, _BackfillPointType.BACKWARDS_EXTREMITY) for event_id, depth in await self.store.get_backfill_points_in_room( room_id=room_id, - current_depth=current_depth, + # Per the docstring, it's best to pad the `current_depth` by the + # number of messages you plan to backfill from these points. + nearby_depth=current_depth + NUMBER_OF_EVENTS_TO_BACKFILL, # We only need to end up with 5 extremities combined with the # insertion event extremities to make the `/backfill` request # but fetch an order of magnitude more to make sure there is @@ -299,12 +307,13 @@ async def _maybe_backfill_inner( # likely not to return anything relevant so we backfill in the background. The # only way, this could return something relevant is if we discover a new branch # of history that extends all the way back to where we are currently paginating - # and it's within the 100 events that are returned from `/backfill`. + # and it's within the `NUMBER_OF_EVENTS_TO_BACKFILL` events that are returned + # from `/backfill`. if not sorted_backfill_points and current_depth != MAX_DEPTH: # Check that we actually have later backfill points, if not just return. have_later_backfill_points = await self.store.get_backfill_points_in_room( room_id=room_id, - current_depth=MAX_DEPTH, + nearby_depth=MAX_DEPTH, limit=1, ) if not have_later_backfill_points: @@ -464,7 +473,10 @@ async def try_backfill(domains: StrCollection) -> bool: try: await self._federation_event_handler.backfill( - dom, room_id, limit=100, extremities=extremities_to_request + dom, + room_id, + limit=NUMBER_OF_EVENTS_TO_BACKFILL, + extremities=extremities_to_request, ) # If this succeeded then we probably already have the # appropriate stuff. diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index 415926eb0a3..d84c58dcf89 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -1199,34 +1199,72 @@ def _get_auth_chain_difference_txn( async def get_backfill_points_in_room( self, room_id: str, - current_depth: int, + nearby_depth: int, limit: int, ) -> list[tuple[str, int]]: """ Get the backward extremities to backfill from in the room along with the approximate depth. - Only returns events that are at a depth lower than or - equal to the `current_depth`. Sorted by depth, highest to lowest (descending) - so the closest events to the `current_depth` are first in the list. - - We ignore extremities that are newer than the user's current scroll position - (ie, those with depth greater than `current_depth`) as: - 1. we don't really care about getting events that have happened - after our current position; and - 2. by the nature of paginating and scrolling back, we have likely - previously tried and failed to backfill from that extremity, so - to avoid getting "stuck" requesting the same backfill repeatedly - we drop those extremities. + Only returns events that are at a depth lower than or equal to the `nearby_depth`. + Sorted by depth, highest to lowest (descending) so the closest events to the + `nearby_depth` are first in the list. + + ### Why `nearby_depth`? + + We find backfill points from the backward extremities in the DAG. Backward + extremities are the oldest events we know of in the room but we only know of + them because some other event referenced them by prev_event and aren't persisted + in our database yet (meaning we don't know their depth specifically). So we can + only do approximate depth comparisons (use the depth of the known events they're + connected to). And we don't know if those backward extremities point to a long + chain/fork of history that could stretch back far enough to be visible. + + This means a naive homeserver implementation that looks for backward extremities <= + depth of the `/messages?dir=b&from=xxx` token may overlook a backfill point that could + reveal more history in the window the user is currently paginating in. + + We consider "nearby" as anything within range of the number of events you plan + to backfill from the given backfill point. This is a good heuristic as since we + plan to backfill N events, the chain of events from a backfill point could + extend back into the visible window. + + Example: + + - Your pagination token represents a scroll position at a depth of `100`. + - We have a backfill point at an approximate depth of `125` + - You plan to backfill `50` events from that backfill point. + + When we pad the token `depth` with the number of messages we plan to backfill, + `100` + `50` = `150`, we find the backfill point at `125` (because <= `150`, our + `nearby_depth`), backfill `50` events to a depth of `75` in the timeline + (exposing new events that we can return `100` -> `75`). + + When we don't pad our token `depth`, `100` is lower than any of the backfill + points so we don't pick any and miss out on backfilling any events. Without + something like MSC3871 to indicate gaps in the timeline, clients will most + likely never know they are missing any events and never try to paginate again. + + Generally though, we ignore extremities that are newer than the user's current + scroll position (ie, those with depth greater than `nearby_depth`) as: + 1. we don't really care about getting events that have happened after our + current position; and + 2. by the nature of paginating and scrolling back, we have likely previously + tried and failed to backfill from that extremity, so to avoid getting + "stuck" requesting the same backfill repeatedly we drop those + extremities. Although we also have `event_failed_pull_attempts` nowadays + to backoff as well. Args: room_id: Room where we want to find the oldest events - current_depth: The depth at the user's current scrollback position + nearby_depth: Typically, this is depth at the user's current scrollback + position + the number of events you plan to backfill from these backfill + points. limit: The max number of backfill points to return Returns: List of (event_id, depth) tuples. Sorted by depth, highest to lowest - (descending) so the closest events to the `current_depth` are first + (descending) so the closest events to the `nearby_depth` are first in the list. """ @@ -1234,12 +1272,12 @@ def get_backfill_points_in_room_txn( txn: LoggingTransaction, room_id: str ) -> list[tuple[str, int]]: # Assemble a tuple lookup of event_id -> depth for the oldest events - # we know of in the room. Backwards extremeties are the oldest + # we know of in the room. Backwards extremities are the oldest # events we know of in the room but we only know of them because # some other event referenced them by prev_event and aren't # persisted in our database yet (meaning we don't know their depth # specifically). So we need to look for the approximate depth from - # the events connected to the current backwards extremeties. + # the events connected to the current backwards extremities. if isinstance(self.database_engine, PostgresEngine): least_function = "LEAST" @@ -1259,7 +1297,7 @@ def get_backfill_points_in_room_txn( ON edge.event_id = event.event_id /** * We find the "oldest" events in the room by looking for - * events connected to backwards extremeties (oldest events + * events connected to backwards extremities (oldest events * in the room that we know of so far). */ INNER JOIN event_backward_extremities AS backward_extrem @@ -1285,16 +1323,19 @@ def get_backfill_points_in_room_txn( AND edge.is_state is FALSE /** * We only want backwards extremities that are older than or at - * the same position of the given `current_depth` (where older + * the same position of the given `nearby_depth` (where older * means less than the given depth) because we're looking backwards - * from the `current_depth` when backfilling. + * from the `nearby_depth` when backfilling. + * + * Keep in mind that `event.depth` is an approximate depth of the + * backward extremity itself. * - * current_depth (ignore events that come after this, ignore 2-4) + * nearby_depth (ignore events that come after this, ignore 2-4) * | * ▼ * [0]<--[1]<--[2]<--[3]<--[4] */ - AND event.depth <= ? /* current_depth */ + AND event.depth <= ? /* nearby_depth */ /** * Exponential back-off (up to the upper bound) so we don't retry the * same backfill point over and over. ex. 2hr, 4hr, 8hr, 16hr, etc. @@ -1312,7 +1353,7 @@ def get_backfill_points_in_room_txn( ) ) /** - * Sort from highest (closest to the `current_depth`) to the lowest depth + * Sort from highest (closest to the `nearby_depth`) to the lowest depth * because the closest are most relevant to backfill from first. * Then tie-break on alphabetical order of the event_ids so we get a * consistent ordering which is nice when asserting things in tests. @@ -1325,7 +1366,7 @@ def get_backfill_points_in_room_txn( sql, ( room_id, - current_depth, + nearby_depth, self.clock.time_msec(), BACKFILL_EVENT_EXPONENTIAL_BACKOFF_MAXIMUM_DOUBLING_STEPS, BACKFILL_EVENT_EXPONENTIAL_BACKOFF_STEP_MILLISECONDS, diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index e4a41cf1ae4..dde17858549 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -43,10 +43,8 @@ from synapse.server import HomeServer from synapse.storage.databases.main.events_worker import EventCacheEntry from synapse.util.clock import Clock -from synapse.util.events import generate_fake_event_id from tests import unittest -from tests.test_utils import event_injection logger = logging.getLogger(__name__) @@ -213,121 +211,6 @@ def test_rejected_state_event_state(self) -> None: self.assertEqual(sg, sg2) - def test_backfill_with_many_backward_extremities(self) -> None: - """ - Check that we can backfill with many backward extremities. - The goal is to make sure that when we only use a portion - of backwards extremities(the magic number is more than 5), - no errors are thrown. - - Regression test, see https://github.com/matrix-org/synapse/pull/11027 - """ - # create the room - user_id = self.register_user("kermit", "test") - tok = self.login("kermit", "test") - - room_id = self.helper.create_room_as(room_creator=user_id, tok=tok) - room_version = self.get_success(self.store.get_room_version(room_id)) - - # we need a user on the remote server to be a member, so that we can send - # extremity-causing events. - remote_server_user_id = f"@user:{self.OTHER_SERVER_NAME}" - self.get_success( - event_injection.inject_member_event( - self.hs, room_id, remote_server_user_id, "join" - ) - ) - - send_result = self.helper.send(room_id, "first message", tok=tok) - ev1 = self.get_success( - self.store.get_event(send_result["event_id"], allow_none=False) - ) - current_state = self.get_success( - self.store.get_events_as_list( - ( - self.get_success(self.store.get_partial_current_state_ids(room_id)) - ).values() - ) - ) - - # Create "many" backward extremities. The magic number we're trying to - # create more than is 5 which corresponds to the number of backward - # extremities we slice off in `_maybe_backfill_inner` - federation_event_handler = self.hs.get_federation_event_handler() - auth_events = [ - ev - for ev in current_state - if (ev.type, ev.state_key) - in {("m.room.create", ""), ("m.room.member", remote_server_user_id)} - ] - for _ in range(8): - event = make_event_from_dict( - self.add_hashes_and_signatures_from_other_server( - { - "origin_server_ts": 1, - "type": "m.room.message", - "content": { - "msgtype": "m.text", - "body": "message connected to fake event", - }, - "room_id": room_id, - "sender": remote_server_user_id, - "prev_events": [ - ev1.event_id, - # We're creating an backward extremity each time thanks - # to this fake event - generate_fake_event_id(), - ], - "auth_events": [ev.event_id for ev in auth_events], - "depth": ev1.depth + 1, - }, - room_version, - ), - room_version, - ) - - # we poke this directly into _process_received_pdu, to avoid the - # federation handler wanting to backfill the fake event. - state_handler = self.hs.get_state_handler() - context = self.get_success( - state_handler.compute_event_context( - event, - state_ids_before_event={ - (e.type, e.state_key): e.event_id for e in current_state - }, - partial_state=False, - ) - ) - self.get_success( - federation_event_handler._process_received_pdu( - self.OTHER_SERVER_NAME, - event, - context, - ) - ) - - # we should now have 8 backwards extremities. - backwards_extremities = self.get_success( - self.store.db_pool.simple_select_list( - "event_backward_extremities", - keyvalues={"room_id": room_id}, - retcols=["event_id"], - ) - ) - self.assertEqual(len(backwards_extremities), 8) - - current_depth = 1 - limit = 100 - - # Make sure backfill still works - self.get_success( - self.hs.get_federation_handler().maybe_backfill( - room_id, - current_depth, - limit, - ) - ) - def test_backfill_ignores_known_events(self) -> None: """ Tests that events that we already know about are ignored when backfilling. From 3f58bc50dfba5768ee43ce48c5e74c25ba0b078a Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 5 May 2026 08:40:17 -0500 Subject: [PATCH 11/92] fix: Cap `WorkerLock` timeout intervals to 60 seconds (#19394) Fixes the symptoms of https://github.com/element-hq/synapse/issues/19315 / https://github.com/element-hq/synapse/issues/19588 but not the underlying reason causing the number to grow so large in the first place. ``` ValueError: Exceeds the limit (4300 digits) for integer string conversion; use sys.set_int_max_str_digits() to increase the limit ``` Copied from the original pull request on [Famedly's Synapse repo](https://github.com/famedly/synapse/pull/221) (with some edits): Basing the time interval around a 5 seconds leaves a big window of waiting especially as this window is doubled each retry, when another worker could be making progress but can not. Right now, the retry interval in seconds looks like `[0.2, 5, 10, 20, 40, 80, 160, 320, (continues to double)]` after which logging should start about excessive times and (relatively quickly) end up with an extremely large retry interval with an unrealistic expectation past the heat death of the universe. 1 year in seconds = 31,536,000. With this change, retry intervals in seconds should look more like: ``` [ 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8, 25.6, 51.2, 60, < never goes higher than this ] ``` Logging about excessive wait times will start at 10 minutes.
Previous breakdown when we were using 15 minutes ``` [ 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8, 25.6, 51.2, 102.4, # 1.7 minutes 204.8, # 3.41 minutes 409.6, # 6.83 minutes 819.2, # 13.65 minutes < logging about excessive times will start here, 13th iteration 900, # 15 minutes < never goes higher than this ] ```
Further suggested work in this area could be to define the cap, the retry interval starting point and the multiplier depending on how frequently this lock should be checked. See data below for reasons why. Increasing the jitter range may also be a good idea --------- Co-authored-by: Eric Eastwood --- changelog.d/19394.bugfix | 1 + synapse/handlers/worker_lock.py | 117 +++++++++++++++++++------- tests/handlers/test_worker_lock.py | 130 +++++++++++++++++++++++++++++ 3 files changed, 216 insertions(+), 32 deletions(-) create mode 100644 changelog.d/19394.bugfix diff --git a/changelog.d/19394.bugfix b/changelog.d/19394.bugfix new file mode 100644 index 00000000000..4ca92cfb328 --- /dev/null +++ b/changelog.d/19394.bugfix @@ -0,0 +1 @@ +Capped the `WorkerLock` time out interval to a maximum of 60 seconds to prevent dealing with excessively long numbers. Contributed by Famedly. diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 1537a18cc05..51be3b5084a 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -54,6 +54,9 @@ # will not disappear under our feet as long as we don't delete the room. NEW_EVENT_DURING_PURGE_LOCK_NAME = "new_event_during_purge_lock" +WORKER_LOCK_MAX_RETRY_INTERVAL = Duration(seconds=60) +WORKER_LOCK_EXCESSIVE_WAITING_WARN_DURATION = Duration(minutes=10) + class WorkerLocksHandler: """A class for waiting on taking out locks, rather than using the storage @@ -206,9 +209,10 @@ class WaitingLock: lock_name: str lock_key: str write: bool | None + start_ts_ms: int = 0 deferred: "defer.Deferred[None]" = attr.Factory(defer.Deferred) _inner_lock: Lock | None = None - _retry_interval: float = 0.1 + _timeout_interval: float = 0.1 _lock_span: "opentracing.Scope" = attr.Factory( lambda: start_active_span("WaitingLock.lock") ) @@ -220,6 +224,7 @@ def release_lock(self) -> None: self.deferred.callback(None) async def __aenter__(self) -> None: + self.start_ts_ms = self.clock.time_msec() self._lock_span.__enter__() with start_active_span("WaitingLock.waiting_for_lock"): @@ -240,19 +245,44 @@ async def __aenter__(self) -> None: break try: - # Wait until the we get notified the lock might have been + # Wait until the notification that the lock might have been # released (by the deferred being resolved). We also - # periodically wake up in case the lock was released but we + # periodically wake up in case the lock was released, but we # weren't notified. with PreserveLoggingContext(): - timeout = self._get_next_retry_interval() await timeout_deferred( deferred=self.deferred, - timeout=timeout, + timeout=self._timeout_interval, clock=self.clock, ) - except Exception: - pass + except defer.TimeoutError: + # Only increment the timeout value if this was an actual timeout + # (defer.TimeoutError) + self._increment_timeout_interval() + + now_ms = self.clock.time_msec() + time_spent_trying_to_lock = Duration( + milliseconds=now_ms - self.start_ts_ms + ) + if ( + time_spent_trying_to_lock.as_millis() + > WORKER_LOCK_EXCESSIVE_WAITING_WARN_DURATION.as_millis() + ): + logger.warning( + "(WaitingLock (%s, %s)) Time spent waiting to acquire lock " + "is getting excessive: %ss. There may be a deadlock.", + self.lock_name, + self.lock_key, + time_spent_trying_to_lock.as_secs(), + ) + + except Exception as e: + logger.warning( + "Caught an exception while waiting on WaitingLock(lock_name=%s, lock_key=%s): %r", + self.lock_name, + self.lock_key, + e, + ) return await self._inner_lock.__aenter__() @@ -273,15 +303,14 @@ async def __aexit__( return r - def _get_next_retry_interval(self) -> float: - next = self._retry_interval - self._retry_interval = max(5, next * 2) - if self._retry_interval > Duration(minutes=10).as_secs(): # >7 iterations - logger.warning( - "Lock timeout is getting excessive: %ss. There may be a deadlock.", - self._retry_interval, - ) - return next * random.uniform(0.9, 1.1) + def _increment_timeout_interval(self) -> float: + next_interval = self._timeout_interval + next_interval = min(WORKER_LOCK_MAX_RETRY_INTERVAL.as_secs(), next_interval * 2) + + # The jitter value is maintained for the timeout, to help avoid a "thundering + # herd" situation when all locks may time out at the same time. + self._timeout_interval = next_interval * random.uniform(0.9, 1.1) + return self._timeout_interval @attr.s(auto_attribs=True, eq=False) @@ -294,10 +323,11 @@ class WaitingMultiLock: store: LockStore handler: WorkerLocksHandler + start_ts_ms: int = 0 deferred: "defer.Deferred[None]" = attr.Factory(defer.Deferred) _inner_lock_cm: AsyncContextManager | None = None - _retry_interval: float = 0.1 + _timeout_interval: float = 0.1 _lock_span: "opentracing.Scope" = attr.Factory( lambda: start_active_span("WaitingLock.lock") ) @@ -309,6 +339,7 @@ def release_lock(self) -> None: self.deferred.callback(None) async def __aenter__(self) -> None: + self.start_ts_ms = self.clock.time_msec() self._lock_span.__enter__() with start_active_span("WaitingLock.waiting_for_lock"): @@ -324,19 +355,42 @@ async def __aenter__(self) -> None: break try: - # Wait until the we get notified the lock might have been + # Wait until the notification that the lock might have been # released (by the deferred being resolved). We also - # periodically wake up in case the lock was released but we + # periodically wake up in case the lock was released, but we # weren't notified. with PreserveLoggingContext(): - timeout = self._get_next_retry_interval() await timeout_deferred( deferred=self.deferred, - timeout=timeout, + timeout=self._timeout_interval, clock=self.clock, ) - except Exception: - pass + except defer.TimeoutError: + # Only increment the timeout value if this was an actual timeout + # (defer.TimeoutError) + self._increment_timeout_interval() + + now_ms = self.clock.time_msec() + time_spent_trying_to_lock = Duration( + milliseconds=now_ms - self.start_ts_ms + ) + if ( + time_spent_trying_to_lock.as_millis() + > WORKER_LOCK_EXCESSIVE_WAITING_WARN_DURATION.as_millis() + ): + logger.warning( + "(WaitingMultiLock (%r)) Time spent waiting to acquire lock " + "is getting excessive: %ss. There may be a deadlock.", + self.lock_names, + time_spent_trying_to_lock.as_secs(), + ) + + except Exception as e: + logger.warning( + "Caught an exception while waiting on WaitingMultiLock(lock_names=%r): %r", + self.lock_names, + e, + ) assert self._inner_lock_cm await self._inner_lock_cm.__aenter__() @@ -360,12 +414,11 @@ async def __aexit__( return r - def _get_next_retry_interval(self) -> float: - next = self._retry_interval - self._retry_interval = max(5, next * 2) - if self._retry_interval > Duration(minutes=10).as_secs(): # >7 iterations - logger.warning( - "Lock timeout is getting excessive: %ss. There may be a deadlock.", - self._retry_interval, - ) - return next * random.uniform(0.9, 1.1) + def _increment_timeout_interval(self) -> float: + next_interval = self._timeout_interval + next_interval = min(WORKER_LOCK_MAX_RETRY_INTERVAL.as_secs(), next_interval * 2) + + # The jitter value is maintained for the timeout, to help avoid a "thundering + # herd" situation when all locks may time out at the same time. + self._timeout_interval = next_interval * random.uniform(0.9, 1.1) + return self._timeout_interval diff --git a/tests/handlers/test_worker_lock.py b/tests/handlers/test_worker_lock.py index 61ff51ff923..74201f41515 100644 --- a/tests/handlers/test_worker_lock.py +++ b/tests/handlers/test_worker_lock.py @@ -26,7 +26,9 @@ from twisted.internet.testing import MemoryReactor from synapse.server import HomeServer +from synapse.storage.databases.main.lock import _RENEWAL_INTERVAL from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests import unittest from tests.replication._base import BaseMultiWorkerStreamTestCase @@ -40,6 +42,7 @@ def prepare( self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer ) -> None: self.worker_lock_handler = self.hs.get_worker_locks_handler() + self.store = self.hs.get_datastores().main def test_wait_for_lock_locally(self) -> None: """Test waiting for a lock on a single worker""" @@ -56,6 +59,66 @@ def test_wait_for_lock_locally(self) -> None: self.get_success(d2) self.get_success(lock2.__aexit__(None, None, None)) + def test_timeouts_for_lock_locally(self) -> None: + """ + Test that we regularly retry to reacquire locks. + + This is a regression test to make sure the lock retry time doesn't balloon to a value + so large it can't even be printed reliably anymore. + """ + + # Create and acquire the first lock + lock1 = self.worker_lock_handler.acquire_lock("name", "key") + self.get_success(lock1.__aenter__()) + + # Create and try to acquire the second lock + lock2 = self.worker_lock_handler.acquire_lock("name", "key") + d2 = defer.ensureDeferred(lock2.__aenter__()) + # Make sure we haven't acquired the lock yet (`lock1` still holds it) + self.assertNoResult(d2) + + # Advance time by an hour (some duration that would previously cause our timeout + # to balloon if it weren't constrained). Max back-off (saturate) + # + # Note: We use `_pump_by` instead of `pump`/`advance` as the `Lock` has an + # internal background looping call that runs every 30 seconds + # (`_RENEWAL_INTERVAL`) to renew the `Lock` and push it's "drop timeout" value + # further out by 2 minutes (`_LOCK_TIMEOUT_MS`). The `Lock` will prematurely + # drop if this renewal is not allowed to run, which sours the test. + # self.pump(amount=Duration(hours=1)) + self._pump_by(amount=Duration(hours=1), by=_RENEWAL_INTERVAL) + + # Make sure we haven't acquired the `lock2` yet (`lock1` still holds it) + self.assertNoResult(d2) + + # Release the first lock (`lock1`). The second lock(`lock2`) should be + # automatically acquired by the `pump()` inside `get_success()` + self.get_success(lock1.__aexit__(None, None, None)) + + # We should now have the lock + self.successResultOf(d2) + + def _pump_by( + self, + *, + amount: Duration = Duration(seconds=0), + by: Duration = Duration(seconds=0.1), + ) -> None: + """ + Like `self.pump()` but you can specify the time increment to advance with until + you reach the time amount. + + Unlike `self.pump()`, this doesn't multiply the time at all. + + Args: + amount: The amount of time to advance + by: The time increment in seconds to advance time by until we reach the `amount` + """ + end_time_s = self.reactor.seconds() + amount.as_secs() + + while self.reactor.seconds() < end_time_s: + self.reactor.advance(by.as_secs()) + def test_lock_contention(self) -> None: """Test lock contention when a lot of locks wait on a single worker""" nb_locks_to_test = 500 @@ -124,3 +187,70 @@ def test_wait_for_lock_worker(self) -> None: self.get_success(d2) self.get_success(lock2.__aexit__(None, None, None)) + + def test_timeouts_for_lock_worker(self) -> None: + """ + Test that we regularly retry to reacquire locks. + + This is a regression test to make sure the lock retry time doesn't balloon to a value + so large it can't even be printed reliably anymore. + """ + worker = self.make_worker_hs( + "synapse.app.generic_worker", + extra_config={ + "redis": {"enabled": True}, + }, + ) + worker_lock_handler = worker.get_worker_locks_handler() + + # Create and acquire the first lock on the main process + lock1 = self.main_worker_lock_handler.acquire_lock("name", "key") + self.get_success(lock1.__aenter__()) + + # Create and try to acquire the second lock on the worker + lock2 = worker_lock_handler.acquire_lock("name", "key") + d2 = defer.ensureDeferred(lock2.__aenter__()) + # Make sure we haven't acquired the lock yet (`lock1` still holds it) + self.assertNoResult(d2) + + # Advance time by an hour (some duration that would previously cause our timeout + # to balloon if it weren't constrained). Max back-off (saturate) + # + # Note: We use `_pump_by` instead of `pump`/`advance` as the `Lock` has an + # internal background looping call that runs every 30 seconds + # (`_RENEWAL_INTERVAL`) to renew the `Lock` and push it's "drop timeout" value + # further out by 2 minutes (`_LOCK_TIMEOUT_MS`). The `Lock` will prematurely + # drop if this renewal is not allowed to run, which sours the test. + # self.pump(amount=Duration(hours=1)) + self._pump_by(amount=Duration(hours=1), by=_RENEWAL_INTERVAL) + + # Make sure we haven't acquired the `lock2` yet (`lock1` still holds it) + self.assertNoResult(d2) + + # Release the first lock (`lock1`). The second lock(`lock2`) should be + # automatically acquired by the `pump()` inside `get_success()` + self.get_success(lock1.__aexit__(None, None, None)) + + # We should now have the lock + self.successResultOf(d2) + + def _pump_by( + self, + *, + amount: Duration = Duration(seconds=0), + by: Duration = Duration(seconds=0.1), + ) -> None: + """ + Like `self.pump()` but you can specify the time increment to advance with until + you reach the time amount. + + Unlike `self.pump()`, this doesn't multiply the time at all. + + Args: + amount: The amount of time to advance + by: The time increment in seconds to advance time by until we reach the `amount` + """ + end_time_s = self.reactor.seconds() + amount.as_secs() + + while self.reactor.seconds() < end_time_s: + self.reactor.advance(by.as_secs()) From 3e6bf10640d25d127399debb4beca2e9129524f4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 6 May 2026 11:38:15 +0100 Subject: [PATCH 12/92] Port `Event.signatures` field to Rust (#19706) This is another stepping stone in porting the event class fully to Rust. The new `Signatures` class is relatively simple, as we actually don't interact with it that much in the code. It does *not* implement `Mapping` or `MutableMapping` as that takes quite a lot of effort that we don't need, even though it would be more ergonomic. --- changelog.d/19706.misc | 1 + rust/Cargo.toml | 6 +- rust/src/events/mod.rs | 2 + rust/src/events/signatures.rs | 348 ++++++++++++++++++ synapse/crypto/event_signing.py | 4 +- synapse/crypto/keyring.py | 12 +- synapse/event_auth.py | 6 +- synapse/events/__init__.py | 14 +- synapse/handlers/message.py | 2 +- synapse/handlers/room_policy.py | 9 +- .../databases/main/events_bg_updates.py | 4 +- synapse/synapse_rust/events.pyi | 31 +- tests/handlers/test_federation.py | 2 +- 13 files changed, 415 insertions(+), 26 deletions(-) create mode 100644 changelog.d/19706.misc create mode 100644 rust/src/events/signatures.rs diff --git a/changelog.d/19706.misc b/changelog.d/19706.misc new file mode 100644 index 00000000000..205abd09d46 --- /dev/null +++ b/changelog.d/19706.misc @@ -0,0 +1 @@ +Port `Event.signatures` field to Rust. diff --git a/rust/Cargo.toml b/rust/Cargo.toml index e6b378a092f..5bdd1947070 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -43,7 +43,7 @@ pyo3-log = "0.13.1" pythonize = "0.27.0" regex = "1.6.0" sha2 = "0.10.8" -serde = { version = "1.0.144", features = ["derive"] } +serde = { version = "1.0.144", features = ["derive", "rc"] } serde_json = { version = "1.0.85", features = ["raw_value"] } ulid = "1.1.2" icu_segmenter = "2.0.0" @@ -58,10 +58,6 @@ tokio = { version = "1.44.2", features = ["rt", "rt-multi-thread"] } once_cell = "1.18.0" itertools = "0.14.0" -[features] -extension-module = ["pyo3/extension-module"] -default = ["extension-module"] - [build-dependencies] blake2 = "0.10.4" hex = "0.4.3" diff --git a/rust/src/events/mod.rs b/rust/src/events/mod.rs index 209efb917be..e42eb97739d 100644 --- a/rust/src/events/mod.rs +++ b/rust/src/events/mod.rs @@ -27,11 +27,13 @@ use pyo3::{ pub mod filter; mod internal_metadata; +pub mod signatures; /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { let child_module = PyModule::new(py, "events")?; child_module.add_class::()?; + child_module.add_class::()?; child_module.add_function(wrap_pyfunction!(filter::event_visible_to_server_py, m)?)?; m.add_submodule(&child_module)?; diff --git a/rust/src/events/signatures.rs b/rust/src/events/signatures.rs new file mode 100644 index 00000000000..0f2acd5c9bb --- /dev/null +++ b/rust/src/events/signatures.rs @@ -0,0 +1,348 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Class for representing event signatures + +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; + +use pyo3::{ + exceptions::{PyKeyError, PyRuntimeError}, + pyclass, pymethods, + types::{PyAnyMethods, PyDict, PyMapping, PyMappingMethods}, + Bound, IntoPyObject, PyAny, PyResult, Python, +}; +use serde::{Deserialize, Serialize}; + +/// A class representing the signatures on an event. +#[pyclass(frozen, skip_from_py_object)] +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(transparent)] +pub struct Signatures { + inner: Arc>>>, +} + +#[pymethods] +impl Signatures { + #[new] + #[pyo3(signature = (signatures = None))] + fn py_new(signatures: Option>>) -> Self { + let mut signatures = signatures.unwrap_or_default(); + + // Prune any entries that have no signatures. + signatures.retain(|_, server_sigs| !server_sigs.is_empty()); + + Self { + inner: Arc::new(RwLock::new(signatures)), + } + } + + /// Check if the signatures contain a signature for the given server name. + fn __contains__(&self, key: Bound<'_, PyAny>) -> PyResult { + let Ok(key) = key.extract::<&str>() else { + return Ok(false); + }; + + let signatures = self + .inner + .read() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + Ok(signatures.contains_key(key)) + } + + /// Get the number of servers that have signatures. + fn __len__(&self) -> PyResult { + let signatures = self + .inner + .read() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + Ok(signatures.len()) + } + + /// Get the signature for the given server name and key ID, if it exists. + fn get_signature(&self, server_name: &str, key_id: &str) -> PyResult> { + let signatures = self + .inner + .read() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + + Ok(signatures + .get(server_name) + .and_then(|server_sigs| server_sigs.get(key_id).cloned())) + } + + /// Get the signatures for the given server name. + fn __getitem__(&self, key: Bound<'_, PyAny>) -> PyResult> { + let Some(server_name) = key.extract::<&str>().ok() else { + return Err(PyKeyError::new_err(key.to_string())); + }; + + let signatures = self + .inner + .read() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + + if let Some(server_sigs) = signatures.get(server_name) { + Ok(server_sigs.clone()) + } else { + Err(PyKeyError::new_err(server_name.to_string())) + } + } + + /// Add a signature for the given server name and key ID. + fn add_signature( + &self, + server_name: String, + key_id: String, + signature: String, + ) -> PyResult<()> { + let mut signatures = self + .inner + .write() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + + signatures + .entry(server_name) + .or_default() + .insert(key_id, signature); + + Ok(()) + } + + /// Update the signatures with the given signatures. + /// + /// Will overwrite all existing signatures for the server names provided. + fn update(&self, other: &Bound<'_, PyMapping>) -> PyResult<()> { + let mut signatures = self + .inner + .write() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + + for list_entry in other.items()? { + let (server_name, server_sigs) = list_entry.extract::<(String, Bound)>()?; + + let mut entry = HashMap::new(); + for list_entry in server_sigs.items()? { + let (key, value) = list_entry.extract::<(String, String)>()?; + entry.insert(key, value); + } + + // Only insert the entry if it has at least one signature. + if !entry.is_empty() { + signatures.insert(server_name, entry); + } else { + signatures.remove(&server_name); + } + } + + Ok(()) + } + + /// Return a copy of the signatures as a dictionary. + fn as_dict<'py>(&self, py: Python<'py>) -> PyResult> { + let signatures = self + .inner + .read() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + + (&*signatures).into_pyobject(py) + } + + fn __repr__(&self) -> PyResult { + let signatures = self + .inner + .read() + .map_err(|_| PyRuntimeError::new_err("Failed to acquire lock"))?; + + Ok(format!("Signatures({signatures:?})")) + } +} + +#[cfg(test)] +mod tests { + use pythonize::pythonize; + + use super::*; + + /// Helper that reads the inner map directly. + fn read_inner(sigs: &Signatures) -> HashMap> { + sigs.inner.read().expect("lock poisoned").clone() + } + + /// Helper to create a server signatures map from a list of (key_id, sig) + /// pairs. + fn make_server_sigs(data: &[(&str, &str)]) -> HashMap { + let mut map = HashMap::new(); + for (key_id, sig) in data { + map.insert((*key_id).to_owned(), (*sig).to_owned()); + } + map + } + + /// Helper to create a `Signatures` object from a list of (server_name, + /// key_id, sig) tuples. + fn create_signatures(data: &[(&str, &str, &str)]) -> Signatures { + let mut map: HashMap> = HashMap::new(); + for (server_name, key_id, sig) in data { + map.entry((*server_name).to_owned()) + .or_default() + .insert((*key_id).to_owned(), (*sig).to_owned()); + } + Signatures::py_new(Some(map)) + } + + #[test] + fn test_new_empty() { + let sigs = Signatures::py_new(None); + assert!(read_inner(&sigs).is_empty()); + assert_eq!(sigs.__len__().unwrap(), 0); + } + + #[test] + fn test_new_with_data() { + let sigs = create_signatures(&[("example.com", "ed25519:key1", "sig1")]); + assert_eq!(sigs.__len__().unwrap(), 1); + assert_eq!( + sigs.get_signature("example.com", "ed25519:key1").unwrap(), + Some("sig1".to_string()) + ); + } + + #[test] + fn test_new_prunes_servers_with_no_signatures() { + let mut data = HashMap::new(); + data.insert("empty.example.com".to_string(), HashMap::new()); + data.insert( + "example.com".to_string(), + make_server_sigs(&[("ed25519:key1", "sig1")]), + ); + + let sigs = Signatures::py_new(Some(data)); + + let inner = read_inner(&sigs); + assert_eq!(inner.len(), 1); + assert!(inner.contains_key("example.com")); + assert!(!inner.contains_key("empty.example.com")); + } + + #[test] + fn test_add_signature() { + let sigs = Signatures::py_new(None); + sigs.add_signature( + "example.com".to_string(), + "ed25519:key1".to_string(), + "sig1".to_string(), + ) + .unwrap(); + + let inner = read_inner(&sigs); + assert_eq!(inner.len(), 1); + assert_eq!( + inner.get("example.com").and_then(|m| m.get("ed25519:key1")), + Some(&"sig1".to_string()) + ); + } + + #[test] + fn test_add_signature_to_existing_server() { + let sigs = create_signatures(&[("example.com", "ed25519:key1", "sig1")]); + sigs.add_signature( + "example.com".to_string(), + "ed25519:key2".to_string(), + "sig2".to_string(), + ) + .unwrap(); + + let inner = read_inner(&sigs); + assert_eq!(inner.len(), 1); + assert_eq!( + inner.get("example.com").and_then(|m| m.get("ed25519:key1")), + Some(&"sig1".to_string()) + ); + assert_eq!( + inner.get("example.com").and_then(|m| m.get("ed25519:key2")), + Some(&"sig2".to_string()) + ); + } + + #[test] + fn test_update_signatures_clobbers_existing() { + let sigs = create_signatures(&[("example.com", "ed25519:key1", "sig1")]); + + // Create a new signatures map with a different signature for the same + // server. + let mut other = HashMap::new(); + other.insert( + "example.com".to_string(), + make_server_sigs(&[("ed25519:key2", "sig2")]), + ); + + // Update the signatures with the new map. + Python::initialize(); + Python::attach(|py| { + let value = pythonize(py, &other).unwrap(); + let value = value.cast::().unwrap(); + + sigs.update(value).unwrap(); + }); + + // Check that the old signature has been replaced with the new one. + let inner = read_inner(&sigs); + assert_eq!(inner.len(), 1); + assert_eq!(inner["example.com"].len(), 1); + assert_eq!(inner["example.com"]["ed25519:key2"], "sig2"); + } + + #[test] + fn test_serialize() { + let mut data = HashMap::new(); + data.insert( + "example.com".to_string(), + make_server_sigs(&[("ed25519:key1", "sig1")]), + ); + let sigs = Signatures::py_new(Some(data)); + + let json = serde_json::to_string(&sigs).unwrap(); + assert_eq!(json, r#"{"example.com":{"ed25519:key1":"sig1"}}"#); + } + + #[test] + fn test_serialize_empty() { + let sigs = Signatures::py_new(None); + let json = serde_json::to_string(&sigs).unwrap(); + assert_eq!(json, "{}"); + } + + #[test] + fn test_deserialize() { + let json = r#"{"example.com":{"ed25519:key1":"sig1"}}"#; + let sigs: Signatures = serde_json::from_str(json).unwrap(); + + let inner = read_inner(&sigs); + assert_eq!(inner.len(), 1); + assert_eq!( + inner.get("example.com").and_then(|m| m.get("ed25519:key1")), + Some(&"sig1".to_string()) + ); + } + + #[test] + fn test_deserialize_empty() { + let sigs: Signatures = serde_json::from_str("{}").unwrap(); + assert!(read_inner(&sigs).is_empty()); + } +} diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index d789c06a9cf..823b6288e82 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -236,9 +236,7 @@ def event_needs_resigning( if sender.domain != server_name: return False want_key_id = verify_key.alg + ":" + verify_key.version - signed_with_current_key_id = ev.signatures.get(server_name, {}).get( - want_key_id, None - ) + signed_with_current_key_id = ev.signatures.get_signature(server_name, want_key_id) if signed_with_current_key_id: return False diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 0d4d5e0e172..36736b45592 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -120,8 +120,18 @@ def from_event( ) -> "VerifyJsonRequest": """Create a VerifyJsonRequest to verify all signatures on an event object for the given server. + + Raises immediately if the event doesn't have any signatures from the + given server. """ - key_ids = list(event.signatures.get(server_name, [])) + if server_name not in event.signatures: + raise SynapseError( + 400, + f"Not signed by {server_name}", + Codes.UNAUTHORIZED, + ) + + key_ids = list(event.signatures[server_name]) return VerifyJsonRequest( server_name, # We defer creating the redacted json object, as it uses a lot more diff --git a/synapse/event_auth.py b/synapse/event_auth.py index ca528ae2358..fd35da8ba09 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -128,7 +128,7 @@ def validate_event_for_room_version(event: "EventBase") -> None: ) # Check the sender's domain has signed the event - if not event.signatures.get(sender_domain): + if sender_domain not in event.signatures: # We allow invites via 3pid to have a sender from a different # HS, as the sender must match the sender of the original # 3pid invite. This is checked further down with the @@ -141,7 +141,7 @@ def validate_event_for_room_version(event: "EventBase") -> None: event_id_domain = get_domain_from_id(event.event_id) # Check the origin domain has signed the event - if not event.signatures.get(event_id_domain): + if event_id_domain not in event.signatures: raise AuthError(403, "Event not signed by sending server") is_invite_via_allow_rule = ( @@ -154,7 +154,7 @@ def validate_event_for_room_version(event: "EventBase") -> None: authoriser_domain = get_domain_from_id( event.content[EventContentFields.AUTHORISING_USER] ) - if not event.signatures.get(authoriser_domain): + if authoriser_domain not in event.signatures: raise AuthError(403, "Event not signed by authorising server") diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index ac9b31cfcf3..fc0f6aadbde 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -44,8 +44,12 @@ StickyEvent, ) from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions -from synapse.synapse_rust.events import EventInternalMetadata -from synapse.types import JsonDict, StateKey, StrCollection +from synapse.synapse_rust.events import EventInternalMetadata, Signatures +from synapse.types import ( + JsonDict, + StateKey, + StrCollection, +) from synapse.util.caches import intern_dict from synapse.util.duration import Duration from synapse.util.frozenutils import freeze @@ -203,7 +207,7 @@ def __init__( assert room_version.event_format == self.format_version self.room_version = room_version - self.signatures = signatures + self.signatures = Signatures(signatures) self.unsigned = unsigned self.rejected_reason = rejected_reason @@ -255,7 +259,9 @@ def get_state_key(self) -> str | None: def get_dict(self) -> JsonDict: d = dict(self._dict) - d.update({"signatures": self.signatures, "unsigned": dict(self.unsigned)}) + d.update( + {"signatures": self.signatures.as_dict(), "unsigned": dict(self.unsigned)} + ) return d diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 4032c7eca97..ed3bce69abe 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -2092,7 +2092,7 @@ async def persist_and_notify_client_events( event.unsigned.pop("room_state", None) # TODO: Make sure the signatures actually are correct. - event.signatures.update(returned_invite.signatures) + event.signatures.update(returned_invite.signatures.as_dict()) if event.content["membership"] == Membership.KNOCK: maybe_upsert_event_field( diff --git a/synapse/handlers/room_policy.py b/synapse/handlers/room_policy.py index 01943e19915..e46e6dc2efc 100644 --- a/synapse/handlers/room_policy.py +++ b/synapse/handlers/room_policy.py @@ -181,9 +181,10 @@ async def is_event_allowed(self, event: EventBase) -> bool: async def _verify_policy_server_signature( self, event: EventBase, policy_server: str, public_key: str ) -> bool: - # check the event is signed with this (via, public_key). - verify_json_req = VerifyJsonRequest.from_event(policy_server, event, 0) try: + # check the event is signed with this (via, public_key). + verify_json_req = VerifyJsonRequest.from_event(policy_server, event, 0) + key_bytes = decode_base64(public_key) verify_key = decode_verify_key_bytes(POLICY_SERVER_KEY_ID, key_bytes) # We would normally use KeyRing.verify_event_for_server but we can't here as we don't @@ -260,9 +261,7 @@ async def ask_policy_server_to_sign_event( # servers need to manually fetch signatures for. This is the code that allows # those events to continue working (because they're legally sent, even if missing # the policy server signature). - event.signatures.setdefault(policy_server.server_name, {}).update( - signature.get(policy_server.server_name, {}) - ) + event.signatures.update(signature) except HttpResponseException as ex: # re-wrap HTTP errors as `SynapseError` so they can be proxied to clients directly raise ex.to_synapse_error() from ex diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index d2623f0760e..c0d218398d9 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -2824,8 +2824,8 @@ def _fetch_next_events_txn( # with the provided old key. if old_verify_key is not None: old_key_id = f"{old_verify_key.alg}:{old_verify_key.version}" - server_sigs = event.signatures.get(self.hs.hostname, {}) - if old_key_id not in server_sigs: + old_sig = event.signatures.get_signature(self.hs.hostname, old_key_id) + if old_sig is None: # Event wasn't signed with this key ID at all, skip. continue diff --git a/synapse/synapse_rust/events.pyi b/synapse/synapse_rust/events.pyi index fe0ca044208..40cf3f59f23 100644 --- a/synapse/synapse_rust/events.pyi +++ b/synapse/synapse_rust/events.pyi @@ -10,7 +10,7 @@ # See the GNU Affero General Public License for more details: # . -from typing import Mapping +from typing import Any, Mapping from synapse.types import JsonDict @@ -154,3 +154,32 @@ def event_visible_to_server( Returns: Whether the server is allowed to see the unredacted event. """ + +class Signatures: + """A class representing the signatures on an event.""" + + def __init__(self, signatures: Mapping[str, Mapping[str, str]] | None = None): ... + def get_signature(self, server_name: str, key_id: str) -> str | None: ... + """Get the signature for the given server name and key ID, if it exists.""" + + def __getitem__(self, server_name: str) -> Mapping[str, str]: ... + """Get the signatures for the given server name. Raises KeyError if there + are no signatures for that server.""" + + def __contains__(self, server_name: Any) -> bool: ... + """Check if there are signatures for the given server name.""" + + def __len__(self) -> int: ... + """Return the number of servers that have signatures.""" + + def add_signature(self, server_name: str, key_id: str, signature: str) -> None: ... + """Add a signature for the given server name and key ID.""" + + def update(self, signatures: Mapping[str, Mapping[str, str]]) -> None: ... + """Update the signatures with the given signatures. + + Will overwrite all existing signatures for the server names provided. + """ + + def as_dict(self) -> dict[str, dict[str, str]]: ... + """Return a copy of the signatures as a dictionary.""" diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index dde17858549..20ffed68f49 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -368,7 +368,7 @@ def _build_and_send_join_event( ) # the auth code requires that a signature exists, but doesn't check that # signature... go figure. - join_event.signatures[other_server] = {"x": "y"} + join_event.signatures.update({other_server: {"x": "y"}}) self.get_success( self.hs.get_federation_event_handler().on_send_membership_event( From 23b8fcf85e9cb9dadb2b8ba2a29a2f717e19f790 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 6 May 2026 18:51:42 +0100 Subject: [PATCH 13/92] Port `Event.unsigned` field to Rust (#19708) Similar to #19706, let's port the `unsigned` field into a Rust class. This does change things a bit in that we now define exactly what unsigned fields that are allowed to be added to an event, and what actually gets persisted. This should be a noop though, as we carefully filter out what unsigned fields we allow in from federation, for example As a side effect of this cleanup, I think this fixes handling `unsigned.age` on events received over federation. --- changelog.d/19708.misc | 1 + rust/src/events/mod.rs | 2 + rust/src/events/unsigned.rs | 429 ++++++++++++++++++ synapse/events/__init__.py | 26 +- synapse/events/utils.py | 3 +- synapse/federation/federation_base.py | 32 +- synapse/federation/federation_client.py | 7 +- synapse/federation/federation_server.py | 17 +- synapse/handlers/message.py | 1 - synapse/storage/databases/main/events.py | 2 +- .../storage/databases/main/events_worker.py | 16 +- synapse/synapse_rust/events.pyi | 32 +- tests/events/test_utils.py | 32 +- tests/federation/test_federation_server.py | 10 +- 14 files changed, 568 insertions(+), 42 deletions(-) create mode 100644 changelog.d/19708.misc create mode 100644 rust/src/events/unsigned.rs diff --git a/changelog.d/19708.misc b/changelog.d/19708.misc new file mode 100644 index 00000000000..308c2b04d0d --- /dev/null +++ b/changelog.d/19708.misc @@ -0,0 +1 @@ +Port `Event.unsigned` field to Rust. diff --git a/rust/src/events/mod.rs b/rust/src/events/mod.rs index e42eb97739d..5f505abb912 100644 --- a/rust/src/events/mod.rs +++ b/rust/src/events/mod.rs @@ -28,12 +28,14 @@ use pyo3::{ pub mod filter; mod internal_metadata; pub mod signatures; +pub mod unsigned; /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { let child_module = PyModule::new(py, "events")?; child_module.add_class::()?; child_module.add_class::()?; + child_module.add_class::()?; child_module.add_function(wrap_pyfunction!(filter::event_visible_to_server_py, m)?)?; m.add_submodule(&child_module)?; diff --git a/rust/src/events/unsigned.rs b/rust/src/events/unsigned.rs new file mode 100644 index 00000000000..c41ed7e6e14 --- /dev/null +++ b/rust/src/events/unsigned.rs @@ -0,0 +1,429 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::sync::{Arc, RwLock, RwLockReadGuard}; + +use pyo3::{ + exceptions::{PyKeyError, PyRuntimeError, PyTypeError}, + pyclass, pymethods, + types::{PyAnyMethods, PyList, PyListMethods, PyMapping}, + Bound, IntoPyObjectExt, PyAny, PyResult, Python, +}; +use pythonize::{depythonize, pythonize}; +use serde::{Deserialize, Serialize}; + +#[pyclass(frozen, skip_from_py_object)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(transparent)] +pub struct Unsigned { + inner: Arc>, +} + +/// The fields in the unsigned data of an event that are persisted in the +/// database. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +struct PersistedUnsignedFields { + #[serde(skip_serializing_if = "Option::is_none")] + age_ts: Option, + #[serde(skip_serializing_if = "Option::is_none")] + replaces_state: Option, + #[serde(skip_serializing_if = "Option::is_none")] + invite_room_state: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + knock_room_state: Option>, +} + +/// The inner representation of the unsigned data of an event, which includes +/// both the fields that are persisted in the database and the fields that are +/// only used in memory. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct UnsignedInner { + #[serde(flatten)] + persisted_fields: PersistedUnsignedFields, + #[serde(skip_serializing_if = "Option::is_none")] + prev_content: Option>, // We use Box to minimise stack space + #[serde(skip_serializing_if = "Option::is_none")] + prev_sender: Option, +} + +/// The fields that exist on the unsigned data of an event. +/// +/// This is used when converting from python to rust, to ensure that if we add a +/// new field we don't forget to add it to all the necessary places. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum UnsignedField { + AgeTs, + ReplacesState, + InviteRoomState, + KnockRoomState, + PrevContent, + PrevSender, +} + +impl std::str::FromStr for UnsignedField { + type Err = (); + + fn from_str(s: &str) -> Result { + match s { + "age_ts" => Ok(Self::AgeTs), + "replaces_state" => Ok(Self::ReplacesState), + "invite_room_state" => Ok(Self::InviteRoomState), + "knock_room_state" => Ok(Self::KnockRoomState), + "prev_content" => Ok(Self::PrevContent), + "prev_sender" => Ok(Self::PrevSender), + _ => Err(()), + } + } +} + +impl Unsigned { + fn py_read(&self) -> PyResult> { + self.inner + .read() + .map_err(|_| PyRuntimeError::new_err("Unsigned lock poisoned")) + } + + fn py_write(&self) -> PyResult> { + self.inner + .write() + .map_err(|_| PyRuntimeError::new_err("Unsigned lock poisoned")) + } +} + +#[pymethods] +impl Unsigned { + #[new] + fn py_new(unsigned: Bound<'_, PyMapping>) -> PyResult { + let inner = depythonize(&unsigned)?; + + Ok(Self { + inner: Arc::new(RwLock::new(inner)), + }) + } + + fn __getitem__<'py>( + &self, + py: Python<'py>, + key: Bound<'_, PyAny>, + ) -> PyResult> { + let key = key + .extract::<&str>() + .map_err(|_| PyTypeError::new_err("Unsigned keys must be strings"))?; + + let field: UnsignedField = key + .parse() + .map_err(|_| PyKeyError::new_err(format!("Unsigned has no key '{key}'")))?; + + let unsigned = self.py_read()?; + + match field { + UnsignedField::AgeTs => Ok(unsigned + .persisted_fields + .age_ts + .ok_or_else(|| PyKeyError::new_err("age_ts"))? + .into_bound_py_any(py)?), + UnsignedField::ReplacesState => Ok((unsigned.persisted_fields.replaces_state) + .as_ref() + .ok_or_else(|| PyKeyError::new_err("replaces_state"))? + .into_bound_py_any(py)?), + UnsignedField::InviteRoomState => Ok(room_state_to_py( + py, + unsigned + .persisted_fields + .invite_room_state + .as_ref() + .ok_or_else(|| PyKeyError::new_err("invite_room_state"))?, + )?), + UnsignedField::KnockRoomState => Ok(room_state_to_py( + py, + unsigned + .persisted_fields + .knock_room_state + .as_ref() + .ok_or_else(|| PyKeyError::new_err("knock_room_state"))?, + )?), + UnsignedField::PrevContent => Ok(pythonize( + py, + unsigned + .prev_content + .as_ref() + .ok_or_else(|| PyKeyError::new_err("prev_content"))?, + )?), + UnsignedField::PrevSender => Ok((unsigned.prev_sender) + .as_ref() + .ok_or_else(|| PyKeyError::new_err("prev_sender"))? + .into_bound_py_any(py)?), + } + } + + fn __contains__(&self, key: Bound<'_, PyAny>) -> PyResult { + let Ok(key) = key.extract::<&str>() else { + return Ok(false); + }; + + let Ok(field) = key.parse::() else { + return Ok(false); + }; + + let unsigned = self.py_read()?; + + let exists = match field { + UnsignedField::AgeTs => unsigned.persisted_fields.age_ts.is_some(), + UnsignedField::ReplacesState => unsigned.persisted_fields.replaces_state.is_some(), + UnsignedField::InviteRoomState => unsigned.persisted_fields.invite_room_state.is_some(), + UnsignedField::KnockRoomState => unsigned.persisted_fields.knock_room_state.is_some(), + UnsignedField::PrevContent => unsigned.prev_content.is_some(), + UnsignedField::PrevSender => unsigned.prev_sender.is_some(), + }; + + Ok(exists) + } + + fn __setitem__(&self, key: Bound<'_, PyAny>, value: Bound<'_, PyAny>) -> PyResult<()> { + let key = key + .extract::<&str>() + .map_err(|_| PyTypeError::new_err("Unsigned keys must be strings"))?; + + let field: UnsignedField = key + .parse() + .map_err(|_| PyKeyError::new_err(format!("Unsigned has no key '{key}'")))?; + + let mut unsigned = self.py_write()?; + + match field { + UnsignedField::AgeTs => unsigned.persisted_fields.age_ts = Some(value.extract()?), + UnsignedField::ReplacesState => { + unsigned.persisted_fields.replaces_state = Some(value.extract()?) + } + UnsignedField::InviteRoomState => { + unsigned.persisted_fields.invite_room_state = Some(room_state_from_py(value)?) + } + UnsignedField::KnockRoomState => { + unsigned.persisted_fields.knock_room_state = Some(room_state_from_py(value)?) + } + UnsignedField::PrevContent => { + unsigned.prev_content = Some(Box::new(depythonize(&value)?)) + } + UnsignedField::PrevSender => unsigned.prev_sender = Some(value.extract()?), + } + + Ok(()) + } + + fn __delitem__(&self, key: Bound<'_, PyAny>) -> PyResult<()> { + let key = key + .extract::<&str>() + .map_err(|_| PyTypeError::new_err("Unsigned keys must be strings"))?; + + let field: UnsignedField = key + .parse() + .map_err(|_| PyKeyError::new_err(format!("Unsigned has no key '{key}'")))?; + + let mut unsigned = self.py_write()?; + + match field { + UnsignedField::AgeTs => unsigned.persisted_fields.age_ts = None, + UnsignedField::ReplacesState => unsigned.persisted_fields.replaces_state = None, + UnsignedField::InviteRoomState => unsigned.persisted_fields.invite_room_state = None, + UnsignedField::KnockRoomState => unsigned.persisted_fields.knock_room_state = None, + UnsignedField::PrevContent => unsigned.prev_content = None, + UnsignedField::PrevSender => unsigned.prev_sender = None, + } + + Ok(()) + } + + #[pyo3(signature = (key, default=None))] + fn get<'py>( + &self, + py: Python<'py>, + key: Bound<'py, PyAny>, + default: Option>, + ) -> PyResult>> { + match self.__getitem__(py, key) { + Ok(value) => Ok(Some(value)), + Err(err) => { + if err.is_instance_of::(py) { + Ok(default) + } else { + Err(err) + } + } + } + } + + fn for_persistence<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(pythonize(py, &self.py_read()?.persisted_fields)?) + } + + fn for_event<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(pythonize(py, &*self.py_read()?)?) + } +} + +fn room_state_to_py<'py>( + py: Python<'py>, + state: &[serde_json::Value], +) -> PyResult> { + let py_list = PyList::empty(py); + + for item in state { + py_list.append(pythonize(py, item)?)?; + } + + py_list.into_bound_py_any(py) +} + +fn room_state_from_py(value: Bound<'_, PyAny>) -> PyResult> { + let py_list = value.cast::()?; + + let mut state = Vec::with_capacity(py_list.len()); + for item in py_list.iter() { + state.push(pythonize::depythonize(&item)?); + } + + Ok(state) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn test_unsigned_field_from_str_valid() { + assert_eq!("age_ts".parse(), Ok(UnsignedField::AgeTs)); + assert_eq!("replaces_state".parse(), Ok(UnsignedField::ReplacesState)); + assert_eq!( + "invite_room_state".parse(), + Ok(UnsignedField::InviteRoomState) + ); + assert_eq!( + "knock_room_state".parse(), + Ok(UnsignedField::KnockRoomState) + ); + assert_eq!("prev_content".parse(), Ok(UnsignedField::PrevContent)); + assert_eq!("prev_sender".parse(), Ok(UnsignedField::PrevSender)); + } + + #[test] + fn test_unsigned_field_from_str_invalid() { + assert_eq!("".parse::(), Err(())); + assert_eq!("unknown".parse::(), Err(())); + // Case-sensitive: upper-case should not match. + assert_eq!("AGE_TS".parse::(), Err(())); + // Must be an exact match, no whitespace. + assert_eq!(" age_ts".parse::(), Err(())); + } + + #[test] + fn test_persisted_fields_serialize_empty_is_empty_object() { + let fields = PersistedUnsignedFields::default(); + let json = serde_json::to_value(&fields).unwrap(); + assert_eq!(json, json!({})); + } + + #[test] + fn test_persisted_fields_serialize_populated() { + let fields = PersistedUnsignedFields { + age_ts: Some(1234), + replaces_state: Some("$prev:example.com".to_string()), + invite_room_state: Some(vec![json!({"type": "m.room.name"})]), + knock_room_state: Some(vec![json!({"type": "m.room.topic"})]), + }; + let json = serde_json::to_value(&fields).unwrap(); + assert_eq!( + json, + json!({ + "age_ts": 1234, + "replaces_state": "$prev:example.com", + "invite_room_state": [{"type": "m.room.name"}], + "knock_room_state": [{"type": "m.room.topic"}], + }) + ); + } + + #[test] + fn test_unsigned_inner_flattens_persisted_fields() { + let inner = UnsignedInner { + persisted_fields: PersistedUnsignedFields { + age_ts: Some(99), + ..Default::default() + }, + prev_content: Some(Box::new(json!({"body": "hi"}))), + prev_sender: Some("@alice:example.com".to_string()), + }; + + let json = serde_json::to_value(&inner).unwrap(); + assert_eq!( + json, + json!({ + "age_ts": 99, + "prev_content": {"body": "hi"}, + "prev_sender": "@alice:example.com", + }) + ); + } + + #[test] + fn test_unsigned_inner_roundtrip() { + let original = UnsignedInner { + persisted_fields: PersistedUnsignedFields { + age_ts: Some(10), + replaces_state: Some("$state:example.com".to_string()), + invite_room_state: None, + knock_room_state: None, + }, + prev_content: Some(Box::new(json!({"membership": "join"}))), + prev_sender: None, + }; + + let json = serde_json::to_string(&original).unwrap(); + let roundtripped: UnsignedInner = serde_json::from_str(&json).unwrap(); + + assert_eq!(roundtripped.persisted_fields.age_ts, Some(10)); + assert_eq!( + roundtripped.persisted_fields.replaces_state.as_deref(), + Some("$state:example.com") + ); + assert_eq!( + roundtripped.prev_content.as_deref(), + Some(&json!({"membership": "join"})) + ); + assert_eq!(roundtripped.prev_sender, None); + } + + #[test] + fn test_unsigned_serializes_transparently() { + // `Unsigned` is `#[serde(transparent)]` over its inner, so serializing + // an empty default should yield an empty object rather than a wrapper. + let unsigned = Unsigned::default(); + let json = serde_json::to_value(&unsigned).unwrap(); + assert_eq!(json, json!({})); + } + + #[test] + fn test_unsigned_deserialize_from_flat_object() { + let json = json!({ + "age_ts": 5, + "prev_sender": "@bob:example.com", + }); + let unsigned: Unsigned = serde_json::from_value(json).unwrap(); + let inner = unsigned.inner.read().unwrap(); + assert_eq!(inner.persisted_fields.age_ts, Some(5)); + assert_eq!(inner.prev_sender.as_deref(), Some("@bob:example.com")); + } +} diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index fc0f6aadbde..0f850d19b1c 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -44,7 +44,7 @@ StickyEvent, ) from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions -from synapse.synapse_rust.events import EventInternalMetadata, Signatures +from synapse.synapse_rust.events import EventInternalMetadata, Signatures, Unsigned from synapse.types import ( JsonDict, StateKey, @@ -208,7 +208,7 @@ def __init__( self.room_version = room_version self.signatures = Signatures(signatures) - self.unsigned = unsigned + self.unsigned = Unsigned(unsigned) self.rejected_reason = rejected_reason self._dict = event_dict @@ -258,9 +258,25 @@ def get_state_key(self) -> str | None: return self._dict.get("state_key") def get_dict(self) -> JsonDict: + """Convert the event to a dictionary suitable for serialisation.""" d = dict(self._dict) d.update( - {"signatures": self.signatures.as_dict(), "unsigned": dict(self.unsigned)} + { + "signatures": self.signatures.as_dict(), + "unsigned": self.unsigned.for_event(), + } + ) + + return d + + def get_dict_for_persistence(self) -> JsonDict: + """Convert the event to a dictionary suitable for persistence.""" + d = dict(self._dict) + d.update( + { + "signatures": self.signatures.as_dict(), + "unsigned": self.unsigned.for_persistence(), + } ) return d @@ -401,7 +417,7 @@ def __init__( for name, sigs in event_dict.pop("signatures", {}).items() } - unsigned = dict(event_dict.pop("unsigned", {})) + unsigned = event_dict.pop("unsigned", {}) # We intern these strings because they turn up a lot (especially when # caching). @@ -455,7 +471,7 @@ def __init__( assert "event_id" not in event_dict - unsigned = dict(event_dict.pop("unsigned", {})) + unsigned = event_dict.pop("unsigned", {}) # We intern these strings because they turn up a lot (especially when # caching). diff --git a/synapse/events/utils.py b/synapse/events/utils.py index f038fb5578d..926c81b83dc 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -47,6 +47,7 @@ from synapse.api.errors import Codes, SynapseError from synapse.api.room_versions import RoomVersion from synapse.logging.opentracing import SynapseTags, set_tag, trace +from synapse.synapse_rust.events import Unsigned from synapse.types import JsonDict, Requester from . import EventBase, FrozenEventV2, StrippedStateEvent, make_event_from_dict @@ -987,7 +988,7 @@ def validate_canonicaljson(value: Any) -> None: def maybe_upsert_event_field( - event: EventBase, container: JsonDict, key: str, value: object + event: EventBase, container: Unsigned, key: str, value: object ) -> bool: """Upsert an event field, but only if this doesn't make the event too large. diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index fe0710a0bf4..1631f021ca1 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -307,20 +307,27 @@ def _is_invite_via_3pid(event: EventBase) -> bool: def parse_events_from_pdu_json( - pdus_json: Sequence[JsonDict], room_version: RoomVersion + pdus_json: Sequence[JsonDict], + room_version: RoomVersion, + received_time: int | None = None, ) -> list[EventBase]: return [ - event_from_pdu_json(pdu_json, room_version) + event_from_pdu_json(pdu_json, room_version, received_time=received_time) for pdu_json in filter_pdus_for_valid_depth(pdus_json) ] -def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventBase: +def event_from_pdu_json( + pdu_json: JsonDict, room_version: RoomVersion, received_time: int | None = None +) -> EventBase: """Construct an EventBase from an event json received over federation Args: pdu_json: pdu as received over federation room_version: The version of the room this event belongs to + received_time: timestamp in ms that the event was received at. If + `None` then any `age` field in the `unsigned` block will be + dropped. Raises: SynapseError: if the pdu is missing required fields or is otherwise @@ -333,6 +340,25 @@ def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventB if "unsigned" in pdu_json: _strip_unsigned_values(pdu_json) + # Handle the `age` field, which is sent by some servers as part of the + # `unsigned` block. We convert this into an `age_ts` field, which is + # what Synapse uses internally. We also remove the `age` field to avoid + # confusion. + # + # c.f. https://github.com/matrix-org/synapse/issues/8429 + unsigned = pdu_json["unsigned"] + age = unsigned.pop("age", None) + + # We check that the `age` is actually an int before using it below. We + # don't error here as the `age` a) doesn't affect the validity of the + # event, and b) is best effort anyway. + if not isinstance(age, int): + age = None + + unsigned.pop("age_ts", None) + if received_time is not None and age is not None: + unsigned["age_ts"] = received_time - int(age) + depth = pdu_json["depth"] if type(depth) is not int: # noqa: E721 raise SynapseError(400, "Depth %r not an intger" % (depth,), Codes.BAD_JSON) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 78a1900c731..2b5ef5fbacc 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -1574,10 +1574,15 @@ async def get_missing_events( min_depth=min_depth, timeout=timeout, ) + received_time = self._clock.time_msec() room_version = await self.store.get_room_version(room_id) - events = parse_events_from_pdu_json(content.get("events", []), room_version) + events = parse_events_from_pdu_json( + content.get("events", []), + room_version, + received_time=received_time, + ) signed_events = await self._check_sigs_and_hash_for_pulled_events_and_fetch( destination, events, room_version=room_version diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 1bbe1444223..60692879755 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -451,16 +451,6 @@ async def _handle_pdus_in_txn( newest_pdu_ts = 0 for p in transaction.pdus: - # FIXME (richardv): I don't think this works: - # https://github.com/matrix-org/synapse/issues/8429 - if "unsigned" in p: - unsigned = p["unsigned"] - if "age" in unsigned: - p["age"] = unsigned["age"] - if "age" in p: - p["age_ts"] = request_time - int(p["age"]) - del p["age"] - # We try and pull out an event ID so that if later checks fail we # can log something sensible. We don't mandate an event ID here in # case future event formats get rid of the key. @@ -488,10 +478,15 @@ async def _handle_pdus_in_txn( continue try: - event = event_from_pdu_json(p, room_version) + event = event_from_pdu_json(p, room_version, received_time=request_time) except SynapseError as e: logger.info("Ignoring PDU for failing to deserialize: %s", e) continue + except Exception as e: + # We catch all exceptions here as we don't want a single bad + # event to cause us to fail the whole transaction. + logger.exception("Error deserializing PDU: %s", e) + continue pdus_by_room.setdefault(room_id, []).append(event) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index ed3bce69abe..0687c9fa790 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -2089,7 +2089,6 @@ async def persist_and_notify_client_events( returned_invite = await federation_handler.send_invite( invitee.domain, event ) - event.unsigned.pop("room_state", None) # TODO: Make sure the signatures actually are correct. event.signatures.update(returned_invite.signatures.as_dict()) diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 12c918eca64..5f6b03a988d 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -2711,7 +2711,7 @@ def _store_event_txn( return def event_dict(event: EventBase) -> JsonDict: - d = event.get_dict() + d = event.get_dict_for_persistence() d.pop("redacted", None) d.pop("redacted_because", None) return d diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index cc79b8042bd..6f26bd17cec 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -779,14 +779,26 @@ async def get_events_as_list( events.append(event) if get_prev_content: - if "replaces_state" in event.unsigned: + # The `event` here might be in the cache, and so might have + # already had the `prev_content` and `prev_sender` fields added + # to its unsigned. + # + # We check if a) we should add the previous content, and b) if + # we have already added it. + replaces_state = "replaces_state" in event.unsigned + has_prev = ( + "prev_content" in event.unsigned and "prev_sender" in event.unsigned + ) + if replaces_state and not has_prev: prev = await self.get_event( event.unsigned["replaces_state"], get_prev_content=False, allow_none=True, ) if prev: - event.unsigned = dict(event.unsigned) + # This mutates the cached event, but that's fine as the + # previous content/sender will be the same for all + # requests for this event. event.unsigned["prev_content"] = prev.content event.unsigned["prev_sender"] = prev.sender diff --git a/synapse/synapse_rust/events.pyi b/synapse/synapse_rust/events.pyi index 40cf3f59f23..5ae2bb880ab 100644 --- a/synapse/synapse_rust/events.pyi +++ b/synapse/synapse_rust/events.pyi @@ -12,7 +12,7 @@ from typing import Any, Mapping -from synapse.types import JsonDict +from synapse.types import JsonDict, JsonMapping class EventInternalMetadata: def __init__(self, internal_metadata_dict: JsonDict): ... @@ -183,3 +183,33 @@ class Signatures: def as_dict(self) -> dict[str, dict[str, str]]: ... """Return a copy of the signatures as a dictionary.""" + +class Unsigned: + """A class representing the unsigned data of an event.""" + + def __init__(self, unsigned_dict: JsonMapping): ... + def __getitem__(self, key: str) -> Any: ... + """Get the value for the given key. + + Raises KeyError if the key is unset or not recognised.""" + + def __setitem__(self, key: str, value: Any) -> None: ... + """Set the value for the given key. + + Raises KeyError if the key is not recognised.""" + + def __delitem__(self, key: str) -> None: ... + """Delete the value for the given key. + + Raises KeyError if the key is unset or not recognised.""" + + def __contains__(self, key: Any) -> bool: ... + def get(self, key: str, default: Any = None) -> Any: ... + """Get the value for the given key, or ``default`` if the key is unset.""" + + def for_persistence(self) -> JsonDict: ... + """Return a dict of the fields that should be persisted to the database.""" + + def for_event(self) -> JsonDict: ... + """Return a dict of all unsigned fields, including those only kept in + memory, suitable for inclusion in an event.""" diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index 12ef42866d2..a402dd70d1f 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -67,25 +67,31 @@ def MockEvent(**kwargs: Any) -> EventBase: class TestMaybeUpsertEventField(stdlib_unittest.TestCase): def test_update_okay(self) -> None: event = make_event_from_dict({"event_id": "$1234"}) - success = maybe_upsert_event_field(event, event.unsigned, "key", "value") + success = maybe_upsert_event_field( + event, event.unsigned, "replaces_state", "value" + ) self.assertTrue(success) - self.assertEqual(event.unsigned["key"], "value") + self.assertEqual(event.unsigned["replaces_state"], "value") def test_update_not_okay(self) -> None: event = make_event_from_dict({"event_id": "$1234"}) LARGE_STRING = "a" * 100_000 - success = maybe_upsert_event_field(event, event.unsigned, "key", LARGE_STRING) + success = maybe_upsert_event_field( + event, event.unsigned, "replaces_state", LARGE_STRING + ) self.assertFalse(success) - self.assertNotIn("key", event.unsigned) + self.assertNotIn("replaces_state", event.unsigned) def test_update_not_okay_leaves_original_value(self) -> None: event = make_event_from_dict( - {"event_id": "$1234", "unsigned": {"key": "value"}} + {"event_id": "$1234", "unsigned": {"replaces_state": "value"}} ) LARGE_STRING = "a" * 100_000 - success = maybe_upsert_event_field(event, event.unsigned, "key", LARGE_STRING) + success = maybe_upsert_event_field( + event, event.unsigned, "replaces_state", LARGE_STRING + ) self.assertFalse(success) - self.assertEqual(event.unsigned["key"], "value") + self.assertEqual(event.unsigned["replaces_state"], "value") class PruneEventTestCase(stdlib_unittest.TestCase): @@ -623,7 +629,7 @@ def test_unsigned_is_copied(self) -> None: { "type": "A", "event_id": "$test:domain", - "unsigned": {"a": 1, "b": 2}, + "unsigned": {"age_ts": 1, "replaces_state": "2"}, }, RoomVersions.V1, {"txn_id": "txn"}, @@ -634,10 +640,14 @@ def test_unsigned_is_copied(self) -> None: self.assertEqual(original.internal_metadata.instance_name, "worker1") cloned = clone_event(original) - cloned.unsigned["b"] = 3 + cloned.unsigned["age_ts"] = 3 - self.assertEqual(original.unsigned, {"a": 1, "b": 2}) - self.assertEqual(cloned.unsigned, {"a": 1, "b": 3}) + self.assertEqual( + original.unsigned.for_event(), {"age_ts": 1, "replaces_state": "2"} + ) + self.assertEqual( + cloned.unsigned.for_event(), {"age_ts": 3, "replaces_state": "2"} + ) self.assertEqual(cloned.internal_metadata.stream_ordering, 1234) self.assertEqual(cloned.internal_metadata.instance_name, "worker1") self.assertEqual(cloned.internal_metadata.txn_id, "txn") diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index a40e0b06807..7b1a2a5adc5 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -754,9 +754,9 @@ def test_strip_event_maintains_allowed_fields(self) -> None: }, } - filtered_event2 = event_from_pdu_json(event2, RoomVersions.V1) - self.assertIn("age", filtered_event2.unsigned) - self.assertEqual(14, filtered_event2.unsigned["age"]) + filtered_event2 = event_from_pdu_json(event2, RoomVersions.V1, received_time=20) + self.assertIn("age_ts", filtered_event2.unsigned) + self.assertEqual(6, filtered_event2.unsigned["age_ts"]) self.assertNotIn("more warez", filtered_event2.unsigned) # Invite_room_state is allowed in events of type m.room.member self.assertIn("invite_room_state", filtered_event2.unsigned) @@ -779,8 +779,8 @@ def test_strip_event_removes_fields_based_on_event_type(self) -> None: "invite_room_state": [], }, } - filtered_event3 = event_from_pdu_json(event3, RoomVersions.V1) - self.assertIn("age", filtered_event3.unsigned) + filtered_event3 = event_from_pdu_json(event3, RoomVersions.V1, received_time=20) + self.assertIn("age_ts", filtered_event3.unsigned) # Invite_room_state field is only permitted in event type m.room.member self.assertNotIn("invite_room_state", filtered_event3.unsigned) self.assertNotIn("more warez", filtered_event3.unsigned) From 0eefdbcb9535d6595ca2b419217404acf57623ac Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 5 May 2026 08:40:17 -0500 Subject: [PATCH 14/92] fix: Cap `WorkerLock` timeout intervals to 60 seconds (#19394) Fixes the symptoms of https://github.com/element-hq/synapse/issues/19315 / https://github.com/element-hq/synapse/issues/19588 but not the underlying reason causing the number to grow so large in the first place. ``` ValueError: Exceeds the limit (4300 digits) for integer string conversion; use sys.set_int_max_str_digits() to increase the limit ``` Copied from the original pull request on [Famedly's Synapse repo](https://github.com/famedly/synapse/pull/221) (with some edits): Basing the time interval around a 5 seconds leaves a big window of waiting especially as this window is doubled each retry, when another worker could be making progress but can not. Right now, the retry interval in seconds looks like `[0.2, 5, 10, 20, 40, 80, 160, 320, (continues to double)]` after which logging should start about excessive times and (relatively quickly) end up with an extremely large retry interval with an unrealistic expectation past the heat death of the universe. 1 year in seconds = 31,536,000. With this change, retry intervals in seconds should look more like: ``` [ 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8, 25.6, 51.2, 60, < never goes higher than this ] ``` Logging about excessive wait times will start at 10 minutes.
Previous breakdown when we were using 15 minutes ``` [ 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8, 25.6, 51.2, 102.4, # 1.7 minutes 204.8, # 3.41 minutes 409.6, # 6.83 minutes 819.2, # 13.65 minutes < logging about excessive times will start here, 13th iteration 900, # 15 minutes < never goes higher than this ] ```
Further suggested work in this area could be to define the cap, the retry interval starting point and the multiplier depending on how frequently this lock should be checked. See data below for reasons why. Increasing the jitter range may also be a good idea --------- Co-authored-by: Eric Eastwood (cherry picked from commit 3f58bc50dfba5768ee43ce48c5e74c25ba0b078a) --- changelog.d/19394.bugfix | 1 + synapse/handlers/worker_lock.py | 117 +++++++++++++++++++------- tests/handlers/test_worker_lock.py | 130 +++++++++++++++++++++++++++++ 3 files changed, 216 insertions(+), 32 deletions(-) create mode 100644 changelog.d/19394.bugfix diff --git a/changelog.d/19394.bugfix b/changelog.d/19394.bugfix new file mode 100644 index 00000000000..4ca92cfb328 --- /dev/null +++ b/changelog.d/19394.bugfix @@ -0,0 +1 @@ +Capped the `WorkerLock` time out interval to a maximum of 60 seconds to prevent dealing with excessively long numbers. Contributed by Famedly. diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 1537a18cc05..51be3b5084a 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -54,6 +54,9 @@ # will not disappear under our feet as long as we don't delete the room. NEW_EVENT_DURING_PURGE_LOCK_NAME = "new_event_during_purge_lock" +WORKER_LOCK_MAX_RETRY_INTERVAL = Duration(seconds=60) +WORKER_LOCK_EXCESSIVE_WAITING_WARN_DURATION = Duration(minutes=10) + class WorkerLocksHandler: """A class for waiting on taking out locks, rather than using the storage @@ -206,9 +209,10 @@ class WaitingLock: lock_name: str lock_key: str write: bool | None + start_ts_ms: int = 0 deferred: "defer.Deferred[None]" = attr.Factory(defer.Deferred) _inner_lock: Lock | None = None - _retry_interval: float = 0.1 + _timeout_interval: float = 0.1 _lock_span: "opentracing.Scope" = attr.Factory( lambda: start_active_span("WaitingLock.lock") ) @@ -220,6 +224,7 @@ def release_lock(self) -> None: self.deferred.callback(None) async def __aenter__(self) -> None: + self.start_ts_ms = self.clock.time_msec() self._lock_span.__enter__() with start_active_span("WaitingLock.waiting_for_lock"): @@ -240,19 +245,44 @@ async def __aenter__(self) -> None: break try: - # Wait until the we get notified the lock might have been + # Wait until the notification that the lock might have been # released (by the deferred being resolved). We also - # periodically wake up in case the lock was released but we + # periodically wake up in case the lock was released, but we # weren't notified. with PreserveLoggingContext(): - timeout = self._get_next_retry_interval() await timeout_deferred( deferred=self.deferred, - timeout=timeout, + timeout=self._timeout_interval, clock=self.clock, ) - except Exception: - pass + except defer.TimeoutError: + # Only increment the timeout value if this was an actual timeout + # (defer.TimeoutError) + self._increment_timeout_interval() + + now_ms = self.clock.time_msec() + time_spent_trying_to_lock = Duration( + milliseconds=now_ms - self.start_ts_ms + ) + if ( + time_spent_trying_to_lock.as_millis() + > WORKER_LOCK_EXCESSIVE_WAITING_WARN_DURATION.as_millis() + ): + logger.warning( + "(WaitingLock (%s, %s)) Time spent waiting to acquire lock " + "is getting excessive: %ss. There may be a deadlock.", + self.lock_name, + self.lock_key, + time_spent_trying_to_lock.as_secs(), + ) + + except Exception as e: + logger.warning( + "Caught an exception while waiting on WaitingLock(lock_name=%s, lock_key=%s): %r", + self.lock_name, + self.lock_key, + e, + ) return await self._inner_lock.__aenter__() @@ -273,15 +303,14 @@ async def __aexit__( return r - def _get_next_retry_interval(self) -> float: - next = self._retry_interval - self._retry_interval = max(5, next * 2) - if self._retry_interval > Duration(minutes=10).as_secs(): # >7 iterations - logger.warning( - "Lock timeout is getting excessive: %ss. There may be a deadlock.", - self._retry_interval, - ) - return next * random.uniform(0.9, 1.1) + def _increment_timeout_interval(self) -> float: + next_interval = self._timeout_interval + next_interval = min(WORKER_LOCK_MAX_RETRY_INTERVAL.as_secs(), next_interval * 2) + + # The jitter value is maintained for the timeout, to help avoid a "thundering + # herd" situation when all locks may time out at the same time. + self._timeout_interval = next_interval * random.uniform(0.9, 1.1) + return self._timeout_interval @attr.s(auto_attribs=True, eq=False) @@ -294,10 +323,11 @@ class WaitingMultiLock: store: LockStore handler: WorkerLocksHandler + start_ts_ms: int = 0 deferred: "defer.Deferred[None]" = attr.Factory(defer.Deferred) _inner_lock_cm: AsyncContextManager | None = None - _retry_interval: float = 0.1 + _timeout_interval: float = 0.1 _lock_span: "opentracing.Scope" = attr.Factory( lambda: start_active_span("WaitingLock.lock") ) @@ -309,6 +339,7 @@ def release_lock(self) -> None: self.deferred.callback(None) async def __aenter__(self) -> None: + self.start_ts_ms = self.clock.time_msec() self._lock_span.__enter__() with start_active_span("WaitingLock.waiting_for_lock"): @@ -324,19 +355,42 @@ async def __aenter__(self) -> None: break try: - # Wait until the we get notified the lock might have been + # Wait until the notification that the lock might have been # released (by the deferred being resolved). We also - # periodically wake up in case the lock was released but we + # periodically wake up in case the lock was released, but we # weren't notified. with PreserveLoggingContext(): - timeout = self._get_next_retry_interval() await timeout_deferred( deferred=self.deferred, - timeout=timeout, + timeout=self._timeout_interval, clock=self.clock, ) - except Exception: - pass + except defer.TimeoutError: + # Only increment the timeout value if this was an actual timeout + # (defer.TimeoutError) + self._increment_timeout_interval() + + now_ms = self.clock.time_msec() + time_spent_trying_to_lock = Duration( + milliseconds=now_ms - self.start_ts_ms + ) + if ( + time_spent_trying_to_lock.as_millis() + > WORKER_LOCK_EXCESSIVE_WAITING_WARN_DURATION.as_millis() + ): + logger.warning( + "(WaitingMultiLock (%r)) Time spent waiting to acquire lock " + "is getting excessive: %ss. There may be a deadlock.", + self.lock_names, + time_spent_trying_to_lock.as_secs(), + ) + + except Exception as e: + logger.warning( + "Caught an exception while waiting on WaitingMultiLock(lock_names=%r): %r", + self.lock_names, + e, + ) assert self._inner_lock_cm await self._inner_lock_cm.__aenter__() @@ -360,12 +414,11 @@ async def __aexit__( return r - def _get_next_retry_interval(self) -> float: - next = self._retry_interval - self._retry_interval = max(5, next * 2) - if self._retry_interval > Duration(minutes=10).as_secs(): # >7 iterations - logger.warning( - "Lock timeout is getting excessive: %ss. There may be a deadlock.", - self._retry_interval, - ) - return next * random.uniform(0.9, 1.1) + def _increment_timeout_interval(self) -> float: + next_interval = self._timeout_interval + next_interval = min(WORKER_LOCK_MAX_RETRY_INTERVAL.as_secs(), next_interval * 2) + + # The jitter value is maintained for the timeout, to help avoid a "thundering + # herd" situation when all locks may time out at the same time. + self._timeout_interval = next_interval * random.uniform(0.9, 1.1) + return self._timeout_interval diff --git a/tests/handlers/test_worker_lock.py b/tests/handlers/test_worker_lock.py index 61ff51ff923..74201f41515 100644 --- a/tests/handlers/test_worker_lock.py +++ b/tests/handlers/test_worker_lock.py @@ -26,7 +26,9 @@ from twisted.internet.testing import MemoryReactor from synapse.server import HomeServer +from synapse.storage.databases.main.lock import _RENEWAL_INTERVAL from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests import unittest from tests.replication._base import BaseMultiWorkerStreamTestCase @@ -40,6 +42,7 @@ def prepare( self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer ) -> None: self.worker_lock_handler = self.hs.get_worker_locks_handler() + self.store = self.hs.get_datastores().main def test_wait_for_lock_locally(self) -> None: """Test waiting for a lock on a single worker""" @@ -56,6 +59,66 @@ def test_wait_for_lock_locally(self) -> None: self.get_success(d2) self.get_success(lock2.__aexit__(None, None, None)) + def test_timeouts_for_lock_locally(self) -> None: + """ + Test that we regularly retry to reacquire locks. + + This is a regression test to make sure the lock retry time doesn't balloon to a value + so large it can't even be printed reliably anymore. + """ + + # Create and acquire the first lock + lock1 = self.worker_lock_handler.acquire_lock("name", "key") + self.get_success(lock1.__aenter__()) + + # Create and try to acquire the second lock + lock2 = self.worker_lock_handler.acquire_lock("name", "key") + d2 = defer.ensureDeferred(lock2.__aenter__()) + # Make sure we haven't acquired the lock yet (`lock1` still holds it) + self.assertNoResult(d2) + + # Advance time by an hour (some duration that would previously cause our timeout + # to balloon if it weren't constrained). Max back-off (saturate) + # + # Note: We use `_pump_by` instead of `pump`/`advance` as the `Lock` has an + # internal background looping call that runs every 30 seconds + # (`_RENEWAL_INTERVAL`) to renew the `Lock` and push it's "drop timeout" value + # further out by 2 minutes (`_LOCK_TIMEOUT_MS`). The `Lock` will prematurely + # drop if this renewal is not allowed to run, which sours the test. + # self.pump(amount=Duration(hours=1)) + self._pump_by(amount=Duration(hours=1), by=_RENEWAL_INTERVAL) + + # Make sure we haven't acquired the `lock2` yet (`lock1` still holds it) + self.assertNoResult(d2) + + # Release the first lock (`lock1`). The second lock(`lock2`) should be + # automatically acquired by the `pump()` inside `get_success()` + self.get_success(lock1.__aexit__(None, None, None)) + + # We should now have the lock + self.successResultOf(d2) + + def _pump_by( + self, + *, + amount: Duration = Duration(seconds=0), + by: Duration = Duration(seconds=0.1), + ) -> None: + """ + Like `self.pump()` but you can specify the time increment to advance with until + you reach the time amount. + + Unlike `self.pump()`, this doesn't multiply the time at all. + + Args: + amount: The amount of time to advance + by: The time increment in seconds to advance time by until we reach the `amount` + """ + end_time_s = self.reactor.seconds() + amount.as_secs() + + while self.reactor.seconds() < end_time_s: + self.reactor.advance(by.as_secs()) + def test_lock_contention(self) -> None: """Test lock contention when a lot of locks wait on a single worker""" nb_locks_to_test = 500 @@ -124,3 +187,70 @@ def test_wait_for_lock_worker(self) -> None: self.get_success(d2) self.get_success(lock2.__aexit__(None, None, None)) + + def test_timeouts_for_lock_worker(self) -> None: + """ + Test that we regularly retry to reacquire locks. + + This is a regression test to make sure the lock retry time doesn't balloon to a value + so large it can't even be printed reliably anymore. + """ + worker = self.make_worker_hs( + "synapse.app.generic_worker", + extra_config={ + "redis": {"enabled": True}, + }, + ) + worker_lock_handler = worker.get_worker_locks_handler() + + # Create and acquire the first lock on the main process + lock1 = self.main_worker_lock_handler.acquire_lock("name", "key") + self.get_success(lock1.__aenter__()) + + # Create and try to acquire the second lock on the worker + lock2 = worker_lock_handler.acquire_lock("name", "key") + d2 = defer.ensureDeferred(lock2.__aenter__()) + # Make sure we haven't acquired the lock yet (`lock1` still holds it) + self.assertNoResult(d2) + + # Advance time by an hour (some duration that would previously cause our timeout + # to balloon if it weren't constrained). Max back-off (saturate) + # + # Note: We use `_pump_by` instead of `pump`/`advance` as the `Lock` has an + # internal background looping call that runs every 30 seconds + # (`_RENEWAL_INTERVAL`) to renew the `Lock` and push it's "drop timeout" value + # further out by 2 minutes (`_LOCK_TIMEOUT_MS`). The `Lock` will prematurely + # drop if this renewal is not allowed to run, which sours the test. + # self.pump(amount=Duration(hours=1)) + self._pump_by(amount=Duration(hours=1), by=_RENEWAL_INTERVAL) + + # Make sure we haven't acquired the `lock2` yet (`lock1` still holds it) + self.assertNoResult(d2) + + # Release the first lock (`lock1`). The second lock(`lock2`) should be + # automatically acquired by the `pump()` inside `get_success()` + self.get_success(lock1.__aexit__(None, None, None)) + + # We should now have the lock + self.successResultOf(d2) + + def _pump_by( + self, + *, + amount: Duration = Duration(seconds=0), + by: Duration = Duration(seconds=0.1), + ) -> None: + """ + Like `self.pump()` but you can specify the time increment to advance with until + you reach the time amount. + + Unlike `self.pump()`, this doesn't multiply the time at all. + + Args: + amount: The amount of time to advance + by: The time increment in seconds to advance time by until we reach the `amount` + """ + end_time_s = self.reactor.seconds() + amount.as_secs() + + while self.reactor.seconds() < end_time_s: + self.reactor.advance(by.as_secs()) From 2d48851438d41bf6e248c7a743c8ffd61c2e1ba3 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 5 May 2026 17:12:11 +0100 Subject: [PATCH 15/92] Prevent pagination ending when a page is full of rejected events (ELEMENTSEC-2025-1636) Fixes: https://github.com/element-hq/synapse/security/advisories/GHSA-6qf2-7x63-mm6v Reviewed-on: https://github.com/element-hq/synapse-private/pull/117 --- synapse/handlers/pagination.py | 11 +-- synapse/storage/databases/main/stream.py | 13 +++- tests/rest/client/test_rooms.py | 86 +++++++++++++++++++++++- 3 files changed, 101 insertions(+), 9 deletions(-) diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 8cbe4b63c88..2bc7efeb5eb 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -566,7 +566,7 @@ async def get_messages( ( events, next_key, - _, + limited, ) = await self.store.paginate_room_events_by_topological_ordering( room_id=room_id, from_key=from_token.room_key, @@ -645,7 +645,7 @@ async def get_messages( ( events, next_key, - _, + limited, ) = await self.store.paginate_room_events_by_topological_ordering( room_id=room_id, from_key=from_token.room_key, @@ -668,11 +668,12 @@ async def get_messages( next_token = from_token.copy_and_replace(StreamKeyType.ROOM, next_key) - # if no events are returned from pagination, that implies - # we have reached the end of the available events. + # if no events are returned from pagination (this page is empty) + # and there aren't any more pages (not limited), + # that implies we have reached the end of the available events. # In that case we do not return end, to tell the client # there is no need for further queries. - if not events: + if not limited and not events: return GetMessagesResult( messages_chunk=[], bundled_aggregations={}, diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index 8fa1e2e5a97..7d14f9f4d80 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -2425,12 +2425,19 @@ async def paginate_room_events_by_topological_ordering( event_filter: If provided filters the events to those that match the filter. Returns: - The results as a list of events, a token that points to the end of - the result set, and a boolean to indicate if there were more events - but we hit the limit. If no events are returned then the end of the + - The results as a list of events; + - a token that points to the end of the result set; and + - a boolean to indicate if there were more events + but we hit the limit (`limited`) + + If no events are returned and `limited` is false, then the end of the stream has been reached (i.e. there are no events between `from_key` and `to_key`). + When `limited` is true, that means that more pagination can be attempted. + Note that `limited` can be true even if no events are returned, + because rejected events are filtered out after the limit check. + When Direction.FORWARDS: from_key < x <= to_key, (ascending order) When Direction.BACKWARDS: from_key >= x > to_key, (descending order) """ diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 221121007da..61e7e87f623 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -66,7 +66,10 @@ from tests import unittest from tests.http.server._base import make_request_with_cancellation_test from tests.storage.test_stream import PaginationTestCase -from tests.test_utils.event_injection import create_event +from tests.test_utils.event_injection import ( + create_event, + inject_event, +) from tests.unittest import override_config from tests.utils import default_config @@ -2371,6 +2374,87 @@ def test_room_message_filter_query_validation(self) -> None: channel.json_body["errcode"], Codes.NOT_JSON, channel.json_body ) + def test_room_messages_paginate_through_rejected_events( + self, + ) -> None: + """Test that pagination continues past a batch of rejected events. + + Regression test for https://github.com/element-hq/synapse/security/advisories/GHSA-6qf2-7x63-mm6v + + Synapse before 1.152.1 had a bug meaning that a batch full of only + rejected events would cause `/messages` to not return any more + pagination tokens, falsely signalling the end of backpagination. + """ + # Send an early message that should not be filtered. + early_event_id = self.helper.send(self.room_id, "early message")["event_id"] + + # Inject a batch of events and mark them as rejected in the database. + # We create more events than a single pagination request would fetch, + # so that one page of backward pagination request would only see rejected events. + for _ in range(3): + event = self.get_success( + inject_event( + self.hs, + room_id=self.room_id, + sender=self.user_id, + type=EventTypes.Message, + content={"body": "filtered event", "msgtype": "m.text"}, + ) + ) + self.get_success( + self.hs.get_datastores().main.db_pool.runInteraction( + "mark_rejected", + self.hs.get_datastores().main.mark_event_rejected_txn, + event.event_id, + "testing", + ) + ) + + # Send a message after all the rejected events. + latest_event_id = self.helper.send(self.room_id, "latest message")["event_id"] + + # Start backpaginating. + channel = self.make_request( + "GET", f"/rooms/{self.room_id}/messages?dir=b&limit=2" + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + + events_in_page = [e["event_id"] for e in channel.json_body["chunk"]] + end_token: str | None = channel.json_body["end"] + + self.assertEqual( + events_in_page, + [latest_event_id], + "The latest event should be included in the first page we see whilst backpaginating", + ) + + event_ids_in_pages: list[list[str]] = [events_in_page] + + # Bound the number of backpagination attempts to 2 + for _ in range(2): + channel = self.make_request( + "GET", f"/rooms/{self.room_id}/messages?from={end_token}&dir=b&limit=2" + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + events_in_page = [e["event_id"] for e in channel.json_body["chunk"]] + event_ids_in_pages.append(events_in_page) + + if early_event_id in events_in_page: + # We have found the event we were looking for + return + + self.assertIn( + "end", + channel.json_body, + f"No `end` token received. Did not find {early_event_id} whilst backpaginating ({latest_event_id = }, {event_ids_in_pages = })", + ) + # Use the end_token in the next iteration + end_token = channel.json_body["end"] + + self.fail( + f"Exhausted backpagination attempts. Did not find {early_event_id} whilst backpaginating ({latest_event_id = }, {event_ids_in_pages = })" + ) + class RoomMessageFilterTestCase(RoomBase): """Tests /rooms/$room_id/messages REST events.""" From d97b5b9e21681792d78c5d58243fc483c664b66e Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Thu, 7 May 2026 13:49:49 +0100 Subject: [PATCH 16/92] 1.152.1 --- CHANGES.md | 8 ++++++++ changelog.d/19394.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 15 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/19394.bugfix diff --git a/CHANGES.md b/CHANGES.md index 3425bcca8c5..d9b3f8b2c12 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,11 @@ +# Synapse 1.152.1 (2026-05-07) + +## Security Fixes + +- Prevent CPU starvation (Denial of Service) under worker lock contention, additionally capping the `WorkerLock` time out interval to a maximum of 60 seconds. Contributed by Famedly. ([\#19394](https://github.com/element-hq/synapse/issues/19394), ELEMENTSEC-2026-1706, [GHSA-8q93-326v-3m7g](https://github.com/element-hq/synapse/security/advisories/GHSA-8q93-326v-3m7g), CVE pending) +- Prevent pagination ending when a page is full of rejected events. (ELEMENTSEC-2025-1636, [GHSA-6qf2-7x63-mm6v](https://github.com/element-hq/synapse/security/advisories/GHSA-6qf2-7x63-mm6v), CVE pending) + + # Synapse 1.152.0 (2026-04-28) No significant changes since 1.152.0rc1. diff --git a/changelog.d/19394.bugfix b/changelog.d/19394.bugfix deleted file mode 100644 index 4ca92cfb328..00000000000 --- a/changelog.d/19394.bugfix +++ /dev/null @@ -1 +0,0 @@ -Capped the `WorkerLock` time out interval to a maximum of 60 seconds to prevent dealing with excessively long numbers. Contributed by Famedly. diff --git a/debian/changelog b/debian/changelog index ff9dfe3e13d..cfefe953e3c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.152.1) stable; urgency=medium + + * New Synapse release 1.152.1. + + -- Synapse Packaging team Thu, 07 May 2026 13:29:05 +0100 + matrix-synapse-py3 (1.152.0) stable; urgency=medium * New Synapse release 1.152.0. diff --git a/pyproject.toml b/pyproject.toml index 9509609ff3b..e9ae6e204dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.152.0" +version = "1.152.1" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 2829a146d32e472fcd2885ed32cb3bbea6f86690 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 7 May 2026 10:36:25 -0500 Subject: [PATCH 17/92] Reduce `WORKER_LOCK_MAX_RETRY_INTERVAL` to 5 seconds (#19755) Better to retry more quickly than have workers wait around. 5 seconds is still a reasonable gap in time to not overwhelm anything. This matters most in cross-worker scenarios. When locks are on the same worker, when the lock holder releases, we signal to other locks (with the same name/key) that they should try reacquiring the lock immediately. But locks on other workers only re-check based on their retry `_timeout_interval`. Updating to 5 seconds to match the previous intentions based on the [flawed code](https://github.com/element-hq/synapse/blob/6100f6e4f7fb0c72f1ae2802683ebc811c0e3a77/synapse/handlers/worker_lock.py#L278). We can assume they were trying to have 5 seconds as the max value to retry. Spawning from https://github.com/element-hq/synapse/pull/19394#discussion_r3168458070 --- changelog.d/19755.misc | 1 + synapse/handlers/worker_lock.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19755.misc diff --git a/changelog.d/19755.misc b/changelog.d/19755.misc new file mode 100644 index 00000000000..6ad478e531c --- /dev/null +++ b/changelog.d/19755.misc @@ -0,0 +1 @@ +Reduce `WORKER_LOCK_MAX_RETRY_INTERVAL` to 5 seconds to reduce idle time after lock is released. diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 51be3b5084a..57792ea53cf 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -54,7 +54,18 @@ # will not disappear under our feet as long as we don't delete the room. NEW_EVENT_DURING_PURGE_LOCK_NAME = "new_event_during_purge_lock" -WORKER_LOCK_MAX_RETRY_INTERVAL = Duration(seconds=60) +WORKER_LOCK_MAX_RETRY_INTERVAL = Duration(seconds=5) +""" +The maximum wait time before retrying to acquire the lock. + +Better to retry more quickly than have workers wait around. 5 seconds is still a +reasonable gap in time to not overwhelm the CPU/Database. + +This matters most in cross-worker scenarios. When locks are on the same worker, when the +lock holder releases, we signal to other locks (with the same name/key) that they +should try reacquiring the lock immediately. But locks on other workers only re-check +based on their retry `_timeout_interval`. +""" WORKER_LOCK_EXCESSIVE_WAITING_WARN_DURATION = Duration(minutes=10) From 4911296fb5e21649aa7d205b2ad9fcc43016d831 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 7 May 2026 10:38:56 -0500 Subject: [PATCH 18/92] Force keyword-only args for `Duration` (prevent footgun) (#19756) So people have to specify which time unit they want to use. Spawning from https://github.com/element-hq/synapse/pull/19394#discussion_r3188418426 --- changelog.d/19756.misc | 1 + synapse/util/duration.py | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 changelog.d/19756.misc diff --git a/changelog.d/19756.misc b/changelog.d/19756.misc new file mode 100644 index 00000000000..2450505b531 --- /dev/null +++ b/changelog.d/19756.misc @@ -0,0 +1 @@ +Force keyword-only args for `Duration` (prevent footgun) so people have to specify which time unit they want to us. diff --git a/synapse/util/duration.py b/synapse/util/duration.py index 135b9808520..a1abe944b53 100644 --- a/synapse/util/duration.py +++ b/synapse/util/duration.py @@ -32,6 +32,33 @@ class Duration(timedelta): ``` """ + # Using `__new__` (instead of `__init__`) because that's what `timedelta` uses + def __new__( + cls, + # The whole goal of overriding `__new__` is to require keyword-only arguments. + # Without this, `Duration(5)` would create a duration represnting 5 *days* + # (timedelta's default), but callers almost certainly want to specify which unit + # like seconds or hours. + *, + days: float = 0, + seconds: float = 0, + microseconds: float = 0, + milliseconds: float = 0, + minutes: float = 0, + hours: float = 0, + weeks: float = 0, + ) -> "Duration": + return super().__new__( + cls, + days=days, + seconds=seconds, + microseconds=microseconds, + milliseconds=milliseconds, + minutes=minutes, + hours=hours, + weeks=weeks, + ) + def as_millis(self) -> int: """Returns the duration in milliseconds.""" return int(self / _ONE_MILLISECOND) From 8dbbc4000b347ed1e4bca494e36dc11c37dd865d Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 8 May 2026 06:20:25 -0500 Subject: [PATCH 19/92] Commit stray Rust change that keeps popping up (`rust/src/canonical_json.rs`) (#19763) (introduced in https://github.com/element-hq/synapse/pull/19739) Seems like some automatic change from `poetry run ./scripts-dev/lint.sh` --- changelog.d/19763.misc | 1 + rust/src/canonical_json.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19763.misc diff --git a/changelog.d/19763.misc b/changelog.d/19763.misc new file mode 100644 index 00000000000..c9e8277e018 --- /dev/null +++ b/changelog.d/19763.misc @@ -0,0 +1 @@ +Lint and format `rust/src/canonical_json.rs`. diff --git a/rust/src/canonical_json.rs b/rust/src/canonical_json.rs index ff1fcd3ee41..4abd0018479 100644 --- a/rust/src/canonical_json.rs +++ b/rust/src/canonical_json.rs @@ -824,7 +824,7 @@ mod tests { // Serialize the keys in the reverse order. for (c, _) in ascii_order.iter().rev() { - map_serializer.serialize_entry(c.into(), &1).unwrap(); + map_serializer.serialize_entry(c, &1).unwrap(); } SerializeMap::end(map_serializer).unwrap(); From 0e508ba80f157e64aab01bb5fca56a9225d4f540 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 8 May 2026 13:22:15 +0100 Subject: [PATCH 20/92] 1.153.0rc1 --- CHANGES.md | 32 +++++++++++++++++++++++++++++++ changelog.d/18475.feature | 1 - changelog.d/19394.bugfix | 1 - changelog.d/19398.bugfix | 1 - changelog.d/19611.bugfix | 1 - changelog.d/19706.misc | 1 - changelog.d/19708.misc | 1 - changelog.d/19711.doc | 1 - changelog.d/19714.bugfix | 1 - changelog.d/19720.feature | 1 - changelog.d/19722.feature | 1 - changelog.d/19727.bugfix | 1 - changelog.d/19737.feature | 1 - changelog.d/19739.misc | 1 - changelog.d/19742.bugfix | 1 - changelog.d/19743.misc | 1 - changelog.d/19755.misc | 1 - changelog.d/19756.misc | 1 - changelog.d/19763.misc | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- schema/synapse-config.schema.yaml | 2 +- 22 files changed, 40 insertions(+), 20 deletions(-) delete mode 100644 changelog.d/18475.feature delete mode 100644 changelog.d/19394.bugfix delete mode 100644 changelog.d/19398.bugfix delete mode 100644 changelog.d/19611.bugfix delete mode 100644 changelog.d/19706.misc delete mode 100644 changelog.d/19708.misc delete mode 100644 changelog.d/19711.doc delete mode 100644 changelog.d/19714.bugfix delete mode 100644 changelog.d/19720.feature delete mode 100644 changelog.d/19722.feature delete mode 100644 changelog.d/19727.bugfix delete mode 100644 changelog.d/19737.feature delete mode 100644 changelog.d/19739.misc delete mode 100644 changelog.d/19742.bugfix delete mode 100644 changelog.d/19743.misc delete mode 100644 changelog.d/19755.misc delete mode 100644 changelog.d/19756.misc delete mode 100644 changelog.d/19763.misc diff --git a/CHANGES.md b/CHANGES.md index d9b3f8b2c12..da26f17606a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,35 @@ +# Synapse 1.153.0rc1 (2026-05-08) + +## Features + +- Make ACLs apply to EDUs per [MSC4163](https://github.com/matrix-org/matrix-spec-proposals/pull/4163). ([\#18475](https://github.com/element-hq/synapse/issues/18475)) +- Stabilize [MSC3266: Room summary API](https://github.com/matrix-org/matrix-spec-proposals/pull/3266), removing the experimental config flag `msc3266_enabled`. Contributed by @dasha-uwu. ([\#19720](https://github.com/element-hq/synapse/issues/19720)) +- Partial [MSC4311](https://github.com/matrix-org/matrix-spec-proposals/pull/4311) implementation: `m.room.create` is now a required part of stripped `invite_state`/`knock_state` . Contributed by @FrenchGithubUser @Famedly. ([\#19722](https://github.com/element-hq/synapse/issues/19722)) +- Expose `tombstoned` and `replacement_room` in room details on admin API endpoint `GET /_synapse/admin/v1/rooms/`. Contributed by Noah Markert. ([\#19737](https://github.com/element-hq/synapse/issues/19737)) + +## Bugfixes + +- Allow self-requested user erasure (upon account deactivation) to succeed even if Synapse has disabled profile changes. Contributed by Famedly. ([\#19398](https://github.com/element-hq/synapse/issues/19398)) +- Fix Synapse not backfilling new history when attempting to use a pagination token near a backward extremity. ([\#19611](https://github.com/element-hq/synapse/issues/19611)) +- Have [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) return a new response immediately if a room subscription has changed and produced a new response. ([\#19714](https://github.com/element-hq/synapse/issues/19714)) +- Fix a bug where when upgrading a room to room version 12, the power level event in the old room got temporarily mutated to remove the user upgrading the room's power. ([\#19727](https://github.com/element-hq/synapse/issues/19727)) +- Fix packaging for Fedora and EPEL caused by unnecessary bumping `authlib` minimum version requirement in `pyproject.toml` file. Contributed by Oleg Girko. ([\#19742](https://github.com/element-hq/synapse/issues/19742)) + +## Improved Documentation + +- Add warning about known problems when configuring `use_frozen_dicts`. ([\#19711](https://github.com/element-hq/synapse/issues/19711)) + +## Internal Changes + +- Port `Event.signatures` field to Rust. ([\#19706](https://github.com/element-hq/synapse/issues/19706)) +- Port `Event.unsigned` field to Rust. ([\#19708](https://github.com/element-hq/synapse/issues/19708)) +- Add a Rust canonical JSON serializer. ([\#19739](https://github.com/element-hq/synapse/issues/19739)) +- Configure Dependabot to only update Python dependencies in the lockfile, unless widening upper bounds. ([\#19743](https://github.com/element-hq/synapse/issues/19743)) +- Reduce `WORKER_LOCK_MAX_RETRY_INTERVAL` to 5 seconds to reduce idle time after lock is released. ([\#19755](https://github.com/element-hq/synapse/issues/19755)) +- Force keyword-only arguments for `Duration` so time units have to be specified. ([\#19756](https://github.com/element-hq/synapse/issues/19756)) +- Lint and format `rust/src/canonical_json.rs`. ([\#19763](https://github.com/element-hq/synapse/issues/19763)) + + # Synapse 1.152.1 (2026-05-07) ## Security Fixes diff --git a/changelog.d/18475.feature b/changelog.d/18475.feature deleted file mode 100644 index 06c13db43e5..00000000000 --- a/changelog.d/18475.feature +++ /dev/null @@ -1 +0,0 @@ -Make ACLs apply to EDUs per [MSC4163](https://github.com/matrix-org/matrix-spec-proposals/pull/4163). diff --git a/changelog.d/19394.bugfix b/changelog.d/19394.bugfix deleted file mode 100644 index 4ca92cfb328..00000000000 --- a/changelog.d/19394.bugfix +++ /dev/null @@ -1 +0,0 @@ -Capped the `WorkerLock` time out interval to a maximum of 60 seconds to prevent dealing with excessively long numbers. Contributed by Famedly. diff --git a/changelog.d/19398.bugfix b/changelog.d/19398.bugfix deleted file mode 100644 index 07679b31aec..00000000000 --- a/changelog.d/19398.bugfix +++ /dev/null @@ -1 +0,0 @@ -Allow user requested erasure to succeed even if Synapse has disabled profile changes. Contributed by Famedly. diff --git a/changelog.d/19611.bugfix b/changelog.d/19611.bugfix deleted file mode 100644 index 4952fd00db5..00000000000 --- a/changelog.d/19611.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix Synapse not backfilling new history when attempting to use a pagination token near a backward extremity. diff --git a/changelog.d/19706.misc b/changelog.d/19706.misc deleted file mode 100644 index 205abd09d46..00000000000 --- a/changelog.d/19706.misc +++ /dev/null @@ -1 +0,0 @@ -Port `Event.signatures` field to Rust. diff --git a/changelog.d/19708.misc b/changelog.d/19708.misc deleted file mode 100644 index 308c2b04d0d..00000000000 --- a/changelog.d/19708.misc +++ /dev/null @@ -1 +0,0 @@ -Port `Event.unsigned` field to Rust. diff --git a/changelog.d/19711.doc b/changelog.d/19711.doc deleted file mode 100644 index d00ee6a908e..00000000000 --- a/changelog.d/19711.doc +++ /dev/null @@ -1 +0,0 @@ -Add warning about known problems when configuring `use_frozen_dicts`. diff --git a/changelog.d/19714.bugfix b/changelog.d/19714.bugfix deleted file mode 100644 index 6aba7b21a61..00000000000 --- a/changelog.d/19714.bugfix +++ /dev/null @@ -1 +0,0 @@ -Have SSS return a new response immediately if a room subscription have changed and produced a new response. diff --git a/changelog.d/19720.feature b/changelog.d/19720.feature deleted file mode 100644 index 97a1d35de5f..00000000000 --- a/changelog.d/19720.feature +++ /dev/null @@ -1 +0,0 @@ -Stabilize MSC3266, removing the experimental config flag `msc3266_enabled`. Add support for stable room summary endpoints. Contributed by @dasha-uwu. diff --git a/changelog.d/19722.feature b/changelog.d/19722.feature deleted file mode 100644 index 30104b7e74d..00000000000 --- a/changelog.d/19722.feature +++ /dev/null @@ -1 +0,0 @@ -Partial [MSC4311](https://github.com/matrix-org/matrix-spec-proposals/pull/4311) implementation: `m.room.create` is now a required part of stripped `invite_state`/`knock_state` . Contributed by @FrenchGithubUser @Famedly. diff --git a/changelog.d/19727.bugfix b/changelog.d/19727.bugfix deleted file mode 100644 index a535bd6aa4b..00000000000 --- a/changelog.d/19727.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where when upgrading a room to v12 the power level event in the old room got mutated to remove the user upgrading the room's power. diff --git a/changelog.d/19737.feature b/changelog.d/19737.feature deleted file mode 100644 index 13bf2405df3..00000000000 --- a/changelog.d/19737.feature +++ /dev/null @@ -1 +0,0 @@ -Exposes `tombstoned` and `replacement_room` in room details on admin API endpoint `GET /_synapse/admin/v1/rooms/`. Contributed by Noah Markert. diff --git a/changelog.d/19739.misc b/changelog.d/19739.misc deleted file mode 100644 index 24562b24fcd..00000000000 --- a/changelog.d/19739.misc +++ /dev/null @@ -1 +0,0 @@ -Add a Rust canonical JSON serializer. diff --git a/changelog.d/19742.bugfix b/changelog.d/19742.bugfix deleted file mode 100644 index 342769b65b7..00000000000 --- a/changelog.d/19742.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix packaging for Fedora and EPEL caused by unnecessary bumping `authlib` minimum version requirement in `pyproject.toml` file. Contributed by Oleg Girko. diff --git a/changelog.d/19743.misc b/changelog.d/19743.misc deleted file mode 100644 index 35c4841386e..00000000000 --- a/changelog.d/19743.misc +++ /dev/null @@ -1 +0,0 @@ -Configure Dependabot to only update Python dependencies in the lockfile, unless widening upper bounds. diff --git a/changelog.d/19755.misc b/changelog.d/19755.misc deleted file mode 100644 index 6ad478e531c..00000000000 --- a/changelog.d/19755.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce `WORKER_LOCK_MAX_RETRY_INTERVAL` to 5 seconds to reduce idle time after lock is released. diff --git a/changelog.d/19756.misc b/changelog.d/19756.misc deleted file mode 100644 index 2450505b531..00000000000 --- a/changelog.d/19756.misc +++ /dev/null @@ -1 +0,0 @@ -Force keyword-only args for `Duration` (prevent footgun) so people have to specify which time unit they want to us. diff --git a/changelog.d/19763.misc b/changelog.d/19763.misc deleted file mode 100644 index c9e8277e018..00000000000 --- a/changelog.d/19763.misc +++ /dev/null @@ -1 +0,0 @@ -Lint and format `rust/src/canonical_json.rs`. diff --git a/debian/changelog b/debian/changelog index cfefe953e3c..6aa3735603a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.153.0~rc1) stable; urgency=medium + + * New Synapse release 1.153.0rc1. + + -- Synapse Packaging team Fri, 08 May 2026 13:05:08 +0100 + matrix-synapse-py3 (1.152.1) stable; urgency=medium * New Synapse release 1.152.1. diff --git a/pyproject.toml b/pyproject.toml index 7ead67c8f5d..b456adfc606 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.152.1" +version = "1.153.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 49207028d1d..8888d2c673a 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/schema/synapse/v1.152/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.153/synapse-config.schema.json type: object properties: modules: From eb2ae9d3da6eb765836e83c37781a2fbba98479e Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 8 May 2026 14:03:41 +0100 Subject: [PATCH 21/92] Tweak changelog --- CHANGES.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index da26f17606a..e7bd8f351ab 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -23,11 +23,10 @@ - Port `Event.signatures` field to Rust. ([\#19706](https://github.com/element-hq/synapse/issues/19706)) - Port `Event.unsigned` field to Rust. ([\#19708](https://github.com/element-hq/synapse/issues/19708)) -- Add a Rust canonical JSON serializer. ([\#19739](https://github.com/element-hq/synapse/issues/19739)) +- Add a Rust canonical JSON serializer. ([\#19739](https://github.com/element-hq/synapse/issues/19739), [\#19763](https://github.com/element-hq/synapse/issues/19763)) - Configure Dependabot to only update Python dependencies in the lockfile, unless widening upper bounds. ([\#19743](https://github.com/element-hq/synapse/issues/19743)) - Reduce `WORKER_LOCK_MAX_RETRY_INTERVAL` to 5 seconds to reduce idle time after lock is released. ([\#19755](https://github.com/element-hq/synapse/issues/19755)) - Force keyword-only arguments for `Duration` so time units have to be specified. ([\#19756](https://github.com/element-hq/synapse/issues/19756)) -- Lint and format `rust/src/canonical_json.rs`. ([\#19763](https://github.com/element-hq/synapse/issues/19763)) # Synapse 1.152.1 (2026-05-07) From c430c16df47229f8ecef6783739accc042fcafbe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 8 May 2026 14:19:03 +0100 Subject: [PATCH 22/92] Port event content to Rust (#19725) Based on #19708. This is on the path to porting the entire event class to Rust, as `event.content` will then return the new Rust class `JsonObject`. This PR adds a pure Rust `JsonObject` class that is a `Mapping` representing a json-style object. It uses `serde_json::Value` as its in-memory representation and `pythonize` for conversion when a field is looked up on the object. I'm not thrilled with the name, but couldn't think of a better one. This also adds `JsonObject` handling to the JSON serialisation functions we use, as well as to the `freeze(..)` function. Reviewable commit-by-commit. --- changelog.d/19725.misc | 1 + rust/src/events/json_object.rs | 488 +++++++++++++++++++++++ rust/src/events/mod.rs | 12 +- synapse/__init__.py | 9 +- synapse/events/__init__.py | 53 +-- synapse/events/utils.py | 2 +- synapse/events/validator.py | 4 +- synapse/handlers/room.py | 9 +- synapse/handlers/stats.py | 4 +- synapse/push/bulk_push_rule_evaluator.py | 4 +- synapse/synapse_rust/events.pyi | 11 +- synapse/util/events.py | 4 +- synapse/util/frozenutils.py | 5 + synapse/util/json.py | 30 +- tests/crypto/test_event_signing.py | 7 +- tests/module_api/test_api.py | 4 +- tests/rest/client/test_rooms.py | 4 +- tests/server.py | 3 +- tests/synapse_rust/test_json_object.py | 149 +++++++ tests/test_state.py | 5 +- tests/test_utils/__init__.py | 4 +- 21 files changed, 749 insertions(+), 63 deletions(-) create mode 100644 changelog.d/19725.misc create mode 100644 rust/src/events/json_object.rs create mode 100644 tests/synapse_rust/test_json_object.py diff --git a/changelog.d/19725.misc b/changelog.d/19725.misc new file mode 100644 index 00000000000..b320f42b9c5 --- /dev/null +++ b/changelog.d/19725.misc @@ -0,0 +1 @@ +Port `Event.content` field to Rust. diff --git a/rust/src/events/json_object.rs b/rust/src/events/json_object.rs new file mode 100644 index 00000000000..2c4be1c87bf --- /dev/null +++ b/rust/src/events/json_object.rs @@ -0,0 +1,488 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::{collections::BTreeMap, sync::Arc}; + +use pyo3::{ + exceptions::{PyKeyError, PyTypeError}, + pyclass, pymethods, + types::{ + PyAnyMethods, PyIterator, PyList, PyListMethods, PyMapping, PySet, PySetMethods, PyTuple, + }, + Bound, IntoPyObject, IntoPyObjectExt, Py, PyAny, PyResult, Python, +}; +use pythonize::{depythonize, pythonize}; +use serde::{Deserialize, Serialize}; + +/// A generic class for representing immutable JSON objects. +/// +/// This is used for representing the `content` field of an event. +/// +/// The basic architecture here is to optimize for two things: +/// 1. Fast access of top-level keys (e.g. `event.content["key"]`) +/// 2. Pure Rust implementation. +#[derive(Serialize, Deserialize, Clone, Default)] +#[pyclass(mapping, frozen)] +#[serde(transparent)] +pub struct JsonObject { + object: Arc, serde_json::Value>>, +} + +#[pymethods] +impl JsonObject { + #[new] + #[pyo3(signature = (content = None))] + fn new<'a, 'py>(content: Option<&'a Bound<'py, PyAny>>) -> PyResult { + let Some(content) = content else { + // If no content is provided, default to an empty object. + return Ok(Self::default()); + }; + + if let Ok(content) = content.cast::() { + // If the content is already a JsonObject, we can just clone the + // underlying map (this is safe as the object is immutable). + return Ok(JsonObject { + object: content.get().object.clone(), + }); + } + + let Ok(content) = content.cast::() else { + return Err(PyTypeError::new_err("'content' must be a mapping")); + }; + + // Use pythonize to try and convert from a mapping. + let content = depythonize(content)?; + Ok(Self { + object: Arc::new(content), + }) + } + + fn __len__(&self) -> usize { + self.object.len() + } + + fn __contains__(&self, key: &Bound<'_, PyAny>) -> bool { + // Match dict semantics: a non-string key is simply "not in" the + // mapping, rather than raising TypeError. + let Ok(key_str) = key.extract::<&str>() else { + return false; + }; + self.object.contains_key(key_str) + } + + fn __getitem__<'py>( + &self, + py: Python<'py>, + key: Bound<'_, PyAny>, + ) -> PyResult> { + // We only ever store string keys, so any non-string lookup is a miss. + // Raise KeyError (not TypeError) to match dict's behaviour. + let Ok(key_str) = key.extract::<&str>() else { + return Err(PyKeyError::new_err(key.unbind())); + }; + let Some(value) = self.object.get(key_str) else { + return Err(PyKeyError::new_err(key.unbind())); + }; + Ok(pythonize(py, value)?) + } + + fn __iter__<'py>(&self, py: Python<'py>) -> PyResult> { + // The easiest way to get an iterator over the keys is to create a + // temporary list and call `iter()` on it. This is not the most + // efficient approach, but is much less boilerplate than implementing a + // custom iterator type. Since the keys are typically small in number + // this should be fine in practice. + let list = PyList::new(py, self.object.keys().map(Box::as_ref))?; + PyIterator::from_object(&list) + } + + // The view classes below each hold a `JsonObject` clone. This is cheap + // because the underlying map is behind an `Arc`, and lets the view outlive + // the originating object (matching dict_keys/values/items semantics in + // Python, which also keep the dict alive). + + fn keys(&self) -> JsonObjectKeysView { + JsonObjectKeysView { + object: self.clone(), + } + } + + fn values(&self) -> JsonObjectValuesView { + JsonObjectValuesView { + object: self.clone(), + } + } + + fn items(&self) -> JsonObjectItemsView { + JsonObjectItemsView { + object: self.clone(), + } + } + + #[pyo3(signature = (key, default=None))] + fn get<'py>( + &self, + py: Python<'py>, + key: Bound<'_, PyAny>, + default: Option>, + ) -> PyResult> { + // Non-string keys can never match, so treat them as a miss and return + // the caller-supplied default rather than raising. + let Ok(key_str) = key.extract::<&str>() else { + return Ok(default.into_pyobject(py)?); + }; + match self.object.get(key_str) { + Some(value) => Ok(pythonize(py, value)?), + None => Ok(default.into_pyobject(py)?), + } + } + + fn __eq__(&self, other: Bound<'_, PyAny>) -> bool { + // We support equality against any Python mapping (e.g. plain dicts), + // so callers can swap a JsonObject in without rewriting comparisons. + let Ok(mapping) = other.cast::() else { + return false; + }; + + let Ok(other_len) = mapping.len() else { + return false; + }; + + if other_len != self.object.len() { + return false; + } + + // We know the "other" is a mapping with the same number of fields as + // us. So we can convert it into a JsonObject and compare the underlying + // maps. + let Ok(other_dict) = depythonize(&other) else { + return false; + }; + + *self.object == other_dict + } + + // Since we implement comparisons with other types, we need to disable + // hashing to avoid violating the invariant that equal objects must have the + // same hash. + // + // Alternatively, we could only allow comparisons with other JsonObjects and + // allow hashing, but a) its nicer to be able to compare with arbitrary + // mappings and b) we don't really need hashing for these objects. + #[classattr] + const __hash__: Option> = None; + + fn __str__(&self) -> String { + serde_json::to_string(&self.object).expect("Value should be serializable") + } + + fn __repr__(&self) -> String { + format!("JsonObject({})", self.__str__()) + } +} + +/// Helper class returned by `JsonObject.keys()` to act as a view into the keys +/// of the object. +/// +/// This needs to both be iterable *and* operate like a set. +#[pyclass(frozen)] +#[derive(Clone)] +pub struct JsonObjectKeysView { + object: JsonObject, +} + +#[pymethods] +impl JsonObjectKeysView { + fn __iter__<'py>(&self, py: Python<'py>) -> PyResult> { + // Create the iterator by making a temporary python list of the keys and + // calling `iter()` on it. + let list = PyList::new(py, self.object.object.keys().map(Box::as_ref))?; + PyIterator::from_object(&list) + } + + fn __len__(&self) -> usize { + self.object.__len__() + } + + fn __contains__(&self, key: &Bound<'_, PyAny>) -> bool { + self.object.__contains__(key) + } + + fn __eq__(&self, other: Bound<'_, PyAny>) -> bool { + let other_len = match other.len() { + Ok(len) => len, + Err(_) => return false, + }; + + if self.object.__len__() != other_len { + return false; + } + + for key in self.object.object.keys() { + if !matches!(other.contains(key.as_ref()), Ok(true)) { + return false; + } + } + + true + } + + // The set operators below match the behaviour of `dict.keys()` in Python: + // they accept any object that supports `__contains__` (for `&`) or is + // iterable (for `|`, `-`, `^`), not just sets. Each returns a fresh + // `PySet` so the caller gets a normal mutable Python set back. + // + // The `__r*__` variants are reflected operators, called by Python when + // the left-hand operand doesn't know how to combine with us. Since these + // operations are commutative for sets (or symmetric in the case of `^`), + // they just delegate. The asymmetric ops (`-`) need a separate impl. + + fn __and__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + // Iterate our (typically small) key set and probe `other`, which may + // be any container supporting `__contains__`. + let mut result = Vec::new(); + + for key in self.object.object.keys() { + if matches!(other.contains(key.as_ref()), Ok(true)) { + result.push(key.as_ref()); + } + } + + PySet::new(py, &result) + } + + fn __rand__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + self.__and__(py, other) + } + + fn __or__<'py>(&self, py: Python<'py>, other: Bound<'_, PyAny>) -> PyResult> { + // Union needs to enumerate both sides, so the right operand must be + // iterable (a bare `__contains__` is not enough). + let Ok(other_iter) = other.try_iter() else { + return Err(PyTypeError::new_err("Right operand must be iterable")); + }; + + let result = PySet::new(py, self.object.object.keys().map(Box::as_ref))?; + + // PySet handles dedup, so we can blindly add every element from the + // other iterable. + for item in other_iter { + let item = item?; + result.add(item)?; + } + + Ok(result) + } + + fn __ror__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + self.__or__(py, other) + } + + fn __sub__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + // `self - other`: keep our keys that are not in `other`. Only `other` + // needs to support `__contains__` here. + let mut result = Vec::new(); + + for key in self.object.object.keys() { + if matches!(other.contains(key.as_ref()), Ok(true)) { + continue; + } + result.push(key.as_ref()); + } + + PySet::new(py, &result) + } + + fn __rsub__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + // `other - self`: we need to enumerate `other`, so it must be + // iterable. Not symmetric with `__sub__`, hence a separate impl. + let Ok(other_iter) = other.try_iter() else { + return Err(PyTypeError::new_err("Left operand must be iterable")); + }; + + let result = PySet::empty(py)?; + + for item in other_iter { + let item = item?; + if self.object.__contains__(&item) { + continue; + } + result.add(item)?; + } + + Ok(result) + } + + fn __xor__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + // Symmetric difference: elements in exactly one side. Implemented as + // two filtered passes — one over our keys, one over `other`. + let Ok(other_iter) = other.try_iter() else { + return Err(PyTypeError::new_err("Right operand must be iterable")); + }; + + let result = PySet::empty(py)?; + + for key in self.object.object.keys() { + if matches!(other.contains(key.as_ref()), Ok(true)) { + continue; + } + result.add(key.as_ref())?; + } + + for item in other_iter { + let item = item?; + if self.object.__contains__(&item) { + continue; + } + result.add(item)?; + } + + Ok(result) + } + + fn __rxor__<'py>( + &self, + py: Python<'py>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + self.__xor__(py, other) + } + + fn isdisjoint(&self, other: Bound<'_, PyAny>) -> bool { + for key in self.object.object.keys() { + if matches!(other.contains(key.as_ref()), Ok(true)) { + return false; + } + } + + true + } +} + +/// Helper class returned by `JsonObject.values()` to act as a view into the +/// values of the object. +#[pyclass(frozen)] +#[derive(Clone)] +pub struct JsonObjectValuesView { + object: JsonObject, +} + +#[pymethods] +impl JsonObjectValuesView { + fn __iter__<'py>(&self, py: Python<'py>) -> PyResult> { + // Create the iterator by making a temporary python list of the keys and + // calling `iter()` on it. + let list = PyList::empty(py); + for v in self.object.object.values() { + let py_value = pythonize(py, v)?.into_bound_py_any(py)?; + list.append(py_value)?; + } + + PyIterator::from_object(&list) + } + + fn __len__(&self) -> usize { + self.object.__len__() + } + + fn __contains__(&self, other: Bound<'_, PyAny>) -> bool { + // We compare by JSON equality rather than Python identity: convert + // the candidate into a `serde_json::Value` once and scan our values. + // Anything that fails to depythonize cannot match by definition. + let other_value: serde_json::Value = match depythonize(&other) { + Ok(v) => v, + Err(_) => return false, + }; + self.object.object.values().any(|v| *v == other_value) + } +} + +/// Helper class returned by `JsonObject.items()` to act as a view into the +/// items of the object. +/// +/// Technically this should be a set-like view according to Python semantics, +/// unless the values are unhashable. Since the values are immutable we could +/// support it, but it's more work and nobody seems to actually use the set +/// operations on `dict_items` in practice. +#[pyclass(frozen)] +#[derive(Clone)] +pub struct JsonObjectItemsView { + object: JsonObject, +} + +#[pymethods] +impl JsonObjectItemsView { + fn __iter__<'py>(&self, py: Python<'py>) -> PyResult> { + // Create the iterator by making a temporary python list of the keys and + // calling `iter()` on it. + let list = PyList::empty(py); + for (k, v) in self.object.object.iter() { + let py_key = k.as_ref().into_bound_py_any(py)?; + let py_value = pythonize(py, v)?.into_bound_py_any(py)?; + let item = PyTuple::new(py, [py_key, py_value])?; + list.append(item)?; + } + + PyIterator::from_object(&list) + } + + fn __len__(&self) -> usize { + self.object.__len__() + } + + fn __contains__(&self, other: Bound<'_, PyAny>) -> bool { + // `(key, value) in items` — only a 2-tuple can possibly match. We + // look the key up directly (avoiding a full scan) and then compare + // the stored value against `value` using JSON equality. + let Ok((key, value)) = other.extract::<(Bound<'_, PyAny>, Bound<'_, PyAny>)>() else { + return false; + }; + let Ok(key_str) = key.extract::<&str>() else { + return false; + }; + let Some(stored) = self.object.object.get(key_str) else { + return false; + }; + let other_value: serde_json::Value = match depythonize(&value) { + Ok(v) => v, + Err(_) => return false, + }; + *stored == other_value + } +} diff --git a/rust/src/events/mod.rs b/rust/src/events/mod.rs index 5f505abb912..e60cdb70786 100644 --- a/rust/src/events/mod.rs +++ b/rust/src/events/mod.rs @@ -21,21 +21,31 @@ //! Classes for representing Events. use pyo3::{ - types::{PyAnyMethods, PyModule, PyModuleMethods}, + types::{PyAnyMethods, PyMapping, PyModule, PyModuleMethods}, wrap_pyfunction, Bound, PyResult, Python, }; pub mod filter; mod internal_metadata; +mod json_object; pub mod signatures; pub mod unsigned; +use json_object::JsonObject; + /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + // Register the `JsonObject` class as a `Mapping` so that `isinstance` works. + PyMapping::register::(py)?; + let child_module = PyModule::new(py, "events")?; child_module.add_class::()?; child_module.add_class::()?; child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; child_module.add_function(wrap_pyfunction!(filter::event_visible_to_server_py, m)?)?; m.add_submodule(&child_module)?; diff --git a/synapse/__init__.py b/synapse/__init__.py index 2bed0608784..3acfc1a0d7f 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -65,7 +65,8 @@ except ImportError: pass -# Teach canonicaljson how to serialise immutabledicts. + +# Teach canonicaljson how to serialise immutabledicts and JsonObjects. try: from canonicaljson import register_preserialisation_callback from immutabledict import immutabledict @@ -79,6 +80,12 @@ def _immutabledict_cb(d: immutabledict) -> dict[str, Any]: return dict(d) register_preserialisation_callback(immutabledict, _immutabledict_cb) + + # Teach canonicaljson how to serialise JsonObjects, which is just to + # convert them to dicts. + from synapse.synapse_rust.events import JsonObject # noqa: E402 + + register_preserialisation_callback(JsonObject, lambda obj: dict(obj)) except ImportError: pass diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 0f850d19b1c..5be0298c30f 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -44,9 +44,15 @@ StickyEvent, ) from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions -from synapse.synapse_rust.events import EventInternalMetadata, Signatures, Unsigned +from synapse.synapse_rust.events import ( + EventInternalMetadata, + JsonObject, + Signatures, + Unsigned, +) from synapse.types import ( JsonDict, + JsonMapping, StateKey, StrCollection, ) @@ -206,17 +212,29 @@ def __init__( ): assert room_version.event_format == self.format_version + if "content" in event_dict: + event_dict["content"] = JsonObject(event_dict["content"]) + + # We intern these strings because they turn up a lot (especially when + # caching). + event_dict = intern_dict(event_dict) + + if USE_FROZEN_DICTS: + frozen_dict = freeze(event_dict) + else: + frozen_dict = event_dict + self.room_version = room_version self.signatures = Signatures(signatures) self.unsigned = Unsigned(unsigned) self.rejected_reason = rejected_reason - self._dict = event_dict + self._dict = frozen_dict self.internal_metadata = EventInternalMetadata(internal_metadata_dict) depth: DictProperty[int] = DictProperty("depth") - content: DictProperty[JsonDict] = DictProperty("content") + content: DictProperty[JsonMapping] = DictProperty("content") hashes: DictProperty[dict[str, str]] = DictProperty("hashes") origin_server_ts: DictProperty[int] = DictProperty("origin_server_ts") sender: DictProperty[str] = DictProperty("sender") @@ -259,7 +277,14 @@ def get_state_key(self) -> str | None: def get_dict(self) -> JsonDict: """Convert the event to a dictionary suitable for serialisation.""" + d = dict(self._dict) + if "content" in d: + # Convert the content (which is a JsonObject) back to a dict. Json + # serialization should handle JsonObjects fine, but for sanities + # sake we want `get_dict()` and `get_pdu_json()` to return plain + # dicts. + d["content"] = dict(self.content) d.update( { "signatures": self.signatures.as_dict(), @@ -419,19 +444,10 @@ def __init__( unsigned = event_dict.pop("unsigned", {}) - # We intern these strings because they turn up a lot (especially when - # caching). - event_dict = intern_dict(event_dict) - - if USE_FROZEN_DICTS: - frozen_dict = freeze(event_dict) - else: - frozen_dict = event_dict - self._event_id = event_dict["event_id"] super().__init__( - frozen_dict, + event_dict, room_version=room_version, signatures=signatures, unsigned=unsigned, @@ -473,19 +489,10 @@ def __init__( unsigned = event_dict.pop("unsigned", {}) - # We intern these strings because they turn up a lot (especially when - # caching). - event_dict = intern_dict(event_dict) - - if USE_FROZEN_DICTS: - frozen_dict = freeze(event_dict) - else: - frozen_dict = event_dict - self._event_id: str | None = None super().__init__( - frozen_dict, + event_dict, room_version=room_version, signatures=signatures, unsigned=unsigned, diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 926c81b83dc..adbede7f160 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -1032,7 +1032,7 @@ def strip_event(event: EventBase) -> JsonDict: return { "type": event.type, "state_key": event.state_key, - "content": event.content, + "content": dict(event.content), "sender": event.sender, } diff --git a/synapse/events/validator.py b/synapse/events/validator.py index ff22b2287f2..d1b5152d77f 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -42,7 +42,7 @@ ) from synapse.http.servlet import validate_json_object from synapse.storage.controllers.state import server_acl_evaluator_from_event -from synapse.types import EventID, JsonDict, RoomID, StrCollection, UserID +from synapse.types import EventID, JsonDict, JsonMapping, RoomID, StrCollection, UserID from synapse.types.rest import RequestBodyModel @@ -245,7 +245,7 @@ def validate_builder(self, event: EventBase | EventBuilder) -> None: self._ensure_state_event(event) - def _ensure_strings(self, d: JsonDict, keys: StrCollection) -> None: + def _ensure_strings(self, d: JsonMapping, keys: StrCollection) -> None: for s in keys: if s not in d: raise SynapseError(400, "'%s' not in content" % (s,)) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 6a5c76c6676..13647caa2a9 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -706,12 +706,12 @@ async def clone_existing_room( spam_check = await self._spam_checker_module_callbacks.user_may_create_room( user_id, { - "creation_content": creation_content, + "creation_content": dict(creation_content), "initial_state": [ { "type": state_key[0], "state_key": state_key[1], - "content": event_content, + "content": dict(event_content), } for state_key, event_content in initial_state.items() ], @@ -1437,7 +1437,7 @@ async def _send_events_for_new_room( room_config: JsonDict, invite_list: list[str], initial_state: MutableStateMap, - creation_content: JsonDict, + creation_content: JsonMapping, room_alias: RoomAlias | None = None, power_level_content_override: JsonDict | None = None, creator_join_profile: JsonDict | None = None, @@ -1508,7 +1508,7 @@ async def _send_events_for_new_room( async def create_event( etype: str, - content: JsonDict, + content: JsonMapping, for_batch: bool, **kwargs: Any, ) -> tuple[EventBase, synapse.events.snapshot.UnpersistedEventContextBase]: @@ -1561,6 +1561,7 @@ async def create_event( if creation_event_with_context is None: # MSC2175 removes the creator field from the create event. if not room_version.implicit_room_creator: + creation_content = dict(creation_content) creation_content["creator"] = creator_id creation_event, unpersisted_creation_context = await create_event( EventTypes.Create, creation_content, False diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index c87b5f854a5..4fd69262b21 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -31,7 +31,7 @@ from synapse.api.constants import EventContentFields, EventTypes, Membership from synapse.metrics import SERVER_NAME_LABEL, event_processing_positions from synapse.storage.databases.main.state_deltas import StateDelta -from synapse.types import JsonDict +from synapse.types import JsonMapping from synapse.util.duration import Duration from synapse.util.events import get_plain_text_topic_from_event_content @@ -195,7 +195,7 @@ async def _handle_deltas( ) continue - event_content: JsonDict = {} + event_content: JsonMapping = {} if delta.event_id is not None: event = await self.store.get_event(delta.event_id, allow_none=True) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 7cf89200a8c..03dd3417440 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -51,7 +51,7 @@ from synapse.storage.invite_rule import InviteRule from synapse.storage.roommember import ProfileInfo from synapse.synapse_rust.push import FilteredPushRules, PushRuleEvaluator -from synapse.types import JsonValue +from synapse.types import JsonMapping, JsonValue from synapse.types.state import StateFilter from synapse.util import unwrapFirstError from synapse.util.async_helpers import gather_results @@ -231,7 +231,7 @@ async def _get_power_levels_and_sender_level( event: EventBase, context: EventContext, event_id_to_event: Mapping[str, EventBase], - ) -> tuple[dict, int | None]: + ) -> tuple[JsonMapping, int | None]: """ Given an event and an event context, get the power level event relevant to the event and the power level of the sender of the event. diff --git a/synapse/synapse_rust/events.pyi b/synapse/synapse_rust/events.pyi index 5ae2bb880ab..5b55d47f0d5 100644 --- a/synapse/synapse_rust/events.pyi +++ b/synapse/synapse_rust/events.pyi @@ -10,7 +10,7 @@ # See the GNU Affero General Public License for more details: # . -from typing import Any, Mapping +from typing import Any, Iterator, Mapping from synapse.types import JsonDict, JsonMapping @@ -213,3 +213,12 @@ class Unsigned: def for_event(self) -> JsonDict: ... """Return a dict of all unsigned fields, including those only kept in memory, suitable for inclusion in an event.""" + +class JsonObject(Mapping[str, Any]): + """Immutable JSON object mapping.""" + + def __init__(self, content_dict: JsonMapping | None = None): ... + def __len__(self) -> int: ... + def __getitem__(self, key: str) -> Any: ... + def __iter__(self) -> Iterator[str]: ... + def __eq__(self, other: object) -> bool: ... diff --git a/synapse/util/events.py b/synapse/util/events.py index 19eca1c1aeb..e7c1c83a37b 100644 --- a/synapse/util/events.py +++ b/synapse/util/events.py @@ -17,7 +17,7 @@ from pydantic import Field, StrictStr, ValidationError, field_validator -from synapse.types import JsonDict +from synapse.types import JsonMapping from synapse.util.pydantic_models import ParseModel from synapse.util.stringutils import random_string @@ -103,7 +103,7 @@ def ignore_invalid_m_topic(cls, m_topic: Any) -> MTopic | None: return None -def get_plain_text_topic_from_event_content(content: JsonDict) -> str | None: +def get_plain_text_topic_from_event_content(content: JsonMapping) -> str | None: """ Given the `content` of an `m.room.topic` event, returns the plain-text topic representation. Prefers pulling plain-text from the newer `m.topic` field if diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py index 0bc27410c6a..92c03690f23 100644 --- a/synapse/util/frozenutils.py +++ b/synapse/util/frozenutils.py @@ -23,6 +23,8 @@ from immutabledict import immutabledict +from synapse.synapse_rust.events import JsonObject + def freeze(o: Any) -> Any: if isinstance(o, dict): @@ -31,6 +33,9 @@ def freeze(o: Any) -> Any: if isinstance(o, immutabledict): return o + if isinstance(o, JsonObject): + return o + if isinstance(o, (bytes, str)): return o diff --git a/synapse/util/json.py b/synapse/util/json.py index b1091704a81..8f8d731c6d1 100644 --- a/synapse/util/json.py +++ b/synapse/util/json.py @@ -20,24 +20,30 @@ from immutabledict import immutabledict +from synapse.synapse_rust.events import JsonObject + def _reject_invalid_json(val: Any) -> None: """Do not allow Infinity, -Infinity, or NaN values in JSON.""" raise ValueError("Invalid JSON value: '%s'" % val) -def _handle_immutabledict(obj: Any) -> dict[Any, Any]: - """Helper for json_encoder. Makes immutabledicts serializable by returning - the underlying dict +def _handle_extra_mappings(obj: Any) -> dict[Any, Any]: + """Helper for json_encoder. Makes immutabledicts and JsonObjects + serializable """ - if type(obj) is immutabledict: - # fishing the protected dict out of the object is a bit nasty, - # but we don't really want the overhead of copying the dict. - try: - # Safety: we catch the AttributeError immediately below. - return obj._dict - except AttributeError: - # If all else fails, resort to making a copy of the immutabledict + match obj: + case immutabledict(): + # fishing the protected dict out of the object is a bit nasty, + # but we don't really want the overhead of copying the dict. + try: + # Safety: we catch the AttributeError immediately below. + return obj._dict + except AttributeError: + # If all else fails, resort to making a copy of the immutabledict + return dict(obj) + case JsonObject(): + # Just convert to a dict. return dict(obj) raise TypeError( "Object of type %s is not JSON serializable" % obj.__class__.__name__ @@ -49,7 +55,7 @@ def _handle_immutabledict(obj: Any) -> dict[Any, Any]: # * produces valid JSON (no NaNs etc) # * reduces redundant whitespace json_encoder = json.JSONEncoder( - allow_nan=False, separators=(",", ":"), default=_handle_immutabledict + allow_nan=False, separators=(",", ":"), default=_handle_extra_mappings ) # Create a custom decoder to reject Python extensions to JSON. diff --git a/tests/crypto/test_event_signing.py b/tests/crypto/test_event_signing.py index 334ff64bc21..6ec0f64ffce 100644 --- a/tests/crypto/test_event_signing.py +++ b/tests/crypto/test_event_signing.py @@ -62,6 +62,7 @@ def test_sign_minimal(self) -> None: "origin_server_ts": 1000000, "signatures": {}, "type": "X", + "content": {}, "unsigned": {"age_ts": 1000000}, } @@ -74,7 +75,7 @@ def test_sign_minimal(self) -> None: self.assertTrue(hasattr(event, "hashes")) self.assertIn("sha256", event.hashes) self.assertEqual( - event.hashes["sha256"], "A6Nco6sqoy18PPfPDVdYvoowfc0PVBk9g9OiyT3ncRM" + event.hashes["sha256"], "mq4QfPPpC+QsBd6eqfVsmJIEz8uvMSVK0+AU67PLESk" ) self.assertTrue(hasattr(event, "signatures")) @@ -82,8 +83,8 @@ def test_sign_minimal(self) -> None: self.assertIn(KEY_NAME, event.signatures["domain"]) self.assertEqual( event.signatures[HOSTNAME][KEY_NAME], - "PBc48yDVszWB9TRaB/+CZC1B+pDAC10F8zll006j+NN" - "fe4PEMWcVuLaG63LFTK9e4rwJE8iLZMPtCKhDTXhpAQ", + "18rGIkd4JJXxw9m+1j3BtN+TmqmLip4VHvFbyXLngpB" + "LXOqbxlQViQABRzep2cODQ2aa5FnFgz+Llt2P03WiAw", ) def test_sign_message(self) -> None: diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py index f1b20a12ecb..2ba5da3b95f 100644 --- a/tests/module_api/test_api.py +++ b/tests/module_api/test_api.py @@ -265,7 +265,7 @@ def test_sending_events_into_room(self) -> None: self.assertEqual(event.type, "m.room.message") self.assertEqual(event.room_id, room_id) self.assertFalse(hasattr(event, "state_key")) - self.assertDictEqual(event.content, content) + self.assertDictEqual(dict(event.content), content) expected_requester = create_requester( user_id, authenticated_entity=self.hs.hostname @@ -301,7 +301,7 @@ def test_sending_events_into_room(self) -> None: self.assertEqual(event.type, "m.room.power_levels") self.assertEqual(event.room_id, room_id) self.assertEqual(event.state_key, "") - self.assertDictEqual(event.content, content) + self.assertDictEqual(dict(event.content), content) # Check that the event was sent self.event_creation_handler.create_and_send_nonmember_event.assert_called_with( diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 61e7e87f623..28872fa06cb 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -59,7 +59,7 @@ sync, ) from synapse.server import HomeServer -from synapse.types import JsonDict, RoomAlias, UserID, create_requester +from synapse.types import JsonDict, JsonMapping, RoomAlias, UserID, create_requester from synapse.util.clock import Clock from synapse.util.stringutils import random_string @@ -1859,7 +1859,7 @@ class SpamCheck: mock_return_value: str | bool | Codes | tuple[Codes, JsonDict] | bool = ( "NOT_SPAM" ) - mock_content: JsonDict | None = None + mock_content: JsonMapping | None = None async def check_event_for_spam( self, diff --git a/tests/server.py b/tests/server.py index 55860701dae..20fcc420815 100644 --- a/tests/server.py +++ b/tests/server.py @@ -101,6 +101,7 @@ from synapse.storage.prepare_database import prepare_database from synapse.types import ISynapseReactor, JsonDict from synapse.util.clock import Clock +from synapse.util.json import json_encoder from tests.utils import ( LEAVE_DB, @@ -422,7 +423,7 @@ def make_request( path = b"/" + path if isinstance(content, dict): - content = json.dumps(content).encode("utf8") + content = json_encoder.encode(content).encode("utf8") if isinstance(content, str): content = content.encode("utf8") diff --git a/tests/synapse_rust/test_json_object.py b/tests/synapse_rust/test_json_object.py new file mode 100644 index 00000000000..77b188eee0c --- /dev/null +++ b/tests/synapse_rust/test_json_object.py @@ -0,0 +1,149 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from synapse.synapse_rust.events import JsonObject +from synapse.util import MutableOverlayMapping + +from tests import unittest + + +class JsonObjectMappingTestCase(unittest.TestCase): + def test_new_and_basic_mapping_behavior(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + self.assertEqual(len(obj), 2) + self.assertTrue("a" in obj) + self.assertTrue("b" in obj) + self.assertFalse("c" in obj) + self.assertFalse(123 in obj) # type: ignore[comparison-overlap] + + def test_getitem_and_key_errors(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + self.assertEqual(obj["a"], 1) + + with self.assertRaises(KeyError): + _ = obj["missing"] + + with self.assertRaises(KeyError): + _ = obj[10] # type: ignore[index] + + def test_iter_keys_values_items(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + iterator = iter(obj) + first = next(iterator) + second = next(iterator) + self.assertCountEqual((first, second), ("a", "b")) + with self.assertRaises(StopIteration): + next(iterator) + + self.assertCountEqual(list(obj.keys()), ["a", "b"]) + self.assertCountEqual(list(obj.values()), [1, 2]) + self.assertCountEqual(list(obj.items()), [("a", 1), ("b", 2)]) + + def test_keys_set_like_behavior(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + # Test 'and' operator. + self.assertEqual(obj.keys() & {"a"}, {"a"}) + self.assertEqual({"a"} & obj.keys(), {"a"}) + self.assertEqual(obj.keys() & {"c"}, set()) + self.assertEqual({"c"} & obj.keys(), set()) + + # Test 'or' operator. + self.assertEqual(obj.keys() | {"a"}, {"a", "b"}) + self.assertEqual({"a"} | obj.keys(), {"a", "b"}) + self.assertEqual(obj.keys() | {"c"}, {"a", "b", "c"}) + self.assertEqual({"c"} | obj.keys(), {"a", "b", "c"}) + + # Test 'xor' operator. + self.assertEqual(obj.keys() ^ {"a"}, {"b"}) + self.assertEqual({"a"} ^ obj.keys(), {"b"}) + self.assertEqual(obj.keys() ^ {"c"}, {"a", "b", "c"}) + self.assertEqual({"c"} ^ obj.keys(), {"a", "b", "c"}) + + # Test 'sub' operator. + self.assertEqual(obj.keys() - {"a"}, {"b"}) + self.assertEqual({"a"} - obj.keys(), set()) + self.assertEqual(obj.keys() - {"c"}, {"a", "b"}) + self.assertEqual({"c"} - obj.keys(), {"c"}) + + def test_values_view(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + values = obj.values() + + self.assertEqual(len(values), 2) + self.assertCountEqual(list(values), [1, 2]) + + self.assertIn(1, values) + self.assertIn(2, values) + self.assertNotIn(3, values) + self.assertNotIn("a", values) + self.assertNotIn(object(), values) + + # Iterating twice should yield the same values. + self.assertCountEqual(list(values), [1, 2]) + + def test_items_view(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + items = obj.items() + + self.assertEqual(len(items), 2) + self.assertCountEqual(list(items), [("a", 1), ("b", 2)]) + + self.assertIn(("a", 1), items) + self.assertIn(("b", 2), items) + self.assertNotIn(("a", 2), items) + self.assertNotIn(("c", 1), items) + self.assertNotIn("a", items) + self.assertNotIn(("a", 1, "extra"), items) + + # Iterating twice should yield the same items. + self.assertCountEqual(list(items), [("a", 1), ("b", 2)]) + + def test_get(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + self.assertEqual(obj.get("a"), 1) + self.assertEqual(obj.get("missing", "fallback"), "fallback") + self.assertEqual(obj.get(5, "fallback"), "fallback") # type: ignore[call-overload] + + def test_eq(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + self.assertEqual(obj, {"a": 1, "b": 2}) + self.assertNotEqual(obj, {"a": 1}) + self.assertNotEqual(obj, ["a", "b"]) + + def test_str_and_repr(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + self.assertEqual(str(obj), r'{"a":1,"b":2}') + self.assertEqual(repr(obj), r'JsonObject({"a":1,"b":2})') + + def test_json_object_constructor(self) -> None: + obj = JsonObject({"a": 1, "b": 2}) + + # Passing in an existing JsonObject should work. + obj2 = JsonObject(obj) + self.assertEqual(obj2, {"a": 1, "b": 2}) + + # Other mapping types should also work. + obj3 = JsonObject(MutableOverlayMapping({"a": 1, "b": 2})) + self.assertEqual(obj3, {"a": 1, "b": 2}) + + # Test that passing a non-mapping raises a TypeError. + with self.assertRaises(TypeError): + JsonObject(123) # type: ignore[arg-type] diff --git a/tests/test_state.py b/tests/test_state.py index 7df95ebf8b4..0ca88aef74c 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -35,7 +35,7 @@ from synapse.events import EventBase, make_event_from_dict from synapse.events.snapshot import EventContext from synapse.state import StateHandler, StateResolutionHandler, _make_state_cache_entry -from synapse.types import MutableStateMap, StateMap +from synapse.types import JsonDict, MutableStateMap, StateMap from synapse.types.state import StateFilter from synapse.util.macaroons import MacaroonGenerator @@ -67,13 +67,14 @@ def create_event( else: name = "<%s, %s>" % (type, event_id) - d = { + d: JsonDict = { "event_id": event_id, "type": type, "sender": "@user_id:example.com", "room_id": "!room_id:example.com", "depth": depth, "prev_events": prev_events or [], + "content": {}, } if state_key is not None: diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py index 0df5a4e6c3a..41707682084 100644 --- a/tests/test_utils/__init__.py +++ b/tests/test_utils/__init__.py @@ -24,7 +24,6 @@ """ import base64 -import json import sys import warnings from binascii import unhexlify @@ -41,6 +40,7 @@ from twisted.web.iweb import IResponse from synapse.types import JsonSerializable +from synapse.util.json import json_encoder if TYPE_CHECKING: from sys import UnraisableHookArgs @@ -127,7 +127,7 @@ def deliverBody(self, protocol: IProtocol) -> None: @classmethod def json(cls, *, code: int = 200, payload: JsonSerializable) -> "FakeResponse": headers = Headers({"Content-Type": ["application/json"]}) - body = json.dumps(payload).encode("utf-8") + body = json_encoder.encode(payload).encode("utf-8") return cls(code=code, body=body, headers=headers) From 5c87faf9e9adbea0c0d6b376f65e6a03533b2326 Mon Sep 17 00:00:00 2001 From: Will Hunt <2072976+Half-Shot@users.noreply.github.com> Date: Mon, 11 May 2026 12:39:38 +0100 Subject: [PATCH 23/92] MSC4452: Preview URL capability (#19715) Implementation of https://github.com/matrix-org/matrix-spec-proposals/pull/4452 --- changelog.d/19715.feature | 3 ++ synapse/config/experimental.py | 4 ++ synapse/config/repository.py | 3 +- synapse/rest/client/capabilities.py | 5 ++ synapse/rest/client/media.py | 20 ++++--- .../rest/media/media_repository_resource.py | 7 ++- synapse/rest/media/preview_url_resource.py | 10 +++- tests/rest/client/test_capabilities.py | 43 ++++++++++++++- tests/rest/client/test_media.py | 54 +++++++++++++++++++ 9 files changed, 135 insertions(+), 14 deletions(-) create mode 100644 changelog.d/19715.feature diff --git a/changelog.d/19715.feature b/changelog.d/19715.feature new file mode 100644 index 00000000000..973fe66e7d0 --- /dev/null +++ b/changelog.d/19715.feature @@ -0,0 +1,3 @@ +Add support for "MSC4452 Preview URL capabilities API" which exposes a `io.element.msc4452.preview_url` capability. +If `experimental_features.msc4452_enabled` is `true`, the `/_matrix/(client/v1/media|media/v3)/preview_url` endpoint +now responds with a 403 status code when the capability is disabled. diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index f7c452bc73b..c958a278fc6 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -613,3 +613,7 @@ def read_config( # Tracked in: https://github.com/element-hq/synapse/issues/19691 # Note that this is only applicable to legacy auth, not MAS integration (OAuth 2.0). self.msc4450_enabled: bool = experimental.get("msc4450_enabled", False) + + # MSC4455: Preview URL capability + # Tracked in: https://github.com/element-hq/synapse/issues/19719 + self.msc4452_enabled: bool = experimental.get("msc4452_enabled", False) diff --git a/synapse/config/repository.py b/synapse/config/repository.py index c87442aace0..cb50d0dc1de 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -242,7 +242,8 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.thumbnail_requirements = parse_thumbnail_requirements( config.get("thumbnail_sizes", DEFAULT_THUMBNAIL_SIZES) ) - self.url_preview_enabled = config.get("url_preview_enabled", False) + self.url_preview_enabled = bool(config.get("url_preview_enabled", False)) + if self.url_preview_enabled: check_requirements("url-preview") diff --git a/synapse/rest/client/capabilities.py b/synapse/rest/client/capabilities.py index 705d74dee13..2be5f5849d7 100644 --- a/synapse/rest/client/capabilities.py +++ b/synapse/rest/client/capabilities.py @@ -77,6 +77,11 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: } } + if self.config.experimental.msc4452_enabled: + response["capabilities"]["io.element.msc4452.preview_url"] = { + "enabled": self.config.media.url_preview_enabled, + } + if self.config.experimental.msc3720_enabled: response["capabilities"]["org.matrix.msc3720.account_status"] = { "enabled": True, diff --git a/synapse/rest/client/media.py b/synapse/rest/client/media.py index 15f58acb955..c740659cdd9 100644 --- a/synapse/rest/client/media.py +++ b/synapse/rest/client/media.py @@ -23,7 +23,12 @@ import logging import re -from synapse.api.errors import Codes, cs_error +from synapse.api.errors import ( + Codes, + SynapseError, + UnrecognizedRequestError, + cs_error, +) from synapse.http.server import ( HttpServer, respond_with_json, @@ -79,11 +84,17 @@ def __init__( self.clock = hs.get_clock() self.media_repo = media_repo self.media_storage = media_storage - assert self.media_repo.url_previewer is not None self.url_previewer = self.media_repo.url_previewer + self.can_respond_403 = hs.config.experimental.msc4452_enabled async def on_GET(self, request: SynapseRequest) -> None: requester = await self.auth.get_user_by_req(request) + if self.url_previewer is None: + # If we have no url_previewer then it has been disabled by the server. + if self.can_respond_403: + raise SynapseError(403, "URL Previews are disabled", Codes.FORBIDDEN) + else: + raise UnrecognizedRequestError(code=404) url = parse_string(request, "url", required=True) ts = parse_integer(request, "ts") if ts is None: @@ -299,10 +310,7 @@ async def on_GET( def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: media_repo = hs.get_media_repository() - if hs.config.media.url_preview_enabled: - PreviewURLServlet(hs, media_repo, media_repo.media_storage).register( - http_server - ) + PreviewURLServlet(hs, media_repo, media_repo.media_storage).register(http_server) MediaConfigResource(hs).register(http_server) ThumbnailResource(hs, media_repo, media_repo.media_storage).register(http_server) DownloadResource(hs, media_repo).register(http_server) diff --git a/synapse/rest/media/media_repository_resource.py b/synapse/rest/media/media_repository_resource.py index 963b9de2523..354144af359 100644 --- a/synapse/rest/media/media_repository_resource.py +++ b/synapse/rest/media/media_repository_resource.py @@ -106,8 +106,7 @@ def register_servlets(http_server: HttpServer, hs: "HomeServer") -> None: ThumbnailResource(hs, media_repo, media_repo.media_storage).register( http_server ) - if hs.config.media.url_preview_enabled: - PreviewUrlResource(hs, media_repo, media_repo.media_storage).register( - http_server - ) + PreviewUrlResource(hs, media_repo, media_repo.media_storage).register( + http_server + ) MediaConfigResource(hs).register(http_server) diff --git a/synapse/rest/media/preview_url_resource.py b/synapse/rest/media/preview_url_resource.py index bfeff2179b8..5a7bdf38f02 100644 --- a/synapse/rest/media/preview_url_resource.py +++ b/synapse/rest/media/preview_url_resource.py @@ -23,6 +23,7 @@ import re from typing import TYPE_CHECKING +from synapse.api.errors import Codes, SynapseError, UnrecognizedRequestError from synapse.http.server import respond_with_json_bytes from synapse.http.servlet import RestServlet, parse_integer, parse_string from synapse.http.site import SynapseRequest @@ -65,12 +66,17 @@ def __init__( self.clock = hs.get_clock() self.media_repo = media_repo self.media_storage = media_storage - assert self.media_repo.url_previewer is not None self.url_previewer = self.media_repo.url_previewer + self.can_respond_403 = hs.config.experimental.msc4452_enabled async def on_GET(self, request: SynapseRequest) -> None: - # XXX: if get_user_by_req fails, what should we do in an async render? requester = await self.auth.get_user_by_req(request) + if self.url_previewer is None: + # If we have no url_previewer then it has been disabled by the server. + if self.can_respond_403: + raise SynapseError(403, "URL Previews are disabled", Codes.FORBIDDEN) + else: + raise UnrecognizedRequestError(code=404) url = parse_string(request, "url", required=True) ts = parse_integer(request, "ts", default=self.clock.time_msec()) og = await self.url_previewer.preview(url, requester.user, ts) diff --git a/tests/rest/client/test_capabilities.py b/tests/rest/client/test_capabilities.py index 2567fee2a4a..c28e0605b54 100644 --- a/tests/rest/client/test_capabilities.py +++ b/tests/rest/client/test_capabilities.py @@ -28,7 +28,12 @@ from synapse.util.clock import Clock from tests import unittest -from tests.unittest import override_config +from tests.unittest import override_config, skip_unless + +try: + import lxml +except ImportError: + lxml = None # type: ignore[assignment] class CapabilitiesTestCase(unittest.HomeserverTestCase): @@ -276,3 +281,39 @@ def test_get_forget_forced_upon_leave_without_auto_forget(self) -> None: self.assertFalse( capabilities["org.matrix.msc4267.forget_forced_upon_leave"]["enabled"] ) + + @override_config( + { + "url_preview_enabled": False, + "experimental_features": {"msc4452_enabled": True}, + } + ) + def test_url_previews_disabled(self) -> None: + access_token = self.get_success( + self.auth_handler.create_access_token_for_user_id( + self.user, device_id=None, valid_until_ms=None + ) + ) + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertFalse(capabilities["io.element.msc4452.preview_url"]["enabled"]) + + @skip_unless(lxml is not None, "Requires lxml") + @override_config( + { + "url_preview_enabled": True, + "url_preview_ip_range_blacklist": ["127.0.0.1"], + "experimental_features": {"msc4452_enabled": True}, + }, + ) + def test_url_previews_enabled(self) -> None: + access_token = self.get_success( + self.auth_handler.create_access_token_for_user_id( + self.user, device_id=None, valid_until_ms=None + ) + ) + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertTrue(capabilities["io.element.msc4452.preview_url"]["enabled"]) diff --git a/tests/rest/client/test_media.py b/tests/rest/client/test_media.py index ec81b1413c2..3409581c5ed 100644 --- a/tests/rest/client/test_media.py +++ b/tests/rest/client/test_media.py @@ -1573,6 +1573,60 @@ def test_blocked_url(self) -> None: self.assertEqual(channel.code, 403, channel.result) +# We test this here because this endpoint must still work +# even if lxml is not installed. +class URLPreviewDisabledTests(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets, + login.register_servlets, + media.register_servlets, + ] + + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + self.register_user("user", "password") + self.tok = self.login("user", "password") + + @override_config( + { + "url_preview_enabled": False, + "experimental_features": {"msc4452_enabled": False}, + } + ) + def test_disabled_previews(self) -> None: + """Tests that disabling URL previews gives back a sane response.""" + channel = self.make_request( + "GET", + "/_matrix/client/v1/media/preview_url?url=" + quote("http://example.com"), + access_token=self.tok, + ) + self.assertEqual(channel.code, 404, channel.result) + self.assertEqual( + channel.json_body, + {"errcode": "M_UNRECOGNIZED", "error": "Unrecognized request"}, + ) + + @override_config( + { + "url_preview_enabled": False, + "experimental_features": {"msc4452_enabled": True}, + } + ) + def test_disabled_previews_with_msc4452(self) -> None: + """Tests that disabling URL previews gives back a sane response.""" + channel = self.make_request( + "GET", + "/_matrix/client/v1/media/preview_url?url=" + quote("http://example.com"), + access_token=self.tok, + ) + self.assertEqual(channel.code, 403, channel.result) + self.assertEqual( + channel.json_body, + {"errcode": "M_FORBIDDEN", "error": "URL Previews are disabled"}, + ) + + class MediaConfigTest(unittest.HomeserverTestCase): servlets = [ media.register_servlets, From b8bd35105fb166aa40d7493a26bcbfff909174c8 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 12 May 2026 10:10:09 -0500 Subject: [PATCH 24/92] Update `WorkerLock` tests to better stress the `WORKER_LOCK_MAX_RETRY_INTERVAL` (#19772) There is no behavioral change, only a change to the tests. See https://github.com/element-hq/synapse/pull/19772#discussion_r3222059105 for an explanation of why the tests needed changing (and diff comments). Follow-up to https://github.com/element-hq/synapse/pull/19394. The test discussion originally happened in https://github.com/element-hq/synapse/pull/19394#discussion_r2789673181 This is spawning from thinking about the problem again. --- changelog.d/19772.misc | 1 + synapse/handlers/worker_lock.py | 8 +-- synapse/storage/databases/main/lock.py | 14 +++-- tests/handlers/test_worker_lock.py | 73 ++++++++++++++++++++--- tests/storage/databases/main/test_lock.py | 12 ++-- 5 files changed, 83 insertions(+), 25 deletions(-) create mode 100644 changelog.d/19772.misc diff --git a/changelog.d/19772.misc b/changelog.d/19772.misc new file mode 100644 index 00000000000..939507f5c35 --- /dev/null +++ b/changelog.d/19772.misc @@ -0,0 +1 @@ +Update `WorkerLock` tests to better stress the `WORKER_LOCK_MAX_RETRY_INTERVAL`. diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 57792ea53cf..a37b04494b5 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -61,10 +61,10 @@ Better to retry more quickly than have workers wait around. 5 seconds is still a reasonable gap in time to not overwhelm the CPU/Database. -This matters most in cross-worker scenarios. When locks are on the same worker, when the -lock holder releases, we signal to other locks (with the same name/key) that they -should try reacquiring the lock immediately. But locks on other workers only re-check -based on their retry `_timeout_interval`. +This matters most when locks go stale as normally, when the lock holder releases, we +signal to other locks (with the same name/key) that they should try reacquiring the lock +immediately. But stale locks are never released and instead forcefully reaped behind the +scenes. """ WORKER_LOCK_EXCESSIVE_WAITING_WARN_DURATION = Duration(minutes=10) diff --git a/synapse/storage/databases/main/lock.py b/synapse/storage/databases/main/lock.py index dd49f98366e..decb74e9943 100644 --- a/synapse/storage/databases/main/lock.py +++ b/synapse/storage/databases/main/lock.py @@ -53,9 +53,9 @@ _RENEWAL_INTERVAL = Duration(seconds=30) # How long before an acquired lock times out. -_LOCK_TIMEOUT_MS = 2 * 60 * 1000 +_LOCK_TIMEOUT = Duration(minutes=2) -_LOCK_REAP_INTERVAL = Duration(milliseconds=_LOCK_TIMEOUT_MS / 10.0) +_LOCK_REAP_INTERVAL = Duration(milliseconds=_LOCK_TIMEOUT.as_millis() / 10.0) class LockStore(SQLBaseStore): @@ -63,7 +63,7 @@ class LockStore(SQLBaseStore): Locks are identified by a name and key. A lock is acquired by inserting into the `worker_locks` table if a) there is no existing row for the name/key or - b) the existing row has a `last_renewed_ts` older than `_LOCK_TIMEOUT_MS`. + b) the existing row has a `last_renewed_ts` older than `_LOCK_TIMEOUT`. When a lock is taken out the instance inserts a random `token`, the instance that holds that token holds the lock until it drops (or times out). @@ -182,7 +182,7 @@ def _try_acquire_lock_txn(txn: LoggingTransaction) -> bool: self._instance_name, token, now, - now - _LOCK_TIMEOUT_MS, + now - _LOCK_TIMEOUT.as_millis(), ), ) @@ -340,7 +340,9 @@ async def _reap_stale_read_write_locks(self) -> None: """ def reap_stale_read_write_locks_txn(txn: LoggingTransaction) -> None: - txn.execute(delete_sql, (self.clock.time_msec() - _LOCK_TIMEOUT_MS,)) + txn.execute( + delete_sql, (self.clock.time_msec() - _LOCK_TIMEOUT.as_millis(),) + ) if txn.rowcount: logger.info("Reaped %d stale locks", txn.rowcount) @@ -489,7 +491,7 @@ async def is_still_valid(self) -> bool: ) return ( last_renewed_ts is not None - and self._clock.time_msec() - _LOCK_TIMEOUT_MS < last_renewed_ts + and self._clock.time_msec() - _LOCK_TIMEOUT.as_millis() < last_renewed_ts ) async def __aenter__(self) -> None: diff --git a/tests/handlers/test_worker_lock.py b/tests/handlers/test_worker_lock.py index 74201f41515..a38adcd4d44 100644 --- a/tests/handlers/test_worker_lock.py +++ b/tests/handlers/test_worker_lock.py @@ -25,8 +25,13 @@ from twisted.internet import defer from twisted.internet.testing import MemoryReactor +from synapse.handlers.worker_lock import WORKER_LOCK_MAX_RETRY_INTERVAL from synapse.server import HomeServer -from synapse.storage.databases.main.lock import _RENEWAL_INTERVAL +from synapse.storage.databases.main.lock import ( + _LOCK_REAP_INTERVAL, + _LOCK_TIMEOUT, + _RENEWAL_INTERVAL, +) from synapse.util.clock import Clock from synapse.util.duration import Duration @@ -83,7 +88,7 @@ def test_timeouts_for_lock_locally(self) -> None: # Note: We use `_pump_by` instead of `pump`/`advance` as the `Lock` has an # internal background looping call that runs every 30 seconds # (`_RENEWAL_INTERVAL`) to renew the `Lock` and push it's "drop timeout" value - # further out by 2 minutes (`_LOCK_TIMEOUT_MS`). The `Lock` will prematurely + # further out by 2 minutes (`_LOCK_TIMEOUT`). The `Lock` will prematurely # drop if this renewal is not allowed to run, which sours the test. # self.pump(amount=Duration(hours=1)) self._pump_by(amount=Duration(hours=1), by=_RENEWAL_INTERVAL) @@ -91,9 +96,34 @@ def test_timeouts_for_lock_locally(self) -> None: # Make sure we haven't acquired the `lock2` yet (`lock1` still holds it) self.assertNoResult(d2) - # Release the first lock (`lock1`). The second lock(`lock2`) should be - # automatically acquired by the `pump()` inside `get_success()` - self.get_success(lock1.__aexit__(None, None, None)) + # Drop the lock without releasing it. If we just normally released the lock + # (`self.get_success(lock1.__aexit__(None, None, None))`), the + # `add_lock_released_callback`/`notify_lock_released` cycle would signal that we + # should re-aquire the lock right away (on the next reactor tick). And we want + # to avoid that as the point of this test is to stress the retry timeout + # interval and `WORKER_LOCK_MAX_RETRY_INTERVAL`. + del lock1 + + # Wait for `lock1` to go stale (it won't be renewed anymore because we deleted + # it just above) + self._pump_by( + amount=_LOCK_TIMEOUT, + by=_RENEWAL_INTERVAL, + ) + + # Wait just enough time so `lock1` is reaped (found stale and forcefully drops + # the lock its holding) + self._pump_by( + amount=_LOCK_REAP_INTERVAL, + by=_RENEWAL_INTERVAL, + ) + + # Wait just enough time so `lock2` tries re-acquiring the lock. Should be no + # longer than our `WORKER_LOCK_MAX_RETRY_INTERVAL`. + self._pump_by( + amount=WORKER_LOCK_MAX_RETRY_INTERVAL, + by=_RENEWAL_INTERVAL, + ) # We should now have the lock self.successResultOf(d2) @@ -219,7 +249,7 @@ def test_timeouts_for_lock_worker(self) -> None: # Note: We use `_pump_by` instead of `pump`/`advance` as the `Lock` has an # internal background looping call that runs every 30 seconds # (`_RENEWAL_INTERVAL`) to renew the `Lock` and push it's "drop timeout" value - # further out by 2 minutes (`_LOCK_TIMEOUT_MS`). The `Lock` will prematurely + # further out by 2 minutes (`_LOCK_TIMEOUT`). The `Lock` will prematurely # drop if this renewal is not allowed to run, which sours the test. # self.pump(amount=Duration(hours=1)) self._pump_by(amount=Duration(hours=1), by=_RENEWAL_INTERVAL) @@ -227,9 +257,34 @@ def test_timeouts_for_lock_worker(self) -> None: # Make sure we haven't acquired the `lock2` yet (`lock1` still holds it) self.assertNoResult(d2) - # Release the first lock (`lock1`). The second lock(`lock2`) should be - # automatically acquired by the `pump()` inside `get_success()` - self.get_success(lock1.__aexit__(None, None, None)) + # Drop the lock without releasing it. If we just normally released the lock + # (`self.get_success(lock1.__aexit__(None, None, None))`), the + # `add_lock_released_callback`/`notify_lock_released` cycle would signal that we + # should re-aquire the lock right away (on the next reactor tick). And we want + # to avoid that as the point of this test is to stress the retry timeout + # interval and `WORKER_LOCK_MAX_RETRY_INTERVAL`. + del lock1 + + # Wait for `lock1` to go stale (it won't be renewed anymore because we deleted + # it just above) + self._pump_by( + amount=_LOCK_TIMEOUT, + by=_RENEWAL_INTERVAL, + ) + + # Wait just enough time so `lock1` is reaped (found stale and forcefully drops + # the lock its holding) + self._pump_by( + amount=_LOCK_REAP_INTERVAL, + by=_RENEWAL_INTERVAL, + ) + + # Wait just enough time so `lock2` tries re-acquiring the lock. Should be no + # longer than our `WORKER_LOCK_MAX_RETRY_INTERVAL`. + self._pump_by( + amount=WORKER_LOCK_MAX_RETRY_INTERVAL, + by=_RENEWAL_INTERVAL, + ) # We should now have the lock self.successResultOf(d2) diff --git a/tests/storage/databases/main/test_lock.py b/tests/storage/databases/main/test_lock.py index 622eb96ded0..c38e0cc834a 100644 --- a/tests/storage/databases/main/test_lock.py +++ b/tests/storage/databases/main/test_lock.py @@ -26,7 +26,7 @@ from twisted.internet.testing import MemoryReactor from synapse.server import HomeServer -from synapse.storage.databases.main.lock import _LOCK_TIMEOUT_MS, _RENEWAL_INTERVAL +from synapse.storage.databases.main.lock import _LOCK_TIMEOUT, _RENEWAL_INTERVAL from synapse.util.clock import Clock from tests import unittest @@ -117,7 +117,7 @@ def test_maintain_lock(self) -> None: self.get_success(lock.__aenter__()) # Wait for ages with the lock, we should not be able to get the lock. - self.reactor.advance(5 * _LOCK_TIMEOUT_MS / 1000) + self.reactor.advance(5 * _LOCK_TIMEOUT.as_secs()) lock2 = self.get_success(self.store.try_acquire_lock("name", "key")) self.assertIsNone(lock2) @@ -138,7 +138,7 @@ def test_timeout_lock(self) -> None: lock._looping_call.stop() # Wait for the lock to timeout. - self.reactor.advance(2 * _LOCK_TIMEOUT_MS / 1000) + self.reactor.advance(2 * _LOCK_TIMEOUT.as_secs()) lock2 = self.get_success(self.store.try_acquire_lock("name", "key")) self.assertIsNotNone(lock2) @@ -154,7 +154,7 @@ def test_drop(self) -> None: del lock # Wait for the lock to timeout. - self.reactor.advance(2 * _LOCK_TIMEOUT_MS / 1000) + self.reactor.advance(2 * _LOCK_TIMEOUT.as_secs()) lock2 = self.get_success(self.store.try_acquire_lock("name", "key")) self.assertIsNotNone(lock2) @@ -402,7 +402,7 @@ def test_timeout_lock(self) -> None: lock._looping_call.stop() # Wait for the lock to timeout. - self.reactor.advance(2 * _LOCK_TIMEOUT_MS / 1000) + self.reactor.advance(2 * _LOCK_TIMEOUT.as_secs()) lock2 = self.get_success( self.store.try_acquire_read_write_lock("name", "key", write=True) @@ -422,7 +422,7 @@ def test_drop(self) -> None: del lock # Wait for the lock to timeout. - self.reactor.advance(2 * _LOCK_TIMEOUT_MS / 1000) + self.reactor.advance(2 * _LOCK_TIMEOUT.as_secs()) lock2 = self.get_success( self.store.try_acquire_read_write_lock("name", "key", write=True) From 5efeac44b27a98bdf73f0fddc37753d8181d5ee2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 May 2026 11:28:06 +0100 Subject: [PATCH 25/92] Handle arbitrary sized integers in `unsigned`. (#19769) Handle arbitrary sized integers in `unsigned` (and other Rust objects that use `serde_json::Value`) --- Cargo.lock | 59 +++--------- changelog.d/19769.bugfix | 1 + rust/Cargo.toml | 14 ++- rust/src/acl/mod.rs | 2 +- rust/src/canonical_json.rs | 132 ++++++++++++++++++++++++--- rust/src/events/internal_metadata.rs | 2 +- rust/src/events/unsigned.rs | 28 +++--- rust/src/push/mod.rs | 6 +- rust/src/room_versions.rs | 13 ++- tests/synapse_rust/test_unsigned.py | 52 +++++++++++ 10 files changed, 225 insertions(+), 84 deletions(-) create mode 100644 changelog.d/19769.bugfix create mode 100644 tests/synapse_rust/test_unsigned.py diff --git a/Cargo.lock b/Cargo.lock index 832d5129fe2..8285c7cf38c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,12 +29,6 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" -[[package]] -name = "autocfg" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" - [[package]] name = "base64" version = "0.22.1" @@ -646,12 +640,6 @@ dependencies = [ "hashbrown", ] -[[package]] -name = "indoc" -version = "2.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" - [[package]] name = "ipnet" version = "2.11.0" @@ -735,15 +723,6 @@ version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" -[[package]] -name = "memoffset" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" -dependencies = [ - "autocfg", -] - [[package]] name = "mime" version = "0.3.17" @@ -821,37 +800,34 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.27.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab53c047fcd1a1d2a8820fe84f05d6be69e9526be40cb03b73f86b6b03e6d87d" +checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" dependencies = [ "anyhow", "bytes", - "indoc", "libc", - "memoffset", "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", "pyo3-macros", - "unindent", ] [[package]] name = "pyo3-build-config" -version = "0.27.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b455933107de8642b4487ed26d912c2d899dec6114884214a0b3bb3be9261ea6" +checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.27.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c85c9cbfaddf651b1221594209aed57e9e5cff63c4d11d1feead529b872a089" +checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" dependencies = [ "libc", "pyo3-build-config", @@ -870,9 +846,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.27.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a5b10c9bf9888125d917fb4d2ca2d25c8df94c7ab5a52e13313a07e050a3b02" +checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -882,9 +858,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.27.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b51720d314836e53327f5871d4c0cfb4fb37cc2c4a11cc71907a86342c40f9" +checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" dependencies = [ "heck", "proc-macro2", @@ -895,12 +871,13 @@ dependencies = [ [[package]] name = "pythonize" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3a8f29db331e28c332c63496cfcbb822aca3d7320bc08b655d7fd0c29c50ede" +checksum = "0b79f670c9626c8b651c0581011b57b6ba6970bb69faf01a7c4c0cfc81c43f95" dependencies = [ "pyo3", "serde", + "serde_json", ] [[package]] @@ -1391,9 +1368,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.13.2" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] name = "thiserror" @@ -1569,12 +1546,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" -[[package]] -name = "unindent" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" - [[package]] name = "untrusted" version = "0.9.0" diff --git a/changelog.d/19769.bugfix b/changelog.d/19769.bugfix new file mode 100644 index 00000000000..05aae443119 --- /dev/null +++ b/changelog.d/19769.bugfix @@ -0,0 +1 @@ +Correctly handle arbitrary precision integers in `unsigned` field of events. diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 5bdd1947070..162bc981820 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -30,7 +30,7 @@ http = "1.1.0" lazy_static = "1.4.0" log = "0.4.17" mime = "0.3.17" -pyo3 = { version = "0.27.2", features = [ +pyo3 = { version = "0.28.3", features = [ "macros", "anyhow", "abi3", @@ -39,12 +39,18 @@ pyo3 = { version = "0.27.2", features = [ # https://docs.rs/pyo3/latest/pyo3/bytes/index.html "bytes", ] } -pyo3-log = "0.13.1" -pythonize = "0.27.0" +pyo3-log = "0.13.3" +pythonize = { version = "0.28.0", features = ["arbitrary_precision"] } regex = "1.6.0" sha2 = "0.10.8" serde = { version = "1.0.144", features = ["derive", "rc"] } -serde_json = { version = "1.0.85", features = ["raw_value"] } +serde_json = { version = "1.0.85", features = [ + "raw_value", + # We need to be able to parse arbitrary precision numbers, as some numbers + # in the database may be out of range of i64 (as Python uses arbitrary + # precision integers). + "arbitrary_precision", +] } ulid = "1.1.2" icu_segmenter = "2.0.0" reqwest = { version = "0.12.15", default-features = false, features = [ diff --git a/rust/src/acl/mod.rs b/rust/src/acl/mod.rs index 57b45475fdd..ab10b9f037d 100644 --- a/rust/src/acl/mod.rs +++ b/rust/src/acl/mod.rs @@ -47,7 +47,7 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> } #[derive(Debug, Clone)] -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] pub struct ServerAclEvaluator { allow_ip_literals: bool, allow: Vec, diff --git a/rust/src/canonical_json.rs b/rust/src/canonical_json.rs index 4abd0018479..94ed6c368d7 100644 --- a/rust/src/canonical_json.rs +++ b/rust/src/canonical_json.rs @@ -29,7 +29,7 @@ use std::{ io::{self, Write}, }; -use serde::ser::SerializeMap; +use serde::{ser::SerializeMap, Serializer as _}; use serde::{ ser::{Error as _, SerializeStruct}, Serialize, @@ -37,7 +37,7 @@ use serde::{ use serde_json::{ ser::{Formatter, Serializer}, value::RawValue, - Value, + Number, Value, }; /// The minimum integer that can be used in canonical JSON. @@ -46,6 +46,12 @@ pub const MIN_VALID_INTEGER: i64 = -(2i64.pow(53)) + 1; /// The maximum integer that can be used in canonical JSON. pub const MAX_VALID_INTEGER: i64 = (2i64.pow(53)) - 1; +/// A token used by `serde_json` to identify its internal `Number` type when the +/// `arbitrary_precision` feature is enabled. This is a copy from serde_json's +/// internal `TOKEN` for `Number`, which unfortunately isn't exported by the +/// crate. +const SERDE_JSON_NUMBER_TOKEN: &str = "$serde_json::private::Number"; + /// Options to control how strict JSON canonicalization is. #[derive(Clone, Debug)] pub struct CanonicalizationOptions { @@ -236,7 +242,7 @@ where type SerializeMap = CanonicalSerializeMap<'a, W>; - type SerializeStruct = CanonicalSerializeMap<'a, W>; + type SerializeStruct = CanonicalSerializeStruct<'a, W>; type SerializeStructVariant = <&'a mut Serializer as serde::Serializer>::SerializeStructVariant; @@ -426,7 +432,7 @@ where fn serialize_struct( self, name: &'static str, - _len: usize, + len: usize, ) -> Result { // We want to disallow `RawValue` as we don't know if its contents is // canonical JSON. @@ -436,10 +442,7 @@ where if name == "$serde_json::private::RawValue" { return Err(Self::Error::custom("`RawValue` is not supported")); } - Ok(CanonicalSerializeMap::new( - &mut self.inner, - self.options.clone(), - )) + CanonicalSerializeStruct::new(name, len, &mut self.inner, self.options.clone()) } fn serialize_struct_variant( @@ -554,7 +557,42 @@ where } } -impl<'a, W> SerializeStruct for CanonicalSerializeMap<'a, W> +/// A helper type for [`CanonicalSerializer`] that serializes structs in +/// lexicographic order. +#[doc(hidden)] +pub struct CanonicalSerializeStruct<'a, W: Write> { + name: &'static str, + // We buffer up the key and serialized value for each field we see. + // The BTreeMap will then serialize in lexicographic order. + map: BTreeMap<&'static str, Box>, + options: CanonicalizationOptions, + // The serializer to use to write the sorted map too. + struct_serializer: + <&'a mut Serializer as serde::Serializer>::SerializeStruct, +} + +impl<'a, W> CanonicalSerializeStruct<'a, W> +where + W: Write, +{ + fn new( + name: &'static str, + len: usize, + ser: &'a mut Serializer, + options: CanonicalizationOptions, + ) -> Result { + let struct_serializer = ser.serialize_struct(name, len)?; + + Ok(Self { + name, + map: BTreeMap::new(), + options, + struct_serializer, + }) + } +} + +impl<'a, W> SerializeStruct for CanonicalSerializeStruct<'a, W> where W: Write, { @@ -566,20 +604,69 @@ where where T: Serialize + ?Sized, { - let key_string = key.to_string(); + // Check if this is the special case of `SERDE_JSON_NUMBER_TOKEN`, + // which is used when serializing numbers with the `arbitrary_precision` + // feature. If so, we can just serialize it directly without + // canonicalizing it first, as `serde_json` will have already serialized + // it in a canonical way. + if key == SERDE_JSON_NUMBER_TOKEN && self.name == SERDE_JSON_NUMBER_TOKEN { + if self.options.enforce_int_range { + // We need to check that the number is in the valid range, as + // `serde_json` won't have done this for us as we're using the + // `arbitrary_precision` feature. + + // The value here will be something that serializes to a JSON + // string containing the number, so we first serialize it to a + // Value and pull the string out, then parse it as a `Number`. + + let serde_val = serde_json::to_value(value)?; + let serde_json::Value::String(number_str) = serde_val else { + return Err(serde_json::Error::custom("invalid number")); + }; + + let number: Number = number_str + .parse() + .map_err(|_| serde_json::Error::custom("invalid number"))?; + + // Now check that the number is an integer in the valid range. + if let Some(int) = number.as_i64() { + assert_integer_in_range(int)?; + } else { + // Can't be cast to an i64, so it must be out of range. + return Err(serde_json::Error::custom("integer out of range")); + } + } + + self.struct_serializer.serialize_field(key, value)?; + return Ok(()); + } // We serialize the value canonically, then store it as a `RawValue` in // the buffer map. let value_string = to_string_canonical(value, self.options.clone())?; - self.map - .insert(key_string, RawValue::from_string(value_string)?); + self.map.insert(key, RawValue::from_string(value_string)?); Ok(()) } - fn end(self) -> Result { - self.map.serialize(self.ser)?; + fn end(mut self) -> Result { + if self.name == SERDE_JSON_NUMBER_TOKEN { + // Map must be empty in this case, as `SERDE_JSON_NUMBER_TOKEN` + // only has one field and we've handled it in `serialize_field`. + if !self.map.is_empty() { + return Err(Self::Error::custom(format!( + "unexpected fields in `{}`", + SERDE_JSON_NUMBER_TOKEN + ))); + } + } + + for (key, value) in self.map { + self.struct_serializer.serialize_field(key, &value)?; + } + + SerializeStruct::end(self.struct_serializer)?; Ok(()) } @@ -737,6 +824,23 @@ mod tests { assert!(to_string_canonical(&-(2i128.pow(60)), CanonicalizationOptions::strict()).is_err()); } + #[test] + fn bigints() { + // Create a `serde_json::Number` that is too big to be represented as an + // i64, but can be represented as a string. + let bigint_string = "10000000000000000000000000000000000000"; + let value: serde_json::Number = bigint_string.parse().unwrap(); + + // This should work with relaxed option. + assert_eq!( + to_string_canonical(&value, CanonicalizationOptions::relaxed()).unwrap(), + bigint_string + ); + + // But should fail with strict option, as it's out of range. + assert!(to_string_canonical(&value, CanonicalizationOptions::strict()).is_err()); + } + #[test] fn backwards_compatibility() { assert_eq!( diff --git a/rust/src/events/internal_metadata.rs b/rust/src/events/internal_metadata.rs index 6fd3d06b00b..4084b8442d2 100644 --- a/rust/src/events/internal_metadata.rs +++ b/rust/src/events/internal_metadata.rs @@ -476,7 +476,7 @@ impl EventInternalMetadataInner { } } -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] #[derive(Clone)] pub struct EventInternalMetadata { inner: Arc>, diff --git a/rust/src/events/unsigned.rs b/rust/src/events/unsigned.rs index c41ed7e6e14..5aa56812c03 100644 --- a/rust/src/events/unsigned.rs +++ b/rust/src/events/unsigned.rs @@ -23,6 +23,7 @@ use pyo3::{ }; use pythonize::{depythonize, pythonize}; use serde::{Deserialize, Serialize}; +use serde_json::Number; #[pyclass(frozen, skip_from_py_object)] #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -36,7 +37,7 @@ pub struct Unsigned { #[derive(Debug, Clone, Serialize, Deserialize, Default)] struct PersistedUnsignedFields { #[serde(skip_serializing_if = "Option::is_none")] - age_ts: Option, + age_ts: Option, #[serde(skip_serializing_if = "Option::is_none")] replaces_state: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -129,11 +130,14 @@ impl Unsigned { let unsigned = self.py_read()?; match field { - UnsignedField::AgeTs => Ok(unsigned - .persisted_fields - .age_ts - .ok_or_else(|| PyKeyError::new_err("age_ts"))? - .into_bound_py_any(py)?), + UnsignedField::AgeTs => { + let age_ts = &unsigned + .persisted_fields + .age_ts + .as_ref() + .ok_or_else(|| PyKeyError::new_err("age_ts"))?; + Ok(pythonize(py, age_ts)?) + } UnsignedField::ReplacesState => Ok((unsigned.persisted_fields.replaces_state) .as_ref() .ok_or_else(|| PyKeyError::new_err("replaces_state"))? @@ -203,7 +207,7 @@ impl Unsigned { let mut unsigned = self.py_write()?; match field { - UnsignedField::AgeTs => unsigned.persisted_fields.age_ts = Some(value.extract()?), + UnsignedField::AgeTs => unsigned.persisted_fields.age_ts = Some(depythonize(&value)?), UnsignedField::ReplacesState => { unsigned.persisted_fields.replaces_state = Some(value.extract()?) } @@ -339,7 +343,7 @@ mod tests { #[test] fn test_persisted_fields_serialize_populated() { let fields = PersistedUnsignedFields { - age_ts: Some(1234), + age_ts: Some(1234.into()), replaces_state: Some("$prev:example.com".to_string()), invite_room_state: Some(vec![json!({"type": "m.room.name"})]), knock_room_state: Some(vec![json!({"type": "m.room.topic"})]), @@ -360,7 +364,7 @@ mod tests { fn test_unsigned_inner_flattens_persisted_fields() { let inner = UnsignedInner { persisted_fields: PersistedUnsignedFields { - age_ts: Some(99), + age_ts: Some(99.into()), ..Default::default() }, prev_content: Some(Box::new(json!({"body": "hi"}))), @@ -382,7 +386,7 @@ mod tests { fn test_unsigned_inner_roundtrip() { let original = UnsignedInner { persisted_fields: PersistedUnsignedFields { - age_ts: Some(10), + age_ts: Some(10.into()), replaces_state: Some("$state:example.com".to_string()), invite_room_state: None, knock_room_state: None, @@ -394,7 +398,7 @@ mod tests { let json = serde_json::to_string(&original).unwrap(); let roundtripped: UnsignedInner = serde_json::from_str(&json).unwrap(); - assert_eq!(roundtripped.persisted_fields.age_ts, Some(10)); + assert_eq!(roundtripped.persisted_fields.age_ts, Some(10.into())); assert_eq!( roundtripped.persisted_fields.replaces_state.as_deref(), Some("$state:example.com") @@ -423,7 +427,7 @@ mod tests { }); let unsigned: Unsigned = serde_json::from_value(json).unwrap(); let inner = unsigned.inner.read().unwrap(); - assert_eq!(inner.persisted_fields.age_ts, Some(5)); + assert_eq!(inner.persisted_fields.age_ts, Some(5.into())); assert_eq!(inner.prev_sender.as_deref(), Some("@bob:example.com")); } } diff --git a/rust/src/push/mod.rs b/rust/src/push/mod.rs index ac9b9c93e42..780d7a8cbd8 100644 --- a/rust/src/push/mod.rs +++ b/rust/src/push/mod.rs @@ -104,7 +104,7 @@ fn get_base_rule_ids() -> HashSet<&'static str> { /// A single push rule for a user. #[derive(Debug, Clone)] -#[pyclass(frozen)] +#[pyclass(frozen, from_py_object)] pub struct PushRule { /// A unique ID for this rule pub rule_id: Cow<'static, str>, @@ -462,7 +462,7 @@ pub struct RelatedEventMatchTypeCondition { /// The collection of push rules for a user. #[derive(Debug, Clone, Default)] -#[pyclass(frozen)] +#[pyclass(frozen, from_py_object)] pub struct PushRules { /// Custom push rules that override a base rule. overridden_base_rules: HashMap, PushRule>, @@ -549,7 +549,7 @@ impl PushRules { /// A wrapper around `PushRules` that checks the enabled state of rules and /// filters out disabled experimental rules. #[derive(Debug, Clone, Default)] -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] pub struct FilteredPushRules { push_rules: PushRules, enabled_map: BTreeMap, diff --git a/rust/src/room_versions.rs b/rust/src/room_versions.rs index dbc962174dd..47473cf200d 100644 --- a/rust/src/room_versions.rs +++ b/rust/src/room_versions.rs @@ -93,7 +93,7 @@ impl PushRuleRoomFlag { } /// An object which describes the unique attributes of a room version. -#[pyclass(frozen, eq, hash, get_all)] +#[pyclass(frozen, eq, hash, get_all, skip_from_py_object)] #[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct RoomVersion { /// The identifier for this version. @@ -622,7 +622,7 @@ const ROOM_VERSION_MSC4242V12: RoomVersion = RoomVersion { /// Note: room versions can be added to this mapping at startup (allowing /// support for experimental room versions to be behind experimental feature /// flags). -#[pyclass(frozen, mapping)] +#[pyclass(frozen, mapping, skip_from_py_object)] #[derive(Clone)] pub struct KnownRoomVersionsMapping { // Note we use a Vec here to ensure that the order of keys is @@ -637,19 +637,22 @@ pub struct KnownRoomVersionsMapping { impl KnownRoomVersionsMapping { /// Add a new room version to the mapping, indicating that this instance /// supports it. - fn add_room_version(&self, version: RoomVersion) -> PyResult<()> { + fn add_room_version(&self, version: Bound<'_, RoomVersion>) -> PyResult<()> { let mut versions = self .versions .write() .map_err(|_| PyRuntimeError::new_err("KnownRoomVersionsMapping lock poisoned"))?; - if versions.iter().any(|v| v.identifier == version.identifier) { + if versions + .iter() + .any(|v| v.identifier == version.get().identifier) + { // We already have this room version, so we don't add it again (as // otherwise we'd end up with duplicates). return Ok(()); } - versions.push(version); + versions.push(*version.get()); Ok(()) } diff --git a/tests/synapse_rust/test_unsigned.py b/tests/synapse_rust/test_unsigned.py new file mode 100644 index 00000000000..5193188f387 --- /dev/null +++ b/tests/synapse_rust/test_unsigned.py @@ -0,0 +1,52 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from synapse.synapse_rust.events import Unsigned + +from tests import unittest + + +class UnsignedTestCase(unittest.TestCase): + def test_prev_content(self) -> None: + """Test that the prev_content field is correctly exposed as a JsonObject.""" + unsigned = Unsigned({"prev_content": {"key1": "value1", "key2": 42}}) + + self.assert_dict(unsigned["prev_content"], {"key1": "value1", "key2": 42}) + + self.assert_dict( + unsigned.for_event(), {"prev_content": {"key1": "value1", "key2": 42}} + ) + + def test_large_age_ts(self) -> None: + """Test that we can handle integers larger than 2^128, which is larger + than the maximum rust native integer size.""" + + large_int = 2**200 + unsigned = Unsigned({"age_ts": large_int}) + + self.assertEqual(unsigned["age_ts"], large_int) + + self.assert_dict(unsigned.for_event(), {"age_ts": large_int}) + + def test_large_integer_in_prev_content(self) -> None: + """Test that we can handle integers larger than 2^128 in the + prev_content field, which is a JsonObject and thus can contain arbitrary + JSON.""" + + large_int = 2**200 + unsigned = Unsigned({"prev_content": {"some_field": large_int}}) + + self.assertEqual(unsigned["prev_content"]["some_field"], large_int) + self.assert_dict( + unsigned.for_event(), {"prev_content": {"some_field": large_int}} + ) From f109c259609532ec6b0c4c9f7da4b25ddfa9aed4 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 13 May 2026 12:01:11 +0100 Subject: [PATCH 26/92] 1.153.0rc2 --- CHANGES.md | 7 +++++++ changelog.d/19769.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 14 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/19769.bugfix diff --git a/CHANGES.md b/CHANGES.md index e7bd8f351ab..3a66c6dd39f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +# Synapse 1.153.0rc2 (2026-05-13) + +## Bugfixes + +- Correctly handle arbitrary precision integers in `unsigned` field of events. The bug was introduced in 1.153.0rc1. ([\#19769](https://github.com/element-hq/synapse/issues/19769)) + + # Synapse 1.153.0rc1 (2026-05-08) ## Features diff --git a/changelog.d/19769.bugfix b/changelog.d/19769.bugfix deleted file mode 100644 index 05aae443119..00000000000 --- a/changelog.d/19769.bugfix +++ /dev/null @@ -1 +0,0 @@ -Correctly handle arbitrary precision integers in `unsigned` field of events. diff --git a/debian/changelog b/debian/changelog index 6aa3735603a..bbd164b2a8b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.153.0~rc2) stable; urgency=medium + + * New Synapse release 1.153.0rc2. + + -- Synapse Packaging team Wed, 13 May 2026 12:00:39 +0100 + matrix-synapse-py3 (1.153.0~rc1) stable; urgency=medium * New Synapse release 1.153.0rc1. diff --git a/pyproject.toml b/pyproject.toml index b456adfc606..a7422e19876 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.153.0rc1" +version = "1.153.0rc2" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 16c17f3a420242e53088337d48b1fb55a86e3a8f Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Wed, 13 May 2026 17:26:16 +0200 Subject: [PATCH 27/92] Add CVE IDs to changelog for 1.152.1. (#19778) Since this is just a change log update, I've removed the entire checklist. Please tell me if this is incorrect. --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d9b3f8b2c12..f0488cd68c9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,8 +2,8 @@ ## Security Fixes -- Prevent CPU starvation (Denial of Service) under worker lock contention, additionally capping the `WorkerLock` time out interval to a maximum of 60 seconds. Contributed by Famedly. ([\#19394](https://github.com/element-hq/synapse/issues/19394), ELEMENTSEC-2026-1706, [GHSA-8q93-326v-3m7g](https://github.com/element-hq/synapse/security/advisories/GHSA-8q93-326v-3m7g), CVE pending) -- Prevent pagination ending when a page is full of rejected events. (ELEMENTSEC-2025-1636, [GHSA-6qf2-7x63-mm6v](https://github.com/element-hq/synapse/security/advisories/GHSA-6qf2-7x63-mm6v), CVE pending) +- Prevent CPU starvation (Denial of Service) under worker lock contention, additionally capping the `WorkerLock` time out interval to a maximum of 60 seconds. Contributed by Famedly. ([\#19394](https://github.com/element-hq/synapse/issues/19394), ELEMENTSEC-2026-1706, [GHSA-8q93-326v-3m7g](https://github.com/element-hq/synapse/security/advisories/GHSA-8q93-326v-3m7g), CVE-2026-45078) +- Prevent pagination ending when a page is full of rejected events. (ELEMENTSEC-2025-1636, [GHSA-6qf2-7x63-mm6v](https://github.com/element-hq/synapse/security/advisories/GHSA-6qf2-7x63-mm6v), CVE-2026-45076) # Synapse 1.152.0 (2026-04-28) From ff0420a03c462fd7a4f67ccca76b58578296d142 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 13 May 2026 12:31:38 -0500 Subject: [PATCH 28/92] Improve `We can't get valid state history.` logging (#19765) Add `event_id` so you can actually correlate everything together in the logs. --- changelog.d/19765.misc | 1 + synapse/handlers/federation_event.py | 24 +++++++++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) create mode 100644 changelog.d/19765.misc diff --git a/changelog.d/19765.misc b/changelog.d/19765.misc new file mode 100644 index 00000000000..7ac5375c7eb --- /dev/null +++ b/changelog.d/19765.misc @@ -0,0 +1 @@ +Improve Synapse logging around when someone encounters `We can't get valid state history.` so you can correlate everything by `event_id`. diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index e314180e122..7cbc52d350e 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -1166,9 +1166,10 @@ async def _compute_event_context_with_maybe_missing_prevs( return await self._state_handler.compute_event_context(event) logger.info( - "Event %s is missing prev_events %s: calculating state for a " - "backwards extremity", + "_compute_event_context_with_maybe_missing_prevs(event_id=%s): Event in room %s is missing prev_events %s: " + "calculating state for a backwards extremity", event_id, + room_id, shortstr(missing_prevs), ) # Calculate the state after each of the previous events, and @@ -1186,16 +1187,21 @@ async def _compute_event_context_with_maybe_missing_prevs( # Ask the remote server for the states we don't # know about - for p in missing_prevs: - logger.info("Requesting state after missing prev_event %s", p) + for missing_prev in missing_prevs: + logger.info( + "_compute_event_context_with_maybe_missing_prevs(event_id=%s): Requesting state from %s for missing prev_event %s", + event_id, + dest, + missing_prev, + ) - with nested_logging_context(p): + with nested_logging_context(missing_prev): # note that if any of the missing prevs share missing state or # auth events, the requests to fetch those events are deduped # by the get_pdu_cache in federation_client. remote_state_map = ( await self._get_state_ids_after_missing_prev_event( - dest, room_id, p + dest, room_id, missing_prev ) ) @@ -1225,7 +1231,11 @@ async def _compute_event_context_with_maybe_missing_prevs( except Exception as e: logger.warning( - "Error attempting to resolve state at missing prev_events: %s", e + "_compute_event_context_with_maybe_missing_prevs(event_id=%s): Error attempting to resolve state from " + "%s for missing prev_events: %s", + event_id, + dest, + e, ) raise FederationError( "ERROR", From b90a0e9fe99b2f66e0bb2911ac88f22437646590 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2026 10:29:16 +0100 Subject: [PATCH 29/92] Use `StrCollection` for `prev_state_events`. (#19777) Convert `prev_state_events` to use `StrCollection` rather than requiring it to be a mutable list. None of the usages require it to be a proper list, and besides, events are immutable and therefore so should `event.prev_state_events`. --- changelog.d/19777.misc | 1 + synapse/events/__init__.py | 2 +- synapse/events/builder.py | 2 +- synapse/handlers/message.py | 9 +++++---- 4 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 changelog.d/19777.misc diff --git a/changelog.d/19777.misc b/changelog.d/19777.misc new file mode 100644 index 00000000000..cd049f8153d --- /dev/null +++ b/changelog.d/19777.misc @@ -0,0 +1 @@ +Use `StrCollection` for `prev_state_events`. diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 5be0298c30f..fcb169e0796 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -618,7 +618,7 @@ class FrozenEventVMSC4242(FrozenEventV4): """FrozenEventVMSC4242, which differs from FrozenEventV4 only in the addition of prev_state_events""" format_version = EventFormatVersions.ROOM_VMSC4242 - prev_state_events: DictProperty[list[str]] = DictProperty("prev_state_events") + prev_state_events: DictProperty[StrCollection] = DictProperty("prev_state_events") def __init__( self, diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 78eb98e1e59..742bb82da99 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -132,7 +132,7 @@ async def build( prev_event_ids: list[str], auth_event_ids: list[str] | None, depth: int | None = None, - prev_state_events: list[str] | None = None, + prev_state_events: StrCollection | None = None, ) -> EventBase: """Transform into a fully signed and hashed event diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 0687c9fa790..56c639f5ef1 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -79,6 +79,7 @@ Requester, RoomAlias, StateMap, + StrCollection, StreamToken, UserID, create_requester, @@ -589,7 +590,7 @@ async def create_event( state_map: StateMap[str] | None = None, for_batch: bool = False, current_state_group: int | None = None, - prev_state_events: list[str] | None = None, + prev_state_events: StrCollection | None = None, delay_id: str | None = None, ) -> tuple[EventBase, UnpersistedEventContextBase]: """ @@ -982,7 +983,7 @@ async def create_and_send_nonmember_event( ignore_shadow_ban: bool = False, outlier: bool = False, depth: int | None = None, - prev_state_events: list[str] | None = None, + prev_state_events: StrCollection | None = None, delay_id: str | None = None, ) -> tuple[EventBase, int]: """ @@ -1127,7 +1128,7 @@ async def _create_and_send_nonmember_event_locked( ignore_shadow_ban: bool = False, outlier: bool = False, depth: int | None = None, - prev_state_events: list[str] | None = None, + prev_state_events: StrCollection | None = None, delay_id: str | None = None, ) -> tuple[EventBase, int]: room_id = event_dict["room_id"] @@ -1253,7 +1254,7 @@ async def create_new_client_event( state_map: StateMap[str] | None = None, for_batch: bool = False, current_state_group: int | None = None, - prev_state_events: list[str] | None = None, + prev_state_events: StrCollection | None = None, ) -> tuple[EventBase, UnpersistedEventContextBase]: """Create a new event for a local client. If bool for_batch is true, will create an event using the prev_event_ids, and will create an event context for From ace8447037a2d206fc1385b2d4969e6fb7445aa0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2026 11:21:00 +0100 Subject: [PATCH 30/92] Tidy up Rust RoomVersion structs (#19766) This is in prep for using the room versions more from Rust. Main changes: - Change it so each room version is defined as a delta to the last one. This is a cosmetic change that makes it easier to ensure the room version definitions are correct (as they're defined as deltas from previous versions). - Move constants to `RoomVersion` constants, like `RoomVersion::V1`, for convenience. - Change visibility of various attributes. --- changelog.d/19766.misc | 1 + rust/src/events/json_object.rs | 8 +- rust/src/room_versions.rs | 755 +++++++++++---------------------- 3 files changed, 257 insertions(+), 507 deletions(-) create mode 100644 changelog.d/19766.misc diff --git a/changelog.d/19766.misc b/changelog.d/19766.misc new file mode 100644 index 00000000000..699852daa21 --- /dev/null +++ b/changelog.d/19766.misc @@ -0,0 +1 @@ +Tidy up Rust `RoomVersion` structs. diff --git a/rust/src/events/json_object.rs b/rust/src/events/json_object.rs index 2c4be1c87bf..0ab54e8dc56 100644 --- a/rust/src/events/json_object.rs +++ b/rust/src/events/json_object.rs @@ -34,7 +34,7 @@ use serde::{Deserialize, Serialize}; /// 1. Fast access of top-level keys (e.g. `event.content["key"]`) /// 2. Pure Rust implementation. #[derive(Serialize, Deserialize, Clone, Default)] -#[pyclass(mapping, frozen)] +#[pyclass(mapping, frozen, skip_from_py_object)] #[serde(transparent)] pub struct JsonObject { object: Arc, serde_json::Value>>, @@ -197,7 +197,7 @@ impl JsonObject { /// of the object. /// /// This needs to both be iterable *and* operate like a set. -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] #[derive(Clone)] pub struct JsonObjectKeysView { object: JsonObject, @@ -397,7 +397,7 @@ impl JsonObjectKeysView { /// Helper class returned by `JsonObject.values()` to act as a view into the /// values of the object. -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] #[derive(Clone)] pub struct JsonObjectValuesView { object: JsonObject, @@ -440,7 +440,7 @@ impl JsonObjectValuesView { /// unless the values are unhashable. Since the values are immutable we could /// support it, but it's more work and nobody seems to actually use the set /// operations on `dict_items` in practice. -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] #[derive(Clone)] pub struct JsonObjectItemsView { object: JsonObject, diff --git a/rust/src/room_versions.rs b/rust/src/room_versions.rs index 47473cf200d..2860bfdbd73 100644 --- a/rust/src/room_versions.rs +++ b/rust/src/room_versions.rs @@ -14,7 +14,11 @@ //! Rust implementation of room version definitions. -use std::sync::{Arc, LazyLock, RwLock}; +use std::{ + fmt::Display, + str::FromStr, + sync::{Arc, LazyLock, RwLock}, +}; use pyo3::{ exceptions::{PyKeyError, PyRuntimeError}, @@ -30,47 +34,47 @@ use pyo3::{ /// versions that they were used or introduced in. /// The concept of an 'event format version' is specific to Synapse (the /// specification does not mention this term.) -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] pub struct EventFormatVersions {} #[pymethods] impl EventFormatVersions { /// $id:server event id format: used for room v1 and v2 #[classattr] - const ROOM_V1_V2: i32 = 1; + pub const ROOM_V1_V2: i32 = 1; /// MSC1659-style $hash event id format: used for room v3 #[classattr] - const ROOM_V3: i32 = 2; + pub const ROOM_V3: i32 = 2; /// MSC1884-style $hash format: introduced for room v4 #[classattr] - const ROOM_V4_PLUS: i32 = 3; + pub const ROOM_V4_PLUS: i32 = 3; /// MSC4291 room IDs as hashes: introduced for room HydraV11 #[classattr] - const ROOM_V11_HYDRA_PLUS: i32 = 4; + pub const ROOM_V11_HYDRA_PLUS: i32 = 4; /// MSC4242 state DAGs: adds prev_state_events, removes auth_events #[classattr] - const ROOM_VMSC4242: i32 = 5; + pub const ROOM_VMSC4242: i32 = 5; } /// Enum to identify the state resolution algorithms. -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] pub struct StateResolutionVersions {} #[pymethods] impl StateResolutionVersions { /// Room v1 state res #[classattr] - const V1: i32 = 1; + pub const V1: i32 = 1; /// MSC1442 state res: room v2 and later #[classattr] - const V2: i32 = 2; + pub const V2: i32 = 2; /// MSC4297 state res #[classattr] - const V2_1: i32 = 3; + pub const V2_1: i32 = 3; } /// Room disposition constants. -#[pyclass(frozen)] +#[pyclass(frozen, skip_from_py_object)] pub struct RoomDisposition {} #[pymethods] @@ -94,18 +98,18 @@ impl PushRuleRoomFlag { /// An object which describes the unique attributes of a room version. #[pyclass(frozen, eq, hash, get_all, skip_from_py_object)] -#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct RoomVersion { /// The identifier for this version. pub identifier: &'static str, - /// One of the RoomDisposition constants. + /// One of the [`RoomDisposition`] constants. pub disposition: &'static str, - /// One of the EventFormatVersions constants. + /// One of the [`EventFormatVersions`] constants. pub event_format: i32, - /// One of the StateResolutionVersions constants. + /// One of the [`StateResolutionVersions`] constants. pub state_res: i32, pub enforce_key_validity: bool, - /// Before MSC2432, m.room.aliases had special auth rules and redaction rules. + /// Before MSC2432, `m.room.aliases` had special auth rules and redaction rules. pub special_case_aliases_auth: bool, /// Strictly enforce canonicaljson, do not allow: /// * Integers outside the range of [-2^53 + 1, 2^53 - 1] @@ -159,459 +163,204 @@ pub struct RoomVersion { pub msc4242_state_dags: bool, } -const ROOM_VERSION_V1: RoomVersion = RoomVersion { - identifier: "1", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V1_V2, - state_res: StateResolutionVersions::V1, - enforce_key_validity: false, - special_case_aliases_auth: true, - strict_canonicaljson: false, - limit_notifications_power_levels: false, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: false, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; - -const ROOM_VERSION_V2: RoomVersion = RoomVersion { - identifier: "2", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V1_V2, - state_res: StateResolutionVersions::V2, - enforce_key_validity: false, - special_case_aliases_auth: true, - strict_canonicaljson: false, - limit_notifications_power_levels: false, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: false, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; - -const ROOM_VERSION_V3: RoomVersion = RoomVersion { - identifier: "3", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V3, - state_res: StateResolutionVersions::V2, - enforce_key_validity: false, - special_case_aliases_auth: true, - strict_canonicaljson: false, - limit_notifications_power_levels: false, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: false, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; - -const ROOM_VERSION_V4: RoomVersion = RoomVersion { - identifier: "4", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: false, - special_case_aliases_auth: true, - strict_canonicaljson: false, - limit_notifications_power_levels: false, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: false, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; - -const ROOM_VERSION_V5: RoomVersion = RoomVersion { - identifier: "5", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: true, - strict_canonicaljson: false, - limit_notifications_power_levels: false, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: false, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; - -const ROOM_VERSION_V6: RoomVersion = RoomVersion { - identifier: "6", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: false, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; - -const ROOM_VERSION_V7: RoomVersion = RoomVersion { - identifier: "7", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: false, - restricted_join_rule_fix: false, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; - -const ROOM_VERSION_V8: RoomVersion = RoomVersion { - identifier: "8", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: true, - restricted_join_rule_fix: false, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; - -const ROOM_VERSION_V9: RoomVersion = RoomVersion { - identifier: "9", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: false, - enforce_int_power_levels: false, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; - -const ROOM_VERSION_V10: RoomVersion = RoomVersion { - identifier: "10", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; - -/// MSC3389 (Redaction changes for events with a relation) based on room version "10". -const ROOM_VERSION_MSC3389V10: RoomVersion = RoomVersion { - identifier: "org.matrix.msc3389.10", - disposition: RoomDisposition::UNSTABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: true, // Changed from v10 - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: true, - msc4242_state_dags: false, -}; - -/// MSC1767 (Extensible Events) based on room version "10". -const ROOM_VERSION_MSC1767V10: RoomVersion = RoomVersion { - identifier: "org.matrix.msc1767.10", - disposition: RoomDisposition::UNSTABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[PushRuleRoomFlag::EXTENSIBLE_EVENTS], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; +impl RoomVersion { + pub const V1: RoomVersion = RoomVersion { + identifier: "1", + disposition: RoomDisposition::STABLE, + event_format: EventFormatVersions::ROOM_V1_V2, + state_res: StateResolutionVersions::V1, + enforce_key_validity: false, + special_case_aliases_auth: true, + strict_canonicaljson: false, + limit_notifications_power_levels: false, + implicit_room_creator: false, + updated_redaction_rules: false, + restricted_join_rule: false, + restricted_join_rule_fix: false, + knock_join_rule: false, + msc3389_relation_redactions: false, + knock_restricted_join_rule: false, + enforce_int_power_levels: false, + msc3931_push_features: &[], + msc3757_enabled: false, + msc4289_creator_power_enabled: false, + msc4291_room_ids_as_hashes: false, + strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, + }; + + pub const V2: RoomVersion = RoomVersion { + identifier: "2", + state_res: StateResolutionVersions::V2, + ..Self::V1 + }; + + pub const V3: RoomVersion = RoomVersion { + identifier: "3", + event_format: EventFormatVersions::ROOM_V3, + ..Self::V2 + }; + + pub const V4: RoomVersion = RoomVersion { + identifier: "4", + event_format: EventFormatVersions::ROOM_V4_PLUS, + ..Self::V3 + }; + + pub const V5: RoomVersion = RoomVersion { + identifier: "5", + enforce_key_validity: true, + ..Self::V4 + }; + + pub const V6: RoomVersion = RoomVersion { + identifier: "6", + special_case_aliases_auth: false, + strict_canonicaljson: true, + limit_notifications_power_levels: true, + ..Self::V5 + }; + + pub const V7: RoomVersion = RoomVersion { + identifier: "7", + knock_join_rule: true, + ..Self::V6 + }; + + pub const V8: RoomVersion = RoomVersion { + identifier: "8", + restricted_join_rule: true, + ..Self::V7 + }; + + pub const V9: RoomVersion = RoomVersion { + identifier: "9", + restricted_join_rule_fix: true, + ..Self::V8 + }; + + pub const V10: RoomVersion = RoomVersion { + identifier: "10", + knock_restricted_join_rule: true, + enforce_int_power_levels: true, + ..Self::V9 + }; + + /// MSC3389 (Redaction changes for events with a relation) based on room version "10". + pub const MSC3389V10: RoomVersion = RoomVersion { + identifier: "org.matrix.msc3389.10", + disposition: RoomDisposition::UNSTABLE, + msc3389_relation_redactions: true, + strict_event_byte_limits_room_versions: true, + ..Self::V10 + }; + + /// MSC1767 (Extensible Events) based on room version "10". + pub const MSC1767V10: RoomVersion = RoomVersion { + identifier: "org.matrix.msc1767.10", + disposition: RoomDisposition::UNSTABLE, + msc3931_push_features: &[PushRuleRoomFlag::EXTENSIBLE_EVENTS], + ..Self::V10 + }; + + /// MSC3757 (Restricting who can overwrite a state event) based on room version "10". + pub const MSC3757V10: RoomVersion = RoomVersion { + identifier: "org.matrix.msc3757.10", + disposition: RoomDisposition::UNSTABLE, + msc3757_enabled: true, + ..Self::V10 + }; + + pub const V11: RoomVersion = RoomVersion { + identifier: "11", + implicit_room_creator: true, // Used by MSC3820 + updated_redaction_rules: true, // Used by MSC3820 + strict_event_byte_limits_room_versions: true, + ..Self::V10 + }; + + /// MSC3757 (Restricting who can overwrite a state event) based on room version "11". + pub const MSC3757V11: RoomVersion = RoomVersion { + identifier: "org.matrix.msc3757.11", + disposition: RoomDisposition::UNSTABLE, + msc3757_enabled: true, + ..Self::V11 + }; + + pub const HYDRA_V11: RoomVersion = RoomVersion { + identifier: "org.matrix.hydra.11", + disposition: RoomDisposition::UNSTABLE, + event_format: EventFormatVersions::ROOM_V11_HYDRA_PLUS, + state_res: StateResolutionVersions::V2_1, + msc4289_creator_power_enabled: true, + msc4291_room_ids_as_hashes: true, + ..Self::V11 + }; + + pub const V12: RoomVersion = RoomVersion { + identifier: "12", + disposition: RoomDisposition::STABLE, + event_format: EventFormatVersions::ROOM_V11_HYDRA_PLUS, + state_res: StateResolutionVersions::V2_1, + msc4289_creator_power_enabled: true, + msc4291_room_ids_as_hashes: true, + ..Self::V11 + }; + + pub const MSC4242V12: RoomVersion = RoomVersion { + identifier: "org.matrix.msc4242.12", + disposition: RoomDisposition::UNSTABLE, + event_format: EventFormatVersions::ROOM_VMSC4242, + msc4242_state_dags: true, + ..Self::V12 + }; +} -/// MSC3757 (Restricting who can overwrite a state event) based on room version "10". -const ROOM_VERSION_MSC3757V10: RoomVersion = RoomVersion { - identifier: "org.matrix.msc3757.10", - disposition: RoomDisposition::UNSTABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: false, - updated_redaction_rules: false, - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: true, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: false, - msc4242_state_dags: false, -}; +impl Display for RoomVersion { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.identifier.fmt(f) + } +} -const ROOM_VERSION_V11: RoomVersion = RoomVersion { - identifier: "11", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: true, // Used by MSC3820 - updated_redaction_rules: true, // Used by MSC3820 - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: true, // Changed from v10 - msc4242_state_dags: false, -}; +impl FromStr for &'static RoomVersion { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "1" => Ok(&RoomVersion::V1), + "2" => Ok(&RoomVersion::V2), + "3" => Ok(&RoomVersion::V3), + "4" => Ok(&RoomVersion::V4), + "5" => Ok(&RoomVersion::V5), + "6" => Ok(&RoomVersion::V6), + "7" => Ok(&RoomVersion::V7), + "8" => Ok(&RoomVersion::V8), + "9" => Ok(&RoomVersion::V9), + "10" => Ok(&RoomVersion::V10), + "11" => Ok(&RoomVersion::V11), + "12" => Ok(&RoomVersion::V12), + "org.matrix.msc1767.10" => Ok(&RoomVersion::MSC1767V10), + "org.matrix.msc3389.10" => Ok(&RoomVersion::MSC3389V10), + "org.matrix.msc3757.10" => Ok(&RoomVersion::MSC3757V10), + "org.matrix.msc3757.11" => Ok(&RoomVersion::MSC3757V11), + "org.matrix.hydra.11" => Ok(&RoomVersion::HYDRA_V11), + "org.matrix.msc4242.12" => Ok(&RoomVersion::MSC4242V12), + _ => Err(anyhow::anyhow!("Unknown room version: {}", s)), + } + } +} -/// MSC3757 (Restricting who can overwrite a state event) based on room version "11". -const ROOM_VERSION_MSC3757V11: RoomVersion = RoomVersion { - identifier: "org.matrix.msc3757.11", - disposition: RoomDisposition::UNSTABLE, - event_format: EventFormatVersions::ROOM_V4_PLUS, - state_res: StateResolutionVersions::V2, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: true, // Used by MSC3820 - updated_redaction_rules: true, // Used by MSC3820 - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: true, - msc4289_creator_power_enabled: false, - msc4291_room_ids_as_hashes: false, - strict_event_byte_limits_room_versions: true, - msc4242_state_dags: false, -}; +impl<'py> IntoPyObject<'py> for &RoomVersion { + type Target = RoomVersion; -const ROOM_VERSION_HYDRA_V11: RoomVersion = RoomVersion { - identifier: "org.matrix.hydra.11", - disposition: RoomDisposition::UNSTABLE, - event_format: EventFormatVersions::ROOM_V11_HYDRA_PLUS, - state_res: StateResolutionVersions::V2_1, // Changed from v11 - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: true, // Used by MSC3820 - updated_redaction_rules: true, // Used by MSC3820 - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: true, // Changed from v11 - msc4291_room_ids_as_hashes: true, // Changed from v11 - strict_event_byte_limits_room_versions: true, - msc4242_state_dags: false, -}; + type Output = Bound<'py, RoomVersion>; -const ROOM_VERSION_V12: RoomVersion = RoomVersion { - identifier: "12", - disposition: RoomDisposition::STABLE, - event_format: EventFormatVersions::ROOM_V11_HYDRA_PLUS, - state_res: StateResolutionVersions::V2_1, // Changed from v11 - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: true, // Used by MSC3820 - updated_redaction_rules: true, // Used by MSC3820 - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: true, // Changed from v11 - msc4291_room_ids_as_hashes: true, // Changed from v11 - strict_event_byte_limits_room_versions: true, - msc4242_state_dags: false, -}; + type Error = PyErr; -const ROOM_VERSION_MSC4242V12: RoomVersion = RoomVersion { - identifier: "org.matrix.msc4242.12", - disposition: RoomDisposition::UNSTABLE, - event_format: EventFormatVersions::ROOM_VMSC4242, - state_res: StateResolutionVersions::V2_1, - enforce_key_validity: true, - special_case_aliases_auth: false, - strict_canonicaljson: true, - limit_notifications_power_levels: true, - implicit_room_creator: true, - updated_redaction_rules: true, - restricted_join_rule: true, - restricted_join_rule_fix: true, - knock_join_rule: true, - msc3389_relation_redactions: false, - knock_restricted_join_rule: true, - enforce_int_power_levels: true, - msc3931_push_features: &[], - msc3757_enabled: false, - msc4289_creator_power_enabled: true, - msc4291_room_ids_as_hashes: true, - strict_event_byte_limits_room_versions: true, - msc4242_state_dags: true, -}; + fn into_pyobject(self, py: Python<'py>) -> Result { + self.clone().into_pyobject(py) + } +} /// Helper class for managing the known room versions, and providing dict-like /// access to them for Python. @@ -652,7 +401,7 @@ impl KnownRoomVersionsMapping { return Ok(()); } - versions.push(*version.get()); + versions.push(version.get().clone()); Ok(()) } @@ -666,7 +415,7 @@ impl KnownRoomVersionsMapping { versions .iter() .find(|v| v.identifier == key) - .copied() + .cloned() .ok_or_else(|| PyKeyError::new_err(key.to_string())) } @@ -698,7 +447,7 @@ impl KnownRoomVersionsMapping { // like a Set *and* has a stable ordering). We don't depend on this, so // for simplicity we just return a list of the items. let versions = self.versions.read().unwrap(); - Ok(versions.iter().map(|v| (v.identifier, *v)).collect()) + Ok(versions.iter().map(|v| (v.identifier, v.clone())).collect()) } #[pyo3(signature = (key, default=None))] @@ -715,8 +464,8 @@ impl KnownRoomVersionsMapping { }; let versions = self.versions.read().unwrap(); - if let Some(version) = versions.iter().find(|v| v.identifier == key).copied() { - return Ok(Some(version.into_bound_py_any(py)?)); + if let Some(version) = versions.iter().find(|v| v.identifier == key) { + return Ok(Some(version.clone().into_bound_py_any(py)?)); } Ok(default) @@ -757,21 +506,21 @@ impl<'py> IntoPyObject<'py> for &KnownRoomVersionsMapping { /// support all experimental room versions. static KNOWN_ROOM_VERSIONS: LazyLock = LazyLock::new(|| { let vec = vec![ - ROOM_VERSION_V1, - ROOM_VERSION_V2, - ROOM_VERSION_V3, - ROOM_VERSION_V4, - ROOM_VERSION_V5, - ROOM_VERSION_V6, - ROOM_VERSION_V7, - ROOM_VERSION_V8, - ROOM_VERSION_V9, - ROOM_VERSION_V10, - ROOM_VERSION_V11, - ROOM_VERSION_V12, - ROOM_VERSION_MSC3757V10, - ROOM_VERSION_MSC3757V11, - ROOM_VERSION_HYDRA_V11, + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + RoomVersion::V11, + RoomVersion::V12, + RoomVersion::MSC3757V10, + RoomVersion::MSC3757V11, + RoomVersion::HYDRA_V11, ]; KnownRoomVersionsMapping { @@ -783,82 +532,82 @@ static KNOWN_ROOM_VERSIONS: LazyLock = LazyLock::new(| /// /// This should contain all room versions that we know about. #[pyclass(frozen)] -pub struct RoomVersions {} +struct RoomVersions {} #[pymethods] #[allow(non_snake_case)] impl RoomVersions { #[classattr] fn V1(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V1.into_py_any(py) + RoomVersion::V1.into_py_any(py) } #[classattr] fn V2(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V2.into_py_any(py) + RoomVersion::V2.into_py_any(py) } #[classattr] fn V3(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V3.into_py_any(py) + RoomVersion::V3.into_py_any(py) } #[classattr] fn V4(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V4.into_py_any(py) + RoomVersion::V4.into_py_any(py) } #[classattr] fn V5(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V5.into_py_any(py) + RoomVersion::V5.into_py_any(py) } #[classattr] fn V6(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V6.into_py_any(py) + RoomVersion::V6.into_py_any(py) } #[classattr] fn V7(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V7.into_py_any(py) + RoomVersion::V7.into_py_any(py) } #[classattr] fn V8(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V8.into_py_any(py) + RoomVersion::V8.into_py_any(py) } #[classattr] fn V9(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V9.into_py_any(py) + RoomVersion::V9.into_py_any(py) } #[classattr] fn V10(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V10.into_py_any(py) + RoomVersion::V10.into_py_any(py) } #[classattr] fn MSC1767v10(py: Python<'_>) -> PyResult> { - ROOM_VERSION_MSC1767V10.into_py_any(py) + RoomVersion::MSC1767V10.into_py_any(py) } #[classattr] fn MSC3389v10(py: Python<'_>) -> PyResult> { - ROOM_VERSION_MSC3389V10.into_py_any(py) + RoomVersion::MSC3389V10.into_py_any(py) } #[classattr] fn MSC3757v10(py: Python<'_>) -> PyResult> { - ROOM_VERSION_MSC3757V10.into_py_any(py) + RoomVersion::MSC3757V10.into_py_any(py) } #[classattr] fn V11(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V11.into_py_any(py) + RoomVersion::V11.into_py_any(py) } #[classattr] fn MSC3757v11(py: Python<'_>) -> PyResult> { - ROOM_VERSION_MSC3757V11.into_py_any(py) + RoomVersion::MSC3757V11.into_py_any(py) } #[classattr] fn HydraV11(py: Python<'_>) -> PyResult> { - ROOM_VERSION_HYDRA_V11.into_py_any(py) + RoomVersion::HYDRA_V11.into_py_any(py) } #[classattr] fn V12(py: Python<'_>) -> PyResult> { - ROOM_VERSION_V12.into_py_any(py) + RoomVersion::V12.into_py_any(py) } #[classattr] fn MSC4242v12(py: Python<'_>) -> PyResult> { - ROOM_VERSION_MSC4242V12.into_py_any(py) + RoomVersion::MSC4242V12.into_py_any(py) } } From 86a1e73ef4cb28640727a3fd1cf77ca8ba052454 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2026 19:02:33 +0100 Subject: [PATCH 31/92] Consolidate MSC4242 state DAG checks via a `TypeIs` helper (#19774) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reason for the change is to make it easier to support these checks when porting event class to Rust. Previously, code that needed to access `prev_state_events` had to combine a `room_version.msc4242_state_dags` boolean check with an `isinstance(event, FrozenEventVMSC4242)` cast (or `cast()`) for the type checker. Introduce `supports_msc4242_state_dag()` in a new `synapse/events/py_protocol.py` which does both in one step via `TypeIs[MSC4242Event]`, removing the need to import the concrete `FrozenEventVMSC4242` class at every call site. `MSC4242Event` is an `EventBase` subclass used purely for type narrowing — it's marked with a metaclass that rejects `isinstance()` to make accidental runtime use loud. No behavioural change: callers continue to gate on the same room version flag and access the same `prev_state_events` attribute. --- changelog.d/19774.misc | 1 + synapse/event_auth.py | 10 +-- synapse/events/py_protocol.py | 88 +++++++++++++++++++ synapse/handlers/admin.py | 6 +- synapse/handlers/message.py | 8 +- synapse/state/__init__.py | 5 +- synapse/storage/controllers/persist_events.py | 23 ++--- synapse/storage/databases/main/events.py | 9 +- tests/events/test_py_protocol.py | 82 +++++++++++++++++ tests/storage/test_msc4242_state_dag.py | 13 +-- 10 files changed, 210 insertions(+), 35 deletions(-) create mode 100644 changelog.d/19774.misc create mode 100644 synapse/events/py_protocol.py create mode 100644 tests/events/test_py_protocol.py diff --git a/changelog.d/19774.misc b/changelog.d/19774.misc new file mode 100644 index 00000000000..5a2cb4d8001 --- /dev/null +++ b/changelog.d/19774.misc @@ -0,0 +1 @@ +Refactor MSC4242 state DAG checks behind a single `TypeIs` helper to avoid scattered `isinstance` casts. diff --git a/synapse/event_auth.py b/synapse/event_auth.py index fd35da8ba09..d9750651fa4 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -61,7 +61,8 @@ EventFormatVersions, RoomVersion, ) -from synapse.events import FrozenEventVMSC4242, is_creator +from synapse.events import is_creator +from synapse.events.py_protocol import supports_msc4242_state_dag from synapse.state import CREATE_KEY from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.types import ( @@ -187,8 +188,8 @@ async def check_state_independent_auth_rules( return # State DAGs 2. Considering the event's prev_state_events: - if event.room_version.msc4242_state_dags: - prev_state_events_ids = set(cast(FrozenEventVMSC4242, event).prev_state_events) + if supports_msc4242_state_dag(event): + prev_state_events_ids = set(event.prev_state_events) # Fetch all of the `prev_state_events` prev_state_events = {} # Try to load the `prev_state_events` from `batched_auth_events` initially as @@ -515,8 +516,7 @@ def _check_create(event: "EventBase") -> None: raise AuthError(403, "Create event has prev events") # State DAGs 1.2 If it has any prev_state_events, reject. - if event.room_version.msc4242_state_dags: - assert isinstance(event, FrozenEventVMSC4242) + if supports_msc4242_state_dag(event): if len(event.prev_state_events) > 0: raise AuthError(403, "Create event has prev state events") diff --git a/synapse/events/py_protocol.py b/synapse/events/py_protocol.py new file mode 100644 index 00000000000..d9ac8c066f5 --- /dev/null +++ b/synapse/events/py_protocol.py @@ -0,0 +1,88 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# +"""Type-narrowing helpers for `EventBase`. + +`EventBase` subclasses are split by room version (e.g. `FrozenEventV4`, +`FrozenEventVMSC4242`), and certain attributes — such as `prev_state_events` +on MSC4242 events — only exist on a subset of those subclasses. Branching on +`room_version.` at runtime tells us *which* subclass we have, but the +type checker can't see that link without an `isinstance` cast at every call +site. + +This module provides "marker" subclasses of `EventBase` (`MSC4242Event`, +etc.) paired with `TypeIs`-returning predicates (`supports_msc4242_state_dag`, +etc.). A single call to the predicate both performs the room-version check +and narrows the type — replacing the `if room_version.foo: assert +isinstance(event, FrozenEventV...)` idiom. + +The marker classes are *type-only*: their metaclass raises on `isinstance` +so they cannot be misused as real runtime classes. Add new markers and +predicates here when a new room-version feature gates access to additional +attributes. +""" + +import abc +from typing import ( + TYPE_CHECKING, + Sequence, +) + +from typing_extensions import TypeIs + +from synapse.events import EventBase + +if TYPE_CHECKING: + from synapse.events.snapshot import EventContext, EventPersistencePair + + +class _DisableIsInstance(abc.ABCMeta): + """Metaclass which disables isinstance checks on classes which use it, by + making isinstance() raise NotImplementedError. + + This is used to prevent isinstance checks on EventProtocol, which is a + helper class used for type narrowing of EventBase objects, but which should + not be used for isinstance checks itself (as its purely type annotation + rather than a real class). + """ + + def __instancecheck__(cls, instance: object) -> bool: + raise NotImplementedError("Instance cannot be used.") + + +class EventProtocol(EventBase, metaclass=_DisableIsInstance): + """Helper subclass that allows type narrowing for `EventBase` objects.""" + + +class MSC4242Event(EventProtocol): + """Marker protocol for events in MSC4242 rooms. This allows us to narrow the + type of events.""" + + prev_state_events: list[str] + + +def supports_msc4242_state_dag(event: EventBase) -> TypeIs[MSC4242Event]: + """Returns true if the given event is in a room that supports state DAGs + (MSC4242)""" + + return event.room_version.msc4242_state_dags + + +def all_supports_msc4242_state_dag( + obj: Sequence["EventPersistencePair"], +) -> TypeIs[Sequence[tuple[MSC4242Event, "EventContext"]]]: + """Returns true if the given sequence of events are all in a room that + supports state DAGs (MSC4242)""" + + return all(event.room_version.msc4242_state_dags for event, _ in obj) diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index 51a752472f1..0b1251f9e3e 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -32,7 +32,8 @@ from synapse.api.constants import Direction, EventTypes, Membership from synapse.api.errors import SynapseError -from synapse.events import EventBase, FrozenEventVMSC4242 +from synapse.events import EventBase +from synapse.events.py_protocol import supports_msc4242_state_dag from synapse.events.utils import FilteredEvent from synapse.types import ( JsonMapping, @@ -495,8 +496,7 @@ async def _redact_all_events( try: prev_state_events = None - if room_version.msc4242_state_dags: - assert isinstance(event, FrozenEventVMSC4242) + if supports_msc4242_state_dag(event): prev_state_events = event.prev_state_events assert prev_state_events is not None, ( "Parent event of redaction has no `prev_state_events` which should be impossible as `prev_state_events` is a required field in MSC4242 rooms" diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 56c639f5ef1..bade219ddf9 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -53,8 +53,9 @@ from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.api.urls import ConsentURIBuilder from synapse.event_auth import validate_event_for_room_version -from synapse.events import EventBase, FrozenEventVMSC4242, relation_from_event +from synapse.events import EventBase, relation_from_event from synapse.events.builder import EventBuilder +from synapse.events.py_protocol import supports_msc4242_state_dag from synapse.events.snapshot import ( EventContext, EventPersistencePair, @@ -1603,8 +1604,7 @@ async def handle_new_client_event( auth_event = event_id_to_event.get(event_id) if auth_event: batched_auth_events[event_id] = auth_event - if event.room_version.msc4242_state_dags: - assert isinstance(event, FrozenEventVMSC4242) + if supports_msc4242_state_dag(event): # State DAG rooms will check that the prev_state_events are not rejected. # To do that, we need to make sure we pass in the prev_state_events as # batched_auth_events, else we will fail the event due to the @@ -1873,7 +1873,7 @@ async def cache_joined_hosts_for_events( state_entry = await self.state.resolve_state_groups_for_events( event.room_id, event_ids=event.prev_state_events - if isinstance(event, FrozenEventVMSC4242) + if supports_msc4242_state_dag(event) else event.prev_event_ids(), ) diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 2f0e3f2c3e5..b22f91b37c6 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -37,7 +37,8 @@ from synapse.api.constants import EventTypes from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, StateResolutionVersions -from synapse.events import EventBase, FrozenEventVMSC4242 +from synapse.events import EventBase +from synapse.events.py_protocol import supports_msc4242_state_dag from synapse.events.snapshot import ( EventContext, UnpersistedEventContext, @@ -315,7 +316,7 @@ async def calculate_context_info( # might redundantly recalculate the state for this event later.) prev_event_ids = frozenset( event.prev_state_events - if isinstance(event, FrozenEventVMSC4242) + if supports_msc4242_state_dag(event) else event.prev_event_ids() ) incomplete_prev_events = await self.store.get_partial_state_events( diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index 7cc6a39639a..62e84f5ac5c 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -34,6 +34,7 @@ Generator, Generic, Iterable, + Sequence, TypeVar, cast, ) @@ -46,7 +47,14 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.errors import SynapseError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS -from synapse.events import EventBase, FrozenEventVMSC4242, event_exists_in_state_dag +from synapse.events import ( + EventBase, + event_exists_in_state_dag, +) +from synapse.events.py_protocol import ( + MSC4242Event, + all_supports_msc4242_state_dag, +) from synapse.events.snapshot import EventContext, EventPersistencePair from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable @@ -637,7 +645,6 @@ async def _persist_event_batch( # Get the room version for the first event. This room version is the same for all events # as events_and_contexts is all for one room. assert len(events_and_contexts) > 0 - room_version = events_and_contexts[0][0].room_version for chunk in chunks: # We can't easily parallelize these since different chunks @@ -648,22 +655,18 @@ async def _persist_event_batch( new_state_dag_extrems = None if not backfilled: - if room_version.msc4242_state_dags: + if all_supports_msc4242_state_dag(chunk): with Measure( self._clock, name="_process_state_dag_forward_extremities_and_state_delta", server_name=self.server_name, ): - assert all( - isinstance(ev, FrozenEventVMSC4242) for ev, _ in chunk - ) ( new_forward_extremities, # for prev_events state_delta_for_room, # for state groups new_state_dag_extrems, # for prev_state_events ) = await self._process_state_dag_forward_extremities_and_state_delta( - room_id, - cast(list[tuple[FrozenEventVMSC4242, EventContext]], chunk), + room_id, chunk ) else: with Measure( @@ -840,7 +843,7 @@ async def _calculate_new_forward_extremities_and_state_delta( async def _process_state_dag_forward_extremities_and_state_delta( self, room_id: str, - event_contexts: list[tuple[FrozenEventVMSC4242, EventContext]], + event_contexts: Sequence[tuple[MSC4242Event, EventContext]], ) -> tuple[set[str] | None, DeltaState | None, set[str] | None]: """Process the forwards extremities for state DAG rooms. Returns: @@ -933,7 +936,7 @@ async def _calculate_new_state_dag_extremities( self, room_id: str, existing_fwd_extrems: frozenset[str], - event_contexts: list[tuple[FrozenEventVMSC4242, EventContext]], + event_contexts: Sequence[tuple[MSC4242Event, EventContext]], ) -> set[str]: """Calculate the new state dag forward extremities. Modifies existing_fwd_extrems. diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 5f6b03a988d..5aab0067fc9 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -48,12 +48,12 @@ from synapse.api.room_versions import RoomVersions from synapse.events import ( EventBase, - FrozenEventVMSC4242, StrippedStateEvent, event_exists_in_state_dag, is_creator, relation_from_event, ) +from synapse.events.py_protocol import MSC4242Event, supports_msc4242_state_dag from synapse.events.snapshot import EventPersistencePair from synapse.events.utils import parse_stripped_state_event from synapse.logging.opentracing import trace @@ -2897,10 +2897,7 @@ def _update_metadata_tables_txn( self._handle_event_relations(txn, event) - if event.room_version.msc4242_state_dags and event_exists_in_state_dag( - event - ): - assert isinstance(event, FrozenEventVMSC4242) + if supports_msc4242_state_dag(event) and event_exists_in_state_dag(event): self._store_state_dag_edges(txn, event) # Store the labels for this event. @@ -2980,7 +2977,7 @@ def local_prefill() -> None: txn.call_after(local_prefill) def _store_state_dag_edges( - self, txn: LoggingTransaction, event: FrozenEventVMSC4242 + self, txn: LoggingTransaction, event: MSC4242Event ) -> None: # the create event has no edge but we still need to persist it as get_state_dag just # yanks all rows in this table. It's a bit gross to store NULL as the prev_state_event_id diff --git a/tests/events/test_py_protocol.py b/tests/events/test_py_protocol.py new file mode 100644 index 00000000000..306e3c17042 --- /dev/null +++ b/tests/events/test_py_protocol.py @@ -0,0 +1,82 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + +from unittest.mock import Mock + +from synapse.api.room_versions import RoomVersion, RoomVersions +from synapse.events import EventBase, FrozenEvent, make_event_from_dict +from synapse.events.py_protocol import ( + EventProtocol, + MSC4242Event, + all_supports_msc4242_state_dag, + supports_msc4242_state_dag, +) + +from tests.unittest import TestCase + + +def _make_event(room_version: RoomVersion) -> EventBase: + """Helper to make an EventBase with the given room version.""" + event_dict = { + "content": {}, + "sender": "@user:example.com", + "type": "m.room.message", + "room_id": "!room:example.com", + } + if room_version.msc4242_state_dags: + event_dict["prev_state_events"] = [] + return make_event_from_dict(event_dict, room_version=room_version) + + +class TestMetaClass(TestCase): + def test_is_instance(self) -> None: + """Test that isinstance checks on EventProtocol raise + NotImplementedError, but that isinstance checks on EventBase and + FrozenEvent still work as normal. + """ + # EventBase and FrozenEvent should work as normal + self.assertFalse(isinstance(object(), EventBase)) + self.assertFalse(isinstance(object(), FrozenEvent)) + + with self.assertRaises(NotImplementedError): + isinstance(object(), EventProtocol) + + with self.assertRaises(NotImplementedError): + isinstance(object(), MSC4242Event) + + +class SupportsMSC4242StateDagTestCase(TestCase): + def test_single_event_msc4242(self) -> None: + """A single event in an MSC4242 room is recognised.""" + ev = _make_event(RoomVersions.MSC4242v12) + self.assertTrue(supports_msc4242_state_dag(ev)) + + def test_single_event_non_msc4242(self) -> None: + """A single event in a non-MSC4242 room is not recognised.""" + ev = _make_event(RoomVersions.V11) + self.assertFalse(supports_msc4242_state_dag(ev)) + + def test_sequence_all_msc4242(self) -> None: + """A sequence of MSC4242 (event, context) pairs is recognised.""" + pairs = [(_make_event(RoomVersions.MSC4242v12), Mock()) for _ in range(3)] + self.assertTrue(all_supports_msc4242_state_dag(pairs)) + + def test_sequence_mixed(self) -> None: + """A sequence containing any non-MSC4242 event is not recognised.""" + pairs = [ + (_make_event(RoomVersions.MSC4242v12), Mock()), + (_make_event(RoomVersions.V11), Mock()), + ] + self.assertFalse(all_supports_msc4242_state_dag(pairs)) diff --git a/tests/storage/test_msc4242_state_dag.py b/tests/storage/test_msc4242_state_dag.py index 8775e5c8eb7..6720c1d6593 100644 --- a/tests/storage/test_msc4242_state_dag.py +++ b/tests/storage/test_msc4242_state_dag.py @@ -19,7 +19,10 @@ from synapse.api.constants import EventTypes from synapse.api.errors import SynapseError from synapse.api.room_versions import RoomVersions -from synapse.events import FrozenEventVMSC4242, make_event_from_dict +from synapse.events import ( + make_event_from_dict, +) +from synapse.events.py_protocol import MSC4242Event, supports_msc4242_state_dag from synapse.events.snapshot import EventContext from synapse.rest.client import room from synapse.server import HomeServer @@ -152,7 +155,7 @@ def _make_event( id: str, prev_state_events: list[str], rejected: bool = False, - ) -> tuple[FrozenEventVMSC4242, EventContext]: + ) -> tuple[MSC4242Event, EventContext]: ev = make_event_from_dict( { "prev_state_events": prev_state_events, @@ -166,8 +169,8 @@ def _make_event( }, room_version=RoomVersions.MSC4242v12, ) - assert isinstance(ev, FrozenEventVMSC4242) - ev._event_id = id + ev._event_id = id # type: ignore[attr-defined] + assert supports_msc4242_state_dag(ev) ctx = Mock() ctx.rejected = rejected return ev, ctx @@ -175,7 +178,7 @@ def _make_event( def _test( self, current_fwds: list[str], - new_events: list[tuple[FrozenEventVMSC4242, EventContext]], + new_events: list[tuple[MSC4242Event, EventContext]], want_new_extrems: set[str], want_raises: bool = False, ) -> None: From b233892a13a6b21648ffe3e333373004216d73a3 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 14 May 2026 14:53:16 -0500 Subject: [PATCH 32/92] Update `wait_for_stream_token(...)` patterns and fix sync fetching with unbounded token (#19644) Spawning from trying to find the proper way to wait for a token, see https://github.com/element-hq/synapse/pull/19558#discussion_r2977673208 - Update `wait_for_stream_token(...)` patterns so validation/sanitization is handled upstream in usage. - Fix sync waiting for bounded token but using unbounded token to fetch data. Noticed while working on adding the new method. Part of https://github.com/element-hq/synapse/issues/19647 --- changelog.d/19644.bugfix | 1 + synapse/handlers/sliding_sync/__init__.py | 17 +++ synapse/handlers/sync.py | 9 ++ synapse/notifier.py | 45 ++++++- synapse/streams/events.py | 10 ++ synapse/types/__init__.py | 3 + tests/handlers/test_sync.py | 14 ++- tests/test_notifier.py | 136 ++++++++++++++++++++++ 8 files changed, 228 insertions(+), 7 deletions(-) create mode 100644 changelog.d/19644.bugfix create mode 100644 tests/test_notifier.py diff --git a/changelog.d/19644.bugfix b/changelog.d/19644.bugfix new file mode 100644 index 00000000000..73ab4bc63e9 --- /dev/null +++ b/changelog.d/19644.bugfix @@ -0,0 +1 @@ +Fix long-standing but niche bug with sync where it could attempt to fetch data with flawed invalid future tokens. diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index a3443b300cc..2e26bfc43a2 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -150,6 +150,23 @@ async def wait_for_sync_for_user( # events or future events if the user is nefariously, manually modifying the # token. if from_token is not None: + # Work around a bug where older Synapse versions gave out tokens "from the + # future", i.e. that are ahead of the tokens persisted in the DB. This could + # also happen if a user is intentionally messing with the token so this also + # acts as sanitization/validation. + # + # If the token has positions ahead of our persisted positions in the + # database (invalid), then we simply use our max persisted position (recover + # gracefully); instead of waiting for a position that may never come around. + # + # FIXME: For Sliding Sync, instead of bounding the token, we should detect + # the invalid future position and raise a `M_UNKNOWN_POS` error. + from_token = SlidingSyncStreamToken( + stream_token=await self.event_sources.bound_future_token( + from_token.stream_token + ), + connection_position=from_token.connection_position, + ) # We need to make sure this worker has caught up with the token. If # this returns false, it means we timed out waiting, and we should # just return an empty response. diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index c88f703ae98..bca51b64b71 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -414,6 +414,15 @@ async def _wait_for_sync_for_user( context.tag = sync_label if since_token is not None: + # Work around a bug where older Synapse versions gave out tokens "from the + # future", i.e. that are ahead of the tokens persisted in the DB. This could + # also happen if a user is intentionally messing with the token so this also + # acts as sanitization/validation. + # + # If the token has positions ahead of our persisted positions in the + # database (invalid), then we simply use our max persisted position (recover + # gracefully); instead of waiting for a position that may never come around. + since_token = await self.event_sources.bound_future_token(since_token) # We need to make sure this worker has caught up with the token. If # this returns false it means we timed out waiting, and we should # just return an empty response. diff --git a/synapse/notifier.py b/synapse/notifier.py index f1cec74462f..6a057ac09fa 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -832,15 +832,49 @@ async def check_for_updates( return result async def wait_for_stream_token(self, stream_token: StreamToken) -> bool: - """Wait for this worker to catch up with the given stream token.""" + """ + Wait for this worker to catch up with the given stream token. + + This is important to ensure that the worker has a proper view of the world + before trying to serve a request. For example, one worker can return a response + with some `next_batch` token, but then the next request goes to another worker + which is behind; if the worker assembles a response up to the token, it could be + missing data in the gap between where it's behind and the requested token. + + ### Inavlid future tokens + + We assume the token has already been validated/sanitized before being passed to + this function to ensure it's not some invalid future token. We consider a token + invalid, if the token has positions ahead of our persisted positions in the + database. This is important as we we don't want to wait for the stream to + advance in those cases (as it may never do so) (it's a waste of time for the + user and server). + + Previously, we would sanitize and `bound_future_token(...)` within this function + but that leads to bad patterns upstream where people can continue to use the + unbounded token. + + While it was possible for older Synapse versions to erroneously give out invalid + future tokens, this is no longer the case and its considered a Synapse + programming error if this ever happens. Validation/sanitization is still + necessary as a user can intentionally mess with numbers in the tokens being + provided. + + Args: + stream_token: The token to wait for. We assume the token has already been + validated/sanitized to ensure it's not some invalid future token (has a + stream position ahead of what is in the DB). (see details above) + + Returns: + True when this worker has caught up + False when we timed out waiting + """ current_token = self.event_sources.get_current_token() + # Return early if we are already caught up if stream_token.is_before_or_eq(current_token): return True - # Work around a bug where older Synapse versions gave out tokens "from - # the future", i.e. that are ahead of the tokens persisted in the DB. - stream_token = await self.event_sources.bound_future_token(stream_token) - + # Start waiting until we've caught up to the `stream_token` start = self.clock.time_msec() logged = False while True: @@ -850,6 +884,7 @@ async def wait_for_stream_token(self, stream_token: StreamToken) -> bool: now = self.clock.time_msec() + # Timed out if now - start > 10_000: return False diff --git a/synapse/streams/events.py b/synapse/streams/events.py index d2720fb9592..f5677a20828 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -149,6 +149,16 @@ async def bound_future_token(self, token: StreamToken) -> StreamToken: ].get_max_allocated_token() if max_token < token_value.get_max_stream_pos(): + # Log *something* as we consider this as a Synapse programming error + # (assuming no malicious user manipulation of the token) (we shouldn't be + # handing out future tokens). + # + # We don't assert as the whole point of bounding is so that we can recover + # gracefully. + # + # Old versions of Synapse could advance streams without persisting anything in + # the DB (fixed in https://github.com/element-hq/synapse/pull/17229) and on + # restart, those updates would be lost. logger.error( "Bounding token from the future '%s': token: %s, bound: %s", key, diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 02889795bbd..8b005ef84d5 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -685,6 +685,9 @@ def is_before_or_eq(self, other_token: Self) -> bool: def bound_stream_token(self, max_stream: int) -> "Self": """Bound the stream positions to a maximum value""" + # Shortcut if we're already under the bound + if self.get_max_stream_pos() <= max_stream: + return self min_pos = min(self.stream, max_stream) return type(self)( diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index b9dee1c9547..e4a4d59aeb3 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -51,6 +51,7 @@ create_requester, ) from synapse.util.clock import Clock +from synapse.util.duration import Duration import tests.unittest import tests.utils @@ -1058,13 +1059,22 @@ def test_wait_for_future_sync_token(self) -> None: ) # This should block waiting for the presence stream to update - self.pump() + # + # Advance time a little bit to make the + # `wait_for_stream_token(...)` sleep loop iterate. + self.reactor.advance(Duration(seconds=2).as_secs()) + # It should still not be done yet self.assertFalse(sync_d.called) # Marking the stream ID as persisted should unblock the request. self.get_success(ctx_mgr.__aexit__(None, None, None)) - self.get_success(sync_d, by=1.0) + # Advance time to make another iteration of `wait_for_stream_token(...)` sleep + # loop so it sees that we're finally caught up now. + self.reactor.advance(Duration(seconds=1).as_secs()) + + # Done waiting + self.get_success(sync_d) @parameterized.expand( [(key,) for key in StreamKeyType.__members__.values()], diff --git a/tests/test_notifier.py b/tests/test_notifier.py new file mode 100644 index 00000000000..c65134e8325 --- /dev/null +++ b/tests/test_notifier.py @@ -0,0 +1,136 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +import logging + +from twisted.internet import defer +from twisted.internet.testing import MemoryReactor + +from synapse.server import HomeServer +from synapse.types import MultiWriterStreamToken, StreamKeyType, StreamToken +from synapse.util.clock import Clock +from synapse.util.duration import Duration + +import tests.unittest + +logger = logging.getLogger(__name__) + + +class NotifierTestCase(tests.unittest.HomeserverTestCase): + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = self.hs.get_datastores().main + self.notifier = self.hs.get_notifier() + + def test_wait_for_stream_token_with_caught_up_token(self) -> None: + """ + Test `wait_for_stream_token` when we receive a token that we are caught up to. + """ + # Create a token + receipt_id_gen = self.store.get_receipts_stream_id_gen() + receipt_token = MultiWriterStreamToken.from_generator(receipt_id_gen) + token = StreamToken.START.copy_and_replace(StreamKeyType.RECEIPT, receipt_token) + + # Function under test + wait_d = defer.ensureDeferred(self.notifier.wait_for_stream_token(token)) + + # Done waiting and caught-up (True) + wait_result = self.get_success(wait_d) + self.assertEqual(wait_result, True) + + def test_wait_for_stream_token_with_future_sync_token(self) -> None: + """ + Test `wait_for_stream_token` when we receive a token that is ahead of our + current token, we'll wait until the stream position advances. + + This can happen if replication streams start lagging, and the client's + previous sync request was serviced by a worker ahead of ours. + """ + # We simulate a lagging stream by getting a stream ID from the ID gen + # and then waiting to mark it as "persisted". + receipt_id_gen = self.store.get_receipts_stream_id_gen() + ctx_mgr = receipt_id_gen.get_next() + receipt_stream_id = self.get_success(ctx_mgr.__aenter__()) + + # Create the new token based on the stream ID above. + current_receipt_token = MultiWriterStreamToken.from_generator(receipt_id_gen) + receipt_token = current_receipt_token.copy_and_advance( + MultiWriterStreamToken(stream=receipt_stream_id) + ) + token = StreamToken.START.copy_and_advance(StreamKeyType.RECEIPT, receipt_token) + + # Function under test + wait_d = defer.ensureDeferred(self.notifier.wait_for_stream_token(token)) + + # This should block waiting for the stream to update + # + # Advance time a little bit to make the + # `wait_for_stream_token(...)` sleep loop iterate. + self.reactor.advance(Duration(seconds=2).as_secs()) + # It should still not be done yet + self.assertFalse(wait_d.called) + + # Marking the stream ID as persisted should unblock the request. + self.get_success(ctx_mgr.__aexit__(None, None, None)) + + # Advance time to make another iteration of + # `wait_for_stream_token(...)` sleep loop so it sees that we're + # finally caught up now. + self.reactor.advance(Duration(seconds=1).as_secs()) + + # Done waiting and caught-up (True) + wait_result = self.get_success(wait_d) + self.assertEqual(wait_result, True) + + def test_wait_for_stream_token_with_future_sync_token_timeout( + self, + ) -> None: + """ + Test `wait_for_stream_token` when we receive a token that is ahead of our + current token, we'll wait until the stream position advances *until* we hit the + timeout. + + This can happen if replication streams start lagging, and the client's + previous sync request was serviced by a worker ahead of ours. + """ + # We simulate a lagging stream by getting a stream ID from the ID gen + # and then waiting to mark it as "persisted". + receipt_id_gen = self.store.get_receipts_stream_id_gen() + ctx_mgr = receipt_id_gen.get_next() + receipt_stream_id = self.get_success(ctx_mgr.__aenter__()) + + # Create the new token based on the stream ID above. + current_receipt_token = MultiWriterStreamToken.from_generator(receipt_id_gen) + receipt_token = current_receipt_token.copy_and_advance( + MultiWriterStreamToken(stream=receipt_stream_id) + ) + token = StreamToken.START.copy_and_advance(StreamKeyType.RECEIPT, receipt_token) + + # Function under test + wait_d = defer.ensureDeferred(self.notifier.wait_for_stream_token(token)) + # Advance time a little bit to make the + # `wait_for_stream_token(...)` sleep loop record 0 as the `start` time. + self.reactor.advance(Duration(seconds=0).as_secs()) + + # This should block waiting for the stream to update + # + # Advance time a little bit to make the + # `wait_for_stream_token(...)` sleep loop iterate. + self.reactor.advance(Duration(seconds=5).as_secs()) + # It should still not be done yet (not enough time to hit the timeout) + self.assertFalse(wait_d.called) + # Advance time past the 10 second timeout (5 + 6 = 11 seconds) to make the + # `wait_for_stream_token(...)` sleep loop give up. + self.reactor.advance(Duration(seconds=6).as_secs()) + + # Make sure we gave up waiting and not caught-up (False) + wait_result = self.get_success(wait_d) + self.assertEqual(wait_result, False) From ff55aff5b292dc3e108b15bd742add730daba89b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 May 2026 10:12:42 +0100 Subject: [PATCH 33/92] Fix up event-construction in tests ahead of the Rust event port (#19781) When we port the `Event` class to Rust, the constructor will check for the existence of required fields. To support that, we tidy up the test code where we construct fake events to add all the required fields. There should be no behavioural changes. Review commit-by-commit. --- changelog.d/19781.misc | 1 + tests/api/test_filtering.py | 86 +++++++------ tests/crypto/test_event_signing.py | 17 +-- tests/events/test_auto_accept_invites.py | 6 +- tests/events/test_utils.py | 104 +++++++++------- tests/federation/test_federation_base.py | 3 + tests/federation/test_federation_client.py | 3 + .../test_federation_out_of_band_membership.py | 30 ++--- tests/federation/test_federation_server.py | 16 +-- tests/federation/transport/test_client.py | 10 ++ tests/handlers/test_federation.py | 26 ++-- tests/handlers/test_federation_event.py | 26 ++-- tests/handlers/test_room_member.py | 10 +- tests/handlers/test_room_policy.py | 19 +-- tests/handlers/test_room_summary.py | 4 +- tests/handlers/test_sync.py | 6 +- tests/push/test_presentable_names.py | 7 +- tests/push/test_push_rule_evaluator.py | 22 +++- .../client/sliding_sync/test_sliding_sync.py | 5 +- tests/rest/client/test_third_party_rules.py | 1 + tests/state/test_v2.py | 25 ++-- tests/state/test_v21.py | 1 + .../databases/main/test_events_worker.py | 6 +- tests/storage/test_events.py | 22 ++-- tests/storage/test_msc4242_state_dag.py | 6 +- tests/storage/test_sliding_sync_tables.py | 5 +- tests/test_event_auth.py | 47 +++---- tests/test_state.py | 5 +- tests/test_utils/event_builders.py | 117 ++++++++++++++++++ 29 files changed, 403 insertions(+), 233 deletions(-) create mode 100644 changelog.d/19781.misc create mode 100644 tests/test_utils/event_builders.py diff --git a/changelog.d/19781.misc b/changelog.d/19781.misc new file mode 100644 index 00000000000..63c2482dedc --- /dev/null +++ b/changelog.d/19781.misc @@ -0,0 +1 @@ +Fix up event-construction in tests ahead of the Rust event port. diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 7742a06b4c9..8c17e98c0dd 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -36,7 +36,7 @@ from synapse.util.frozenutils import freeze from tests import unittest -from tests.events.test_utils import MockEvent +from tests.test_utils.event_builders import make_test_event user_id = UserID.from_string("@test_user:test") user2_id = UserID.from_string("@test_user2:test") @@ -134,18 +134,22 @@ def test_limits_are_applied(self) -> None: def test_definition_types_works_with_literals(self) -> None: definition = {"types": ["m.room.message", "org.matrix.foo.bar"]} - event = MockEvent(sender="@foo:bar", type="m.room.message", room_id="!foo:bar") + event = make_test_event( + sender="@foo:bar", type="m.room.message", room_id="!foo:bar" + ) self.assertTrue(Filter(self.hs, definition)._check(event)) def test_definition_types_works_with_wildcards(self) -> None: definition = {"types": ["m.*", "org.matrix.foo.bar"]} - event = MockEvent(sender="@foo:bar", type="m.room.message", room_id="!foo:bar") + event = make_test_event( + sender="@foo:bar", type="m.room.message", room_id="!foo:bar" + ) self.assertTrue(Filter(self.hs, definition)._check(event)) def test_definition_types_works_with_unknowns(self) -> None: definition = {"types": ["m.room.message", "org.matrix.foo.bar"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="now.for.something.completely.different", room_id="!foo:bar", @@ -154,19 +158,23 @@ def test_definition_types_works_with_unknowns(self) -> None: def test_definition_not_types_works_with_literals(self) -> None: definition = {"not_types": ["m.room.message", "org.matrix.foo.bar"]} - event = MockEvent(sender="@foo:bar", type="m.room.message", room_id="!foo:bar") + event = make_test_event( + sender="@foo:bar", type="m.room.message", room_id="!foo:bar" + ) self.assertFalse(Filter(self.hs, definition)._check(event)) def test_definition_not_types_works_with_wildcards(self) -> None: definition = {"not_types": ["m.room.message", "org.matrix.*"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="org.matrix.custom.event", room_id="!foo:bar" ) self.assertFalse(Filter(self.hs, definition)._check(event)) def test_definition_not_types_works_with_unknowns(self) -> None: definition = {"not_types": ["m.*", "org.*"]} - event = MockEvent(sender="@foo:bar", type="com.nom.nom.nom", room_id="!foo:bar") + event = make_test_event( + sender="@foo:bar", type="com.nom.nom.nom", room_id="!foo:bar" + ) self.assertTrue(Filter(self.hs, definition)._check(event)) def test_definition_not_types_takes_priority_over_types(self) -> None: @@ -174,33 +182,35 @@ def test_definition_not_types_takes_priority_over_types(self) -> None: "not_types": ["m.*", "org.*"], "types": ["m.room.message", "m.room.topic"], } - event = MockEvent(sender="@foo:bar", type="m.room.topic", room_id="!foo:bar") + event = make_test_event( + sender="@foo:bar", type="m.room.topic", room_id="!foo:bar" + ) self.assertFalse(Filter(self.hs, definition)._check(event)) def test_definition_senders_works_with_literals(self) -> None: definition = {"senders": ["@flibble:wibble"]} - event = MockEvent( + event = make_test_event( sender="@flibble:wibble", type="com.nom.nom.nom", room_id="!foo:bar" ) self.assertTrue(Filter(self.hs, definition)._check(event)) def test_definition_senders_works_with_unknowns(self) -> None: definition = {"senders": ["@flibble:wibble"]} - event = MockEvent( + event = make_test_event( sender="@challenger:appears", type="com.nom.nom.nom", room_id="!foo:bar" ) self.assertFalse(Filter(self.hs, definition)._check(event)) def test_definition_not_senders_works_with_literals(self) -> None: definition = {"not_senders": ["@flibble:wibble"]} - event = MockEvent( + event = make_test_event( sender="@flibble:wibble", type="com.nom.nom.nom", room_id="!foo:bar" ) self.assertFalse(Filter(self.hs, definition)._check(event)) def test_definition_not_senders_works_with_unknowns(self) -> None: definition = {"not_senders": ["@flibble:wibble"]} - event = MockEvent( + event = make_test_event( sender="@challenger:appears", type="com.nom.nom.nom", room_id="!foo:bar" ) self.assertTrue(Filter(self.hs, definition)._check(event)) @@ -210,21 +220,21 @@ def test_definition_not_senders_takes_priority_over_senders(self) -> None: "not_senders": ["@misspiggy:muppets"], "senders": ["@kermit:muppets", "@misspiggy:muppets"], } - event = MockEvent( + event = make_test_event( sender="@misspiggy:muppets", type="m.room.topic", room_id="!foo:bar" ) self.assertFalse(Filter(self.hs, definition)._check(event)) def test_definition_rooms_works_with_literals(self) -> None: definition = {"rooms": ["!secretbase:unknown"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown" ) self.assertTrue(Filter(self.hs, definition)._check(event)) def test_definition_rooms_works_with_unknowns(self) -> None: definition = {"rooms": ["!secretbase:unknown"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!anothersecretbase:unknown", @@ -233,7 +243,7 @@ def test_definition_rooms_works_with_unknowns(self) -> None: def test_definition_not_rooms_works_with_literals(self) -> None: definition = {"not_rooms": ["!anothersecretbase:unknown"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!anothersecretbase:unknown", @@ -242,7 +252,7 @@ def test_definition_not_rooms_works_with_literals(self) -> None: def test_definition_not_rooms_works_with_unknowns(self) -> None: definition = {"not_rooms": ["!secretbase:unknown"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!anothersecretbase:unknown", @@ -254,7 +264,7 @@ def test_definition_not_rooms_takes_priority_over_rooms(self) -> None: "not_rooms": ["!secretbase:unknown"], "rooms": ["!secretbase:unknown"], } - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown" ) self.assertFalse(Filter(self.hs, definition)._check(event)) @@ -268,7 +278,7 @@ def test_definition_combined_event(self) -> None: "types": ["m.room.message", "muppets.kermit.*"], "not_types": ["muppets.misspiggy.*"], } - event = MockEvent( + event = make_test_event( sender="@kermit:muppets", # yup type="m.room.message", # yup room_id="!stage:unknown", # yup @@ -284,7 +294,7 @@ def test_definition_combined_event_bad_sender(self) -> None: "types": ["m.room.message", "muppets.kermit.*"], "not_types": ["muppets.misspiggy.*"], } - event = MockEvent( + event = make_test_event( sender="@misspiggy:muppets", # nope type="m.room.message", # yup room_id="!stage:unknown", # yup @@ -300,7 +310,7 @@ def test_definition_combined_event_bad_room(self) -> None: "types": ["m.room.message", "muppets.kermit.*"], "not_types": ["muppets.misspiggy.*"], } - event = MockEvent( + event = make_test_event( sender="@kermit:muppets", # yup type="m.room.message", # yup room_id="!piggyshouse:muppets", # nope @@ -316,7 +326,7 @@ def test_definition_combined_event_bad_type(self) -> None: "types": ["m.room.message", "muppets.kermit.*"], "not_types": ["muppets.misspiggy.*"], } - event = MockEvent( + event = make_test_event( sender="@kermit:muppets", # yup type="muppets.misspiggy.kisses", # nope room_id="!stage:unknown", # yup @@ -325,7 +335,7 @@ def test_definition_combined_event_bad_type(self) -> None: def test_filter_labels(self) -> None: definition = {"org.matrix.labels": ["#fun"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -334,7 +344,7 @@ def test_filter_labels(self) -> None: self.assertTrue(Filter(self.hs, definition)._check(event)) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -344,7 +354,7 @@ def test_filter_labels(self) -> None: self.assertFalse(Filter(self.hs, definition)._check(event)) # check it works with frozen dictionaries too - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -354,7 +364,7 @@ def test_filter_labels(self) -> None: def test_filter_not_labels(self) -> None: definition = {"org.matrix.not_labels": ["#fun"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -363,7 +373,7 @@ def test_filter_not_labels(self) -> None: self.assertFalse(Filter(self.hs, definition)._check(event)) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -375,7 +385,7 @@ def test_filter_not_labels(self) -> None: @unittest.override_config({"experimental_features": {"msc3874_enabled": True}}) def test_filter_rel_type(self) -> None: definition = {"org.matrix.msc3874.rel_types": ["m.thread"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -384,7 +394,7 @@ def test_filter_rel_type(self) -> None: self.assertFalse(Filter(self.hs, definition)._check(event)) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -393,7 +403,7 @@ def test_filter_rel_type(self) -> None: self.assertFalse(Filter(self.hs, definition)._check(event)) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -405,7 +415,7 @@ def test_filter_rel_type(self) -> None: @unittest.override_config({"experimental_features": {"msc3874_enabled": True}}) def test_filter_not_rel_type(self) -> None: definition = {"org.matrix.msc3874.not_rel_types": ["m.thread"]} - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -414,7 +424,7 @@ def test_filter_not_rel_type(self) -> None: self.assertFalse(Filter(self.hs, definition)._check(event)) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -423,7 +433,7 @@ def test_filter_not_rel_type(self) -> None: self.assertTrue(Filter(self.hs, definition)._check(event)) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown", @@ -494,7 +504,9 @@ def test_filter_room_state_match(self) -> None: user_id=user_id, user_filter=user_filter_json ) ) - event = MockEvent(sender="@foo:bar", type="m.room.topic", room_id="!foo:bar") + event = make_test_event( + sender="@foo:bar", type="m.room.topic", room_id="!foo:bar" + ) events = [event] user_filter = self.get_success( @@ -511,7 +523,7 @@ def test_filter_room_state_no_match(self) -> None: user_id=user_id, user_filter=user_filter_json ) ) - event = MockEvent( + event = make_test_event( sender="@foo:bar", type="org.matrix.custom.event", room_id="!foo:bar" ) events = [event] @@ -542,14 +554,14 @@ def test_filter_rooms(self) -> None: def test_filter_relations(self) -> None: events = [ # An event without a relation. - MockEvent( + make_test_event( event_id="$no_relation", sender="@foo:bar", type="org.matrix.custom.event", room_id="!foo:bar", ), # An event with a relation. - MockEvent( + make_test_event( event_id="$with_relation", sender="@foo:bar", type="org.matrix.custom.event", diff --git a/tests/crypto/test_event_signing.py b/tests/crypto/test_event_signing.py index 6ec0f64ffce..e4621b1c2bc 100644 --- a/tests/crypto/test_event_signing.py +++ b/tests/crypto/test_event_signing.py @@ -34,10 +34,11 @@ event_needs_resigning, resign_event, ) -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase from synapse.types import JsonDict from tests import unittest +from tests.test_utils.event_builders import make_test_event # Perform these tests using given secret key so we get entirely deterministic # signatures output that we can test against. @@ -70,7 +71,7 @@ def test_sign_minimal(self) -> None: RoomVersions.V1, event_dict, HOSTNAME, self.signing_key ) - event = make_event_from_dict(event_dict) + event = make_test_event(event_dict) self.assertTrue(hasattr(event, "hashes")) self.assertIn("sha256", event.hashes) @@ -103,7 +104,7 @@ def test_sign_message(self) -> None: RoomVersions.V1, event_dict, HOSTNAME, self.signing_key ) - event = make_event_from_dict(event_dict) + event = make_test_event(event_dict) self.assertTrue(hasattr(event, "hashes")) self.assertIn("sha256", event.hashes) @@ -141,7 +142,7 @@ def test_resign(self) -> None: add_hashes_and_signatures( RoomVersions.V1, event_dict, HOSTNAME, self.signing_key ) - event = make_event_from_dict(event_dict) + event = make_test_event(event_dict) self.assertIn(HOSTNAME, event.signatures) self.assertIn(KEY_NAME, event.signatures[HOSTNAME]) signature = event.signatures[HOSTNAME][KEY_NAME] @@ -171,7 +172,7 @@ def test_resign(self) -> None: "signatures": {}, "unsigned": {"age_ts": 1000000}, } - event = make_event_from_dict(event_dict) + event = make_test_event(event_dict) resigned_event = resign_event(event, HOSTNAME, signing_key_2) self.assertIn(HOSTNAME, resigned_event["signatures"]) self.assertIn(key_name_2, resigned_event["signatures"][HOSTNAME]) @@ -188,7 +189,7 @@ def test_event_needs_resigning(self) -> None: "unsigned": {"age_ts": 1000000}, } internal_metadata: JsonDict = {} - event_that_needs_resigning = make_event_from_dict( + event_that_needs_resigning = make_test_event( event_that_needs_resigning_dict, RoomVersions.V1, internal_metadata, @@ -207,7 +208,7 @@ class TestCase(TypedDict): events_that_dont_need_resigning: list[TestCase] = [ { "name": "sender domain isn't ours", - "event": make_event_from_dict( + "event": make_test_event( {**event_that_needs_resigning_dict, "sender": "@u:somewhereelse"}, RoomVersions.V1, internal_metadata, @@ -215,7 +216,7 @@ class TestCase(TypedDict): }, { "name": "already signed with this key", - "event": make_event_from_dict( + "event": make_test_event( { **event_that_needs_resigning_dict, "signatures": { diff --git a/tests/events/test_auto_accept_invites.py b/tests/events/test_auto_accept_invites.py index e0ebdf0bcac..632d1dc4f64 100644 --- a/tests/events/test_auto_accept_invites.py +++ b/tests/events/test_auto_accept_invites.py @@ -33,7 +33,6 @@ from synapse.config._base import RootConfig from synapse.config.auto_accept_invites import AutoAcceptInvitesConfig from synapse.events.auto_accept_invites import InviteAutoAccepter -from synapse.federation.federation_base import event_from_pdu_json from synapse.handlers.sync import JoinedSyncResult, SyncRequestKey from synapse.module_api import ModuleApi from synapse.rest import admin @@ -43,6 +42,7 @@ from synapse.util.clock import Clock from tests.handlers.test_sync import generate_sync_config +from tests.test_utils.event_builders import make_test_pdu_event from tests.unittest import ( FederatingHomeserverTestCase, HomeserverTestCase, @@ -182,7 +182,7 @@ def test_invite_from_remote_user(self) -> None: ) room_version = self.get_success(self.store.get_room_version(room_id)) - invite_event = event_from_pdu_json( + invite_event = make_test_pdu_event( { "type": EventTypes.Member, "content": {"membership": "invite"}, @@ -308,7 +308,7 @@ def test_accept_invite_local_user( remote_server = "otherserver" remote_user = "@otheruser:" + remote_server - invite_event = event_from_pdu_json( + invite_event = make_test_pdu_event( { "type": EventTypes.Member, "content": {"membership": "invite"}, diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index a402dd70d1f..8f78ae49448 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -26,7 +26,7 @@ from synapse.api.constants import EventContentFields from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase from synapse.events.utils import ( FilteredEvent, PowerLevelsContent, @@ -42,31 +42,16 @@ from synapse.types import JsonDict, create_requester from synapse.util.frozenutils import freeze +from tests.test_utils.event_builders import make_test_event from tests.unittest import HomeserverTestCase if TYPE_CHECKING: from synapse.server import HomeServer -def MockEvent(**kwargs: Any) -> EventBase: - if "event_id" not in kwargs: - kwargs["event_id"] = "fake_event_id" - if "type" not in kwargs: - kwargs["type"] = "fake_type" - if "content" not in kwargs: - kwargs["content"] = {} - - # Move internal metadata out so we can call make_event properly - internal_metadata = kwargs.get("internal_metadata") - if internal_metadata is not None: - kwargs.pop("internal_metadata") - - return make_event_from_dict(kwargs, internal_metadata_dict=internal_metadata) - - class TestMaybeUpsertEventField(stdlib_unittest.TestCase): def test_update_okay(self) -> None: - event = make_event_from_dict({"event_id": "$1234"}) + event = make_test_event({"event_id": "$1234"}) success = maybe_upsert_event_field( event, event.unsigned, "replaces_state", "value" ) @@ -74,7 +59,7 @@ def test_update_okay(self) -> None: self.assertEqual(event.unsigned["replaces_state"], "value") def test_update_not_okay(self) -> None: - event = make_event_from_dict({"event_id": "$1234"}) + event = make_test_event({"event_id": "$1234"}) LARGE_STRING = "a" * 100_000 success = maybe_upsert_event_field( event, event.unsigned, "replaces_state", LARGE_STRING @@ -83,7 +68,7 @@ def test_update_not_okay(self) -> None: self.assertNotIn("replaces_state", event.unsigned) def test_update_not_okay_leaves_original_value(self) -> None: - event = make_event_from_dict( + event = make_test_event( {"event_id": "$1234", "unsigned": {"replaces_state": "value"}} ) LARGE_STRING = "a" * 100_000 @@ -95,6 +80,20 @@ def test_update_not_okay_leaves_original_value(self) -> None: class PruneEventTestCase(stdlib_unittest.TestCase): + # Fields that `make_test_event` fills in by default and that `prune_event` + # preserves as spec-required keep fields. Pruning tests only spell out the + # fields they care about; these are merged into the expected dict so each + # test stays focused on what it is actually checking. + _DEFAULT_KEPT_FIELDS: JsonDict = { + "sender": "@test:test", + "room_id": "!test:test", + "depth": 1, + "origin_server_ts": 1, + "hashes": {}, + "auth_events": [], + "prev_events": [], + } + def run_test(self, evdict: JsonDict, matchdict: JsonDict, **kwargs: Any) -> None: """ Asserts that a new event constructed with `evdict` will look like @@ -105,8 +104,9 @@ def run_test(self, evdict: JsonDict, matchdict: JsonDict, **kwargs: Any) -> None matchdict: The expected resulting dictionary. kwargs: Additional keyword arguments used to create the event. """ + expected = {**self._DEFAULT_KEPT_FIELDS, **matchdict} self.assertEqual( - prune_event(make_event_from_dict(evdict, **kwargs)).get_dict(), matchdict + prune_event(make_test_event(evdict, **kwargs)).get_dict(), expected ) def test_minimal(self) -> None: @@ -123,9 +123,6 @@ def test_minimal(self) -> None: def test_basic_keys(self) -> None: """Ensure that the keys that should be untouched are kept.""" - # Note that some of the values below don't really make sense, but the - # pruning of events doesn't worry about the values of any fields (with - # the exception of the content field). self.run_test( { "event_id": "$3:domain", @@ -134,12 +131,12 @@ def test_basic_keys(self) -> None: "sender": "@2:domain", "state_key": "B", "content": {"other_key": "foo"}, - "hashes": "hashes", + "hashes": {"sha256": "abc"}, "signatures": {"domain": {"algo:1": "sigs"}}, "depth": 4, - "prev_events": "prev_events", + "prev_events": [], "prev_state": "prev_state", - "auth_events": "auth_events", + "auth_events": [], "origin": "domain", # historical top-level field that still exists on old events "origin_server_ts": 1234, "membership": "join", @@ -152,11 +149,11 @@ def test_basic_keys(self) -> None: "room_id": "!1:domain", "sender": "@2:domain", "state_key": "B", - "hashes": "hashes", + "hashes": {"sha256": "abc"}, "depth": 4, - "prev_events": "prev_events", + "prev_events": [], "prev_state": "prev_state", - "auth_events": "auth_events", + "auth_events": [], "origin": "domain", # historical top-level field that still exists on old events "origin_server_ts": 1234, "membership": "join", @@ -625,7 +622,7 @@ def test_relations(self) -> None: class CloneEventTestCase(stdlib_unittest.TestCase): def test_unsigned_is_copied(self) -> None: - original = make_event_from_dict( + original = make_test_event( { "type": "A", "event_id": "$test:domain", @@ -679,7 +676,8 @@ def serialize( def test_event_fields_works_with_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent(sender="@alice:localhost", room_id="!foo:bar"), ["room_id"] + make_test_event(sender="@alice:localhost", room_id="!foo:bar"), + ["room_id"], ), {"room_id": "!foo:bar"}, ) @@ -687,7 +685,7 @@ def test_event_fields_works_with_keys(self) -> None: def test_event_fields_works_with_nested_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( sender="@alice:localhost", room_id="!foo:bar", content={"body": "A message"}, @@ -700,7 +698,7 @@ def test_event_fields_works_with_nested_keys(self) -> None: def test_event_fields_works_with_dot_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( sender="@alice:localhost", room_id="!foo:bar", content={"key.with.dots": {}}, @@ -713,7 +711,7 @@ def test_event_fields_works_with_dot_keys(self) -> None: def test_event_fields_works_with_nested_dot_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( sender="@alice:localhost", room_id="!foo:bar", content={ @@ -729,7 +727,7 @@ def test_event_fields_works_with_nested_dot_keys(self) -> None: def test_event_fields_nops_with_unknown_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( sender="@alice:localhost", room_id="!foo:bar", content={"foo": "bar"}, @@ -742,7 +740,7 @@ def test_event_fields_nops_with_unknown_keys(self) -> None: def test_event_fields_nops_with_non_dict_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( sender="@alice:localhost", room_id="!foo:bar", content={"foo": ["I", "am", "an", "array"]}, @@ -755,7 +753,7 @@ def test_event_fields_nops_with_non_dict_keys(self) -> None: def test_event_fields_nops_with_array_keys(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( sender="@alice:localhost", room_id="!foo:bar", content={"foo": ["I", "am", "an", "array"]}, @@ -768,7 +766,7 @@ def test_event_fields_nops_with_array_keys(self) -> None: def test_event_fields_all_fields_if_empty(self) -> None: self.assertEqual( self.serialize( - MockEvent( + make_test_event( type="foo", event_id="test", room_id="!foo:bar", @@ -782,6 +780,9 @@ def test_event_fields_all_fields_if_empty(self) -> None: "room_id": "!foo:bar", "content": {"foo": "bar"}, "unsigned": {}, + "sender": "@test:test", + "user_id": "@test:test", + "origin_server_ts": 1, }, ) @@ -799,12 +800,12 @@ def test_event_flagged_for_admins(self) -> None: # Default behaviour should be *not* to include it self.assertEqual( self.serialize( - MockEvent( + make_test_event( type="foo", event_id="test", room_id="!foo:bar", content={"foo": "bar"}, - internal_metadata={"soft_failed": True}, + internal_metadata_dict={"soft_failed": True}, ), [], ), @@ -814,18 +815,21 @@ def test_event_flagged_for_admins(self) -> None: "room_id": "!foo:bar", "content": {"foo": "bar"}, "unsigned": {}, + "sender": "@test:test", + "user_id": "@test:test", + "origin_server_ts": 1, }, ) # When asked though, we should set it self.assertEqual( self.serialize( - MockEvent( + make_test_event( type="foo", event_id="test", room_id="!foo:bar", content={"foo": "bar"}, - internal_metadata={"soft_failed": True}, + internal_metadata_dict={"soft_failed": True}, ), [], True, @@ -836,16 +840,19 @@ def test_event_flagged_for_admins(self) -> None: "room_id": "!foo:bar", "content": {"foo": "bar"}, "unsigned": {"io.element.synapse.soft_failed": True}, + "sender": "@test:test", + "user_id": "@test:test", + "origin_server_ts": 1, }, ) self.assertEqual( self.serialize( - MockEvent( + make_test_event( type="foo", event_id="test", room_id="!foo:bar", content={"foo": "bar"}, - internal_metadata={ + internal_metadata_dict={ "soft_failed": True, "policy_server_spammy": True, }, @@ -862,6 +869,9 @@ def test_event_flagged_for_admins(self) -> None: "io.element.synapse.soft_failed": True, "io.element.synapse.policy_server_spammy": True, }, + "sender": "@test:test", + "user_id": "@test:test", + "origin_server_ts": 1, }, ) @@ -896,7 +906,7 @@ def test_redacted_because_is_filtered_out(self) -> None: redaction_id = "$redaction_event_id" - event = MockEvent( + event = make_test_event( type="foo", event_id="test", room_id="!foo:bar", @@ -904,7 +914,7 @@ def test_redacted_because_is_filtered_out(self) -> None: ) event.internal_metadata.redacted_by = redaction_id - redaction_event = MockEvent( + redaction_event = make_test_event( type="m.room.redaction", event_id=redaction_id, content={"redacts": "test"}, diff --git a/tests/federation/test_federation_base.py b/tests/federation/test_federation_base.py index 1bc1da1feb7..8b8c65a0c5e 100644 --- a/tests/federation/test_federation_base.py +++ b/tests/federation/test_federation_base.py @@ -36,6 +36,9 @@ def test_events_signed_by_banned_key_are_refused(self) -> None: "type": "m.room.message", "room_id": "!r:domain", "sender": f"@u:{self.hs.config.server.server_name}", + "auth_events": [], + "prev_events": [], + "depth": 1, "signatures": {}, "unsigned": {"age_ts": 1000000}, } diff --git a/tests/federation/test_federation_client.py b/tests/federation/test_federation_client.py index 0535aed1073..656665cb424 100644 --- a/tests/federation/test_federation_client.py +++ b/tests/federation/test_federation_client.py @@ -73,6 +73,7 @@ def test_get_room_state(self) -> None: "content": {"creator": self.creator}, "prev_events": [], "auth_events": [], + "depth": 1, "origin_server_ts": 500, } ) @@ -85,6 +86,7 @@ def test_get_room_state(self) -> None: "content": {"membership": "join"}, "prev_events": [], "auth_events": [], + "depth": 2, "origin_server_ts": 600, } ) @@ -97,6 +99,7 @@ def test_get_room_state(self) -> None: "content": {}, "prev_events": [], "auth_events": [], + "depth": 3, "origin_server_ts": 700, } ) diff --git a/tests/federation/test_federation_out_of_band_membership.py b/tests/federation/test_federation_out_of_band_membership.py index a1ab72b7a18..1707081863a 100644 --- a/tests/federation/test_federation_out_of_band_membership.py +++ b/tests/federation/test_federation_out_of_band_membership.py @@ -33,11 +33,8 @@ from synapse.api.constants import EventContentFields, EventTypes, Membership from synapse.api.room_versions import RoomVersion, RoomVersions -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase from synapse.events.utils import strip_event -from synapse.federation.federation_base import ( - event_from_pdu_json, -) from synapse.federation.transport.client import SendJoinResponse from synapse.http.matrixfederationclient import ( ByteParser, @@ -53,6 +50,7 @@ from synapse.util.clock import Clock from tests import unittest +from tests.test_utils.event_builders import make_test_event, make_test_pdu_event from tests.utils import test_timeout logger = logging.getLogger(__name__) @@ -80,9 +78,7 @@ def required_state_json_to_state_map(required_state: Any) -> StateMap[EventBase] "Each event in `required_state` should have a string `state_key`" ) - state_map[(event_type, event_state_key)] = make_event_from_dict( - state_event_dict - ) + state_map[(event_type, event_state_key)] = make_test_event(state_event_dict) else: # Yell because we're in a test and this is unexpected raise AssertionError("`required_state` should be a list of event dicts") @@ -195,7 +191,7 @@ def _invite_local_user_to_remote_room_and_join(self) -> RemoteRoomJoinResult: remote_room_id = f"!remote-room:{self.OTHER_SERVER_NAME}" room_version = RoomVersions.V10 - room_create_event = make_event_from_dict( + room_create_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "room_id": remote_room_id, @@ -217,7 +213,7 @@ def _invite_local_user_to_remote_room_and_join(self) -> RemoteRoomJoinResult: room_version=room_version, ) - creator_membership_event = make_event_from_dict( + creator_membership_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "room_id": remote_room_id, @@ -235,7 +231,7 @@ def _invite_local_user_to_remote_room_and_join(self) -> RemoteRoomJoinResult: ) # From the remote homeserver, invite user1 on the local homserver - user1_invite_membership_event = make_event_from_dict( + user1_invite_membership_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "room_id": remote_room_id, @@ -297,7 +293,7 @@ def _invite_local_user_to_remote_room_and_join(self) -> RemoteRoomJoinResult: # Prevent tight-looping to allow the `test_timeout` to work time.sleep(0.1) - user1_join_membership_event_template = make_event_from_dict( + user1_join_membership_event_template = make_test_event( { "room_id": remote_room_id, "sender": local_user1_id, @@ -377,7 +373,7 @@ async def put_json( and parser is not None ): # As the remote server, we need to sign the event before sending it back - user1_join_membership_event_signed = make_event_from_dict( + user1_join_membership_event_signed = make_test_event( self.add_hashes_and_signatures_from_other_server(data), room_version=room_version, ) @@ -404,12 +400,12 @@ async def put_json( if path.startswith("/_matrix/federation/v1/send/") and data is not None: for pdu in data.get("pdus", []): - event = event_from_pdu_json(pdu, room_version) + event = make_test_pdu_event(pdu, room_version) collected_pdus_from_hs1_federation_send.add(event.event_id) # Just acknowledge everything hs1 is trying to send hs2 return { - event_from_pdu_json(pdu, room_version).event_id: {} + make_test_pdu_event(pdu, room_version).event_id: {} for pdu in data.get("pdus", []) } @@ -521,12 +517,12 @@ async def put_json( ) -> JsonDict | T: if path.startswith("/_matrix/federation/v1/send/") and data is not None: for pdu in data.get("pdus", []): - event = event_from_pdu_json(pdu, room_version) + event = make_test_pdu_event(pdu, room_version) collected_pdus_from_hs1_federation_send.add(event.event_id) # Just acknowledge everything hs1 is trying to send hs2 return { - event_from_pdu_json(pdu, room_version).event_id: {} + make_test_pdu_event(pdu, room_version).event_id: {} for pdu in data.get("pdus", []) } @@ -538,7 +534,7 @@ async def put_json( self.federation_http_client.put_json.side_effect = put_json # From the remote homeserver, invite user2 on the local homserver - user2_invite_membership_event = make_event_from_dict( + user2_invite_membership_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "room_id": remote_room_id, diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index 7b1a2a5adc5..1bf0817bd24 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -31,8 +31,7 @@ from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.config.server import DEFAULT_ROOM_VERSION from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import EventBase, make_event_from_dict -from synapse.federation.federation_base import event_from_pdu_json +from synapse.events import EventBase from synapse.http.types import QueryParams from synapse.logging.context import LoggingContext from synapse.rest import admin @@ -43,6 +42,7 @@ from synapse.util.clock import Clock from tests import unittest +from tests.test_utils.event_builders import make_test_event, make_test_pdu_event from tests.unittest import override_config logger = logging.getLogger(__name__) @@ -95,7 +95,7 @@ async def failing_handler(_origin: str, _content: JsonDict) -> None: def _create_acl_event(content: JsonDict) -> EventBase: - return make_event_from_dict( + return make_test_event( { "room_id": "!a:b", "event_id": "$a:b", @@ -147,7 +147,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: # Join a remote user to the room that will attempt to send bad events self.remote_bad_user_id = f"@baduser:{self.OTHER_SERVER_NAME}" - self.remote_bad_user_join_event = make_event_from_dict( + self.remote_bad_user_join_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "room_id": self.room_id, @@ -212,7 +212,7 @@ async def post_json( ) # Now lie about an event's prev_events - lying_event = make_event_from_dict( + lying_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "room_id": self.room_id, @@ -732,7 +732,7 @@ def test_strip_unauthorized_unsigned_values(self) -> None: "auth_events": [], "unsigned": {"malicious garbage": "hackz", "more warez": "more hackz"}, } - filtered_event = event_from_pdu_json(event1, RoomVersions.V1) + filtered_event = make_test_pdu_event(event1, RoomVersions.V1) # Make sure unauthorized fields are stripped from unsigned self.assertNotIn("more warez", filtered_event.unsigned) @@ -754,7 +754,7 @@ def test_strip_event_maintains_allowed_fields(self) -> None: }, } - filtered_event2 = event_from_pdu_json(event2, RoomVersions.V1, received_time=20) + filtered_event2 = make_test_pdu_event(event2, RoomVersions.V1, received_time=20) self.assertIn("age_ts", filtered_event2.unsigned) self.assertEqual(6, filtered_event2.unsigned["age_ts"]) self.assertNotIn("more warez", filtered_event2.unsigned) @@ -779,7 +779,7 @@ def test_strip_event_removes_fields_based_on_event_type(self) -> None: "invite_room_state": [], }, } - filtered_event3 = event_from_pdu_json(event3, RoomVersions.V1, received_time=20) + filtered_event3 = make_test_pdu_event(event3, RoomVersions.V1, received_time=20) self.assertIn("age_ts", filtered_event3.unsigned) # Invite_room_state field is only permitted in event type m.room.member self.assertNotIn("invite_room_state", filtered_event3.unsigned) diff --git a/tests/federation/transport/test_client.py b/tests/federation/transport/test_client.py index 9a6bbabd351..881baa02a9a 100644 --- a/tests/federation/transport/test_client.py +++ b/tests/federation/transport/test_client.py @@ -36,7 +36,16 @@ class SendJoinParserTestCase(TestCase): def test_two_writes(self) -> None: """Test that the parser can sensibly deserialise an input given in two slices.""" parser = SendJoinParser(RoomVersions.V1, True) + common_event_fields = { + "sender": "@user:example.org", + "depth": 1, + "origin_server_ts": 1, + "hashes": {}, + "auth_events": [], + "prev_events": [], + } parent_event = { + **common_event_fields, "content": { "see_room_version_spec": "The event format changes depending on the room version." }, @@ -45,6 +54,7 @@ def test_two_writes(self) -> None: "type": "m.room.minimal_pdu", } state = { + **common_event_fields, "content": { "see_room_version_spec": "The event format changes depending on the room version." }, diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 20ffed68f49..794c0a3185f 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -35,8 +35,7 @@ SynapseError, ) from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase, make_event_from_dict -from synapse.federation.federation_base import event_from_pdu_json +from synapse.events import EventBase from synapse.federation.federation_client import SendJoinResult from synapse.rest import admin from synapse.rest.client import login, room @@ -45,6 +44,7 @@ from synapse.util.clock import Clock from tests import unittest +from tests.test_utils.event_builders import make_test_event, make_test_pdu_event logger = logging.getLogger(__name__) @@ -132,7 +132,7 @@ def test_rejected_message_event_state(self) -> None: ) # build and send an event which will be rejected - ev = event_from_pdu_json( + ev = make_test_pdu_event( { "type": EventTypes.Message, "content": {}, @@ -183,7 +183,7 @@ def test_rejected_state_event_state(self) -> None: ) # build and send an event which will be rejected - ev = event_from_pdu_json( + ev = make_test_pdu_event( { "type": "org.matrix.test", "state_key": "test_key", @@ -227,7 +227,7 @@ def test_backfill_ignores_known_events(self) -> None: room_version = self.get_success(self.store.get_room_version(room_id)) # Build an event to backfill - event = event_from_pdu_json( + event = make_test_pdu_event( { "type": EventTypes.Message, "content": {"body": "hello world", "msgtype": "m.text"}, @@ -324,7 +324,7 @@ def test_invite_by_user_ratelimit(self) -> None: def create_invite() -> EventBase: room_id = self.helper.create_room_as(room_creator=user_id, tok=tok) room_version = self.get_success(self.store.get_room_version(room_id)) - return event_from_pdu_json( + return make_test_pdu_event( { "type": EventTypes.Member, "content": {"membership": "invite"}, @@ -386,7 +386,7 @@ def _build_and_send_join_event( class EventFromPduTestCase(TestCase): def test_valid_json(self) -> None: """Valid JSON should be turned into an event.""" - ev = event_from_pdu_json( + ev = make_test_pdu_event( { "type": EventTypes.Message, "content": {"bool": True, "null": None, "int": 1, "str": "foobar"}, @@ -413,7 +413,7 @@ def test_invalid_numbers(self) -> None: float("nan"), ]: with self.assertRaises(SynapseError): - event_from_pdu_json( + make_test_pdu_event( { "type": EventTypes.Message, "content": {"foo": value}, @@ -430,7 +430,7 @@ def test_invalid_numbers(self) -> None: def test_invalid_nested(self) -> None: """List and dictionaries are recursively searched.""" with self.assertRaises(SynapseError): - event_from_pdu_json( + make_test_pdu_event( { "type": EventTypes.Message, "content": {"foo": [{"bar": 2**56}]}, @@ -457,7 +457,7 @@ def test_failed_partial_join_is_clean(self) -> None: room_id = "!room:example.com" - EVENT_CREATE = make_event_from_dict( + EVENT_CREATE = make_test_event( { "room_id": room_id, "type": "m.room.create", @@ -470,7 +470,7 @@ def test_failed_partial_join_is_clean(self) -> None: }, room_version=RoomVersions.V10, ) - EVENT_CREATOR_MEMBERSHIP = make_event_from_dict( + EVENT_CREATOR_MEMBERSHIP = make_test_event( { "room_id": room_id, "type": "m.room.member", @@ -484,7 +484,7 @@ def test_failed_partial_join_is_clean(self) -> None: }, room_version=RoomVersions.V10, ) - EVENT_INVITATION_MEMBERSHIP = make_event_from_dict( + EVENT_INVITATION_MEMBERSHIP = make_test_event( { "room_id": room_id, "type": "m.room.member", @@ -501,7 +501,7 @@ def test_failed_partial_join_is_clean(self) -> None: }, room_version=RoomVersions.V10, ) - membership_event = make_event_from_dict( + membership_event = make_test_event( { "room_id": room_id, "type": "m.room.member", diff --git a/tests/handlers/test_federation_event.py b/tests/handlers/test_federation_event.py index 1aaa86e2e87..9da9f52fb6f 100644 --- a/tests/handlers/test_federation_event.py +++ b/tests/handlers/test_federation_event.py @@ -28,7 +28,6 @@ check_state_dependent_auth_rules, check_state_independent_auth_rules, ) -from synapse.events import make_event_from_dict from synapse.events.snapshot import EventContext from synapse.federation.transport.client import StateRequestResponse from synapse.logging.context import LoggingContext @@ -42,6 +41,7 @@ from tests import unittest from tests.test_utils import event_injection +from tests.test_utils.event_builders import make_test_event class FederationEventHandlerTests(unittest.FederatingHomeserverTestCase): @@ -127,7 +127,7 @@ def _test_process_pulled_event_with_missing_state( # mock up a load of state events which we are missing state_events = [ - make_event_from_dict( + make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_state_type", @@ -154,7 +154,7 @@ def _test_process_pulled_event_with_missing_state( # mock up a prev event. # Depending on the test, we either persist this upfront (as an outlier), # or let the server request it. - prev_event = make_event_from_dict( + prev_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_regular_type", @@ -191,7 +191,7 @@ async def get_event( self.mock_federation_transport_client.get_event.side_effect = get_event # mock up a regular event to pass into _process_pulled_event - pulled_event = make_event_from_dict( + pulled_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_regular_type", @@ -301,7 +301,7 @@ def test_process_pulled_event_records_failed_backfill_attempts( ) ) - pulled_event = make_event_from_dict( + pulled_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_regular_type", @@ -421,7 +421,7 @@ def test_process_pulled_event_clears_backfill_attempts_after_being_successfully_ member_event.event_id, ] - pulled_event = make_event_from_dict( + pulled_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_regular_type", @@ -524,7 +524,7 @@ def test_backfill_signature_failure_does_not_fetch_same_prev_event_later( # We purposely don't run `add_hashes_and_signatures_from_other_server` # over this because we want the signature check to fail. - pulled_event_without_signatures = make_event_from_dict( + pulled_event_without_signatures = make_test_event( { "type": "test_regular_type", "room_id": room_id, @@ -540,7 +540,7 @@ def test_backfill_signature_failure_does_not_fetch_same_prev_event_later( # Create a regular event that should pass except for the # `pulled_event_without_signatures` in the `prev_event`. - pulled_event = make_event_from_dict( + pulled_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_regular_type", @@ -720,7 +720,7 @@ def test_backfill_process_previously_failed_pull_attempt_event_in_the_background ] # Create a regular event that should process - pulled_event = make_event_from_dict( + pulled_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "test_regular_type", @@ -878,7 +878,7 @@ def test_process_pulled_event_with_rejected_missing_state(self) -> None: # accepted, but the local homeserver will reject. next_depth = 100 next_timestamp = other_member_event.origin_server_ts + 100 - rejected_power_levels_event = make_event_from_dict( + rejected_power_levels_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "m.room.power_levels", @@ -927,7 +927,7 @@ def test_process_pulled_event_with_rejected_missing_state(self) -> None: # Then we create a kick event for a local user that cites the rejected power # levels event in its auth events. The kick event will be rejected solely # because of the rejected auth event and would otherwise be accepted. - rejected_kick_event = make_event_from_dict( + rejected_kick_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "m.room.member", @@ -1042,7 +1042,7 @@ def test_process_pulled_event_with_rejected_missing_state(self) -> None: # Create a missing event, so that the local homeserver has to do a `/state` or # `/state_ids` request to pull state from the remote homeserver. - missing_event = make_event_from_dict( + missing_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "m.room.message", @@ -1067,7 +1067,7 @@ def test_process_pulled_event_with_rejected_missing_state(self) -> None: # The pulled event has two prev events, one of which is missing. We will make a # `/state` or `/state_ids` request to the remote homeserver to ask it for the # state before the missing prev event. - pulled_event = make_event_from_dict( + pulled_event = make_test_event( self.add_hashes_and_signatures_from_other_server( { "type": "m.room.message", diff --git a/tests/handlers/test_room_member.py b/tests/handlers/test_room_member.py index 3890abdbc83..b550c2420b5 100644 --- a/tests/handlers/test_room_member.py +++ b/tests/handlers/test_room_member.py @@ -9,9 +9,6 @@ from synapse.api.errors import Codes, LimitExceededError, SynapseError from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.events import FrozenEventV3 -from synapse.federation.federation_base import ( - event_from_pdu_json, -) from synapse.federation.federation_client import SendJoinResult from synapse.server import HomeServer from synapse.types import UserID, create_requester @@ -19,6 +16,7 @@ from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.server import make_request +from tests.test_utils.event_builders import make_test_pdu_event from tests.unittest import ( FederatingHomeserverTestCase, HomeserverTestCase, @@ -549,7 +547,7 @@ def test_msc4155_block_invite_remote(self) -> None: ) room_version = self.get_success(self.store.get_room_version(room_id)) - invite_event = event_from_pdu_json( + invite_event = make_test_pdu_event( { "type": EventTypes.Member, "content": {"membership": "invite"}, @@ -595,7 +593,7 @@ def test_msc4155_block_invite_remote_server(self) -> None: ) room_version = self.get_success(self.store.get_room_version(room_id)) - invite_event = event_from_pdu_json( + invite_event = make_test_pdu_event( { "type": EventTypes.Member, "content": {"membership": "invite"}, @@ -710,7 +708,7 @@ def test_msc4380_block_invite_remote(self) -> None: ) room_version = self.get_success(self.store.get_room_version(room_id)) - invite_event = event_from_pdu_json( + invite_event = make_test_pdu_event( { "type": EventTypes.Member, "content": {"membership": "invite"}, diff --git a/tests/handlers/test_room_policy.py b/tests/handlers/test_room_policy.py index 9adf0e38c25..2d3e917aef0 100644 --- a/tests/handlers/test_room_policy.py +++ b/tests/handlers/test_room_policy.py @@ -22,7 +22,7 @@ from synapse.api.constants import EventTypes from synapse.api.errors import HttpResponseException, SynapseError from synapse.crypto.event_signing import compute_event_signature -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase from synapse.handlers.room_policy import POLICY_SERVER_KEY_ID from synapse.rest import admin from synapse.rest.client import filter, login, room, sync @@ -32,6 +32,7 @@ from tests import unittest from tests.test_utils import event_injection +from tests.test_utils.event_builders import make_test_event class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): @@ -75,7 +76,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.signing_key = signedjson.key.generate_signing_key("policy_server") # Create some sample events - self.spammy_event = make_event_from_dict( + self.spammy_event = make_test_event( room_version=room_version, internal_metadata_dict={}, event_dict={ @@ -88,7 +89,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: }, }, ) - self.not_spammy_event = make_event_from_dict( + self.not_spammy_event = make_test_event( room_version=room_version, internal_metadata_dict={}, event_dict={ @@ -272,7 +273,7 @@ def test_spammy_event_is_spam(self) -> None: def test_signed_event_is_not_spam(self) -> None: verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) self._add_policy_server_to_room(public_key=verify_key_str) - event = make_event_from_dict( + event = make_test_event( room_version=self.room_version, internal_metadata_dict={}, event_dict={ @@ -302,7 +303,7 @@ def test_signed_event_is_not_spam(self) -> None: def test_ask_policy_server_to_sign_event_ok(self) -> None: verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) self._add_policy_server_to_room(public_key=verify_key_str) - event = make_event_from_dict( + event = make_test_event( room_version=self.room_version, internal_metadata_dict={}, event_dict={ @@ -324,7 +325,7 @@ def test_ask_policy_server_to_sign_event_ok(self) -> None: def test_ask_policy_server_to_sign_event_refuses(self) -> None: verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) self._add_policy_server_to_room(public_key=verify_key_str) - event = make_event_from_dict( + event = make_test_event( room_version=self.room_version, internal_metadata_dict={}, event_dict={ @@ -353,7 +354,7 @@ def test_ask_policy_server_to_sign_event_refuses(self) -> None: def test_ask_policy_server_to_sign_event_cannot_reach(self) -> None: verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) self._add_policy_server_to_room(public_key=verify_key_str) - event = make_event_from_dict( + event = make_test_event( room_version=self.room_version, internal_metadata_dict={}, event_dict={ @@ -379,7 +380,7 @@ def test_ask_policy_server_to_sign_event_wrong_sig(self) -> None: verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) self._add_policy_server_to_room(public_key=verify_key_str) self.mock_federation_transport_client.ask_policy_server_to_sign_event.side_effect = self.policy_server_signs_event_with_wrong_key - unverified_event = make_event_from_dict( + unverified_event = make_test_event( room_version=self.room_version, internal_metadata_dict={}, event_dict={ @@ -398,7 +399,7 @@ def test_ask_policy_server_to_sign_event_wrong_sig(self) -> None: ) self.assertEqual(len(unverified_event.signatures), 1) - verified_event = make_event_from_dict( + verified_event = make_test_event( room_version=self.room_version, internal_metadata_dict={}, event_dict={ diff --git a/tests/handlers/test_room_summary.py b/tests/handlers/test_room_summary.py index ee65cb1afbb..49076e69d8c 100644 --- a/tests/handlers/test_room_summary.py +++ b/tests/handlers/test_room_summary.py @@ -35,7 +35,6 @@ ) from synapse.api.errors import AuthError, NotFoundError, SynapseError from synapse.api.room_versions import RoomVersions -from synapse.events import make_event_from_dict from synapse.federation.transport.client import TransportLayerClient from synapse.handlers.room_summary import _child_events_comparison_key, _RoomEntry from synapse.rest import admin @@ -45,6 +44,7 @@ from synapse.util.clock import Clock from tests import unittest +from tests.test_utils.event_builders import make_test_event from tests.unittest import override_config @@ -217,7 +217,7 @@ def _poke_fed_invite(self, room_id: str, from_user: str) -> None: # Poke an invite over federation into the database. fed_handler = self.hs.get_federation_handler() fed_hostname = UserID.from_string(from_user).domain - event = make_event_from_dict( + event = make_test_event( { "room_id": room_id, "event_id": "!abcd:" + fed_hostname, diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index e4a4d59aeb3..d2b2523321b 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -32,7 +32,6 @@ from synapse.api.room_versions import RoomVersion, RoomVersions from synapse.events import EventBase from synapse.events.snapshot import EventContext -from synapse.federation.federation_base import event_from_pdu_json from synapse.handlers.sync import ( SyncConfig, SyncRequestKey, @@ -55,6 +54,7 @@ import tests.unittest import tests.utils +from tests.test_utils.event_builders import make_test_pdu_event _request_key = 0 @@ -913,7 +913,7 @@ async def _check_sigs_and_hash_for_pulled_events_and_fetch( prev_events = self.get_success(self.store.get_prev_events_for_room(room_id)) # create a call invite event - call_event = event_from_pdu_json( + call_event = make_test_pdu_event( { "type": EventTypes.CallInvite, "content": {}, @@ -961,7 +961,7 @@ async def _check_sigs_and_hash_for_pulled_events_and_fetch( priv_prev_events = self.get_success( self.store.get_prev_events_for_room(private_room_id) ) - private_call_event = event_from_pdu_json( + private_call_event = make_test_pdu_event( { "type": EventTypes.CallInvite, "content": {}, diff --git a/tests/push/test_presentable_names.py b/tests/push/test_presentable_names.py index 2558f2c0b26..2d382c8fabf 100644 --- a/tests/push/test_presentable_names.py +++ b/tests/push/test_presentable_names.py @@ -23,11 +23,12 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase, FrozenEvent +from synapse.events import EventBase from synapse.push.presentable_names import calculate_room_name from synapse.types import StateKey, StateMap from tests import unittest +from tests.test_utils.event_builders import make_test_event class MockDataStore: @@ -44,7 +45,7 @@ def __init__(self, events: Iterable[tuple[StateKey, dict]]): self._events = {} for i, (event_id, content) in enumerate(events): - self._events[event_id] = FrozenEvent( + self._events[event_id] = make_test_event( { "event_id": "$event_id", "type": event_id[0], @@ -59,7 +60,7 @@ def __init__(self, events: Iterable[tuple[StateKey, dict]]): async def get_event( self, event_id: str, allow_none: bool = False - ) -> FrozenEvent | None: + ) -> EventBase | None: assert allow_none, "Mock not configured for allow_none = False" # Decode the state key from the event ID. diff --git a/tests/push/test_push_rule_evaluator.py b/tests/push/test_push_rule_evaluator.py index a786d74bf1a..2e389710b95 100644 --- a/tests/push/test_push_rule_evaluator.py +++ b/tests/push/test_push_rule_evaluator.py @@ -27,7 +27,6 @@ from synapse.api.constants import EventTypes, HistoryVisibility, Membership from synapse.api.room_versions import RoomVersions from synapse.appservice import ApplicationService -from synapse.events import FrozenEvent, make_event_from_dict from synapse.push.bulk_push_rule_evaluator import _flatten_dict from synapse.push.httppusher import tweaks_for_actions from synapse.rest import admin @@ -40,6 +39,7 @@ from synapse.util.frozenutils import freeze from tests import unittest +from tests.test_utils.event_builders import make_test_event from tests.test_utils.event_injection import create_event, inject_member_event @@ -81,7 +81,7 @@ def test_non_string(self) -> None: def test_event(self) -> None: """Events can also be flattened.""" - event = make_event_from_dict( + event = make_test_event( { "room_id": "!test:test", "type": "m.room.message", @@ -103,6 +103,10 @@ def test_event(self) -> None: "room_id": "!test:test", "sender": "@alice:test", "type": "m.room.message", + "depth": 1, + "origin_server_ts": 1, + "auth_events": [], + "prev_events": [], } self.assertEqual(expected, _flatten_dict(event)) @@ -121,24 +125,32 @@ def test_extensible_events(self) -> None: } # For a current room version, there's no special behavior. - event = make_event_from_dict(event_dict, room_version=RoomVersions.V8) + event = make_test_event(event_dict, room_version=RoomVersions.V8) expected = { "room_id": "!test:test", "sender": "@alice:test", "type": "m.room.message", "content.org\\.matrix\\.msc1767\\.markup": [], + "depth": 1, + "origin_server_ts": 1, + "auth_events": [], + "prev_events": [], } self.assertEqual(expected, _flatten_dict(event)) # For a room version with extensible events, they parse out the text/plain # to a content.body property. - event = make_event_from_dict(event_dict, room_version=RoomVersions.MSC1767v10) + event = make_test_event(event_dict, room_version=RoomVersions.MSC1767v10) expected = { "content.body": "hello world!", "room_id": "!test:test", "sender": "@alice:test", "type": "m.room.message", "content.org\\.matrix\\.msc1767\\.markup": [], + "depth": 1, + "origin_server_ts": 1, + "auth_events": [], + "prev_events": [], } self.assertEqual(expected, _flatten_dict(event)) @@ -152,7 +164,7 @@ def _get_evaluator( msc4210: bool = False, msc4306: bool = False, ) -> PushRuleEvaluator: - event = FrozenEvent( + event = make_test_event( { "event_id": "$event_id", "type": "m.room.history_visibility", diff --git a/tests/rest/client/sliding_sync/test_sliding_sync.py b/tests/rest/client/sliding_sync/test_sliding_sync.py index ac8dfd37d8d..ebf41cd87c3 100644 --- a/tests/rest/client/sliding_sync/test_sliding_sync.py +++ b/tests/rest/client/sliding_sync/test_sliding_sync.py @@ -30,7 +30,7 @@ RoomTypes, ) from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase, StrippedStateEvent, make_event_from_dict +from synapse.events import EventBase, StrippedStateEvent from synapse.events.snapshot import EventContext from synapse.handlers.sliding_sync import StateValues from synapse.rest.client import account_data, devices, login, receipts, room, sync @@ -47,6 +47,7 @@ from tests import unittest from tests.server import FakeChannel, TimedOutException +from tests.test_utils.event_builders import make_test_event from tests.test_utils.event_injection import create_event logger = logging.getLogger(__name__) @@ -308,7 +309,7 @@ def _create_remote_invite_room_for_user( "invite_room_state": serialized_stripped_state_events } - invite_event = make_event_from_dict( + invite_event = make_test_event( invite_event_dict, room_version=RoomVersions.V10, ) diff --git a/tests/rest/client/test_third_party_rules.py b/tests/rest/client/test_third_party_rules.py index 1709b27f67c..b4fa71ece8e 100644 --- a/tests/rest/client/test_third_party_rules.py +++ b/tests/rest/client/test_third_party_rules.py @@ -532,6 +532,7 @@ def _send_event_over_federation(self) -> None: "origin_server_ts": self.clock.time_msec(), "prev_events": [], "auth_events": [], + "hashes": {}, "signatures": {}, "unsigned": {}, } diff --git a/tests/state/test_v2.py b/tests/state/test_v2.py index 85ce5bede25..8b3b919f441 100644 --- a/tests/state/test_v2.py +++ b/tests/state/test_v2.py @@ -33,7 +33,7 @@ from synapse.api.constants import EventTypes, JoinRules, Membership from synapse.api.room_versions import RoomVersions from synapse.event_auth import auth_types_for_event -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase from synapse.state.v2 import ( _get_auth_chain_difference, _get_power_level_for_sender, @@ -45,6 +45,7 @@ from synapse.util.duration import Duration from tests import unittest +from tests.test_utils.event_builders import make_test_event ALICE = "@alice:example.com" BOB = "@bob:example.com" @@ -119,7 +120,7 @@ def to_event(self, auth_events: list[str], prev_events: list[str]) -> EventBase: if self.state_key is not None: event_dict["state_key"] = self.state_key - return make_event_from_dict(event_dict) + return make_test_event(event_dict) # All graphs start with this set of events @@ -878,7 +879,7 @@ def test_get_power_level_for_sender(self) -> None: on room version""" store = TestStateResolutionStore({}) for room_version in [RoomVersions.V10, RoomVersions.V11]: - create_event = make_event_from_dict( + create_event = make_test_event( { "room_id": ROOM_ID, "sender": ALICE, @@ -894,9 +895,9 @@ def test_get_power_level_for_sender(self) -> None: else {} ), }, - room_version, + room_version=room_version, ) - member_event = make_event_from_dict( + member_event = make_test_event( { "room_id": ROOM_ID, "sender": ALICE, @@ -908,9 +909,9 @@ def test_get_power_level_for_sender(self) -> None: "auth_events": [create_event.event_id], "prev_events": [create_event.event_id], }, - room_version, + room_version=room_version, ) - pl_event = make_event_from_dict( + pl_event = make_test_event( { "room_id": ROOM_ID, "sender": ALICE, @@ -926,7 +927,7 @@ def test_get_power_level_for_sender(self) -> None: "auth_events": [create_event.event_id, member_event.event_id], "prev_events": [member_event.event_id], }, - room_version, + room_version=room_version, ) event_map = { @@ -940,7 +941,7 @@ def test_get_power_level_for_sender(self) -> None: CHARLIE: 10, } for user_id, want_pl in want_pls.items(): - test_event = make_event_from_dict( + test_event = make_test_event( { "room_id": ROOM_ID, "sender": user_id, @@ -954,7 +955,7 @@ def test_get_power_level_for_sender(self) -> None: ], "prev_events": [pl_event.event_id], }, - room_version, + room_version=room_version, ) event_map[test_event.event_id] = test_event got_pl = self.successResultOf( @@ -977,7 +978,7 @@ def test_get_power_level_for_sender(self) -> None: CHARLIE: 0, } for user_id, want_pl in want_pls.items(): - test_event = make_event_from_dict( + test_event = make_test_event( { "room_id": ROOM_ID, "sender": user_id, @@ -991,7 +992,7 @@ def test_get_power_level_for_sender(self) -> None: ], "prev_events": [pl_event.event_id], }, - room_version, + room_version=room_version, ) got_pl = self.successResultOf( defer.ensureDeferred( diff --git a/tests/state/test_v21.py b/tests/state/test_v21.py index 9f607176c92..6b4d2837abc 100644 --- a/tests/state/test_v21.py +++ b/tests/state/test_v21.py @@ -495,6 +495,7 @@ def create_event( "prev_events": prev_events, "auth_events": auth_events, "origin_server_ts": monotonic_timestamp(), + "hashes": {}, } if event_type != EventTypes.Create: if room_id is None: diff --git a/tests/storage/databases/main/test_events_worker.py b/tests/storage/databases/main/test_events_worker.py index c786271c09a..dd647d29d6a 100644 --- a/tests/storage/databases/main/test_events_worker.py +++ b/tests/storage/databases/main/test_events_worker.py @@ -28,7 +28,6 @@ from twisted.internet.testing import MemoryReactor from synapse.api.room_versions import EventFormatVersions, RoomVersions -from synapse.events import make_event_from_dict from synapse.logging.context import LoggingContext from synapse.rest import admin from synapse.rest.client import login, room @@ -42,6 +41,7 @@ from synapse.util.clock import Clock from tests import unittest +from tests.test_utils.event_builders import make_test_event from tests.test_utils.event_injection import create_event, inject_event @@ -377,7 +377,7 @@ def _populate_events(self) -> None: "type": f"test {idx}", "room_id": self.room_id, } - event = make_event_from_dict(event_json, room_version=RoomVersions.V4) + event = make_test_event(event_json, room_version=RoomVersions.V4) event_id = event.event_id self.get_success( self.store.db_pool.simple_upsert( @@ -400,7 +400,7 @@ def _populate_events(self) -> None: {"event_id": event_id}, { "room_id": self.room_id, - "json": json.dumps(event_json), + "json": json.dumps(event.get_dict()), "internal_metadata": "{}", "format_version": EventFormatVersions.ROOM_V4_PLUS, }, diff --git a/tests/storage/test_events.py b/tests/storage/test_events.py index 7d1c96f97f6..c511ac9252e 100644 --- a/tests/storage/test_events.py +++ b/tests/storage/test_events.py @@ -26,13 +26,13 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.room_versions import RoomVersions from synapse.events import EventBase -from synapse.federation.federation_base import event_from_pdu_json from synapse.rest import admin from synapse.rest.client import login, room from synapse.server import HomeServer from synapse.types import StateMap from synapse.util.clock import Clock +from tests.test_utils.event_builders import make_test_pdu_event from tests.unittest import HomeserverTestCase logger = logging.getLogger(__name__) @@ -142,7 +142,7 @@ def prepare( # Fudge a remote event and persist it. This will be the extremity before # the gap. - self.remote_event_1 = event_from_pdu_json( + self.remote_event_1 = make_test_pdu_event( { "type": EventTypes.Message, "state_key": "@user:other", @@ -198,7 +198,7 @@ def test_prune_gap(self) -> None: # Fudge a second event which points to an event we don't have. This is a # state event so that the state changes (otherwise we won't prune the # extremity as they'll have the same state group). - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": "@user:other", @@ -237,7 +237,7 @@ def test_do_not_prune_gap_if_state_different(self) -> None: ) # Fudge a second event which points to an event we don't have. - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Message, "state_key": "@user:other", @@ -286,7 +286,7 @@ def test_prune_gap_if_old(self) -> None: # Fudge a second event which points to an event we don't have. This is a # state event so that the state changes (otherwise we won't prune the # extremity as they'll have the same state group). - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": "@user:other2", @@ -322,7 +322,7 @@ def test_do_not_prune_gap_if_other_server(self) -> None: # Fudge a second event which points to an event we don't have. This is a # state event so that the state changes (otherwise we won't prune the # extremity as they'll have the same state group). - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": "@user:other2", @@ -367,7 +367,7 @@ def test_prune_gap_if_dummy_remote(self) -> None: # Fudge a second event which points to an event we don't have. This is a # state event so that the state changes (otherwise we won't prune the # extremity as they'll have the same state group). - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": "@user:other2", @@ -415,7 +415,7 @@ def test_prune_gap_if_dummy_local(self) -> None: # Fudge a second event which points to an event we don't have. This is a # state event so that the state changes (otherwise we won't prune the # extremity as they'll have the same state group). - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": "@user:other2", @@ -455,7 +455,7 @@ def test_do_not_prune_gap_if_not_dummy(self) -> None: # Fudge a second event which points to an event we don't have. This is a # state event so that the state changes (otherwise we won't prune the # extremity as they'll have the same state group). - remote_event_2 = event_from_pdu_json( + remote_event_2 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": "@user:other2", @@ -514,7 +514,7 @@ def test_remote_user_rooms_cache_invalidated(self) -> None: # Fudge a join event for a remote user. remote_user = "@user:other" - remote_event_1 = event_from_pdu_json( + remote_event_1 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": remote_user, @@ -561,7 +561,7 @@ def test_room_remote_user_cache_invalidated(self) -> None: # Fudge a join event for a remote user. remote_user = "@user:other" - remote_event_1 = event_from_pdu_json( + remote_event_1 = make_test_pdu_event( { "type": EventTypes.Member, "state_key": remote_user, diff --git a/tests/storage/test_msc4242_state_dag.py b/tests/storage/test_msc4242_state_dag.py index 6720c1d6593..2150bc09960 100644 --- a/tests/storage/test_msc4242_state_dag.py +++ b/tests/storage/test_msc4242_state_dag.py @@ -19,15 +19,13 @@ from synapse.api.constants import EventTypes from synapse.api.errors import SynapseError from synapse.api.room_versions import RoomVersions -from synapse.events import ( - make_event_from_dict, -) from synapse.events.py_protocol import MSC4242Event, supports_msc4242_state_dag from synapse.events.snapshot import EventContext from synapse.rest.client import room from synapse.server import HomeServer from synapse.util.clock import Clock +from tests.test_utils.event_builders import make_test_event from tests.unittest import HomeserverTestCase, override_config @@ -156,7 +154,7 @@ def _make_event( prev_state_events: list[str], rejected: bool = False, ) -> tuple[MSC4242Event, EventContext]: - ev = make_event_from_dict( + ev = make_test_event( { "prev_state_events": prev_state_events, "content": { diff --git a/tests/storage/test_sliding_sync_tables.py b/tests/storage/test_sliding_sync_tables.py index 39226ff9bef..67b8d1138f9 100644 --- a/tests/storage/test_sliding_sync_tables.py +++ b/tests/storage/test_sliding_sync_tables.py @@ -27,7 +27,7 @@ from synapse.api.constants import EventContentFields, EventTypes, Membership, RoomTypes from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase, StrippedStateEvent, make_event_from_dict +from synapse.events import EventBase, StrippedStateEvent from synapse.events.snapshot import EventContext from synapse.rest import admin from synapse.rest.client import login, room, sync @@ -46,6 +46,7 @@ from synapse.util.clock import Clock from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase +from tests.test_utils.event_builders import make_test_event from tests.test_utils.event_injection import create_event logger = logging.getLogger(__name__) @@ -240,7 +241,7 @@ def _retract_remote_invite_for_user( "prev_events": [], } - kick_event = make_event_from_dict( + kick_event = make_test_event( kick_event_dict, room_version=RoomVersions.V10, ) diff --git a/tests/test_event_auth.py b/tests/test_event_auth.py index 9258f0d4dc1..4537186ee6d 100644 --- a/tests/test_event_auth.py +++ b/tests/test_event_auth.py @@ -29,11 +29,12 @@ from synapse.api.constants import EventContentFields, RejectedReason from synapse.api.errors import AuthError, SynapseError from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions -from synapse.events import EventBase, event_exists_in_state_dag, make_event_from_dict +from synapse.events import EventBase, event_exists_in_state_dag from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.types import JsonDict, get_domain_from_id from tests.test_utils import get_awaitable_result +from tests.test_utils.event_builders import make_test_event class _StubEventSourceStore: @@ -128,7 +129,7 @@ def test_create_event_with_prev_events(self) -> None: # we make both a good event and a bad event, to check that we are rejecting # the bad event for the reason we think we are. - good_event = make_event_from_dict( + good_event = make_test_event( { "room_id": TEST_ROOM_ID, "type": "m.room.create", @@ -143,7 +144,7 @@ def test_create_event_with_prev_events(self) -> None: }, room_version=RoomVersions.V9, ) - bad_event = make_event_from_dict( + bad_event = make_test_event( {**good_event.get_dict(), "prev_events": ["$fakeevent"]}, room_version=RoomVersions.V9, ) @@ -387,7 +388,7 @@ def test_msc4242_state_dag_rules(self) -> None: creator = "@creator:example.com" room_version = RoomVersions.MSC4242v12 - create_event = make_event_from_dict( + create_event = make_test_event( { "type": "m.room.create", "sender": creator, @@ -398,7 +399,7 @@ def test_msc4242_state_dag_rules(self) -> None: }, room_version, ) - create_event_2 = make_event_from_dict( + create_event_2 = make_test_event( { "type": "m.room.create", "sender": creator, @@ -411,7 +412,7 @@ def test_msc4242_state_dag_rules(self) -> None: ) room_id = create_event.room_id another_room_id = create_event_2.room_id - join_event = make_event_from_dict( + join_event = make_test_event( { "room_id": room_id, "type": "m.room.member", @@ -424,7 +425,7 @@ def test_msc4242_state_dag_rules(self) -> None: room_version, {"calculated_auth_event_ids": [create_event.event_id]}, ) - event_in_another_room = make_event_from_dict( + event_in_another_room = make_test_event( { "room_id": another_room_id, "type": "m.room.join_rules", @@ -437,7 +438,7 @@ def test_msc4242_state_dag_rules(self) -> None: room_version, {"calculated_auth_event_ids": [create_event.event_id, join_event.event_id]}, ) - msg_event = make_event_from_dict( + msg_event = make_test_event( { "room_id": room_id, "type": "m.room.message", @@ -449,7 +450,7 @@ def test_msc4242_state_dag_rules(self) -> None: room_version, {"calculated_auth_event_ids": [create_event.event_id, join_event.event_id]}, ) - rejected_event = make_event_from_dict( + rejected_event = make_test_event( { "room_id": room_id, "type": "m.room.name", @@ -470,7 +471,7 @@ def test_msc4242_state_dag_rules(self) -> None: RejectingTestCase( name="create event has prev_state_events", events_in_store=[], - test_event=make_event_from_dict( + test_event=make_test_event( { "type": "m.room.create", "sender": creator, @@ -486,7 +487,7 @@ def test_msc4242_state_dag_rules(self) -> None: RejectingTestCase( name="prev_state_event belongs in a different room", events_in_store=[create_event, join_event, event_in_another_room], - test_event=make_event_from_dict( + test_event=make_test_event( { "room_id": room_id, "type": "m.room.name", @@ -508,7 +509,7 @@ def test_msc4242_state_dag_rules(self) -> None: RejectingTestCase( name="prev_state_event is a message event", events_in_store=[create_event, join_event, msg_event], - test_event=make_event_from_dict( + test_event=make_test_event( { "room_id": room_id, "type": "m.room.name", @@ -530,7 +531,7 @@ def test_msc4242_state_dag_rules(self) -> None: RejectingTestCase( name="prev_state_event was rejected", events_in_store=[create_event, join_event, rejected_event], - test_event=make_event_from_dict( + test_event=make_test_event( { "room_id": room_id, "type": "m.room.name", @@ -892,7 +893,7 @@ def test_join_rules_msc3083_restricted(self) -> None: def test_room_v10_rejects_string_power_levels(self) -> None: pl_event_content = {"users_default": "42"} - pl_event = make_event_from_dict( + pl_event = make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(RoomVersions.V10), @@ -906,7 +907,7 @@ def test_room_v10_rejects_string_power_levels(self) -> None: ) pl_event2_content = {"events": {"m.room.name": "42", "m.room.power_levels": 42}} - pl_event2 = make_event_from_dict( + pl_event2 = make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(RoomVersions.V10), @@ -936,7 +937,7 @@ def test_room_v10_rejects_other_non_integer_power_levels(self) -> None: """ def create_event(pl_event_content: dict[str, Any]) -> EventBase: - return make_event_from_dict( + return make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(RoomVersions.V10), @@ -1049,7 +1050,7 @@ def check_events(events: list[dict], should_exist: bool) -> None: "signatures": {"test.com": {"ed25519:0": "some9signature"}}, } base.update(ev) - event = make_event_from_dict(base, RoomVersions.V10) + event = make_test_event(base, RoomVersions.V10) got = event_exists_in_state_dag(event) self.assertEqual( got, should_exist, f"{ev} should_exist={should_exist} but got {got}" @@ -1068,7 +1069,7 @@ def _create_event( room_version: RoomVersion, user_id: str, ) -> EventBase: - return make_event_from_dict( + return make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(room_version), @@ -1089,7 +1090,7 @@ def _member_event( sender: str | None = None, additional_content: dict | None = None, ) -> EventBase: - return make_event_from_dict( + return make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(room_version), @@ -1122,7 +1123,7 @@ def _power_levels_event( sender: str, content: JsonDict, ) -> EventBase: - return make_event_from_dict( + return make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(room_version), @@ -1145,7 +1146,7 @@ def _alias_event(room_version: RoomVersion, sender: str, **kwargs: Any) -> Event "content": {"aliases": []}, } data.update(**kwargs) - return make_event_from_dict(data, room_version=room_version) + return make_test_event(data, room_version=room_version) def _build_auth_dict_for_room_version( @@ -1164,7 +1165,7 @@ def _random_state_event( ) -> EventBase: if auth_events is None: auth_events = [] - return make_event_from_dict( + return make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(room_version), @@ -1181,7 +1182,7 @@ def _random_state_event( def _join_rules_event( room_version: RoomVersion, sender: str, join_rule: str ) -> EventBase: - return make_event_from_dict( + return make_test_event( { "room_id": TEST_ROOM_ID, **_maybe_get_event_id_dict_for_room_version(room_version), diff --git a/tests/test_state.py b/tests/test_state.py index 0ca88aef74c..0a6720d6ef4 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -32,7 +32,7 @@ from synapse.api.auth.internal import InternalAuth from synapse.api.constants import EventTypes, Membership from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase from synapse.events.snapshot import EventContext from synapse.state import StateHandler, StateResolutionHandler, _make_state_cache_entry from synapse.types import JsonDict, MutableStateMap, StateMap @@ -41,6 +41,7 @@ from tests import unittest from tests.server import get_clock +from tests.test_utils.event_builders import make_test_event from tests.utils import default_config _next_event_id = 1000 @@ -82,7 +83,7 @@ def create_event( d.update(kwargs) - return make_event_from_dict(d) + return make_test_event(d) class _DummyStore: diff --git a/tests/test_utils/event_builders.py b/tests/test_utils/event_builders.py new file mode 100644 index 00000000000..a8eb586c1f2 --- /dev/null +++ b/tests/test_utils/event_builders.py @@ -0,0 +1,117 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from typing import TypedDict + +from typing_extensions import NotRequired, Unpack + +from synapse.api.room_versions import ( + RoomVersion, + RoomVersions, +) +from synapse.events import EventBase, make_event_from_dict +from synapse.federation.federation_base import event_from_pdu_json +from synapse.types import JsonDict + + +def default_event_fields(room_version: RoomVersion) -> JsonDict: + """Return default values for every field required by `room_version`.""" + + # We need to include entries for every required field for the room version. + # Note that they don't necessarily have to be valid values, just enough to + # allow us to construct the event class. (Ideally we'd build a fully valid + # event, but this is fine for now.) + defaults: JsonDict = { + "type": "m.test", + "sender": "@test:test", + "content": {}, + "depth": 1, + "origin_server_ts": 1, + "hashes": {}, + "prev_events": [], + "room_id": "!test:test", + } + + # MSC4242 versions require prev_state_events but not auth_events. + if room_version.msc4242_state_dags: + defaults["prev_state_events"] = [] + else: + defaults["auth_events"] = [] + + if room_version == RoomVersions.V1: + # V1 requires an event_id field, but later versions don't. + defaults["event_id"] = "$test_event_id:matrix.org" + + return defaults + + +def make_test_event( + event_dict: JsonDict | None = None, + room_version: RoomVersion = RoomVersions.V1, + internal_metadata_dict: JsonDict | None = None, + rejected_reason: str | None = None, + **fields: Unpack["_EventFields"], +) -> EventBase: + """Build an `EventBase` with defaults for the strict-required fields. + + Pass an `event_dict` and/or `**fields` keyword arguments — both are + merged on top of the format-version defaults from + `default_event_fields`. Explicit values win over defaults, and + `**fields` wins over `event_dict` so call sites can override a + shared base dict with one-off tweaks. + """ + merged: JsonDict = { + **default_event_fields(room_version), + **(event_dict or {}), + **fields, + } + return make_event_from_dict( + merged, + room_version=room_version, + internal_metadata_dict=internal_metadata_dict, + rejected_reason=rejected_reason, + ) + + +def make_test_pdu_event( + pdu: JsonDict, + room_version: RoomVersion, + received_time: int | None = None, +) -> EventBase: + """Wrapper around `event_from_pdu_json` for test PDU dicts. + + Federation-side test fixtures often omit fields the strict Rust ctor + requires (e.g. `hashes`, `auth_events`, `prev_events`, `depth`) + because those tests focus on transport/auth flow rather than event + well-formedness. This helper layers in the same format-version + defaults as `make_test_event` before delegating. + """ + pdu = {**default_event_fields(room_version), **pdu} + return event_from_pdu_json(pdu, room_version, received_time=received_time) + + +class _EventFields(TypedDict): + """Type for `kwargs` in `make_test_event`.""" + + event_id: NotRequired[str] + type: NotRequired[str] + sender: NotRequired[str] + content: NotRequired[JsonDict] + depth: NotRequired[int] + origin_server_ts: NotRequired[int] + hashes: NotRequired[dict[str, str]] + auth_events: NotRequired[list[str]] + prev_events: NotRequired[list[str]] + prev_state_events: NotRequired[list[str]] + room_id: NotRequired[str] From cf64199ea0ebdd7181d8145f5c5a9e904a073a73 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 May 2026 10:24:49 +0000 Subject: [PATCH 34/92] Bump urllib3 from 2.6.3 to 2.7.0 (#19771) Signed-off-by: dependabot[bot] --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index d66b175b0ab..5c8057e675f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3586,14 +3586,14 @@ files = [ [[package]] name = "urllib3" -version = "2.6.3" +version = "2.7.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4"}, - {file = "urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed"}, + {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, + {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, ] [package.extras] From 5c8419eed7aba38aa0799792adaf915ad62c7fe6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 May 2026 10:26:51 +0000 Subject: [PATCH 35/92] Bump authlib from 1.6.11 to 1.6.12 (#19776) Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5c8057e675f..e56155bf702 100644 --- a/poetry.lock +++ b/poetry.lock @@ -26,15 +26,15 @@ files = [ [[package]] name = "authlib" -version = "1.6.11" +version = "1.6.12" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = true python-versions = ">=3.9" groups = ["main"] markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" files = [ - {file = "authlib-1.6.11-py2.py3-none-any.whl", hash = "sha256:c8687a9a26451c51a34a06fa17bb97cb15bba46a6a626755e2d7f50da8bff3e3"}, - {file = "authlib-1.6.11.tar.gz", hash = "sha256:64db35b9b01aeccb4715a6c9a6613a06f2bd7be2ab9d2eb89edd1dfc7580a38f"}, + {file = "authlib-1.6.12-py2.py3-none-any.whl", hash = "sha256:e9229ad7fde610b139dd12f5edbe97eab9ee78bfb85691247e767727850b99ab"}, + {file = "authlib-1.6.12.tar.gz", hash = "sha256:0656d8482f28fc8221929d5f35b2bde5d13e10555ebc06b4561b0d622e83b1bd"}, ] [package.dependencies] From 9ce68a6a4a2913cf399d058ef119f65b88e93fa9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 May 2026 10:29:39 +0000 Subject: [PATCH 36/92] Bump gitpython from 3.1.47 to 3.1.50 (#19767) Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index e56155bf702..78f18caadf9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -582,14 +582,14 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.47" +version = "3.1.50" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "gitpython-3.1.47-py3-none-any.whl", hash = "sha256:489f590edfd6d20571b2c0e72c6a6ac6915ee8b8cd04572330e3842207a78905"}, - {file = "gitpython-3.1.47.tar.gz", hash = "sha256:dba27f922bd2b42cb54c87a8ab3cb6beb6bf07f3d564e21ac848913a05a8a3cd"}, + {file = "gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9"}, + {file = "gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc"}, ] [package.dependencies] From 3f0f03d53653bc97a35e76baf8c98da533516d75 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 15 May 2026 11:36:47 +0100 Subject: [PATCH 37/92] Revert "Send a SSS response immediately if the config has changed and there are new results to sync (#19714)" (#19784) Reverts: #19714 Opens: #19783 Closes: https://github.com/element-hq/backend-internal/issues/242 Related: #18880 (the performance problem that is aggravated by #19714) This reverts commit 2691d0b8b143e01987504210e320198560371435. --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19784.bugfix | 1 + synapse/handlers/sliding_sync/__init__.py | 52 ++++---- synapse/handlers/sliding_sync/room_lists.py | 12 +- .../sliding_sync/test_room_subscriptions.py | 119 ------------------ 4 files changed, 29 insertions(+), 155 deletions(-) create mode 100644 changelog.d/19784.bugfix diff --git a/changelog.d/19784.bugfix b/changelog.d/19784.bugfix new file mode 100644 index 00000000000..c68524d57a9 --- /dev/null +++ b/changelog.d/19784.bugfix @@ -0,0 +1 @@ +Revert 'Have [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) return a new response immediately if a room subscription has changed and produced a new response. ([\#19714](https://github.com/element-hq/synapse/issues/19714))' due to performance problems. diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index 2e26bfc43a2..ddce04d5b32 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -184,38 +184,34 @@ async def wait_for_sync_for_user( timeout_ms -= after_wait_ts - before_wait_ts timeout_ms = max(timeout_ms, 0) - # Compute a response immediately. We always need to do this before - # waiting for new data (unlike in /v3/sync), as the request config might - # have changed (e.g. new room subscriptions, etc). - now_token = self.event_sources.get_current_token() - result = await self.current_sync_for_user( - sync_config, - from_token=from_token, - to_token=now_token, - ) - - # Return immediately if we have a result, the timeout is 0, or this is - # an initial sync. - if result or timeout_ms == 0 or from_token is None: - return result, did_wait - - # Otherwise, we wait for something to happen and report it to the user. - async def current_sync_callback( - before_token: StreamToken, after_token: StreamToken - ) -> SlidingSyncResult: - return await self.current_sync_for_user( + # We're going to respond immediately if the timeout is 0 or if this is an + # initial sync (without a `from_token`) so we can avoid calling + # `notifier.wait_for_events()`. + if timeout_ms == 0 or from_token is None: + now_token = self.event_sources.get_current_token() + result = await self.current_sync_for_user( sync_config, from_token=from_token, - to_token=after_token, + to_token=now_token, ) + else: + # Otherwise, we wait for something to happen and report it to the user. + async def current_sync_callback( + before_token: StreamToken, after_token: StreamToken + ) -> SlidingSyncResult: + return await self.current_sync_for_user( + sync_config, + from_token=from_token, + to_token=after_token, + ) - result = await self.notifier.wait_for_events( - sync_config.user.to_string(), - timeout_ms, - current_sync_callback, - from_token=now_token, - ) - did_wait = True + result = await self.notifier.wait_for_events( + sync_config.user.to_string(), + timeout_ms, + current_sync_callback, + from_token=from_token.stream_token, + ) + did_wait = True return result, did_wait diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index 216ef3b0710..8969d915836 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -852,15 +852,11 @@ async def _filter_relevant_rooms_to_send( previous_connection_state.room_configs.get(room_id) ) if prev_room_sync_config is not None: - # Always include rooms whose effective config has - # expanded. This covers timeline-limit increases and - # required-state additions introduced by room - # subscriptions overriding list-derived params. + # Always include rooms whose timeline limit has increased. + # (see the "XXX: Odd behavior" described below) if ( - prev_room_sync_config.combine_room_sync_config( - room_config - ) - != prev_room_sync_config + prev_room_sync_config.timeline_limit + < room_config.timeline_limit ): rooms_should_send.add(room_id) continue diff --git a/tests/rest/client/sliding_sync/test_room_subscriptions.py b/tests/rest/client/sliding_sync/test_room_subscriptions.py index d970af367d3..811478f1ba7 100644 --- a/tests/rest/client/sliding_sync/test_room_subscriptions.py +++ b/tests/rest/client/sliding_sync/test_room_subscriptions.py @@ -22,7 +22,6 @@ from synapse.api.constants import EventTypes, HistoryVisibility from synapse.rest.client import login, room, sync from synapse.server import HomeServer -from synapse.types import JsonDict from synapse.util.clock import Clock from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase @@ -127,124 +126,6 @@ def test_room_subscriptions_with_join_membership(self) -> None: response_body["rooms"][room_id1], ) - def test_room_subscription_required_state_expansion_returns_immediately( - self, - ) -> None: - """ - Test that adding a room subscription with stronger params than the list causes an - incremental long-poll to return immediately, even without new stream activity. - """ - user1_id = self.register_user("user1", "pass") - user1_tok = self.login(user1_id, "pass") - - room_id1 = self.helper.create_room_as(user1_id, tok=user1_tok) - - sync_body: JsonDict = { - "lists": { - "foo-list": { - "ranges": [[0, 0]], - "required_state": [], - "timeline_limit": 0, - } - }, - "conn_id": "conn_id", - } - _, from_token = self.do_sync(sync_body, tok=user1_tok) - - sync_body["room_subscriptions"] = { - room_id1: { - "required_state": [ - [EventTypes.Create, ""], - ], - "timeline_limit": 0, - } - } - - channel = self.make_request( - "POST", - self.sync_endpoint + f"?timeout=10000&pos={from_token}", - content=sync_body, - access_token=user1_tok, - await_result=False, - ) - channel.await_result(timeout_ms=3000) - self.assertEqual(channel.code, 200, channel.json_body) - - state_map = self.get_success( - self.storage_controllers.state.get_current_state(room_id1) - ) - - room_response = channel.json_body["rooms"][room_id1] - self.assertNotIn("initial", room_response) - self._assertRequiredStateIncludes( - room_response["required_state"], - { - state_map[(EventTypes.Create, "")], - }, - exact=True, - ) - - def test_room_subscription_required_state_change_returns_immediately(self) -> None: - """ - Test that expanding an existing room subscription's required state causes an - incremental long-poll to return immediately, even without new stream activity. - """ - user1_id = self.register_user("user1", "pass") - user1_tok = self.login(user1_id, "pass") - - room_id1 = self.helper.create_room_as( - user1_id, tok=user1_tok, extra_content={"name": "Foo"} - ) - - sync_body: JsonDict = { - "room_subscriptions": { - room_id1: { - "required_state": [ - [EventTypes.Create, ""], - ], - "timeline_limit": 0, - } - }, - "conn_id": "conn_id", - } - response_body, from_token = self.do_sync(sync_body, tok=user1_tok) - - state_map = self.get_success( - self.storage_controllers.state.get_current_state(room_id1) - ) - self._assertRequiredStateIncludes( - response_body["rooms"][room_id1]["required_state"], - { - state_map[(EventTypes.Create, "")], - }, - exact=True, - ) - - sync_body["room_subscriptions"][room_id1]["required_state"] = [ - [EventTypes.Create, ""], - [EventTypes.Name, ""], - ] - - channel = self.make_request( - "POST", - self.sync_endpoint + f"?timeout=10000&pos={from_token}", - content=sync_body, - access_token=user1_tok, - await_result=False, - ) - channel.await_result(timeout_ms=3000) - self.assertEqual(channel.code, 200, channel.json_body) - - room_response = channel.json_body["rooms"][room_id1] - self.assertNotIn("initial", room_response) - self._assertRequiredStateIncludes( - room_response["required_state"], - { - state_map[(EventTypes.Name, "")], - }, - exact=True, - ) - def test_room_subscriptions_with_leave_membership(self) -> None: """ Test `room_subscriptions` with a leave room should give us timeline and state From 71e1da976cfbfbdf4be2598a84d6a22671c1bf46 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 15 May 2026 11:36:47 +0100 Subject: [PATCH 38/92] Revert "Send a SSS response immediately if the config has changed and there are new results to sync (#19714)" (#19784) Reverts: #19714 Opens: #19783 Closes: https://github.com/element-hq/backend-internal/issues/242 Related: #18880 (the performance problem that is aggravated by #19714) This reverts commit 2691d0b8b143e01987504210e320198560371435. --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19784.bugfix | 1 + synapse/handlers/sliding_sync/__init__.py | 52 ++++---- synapse/handlers/sliding_sync/room_lists.py | 12 +- .../sliding_sync/test_room_subscriptions.py | 119 ------------------ 4 files changed, 29 insertions(+), 155 deletions(-) create mode 100644 changelog.d/19784.bugfix diff --git a/changelog.d/19784.bugfix b/changelog.d/19784.bugfix new file mode 100644 index 00000000000..c68524d57a9 --- /dev/null +++ b/changelog.d/19784.bugfix @@ -0,0 +1 @@ +Revert 'Have [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) return a new response immediately if a room subscription has changed and produced a new response. ([\#19714](https://github.com/element-hq/synapse/issues/19714))' due to performance problems. diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index a3443b300cc..1cc587d4a7b 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -167,38 +167,34 @@ async def wait_for_sync_for_user( timeout_ms -= after_wait_ts - before_wait_ts timeout_ms = max(timeout_ms, 0) - # Compute a response immediately. We always need to do this before - # waiting for new data (unlike in /v3/sync), as the request config might - # have changed (e.g. new room subscriptions, etc). - now_token = self.event_sources.get_current_token() - result = await self.current_sync_for_user( - sync_config, - from_token=from_token, - to_token=now_token, - ) - - # Return immediately if we have a result, the timeout is 0, or this is - # an initial sync. - if result or timeout_ms == 0 or from_token is None: - return result, did_wait - - # Otherwise, we wait for something to happen and report it to the user. - async def current_sync_callback( - before_token: StreamToken, after_token: StreamToken - ) -> SlidingSyncResult: - return await self.current_sync_for_user( + # We're going to respond immediately if the timeout is 0 or if this is an + # initial sync (without a `from_token`) so we can avoid calling + # `notifier.wait_for_events()`. + if timeout_ms == 0 or from_token is None: + now_token = self.event_sources.get_current_token() + result = await self.current_sync_for_user( sync_config, from_token=from_token, - to_token=after_token, + to_token=now_token, ) + else: + # Otherwise, we wait for something to happen and report it to the user. + async def current_sync_callback( + before_token: StreamToken, after_token: StreamToken + ) -> SlidingSyncResult: + return await self.current_sync_for_user( + sync_config, + from_token=from_token, + to_token=after_token, + ) - result = await self.notifier.wait_for_events( - sync_config.user.to_string(), - timeout_ms, - current_sync_callback, - from_token=now_token, - ) - did_wait = True + result = await self.notifier.wait_for_events( + sync_config.user.to_string(), + timeout_ms, + current_sync_callback, + from_token=from_token.stream_token, + ) + did_wait = True return result, did_wait diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index 216ef3b0710..8969d915836 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -852,15 +852,11 @@ async def _filter_relevant_rooms_to_send( previous_connection_state.room_configs.get(room_id) ) if prev_room_sync_config is not None: - # Always include rooms whose effective config has - # expanded. This covers timeline-limit increases and - # required-state additions introduced by room - # subscriptions overriding list-derived params. + # Always include rooms whose timeline limit has increased. + # (see the "XXX: Odd behavior" described below) if ( - prev_room_sync_config.combine_room_sync_config( - room_config - ) - != prev_room_sync_config + prev_room_sync_config.timeline_limit + < room_config.timeline_limit ): rooms_should_send.add(room_id) continue diff --git a/tests/rest/client/sliding_sync/test_room_subscriptions.py b/tests/rest/client/sliding_sync/test_room_subscriptions.py index d970af367d3..811478f1ba7 100644 --- a/tests/rest/client/sliding_sync/test_room_subscriptions.py +++ b/tests/rest/client/sliding_sync/test_room_subscriptions.py @@ -22,7 +22,6 @@ from synapse.api.constants import EventTypes, HistoryVisibility from synapse.rest.client import login, room, sync from synapse.server import HomeServer -from synapse.types import JsonDict from synapse.util.clock import Clock from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase @@ -127,124 +126,6 @@ def test_room_subscriptions_with_join_membership(self) -> None: response_body["rooms"][room_id1], ) - def test_room_subscription_required_state_expansion_returns_immediately( - self, - ) -> None: - """ - Test that adding a room subscription with stronger params than the list causes an - incremental long-poll to return immediately, even without new stream activity. - """ - user1_id = self.register_user("user1", "pass") - user1_tok = self.login(user1_id, "pass") - - room_id1 = self.helper.create_room_as(user1_id, tok=user1_tok) - - sync_body: JsonDict = { - "lists": { - "foo-list": { - "ranges": [[0, 0]], - "required_state": [], - "timeline_limit": 0, - } - }, - "conn_id": "conn_id", - } - _, from_token = self.do_sync(sync_body, tok=user1_tok) - - sync_body["room_subscriptions"] = { - room_id1: { - "required_state": [ - [EventTypes.Create, ""], - ], - "timeline_limit": 0, - } - } - - channel = self.make_request( - "POST", - self.sync_endpoint + f"?timeout=10000&pos={from_token}", - content=sync_body, - access_token=user1_tok, - await_result=False, - ) - channel.await_result(timeout_ms=3000) - self.assertEqual(channel.code, 200, channel.json_body) - - state_map = self.get_success( - self.storage_controllers.state.get_current_state(room_id1) - ) - - room_response = channel.json_body["rooms"][room_id1] - self.assertNotIn("initial", room_response) - self._assertRequiredStateIncludes( - room_response["required_state"], - { - state_map[(EventTypes.Create, "")], - }, - exact=True, - ) - - def test_room_subscription_required_state_change_returns_immediately(self) -> None: - """ - Test that expanding an existing room subscription's required state causes an - incremental long-poll to return immediately, even without new stream activity. - """ - user1_id = self.register_user("user1", "pass") - user1_tok = self.login(user1_id, "pass") - - room_id1 = self.helper.create_room_as( - user1_id, tok=user1_tok, extra_content={"name": "Foo"} - ) - - sync_body: JsonDict = { - "room_subscriptions": { - room_id1: { - "required_state": [ - [EventTypes.Create, ""], - ], - "timeline_limit": 0, - } - }, - "conn_id": "conn_id", - } - response_body, from_token = self.do_sync(sync_body, tok=user1_tok) - - state_map = self.get_success( - self.storage_controllers.state.get_current_state(room_id1) - ) - self._assertRequiredStateIncludes( - response_body["rooms"][room_id1]["required_state"], - { - state_map[(EventTypes.Create, "")], - }, - exact=True, - ) - - sync_body["room_subscriptions"][room_id1]["required_state"] = [ - [EventTypes.Create, ""], - [EventTypes.Name, ""], - ] - - channel = self.make_request( - "POST", - self.sync_endpoint + f"?timeout=10000&pos={from_token}", - content=sync_body, - access_token=user1_tok, - await_result=False, - ) - channel.await_result(timeout_ms=3000) - self.assertEqual(channel.code, 200, channel.json_body) - - room_response = channel.json_body["rooms"][room_id1] - self.assertNotIn("initial", room_response) - self._assertRequiredStateIncludes( - room_response["required_state"], - { - state_map[(EventTypes.Name, "")], - }, - exact=True, - ) - def test_room_subscriptions_with_leave_membership(self) -> None: """ Test `room_subscriptions` with a leave room should give us timeline and state From 0ff50720d899e6f867b5694c0f05be1561d650f6 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 15 May 2026 11:43:12 +0100 Subject: [PATCH 39/92] 1.153.0rc3 --- CHANGES.md | 7 +++++++ changelog.d/19784.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 14 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/19784.bugfix diff --git a/CHANGES.md b/CHANGES.md index f722a1e7037..96a9fadbfb6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +# Synapse 1.153.0rc3 (2026-05-15) + +## Bugfixes + +- Revert 'Have [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) return a new response immediately if a room subscription has changed and produced a new response. ([\#19714](https://github.com/element-hq/synapse/issues/19714))' (introduced in 1.153.0rc1) due to performance problems. ([\#19784](https://github.com/element-hq/synapse/issues/19784)) + + # Synapse 1.153.0rc2 (2026-05-13) ## Bugfixes diff --git a/changelog.d/19784.bugfix b/changelog.d/19784.bugfix deleted file mode 100644 index c68524d57a9..00000000000 --- a/changelog.d/19784.bugfix +++ /dev/null @@ -1 +0,0 @@ -Revert 'Have [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) return a new response immediately if a room subscription has changed and produced a new response. ([\#19714](https://github.com/element-hq/synapse/issues/19714))' due to performance problems. diff --git a/debian/changelog b/debian/changelog index bbd164b2a8b..81f080bead3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.153.0~rc3) stable; urgency=medium + + * New Synapse release 1.153.0rc3. + + -- Synapse Packaging team Fri, 15 May 2026 11:42:29 +0100 + matrix-synapse-py3 (1.153.0~rc2) stable; urgency=medium * New Synapse release 1.153.0rc2. diff --git a/pyproject.toml b/pyproject.toml index a7422e19876..fed1a23f4a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.153.0rc2" +version = "1.153.0rc3" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 0c6e0f79e5fef944ceca5f74404ec4a44be310bd Mon Sep 17 00:00:00 2001 From: mhlas7 Date: Fri, 15 May 2026 08:29:53 -0700 Subject: [PATCH 40/92] doc: Enhance update_profile_information documentation with picture claim (#19508) Added details how synapse syncs the picture claim when update_profile_information setting is true. Addresses #17836 --------- Co-authored-by: Michael Hlas <3398654+mhlas7@users.noreply.github.com> --- changelog.d/19508.doc | 1 + docs/usage/configuration/config_documentation.md | 5 ++++- schema/synapse-config.schema.yaml | 14 +++++++++----- 3 files changed, 14 insertions(+), 6 deletions(-) create mode 100644 changelog.d/19508.doc diff --git a/changelog.d/19508.doc b/changelog.d/19508.doc new file mode 100644 index 00000000000..25501163414 --- /dev/null +++ b/changelog.d/19508.doc @@ -0,0 +1 @@ +Added details about how Synapse syncs the picture claim when `update_profile_information` setting is true. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index e58a2dcf10b..d028d65fe33 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -3788,7 +3788,10 @@ This setting has the following sub-options: Defaults to `null`. -* `update_profile_information` (boolean): Use this setting to keep a user's profile fields in sync with information from the identity provider. Currently only syncing the displayname is supported. Fields are checked on every SSO login, and are updated if necessary. Note that enabling this option will override user profile information, regardless of whether users have opted-out of syncing that information when first signing in. Defaults to `false`. +* `update_profile_information` (boolean): Use this setting to keep a user's profile fields in sync with information from the identity provider. Fields are checked on every SSO login, and are updated if necessary. Note that enabling this option will override user profile information, regardless of whether users have opted-out of syncing that information when first signing in. Fields that will be synced: + * displayname + * picture - only if Synapse media repository is running in the main + process (i.e. not workerized) and media is stored locally Defaults to `false`. Example configuration: ```yaml diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 8888d2c673a..8b8d57b9bf1 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -4620,11 +4620,15 @@ properties: type: boolean description: >- Use this setting to keep a user's profile fields in sync with - information from the identity provider. Currently only syncing the - displayname is supported. Fields are checked on every SSO login, and - are updated if necessary. Note that enabling this option will override - user profile information, regardless of whether users have opted-out - of syncing that information when first signing in. + information from the identity provider. Fields are checked on every + SSO login, and are updated if necessary. Note that enabling this + option will override user profile information, regardless of whether + users have opted-out of syncing that information when first signing + in. + Fields that will be synced: + * displayname + * picture - only if Synapse media repository is running in the main + process (i.e. not workerized) and media is stored locally default: false examples: - client_whitelist: From 19f636244ce3a4fa6970f5abe970d105d963e0e6 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 15 May 2026 11:49:11 -0500 Subject: [PATCH 41/92] Prefer close backfill points (absolute distance) (#19748) This isn't fixing any particular issue. It's just a follow-up I thought about after merging https://github.com/element-hq/synapse/pull/19611 since we're now also dealing with backfill points in the nearby range ahead of the `current_depth`. And it's possible that the previous sort could bias to all nearby backfill points ahead of the `current_depth` that don't extend into the visible window of events we're paginating through. --- changelog.d/19748.misc | 1 + synapse/handlers/federation.py | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 changelog.d/19748.misc diff --git a/changelog.d/19748.misc b/changelog.d/19748.misc new file mode 100644 index 00000000000..eedd4e92a26 --- /dev/null +++ b/changelog.d/19748.misc @@ -0,0 +1 @@ +Prefer close backfill points (absolute distance). diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 166a02d7c7e..ba83d4fd268 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -275,11 +275,22 @@ async def _maybe_backfill_inner( ) ] - # we now have a list of potential places to backpaginate from. We prefer to - # start with the most recent (ie, max depth), so let's sort the list. + # we now have a list of potential places to backpaginate from. Figure out which + # ones we should prefer, so let's sort the list. sorted_backfill_points: list[_BackfillPoint] = sorted( backwards_extremities, - key=lambda e: -int(e.depth), + key=lambda e: ( + # Prefer backfill points that are closer to the `current_depth` + # (absolute distance) + abs(current_depth - e.depth), + # For the tie-break, we care about events that are actually in the past + # as they're more likely to reveal history that we can return (something + # absolutely in the past is better than something can potentially extend + # into the past). + # + # This sorts ascending so 0 sorts before 1 + 0 if current_depth >= e.depth else 1, + ), ) logger.debug( @@ -300,7 +311,7 @@ async def _maybe_backfill_inner( str(len(sorted_backfill_points)), ) - # If we have no backfill points lower than the `current_depth` then either we + # If we have no backfill points lower than the `nearby_depth` then either we # can a) bail or b) still attempt to backfill. We opt to try backfilling anyway # just in case we do get relevant events. This is good for eventual consistency # sake but we don't need to block the client for something that is just as From 8eb220a5e25c57da979d5b0bc8dbc7b5b657e501 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 15 May 2026 13:51:03 -0500 Subject: [PATCH 42/92] Replace `wait_for_quarantined_media_stream_id(...)` with standard `wait_for_stream_token(...)` (#19764) In order to be able to use `wait_for_stream_token(...)`, we have to add the `quarantined_media` stream to the `StreamToken`. Even though we don't care about `/sync`'ing `quarantined_media`, this aligns with the future where all endpoints should probably use `StreamToken`, see https://github.com/element-hq/synapse/issues/19647 Follow-up to https://github.com/element-hq/synapse/pull/19558 and https://github.com/element-hq/synapse/pull/19644 --- changelog.d/19764.misc | 1 + synapse/rest/admin/media.py | 23 ++++++-- synapse/storage/databases/main/room.py | 80 ++++---------------------- synapse/streams/events.py | 3 + synapse/types/__init__.py | 27 +++++++-- tests/rest/admin/test_room.py | 4 +- tests/rest/client/test_rooms.py | 4 +- 7 files changed, 59 insertions(+), 83 deletions(-) create mode 100644 changelog.d/19764.misc diff --git a/changelog.d/19764.misc b/changelog.d/19764.misc new file mode 100644 index 00000000000..8704e3eed64 --- /dev/null +++ b/changelog.d/19764.misc @@ -0,0 +1 @@ +Replace unique `quarantined_media` waiting patterns with standard `wait_for_stream_token(...)`. diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py index 1633cca884a..35454c15221 100644 --- a/synapse/rest/admin/media.py +++ b/synapse/rest/admin/media.py @@ -43,7 +43,13 @@ from synapse.storage.databases.main.media_repository import ( MediaSortOrder, ) -from synapse.types import JsonDict, UserID +from synapse.types import ( + JsonDict, + MultiWriterStreamToken, + StreamKeyType, + StreamToken, + UserID, +) if TYPE_CHECKING: from synapse.server import HomeServer @@ -243,6 +249,7 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.server_name = hs.hostname self.replication = hs.get_replication_data_handler() + self.notifier = hs.get_notifier() async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) @@ -256,8 +263,7 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: # The caller is trying to get future data, which we don't allow because # we know it's an invalid state that should never happen. We could # wait until we reach the token but we might as well not waste our - # resources on that which is why `wait_for_quarantined_media_stream_id(...)` - # has assertions around this. + # resources on that. raise SynapseError( HTTPStatus.BAD_REQUEST, "The `from` token is considered invalid because it includes stream positions " @@ -268,9 +274,16 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: errcode=Codes.INVALID_PARAM, ) + # Create a `StreamToken` that's compatible with `wait_for_stream_token`. + # + # FIXME: Ideally, this endpoint would use a `StreamToken` to begin with + from_token = StreamToken.START.copy_and_replace( + StreamKeyType.QUARANTINED_MEDIA, MultiWriterStreamToken(stream=from_id) + ) + # We need to wait to ensure that our current worker is actually caught up with # the stream position, otherwise we might not return what we think we're returning. - if not await self.store.wait_for_quarantined_media_stream_id(from_id): + if not await self.notifier.wait_for_stream_token(from_token): raise SynapseError( HTTPStatus.INTERNAL_SERVER_ERROR, "Timed out while waiting for the worker serving this request to catch up to the given " @@ -280,7 +293,7 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: errcode=Codes.UNKNOWN, ) - to_id = await self.store.get_current_quarantined_media_stream_id() + to_id = self.store.get_current_quarantined_media_stream_id() changes = await self.store.get_quarantined_media_changes( from_id=from_id, to_id=to_id, diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index a0c42082f0e..95aa2cb7dcf 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -62,12 +62,12 @@ from synapse.storage.util.id_generators import IdGenerator, MultiWriterIdGenerator from synapse.types import ( JsonDict, + MultiWriterStreamToken, RetentionPolicy, StrCollection, ThirdPartyInstanceID, ) from synapse.util.caches.descriptors import cached, cachedList -from synapse.util.duration import Duration from synapse.util.json import json_encoder from synapse.util.stringutils import MXC_REGEX @@ -1302,7 +1302,15 @@ def _get_media_ids_by_user_txn( return local_media_ids - async def get_current_quarantined_media_stream_id(self) -> int: + def get_quarantined_media_stream_token(self) -> MultiWriterStreamToken: + return MultiWriterStreamToken.from_generator( + self._quarantined_media_changes_id_gen + ) + + def get_quarantined_media_stream_id_generator(self) -> MultiWriterIdGenerator: + return self._quarantined_media_changes_id_gen + + def get_current_quarantined_media_stream_id(self) -> int: """Gets the position of the quarantined media changes stream. Returns: @@ -1318,74 +1326,6 @@ async def get_max_allocated_quarantined_media_stream_id(self) -> int: """ return await self._quarantined_media_changes_id_gen.get_max_allocated_token() - async def wait_for_quarantined_media_stream_id(self, target_id: int) -> bool: - """Waits until the quarantined media changes stream reaches the given stream ID. - - See https://github.com/element-hq/synapse/pull/19644 for more details. - - TODO: Replace function and call sites with https://github.com/element-hq/synapse/pull/19644 - - Args: - target_id: The stream ID to wait for. - - Returns: - True when caught up to the target stream ID. - False when timing out while waiting. - """ - # We ideally would use something like `wait_for_stream_position` in the meantime, - # but that short circuits if the instance name matches the current instance name. - # Doing so means that if *another* writer is actually leading the to_id, then we'll - # assume that we're caught up when we aren't. - # - # NOTE: Because this is implemented to wait for stream positions by integer ID, - # we're technically waiting for *all* workers to catch up rather than just waiting - # for *our* worker to catch up. This is okay for now because the quarantined media - # stream should be pretty fast to update, and if it's not then the only thing we're - # affecting is an admin API that probably has a tool automatically retrying requests - # anyway. https://github.com/element-hq/synapse/pull/19644 does the waiting properly - # so this should be replaced by that (or similar). - - # Get the minimum shared position/ID across all workers - current_id = self._quarantined_media_changes_id_gen.get_current_token() - if current_id >= target_id: - return True # nothing to wait for: we're already caught up. - - # "This should never happen". Tokens we hand out via the API should exist. If they - # don't, then we're in a bad state and need to explode. - max_persisted_position = ( - await self._quarantined_media_changes_id_gen.get_max_allocated_token() - ) - assert max_persisted_position >= target_id, ( - f"Unable to wait for invalid future token (token={target_id} has positions " - f"ahead of our max persisted position={max_persisted_position})" - ) - - # Start waiting until we've caught up to the `stream_token` - start = self.clock.time_msec() - logged = False - while True: - # Like above, get the minimum shared ID across all workers - current_id = self._quarantined_media_changes_id_gen.get_current_token() - if current_id >= target_id: - return True - - now = self.clock.time_msec() - - # Timed out - if now - start > 10_000: - return False - - if not logged: - logger.info( - "Waiting for current token to reach %s; currently at %s", - target_id, - current_id, - ) - logged = True - - # TODO: be better - await self.clock.sleep(Duration(milliseconds=500)) - async def get_quarantined_media_changes( self, *, from_id: int, to_id: int, limit: int ) -> list[QuarantinedMediaUpdate]: diff --git a/synapse/streams/events.py b/synapse/streams/events.py index f5677a20828..36490fcb355 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -85,6 +85,7 @@ def get_current_token(self) -> StreamToken: ) thread_subscriptions_key = self.store.get_max_thread_subscriptions_stream_id() sticky_events_key = self.store.get_max_sticky_events_stream_id() + quarantined_media_key = self.store.get_quarantined_media_stream_token() token = StreamToken( room_key=self.sources.room.get_current_key(), @@ -100,6 +101,7 @@ def get_current_token(self) -> StreamToken: un_partial_stated_rooms_key=un_partial_stated_rooms_key, thread_subscriptions_key=thread_subscriptions_key, sticky_events_key=sticky_events_key, + quarantined_media_key=quarantined_media_key, ) return token @@ -128,6 +130,7 @@ async def bound_future_token(self, token: StreamToken) -> StreamToken: StreamKeyType.UN_PARTIAL_STATED_ROOMS: self.store.get_un_partial_stated_rooms_id_generator(), StreamKeyType.THREAD_SUBSCRIPTIONS: self.store.get_thread_subscriptions_stream_id_generator(), StreamKeyType.STICKY_EVENTS: self.store.get_sticky_events_stream_id_generator(), + StreamKeyType.QUARANTINED_MEDIA: self.store.get_quarantined_media_stream_id_generator(), } for _, key in StreamKeyType.__members__.items(): diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 8b005ef84d5..8537a63bdec 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -1060,6 +1060,7 @@ class StreamKeyType(Enum): UN_PARTIAL_STATED_ROOMS = "un_partial_stated_rooms_key" THREAD_SUBSCRIPTIONS = "thread_subscriptions_key" STICKY_EVENTS = "sticky_events_key" + QUARANTINED_MEDIA = "quarantined_media_key" @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -1067,7 +1068,7 @@ class StreamToken: """A collection of keys joined together by underscores in the following order and which represent the position in their respective streams. - ex. `s2633508_17_338_6732159_1082514_541479_274711_265584_1_379_4242` + ex. `s2633508_17_338_6732159_1082514_541479_274711_265584_1_379_4242_4141_4343` 1. `room_key`: `s2633508` which is a `RoomStreamToken` - `RoomStreamToken`'s can also look like `t426-2633508` or `m56~2.58~3.59` - See the docstring for `RoomStreamToken` for more details. @@ -1082,12 +1083,13 @@ class StreamToken: 10. `un_partial_stated_rooms_key`: `379` 11. `thread_subscriptions_key`: 4242 12. `sticky_events_key`: 4141 + 13. `quarantined_media_key`: 4343 You can see how many of these keys correspond to the various fields in a "/sync" response: ```json { - "next_batch": "s12_4_0_1_1_1_1_4_1_1", + "next_batch": "s12_4_0_1_1_1_1_4_1_1_1_1_1", "presence": { "events": [] }, @@ -1099,7 +1101,7 @@ class StreamToken: "!QrZlfIDQLNLdZHqTnt:hs1": { "timeline": { "events": [], - "prev_batch": "s10_4_0_1_1_1_1_4_1_1", + "prev_batch": "s10_4_0_1_1_1_1_4_1_1_1_1_1", "limited": false }, "state": { @@ -1142,6 +1144,9 @@ class StreamToken: un_partial_stated_rooms_key: int thread_subscriptions_key: int sticky_events_key: int + quarantined_media_key: MultiWriterStreamToken = attr.ib( + validator=attr.validators.instance_of(MultiWriterStreamToken) + ) _SEPARATOR = "_" START: ClassVar["StreamToken"] @@ -1171,6 +1176,7 @@ async def from_string(cls, store: "DataStore", string: str) -> "StreamToken": un_partial_stated_rooms_key, thread_subscriptions_key, sticky_events_key, + quarantined_media_key, ) = keys return cls( @@ -1188,6 +1194,9 @@ async def from_string(cls, store: "DataStore", string: str) -> "StreamToken": un_partial_stated_rooms_key=int(un_partial_stated_rooms_key), thread_subscriptions_key=int(thread_subscriptions_key), sticky_events_key=int(sticky_events_key), + quarantined_media_key=await MultiWriterStreamToken.parse( + store, quarantined_media_key + ), ) except CancelledError: raise @@ -1212,6 +1221,7 @@ async def to_string(self, store: "DataStore") -> str: str(self.un_partial_stated_rooms_key), str(self.thread_subscriptions_key), str(self.sticky_events_key), + await self.quarantined_media_key.to_string(store), ] ) @@ -1241,6 +1251,12 @@ def copy_and_advance(self, key: StreamKeyType, new_value: Any) -> "StreamToken": self.device_list_key.copy_and_advance(new_value), ) return new_token + elif key == StreamKeyType.QUARANTINED_MEDIA: + new_token = self.copy_and_replace( + StreamKeyType.QUARANTINED_MEDIA, + self.quarantined_media_key.copy_and_advance(new_value), + ) + return new_token new_token = self.copy_and_replace(key, new_value) new_id = new_token.get_field(key) @@ -1263,6 +1279,7 @@ def get_field( key: Literal[ StreamKeyType.RECEIPT, StreamKeyType.DEVICE_LIST, + StreamKeyType.QUARANTINED_MEDIA, ], ) -> MultiWriterStreamToken: ... @@ -1334,7 +1351,8 @@ def __str__(self) -> str: f"account_data: {self.account_data_key}, push_rules: {self.push_rules_key}, " f"to_device: {self.to_device_key}, device_list: {self.device_list_key}, " f"groups: {self.groups_key}, un_partial_stated_rooms: {self.un_partial_stated_rooms_key}," - f"thread_subscriptions: {self.thread_subscriptions_key}, sticky_events: {self.sticky_events_key})" + f"thread_subscriptions: {self.thread_subscriptions_key}, sticky_events: {self.sticky_events_key}" + f"quarantined_media: {self.quarantined_media_key})" ) @@ -1351,6 +1369,7 @@ def __str__(self) -> str: un_partial_stated_rooms_key=0, thread_subscriptions_key=0, sticky_events_key=0, + quarantined_media_key=MultiWriterStreamToken(stream=0), ) diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index 507cf10c5de..c4e4170c6f9 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -2549,7 +2549,7 @@ def test_timestamp_to_event(self) -> None: def test_topo_token_is_accepted(self) -> None: """Test Topo Token is accepted.""" - token = "t1-0_0_0_0_0_0_0_0_0_0_0_0" + token = "t1-0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/_synapse/admin/v1/rooms/%s/messages?from=%s" % (self.room_id, token), @@ -2563,7 +2563,7 @@ def test_topo_token_is_accepted(self) -> None: def test_stream_token_is_accepted_for_fwd_pagianation(self) -> None: """Test that stream token is accepted for forward pagination.""" - token = "s0_0_0_0_0_0_0_0_0_0_0_0" + token = "s0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/_synapse/admin/v1/rooms/%s/messages?from=%s" % (self.room_id, token), diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 28872fa06cb..10325c536a0 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -2248,7 +2248,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.room_id = self.helper.create_room_as(self.user_id) def test_topo_token_is_accepted(self) -> None: - token = "t1-0_0_0_0_0_0_0_0_0_0_0_0" + token = "t1-0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token) ) @@ -2259,7 +2259,7 @@ def test_topo_token_is_accepted(self) -> None: self.assertTrue("end" in channel.json_body) def test_stream_token_is_accepted_for_fwd_pagianation(self) -> None: - token = "s0_0_0_0_0_0_0_0_0_0_0_0" + token = "s0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token) ) From d8b4ffdf2d987a04ab1f98d8e67388a36f79a017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan?= <166950915+gaetan-sbt@users.noreply.github.com> Date: Mon, 18 May 2026 10:27:10 +0100 Subject: [PATCH 43/92] Fix validation of frozen message event with mentions. (#19634) Fixes: #19689 # What This PR fixes a bug I found when I run synapse (from dockerhub) and register a `check_event_allowed` callback and my client makes use of the mentions field in messages (`cinny:latest`). The bug doesn't appear when the `check_event_allowed` callback is not loaded. After some digging I noticed that the current validation of the mentions doesn't work when an event has been frozen with `event.freeze()`. For the messages this seems to happen when a the `check_event_allowed` is registered (but not otherwise), see [where the event is frozen for check_event_allowed callback](https://github.com/element-hq/synapse/blob/b0fc0b7a612a42e6f15b87dee2a1db4c383645fb/synapse/module_api/callbacks/third_party_event_rules_callbacks.py#L289) and [where the validation function is called](https://github.com/element-hq/synapse/blob/b0fc0b7a612a42e6f15b87dee2a1db4c383645fb/synapse/handlers/message.py#L1404). To have a minimal reproduction example, the following scripts fails on `develop` but succeeds in this branch: ``` python from synapse.api.room_versions import RoomVersions from synapse.events import EventBase, make_event_from_dict from synapse.events.validator import EventValidator from tests.utils import default_config def make_message_event(content: dict) -> EventBase: return make_event_from_dict( { "room_id": "!room:test", "type": "m.room.message", "sender": "@alice:test", "content": content, "auth_events": [], "prev_events": [], "hashes": {"sha256": "aGVsbG8="}, "signatures": {}, "depth": 1, "origin_server_ts": 1000, }, room_version=RoomVersions.V9, ) event = make_message_event( { "msgtype": "m.text", "body": "@moderator:example.com hello", "m.mentions": {"user_ids": ["@moderator:jailbreak-challenge.aqtiveguard.com"]}, } ) EventValidator().validate_new(event, default_config) # Ok event.freeze() EventValidator().validate_new(event, default_config) # throws # pydantic_core._pydantic_core.ValidationError: 1 validation error for Mentions # Input should be a valid dictionary or instance of Mentions [type=model_type, input_value=immutabledict({'user_ids'...nge.aqtiveguard.com',)}), input_type=immutabledict] # For further information visit https://errors.pydantic.dev/2.12/v/model_type ``` # How I made the validation logic also validate the transformation performed by the freezing process, namely: - `immutabledict` validates as `dict`. (was already implemented for POWER_LEVELS) - `tuple` validates as array (added this to the validator in this PR). --------- Co-authored-by: Eric Eastwood Co-authored-by: Olivier 'reivilibre --- changelog.d/19634.bugfix | 1 + synapse/events/validator.py | 81 ++++++++++++++++++++++------------ tests/events/test_validator.py | 50 +++++++++++++++++++++ 3 files changed, 103 insertions(+), 29 deletions(-) create mode 100644 changelog.d/19634.bugfix create mode 100644 tests/events/test_validator.py diff --git a/changelog.d/19634.bugfix b/changelog.d/19634.bugfix new file mode 100644 index 00000000000..e8fcb435701 --- /dev/null +++ b/changelog.d/19634.bugfix @@ -0,0 +1 @@ +Fix bug where Synapse would return 400 (`M_BAD_JSON`) when sending a message with `mentions` field and Synapse module `check_event_allowed` callback registered (frozen event). Contributed by @gaetan-sbt. \ No newline at end of file diff --git a/synapse/events/validator.py b/synapse/events/validator.py index d1b5152d77f..e8d6cc97109 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -22,7 +22,6 @@ from typing import cast import jsonschema -from pydantic import Field, StrictBool, StrictStr from synapse.api.constants import ( MAX_ALIAS_LENGTH, @@ -40,10 +39,8 @@ CANONICALJSON_MIN_INT, validate_canonicaljson, ) -from synapse.http.servlet import validate_json_object from synapse.storage.controllers.state import server_acl_evaluator_from_event from synapse.types import EventID, JsonDict, JsonMapping, RoomID, StrCollection, UserID -from synapse.types.rest import RequestBodyModel class EventValidator: @@ -116,29 +113,18 @@ def validate_new(self, event: EventBase, config: HomeServerConfig) -> None: cls=POWER_LEVELS_VALIDATOR, ) except jsonschema.ValidationError as e: - if e.path: - # example: "users_default": '0' is not of type 'integer' - # cast safety: path entries can be integers, if we fail to validate - # items in an array. However, the POWER_LEVELS_SCHEMA doesn't expect - # to see any arrays. - message = ( - '"' + cast(str, e.path[-1]) + '": ' + e.message # noqa: B306 - ) - # jsonschema.ValidationError.message is a valid attribute - else: - # example: '0' is not of type 'integer' - message = e.message # noqa: B306 - # jsonschema.ValidationError.message is a valid attribute - - raise SynapseError( - code=400, - msg=message, - errcode=Codes.BAD_JSON, - ) + raise _validation_error_to_api_error(e) # If the event contains a mentions key, validate it. if EventContentFields.MENTIONS in event.content: - validate_json_object(event.content[EventContentFields.MENTIONS], Mentions) + try: + jsonschema.validate( + instance=event.content[EventContentFields.MENTIONS], + schema=MENTIONS_SCHEMA, + cls=MENTIONS_VALIDATOR, + ) + except jsonschema.ValidationError as e: + raise _validation_error_to_api_error(e) def _validate_retention(self, event: EventBase) -> None: """Checks that an event that defines the retention policy for a room respects the @@ -284,10 +270,16 @@ def _ensure_state_event(self, event: EventBase | EventBuilder) -> None: }, } - -class Mentions(RequestBodyModel): - user_ids: list[StrictStr] = Field(default_factory=list) - room: StrictBool = False +MENTIONS_SCHEMA = { + "type": "object", + "properties": { + "user_ids": { + "type": "array", + "items": {"type": "string"}, + }, + "room": {"type": "boolean"}, + }, +} # This could return something newer than Draft 7, but that's the current "latest" @@ -295,14 +287,45 @@ class Mentions(RequestBodyModel): def _create_validator(schema: JsonDict) -> type[jsonschema.Draft7Validator]: validator = jsonschema.validators.validator_for(schema) - # by default jsonschema does not consider a immutabledict to be an object so - # we need to use a custom type checker + # by default jsonschema does not consider a immutabledict to be an object, or + # a tuple to be an array (frozenutils freezes lists to tuples), so we need a + # custom type checker for both. # https://python-jsonschema.readthedocs.io/en/stable/validate/?highlight=object#validating-with-additional-types type_checker = validator.TYPE_CHECKER.redefine( "object", lambda checker, thing: isinstance(thing, collections.abc.Mapping) + ).redefine( + "array", + lambda checker, thing: isinstance(thing, collections.abc.Sequence), ) return jsonschema.validators.extend(validator, type_checker=type_checker) +def _validation_error_to_api_error(err: jsonschema.ValidationError) -> SynapseError: + """ + Converts a JSONSchema `ValidationError` to a `SynapseError` that can be thrown + to give a Matrix API-compatible 400 Bad Request response with `M_BAD_JSON` code + and a descriptive error message. + """ + if err.path: + # example: "users_default": '0' is not of type 'integer' + # cast safety: path entries can be integers, if we fail to validate + # items in an array. However, the POWER_LEVELS_SCHEMA doesn't expect + # to see any arrays. + message = '"' + cast(str, err.path[-1]) + '": ' + err.message + # jsonschema.ValidationError.message is a valid attribute + else: + # example: '0' is not of type 'integer' + message = err.message + # jsonschema.ValidationError.message is a valid attribute + + return SynapseError( + code=400, + msg=message, + errcode=Codes.BAD_JSON, + ) + + POWER_LEVELS_VALIDATOR = _create_validator(POWER_LEVELS_SCHEMA) + +MENTIONS_VALIDATOR = _create_validator(MENTIONS_SCHEMA) diff --git a/tests/events/test_validator.py b/tests/events/test_validator.py new file mode 100644 index 00000000000..3810fdb3da6 --- /dev/null +++ b/tests/events/test_validator.py @@ -0,0 +1,50 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +from synapse.api.room_versions import RoomVersions +from synapse.events import make_event_from_dict +from synapse.events.validator import EventValidator + +from tests.unittest import HomeserverTestCase + + +class EventValidatorTestCase(HomeserverTestCase): + def test_validate_new_with_mentions_succeeds_even_when_frozen(self) -> None: + """ + Test that `EventValidator.validate_new` accepts an event with valid `m.mentions` + content even when the event is frozen. + """ + event = make_event_from_dict( + { + "room_id": "!room:test", + "type": "m.room.message", + "sender": "@alice:example.com", + "content": { + "msgtype": "m.text", + "body": "@alice:example.com hello", + "m.mentions": {"user_ids": ["@alice:example.com"]}, + }, + "auth_events": [], + "prev_events": [], + "hashes": {"sha256": "aGVsbG8="}, + "signatures": {}, + "depth": 1, + "origin_server_ts": 1000, + }, + room_version=RoomVersions.V9, + ) + # Sanity check that the event is valid before freezing + EventValidator().validate_new(event, self.hs.config) + event.freeze() + # Event should still be valid after freezing + EventValidator().validate_new(event, self.hs.config) From 4d0e4ff9359ecf40997a5f03890cf2aece410319 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Mon, 18 May 2026 12:15:57 +0100 Subject: [PATCH 44/92] Fix `/sync` failing when MSC4354 Sticky Events are enabled and the sync request filters out Ephemeral Data Units (EDUs). (#19787) Fixes: #19779 Fixes: https://github.com/element-hq/synapse/issues/19618 See also: #19786 (which would have caught this, but currently has too many findings to enable) Fix UnboundLocalError when MSC4354 is enabled in sync and all EDUs are filtered out --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19787.bugfix | 1 + synapse/handlers/sync.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 changelog.d/19787.bugfix diff --git a/changelog.d/19787.bugfix b/changelog.d/19787.bugfix new file mode 100644 index 00000000000..26bd3e2252d --- /dev/null +++ b/changelog.d/19787.bugfix @@ -0,0 +1 @@ +Fix `/sync` failing when [MSC4354 Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354) are enabled and the sync request filters out Ephemeral Data Units (EDUs). \ No newline at end of file diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index bca51b64b71..9ecfe0da0f2 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -2236,19 +2236,23 @@ async def _generate_sync_entry_for_rooms( if block_all_room_ephemeral: ephemeral_by_room: dict[str, list[JsonDict]] = {} else: - now_token, ephemeral_by_room = await self.ephemeral_by_room( + ( + sync_result_builder.now_token, + ephemeral_by_room, + ) = await self.ephemeral_by_room( sync_result_builder, now_token=sync_result_builder.now_token, since_token=sync_result_builder.since_token, ) - sync_result_builder.now_token = now_token sticky_by_room: dict[str, list[str]] = {} if self.hs_config.experimental.msc4354_enabled: - now_token, sticky_by_room = await self.sticky_events_by_room( - sync_result_builder, now_token, since_token + ( + sync_result_builder.now_token, + sticky_by_room, + ) = await self.sticky_events_by_room( + sync_result_builder, sync_result_builder.now_token, since_token ) - sync_result_builder.now_token = now_token # 2. We check up front if anything has changed, if it hasn't then there is # no point in going further. From ec4950b438f17efd7ceaf3fd1e8fa2b9c7d41ac4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 May 2026 16:10:40 +0000 Subject: [PATCH 45/92] Bump types-jsonschema from 4.26.0.20260202 to 4.26.0.20260508 (#19683) Signed-off-by: dependabot[bot] --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 78f18caadf9..ab790015bb6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3417,14 +3417,14 @@ types-webencodings = "*" [[package]] name = "types-jsonschema" -version = "4.26.0.20260202" +version = "4.26.0.20260508" description = "Typing stubs for jsonschema" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "types_jsonschema-4.26.0.20260202-py3-none-any.whl", hash = "sha256:41c95343abc4de9264e333a55e95dfb4d401e463856d0164eec9cb182e8746da"}, - {file = "types_jsonschema-4.26.0.20260202.tar.gz", hash = "sha256:29831baa4308865a9aec547a61797a06fc152b0dac8dddd531e002f32265cb07"}, + {file = "types_jsonschema-4.26.0.20260508-py3-none-any.whl", hash = "sha256:4ec1dea0a757c8c2e2aa7bc085612fb54e1ae9562428d5da6f26dd7a0f24dbc2"}, + {file = "types_jsonschema-4.26.0.20260508.tar.gz", hash = "sha256:ae0be85ac6ec0cb94a98f75f876b0620cf2afa3e37fdf8460203f4d05f745acb"}, ] [package.dependencies] From 7b1c4da5dfbe48fa5e0dabc3933aed54f40e1644 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 19 May 2026 14:13:03 +0100 Subject: [PATCH 46/92] 1.153.0 --- CHANGES.md | 5 +++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 96a9fadbfb6..95ab39d0e3d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,8 @@ +# Synapse 1.153.0 (2026-05-19) + +No significant changes since 1.153.0rc3. + + # Synapse 1.153.0rc3 (2026-05-15) ## Bugfixes diff --git a/debian/changelog b/debian/changelog index 81f080bead3..b1a4e04bc38 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.153.0) stable; urgency=medium + + * New Synapse release 1.153.0. + + -- Synapse Packaging team Tue, 19 May 2026 14:12:51 +0100 + matrix-synapse-py3 (1.153.0~rc3) stable; urgency=medium * New Synapse release 1.153.0rc3. diff --git a/pyproject.toml b/pyproject.toml index fed1a23f4a0..66cd3e83b5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.153.0rc3" +version = "1.153.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 87095ae500ef1371346f57ef6dfb29231e822cb9 Mon Sep 17 00:00:00 2001 From: FrenchGithubUser Date: Wed, 20 May 2026 12:06:16 +0200 Subject: [PATCH 47/92] fix: invalidate access token cache on device deletion (#19483) when an access token had a refresh token associated to it in the database, deleting this refresh token (for example when deleting the device using it) would cascade delete the access token, which wouldn't be returned by the sql query that was supposed to delete it on its own, and an empty array was passed to the cache invalidation function. --- changelog.d/19483.bugfix | 1 + .../storage/databases/main/registration.py | 19 ++-- tests/rest/client/test_auth.py | 90 ++++++++++++++++++- 3 files changed, 100 insertions(+), 10 deletions(-) create mode 100644 changelog.d/19483.bugfix diff --git a/changelog.d/19483.bugfix b/changelog.d/19483.bugfix new file mode 100644 index 00000000000..9e4fb209965 --- /dev/null +++ b/changelog.d/19483.bugfix @@ -0,0 +1 @@ +Fix access token cache not being invalidated for sessions using refresh tokens. Contributed by @FrenchGithubUser @ Famedly. diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index 7b0de924351..2455057b027 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -2474,14 +2474,6 @@ async def user_delete_access_tokens_for_devices( def user_delete_access_tokens_for_devices_txn( txn: LoggingTransaction, batch_device_ids: StrCollection ) -> list[tuple[str, int, str | None]]: - self.db_pool.simple_delete_many_txn( - txn, - table="refresh_tokens", - keyvalues={"user_id": user_id}, - column="device_id", - values=batch_device_ids, - ) - clause, args = make_in_list_sql_clause( txn.database_engine, "device_id", batch_device_ids ) @@ -2500,6 +2492,17 @@ def user_delete_access_tokens_for_devices_txn( self.get_user_by_access_token, [(t[0],) for t in tokens_and_devices], ) + # Delete access tokens first, before refresh tokens. + # This ensures we can capture the deleted access tokens for cache invalidation + # before any CASCADE deletes occur. + self.db_pool.simple_delete_many_txn( + txn, + table="refresh_tokens", + keyvalues={"user_id": user_id}, + column="device_id", + values=batch_device_ids, + ) + return tokens_and_devices results = [] diff --git a/tests/rest/client/test_auth.py b/tests/rest/client/test_auth.py index c266937a65a..4eb918d5c5c 100644 --- a/tests/rest/client/test_auth.py +++ b/tests/rest/client/test_auth.py @@ -30,7 +30,14 @@ from synapse.api.constants import ApprovalNoticeMedium, LoginType from synapse.api.errors import Codes, SynapseError from synapse.handlers.ui_auth.checkers import UserInteractiveAuthChecker -from synapse.rest.client import account, auth, devices, login, logout, register +from synapse.rest.client import ( + account, + auth, + devices, + login, + logout, + register, +) from synapse.rest.synapse.client import build_synapse_client_resource_tree from synapse.server import HomeServer from synapse.storage.database import LoggingTransaction @@ -765,10 +772,11 @@ class RefreshAuthTests(unittest.HomeserverTestCase): servlets = [ auth.register_servlets, account.register_servlets, + devices.register_servlets, login.register_servlets, logout.register_servlets, - synapse.rest.admin.register_servlets_for_client_rest_resource, register.register_servlets, + synapse.rest.admin.register_servlets_for_client_rest_resource, ] hijack_auth = False @@ -1309,6 +1317,84 @@ def _txn(txn: LoggingTransaction) -> int: self.assertEqual(_table_length("access_tokens"), 0) self.assertEqual(_table_length("refresh_tokens"), 0) + def test_token_invalid_after_refresh_token_issued_and_device_removed(self) -> None: + """ + Test that an access token is invalidated after the device (which had a + refresh token) is removed by another device. + The removal of a refresh token cascade deletes the associated access + token in the db, which can make cache invalidation fail, if not handled + properly. This test will catch such behavior if it ever happens again. + 1. User logs in with device1 + 2. User logs in with device2 and requests a refresh token + 3. Device2 calls /whoami (should work) + 4. Device1 removes device2 + 5. Device2 calls /whoami (should fail) + """ + # Login with device1 + device1_tok = self.login("test", self.user_pass, device_id="device1") + + # Login with device2 and request a refresh token + login_response = self.make_request( + "POST", + "/_matrix/client/r0/login", + { + "type": "m.login.password", + "user": "test", + "password": self.user_pass, + "device_id": "device2", + "refresh_token": True, + }, + ) + self.assertEqual(login_response.code, HTTPStatus.OK, login_response.result) + device2_tok = login_response.json_body["access_token"] + device2_id = login_response.json_body["device_id"] + self.assertEqual(device2_id, "device2") + + # Device2 calls /whoami (should work) + channel = self.make_request( + "GET", + "/_matrix/client/v3/account/whoami", + access_token=device2_tok, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + + # Device1 removes device2 + # First, attempt to delete device2 + delete_channel = self.make_request( + "DELETE", + f"devices/{device2_id}", + access_token=device1_tok, + ) + self.assertEqual( + delete_channel.code, HTTPStatus.UNAUTHORIZED, delete_channel.result + ) + session = delete_channel.json_body["session"] + + # Complete the UI auth flow + delete_channel = self.make_request( + "DELETE", + f"devices/{device2_id}", + content={ + "auth": { + "type": "m.login.password", + "user": "test", + "password": self.user_pass, + "session": session, + }, + }, + access_token=device1_tok, + ) + self.assertEqual(delete_channel.code, HTTPStatus.OK, delete_channel.result) + + # Device2 calls /whoami (should fail) + channel = self.make_request( + "GET", + "/_matrix/client/v3/account/whoami", + access_token=device2_tok, + ) + self.assertEqual(channel.code, HTTPStatus.UNAUTHORIZED, channel.result) + self.assertEqual(channel.json_body["errcode"], "M_UNKNOWN_TOKEN") + def oidc_config( id: str, with_localpart_template: bool, **kwargs: Any From be03be7b50ca914d73757a00d2819807187aa3a8 Mon Sep 17 00:00:00 2001 From: Oleg Girko Date: Wed, 20 May 2026 11:47:43 +0100 Subject: [PATCH 48/92] Partially revert "Bump attrs from 25.4.0 to 26.1.0 (#19684)" (#19789) Accidental bump broke build for Fedora and RHEL. This reverts commit 2e9d6f7f35df53f510b689c09d3a9ace82a0d840. As discussed in the [Synapse Package Maintainers](https://matrix.to/#/!rh9Uxk45AsPongyP3ypgpsCmuIufiggD6mDXFWh4_FM/$0mdulZEyJFdI6bwS8GFwYnFt-zmpyCyx2DwcA8JyuY8?via=jki.re&via=matrix.org&via=element.io) room (private) --- changelog.d/19789.bugfix | 1 + poetry.lock | 53 ++++++++++++++++++++-------------------- pyproject.toml | 2 +- 3 files changed, 29 insertions(+), 27 deletions(-) create mode 100644 changelog.d/19789.bugfix diff --git a/changelog.d/19789.bugfix b/changelog.d/19789.bugfix new file mode 100644 index 00000000000..f6c325ec824 --- /dev/null +++ b/changelog.d/19789.bugfix @@ -0,0 +1 @@ +Fix packaging for Fedora and EPEL caused by unnecessary bumping `attrs` minimum version requirement in `pyproject.toml` file. Contributed by Oleg Girko. diff --git a/poetry.lock b/poetry.lock index ab790015bb6..92c25d93085 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,7 +31,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" +markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" files = [ {file = "authlib-1.6.12-py2.py3-none-any.whl", hash = "sha256:e9229ad7fde610b139dd12f5edbe97eab9ee78bfb85691247e767727850b99ab"}, {file = "authlib-1.6.12.tar.gz", hash = "sha256:0656d8482f28fc8221929d5f35b2bde5d13e10555ebc06b4561b0d622e83b1bd"}, @@ -62,7 +62,7 @@ description = "Backport of CPython tarfile module" optional = false python-versions = ">=3.8" groups = ["dev"] -markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" +markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34"}, {file = "backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"}, @@ -531,7 +531,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -556,7 +556,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "elementpath-4.8.0-py3-none-any.whl", hash = "sha256:5393191f84969bcf8033b05ec4593ef940e58622ea13cefe60ecefbbf09d58d9"}, {file = "elementpath-4.8.0.tar.gz", hash = "sha256:5822a2560d99e2633d95f78694c7ff9646adaa187db520da200a8e9479dc46ae"}, @@ -606,7 +606,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:f525734382a47f9828c9d6a1501522c78d5935466d8e2be1a41ba40ca5bb922b"}, {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:6e2e1024f0a021777740cb7c633a0efb2c4a4bc570f508223a8dcbcf79f99ef9"}, @@ -889,7 +889,7 @@ description = "Read metadata from Python packages" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" +markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151"}, {file = "importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb"}, @@ -930,7 +930,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1122,7 +1122,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1239,7 +1239,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"url-preview\"" +markers = "extra == \"url-preview\" or extra == \"all\"" files = [ {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e77dd455b9a16bbd2a5036a63ddbd479c19572af81b624e79ef422f929eef388"}, {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d444858b9f07cefff6455b983aea9a67f7462ba1f6cbe4a21e8bf6791bf2153"}, @@ -1553,7 +1553,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "matrix_synapse_ldap3-0.4.0-py3-none-any.whl", hash = "sha256:bf080037230d2af5fd3639cb87266de65c1cad7a68ea206278c5b4bf9c1a17f3"}, {file = "matrix_synapse_ldap3-0.4.0.tar.gz", hash = "sha256:cff52ba780170de5e6e8af42863d2648ee23f3bf0a9fea6db52372f9fc00be2b"}, @@ -1834,7 +1834,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -2032,7 +2032,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"postgres\"" +markers = "extra == \"postgres\" or extra == \"all\"" files = [ {file = "psycopg2-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:103e857f46bb76908768ead4e2d0ba1d1a130e7b8ed77d3ae91e8b33481813e8"}, {file = "psycopg2-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:210daed32e18f35e3140a1ebe059ac29209dd96468f2f7559aa59f75ee82a5cb"}, @@ -2050,7 +2050,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -2066,7 +2066,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2348,7 +2348,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"cache-memory\"" +markers = "extra == \"cache-memory\" or extra == \"all\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2480,7 +2480,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2505,7 +2505,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -2533,7 +2533,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pytz-2026.1.post1-py2.py3-none-any.whl", hash = "sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a"}, {file = "pytz-2026.1.post1.tar.gz", hash = "sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1"}, @@ -2938,7 +2938,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"sentry\"" +markers = "extra == \"sentry\" or extra == \"all\"" files = [ {file = "sentry_sdk-2.57.0-py2.py3-none-any.whl", hash = "sha256:812c8bf5ff3d2f0e89c82f5ce80ab3a6423e102729c4706af7413fd1eb480585"}, {file = "sentry_sdk-2.57.0.tar.gz", hash = "sha256:4be8d1e71c32fb27f79c577a337ac8912137bba4bcbc64a4ec1da4d6d8dc5199"}, @@ -3138,7 +3138,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -3154,7 +3154,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "thrift-0.22.0.tar.gz", hash = "sha256:42e8276afbd5f54fe1d364858b6877bc5e5a4a5ed69f6a005b94ca4918fe1466"}, ] @@ -3220,6 +3220,7 @@ files = [ {file = "tomli-2.4.0-py3-none-any.whl", hash = "sha256:1f776e7d669ebceb01dee46484485f43a4048746235e683bcdffacdf1fb4785a"}, {file = "tomli-2.4.0.tar.gz", hash = "sha256:aa89c3f6c277dd275d8e243ad24f3b5e701491a860d5121f2cdd399fbb31fc9c"}, ] +markers = {main = "python_version < \"3.14\""} [[package]] name = "tornado" @@ -3228,7 +3229,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:487dc9cc380e29f58c7ab88f9e27cdeef04b2140862e5076a66fb6bb68bb1bfa"}, {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:65a7f1d46d4bb41df1ac99f5fcb685fb25c7e61613742d5108b010975a9a6521"}, @@ -3360,7 +3361,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3621,7 +3622,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "xmlschema-2.5.1-py3-none-any.whl", hash = "sha256:ec2b2a15c8896c1fcd14dcee34ca30032b99456c3c43ce793fdb9dca2fb4b869"}, {file = "xmlschema-2.5.1.tar.gz", hash = "sha256:4f7497de6c8b6dc2c28ad7b9ed6e21d186f4afe248a5bea4f54eedab4da44083"}, @@ -3642,7 +3643,7 @@ description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" +markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, @@ -3755,4 +3756,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<4.0.0" -content-hash = "d97bee07fec0f4048d964aa7127a50813920bce77b00e5191aa1815f83922c85" +content-hash = "ef0540b89c417a69668f551688bd0974256ea7a580044f3954a76bdf0d8fe7c9" diff --git a/pyproject.toml b/pyproject.toml index 66cd3e83b5e..e92e7b5c21f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ dependencies = [ "prometheus-client>=0.6.0", # we use `order`, which arrived in attrs 19.2.0. # Note: 21.1.0 broke `/sync`, see https://github.com/matrix-org/synapse/issues/9936 - "attrs>=26.1.0,!=21.1.0", + "attrs>=19.2.0,!=21.1.0", "netaddr>=0.7.18", # Jinja 2.x is incompatible with MarkupSafe>=2.1. To ensure that admins do not # end up with a broken installation, with recent MarkupSafe but old Jinja, we From 966e193e4e1900bc28891ed697aee30ab5a72dbd Mon Sep 17 00:00:00 2001 From: Joe Groocock Date: Wed, 20 May 2026 12:10:54 +0100 Subject: [PATCH 49/92] Fix sending heroes in SSS when `m.room.name=""` (#19468) As per the spec, a room with m.room.name value that is absent, null or empty should be treated as if there is no m.room.name event at all: https://spec.matrix.org/v1.17/client-server-api/#mroomname This fetches the full m.room.name event and checks the content.name instead of only checking the existence of the m.room.name event. This results in correctly sending heroes for those rooms. Fixes: https://github.com/element-hq/synapse/issues/19447 Signed-off-by: Joe Groocock --- changelog.d/19468.bugfix | 1 + synapse/handlers/sliding_sync/__init__.py | 51 +++++++------- .../client/sliding_sync/test_rooms_meta.py | 70 +++++++++++++++++++ 3 files changed, 98 insertions(+), 24 deletions(-) create mode 100644 changelog.d/19468.bugfix diff --git a/changelog.d/19468.bugfix b/changelog.d/19468.bugfix new file mode 100644 index 00000000000..003716d296e --- /dev/null +++ b/changelog.d/19468.bugfix @@ -0,0 +1 @@ +Fix a bug in [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) that could prevent user avatars from showing if the room had an empty name. diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index ddce04d5b32..df1270f7f97 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -872,20 +872,10 @@ async def get_room_sync_data( # For incremental syncs, we can do this first to determine if something relevant # has changed and strategically avoid fetching other costly things. room_state_delta_id_map: MutableStateMap[str] = {} - name_event_id: str | None = None membership_changed = False name_changed = False avatar_changed = False - if initial: - # Check whether the room has a name set - name_state_ids = await self.get_current_state_ids_at( - room_id=room_id, - room_membership_for_user_at_to_token=room_membership_for_user_at_to_token, - state_filter=StateFilter.from_types([(EventTypes.Name, "")]), - to_token=to_token, - ) - name_event_id = name_state_ids.get((EventTypes.Name, "")) - else: + if not initial: assert from_bound is not None # TODO: Limit the number of state events we're about to send down @@ -933,6 +923,27 @@ async def get_room_sync_data( ): avatar_changed = True + # If a room has an m.room.name event with an absent, null, or empty + # name field, it should be treated the same as a room with no + # m.room.name event. + # https://spec.matrix.org/v1.17/client-server-api/#mroomname + # + # TODO: Should we also check for `EventTypes.CanonicalAlias` + # (`m.room.canonical_alias`) as a fallback for the room name? see + # https://github.com/matrix-org/matrix-spec-proposals/pull/4186/changes#r2860107511 + room_name: str | None = None + if initial or name_changed: + # Check whether the room has a name set + name_states = await self.get_current_state_at( + room_id=room_id, + room_membership_for_user_at_to_token=room_membership_for_user_at_to_token, + state_filter=StateFilter.from_types([(EventTypes.Name, "")]), + to_token=to_token, + ) + name_event = name_states.get((EventTypes.Name, "")) + if name_event is not None: + room_name = name_event.content.get("name") + # We only need the room summary for calculating heroes, however if we do # fetch it then we can use it to calculate `joined_count` and # `invited_count`. @@ -949,12 +960,13 @@ async def get_room_sync_data( hero_user_ids: list[str] = [] # TODO: Should we also check for `EventTypes.CanonicalAlias` # (`m.room.canonical_alias`) as a fallback for the room name? see - # https://github.com/matrix-org/matrix-spec-proposals/pull/3575#discussion_r1671260153 + # https://github.com/matrix-org/matrix-spec-proposals/pull/4186/changes#r2860107511 # - # We need to fetch the `heroes` if the room name is not set. But we only need to - # get them on initial syncs (or the first time we send down the room) or if the + # We need to fetch the `heroes` if the room name is not set (taking + # care to treat an empty string as unset). But we only need to get them + # on initial syncs (or the first time we send down the room) or if the # membership has changed which may change the heroes. - if name_event_id is None and (initial or (not initial and membership_changed)): + if not room_name and (initial or (not initial and membership_changed)): # We need the room summary to extract the heroes from if room_membership_for_user_at_to_token.membership != Membership.JOIN: # TODO: Figure out how to get the membership summary for left/banned rooms @@ -1332,15 +1344,6 @@ async def get_room_sync_data( if required_state_filter != StateFilter.none(): required_room_state = required_state_filter.filter_state(room_state) - # Find the room name and avatar from the state - room_name: str | None = None - # TODO: Should we also check for `EventTypes.CanonicalAlias` - # (`m.room.canonical_alias`) as a fallback for the room name? see - # https://github.com/matrix-org/matrix-spec-proposals/pull/3575#discussion_r1671260153 - name_event = room_state.get((EventTypes.Name, "")) - if name_event is not None: - room_name = name_event.content.get("name") - room_avatar: str | None = None avatar_event = room_state.get((EventTypes.RoomAvatar, "")) if avatar_event is not None: diff --git a/tests/rest/client/sliding_sync/test_rooms_meta.py b/tests/rest/client/sliding_sync/test_rooms_meta.py index 4ff07b4d261..b1b771ef848 100644 --- a/tests/rest/client/sliding_sync/test_rooms_meta.py +++ b/tests/rest/client/sliding_sync/test_rooms_meta.py @@ -631,6 +631,76 @@ def test_rooms_meta_heroes_max(self) -> None: # We didn't request any state so we shouldn't see any `required_state` self.assertIsNone(response_body["rooms"][room_id1].get("required_state")) + def test_rooms_meta_heroes_empty_room_name(self) -> None: + """ + Test that the `rooms` `heroes` are included when the room name is an + empty string (i.e. unset as per the spec) + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + user3_id = self.register_user("user3", "pass") + _user3_tok = self.login(user3_id, "pass") + + room_id = self.helper.create_room_as( + user2_id, + tok=user2_tok, + extra_content={ + # https://spec.matrix.org/v1.17/client-server-api/#mroomname + # > If a room has an m.room.name event with an absent, null, or + # > empty name field, it should be treated the same as a room + # > with no m.room.name event. + "name": "", + }, + ) + self.helper.join(room_id, user1_id, tok=user1_tok) + # User3 is invited + self.helper.invite(room_id, src=user2_id, targ=user3_id, tok=user2_tok) + + # Make the Sliding Sync request + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [], + "timeline_limit": 1, + } + } + } + response_body, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Room has an empty name so we should see `heroes` populated + self.assertEqual(response_body["rooms"][room_id]["initial"], True) + self.assertIsNone(response_body["rooms"][room_id].get("name")) + self.assertCountEqual( + [ + hero["user_id"] + for hero in response_body["rooms"][room_id].get("heroes", []) + ], + # Heroes shouldn't include the user themselves (we shouldn't see user1) + [user2_id, user3_id], + ) + self.assertEqual( + response_body["rooms"][room_id]["joined_count"], + 2, + ) + self.assertEqual( + response_body["rooms"][room_id]["invited_count"], + 1, + ) + + # We didn't request any state so we shouldn't see any `required_state` + self.assertIsNone(response_body["rooms"][room_id].get("required_state")) + + # Send a message to make the room come down sync + self.helper.send(room_id, "message in room", tok=user2_tok) + + # Incremental sync + incremental_body, _ = self.do_sync(sync_body, since=from_token, tok=user1_tok) + self.assertNotIn("name", incremental_body["rooms"][room_id]) + self.assertNotIn("heroes", incremental_body["rooms"][room_id]) + def test_rooms_meta_heroes_when_banned(self) -> None: """ Test that the `rooms` `heroes` are included in the response when the room From 57299fd5bdb1a563c41b03da1e75b8a55413701c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 May 2026 13:32:08 +0000 Subject: [PATCH 50/92] Bump lxml from 6.0.2 to 6.1.0 (#19716) Signed-off-by: dependabot[bot] --- poetry.lock | 329 +++++++++++++++++++++++++--------------------------- 1 file changed, 161 insertions(+), 168 deletions(-) diff --git a/poetry.lock b/poetry.lock index 92c25d93085..6093b13c02f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,7 +31,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" files = [ {file = "authlib-1.6.12-py2.py3-none-any.whl", hash = "sha256:e9229ad7fde610b139dd12f5edbe97eab9ee78bfb85691247e767727850b99ab"}, {file = "authlib-1.6.12.tar.gz", hash = "sha256:0656d8482f28fc8221929d5f35b2bde5d13e10555ebc06b4561b0d622e83b1bd"}, @@ -62,7 +62,7 @@ description = "Backport of CPython tarfile module" optional = false python-versions = ">=3.8" groups = ["dev"] -markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" files = [ {file = "backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34"}, {file = "backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"}, @@ -531,7 +531,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -556,7 +556,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "elementpath-4.8.0-py3-none-any.whl", hash = "sha256:5393191f84969bcf8033b05ec4593ef940e58622ea13cefe60ecefbbf09d58d9"}, {file = "elementpath-4.8.0.tar.gz", hash = "sha256:5822a2560d99e2633d95f78694c7ff9646adaa187db520da200a8e9479dc46ae"}, @@ -606,7 +606,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:f525734382a47f9828c9d6a1501522c78d5935466d8e2be1a41ba40ca5bb922b"}, {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:6e2e1024f0a021777740cb7c633a0efb2c4a4bc570f508223a8dcbcf79f99ef9"}, @@ -889,7 +889,7 @@ description = "Read metadata from Python packages" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" files = [ {file = "importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151"}, {file = "importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb"}, @@ -930,7 +930,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1122,7 +1122,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1234,153 +1234,147 @@ files = [ [[package]] name = "lxml" -version = "6.0.2" +version = "6.1.0" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"url-preview\" or extra == \"all\"" -files = [ - {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e77dd455b9a16bbd2a5036a63ddbd479c19572af81b624e79ef422f929eef388"}, - {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d444858b9f07cefff6455b983aea9a67f7462ba1f6cbe4a21e8bf6791bf2153"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f952dacaa552f3bb8834908dddd500ba7d508e6ea6eb8c52eb2d28f48ca06a31"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:71695772df6acea9f3c0e59e44ba8ac50c4f125217e84aab21074a1a55e7e5c9"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:17f68764f35fd78d7c4cc4ef209a184c38b65440378013d24b8aecd327c3e0c8"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:058027e261afed589eddcfe530fcc6f3402d7fd7e89bfd0532df82ebc1563dba"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8ffaeec5dfea5881d4c9d8913a32d10cfe3923495386106e4a24d45300ef79c"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:f2e3b1a6bb38de0bc713edd4d612969dd250ca8b724be8d460001a387507021c"}, - {file = "lxml-6.0.2-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d6690ec5ec1cce0385cb20896b16be35247ac8c2046e493d03232f1c2414d321"}, - {file = "lxml-6.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f2a50c3c1d11cad0ebebbac357a97b26aa79d2bcaf46f256551152aa85d3a4d1"}, - {file = "lxml-6.0.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:3efe1b21c7801ffa29a1112fab3b0f643628c30472d507f39544fd48e9549e34"}, - {file = "lxml-6.0.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:59c45e125140b2c4b33920d21d83681940ca29f0b83f8629ea1a2196dc8cfe6a"}, - {file = "lxml-6.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:452b899faa64f1805943ec1c0c9ebeaece01a1af83e130b69cdefeda180bb42c"}, - {file = "lxml-6.0.2-cp310-cp310-win32.whl", hash = "sha256:1e786a464c191ca43b133906c6903a7e4d56bef376b75d97ccbb8ec5cf1f0a4b"}, - {file = "lxml-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:dacf3c64ef3f7440e3167aa4b49aa9e0fb99e0aa4f9ff03795640bf94531bcb0"}, - {file = "lxml-6.0.2-cp310-cp310-win_arm64.whl", hash = "sha256:45f93e6f75123f88d7f0cfd90f2d05f441b808562bf0bc01070a00f53f5028b5"}, - {file = "lxml-6.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:13e35cbc684aadf05d8711a5d1b5857c92e5e580efa9a0d2be197199c8def607"}, - {file = "lxml-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b1675e096e17c6fe9c0e8c81434f5736c0739ff9ac6123c87c2d452f48fc938"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8ac6e5811ae2870953390452e3476694196f98d447573234592d30488147404d"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5aa0fc67ae19d7a64c3fe725dc9a1bb11f80e01f78289d05c6f62545affec438"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de496365750cc472b4e7902a485d3f152ecf57bd3ba03ddd5578ed8ceb4c5964"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:200069a593c5e40b8f6fc0d84d86d970ba43138c3e68619ffa234bc9bb806a4d"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d2de809c2ee3b888b59f995625385f74629707c9355e0ff856445cdcae682b7"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:b2c3da8d93cf5db60e8858c17684c47d01fee6405e554fb55018dd85fc23b178"}, - {file = "lxml-6.0.2-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:442de7530296ef5e188373a1ea5789a46ce90c4847e597856570439621d9c553"}, - {file = "lxml-6.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2593c77efde7bfea7f6389f1ab249b15ed4aa5bc5cb5131faa3b843c429fbedb"}, - {file = "lxml-6.0.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3e3cb08855967a20f553ff32d147e14329b3ae70ced6edc2f282b94afbc74b2a"}, - {file = "lxml-6.0.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2ed6c667fcbb8c19c6791bbf40b7268ef8ddf5a96940ba9404b9f9a304832f6c"}, - {file = "lxml-6.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b8f18914faec94132e5b91e69d76a5c1d7b0c73e2489ea8929c4aaa10b76bbf7"}, - {file = "lxml-6.0.2-cp311-cp311-win32.whl", hash = "sha256:6605c604e6daa9e0d7f0a2137bdc47a2e93b59c60a65466353e37f8272f47c46"}, - {file = "lxml-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e5867f2651016a3afd8dd2c8238baa66f1e2802f44bc17e236f547ace6647078"}, - {file = "lxml-6.0.2-cp311-cp311-win_arm64.whl", hash = "sha256:4197fb2534ee05fd3e7afaab5d8bfd6c2e186f65ea7f9cd6a82809c887bd1285"}, - {file = "lxml-6.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a59f5448ba2ceccd06995c95ea59a7674a10de0810f2ce90c9006f3cbc044456"}, - {file = "lxml-6.0.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e8113639f3296706fbac34a30813929e29247718e88173ad849f57ca59754924"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a8bef9b9825fa8bc816a6e641bb67219489229ebc648be422af695f6e7a4fa7f"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:65ea18d710fd14e0186c2f973dc60bb52039a275f82d3c44a0e42b43440ea534"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c371aa98126a0d4c739ca93ceffa0fd7a5d732e3ac66a46e74339acd4d334564"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:700efd30c0fa1a3581d80a748157397559396090a51d306ea59a70020223d16f"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c33e66d44fe60e72397b487ee92e01da0d09ba2d66df8eae42d77b6d06e5eba0"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90a345bbeaf9d0587a3aaffb7006aa39ccb6ff0e96a57286c0cb2fd1520ea192"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:064fdadaf7a21af3ed1dcaa106b854077fbeada827c18f72aec9346847cd65d0"}, - {file = "lxml-6.0.2-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbc74f42c3525ac4ffa4b89cbdd00057b6196bcefe8bce794abd42d33a018092"}, - {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6ddff43f702905a4e32bc24f3f2e2edfe0f8fde3277d481bffb709a4cced7a1f"}, - {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6da5185951d72e6f5352166e3da7b0dc27aa70bd1090b0eb3f7f7212b53f1bb8"}, - {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:57a86e1ebb4020a38d295c04fc79603c7899e0df71588043eb218722dabc087f"}, - {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2047d8234fe735ab77802ce5f2297e410ff40f5238aec569ad7c8e163d7b19a6"}, - {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f91fd2b2ea15a6800c8e24418c0775a1694eefc011392da73bc6cef2623b322"}, - {file = "lxml-6.0.2-cp312-cp312-win32.whl", hash = "sha256:3ae2ce7d6fedfb3414a2b6c5e20b249c4c607f72cb8d2bb7cc9c6ec7c6f4e849"}, - {file = "lxml-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:72c87e5ee4e58a8354fb9c7c84cbf95a1c8236c127a5d1b7683f04bed8361e1f"}, - {file = "lxml-6.0.2-cp312-cp312-win_arm64.whl", hash = "sha256:61cb10eeb95570153e0c0e554f58df92ecf5109f75eacad4a95baa709e26c3d6"}, - {file = "lxml-6.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9b33d21594afab46f37ae58dfadd06636f154923c4e8a4d754b0127554eb2e77"}, - {file = "lxml-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6c8963287d7a4c5c9a432ff487c52e9c5618667179c18a204bdedb27310f022f"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1941354d92699fb5ffe6ed7b32f9649e43c2feb4b97205f75866f7d21aa91452"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bb2f6ca0ae2d983ded09357b84af659c954722bbf04dea98030064996d156048"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb2a12d704f180a902d7fa778c6d71f36ceb7b0d317f34cdc76a5d05aa1dd1df"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:6ec0e3f745021bfed19c456647f0298d60a24c9ff86d9d051f52b509663feeb1"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:846ae9a12d54e368933b9759052d6206a9e8b250291109c48e350c1f1f49d916"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef9266d2aa545d7374938fb5c484531ef5a2ec7f2d573e62f8ce722c735685fd"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:4077b7c79f31755df33b795dc12119cb557a0106bfdab0d2c2d97bd3cf3dffa6"}, - {file = "lxml-6.0.2-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a7c5d5e5f1081955358533be077166ee97ed2571d6a66bdba6ec2f609a715d1a"}, - {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8f8d0cbd0674ee89863a523e6994ac25fd5be9c8486acfc3e5ccea679bad2679"}, - {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:2cbcbf6d6e924c28f04a43f3b6f6e272312a090f269eff68a2982e13e5d57659"}, - {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:dfb874cfa53340009af6bdd7e54ebc0d21012a60a4e65d927c2e477112e63484"}, - {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fb8dae0b6b8b7f9e96c26fdd8121522ce5de9bb5538010870bd538683d30e9a2"}, - {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:358d9adae670b63e95bc59747c72f4dc97c9ec58881d4627fe0120da0f90d314"}, - {file = "lxml-6.0.2-cp313-cp313-win32.whl", hash = "sha256:e8cd2415f372e7e5a789d743d133ae474290a90b9023197fd78f32e2dc6873e2"}, - {file = "lxml-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:b30d46379644fbfc3ab81f8f82ae4de55179414651f110a1514f0b1f8f6cb2d7"}, - {file = "lxml-6.0.2-cp313-cp313-win_arm64.whl", hash = "sha256:13dcecc9946dca97b11b7c40d29fba63b55ab4170d3c0cf8c0c164343b9bfdcf"}, - {file = "lxml-6.0.2-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:b0c732aa23de8f8aec23f4b580d1e52905ef468afb4abeafd3fec77042abb6fe"}, - {file = "lxml-6.0.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4468e3b83e10e0317a89a33d28f7aeba1caa4d1a6fd457d115dd4ffe90c5931d"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:abd44571493973bad4598a3be7e1d807ed45aa2adaf7ab92ab7c62609569b17d"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:370cd78d5855cfbffd57c422851f7d3864e6ae72d0da615fca4dad8c45d375a5"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:901e3b4219fa04ef766885fb40fa516a71662a4c61b80c94d25336b4934b71c0"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:a4bf42d2e4cf52c28cc1812d62426b9503cdb0c87a6de81442626aa7d69707ba"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2c7fdaa4d7c3d886a42534adec7cfac73860b89b4e5298752f60aa5984641a0"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98a5e1660dc7de2200b00d53fa00bcd3c35a3608c305d45a7bbcaf29fa16e83d"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:dc051506c30b609238d79eda75ee9cab3e520570ec8219844a72a46020901e37"}, - {file = "lxml-6.0.2-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8799481bbdd212470d17513a54d568f44416db01250f49449647b5ab5b5dccb9"}, - {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9261bb77c2dab42f3ecd9103951aeca2c40277701eb7e912c545c1b16e0e4917"}, - {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:65ac4a01aba353cfa6d5725b95d7aed6356ddc0a3cd734de00124d285b04b64f"}, - {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:b22a07cbb82fea98f8a2fd814f3d1811ff9ed76d0fc6abc84eb21527596e7cc8"}, - {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d759cdd7f3e055d6bc8d9bec3ad905227b2e4c785dc16c372eb5b5e83123f48a"}, - {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:945da35a48d193d27c188037a05fec5492937f66fb1958c24fc761fb9d40d43c"}, - {file = "lxml-6.0.2-cp314-cp314-win32.whl", hash = "sha256:be3aaa60da67e6153eb15715cc2e19091af5dc75faef8b8a585aea372507384b"}, - {file = "lxml-6.0.2-cp314-cp314-win_amd64.whl", hash = "sha256:fa25afbadead523f7001caf0c2382afd272c315a033a7b06336da2637d92d6ed"}, - {file = "lxml-6.0.2-cp314-cp314-win_arm64.whl", hash = "sha256:063eccf89df5b24e361b123e257e437f9e9878f425ee9aae3144c77faf6da6d8"}, - {file = "lxml-6.0.2-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:6162a86d86893d63084faaf4ff937b3daea233e3682fb4474db07395794fa80d"}, - {file = "lxml-6.0.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:414aaa94e974e23a3e92e7ca5b97d10c0cf37b6481f50911032c69eeb3991bba"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48461bd21625458dd01e14e2c38dd0aea69addc3c4f960c30d9f59d7f93be601"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:25fcc59afc57d527cfc78a58f40ab4c9b8fd096a9a3f964d2781ffb6eb33f4ed"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5179c60288204e6ddde3f774a93350177e08876eaf3ab78aa3a3649d43eb7d37"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:967aab75434de148ec80597b75062d8123cadf2943fb4281f385141e18b21338"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d100fcc8930d697c6561156c6810ab4a508fb264c8b6779e6e61e2ed5e7558f9"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ca59e7e13e5981175b8b3e4ab84d7da57993eeff53c07764dcebda0d0e64ecd"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:957448ac63a42e2e49531b9d6c0fa449a1970dbc32467aaad46f11545be9af1d"}, - {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b7fc49c37f1786284b12af63152fe1d0990722497e2d5817acfe7a877522f9a9"}, - {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e19e0643cc936a22e837f79d01a550678da8377d7d801a14487c10c34ee49c7e"}, - {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:1db01e5cf14345628e0cbe71067204db658e2fb8e51e7f33631f5f4735fefd8d"}, - {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:875c6b5ab39ad5291588aed6925fac99d0097af0dd62f33c7b43736043d4a2ec"}, - {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cdcbed9ad19da81c480dfd6dd161886db6096083c9938ead313d94b30aadf272"}, - {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:80dadc234ebc532e09be1975ff538d154a7fa61ea5031c03d25178855544728f"}, - {file = "lxml-6.0.2-cp314-cp314t-win32.whl", hash = "sha256:da08e7bb297b04e893d91087df19638dc7a6bb858a954b0cc2b9f5053c922312"}, - {file = "lxml-6.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:252a22982dca42f6155125ac76d3432e548a7625d56f5a273ee78a5057216eca"}, - {file = "lxml-6.0.2-cp314-cp314t-win_arm64.whl", hash = "sha256:bb4c1847b303835d89d785a18801a883436cdfd5dc3d62947f9c49e24f0f5a2c"}, - {file = "lxml-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a656ca105115f6b766bba324f23a67914d9c728dafec57638e2b92a9dcd76c62"}, - {file = "lxml-6.0.2-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c54d83a2188a10ebdba573f16bd97135d06c9ef60c3dc495315c7a28c80a263f"}, - {file = "lxml-6.0.2-cp38-cp38-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:1ea99340b3c729beea786f78c38f60f4795622f36e305d9c9be402201efdc3b7"}, - {file = "lxml-6.0.2-cp38-cp38-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:af85529ae8d2a453feee4c780d9406a5e3b17cee0dd75c18bd31adcd584debc3"}, - {file = "lxml-6.0.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:fe659f6b5d10fb5a17f00a50eb903eb277a71ee35df4615db573c069bcf967ac"}, - {file = "lxml-6.0.2-cp38-cp38-win32.whl", hash = "sha256:5921d924aa5468c939d95c9814fa9f9b5935a6ff4e679e26aaf2951f74043512"}, - {file = "lxml-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:0aa7070978f893954008ab73bb9e3c24a7c56c054e00566a21b553dc18105fca"}, - {file = "lxml-6.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2c8458c2cdd29589a8367c09c8f030f1d202be673f0ca224ec18590b3b9fb694"}, - {file = "lxml-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3fee0851639d06276e6b387f1c190eb9d7f06f7f53514e966b26bae46481ec90"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b2142a376b40b6736dfc214fd2902409e9e3857eff554fed2d3c60f097e62a62"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a6b5b39cc7e2998f968f05309e666103b53e2edd01df8dc51b90d734c0825444"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4aec24d6b72ee457ec665344a29acb2d35937d5192faebe429ea02633151aad"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:b42f4d86b451c2f9d06ffb4f8bbc776e04df3ba070b9fe2657804b1b40277c48"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6cdaefac66e8b8f30e37a9b4768a391e1f8a16a7526d5bc77a7928408ef68e93"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:b738f7e648735714bbb82bdfd030203360cfeab7f6e8a34772b3c8c8b820568c"}, - {file = "lxml-6.0.2-cp39-cp39-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daf42de090d59db025af61ce6bdb2521f0f102ea0e6ea310f13c17610a97da4c"}, - {file = "lxml-6.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:66328dabea70b5ba7e53d94aa774b733cf66686535f3bc9250a7aab53a91caaf"}, - {file = "lxml-6.0.2-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:e237b807d68a61fc3b1e845407e27e5eb8ef69bc93fe8505337c1acb4ee300b6"}, - {file = "lxml-6.0.2-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:ac02dc29fd397608f8eb15ac1610ae2f2f0154b03f631e6d724d9e2ad4ee2c84"}, - {file = "lxml-6.0.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:817ef43a0c0b4a77bd166dc9a09a555394105ff3374777ad41f453526e37f9cb"}, - {file = "lxml-6.0.2-cp39-cp39-win32.whl", hash = "sha256:bc532422ff26b304cfb62b328826bd995c96154ffd2bac4544f37dbb95ecaa8f"}, - {file = "lxml-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:995e783eb0374c120f528f807443ad5a83a656a8624c467ea73781fc5f8a8304"}, - {file = "lxml-6.0.2-cp39-cp39-win_arm64.whl", hash = "sha256:08b9d5e803c2e4725ae9e8559ee880e5328ed61aa0935244e0515d7d9dbec0aa"}, - {file = "lxml-6.0.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:e748d4cf8fef2526bb2a589a417eba0c8674e29ffcb570ce2ceca44f1e567bf6"}, - {file = "lxml-6.0.2-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4ddb1049fa0579d0cbd00503ad8c58b9ab34d1254c77bc6a5576d96ec7853dba"}, - {file = "lxml-6.0.2-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cb233f9c95f83707dae461b12b720c1af9c28c2d19208e1be03387222151daf5"}, - {file = "lxml-6.0.2-pp310-pypy310_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc456d04db0515ce3320d714a1eac7a97774ff0849e7718b492d957da4631dd4"}, - {file = "lxml-6.0.2-pp310-pypy310_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2613e67de13d619fd283d58bda40bff0ee07739f624ffee8b13b631abf33083d"}, - {file = "lxml-6.0.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:24a8e756c982c001ca8d59e87c80c4d9dcd4d9b44a4cbeb8d9be4482c514d41d"}, - {file = "lxml-6.0.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1c06035eafa8404b5cf475bb37a9f6088b0aca288d4ccc9d69389750d5543700"}, - {file = "lxml-6.0.2-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c7d13103045de1bdd6fe5d61802565f1a3537d70cd3abf596aa0af62761921ee"}, - {file = "lxml-6.0.2-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0a3c150a95fbe5ac91de323aa756219ef9cf7fde5a3f00e2281e30f33fa5fa4f"}, - {file = "lxml-6.0.2-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60fa43be34f78bebb27812ed90f1925ec99560b0fa1decdb7d12b84d857d31e9"}, - {file = "lxml-6.0.2-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21c73b476d3cfe836be731225ec3421fa2f048d84f6df6a8e70433dff1376d5a"}, - {file = "lxml-6.0.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:27220da5be049e936c3aca06f174e8827ca6445a4353a1995584311487fc4e3e"}, - {file = "lxml-6.0.2.tar.gz", hash = "sha256:cd79f3367bd74b317dda655dc8fcfa304d9eb6e4fb06b7168c5cf27f96e0cd62"}, +markers = "extra == \"all\" or extra == \"url-preview\"" +files = [ + {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:41dcc4c7b10484257cbd6c37b83ddb26df2b0e5aff5ac00d095689015af868ec"}, + {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a31286dbb5e74c8e9a5344465b77ab4c5bd511a253b355b5ca2fae7e579fafec"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1bc4cc83fb7f66ffb16f74d6dd0162e144333fc36ebcce32246f80c8735b2551"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:20cf4d0651987c906a2f5cba4e3a8d6ba4bfdf973cfe2a96c0d6053888ea2ecd"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ffb34ea45a82dd637c2c97ae1bbb920850c1e59bcae79ce1c15af531d83e7215"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a1d9b99e5b2597e4f5aed2484fef835256fa1b68a19e4265c97628ef4bf8bcf4"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux_2_28_i686.whl", hash = "sha256:d43aa26dcda363f21e79afa0668f5029ed7394b3bb8c92a6927a3d34e8b610ea"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:6262b87f9e5c1e5fe501d6c153247289af42eb44ad7660b9b3de17baaf92d6f6"}, + {file = "lxml-6.1.0-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d1392c569c032f78a11a25d1de1c43fff13294c793b39e19d84fade3045cbbc3"}, + {file = "lxml-6.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:045e387d1f4f42a418380930fa3f45c73c9b392faf67e495e58902e68e8f44a7"}, + {file = "lxml-6.1.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:9f93d5b8b07f73e8c77e3c6556a3db269918390c804b5e5fcdd4858232cc8f16"}, + {file = "lxml-6.1.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:de550d129f18d8ab819651ffe4f38b1b713c7e116707de3c0c6400d0ef34fbc1"}, + {file = "lxml-6.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c08da09dc003c9e8c70e06b53a11db6fb3b250c21c4236b03c7d7b443c318e7a"}, + {file = "lxml-6.1.0-cp310-cp310-win32.whl", hash = "sha256:37448bf9c7d7adfc5254763901e2bbd6bb876228dfc1fc7f66e58c06368a7544"}, + {file = "lxml-6.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:2593a0a6621545b9095b71ad74ed4226eba438a7d9fc3712a99bdb15508cf93a"}, + {file = "lxml-6.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:e80807d72f96b96ad5588cb85c75616e4f2795a7737d4630784c51497beb7776"}, + {file = "lxml-6.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:cec05be8c876f92a5aa07b01d60bbb4d11cfbdd654cad0561c0d7b5c043a61b9"}, + {file = "lxml-6.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9c03e048b6ce8e77b09c734e931584894ecd58d08296804ca2d0b184c933ce50"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:942454ff253da14218f972b23dc72fa4edf6c943f37edd19cd697618b626fac5"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d036ee7b99d5148072ac7c9b847193decdfeac633db350363f7bce4fff108f0e"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ae5d8d5427f3cc317e7950f2da7ad276df0cfa37b8de2f5658959e618ea8512"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:363e47283bde87051b821826e71dde47f107e08614e1aa312ba0c5711e77738c"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux_2_28_i686.whl", hash = "sha256:f504d861d9f2a8f94020130adac88d66de93841707a23a86244263d1e54682f5"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:23a5dc68e08ed13331d61815c08f260f46b4a60fdd1640bbeb82cf89a9d90289"}, + {file = "lxml-6.1.0-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f15401d8d3dbf239e23c818afc10c7207f7b95f9a307e092122b6f86dd43209a"}, + {file = "lxml-6.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fcf3da95e93349e0647d48d4b36a12783105bcc74cb0c416952f9988410846a3"}, + {file = "lxml-6.1.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:0d082495c5fcf426e425a6e28daaba1fcb6d8f854a4ff01effb1f1f381203eb9"}, + {file = "lxml-6.1.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:e3c4f84b24a1fcba435157d111c4b755099c6ff00a3daee1ad281817de75ed11"}, + {file = "lxml-6.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:976a6b39b1b13e8c354ad8d3f261f3a4ac6609518af91bdb5094760a08f132c4"}, + {file = "lxml-6.1.0-cp311-cp311-win32.whl", hash = "sha256:857efde87d365706590847b916baff69c0bc9252dc5af030e378c9800c0b10e3"}, + {file = "lxml-6.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:183bfb45a493081943be7ea2b5adfc2b611e1cf377cefa8b8a8be404f45ef9a7"}, + {file = "lxml-6.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:19f4164243fc206d12ed3d866e80e74f5bc3627966520da1a5f97e42c32a3f39"}, + {file = "lxml-6.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d2f17a16cd8751e8eb233a7e41aecdf8e511712e00088bf9be455f604cd0d28d"}, + {file = "lxml-6.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f0cea5b1d3e6e77d71bd2b9972eb2446221a69dc52bb0b9c3c6f6e5700592d93"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fc46da94826188ed45cb53bd8e3fc076ae22675aea2087843d4735627f867c6d"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9147d8e386ec3b82c3b15d88927f734f565b0aaadef7def562b853adca45784a"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5715e0e28736a070f3f34a7ccc09e2fdcba0e3060abbcf61a1a5718ff6d6b105"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4937460dc5df0cdd2f06a86c285c28afda06aefa3af949f9477d3e8df430c485"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc783ee3147e60a25aa0445ea82b3e8aabb83b240f2b95d32cb75587ff781814"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:40d9189f80075f2e1f88db21ef815a2b17b28adf8e50aaf5c789bfe737027f32"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:05b9b8787e35bec69e68daf4952b2e6dfcfb0db7ecf1a06f8cdfbbac4eb71aad"}, + {file = "lxml-6.1.0-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0f0f08beb0182e3e9a86fae124b3c47a7b41b7b69b225e1377db983802404e54"}, + {file = "lxml-6.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73becf6d8c81d4c76b1014dbd3584cb26d904492dcf73ca85dc8bff08dcd6d2d"}, + {file = "lxml-6.1.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1ae225f66e5938f4fa29d37e009a3bb3b13032ac57eb4eb42afa44f6e4054e69"}, + {file = "lxml-6.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:690022c7fae793b0489aa68a658822cea83e0d5933781811cabbf5ea3bcfe73d"}, + {file = "lxml-6.1.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:63aeafc26aac0be8aff14af7871249e87ea1319be92090bfd632ec68e03b16a5"}, + {file = "lxml-6.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:264c605ab9c0e4aa1a679636f4582c4d3313700009fac3ec9c3412ed0d8f3e1d"}, + {file = "lxml-6.1.0-cp312-cp312-win32.whl", hash = "sha256:56971379bc5ee8037c5a0f09fa88f66cdb7d37c3e38af3e45cf539f41131ac1f"}, + {file = "lxml-6.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:bba078de0031c219e5dd06cf3e6bf8fb8e6e64a77819b358f53bb132e3e03366"}, + {file = "lxml-6.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:c3592631e652afa34999a088f98ba7dfc7d6aff0d535c410bea77a71743f3819"}, + {file = "lxml-6.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a0092f2b107b69601adf562a57c956fbb596e05e3e6651cabd3054113b007e45"}, + {file = "lxml-6.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fc7140d7a7386e6b545d41b7358f4d02b656d4053f5fa6859f92f4b9c2572c4d"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:419c58fc92cc3a2c3fa5f78c63dbf5da70c1fa9c1b25f25727ecee89a96c7de2"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:37fabd1452852636cf38ecdcc9dd5ca4bba7a35d6c53fa09725deeb894a87491"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2853c8b2170cc6cd54a6b4d50d2c1a8a7aeca201f23804b4898525c7a152cfc"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8e369cbd690e788c8d15e56222d91a09c6a417f49cbc543040cba0fe2e25a79e"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e69aa6805905807186eb00e66c6d97a935c928275182eb02ee40ba00da9623b2"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:4bd1bdb8a9e0e2dd229de19b5f8aebac80e916921b4b2c6ef8a52bc131d0c1f9"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:cbd7b79cdcb4986ad78a2662625882747f09db5e4cd7b2ae178a88c9c51b3dfe"}, + {file = "lxml-6.1.0-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:43e4d297f11080ec9d64a4b1ad7ac02b4484c9f0e2179d9c4ef78e886e747b88"}, + {file = "lxml-6.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cc16682cc987a3da00aa56a3aa3075b08edb10d9b1e476938cfdbee8f3b67181"}, + {file = "lxml-6.1.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:d6d8efe71429635f0559579092bb5e60560d7b9115ee38c4adbea35632e7fa24"}, + {file = "lxml-6.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7e39ab3a28af7784e206d8606ec0e4bcad0190f63a492bca95e94e5a4aef7f6e"}, + {file = "lxml-6.1.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:9eb667bf50856c4a58145f8ca2d5e5be160191e79eb9e30855a476191b3c3495"}, + {file = "lxml-6.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7f4a77d6f7edf9230cee3e1f7f6764722a41604ee5681844f18db9a81ea0ec33"}, + {file = "lxml-6.1.0-cp313-cp313-win32.whl", hash = "sha256:28902146ffbe5222df411c5d19e5352490122e14447e98cd118907ee3fd6ee62"}, + {file = "lxml-6.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:4a1503c56e4e2b38dc76f2f2da7bae69670c0f1933e27cfa34b2fa5876410b16"}, + {file = "lxml-6.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:e0af85773850417d994d019741239b901b22c6680206f46a34766926e466141d"}, + {file = "lxml-6.1.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:ab863fd37458fed6456525f297d21239d987800c46e67da5ef04fc6b3dd93ac8"}, + {file = "lxml-6.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6fd8b1df8254ff4fd93fd31da1fc15770bde23ac045be9bb1f87425702f61cc9"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:47024feaae386a92a146af0d2aeed65229bf6fff738e6a11dda6b0015fb8fd03"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3f00972f84450204cd5d93a5395965e348956aaceaadec693a22ec743f8ae3eb"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97faa0860e13b05b15a51fb4986421ef7a30f0b3334061c416e0981e9450ca4c"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:972a6451204798675407beaad97b868d0c733d9a74dafefc63120b81b8c2de28"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fe022f20bc4569ec66b63b3fb275a3d628d9d32da6326b2982584104db6d3086"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:75c4c7c619a744f972f4451bf5adf6d0fb00992a1ffc9fd78e13b0bc817cc99f"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:3648f20d25102a22b6061c688beb3a805099ea4beb0a01ce62975d926944d292"}, + {file = "lxml-6.1.0-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:77b9f99b17cbf14026d1e618035077060fc7195dd940d025149f3e2e830fbfcb"}, + {file = "lxml-6.1.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:32662519149fd7a9db354175aa5e417d83485a8039b8aaa62f873ceee7ea4cad"}, + {file = "lxml-6.1.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:73d658216fc173cf2c939e90e07b941c5e12736b0bf6a99e7af95459cfe8eabb"}, + {file = "lxml-6.1.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ac4db068889f8772a4a698c5980ec302771bb545e10c4b095d4c8be26749616f"}, + {file = "lxml-6.1.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:45e9dfbd1b661eb64ba0d4dbe762bd210c42d86dd1e5bd2bdf89d634231beb43"}, + {file = "lxml-6.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:89e8d73d09ac696a5ba42ec69787913d53284f12092f651506779314f10ba585"}, + {file = "lxml-6.1.0-cp314-cp314-win32.whl", hash = "sha256:ebe33f4ec1b2de38ceb225a1749a2965855bffeef435ba93cd2d5d540783bf2f"}, + {file = "lxml-6.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:398443df51c538bd578529aa7e5f7afc6c292644174b47961f3bf87fe5741120"}, + {file = "lxml-6.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:8c8984e1d8c4b3949e419158fda14d921ff703a9ed8a47236c6eb7a2b6cb4946"}, + {file = "lxml-6.1.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:1081dd10bc6fa437db2500e13993abf7cc30716d0a2f40e65abb935f02ec559c"}, + {file = "lxml-6.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:dabecc48db5f42ba348d1f5d5afdc54c6c4cc758e676926c7cd327045749517d"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e3dd5fe19c9e0ac818a9c7f132a5e43c1339ec1cbbfecb1a938bd3a47875b7c9"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9e7b0a4ca6dcc007a4cef00a761bba2dea959de4bd2df98f926b33c92ca5dfb9"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d27bbe326c6b539c64b42638b18bc6003a8d88f76213a97ac9ed4f885efeab7"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4e425db0c5445ef0ad56b0eec54f89b88b2d884656e536a90b2f52aecb4ca86"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b89b098105b8599dc57adac95d1813409ac476d3c948a498775d3d0c6124bfb"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:c4a699432846df86cc3de502ee85f445ebad748a1c6021d445f3e514d2cd4b1c"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:30e7b2ed63b6c8e97cca8af048589a788ab5c9c905f36d9cf1c2bb549f450d2f"}, + {file = "lxml-6.1.0-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:022981127642fe19866d2907d76241bb07ed21749601f727d5d5dd1ce5d1b773"}, + {file = "lxml-6.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:23cad0cc86046d4222f7f418910e46b89971c5a45d3c8abfad0f64b7b05e4a9b"}, + {file = "lxml-6.1.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:21c3302068f50d1e8728c67c87ba92aa87043abee517aa2576cca1855326b405"}, + {file = "lxml-6.1.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:be10838781cb3be19251e276910cd508fe127e27c3242e50521521a0f3781690"}, + {file = "lxml-6.1.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:2173a7bffe97667bbf0767f8a99e587740a8c56fdf3befac4b09cb29a80276fd"}, + {file = "lxml-6.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c6854e9cf99c84beb004eecd7d3a3868ef1109bf2b1df92d7bc11e96a36c2180"}, + {file = "lxml-6.1.0-cp314-cp314t-win32.whl", hash = "sha256:00750d63ef0031a05331b9223463b1c7c02b9004cef2346a5b2877f0f9494dd2"}, + {file = "lxml-6.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:80410c3a7e3c617af04de17caa9f9f20adaa817093293d69eae7d7d0522836f5"}, + {file = "lxml-6.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:26dd9f57ee3bd41e7d35b4c98a2ffd89ed11591649f421f0ec19f67d50ec67ac"}, + {file = "lxml-6.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b6c2f225662bc5ad416bdd06f72ca301b31b39ce4261f0e0097017fc2891b940"}, + {file = "lxml-6.1.0-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a86f06f059e22a0d574990ee2df24ede03f7f3c68c1336293eee9536c4c776cd"}, + {file = "lxml-6.1.0-cp38-cp38-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:468479e52ecf3ec23799c863336d02c05fc2f7ffd1a1424eeeb9a28d4eb69d13"}, + {file = "lxml-6.1.0-cp38-cp38-manylinux_2_28_i686.whl", hash = "sha256:a02ca8fe48815bddcfca3248efe54451abb9dbf2f7d1c5744c8aa4142d476919"}, + {file = "lxml-6.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:bb40648d96157f9081886defe13eac99253e663be969ff938a9289eff6e47b72"}, + {file = "lxml-6.1.0-cp38-cp38-win32.whl", hash = "sha256:1dd6a1c3ad4cb674f44525d9957f3e9c209bb6dd9213245195167a281fcc2bdc"}, + {file = "lxml-6.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:4e2c54d6b47361d0f1d3bc8d4e082ad87201e56ccdcca4d3b9ee3644ff595ec8"}, + {file = "lxml-6.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:920354904d1cb86577d4b3cfe2830c2dbe81d6f4449e57ada428f1609b5985f7"}, + {file = "lxml-6.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c871299c595ee004d186f61840f0bfc4941aa3f17c8ba4a565ead7e4f4f820ee"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d0d799ff958655781296ec870d5e2448e75150da2b3d07f13ff5b0c2c35beefd"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ba11752e346bd804ea312ec2eea2532dfa8b8d3261d81a32ef9e6ab16256280"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26c5272c6a4bf4cf32d3f5a7890c942b0e04438691157d341616d02cca74d4bd"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c53fa3a5a52122d590e847a57ccf955557b9634a7f99ff5a35131321b0a85317"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux_2_28_i686.whl", hash = "sha256:76b958b4ea3104483c20f74866d55aa056546e15ebe83dd7aecd63698f43b755"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:8c11b984b5ce6add4dccc7144c7be5d364d298f15b0c6a57da1991baedc750ce"}, + {file = "lxml-6.1.0-cp39-cp39-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d3829a6e6fd550a219564912d4002c537f65da4c6ae4e093cc34462f4fa027ad"}, + {file = "lxml-6.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:52b0ac6903cf74ebf997eb8c682d2fbac7d1ab7e4c552413eec55868a9b73f39"}, + {file = "lxml-6.1.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:29f5c00cb7d752bce2c70ebd2d31b0a42f9499ffdd3ecb2f31a5b73ee43031ad"}, + {file = "lxml-6.1.0-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:c748ebcb6877de89f48ab90ca96642ac458fff5dec291a2b9337cd4d0934e383"}, + {file = "lxml-6.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:08950a23f296b3f83521577274e3d3b0f3d739bf2e68d01a752e4288bc50d286"}, + {file = "lxml-6.1.0-cp39-cp39-win32.whl", hash = "sha256:11a873c77a181b4fef9c2e357d08ed399542c2af1390101da66720a19c7c9618"}, + {file = "lxml-6.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:81ff55c70b67d19d52b6fd118a114c0a4c97d799cd3089ff9bd9e2ff4b414ee2"}, + {file = "lxml-6.1.0-cp39-cp39-win_arm64.whl", hash = "sha256:481d6e2104285d9add34f41b42b247b76b61c5b5c26c303c2e9707bbf8bd9a64"}, + {file = "lxml-6.1.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:546b66c0dd1bb8d9fa89d7123e5fa19a8aff3a1f2141eb22df96112afb17b842"}, + {file = "lxml-6.1.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5cfa1a34df366d9dc0d5eaf420f4cf2bb1e1bebe1066d1c2fc28c179f8a4004c"}, + {file = "lxml-6.1.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db88156fcf544cdbf0d95588051515cfdfd4c876fc66444eb98bceb5d6db76de"}, + {file = "lxml-6.1.0-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:07f98f5496f96bf724b1e3c933c107f0cbf2745db18c03d2e13a291c3afd2635"}, + {file = "lxml-6.1.0-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4642e04449a1e164b5ff71ffd901ddb772dfabf5c9adf1b7be5dffe1212bc037"}, + {file = "lxml-6.1.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:7da13bb6fbadfafb474e0226a30570a3445cfd47c86296f2446dafbd77079ace"}, + {file = "lxml-6.1.0.tar.gz", hash = "sha256:bfd57d8008c4965709a919c3e9a98f76c2c7cb319086b3d26858250620023b13"}, ] [package.extras] @@ -1553,7 +1547,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "matrix_synapse_ldap3-0.4.0-py3-none-any.whl", hash = "sha256:bf080037230d2af5fd3639cb87266de65c1cad7a68ea206278c5b4bf9c1a17f3"}, {file = "matrix_synapse_ldap3-0.4.0.tar.gz", hash = "sha256:cff52ba780170de5e6e8af42863d2648ee23f3bf0a9fea6db52372f9fc00be2b"}, @@ -1834,7 +1828,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -2032,7 +2026,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"postgres\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"postgres\"" files = [ {file = "psycopg2-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:103e857f46bb76908768ead4e2d0ba1d1a130e7b8ed77d3ae91e8b33481813e8"}, {file = "psycopg2-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:210daed32e18f35e3140a1ebe059ac29209dd96468f2f7559aa59f75ee82a5cb"}, @@ -2050,7 +2044,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -2066,7 +2060,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2348,7 +2342,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"cache-memory\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"cache-memory\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2480,7 +2474,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2505,7 +2499,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -2533,7 +2527,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pytz-2026.1.post1-py2.py3-none-any.whl", hash = "sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a"}, {file = "pytz-2026.1.post1.tar.gz", hash = "sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1"}, @@ -2938,7 +2932,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"sentry\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"sentry\"" files = [ {file = "sentry_sdk-2.57.0-py2.py3-none-any.whl", hash = "sha256:812c8bf5ff3d2f0e89c82f5ce80ab3a6423e102729c4706af7413fd1eb480585"}, {file = "sentry_sdk-2.57.0.tar.gz", hash = "sha256:4be8d1e71c32fb27f79c577a337ac8912137bba4bcbc64a4ec1da4d6d8dc5199"}, @@ -3138,7 +3132,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -3154,7 +3148,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "thrift-0.22.0.tar.gz", hash = "sha256:42e8276afbd5f54fe1d364858b6877bc5e5a4a5ed69f6a005b94ca4918fe1466"}, ] @@ -3220,7 +3214,6 @@ files = [ {file = "tomli-2.4.0-py3-none-any.whl", hash = "sha256:1f776e7d669ebceb01dee46484485f43a4048746235e683bcdffacdf1fb4785a"}, {file = "tomli-2.4.0.tar.gz", hash = "sha256:aa89c3f6c277dd275d8e243ad24f3b5e701491a860d5121f2cdd399fbb31fc9c"}, ] -markers = {main = "python_version < \"3.14\""} [[package]] name = "tornado" @@ -3229,7 +3222,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:487dc9cc380e29f58c7ab88f9e27cdeef04b2140862e5076a66fb6bb68bb1bfa"}, {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:65a7f1d46d4bb41df1ac99f5fcb685fb25c7e61613742d5108b010975a9a6521"}, @@ -3361,7 +3354,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3622,7 +3615,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "xmlschema-2.5.1-py3-none-any.whl", hash = "sha256:ec2b2a15c8896c1fcd14dcee34ca30032b99456c3c43ce793fdb9dca2fb4b869"}, {file = "xmlschema-2.5.1.tar.gz", hash = "sha256:4f7497de6c8b6dc2c28ad7b9ed6e21d186f4afe248a5bea4f54eedab4da44083"}, @@ -3643,7 +3636,7 @@ description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" files = [ {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, From 4655b435ee6e83518cd9ea6941d403be1cdf3a63 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 20 May 2026 16:58:39 +0100 Subject: [PATCH 51/92] Follow #19468 (Fix sending heroes in Sliding Sync) with small tweaks (#19791) Follows: #19468 The main change is from this comment https://github.com/element-hq/synapse/pull/19468#discussion_r2810364196 I am pretty sure it's safe and was tempted to add it to that PR, but for easier bisection and reversion in case it goes wrong, thought a separate commit would be the best. The other drive-by change is a boolean logic simplification Simplify condition (boolean equivalence) Don't fetch name state from `meta_room_state` since it's no longer used there --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19791.bugfix | 1 + synapse/handlers/sliding_sync/__init__.py | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) create mode 100644 changelog.d/19791.bugfix diff --git a/changelog.d/19791.bugfix b/changelog.d/19791.bugfix new file mode 100644 index 00000000000..003716d296e --- /dev/null +++ b/changelog.d/19791.bugfix @@ -0,0 +1 @@ +Fix a bug in [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) that could prevent user avatars from showing if the room had an empty name. diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index df1270f7f97..4d6287b147c 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -966,7 +966,7 @@ async def get_room_sync_data( # care to treat an empty string as unset). But we only need to get them # on initial syncs (or the first time we send down the room) or if the # membership has changed which may change the heroes. - if not room_name and (initial or (not initial and membership_changed)): + if not room_name and (initial or membership_changed): # We need the room summary to extract the heroes from if room_membership_for_user_at_to_token.membership != Membership.JOIN: # TODO: Figure out how to get the membership summary for left/banned rooms @@ -1185,8 +1185,6 @@ async def get_room_sync_data( (EventTypes.Member, hero_user_id) for hero_user_id in hero_user_ids ] meta_room_state = list(hero_room_state) - if initial or name_changed: - meta_room_state.append((EventTypes.Name, "")) if initial or avatar_changed: meta_room_state.append((EventTypes.RoomAvatar, "")) From 0e39c0c8f684d471d53b2a8e459e997f51d114a9 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 26 May 2026 22:12:20 +0300 Subject: [PATCH 52/92] Fix policy server signature merging again (#19797) Fixes #19796 --- changelog.d/19797.bugfix | 1 + synapse/handlers/room_policy.py | 8 +-- tests/handlers/test_room_policy.py | 84 +++++++++++++++++++++++++++++- 3 files changed, 88 insertions(+), 5 deletions(-) create mode 100644 changelog.d/19797.bugfix diff --git a/changelog.d/19797.bugfix b/changelog.d/19797.bugfix new file mode 100644 index 00000000000..4d769cb1a41 --- /dev/null +++ b/changelog.d/19797.bugfix @@ -0,0 +1 @@ +Fix merging signatures when a policy server is running under the same server name as Synapse. The bug was re-introduced in v1.153.0rc1 after being fixed earlier in v1.151.0rc1. Contributed by @tulir @ Beeper. diff --git a/synapse/handlers/room_policy.py b/synapse/handlers/room_policy.py index e46e6dc2efc..d19815d6e21 100644 --- a/synapse/handlers/room_policy.py +++ b/synapse/handlers/room_policy.py @@ -251,8 +251,8 @@ async def ask_policy_server_to_sign_event( # Note: if the policy server and event sender are the same server, the sender # might not have added policy server signatures to the event for whatever reason. # When this happens, we don't want to obliterate the event's existing signatures - # because the event will fail authorization. This is why we add defaults rather - # than simply `update` the signatures on the event. + # because the event will fail authorization. This is why we add items individually + # rather than simply `update` the signatures on the event. # # This situation can happen if the homeserver and policy server parts are # logically the same server, but run by different software. For example, Synapse @@ -261,7 +261,9 @@ async def ask_policy_server_to_sign_event( # servers need to manually fetch signatures for. This is the code that allows # those events to continue working (because they're legally sent, even if missing # the policy server signature). - event.signatures.update(signature) + signatures = signature.get(policy_server.server_name, {}) + for key_id, sig in signatures.items(): + event.signatures.add_signature(policy_server.server_name, key_id, sig) except HttpResponseException as ex: # re-wrap HTTP errors as `SynapseError` so they can be proxied to clients directly raise ex.to_synapse_error() from ex diff --git a/tests/handlers/test_room_policy.py b/tests/handlers/test_room_policy.py index 2d3e917aef0..4f2188b8e71 100644 --- a/tests/handlers/test_room_policy.py +++ b/tests/handlers/test_room_policy.py @@ -27,6 +27,7 @@ from synapse.rest import admin from synapse.rest.client import filter, login, room, sync from synapse.server import HomeServer +from synapse.synapse_rust.events import Signatures from synapse.types import JsonDict, UserID from synapse.util.clock import Clock @@ -113,7 +114,15 @@ async def policy_server_signs_event( self.OTHER_SERVER_NAME, self.signing_key, ) - return sigs + # Only return the new signature like the policy server spec says, + # not any others that were already in the event + return { + self.OTHER_SERVER_NAME: { + POLICY_SERVER_KEY_ID: sigs[self.OTHER_SERVER_NAME][ + POLICY_SERVER_KEY_ID + ] + } + } async def policy_server_signs_event_with_wrong_key( destination: str, pdu: EventBase, timeout: int | None = None @@ -169,6 +178,19 @@ def _add_policy_server_to_room(self, public_key: str | None = None) -> None: state_key="", ) + def _sign_with_random_key(self, server_name: str, event: EventBase) -> None: + non_policyserver_key = signedjson.key.generate_signing_key( + "non_policyserver_key" + ) + event.signatures = Signatures( + compute_event_signature( + event.room_version, + event.get_dict(), + server_name, + non_policyserver_key, + ) + ) + def test_no_policy_event_set(self) -> None: # We don't need to modify the room state at all - we're testing the default # case where a room doesn't use a policy server. @@ -316,11 +338,69 @@ def test_ask_policy_server_to_sign_event_ok(self) -> None: }, }, ) + # Sign the event as the origin server first, since that's what events passed to + # ask_policy_server_to_sign_event will generally look like. The exact key used + # here isn't important. + self._sign_with_random_key("example.org", event) self.mock_federation_transport_client.ask_policy_server_to_sign_event.side_effect = self.policy_server_signs_event self.get_success( self.handler.ask_policy_server_to_sign_event(event, verify=True) ) - self.assertEqual(len(event.signatures), 1) + # Standard success case: event has signatures from the origin and the policy server + self.assertEqual( + { + server: len(signatures) + for server, signatures in event.signatures.as_dict().items() + }, + {"example.org": 1, self.OTHER_SERVER_NAME: 1}, + f"Expected signatures for the origin homeserver (example.org) and policy server ({self.OTHER_SERVER_NAME})", + ) + + def test_ask_origin_server_to_sign_event_doesnt_replace_signatures(self) -> None: + """ + ``ask_policy_server_to_sign_event`` has had bugs where it accidentally overwrote + the origin server's signature in the case where the origin server has the same + server name as the policy server (each have their own signing key). This test is + otherwise equivalent to the success case test above, but the server name for + origin event sending server and the policy server are the same and we want to + ensure both signatures are preserved. + """ + verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) + self._add_policy_server_to_room(public_key=verify_key_str) + event = make_test_event( + room_version=self.room_version, + internal_metadata_dict={}, + event_dict={ + "room_id": self.room_id, + "type": "m.room.message", + "sender": "@spammy:" + self.OTHER_SERVER_NAME, + "content": { + "msgtype": "m.text", + "body": "This is another signed event.", + }, + }, + ) + # Sign the event as the origin server that sent the event, which in this case + # has the same server name as the policy server. We're using a different key + # than `self.signing_key` (for the policy server), as the ed25519:policy_server + # key is only used for policy server signatures, not any other federation traffic + # even when the origin server and policy are logically the same server. + self._sign_with_random_key(self.OTHER_SERVER_NAME, event) + self.mock_federation_transport_client.ask_policy_server_to_sign_event.side_effect = self.policy_server_signs_event + self.get_success( + self.handler.ask_policy_server_to_sign_event(event, verify=True) + ) + # Less common success case: the event origin server is logically the same as + # the policy server, so there will be two signatures from one server name. + # It's important to make sure both signatures are preserved. + self.assertEqual( + { + server: len(signatures) + for server, signatures in event.signatures.as_dict().items() + }, + {self.OTHER_SERVER_NAME: 2}, + f"Expected 2 signatures for the origin server and policy server under the same server name ({self.OTHER_SERVER_NAME}) but with different keys", + ) def test_ask_policy_server_to_sign_event_refuses(self) -> None: verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) From 82d66fa0f189a3857ec059ec9e5da6b3ffdcc874 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 27 May 2026 12:27:43 +0100 Subject: [PATCH 53/92] 1.154.0rc1 --- CHANGES.md | 37 +++++++++++++++++++++++++++++++ changelog.d/19468.bugfix | 1 - changelog.d/19483.bugfix | 1 - changelog.d/19508.doc | 1 - changelog.d/19634.bugfix | 1 - changelog.d/19644.bugfix | 1 - changelog.d/19715.feature | 3 --- changelog.d/19725.misc | 1 - changelog.d/19748.misc | 1 - changelog.d/19764.misc | 1 - changelog.d/19765.misc | 1 - changelog.d/19766.misc | 1 - changelog.d/19772.misc | 1 - changelog.d/19774.misc | 1 - changelog.d/19777.misc | 1 - changelog.d/19781.misc | 1 - changelog.d/19784.bugfix | 1 - changelog.d/19787.bugfix | 1 - changelog.d/19789.bugfix | 1 - changelog.d/19791.bugfix | 1 - changelog.d/19797.bugfix | 1 - debian/changelog | 6 +++++ pyproject.toml | 2 +- schema/synapse-config.schema.yaml | 2 +- 24 files changed, 45 insertions(+), 24 deletions(-) delete mode 100644 changelog.d/19468.bugfix delete mode 100644 changelog.d/19483.bugfix delete mode 100644 changelog.d/19508.doc delete mode 100644 changelog.d/19634.bugfix delete mode 100644 changelog.d/19644.bugfix delete mode 100644 changelog.d/19715.feature delete mode 100644 changelog.d/19725.misc delete mode 100644 changelog.d/19748.misc delete mode 100644 changelog.d/19764.misc delete mode 100644 changelog.d/19765.misc delete mode 100644 changelog.d/19766.misc delete mode 100644 changelog.d/19772.misc delete mode 100644 changelog.d/19774.misc delete mode 100644 changelog.d/19777.misc delete mode 100644 changelog.d/19781.misc delete mode 100644 changelog.d/19784.bugfix delete mode 100644 changelog.d/19787.bugfix delete mode 100644 changelog.d/19789.bugfix delete mode 100644 changelog.d/19791.bugfix delete mode 100644 changelog.d/19797.bugfix diff --git a/CHANGES.md b/CHANGES.md index 95ab39d0e3d..9ad7dfbdad6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,40 @@ +# Synapse 1.154.0rc1 (2026-05-27) + +## Features + +- Add support for [MSC4452: Preview URL capabilities API](https://github.com/matrix-org/matrix-spec-proposals/pull/4452) which exposes a `io.element.msc4452.preview_url` capability. + If `experimental_features.msc4452_enabled` is `true`, the `/_matrix/(client/v1/media|media/v3)/preview_url` endpoint + now responds with a 403 status code when the capability is disabled. ([\#19715](https://github.com/element-hq/synapse/issues/19715)) + +## Bugfixes + +- Fix a bug in [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) that could prevent user avatars from showing if the room had an empty name. ([\#19468](https://github.com/element-hq/synapse/issues/19468), [\#19791](https://github.com/element-hq/synapse/issues/19791)) +- Fix access token cache not being invalidated for sessions using refresh tokens. Contributed by @FrenchGithubUser @ Famedly. ([\#19483](https://github.com/element-hq/synapse/issues/19483)) +- Fix bug where Synapse would return 400 (`M_BAD_JSON`) when sending a message with `mentions` field and Synapse module `check_event_allowed` callback registered (frozen event). Contributed by @gaetan-sbt. ([\#19634](https://github.com/element-hq/synapse/issues/19634)) +- Fix long-standing but niche bug with sync where it could attempt to fetch data with flawed invalid future tokens. ([\#19644](https://github.com/element-hq/synapse/issues/19644)) +- Fix `/sync` failing when [MSC4354 Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354) are enabled and the sync request filters out Ephemeral Data Units (EDUs). ([\#19787](https://github.com/element-hq/synapse/issues/19787)) +- Fix packaging for Fedora and EPEL caused by unnecessary bumping `attrs` minimum version requirement in `pyproject.toml` file. Contributed by Oleg Girko. ([\#19789](https://github.com/element-hq/synapse/issues/19789)) +- Fix merging signatures when a policy server is running under the same server name as Synapse. The bug was re-introduced in v1.153.0rc1 after being fixed earlier in v1.151.0rc1. Contributed by @tulir @ Beeper. ([\#19797](https://github.com/element-hq/synapse/issues/19797)) + +## Improved Documentation + +- Added details about how Synapse syncs the picture claim when `update_profile_information` setting is true. ([\#19508](https://github.com/element-hq/synapse/issues/19508)) + +## Internal Changes + +- Port `Event.content` field to Rust. ([\#19725](https://github.com/element-hq/synapse/issues/19725)) +- Prefer close backfill points (absolute distance). ([\#19748](https://github.com/element-hq/synapse/issues/19748)) +- Replace unique `quarantined_media` waiting patterns with standard `wait_for_stream_token(...)`. ([\#19764](https://github.com/element-hq/synapse/issues/19764)) +- Improve Synapse logging around when someone encounters `We can't get valid state history.` so you can correlate everything by `event_id`. ([\#19765](https://github.com/element-hq/synapse/issues/19765)) +- Tidy up Rust `RoomVersion` structs. ([\#19766](https://github.com/element-hq/synapse/issues/19766)) +- Update `WorkerLock` tests to better stress the `WORKER_LOCK_MAX_RETRY_INTERVAL`. ([\#19772](https://github.com/element-hq/synapse/issues/19772)) +- Refactor [MSC4242: State DAG](https://github.com/matrix-org/matrix-spec-proposals/pull/4242) checks behind a single `TypeIs` helper to avoid scattered `isinstance` casts. ([\#19774](https://github.com/element-hq/synapse/issues/19774)) +- Use `StrCollection` for `prev_state_events`. ([\#19777](https://github.com/element-hq/synapse/issues/19777)) +- Fix up the construction of events in tests, ahead of the Rust event port. ([\#19781](https://github.com/element-hq/synapse/issues/19781)) + + + + # Synapse 1.153.0 (2026-05-19) No significant changes since 1.153.0rc3. diff --git a/changelog.d/19468.bugfix b/changelog.d/19468.bugfix deleted file mode 100644 index 003716d296e..00000000000 --- a/changelog.d/19468.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug in [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) that could prevent user avatars from showing if the room had an empty name. diff --git a/changelog.d/19483.bugfix b/changelog.d/19483.bugfix deleted file mode 100644 index 9e4fb209965..00000000000 --- a/changelog.d/19483.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix access token cache not being invalidated for sessions using refresh tokens. Contributed by @FrenchGithubUser @ Famedly. diff --git a/changelog.d/19508.doc b/changelog.d/19508.doc deleted file mode 100644 index 25501163414..00000000000 --- a/changelog.d/19508.doc +++ /dev/null @@ -1 +0,0 @@ -Added details about how Synapse syncs the picture claim when `update_profile_information` setting is true. diff --git a/changelog.d/19634.bugfix b/changelog.d/19634.bugfix deleted file mode 100644 index e8fcb435701..00000000000 --- a/changelog.d/19634.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where Synapse would return 400 (`M_BAD_JSON`) when sending a message with `mentions` field and Synapse module `check_event_allowed` callback registered (frozen event). Contributed by @gaetan-sbt. \ No newline at end of file diff --git a/changelog.d/19644.bugfix b/changelog.d/19644.bugfix deleted file mode 100644 index 73ab4bc63e9..00000000000 --- a/changelog.d/19644.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix long-standing but niche bug with sync where it could attempt to fetch data with flawed invalid future tokens. diff --git a/changelog.d/19715.feature b/changelog.d/19715.feature deleted file mode 100644 index 973fe66e7d0..00000000000 --- a/changelog.d/19715.feature +++ /dev/null @@ -1,3 +0,0 @@ -Add support for "MSC4452 Preview URL capabilities API" which exposes a `io.element.msc4452.preview_url` capability. -If `experimental_features.msc4452_enabled` is `true`, the `/_matrix/(client/v1/media|media/v3)/preview_url` endpoint -now responds with a 403 status code when the capability is disabled. diff --git a/changelog.d/19725.misc b/changelog.d/19725.misc deleted file mode 100644 index b320f42b9c5..00000000000 --- a/changelog.d/19725.misc +++ /dev/null @@ -1 +0,0 @@ -Port `Event.content` field to Rust. diff --git a/changelog.d/19748.misc b/changelog.d/19748.misc deleted file mode 100644 index eedd4e92a26..00000000000 --- a/changelog.d/19748.misc +++ /dev/null @@ -1 +0,0 @@ -Prefer close backfill points (absolute distance). diff --git a/changelog.d/19764.misc b/changelog.d/19764.misc deleted file mode 100644 index 8704e3eed64..00000000000 --- a/changelog.d/19764.misc +++ /dev/null @@ -1 +0,0 @@ -Replace unique `quarantined_media` waiting patterns with standard `wait_for_stream_token(...)`. diff --git a/changelog.d/19765.misc b/changelog.d/19765.misc deleted file mode 100644 index 7ac5375c7eb..00000000000 --- a/changelog.d/19765.misc +++ /dev/null @@ -1 +0,0 @@ -Improve Synapse logging around when someone encounters `We can't get valid state history.` so you can correlate everything by `event_id`. diff --git a/changelog.d/19766.misc b/changelog.d/19766.misc deleted file mode 100644 index 699852daa21..00000000000 --- a/changelog.d/19766.misc +++ /dev/null @@ -1 +0,0 @@ -Tidy up Rust `RoomVersion` structs. diff --git a/changelog.d/19772.misc b/changelog.d/19772.misc deleted file mode 100644 index 939507f5c35..00000000000 --- a/changelog.d/19772.misc +++ /dev/null @@ -1 +0,0 @@ -Update `WorkerLock` tests to better stress the `WORKER_LOCK_MAX_RETRY_INTERVAL`. diff --git a/changelog.d/19774.misc b/changelog.d/19774.misc deleted file mode 100644 index 5a2cb4d8001..00000000000 --- a/changelog.d/19774.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor MSC4242 state DAG checks behind a single `TypeIs` helper to avoid scattered `isinstance` casts. diff --git a/changelog.d/19777.misc b/changelog.d/19777.misc deleted file mode 100644 index cd049f8153d..00000000000 --- a/changelog.d/19777.misc +++ /dev/null @@ -1 +0,0 @@ -Use `StrCollection` for `prev_state_events`. diff --git a/changelog.d/19781.misc b/changelog.d/19781.misc deleted file mode 100644 index 63c2482dedc..00000000000 --- a/changelog.d/19781.misc +++ /dev/null @@ -1 +0,0 @@ -Fix up event-construction in tests ahead of the Rust event port. diff --git a/changelog.d/19784.bugfix b/changelog.d/19784.bugfix deleted file mode 100644 index c68524d57a9..00000000000 --- a/changelog.d/19784.bugfix +++ /dev/null @@ -1 +0,0 @@ -Revert 'Have [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) return a new response immediately if a room subscription has changed and produced a new response. ([\#19714](https://github.com/element-hq/synapse/issues/19714))' due to performance problems. diff --git a/changelog.d/19787.bugfix b/changelog.d/19787.bugfix deleted file mode 100644 index 26bd3e2252d..00000000000 --- a/changelog.d/19787.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix `/sync` failing when [MSC4354 Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354) are enabled and the sync request filters out Ephemeral Data Units (EDUs). \ No newline at end of file diff --git a/changelog.d/19789.bugfix b/changelog.d/19789.bugfix deleted file mode 100644 index f6c325ec824..00000000000 --- a/changelog.d/19789.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix packaging for Fedora and EPEL caused by unnecessary bumping `attrs` minimum version requirement in `pyproject.toml` file. Contributed by Oleg Girko. diff --git a/changelog.d/19791.bugfix b/changelog.d/19791.bugfix deleted file mode 100644 index 003716d296e..00000000000 --- a/changelog.d/19791.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug in [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) that could prevent user avatars from showing if the room had an empty name. diff --git a/changelog.d/19797.bugfix b/changelog.d/19797.bugfix deleted file mode 100644 index 4d769cb1a41..00000000000 --- a/changelog.d/19797.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix merging signatures when a policy server is running under the same server name as Synapse. The bug was re-introduced in v1.153.0rc1 after being fixed earlier in v1.151.0rc1. Contributed by @tulir @ Beeper. diff --git a/debian/changelog b/debian/changelog index b1a4e04bc38..4c3999413b6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.154.0~rc1) stable; urgency=medium + + * New Synapse release 1.154.0rc1. + + -- Synapse Packaging team Wed, 27 May 2026 12:23:54 +0100 + matrix-synapse-py3 (1.153.0) stable; urgency=medium * New Synapse release 1.153.0. diff --git a/pyproject.toml b/pyproject.toml index e92e7b5c21f..76ccbbe946a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.153.0" +version = "1.154.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 8b8d57b9bf1..dc57cfeea50 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/schema/synapse/v1.153/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.154/synapse-config.schema.json type: object properties: modules: From f9f08db4c3dab5bec742ea2476ed815806c584c8 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 27 May 2026 14:27:37 +0100 Subject: [PATCH 54/92] Changelog tweaks --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9ad7dfbdad6..c3ed13ecb51 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,8 +10,8 @@ - Fix a bug in [MSC4186: Simplified Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) that could prevent user avatars from showing if the room had an empty name. ([\#19468](https://github.com/element-hq/synapse/issues/19468), [\#19791](https://github.com/element-hq/synapse/issues/19791)) - Fix access token cache not being invalidated for sessions using refresh tokens. Contributed by @FrenchGithubUser @ Famedly. ([\#19483](https://github.com/element-hq/synapse/issues/19483)) -- Fix bug where Synapse would return 400 (`M_BAD_JSON`) when sending a message with `mentions` field and Synapse module `check_event_allowed` callback registered (frozen event). Contributed by @gaetan-sbt. ([\#19634](https://github.com/element-hq/synapse/issues/19634)) -- Fix long-standing but niche bug with sync where it could attempt to fetch data with flawed invalid future tokens. ([\#19644](https://github.com/element-hq/synapse/issues/19644)) +- Fix bug where Synapse would return 400 (`M_BAD_JSON`) when sending a message with a `mentions` field and Synapse module `check_event_allowed` callback registered (frozen event). Contributed by @gaetan-sbt. ([\#19634](https://github.com/element-hq/synapse/issues/19634)) +- Fix long-standing but niche bug with `/sync` where it could attempt to fetch data with flawed invalid future tokens. ([\#19644](https://github.com/element-hq/synapse/issues/19644)) - Fix `/sync` failing when [MSC4354 Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354) are enabled and the sync request filters out Ephemeral Data Units (EDUs). ([\#19787](https://github.com/element-hq/synapse/issues/19787)) - Fix packaging for Fedora and EPEL caused by unnecessary bumping `attrs` minimum version requirement in `pyproject.toml` file. Contributed by Oleg Girko. ([\#19789](https://github.com/element-hq/synapse/issues/19789)) - Fix merging signatures when a policy server is running under the same server name as Synapse. The bug was re-introduced in v1.153.0rc1 after being fixed earlier in v1.151.0rc1. Contributed by @tulir @ Beeper. ([\#19797](https://github.com/element-hq/synapse/issues/19797)) From 65ddc7118ab556dd2dbfa47ae9175e8fa67d4566 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 May 2026 17:33:26 +0000 Subject: [PATCH 55/92] Bump go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp from 1.34.0 to 1.43.0 in /complement (#19673) Signed-off-by: dependabot[bot] --- complement/go.mod | 24 +++++++------- complement/go.sum | 84 +++++++++++++++++++++++------------------------ 2 files changed, 54 insertions(+), 54 deletions(-) diff --git a/complement/go.mod b/complement/go.mod index d1ccf6b4962..5ebf19bbf6f 100644 --- a/complement/go.mod +++ b/complement/go.mod @@ -1,8 +1,6 @@ module github.com/element-hq/synapse -go 1.24.1 - -toolchain go1.24.4 +go 1.25.0 require ( github.com/matrix-org/complement v0.0.0-20251120181401-44111a2a8a9d @@ -16,7 +14,7 @@ require ( github.com/hashicorp/go-set/v3 v3.0.0 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect github.com/oleiade/lane/v2 v2.0.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.34.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect gotest.tools/v3 v3.4.0 // indirect ) @@ -47,13 +45,15 @@ require ( github.com/tidwall/sjson v1.2.5 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect - go.opentelemetry.io/otel v1.41.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 // indirect - go.opentelemetry.io/otel/metric v1.41.0 // indirect - go.opentelemetry.io/otel/trace v1.41.0 // indirect - go.opentelemetry.io/proto/otlp v1.7.0 // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/sys v0.38.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect + golang.org/x/crypto v0.49.0 // indirect + golang.org/x/net v0.52.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.42.0 // indirect golang.org/x/time v0.11.0 // indirect - google.golang.org/protobuf v1.36.6 // indirect + golang.org/x/tools v0.42.0 // indirect ) diff --git a/complement/go.sum b/complement/go.sum index f82487882b3..25c69226695 100644 --- a/complement/go.sum +++ b/complement/go.sum @@ -2,8 +2,8 @@ github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEK github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= @@ -38,8 +38,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/go-set/v3 v3.0.0 h1:CaJBQvQCOWoftrBcDt7Nwgo0kdpmrKxar/x2o6pV9JA= @@ -101,55 +101,55 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= -go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9HsPl42NIXFIFSUSSs0fiqra0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0/go.mod h1:90PoxvaEB5n6AOdZvi+yWJQoE95U8Dhhw2bSyRqnTD0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.34.0 h1:BEj3SPM81McUZHYjRS5pEgNgnmzGJ5tRpU5krWnV8Bs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.34.0/go.mod h1:9cKLGBDzI/F3NoHLQGm4ZrYdIHsvGt6ej6hUowxY0J4= -go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= -go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= -go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= -go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= -go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= -go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= -go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= -go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= -go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= -go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= -golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= -golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -157,20 +157,20 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= -golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a h1:SGktgSolFCo75dnHJF2yMvnns6jCmHFJ0vE4Vn2JKvQ= -google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a/go.mod h1:a77HrdMjoeKbnd2jmgcWdaS++ZLZAEq3orIOAEIKiVw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= -google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0= From 0a6aed8df273d599ca477364c0279960a9d9cecc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 May 2026 17:35:45 +0000 Subject: [PATCH 56/92] Bump rand from 0.9.2 to 0.9.4 (#19687) Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8285c7cf38c..761e99b3e65 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -952,9 +952,9 @@ checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha", "rand_core", From a6b8a19ac72e17211777455b817ff1e23855bf31 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 28 May 2026 20:44:17 +0300 Subject: [PATCH 57/92] Add workaround for missing events in `_should_perform_remote_join` (#19730) --- changelog.d/19730.bugfix | 1 + synapse/handlers/room_member.py | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 changelog.d/19730.bugfix diff --git a/changelog.d/19730.bugfix b/changelog.d/19730.bugfix new file mode 100644 index 00000000000..074405d9442 --- /dev/null +++ b/changelog.d/19730.bugfix @@ -0,0 +1 @@ +Work around bug that sometimes breaks joining restricted rooms that require a remote join. Contributed by @tulir @ Beeper. diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 236c8ca03c7..8a19dba5ee5 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -1355,7 +1355,15 @@ async def _should_perform_remote_join( current_state = { state_key: event_map[event_id] for state_key, event_id in state_before_join.items() + # TODO figure out why events present in state_before_join are sometimes not found in event_map + # See https://github.com/element-hq/synapse/issues/19465 + if event_id in event_map } + if len(current_state) < len(state_before_join): + logger.warning( + "Some events from state_before_join were not found in event_map: %s", + set(state_before_join.values()) - set(event_map.keys()), + ) servers_that_can_issue_invite = get_servers_from_users( get_users_which_can_issue_invite(current_state) ) From c45096e7e82de5c550c169253b81a4fc01f203e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 May 2026 09:00:39 +0000 Subject: [PATCH 58/92] Bump sigstore/cosign-installer from 4.1.1 to 4.1.2 in the minor-and-patches group (#19803) Signed-off-by: dependabot[bot] --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c451e493011..e343d950b30 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -186,7 +186,7 @@ jobs: uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Install Cosign - uses: sigstore/cosign-installer@cad07c2e89fa2edd6e2d7bab4c1aa38e53f76003 # v4.1.1 + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - name: Calculate docker image tag uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 From 306d8b23bd405802ef668e99420d37f6f8a2ecec Mon Sep 17 00:00:00 2001 From: shcherbak Date: Fri, 29 May 2026 13:57:33 +0300 Subject: [PATCH 59/92] Feat/gcp json formatter (#19775) Co-authored-by: Andrew Morgan --- changelog.d/19775.misc | 1 + docs/structured_logging.md | 41 ++++++++++++++++ synapse/logging/__init__.py | 8 +++- synapse/logging/_terse_json.py | 29 ++++++++++++ tests/logging/test_terse_json.py | 80 +++++++++++++++++++++++++++++++- 5 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 changelog.d/19775.misc diff --git a/changelog.d/19775.misc b/changelog.d/19775.misc new file mode 100644 index 00000000000..3ff0fd34b9c --- /dev/null +++ b/changelog.d/19775.misc @@ -0,0 +1 @@ +Add `GcpJsonFormatter` logging formatter for use with Google Cloud Logging and GKE deployments. \ No newline at end of file diff --git a/docs/structured_logging.md b/docs/structured_logging.md index 761d6466dd2..c6916670aab 100644 --- a/docs/structured_logging.md +++ b/docs/structured_logging.md @@ -78,3 +78,44 @@ loggers: The above logging config will set Synapse as 'INFO' logging level by default, with the SQL layer at 'WARNING', and will log JSON formatted messages to a remote endpoint at 10.1.2.3:9999. + +## Google Cloud Logging (GKE) + +When running Synapse on GKE, use `synapse.logging.GcpJsonFormatter`. It outputs +JSON to stdout with a `severity` field that Google Cloud Logging maps to the +correct per-entry severity. Without this, GKE assigns `ERROR` to everything +written to stderr regardless of the actual Python log level. + +Example output: + +```json +{"severity":"INFO","message":"Processed request: 3.481sec 200 GET /sync","logger":"synapse.access.http.8008","time":"2026-05-12T13:40:37.829Z"} +``` + +Configuration: + +```yaml +version: 1 +disable_existing_loggers: false + +formatters: + gcp_json: + class: synapse.logging.GcpJsonFormatter + +handlers: + console: + class: logging.StreamHandler + formatter: gcp_json + stream: ext://sys.stdout + +loggers: + synapse.storage.SQL: + level: WARNING + twisted: + handlers: [console] + propagate: false + +root: + level: INFO + handlers: [console] +``` diff --git a/synapse/logging/__init__.py b/synapse/logging/__init__.py index 15b92d7ef3d..4058561e5ec 100644 --- a/synapse/logging/__init__.py +++ b/synapse/logging/__init__.py @@ -22,10 +22,14 @@ import logging from synapse.logging._remote import RemoteHandler -from synapse.logging._terse_json import JsonFormatter, TerseJsonFormatter +from synapse.logging._terse_json import ( + GcpJsonFormatter, + JsonFormatter, + TerseJsonFormatter, +) # These are imported to allow for nicer logging configuration files. -__all__ = ["RemoteHandler", "JsonFormatter", "TerseJsonFormatter"] +__all__ = ["RemoteHandler", "JsonFormatter", "TerseJsonFormatter", "GcpJsonFormatter"] # Debug logger for https://github.com/matrix-org/synapse/issues/9533 etc issue9533_logger = logging.getLogger("synapse.9533_debug") diff --git a/synapse/logging/_terse_json.py b/synapse/logging/_terse_json.py index d9ff70b2520..afa3288809c 100644 --- a/synapse/logging/_terse_json.py +++ b/synapse/logging/_terse_json.py @@ -25,6 +25,7 @@ import json import logging +from datetime import datetime, timezone _encoder = json.JSONEncoder(ensure_ascii=False, separators=(",", ":")) @@ -93,3 +94,31 @@ def format(self, record: logging.LogRecord) -> str: } return self._format(record, event) + + +class GcpJsonFormatter(logging.Formatter): + """JSON formatter compatible with Google Cloud Logging structured logging. + + Outputs `severity` (not `level`) so GCL correctly maps each log record to + the right severity instead of inheriting ERROR from stderr. + """ + + def format(self, record: logging.LogRecord) -> str: + msg = record.getMessage() + if record.exc_info: + if not record.exc_text: + record.exc_text = self.formatException(record.exc_info) + if record.exc_text: + msg = f"{msg}\n{record.exc_text}" + + event = { + "severity": record.levelname, + "message": msg, + "logger": record.name, + "time": datetime.fromtimestamp(record.created, tz=timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%S.%f" + )[:-3] + + "Z", + } + + return _encoder.encode(event) diff --git a/tests/logging/test_terse_json.py b/tests/logging/test_terse_json.py index a857737ddfe..433408fee68 100644 --- a/tests/logging/test_terse_json.py +++ b/tests/logging/test_terse_json.py @@ -28,7 +28,11 @@ from twisted.web.server import Request from synapse.http.site import SynapseRequest -from synapse.logging._terse_json import JsonFormatter, TerseJsonFormatter +from synapse.logging._terse_json import ( + GcpJsonFormatter, + JsonFormatter, + TerseJsonFormatter, +) from synapse.logging.context import LoggingContext, LoggingContextFilter from synapse.types import JsonDict @@ -251,3 +255,77 @@ def test_with_exception(self) -> None: self.assertEqual(log["log"], "Hello there, wally!") self.assertEqual(log["exc_type"], "ValueError") self.assertEqual(log["exc_value"], "That's wrong, you wally!") + + +class GcpJsonFormatterTestCase(LoggerCleanupMixin, TestCase): + def setUp(self) -> None: + self.output = StringIO() + + def get_log_line(self) -> JsonDict: + data = self.output.getvalue() + logs = data.splitlines() + self.assertEqual(len(logs), 1) + self.assertEqual(data.count("\n"), 1) + return json.loads(logs[0]) + + def test_gcp_json_output(self) -> None: + """ + GcpJsonFormatter produces exactly the four fields GCL expects. + """ + handler = logging.StreamHandler(self.output) + handler.setFormatter(GcpJsonFormatter()) + logger = self.get_logger(handler) + + logger.info("Hello there, %s!", "wally") + + log = self.get_log_line() + + self.assertIncludes( + log.keys(), {"severity", "message", "logger", "time"}, exact=True + ) + self.assertEqual(log["message"], "Hello there, wally!") + self.assertEqual(log["severity"], "INFO") + self.assertTrue(log["time"].endswith("Z")) + + def test_severity_levels(self) -> None: + """ + Python log levels are mapped to their GCL severity equivalents. + """ + cases = [ + (logging.DEBUG, "DEBUG"), + (logging.INFO, "INFO"), + (logging.WARNING, "WARNING"), + (logging.ERROR, "ERROR"), + (logging.CRITICAL, "CRITICAL"), + ] + for level, expected_severity in cases: + self.output = StringIO() + handler = logging.StreamHandler(self.output) + handler.setFormatter(GcpJsonFormatter()) + logger = self.get_logger(handler) + logger.setLevel(level) + logger.log(level, "test") + log = self.get_log_line() + self.assertEqual(log["severity"], expected_severity, f"level={level}") + + def test_gcp_json_with_exception(self) -> None: + """ + Exception info is appended to the message field, not separate keys. + """ + handler = logging.StreamHandler(self.output) + handler.setFormatter(GcpJsonFormatter()) + logger = self.get_logger(handler) + + try: + raise ValueError("That's wrong, you wally!") + except ValueError: + logger.exception("Hello there, %s!", "wally") + + log = self.get_log_line() + + self.assertIncludes( + log.keys(), {"severity", "message", "logger", "time"}, exact=True + ) + self.assertIn("Hello there, wally!", log["message"]) + self.assertIn("ValueError", log["message"]) + self.assertIn("That's wrong, you wally!", log["message"]) From 2aef6c33a842fe7f50e0a8492a0396412aed5115 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 1 Jun 2026 11:34:12 +0100 Subject: [PATCH 60/92] Add logging to help diagnose missing to-device messages (#19801) Some attempts to debug https://github.com/element-hq/synapse/issues/19795. --------- Co-authored-by: Eric Eastwood Co-authored-by: Eric Eastwood --- changelog.d/19801.misc | 1 + synapse/replication/tcp/resource.py | 102 +++++++++--------- synapse/storage/databases/main/deviceinbox.py | 101 ++++++++++++----- synapse/storage/util/id_generators.py | 26 ++++- 4 files changed, 151 insertions(+), 79 deletions(-) create mode 100644 changelog.d/19801.misc diff --git a/changelog.d/19801.misc b/changelog.d/19801.misc new file mode 100644 index 00000000000..56ec0f7ddca --- /dev/null +++ b/changelog.d/19801.misc @@ -0,0 +1 @@ +Add more logging to the to-device message replication stream. diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index 36dd39ed672..95364778cc0 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -182,13 +182,10 @@ async def _run_notifier_loop(self) -> None: self.command_handler.will_announce_positions() for stream in all_streams: - self.command_handler.send_command( - PositionCommand( - stream.NAME, - self._instance_name, - stream.last_token, - stream.last_token, - ) + self._send_position_command( + stream_name=stream.NAME, + prev_token=stream.last_token, + new_token=stream.last_token, ) for stream in all_streams: @@ -205,9 +202,9 @@ async def _run_notifier_loop(self) -> None: last_token = stream.last_token logger.debug( - "Getting stream: %s: %s -> %s", + "Getting stream updates for %s: %s -> %s", stream.NAME, - stream.last_token, + last_token, stream.current_token(self._instance_name), ) try: @@ -217,26 +214,7 @@ async def _run_notifier_loop(self) -> None: logger.info("Failed to handle stream %s", stream.NAME) raise - logger.debug( - "Sending %d updates", - len(updates), - ) - - if updates: - logger.info( - "Streaming: %s -> %s (limited: %s, updates: %s, max token: %s)", - stream.NAME, - updates[-1][0], - limited, - len(updates), - current_token, - ) - stream_updates_counter.labels( - stream_name=stream.NAME, - **{SERVER_NAME_LABEL: self.server_name}, - ).inc(len(updates)) - - else: + if not updates: # The token has advanced but there is no data to # send, so we send a `POSITION` to inform other # workers of the updated position. @@ -266,21 +244,28 @@ async def _run_notifier_loop(self) -> None: # POSITION with last token of X+1, which will # cause them to check if there were any missing # updates between X and X+1. - logger.info( - "Sending position: %s -> %s", - stream.NAME, - current_token, - ) - self.command_handler.send_command( - PositionCommand( - stream.NAME, - self._instance_name, - last_token, - current_token, - ) + self._send_position_command( + stream_name=stream.NAME, + prev_token=last_token, + new_token=current_token, ) continue + logger.info( + "Sending update for %s: %s -> %s (limited: %s, updates: %s, max token: %s)", + stream.NAME, + last_token, + updates[-1][0], + limited, + len(updates), + current_token, + ) + + stream_updates_counter.labels( + stream_name=stream.NAME, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc(len(updates)) + # Some streams return multiple rows with the same stream IDs, # we need to make sure they get sent out in batches. We do # this by setting the current token to all but the last of @@ -300,18 +285,10 @@ async def _run_notifier_loop(self) -> None: # token, in which case we want to send out a `POSITION` # to tell other workers the actual current position. if updates[-1][0] < current_token: - logger.info( - "Sending position: %s -> %s", - stream.NAME, - current_token, - ) - self.command_handler.send_command( - PositionCommand( - stream.NAME, - self._instance_name, - updates[-1][0], - current_token, - ) + self._send_position_command( + stream_name=stream.NAME, + prev_token=updates[-1][0], + new_token=current_token, ) logger.debug("No more pending updates, breaking poke loop") @@ -319,6 +296,25 @@ async def _run_notifier_loop(self) -> None: self.pending_updates = False self.is_looping = False + def _send_position_command( + self, *, stream_name: str, prev_token: int, new_token: int + ) -> None: + """Send a POSITION command over replication""" + logger.info( + "Sending position for %s: %s -> %s", + stream_name, + prev_token, + new_token, + ) + self.command_handler.send_command( + PositionCommand( + stream_name, + self._instance_name, + prev_token, + new_token, + ) + ) + def _batch_updates( updates: list[tuple[Token, StreamRow]], diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index fc61f46c1c5..a3806fd1123 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -897,10 +897,20 @@ def _add_messages_to_local_device_inbox_txn( ) -> None: assert self._can_write_to_device - local_by_user_then_device = {} + # A map from user id, to device id, to a pair of (serialized message, msgid). + local_by_user_then_device: dict[str, dict[str, tuple[str, str]]] = {} + for user_id, messages_by_device in messages_by_user_then_device.items(): - messages_json_for_user = {} + # Mesages to send to this specific user. A map + # from device id, to a pair of (serialized message, msgid). + messages_json_for_user: dict[str, tuple[str, str]] = {} + devices = list(messages_by_device.keys()) + if not devices: + # No to-device messages for this user. (For example, someone has + # hit `/sendToDevice` with an empty {device: message} dict.) + continue + if len(devices) == 1 and devices[0] == "*": # Handle wildcard device_ids. # We exclude hidden devices (such as cross-signing keys) here as they are @@ -912,15 +922,28 @@ def _add_messages_to_local_device_inbox_txn( retcol="device_id", ) - message_json = json_encoder.encode(messages_by_device["*"]) + # Don't bother to serialize if there are no devices for this user + if not devices: + if issue9533_logger.isEnabledFor(logging.DEBUG): + msgid = _get_msgid_for_message(messages_by_device["*"]) + issue9533_logger.debug( + "Dropping wildcard to-device message for user %s with no devices (msgid %s)", + user_id, + msgid, + ) + continue + + message_json, msgid = _serialize_to_device_message( + user_id=user_id, device_id="*", msg=messages_by_device["*"] + ) for device_id in devices: # Add the message for all devices for this user on this # server. - messages_json_for_user[device_id] = message_json + messages_json_for_user[device_id] = (message_json, msgid) else: - if not devices: - continue - + # Query the database to determine which of the target devices actually + # exist. + # # We exclude hidden devices (such as cross-signing keys) here as they are # not expected to receive to-device messages. rows = cast( @@ -938,19 +961,25 @@ def _add_messages_to_local_device_inbox_txn( for (device_id,) in rows: # Only insert into the local inbox if the device exists on # this server - with start_active_span("serialise_to_device_message"): - msg = messages_by_device[device_id] - set_tag(SynapseTags.TO_DEVICE_TYPE, msg["type"]) - set_tag(SynapseTags.TO_DEVICE_SENDER, msg["sender"]) - set_tag(SynapseTags.TO_DEVICE_RECIPIENT, user_id) - set_tag(SynapseTags.TO_DEVICE_RECIPIENT_DEVICE, device_id) - set_tag( - SynapseTags.TO_DEVICE_MSGID, - msg["content"].get(EventContentFields.TO_DEVICE_MSGID), - ) - message_json = json_encoder.encode(msg) + msg = messages_by_device[device_id] + message_json, msgid = _serialize_to_device_message( + user_id=user_id, device_id=device_id, msg=msg + ) + messages_json_for_user[device_id] = (message_json, msgid) - messages_json_for_user[device_id] = message_json + if issue9533_logger.isEnabledFor(logging.DEBUG): + # Log any messages we are dropping + unmapped_devices = ( + messages_by_device.keys() - messages_json_for_user.keys() + ) + if unmapped_devices: + issue9533_logger.debug( + "Dropping to-device messages for unknown devices: %s", + [ + f"{user_id}/{device_id} (msgid {_get_msgid_for_message(messages_by_device[device_id])})" + for device_id in unmapped_devices + ], + ) if messages_json_for_user: local_by_user_then_device[user_id] = messages_json_for_user @@ -965,22 +994,21 @@ def _add_messages_to_local_device_inbox_txn( values=[ (user_id, device_id, stream_id, message_json, self._instance_name) for user_id, messages_by_device in local_by_user_then_device.items() - for device_id, message_json in messages_by_device.items() + for device_id, (message_json, _msgid) in messages_by_device.items() ], ) if issue9533_logger.isEnabledFor(logging.DEBUG): issue9533_logger.debug( - "Stored to-device messages with stream_id %i: %s", + "Storing to-device messages with stream_id %i: %s", stream_id, [ - f"{user_id}/{device_id} (msgid " - f"{msg['content'].get(EventContentFields.TO_DEVICE_MSGID)})" + f"{user_id}/{device_id} (msgid {msgid})" for ( user_id, messages_by_device, - ) in messages_by_user_then_device.items() - for (device_id, msg) in messages_by_device.items() + ) in local_by_user_then_device.items() + for (device_id, (_msg, msgid)) in messages_by_device.items() ], ) @@ -1066,6 +1094,29 @@ def get_devices_with_messages_txn( return results +def _serialize_to_device_message( + *, user_id: str, device_id: str, msg: JsonDict +) -> tuple[str, str]: + """Serialiize a to-device message, ready to add to the device_inbox table. + + Returns a tuple (message_json, msgid). + """ + with start_active_span("serialise_to_device_message"): + msgid = _get_msgid_for_message(msg) + set_tag(SynapseTags.TO_DEVICE_TYPE, msg["type"]) + set_tag(SynapseTags.TO_DEVICE_SENDER, msg["sender"]) + set_tag(SynapseTags.TO_DEVICE_RECIPIENT, user_id) + set_tag(SynapseTags.TO_DEVICE_RECIPIENT_DEVICE, device_id) + set_tag(SynapseTags.TO_DEVICE_MSGID, msgid) + message_json = json_encoder.encode(msg) + return message_json, msgid + + +def _get_msgid_for_message(msg: JsonDict) -> str: + """Extract the message ID from a to-device message.""" + return str(msg["content"].get(EventContentFields.TO_DEVICE_MSGID, "")) + + class DeviceInboxBackgroundUpdateStore(SQLBaseStore): DEVICE_INBOX_STREAM_ID = "device_inbox_stream_drop" REMOVE_DEAD_DEVICES_FROM_INBOX = "remove_dead_devices_from_device_inbox" diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index a1afe6d2aee..1e053be6aff 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -38,6 +38,7 @@ import attr from sortedcontainers import SortedList, SortedSet +from synapse.logging import issue9533_logger from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.database import ( DatabasePool, @@ -774,6 +775,14 @@ def _add_persisted_position(self, new_id: int) -> None: # We move the current min position up if the minimum current positions # of all instances is higher (since by definition all positions less # that that have been persisted). + # + # If we are one of several writers, then we don't need to factor our own + # `_current_position` into `_persisted_upto_position` unless we have unfinished + # writes (since we know that any future write that happens locally will have + # a higher stream ID than any of the other writers' current positions). In other + # words, when we have no outstanding writes, then the new `_persisted_upto_position` + # can be the minimum of all *other* writers' current positions, + # our_current_position = self._current_positions.get(self._instance_name, 0) min_curr = min( ( @@ -783,7 +792,6 @@ def _add_persisted_position(self, new_id: int) -> None: ), default=our_current_position, ) - if our_current_position and (self._unfinished_ids or self._in_flight_fetches): min_curr = min(min_curr, our_current_position) @@ -820,6 +828,22 @@ def _add_persisted_position(self, new_id: int) -> None: # do. break + # Hacky debug logging to attempt to trace https://github.com/element-hq/synapse/issues/19795 + if ( + issue9533_logger.isEnabledFor(logging.DEBUG) + and self._stream_name == "to_device" + ): + issue9533_logger.debug( + "stream_id=%i now persisted for stream=%s; _current_positions=%s _unfinished_ids=%s, _known_persisted_positions=%s _persisted_upto_position=%i min_curr=%i", + new_id, + self._stream_name, + self._current_positions, + self._unfinished_ids, + self._known_persisted_positions, + self._persisted_upto_position, + min_curr, + ) + def _update_stream_positions_table_txn(self, txn: Cursor) -> None: """Update the `stream_positions` table with newly persisted position.""" From 71e07d4c75e1dad297bc505b249acd556bf5b253 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:38:08 +0200 Subject: [PATCH 61/92] Bump hashicorp/vault-action from 3.4.0 to 4.0.0 (#19804) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [hashicorp/vault-action](https://github.com/hashicorp/vault-action) from 3.4.0 to 4.0.0.
Release notes

Sourced from hashicorp/vault-action's releases.

v4.0.0

4.0.0 (May 12, 2026)

Improvements:

  • Bump node runtime from node20 to node24 GH-604
  • Fix leading slash in secret paths causing HTTP 400 errors (e.g. /cubbyhole/test → v1/cubbyhole/test instead of v1//cubbyhole/test)
  • bump jsrsasign from 11.1.0 to 11.1.3
  • bump body-parser from 1.20.3 to 1.20.5
  • bump qs from 6.13.0 to 6.15.1
  • bump http-errors from 2.0.0 to 2.0.1
  • bump minimatch from 3.1.2 to 3.1.5
  • bump underscore from 1.13.4 to 1.13.8
Changelog

Sourced from hashicorp/vault-action's changelog.

4.0.0 (May 12, 2026)

Improvements:

  • Bump node runtime from node20 to node24 GH-604
  • Fix leading slash in secret paths causing HTTP 400 errors (e.g. /cubbyhole/test → v1/cubbyhole/test instead of v1//cubbyhole/test)
  • bump jsrsasign from 11.1.0 to 11.1.3
  • bump body-parser from 1.20.3 to 1.20.5
  • bump qs from 6.13.0 to 6.15.1
  • bump http-errors from 2.0.0 to 2.0.1
  • bump minimatch from 3.1.2 to 3.1.5
  • bump underscore from 1.13.4 to 1.13.8

3.4.0 (June 13, 2025)

Bugs:

Improvements:

3.3.0 (March 3, 2025)

Features:

  • Wildcard secret imports can use ** to retain case of exported env keys GH-545

3.2.0 (March 3, 2025)

Improvements:

  • Add retry for jwt auth login to fix intermittent login failures GH-574

3.1.0 (January 9, 2025)

Improvements:

  • fix wildcard handling when field contains dot GH-542
  • bump body-parser from 1.20.0 to 1.20.3
  • bump braces from 3.0.2 to 3.0.3
  • bump cross-spawn from 7.0.3 to 7.0.6
  • bump micromatch from 4.0.5 to 4.0.8

Features:

  • secretId is no longer required for approle to support advanced use cases like machine login when bind_secret_id is false. GH-522
  • Use pki configuration to generate certificates from Vault GH-564

3.0.0 (February 15, 2024)

... (truncated)

Commits
  • 892a268 Update copywrite headers for v.4.0.0 release (#607)
  • a7ffa26 Prepare for release v4.0.0 (#606)
  • a049f01 [COMPLIANCE] Add/Update Copyright Headers (#605)
  • 95977a3 Adding team-vault-consumption as CODEOWNERS (#600)
  • 7e48e56 Upgrade Node.js to 24 and update dependencies (#604)
  • 79632e3 [COMPLIANCE] Add Copyright and License Headers (Batch 1 of 1) (#589)
  • 734c523 README.md: Removing jwtGithubAudience default (#590)
  • 2c58270 [Compliance] - PR Template Changes Required (#586)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=hashicorp/vault-action&package-manager=github_actions&previous-version=3.4.0&new-version=4.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e343d950b30..801d369c483 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -67,7 +67,7 @@ jobs: - name: Get team registry token id: import-secrets - uses: hashicorp/vault-action@4c06c5ccf5c0761b6029f56cfb1dcf5565918a3b # v3.4.0 + uses: hashicorp/vault-action@892a26828f195e65540a40b4768ae4571f51ebfc # v4.0.0 with: url: https://vault.infra.ci.i.element.dev role: ${{ steps.vault-jwt-role.outputs.role_name }} @@ -164,7 +164,7 @@ jobs: - name: Get team registry token id: import-secrets - uses: hashicorp/vault-action@4c06c5ccf5c0761b6029f56cfb1dcf5565918a3b # v3.4.0 + uses: hashicorp/vault-action@892a26828f195e65540a40b4768ae4571f51ebfc # v4.0.0 with: url: https://vault.infra.ci.i.element.dev role: ${{ steps.vault-jwt-role.outputs.role_name }} From 6c3ba2205bcc78187dcbdee25b3950c6be92da83 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:45:41 +0200 Subject: [PATCH 62/92] Bump idna from 3.11 to 3.15 (#19790) Bumps [idna](https://github.com/kjd/idna) from 3.11 to 3.15.
Changelog

Sourced from idna's changelog.

3.15 (2026-05-12)

  • Enforce DNS-length cap on individual labels early in check_label, short-circuiting contextual-rule processing for oversized input while staying compatible with UTS 46 usage.
  • Tidy core helpers: hoist bidi category sets to module-level frozensets (avoiding per-codepoint list construction), simplify length checks, and reuse the shared _unicode_dots_re from idna.core in the codec module.
  • Use raise ... from err for proper exception chaining and switch internal string formatting to f-strings.
  • Allow flit_core 4.x in the build backend.
  • Expand the ruff lint set (flake8-bugbear, flake8-simplify, pyupgrade, perflint) and apply the surfaced fixes; pin lint CI to Python 3.14.
  • Add Dependabot configuration for GitHub Actions.
  • Convert README and HISTORY from reStructuredText to Markdown.
  • Reference CVE-2026-45409 for the 3.14 advisory in place of the initial GHSA identifier.

Thanks to Felix Yan, Stan Ulbrych, and metsw24-max for contributions to this release.

3.14 (2026-05-10)

  • Removed opportunity to process long inputs into quadratic time by rejecting oversize inputs up-front. Closes a bypass of the CVE-2024-3651 mitigation. [CVE-2026-45409]

Thanks to Stan Ulbrych for reporting the issue.

3.13 (2026-04-22)

  • Correct classification error for codepoint U+A7F1

3.12 (2026-04-21)

  • Update to Unicode 17.0.0.
  • Issue a deprecation warning for the transitional argument.
  • Added lazy-loading to provide some performance improvements.
  • Removed vestiges of code related to Python 2 support, including segmentation of data structures specific to Jython.

Thanks to Rodrigo Nogueira for contributions to this release.

Commits
  • af30a09 Release 3.15
  • 30314d4 Pre-release 3.15rc0
  • 05d4b21 Merge pull request #237 from kjd/convert-docs-to-markdown
  • 2987fdb Convert README and HISTORY from reStructuredText to Markdown
  • 59fa800 Merge pull request #236 from kjd/dependabot/github_actions/actions-f3e34333ea
  • def6983 Merge branch 'master' into dependabot/github_actions/actions-f3e34333ea
  • bbd8004 Merge pull request #234 from StanFromIreland/patch-1
  • edd07c0 Bump github/codeql-action from 3.35.2 to 4.35.2 in the actions group
  • 5557db0 Merge branch 'master' into patch-1
  • f11746c Merge pull request #235 from StanFromIreland/patch-2
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=idna&package-manager=pip&previous-version=3.11&new-version=3.15)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/element-hq/synapse/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 6093b13c02f..4afbaad2c15 100644 --- a/poetry.lock +++ b/poetry.lock @@ -752,18 +752,18 @@ test = ["coverage[toml]", "pretend", "pytest", "pytest-cov"] [[package]] name = "idna" -version = "3.11" +version = "3.15" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.8" groups = ["main", "dev"] files = [ - {file = "idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea"}, - {file = "idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902"}, + {file = "idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8"}, + {file = "idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc"}, ] [package.extras] -all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] +all = ["mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] [[package]] name = "ijson" From 9e2a076144601e07f253cb335f2b0190417b57cb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Jun 2026 11:05:38 +0100 Subject: [PATCH 63/92] Port Event class to Rust (#19701) Ports the event class to Rust. The main difference here are: 1. There is now a single event class 2. We now validate a lot more at event construction time than we previously did (we basically checked nothing before). This required some changes to the tests, including https://github.com/matrix-org/sytest/pull/1423 Reviewable commit-by-commit. ### Overview of Event Rust structure The format of the event struct in Rust is quite different than that in Python. The top-level looks like: ```rust pub struct Event { /// The parsed event JSON. fields: FormattedEvent, /// The event ID. For format v1 this is read directly from the JSON; /// for v2+ it is computed from the canonical-JSON hash at /// construction time and cached here. event_id: Arc, /// Synapse-internal per-event state that lives outside the federated /// JSON (e.g. outlier flag, soft-failure, stream positions). #[pyo3(get)] internal_metadata: EventInternalMetadata, /// The room version this event was parsed for. #[pyo3(get)] room_version: &'static RoomVersion, /// `None` for accepted events; otherwise a short reason set by auth /// when the event was rejected. rejected_reason: Option>, } ``` which includes the actual parsed event in `FormattedEvent`, plus the rest of the event metadata. ```rust pub struct FormattedEvent> { #[serde(default)] pub signatures: Signatures, #[serde(default)] pub unsigned: Unsigned, #[serde(flatten)] pub specific_fields: E, #[serde(flatten)] pub common_fields: Arc, } ``` The struct is further split into the common fields, format specific fields, plus the signatures and unsigned. We split out the signature and unsigned fields as they are mutable, so when we clone the event we can still share the common and specific fields and only copy signature and unsigned. The `specific_fields` are the fields that depend on the format version. They can either be a specific format (e.g. `E = EventFormatV1`) or a type-erased enum `EventFormatEnum` that is across all room versions: ```rust pub enum EventFormatEnum { V1(EventFormatV1), V2V3(EventFormatV2V3), V4(EventFormatV4), VMSC4242(EventFormatVMSC4242), } ``` For example: ```rust /// Shared flat-list encoding of `auth_events` and `prev_events`, reused /// by every format from v2/v3 onwards. #[derive(Serialize, Deserialize)] pub struct SimpleAuthPrevEvents { pub auth_events: Vec, pub prev_events: Vec, } /// Version-specific fields for room versions 3-10. #[derive(Serialize, Deserialize)] pub struct EventFormatV2V3 { pub room_id: Box, #[serde(flatten)] pub auth_prev_events: SimpleAuthPrevEvents, } ``` ### Dev notes As discussed in [`#element-backend-internal:matrix.org`](https://matrix.to/#/!SGNQGPGUwtcPBUotTL:matrix.org/$3gTjDO440GbAz57cXcCawwiyFLiD0crrarvS1uhzKOY?via=jki.re&via=element.io&via=matrix.org) --------- Co-authored-by: Eric Eastwood --- changelog.d/19701.misc | 1 + rust/src/duration.rs | 32 +- rust/src/events/constants.rs | 143 ++ rust/src/events/formats/mod.rs | 268 ++++ rust/src/events/formats/v1.rs | 54 + rust/src/events/formats/v2v3.rs | 60 + rust/src/events/formats/v4.rs | 156 +++ rust/src/events/formats/vmsc4242.rs | 94 ++ rust/src/events/internal_metadata.rs | 11 +- rust/src/events/json_object.rs | 6 + rust/src/events/mod.rs | 888 +++++++++++- rust/src/events/signatures.rs | 15 +- rust/src/events/unsigned.rs | 13 +- rust/src/events/utils.rs | 1191 +++++++++++++++++ rust/src/json.rs | 218 +++ rust/src/lib.rs | 1 + synapse/crypto/event_signing.py | 5 +- synapse/crypto/keyring.py | 4 +- synapse/event_auth.py | 2 + synapse/events/__init__.py | 670 +--------- synapse/events/py_protocol.py | 34 +- synapse/events/utils.py | 175 +-- synapse/federation/federation_client.py | 8 +- .../third_party_event_rules_callbacks.py | 5 - synapse/rest/admin/rooms.py | 10 +- synapse/storage/controllers/persist_events.py | 15 +- .../storage/databases/main/censor_events.py | 10 +- synapse/storage/databases/main/events.py | 4 +- .../databases/main/events_bg_updates.py | 4 +- .../storage/databases/main/events_worker.py | 26 +- synapse/synapse_rust/events.pyi | 166 ++- tests/events/test_py_protocol.py | 27 +- tests/events/test_validator.py | 9 +- tests/handlers/test_device.py | 12 +- tests/handlers/test_room_member.py | 6 +- tests/handlers/test_room_policy.py | 3 +- tests/module_api/test_api.py | 4 +- tests/replication/storage/test_events.py | 1 + tests/rest/client/test_third_party_rules.py | 5 +- tests/state/test_v2.py | 3 - tests/storage/test_msc4242_state_dag.py | 29 +- tests/storage/test_redaction.py | 15 +- tests/storage/test_stream.py | 8 +- tests/test_event_auth.py | 8 +- tests/test_utils/event_builders.py | 14 + 45 files changed, 3454 insertions(+), 979 deletions(-) create mode 100644 changelog.d/19701.misc create mode 100644 rust/src/events/constants.rs create mode 100644 rust/src/events/formats/mod.rs create mode 100644 rust/src/events/formats/v1.rs create mode 100644 rust/src/events/formats/v2v3.rs create mode 100644 rust/src/events/formats/v4.rs create mode 100644 rust/src/events/formats/vmsc4242.rs create mode 100644 rust/src/events/utils.rs create mode 100644 rust/src/json.rs diff --git a/changelog.d/19701.misc b/changelog.d/19701.misc new file mode 100644 index 00000000000..4663e8b9611 --- /dev/null +++ b/changelog.d/19701.misc @@ -0,0 +1 @@ +Port the python Event classes to Rust. diff --git a/rust/src/duration.rs b/rust/src/duration.rs index a3dbe919b28..6c2e2653d11 100644 --- a/rust/src/duration.rs +++ b/rust/src/duration.rs @@ -29,19 +29,41 @@ fn duration_module(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { } /// Mirrors the `synapse.util.duration.Duration` Python class. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct SynapseDuration { - microseconds: u64, + milliseconds: u64, } impl SynapseDuration { - /// For now we only need to create durations from milliseconds. - pub fn from_milliseconds(milliseconds: u64) -> Self { + /// Creates a `SynapseDuration` from a number of milliseconds. + pub const fn from_milliseconds(milliseconds: u64) -> Self { + Self { milliseconds } + } + + /// Creates a `SynapseDuration` from a number of hours. + pub const fn from_hours(hours: u32) -> Self { + // We take a u32 here so that we know the multiplication won't overflow. + // We could instead panic, but that is unstable in a const context (for + // the current MSRV 1.82). Self { - microseconds: milliseconds * 1_000, + milliseconds: (hours as u64) * 3_600_000, } } } +impl<'py> IntoPyObject<'py> for SynapseDuration { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let duration_module = duration_module(py)?; + let kwargs = [("milliseconds", self.milliseconds)].into_py_dict(py)?; + let duration_instance = duration_module.call_method("Duration", (), Some(&kwargs))?; + Ok(duration_instance.into_bound()) + } +} + impl<'py> IntoPyObject<'py> for &SynapseDuration { type Target = PyAny; type Output = Bound<'py, Self::Target>; @@ -49,7 +71,7 @@ impl<'py> IntoPyObject<'py> for &SynapseDuration { fn into_pyobject(self, py: Python<'py>) -> Result { let duration_module = duration_module(py)?; - let kwargs = [("microseconds", self.microseconds)].into_py_dict(py)?; + let kwargs = [("milliseconds", self.milliseconds)].into_py_dict(py)?; let duration_instance = duration_module.call_method("Duration", (), Some(&kwargs))?; Ok(duration_instance.into_bound()) } diff --git a/rust/src/events/constants.rs b/rust/src/events/constants.rs new file mode 100644 index 00000000000..811794d48c3 --- /dev/null +++ b/rust/src/events/constants.rs @@ -0,0 +1,143 @@ +//! Matrix Events +//! +//! Contains types and utilities for working with Matrix events. + +/// Maximum size of a PDU +pub const MAX_PDU_SIZE_BYTES: usize = 65_536; + +/// Event Types +pub mod event_type { + /// Event type: m.room.member + pub const M_ROOM_MEMBER: &str = "m.room.member"; + /// Event type: m.room.create + pub const M_ROOM_CREATE: &str = "m.room.create"; + /// Event type: m.room.join_rules + pub const M_ROOM_JOIN_RULES: &str = "m.room.join_rules"; + /// Event type: m.room.power_levels + pub const M_ROOM_POWER_LEVELS: &str = "m.room.power_levels"; + /// Event type: m.room.aliases + pub const M_ROOM_ALIASES: &str = "m.room.aliases"; + /// Event type: m.room.history_visibility + pub const M_ROOM_HISTORY_VISIBILITY: &str = "m.room.history_visibility"; + /// Event type: m.room.redaction + pub const M_ROOM_REDACTION: &str = "m.room.redaction"; +} + +/// Event Fields +pub mod event_field { + /// Event field: auth_events + pub const AUTH_EVENTS: &str = "auth_events"; + /// Event field: content + pub const CONTENT: &str = "content"; + /// Event field: depth + pub const DEPTH: &str = "depth"; + /// Event field: hashes + pub const HASHES: &str = "hashes"; + /// Event field: origin_server_ts + pub const ORIGIN_SERVER_TS: &str = "origin_server_ts"; + /// Event field: prev_events + pub const PREV_EVENTS: &str = "prev_events"; + /// Event field: room_id + pub const ROOM_ID: &str = "room_id"; + /// Event field: sender + pub const SENDER: &str = "sender"; + /// Event field: signatures + pub const SIGNATURES: &str = "signatures"; + /// Event field: state_key + pub const STATE_KEY: &str = "state_key"; + /// Event field: type + pub const TYPE: &str = "type"; + /// Event field: unsigned + pub const UNSIGNED: &str = "unsigned"; + /// Event field: event_id + pub const EVENT_ID: &str = "event_id"; + /// Event field: origin + pub const ORIGIN: &str = "origin"; + /// Event field: prev_state + pub const PREV_STATE: &str = "prev_state"; + /// Event field: membership + pub const MEMBERSHIP: &str = "membership"; + /// Event field: replaces_state + pub const REPLACES_STATE: &str = "replaces_state"; + /// Event field: msc4354_sticky + pub const MSC4354_STICKY: &str = "msc4354_sticky"; + // Event field: prev_state_events + pub const PREV_STATE_EVENTS: &str = "prev_state_events"; + // Event field: m.relates_to + pub const M_RELATES_TO: &str = "m.relates_to"; +} + +pub mod unsigned_field { + /// Unsigned field: age + pub const AGE: &str = "age"; + /// Unsigned field: age_ts + pub const AGE_TS: &str = "age_ts"; + /// Unsigned field: redacted_because + pub const REDACTED_BECAUSE: &str = "redacted_because"; +} + +/// Membership Event Fields +pub mod membership_field { + /// Membership event field: membership + pub const MEMBERSHIP: &str = "membership"; + /// Membership event field: join_authorised_via_users_server + pub const JOIN_AUTHORISED_VIA_USERS_SERVER: &str = "join_authorised_via_users_server"; + /// Membership event field: third_party_invite + pub const THIRD_PARTY_INVITE: &str = "third_party_invite"; + /// Membership event field: signed + pub const SIGNED: &str = "signed"; +} + +/// Create Event Fields +pub mod create_field { + /// Create event field: creator + pub const CREATOR: &str = "creator"; +} + +/// Join Rules Event Fields +pub mod join_rules_field { + /// Join Rules event field: join_rule + pub const JOIN_RULE: &str = "join_rule"; + /// Join Rules event field: allow + pub const ALLOW: &str = "allow"; +} + +/// Power Levels Event Fields +pub mod power_levels_field { + /// Power Levels event field: users + pub const USERS: &str = "users"; + /// Power Levels event field: users_default + pub const USERS_DEFAULT: &str = "users_default"; + /// Power Levels event field: events + pub const EVENTS: &str = "events"; + /// Power Levels event field: events_default + pub const EVENTS_DEFAULT: &str = "events_default"; + /// Power Levels event field: state_default + pub const STATE_DEFAULT: &str = "state_default"; + /// Power Levels event field: ban + pub const BAN: &str = "ban"; + /// Power Levels event field: kick + pub const KICK: &str = "kick"; + /// Power Levels event field: redact + pub const REDACT: &str = "redact"; + /// Power Levels event field: invite + pub const INVITE: &str = "invite"; +} + +/// Aliases Event Fields +pub mod aliases_field { + /// Aliases event field: aliases + pub const ALIASES: &str = "aliases"; +} + +/// History Visibility Event Fields +pub mod history_visibility_field { + /// History Visibility event field: history_visibility + pub const HISTORY_VISIBILITY: &str = "history_visibility"; +} + +/// Redaction Event Fields +pub mod redaction_field { + /// Redacts event field: redacts + pub const REDACTS: &str = "redacts"; +} diff --git a/rust/src/events/formats/mod.rs b/rust/src/events/formats/mod.rs new file mode 100644 index 00000000000..10b16a5436c --- /dev/null +++ b/rust/src/events/formats/mod.rs @@ -0,0 +1,268 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Over-the-wire representations of Matrix events, parameterised by event +//! format version (i.e. the structure we parse the event JSON into). +//! +//! # Design +//! +//! The shape of an event's JSON varies with the room version, but there are +//! many common fields across all of them — `type`, `sender`, `content`, etc. +//! +//! We model this with a single [`FormattedEvent`] container that is generic +//! over the format-specific tail `E`. Serde `#[serde(flatten)]` merges the +//! common and specific halves into a single JSON object over the wire, while +//! keeping them as distinct structs in Rust. This lets version-agnostic code +//! (field getters, the `unsigned` accessor, …) read [`EventCommonFields`] +//! directly, and only the small amount of version-aware logic (auth-event +//! derivation, room-ID lookup, validation) needs to match on the format. +//! +//! The default `E` parameter is the type-erased [`EventFormatEnum`], which can +//! contain any known format. The [`FormattedEvent::into_general`] method allows +//! converting from a specific format to the general enum. +//! +//! The `signatures` and `unsigned` fields are kept distinct from the +//! common/specific as they allow mutation. When copying an event they need to +//! be deep-copied, but the common/specific fields (which are immutable) can be +//! shared. +//! +//! # Format variants +//! +//! Different room versions have different over-the-wire formats, which is +//! tracked by [`crate::room_versions::RoomVersion::event_format`] field. +//! +//! Each format struct owns only its version-specific fields and any +//! validation/derivation logic; the rest lives in [`EventCommonFields`]. The +//! [`EventFormatEnum`] sum type erases the generic parameter when an `Event` +//! needs to be stored alongside others of unknown room version. +//! +//! Note that any fields not recognised by the format-specific struct or by the +//! common fields are captured into [`EventCommonFields::other_fields`] and +//! round-tripped losslessly. This is useful for capturing optional fields that +//! don't need to be parsed up front. Generally, optional fields should be +//! handled via `other_fields`, as this saves space when they are not present. +//! +//! # Serialization and deserialization +//! +//! Deserializing a Matrix Event from JSON is done by specifying the expected +//! format struct (e.g. [`FormattedEvent`]), which enforces the +//! invariants of that format at parse time. This can then be converted into the +//! version-agnostic [`FormattedEvent`] with the +//! [`FormattedEvent::into_general`] method, which erases the format-specific +//! type but keeps the parsed fields intact. +//! +//! Serializing a [`FormattedEvent`] produces the correct Matrix Event JSON +//! shape for the format variant it contains. + +use std::{collections::HashMap, sync::Arc}; + +use anyhow::Error; +use serde::{Deserialize, Serialize}; + +use crate::{ + events::{json_object::JsonObject, signatures::Signatures, unsigned::Unsigned}, + json::AllowMissing, +}; + +mod v1; +mod v2v3; +mod v4; +mod vmsc4242; + +pub use v1::EventFormatV1; +pub use v2v3::EventFormatV2V3; +pub use v4::EventFormatV4; +pub use vmsc4242::EventFormatVMSC4242; + +/// A parsed Matrix event in its over-the-wire layout. +/// +/// `E` is the format-specific tail. Code that deserialises a known +/// room version picks a concrete `E` (e.g. `FormattedEvent`); +/// the default `Arc` is used once the event has been +/// boxed into the version-agnostic [`Event`](crate::events::Event) +/// pyclass. +/// +/// The `signatures` and `unsigned` fields are kept separate from the other +/// fields as they are mutable (and must be deep-copied if the event is cloned). +/// `common_fields` and `specific_fields` are both `#[serde(flatten)]`ed so that +/// the serialised JSON is a single flat object matching the Matrix spec. +#[derive(Serialize, Deserialize)] +pub struct FormattedEvent> { + /// The event's signatures. + /// + /// Kept separate from common/specific fields as this this is a mutable + /// field. + #[serde(default)] + pub signatures: Signatures, + + /// The event's unsigned data. + /// + /// Kept separate from common/specific fields as this this is a mutable + /// field. + #[serde(default)] + pub unsigned: Unsigned, + + /// The format-specific fields of the event. This is an immutable field. + #[serde(flatten)] + pub specific_fields: E, + + /// The fields common to all event formats. This is an immutable field. + #[serde(flatten)] + pub common_fields: Arc, +} + +impl FormattedEvent { + /// Creates a deep copy of this event, allowing the signatures and unsigned + /// to be mutated without affecting the original. + /// + /// The common and specific fields are shared between the copy and the + /// original, as they are immutable. + pub fn deep_copy(&self) -> FormattedEvent { + FormattedEvent { + signatures: self.signatures.deep_copy(), + unsigned: self.unsigned.deep_copy(), + // These fields can safely be shared among all of the copies as they + // are immutable (they're behind an Arc and so you can't get a + // mutable reference and they have no interior mutability) and these + // write protections extend into Python land as well (i.e. you can't + // accidentally do the wrong thing and mutate) + specific_fields: Arc::clone(&self.specific_fields), + common_fields: Arc::clone(&self.common_fields), + } + } + + pub fn validate(&self) -> Result<(), Error> { + match &*self.specific_fields { + EventFormatEnum::V1(format) => format.validate(&self.common_fields), + EventFormatEnum::V2V3(format) => format.validate(&self.common_fields), + EventFormatEnum::V4(format) => format.validate(&self.common_fields), + EventFormatEnum::VMSC4242(format) => format.validate(&self.common_fields), + } + } +} + +impl FormattedEvent +where + E: Into, +{ + /// Transforms a container of a specific event format into a container of + /// the enum type. + pub fn into_general(self) -> FormattedEvent { + let format: Arc = Arc::new(self.specific_fields.into()); + FormattedEvent { + signatures: self.signatures, + unsigned: self.unsigned, + specific_fields: format, + common_fields: self.common_fields, + } + } +} + +impl From> for FormattedEvent +where + E: Into, +{ + fn from(container: FormattedEvent) -> Self { + container.into_general() + } +} + +/// Fields that appear in every supported event format. +/// +/// Anything not recognised by the format-specific tail or by the fields +/// named here is captured into `other_fields` so events round-trip +/// losslessly even when they carry experimental or future-version +/// keys. +#[derive(Serialize, Deserialize)] +pub struct EventCommonFields { + pub content: JsonObject, + pub depth: i64, + pub hashes: HashMap, Box>, + pub origin_server_ts: i64, + pub sender: Box, + #[serde( + default, + with = "crate::json::allow_missing", + skip_serializing_if = "AllowMissing::is_absent" + )] + pub state_key: AllowMissing>, + + /// The `type` field of the event (we use `type_` in Rust to avoid the + /// reserved keyword). + #[serde(rename = "type")] + pub type_: Box, + + /// All other fields that are not required/parsed by the specific/common + /// fields. This allows us to round-trip events that contain extra fields. + /// + /// Generally, optional fields should be handled via `other_fields`, as this + /// saves space when they are not present. However, that does mean we don't + /// do any type-checking until they get used. + #[serde(flatten)] + pub other_fields: HashMap, serde_json::Value>, +} + +impl EventCommonFields { + /// Helper method to check if the event is a state event and return the + /// tuple of `(type, state_key)` if so. + fn type_state_key_tuple(&self) -> Option<(&str, &str)> { + if let AllowMissing::Some(state_key) = &self.state_key { + Some((&self.type_, state_key)) + } else { + None + } + } +} + +/// Type-erased version-specific tail. +/// +/// Used as the default `E` parameter on [`FormattedEvent`] so the +/// pyclass [`Event`](crate::events::Event) can hold any room version +/// behind a single type. The enum is `#[serde(untagged)]` because the +/// discriminator (the room version) lives outside the JSON; in +/// practice the only direction this is serialised in is `Event -> +/// JSON`, where the chosen variant alone determines the shape. +#[derive(Serialize)] +#[serde(untagged)] +pub enum EventFormatEnum { + V1(EventFormatV1), + V2V3(EventFormatV2V3), + V4(EventFormatV4), + VMSC4242(EventFormatVMSC4242), +} + +impl From for EventFormatEnum { + fn from(format: EventFormatV1) -> Self { + EventFormatEnum::V1(format) + } +} + +impl From for EventFormatEnum { + fn from(format: EventFormatV2V3) -> Self { + EventFormatEnum::V2V3(format) + } +} + +impl From for EventFormatEnum { + fn from(format: EventFormatV4) -> Self { + EventFormatEnum::V4(format) + } +} + +impl From for EventFormatEnum { + fn from(format: EventFormatVMSC4242) -> Self { + EventFormatEnum::VMSC4242(format) + } +} diff --git a/rust/src/events/formats/v1.rs b/rust/src/events/formats/v1.rs new file mode 100644 index 00000000000..09caabc597b --- /dev/null +++ b/rust/src/events/formats/v1.rs @@ -0,0 +1,54 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Event format v1 (room versions 1 and 2). +//! +//! Distinguishing features compared to later formats: +//! +//! - `auth_events` and `prev_events` are `[event_id, hashes]` pairs +//! rather than flat lists of IDs. +//! - `event_id` is carried explicitly in the event JSON, rather than +//! being derived from the canonical-JSON hash. +//! - `room_id` is always present. + +use std::{collections::HashMap, sync::Arc}; + +use anyhow::Error; +use serde::{Deserialize, Serialize}; + +use crate::events::formats::EventCommonFields; + +/// Version-specific fields for room versions 1 and 2. +#[derive(Serialize, Deserialize)] +pub struct EventFormatV1 { + pub auth_events: Vec<(String, HashMap)>, + pub prev_events: Vec<(String, HashMap)>, + pub room_id: Arc, + pub event_id: Arc, +} + +impl EventFormatV1 { + pub fn validate(&self, _common_fields: &EventCommonFields) -> Result<(), Error> { + Ok(()) + } + + pub fn auth_event_ids(&self) -> Vec { + self.auth_events.iter().map(|(id, _)| id.clone()).collect() + } + + pub fn prev_event_ids(&self) -> Vec { + self.prev_events.iter().map(|(id, _)| id.clone()).collect() + } +} diff --git a/rust/src/events/formats/v2v3.rs b/rust/src/events/formats/v2v3.rs new file mode 100644 index 00000000000..a62c68a4e4f --- /dev/null +++ b/rust/src/events/formats/v2v3.rs @@ -0,0 +1,60 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Event format v2/v3 (room versions 3 through 10). +//! +//! Differences from v1: +//! +//! - `auth_events` and `prev_events` are flat `Vec` lists of event IDs +//! rather than `[id, hashes]` pairs. +//! - `event_id` is no longer in the event JSON; it is derived from the +//! canonical-JSON hash at parse time. +//! +//! Note that the difference between event format v2 and v3 is purely in the +//! base64 encoding of the event ID, so the same struct can be used for both +//! formats. +//! +//! [`SimpleAuthPrevEvents`] is shared with [`v4`](super::v4) since the +//! flat-list encoding carries forward unchanged. + +use std::sync::Arc; + +use anyhow::{bail, Error}; +use serde::{Deserialize, Serialize}; + +use crate::events::formats::EventCommonFields; + +/// Version-specific fields for room versions 3-10. +#[derive(Serialize, Deserialize)] +pub struct EventFormatV2V3 { + pub room_id: Arc, + pub auth_events: Vec, + pub prev_events: Vec, +} + +impl EventFormatV2V3 { + pub fn validate(&self, common_fields: &EventCommonFields) -> Result<(), Error> { + // Ensure that we don't have an event_id set. + if common_fields.other_fields.contains_key("event_id") { + bail!("v2/v3 events must not have an explicit event_id"); + } + + Ok(()) + } + + pub fn auth_event_ids(&self) -> Vec { + self.auth_events.clone() + } +} diff --git a/rust/src/events/formats/v4.rs b/rust/src/events/formats/v4.rs new file mode 100644 index 00000000000..22a1a677f29 --- /dev/null +++ b/rust/src/events/formats/v4.rs @@ -0,0 +1,156 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Event format v4 (room version 11). +//! +//! The main change from v2/v3 is that `room_id` becomes optional: an +//! `m.room.create` event no longer carries an explicit room ID, and the +//! room ID is instead *derived* from the create event's ID by replacing +//! the leading `$` with `!`. Conversely, every non-create event still +//! has an explicit `room_id`, and the create event is implicitly +//! included in the auth chain of every non-create event (so it does not +//! need to appear in `auth_events`). +//! +//! [`EventFormatV4::validate`] enforces these invariants at parse time; +//! [`EventFormatV4::room_id`] and [`EventFormatV4::auth_event_ids`] +//! expose the derived values to callers. + +use std::sync::Arc; + +use anyhow::{bail, ensure, Error}; +use serde::{Deserialize, Serialize}; + +use crate::{ + events::{constants::event_type::M_ROOM_CREATE, formats::EventCommonFields}, + json::AllowMissing, +}; + +/// Version-specific fields for room version 11. +#[derive(Serialize, Deserialize)] +pub struct EventFormatV4 { + #[serde( + default, + with = "crate::json::allow_missing", + skip_serializing_if = "AllowMissing::is_absent" + )] + pub room_id: AllowMissing>, + pub auth_events: Vec, + pub prev_events: Vec, +} + +impl EventFormatV4 { + pub fn validate(&self, common_fields: &EventCommonFields) -> Result<(), Error> { + // Ensure that we don't have an event_id set. + if common_fields.other_fields.contains_key("event_id") { + bail!("v4 events must not have an explicit event_id"); + } + + Ok(()) + } + + pub fn room_id( + &self, + event_id: &str, + common_fields: &EventCommonFields, + ) -> Result, Error> { + get_room_id_for_optional_room_id(self.room_id.as_ref_opt(), event_id, common_fields) + } + + pub fn auth_event_ids(&self, common_fields: &EventCommonFields) -> Result, Error> { + let is_create = common_fields.type_state_key_tuple() == Some((M_ROOM_CREATE, "")); + + if is_create { + // The create event itself has no implicit auth events. + return Ok(self.auth_events.clone()); + } + + // For non-create events, the create event is implicitly part of + // the auth chain. Derive its event ID from the room ID by + // replacing the leading '!' with '$'. + let room_id = self + .room_id + .as_deref_opt() + .ok_or_else(|| anyhow::anyhow!("non-create event has no room_id"))?; + + let mut create_event_id = String::with_capacity(room_id.len()); + create_event_id.push('$'); + create_event_id.push_str(&room_id[1..]); + + ensure!( + !self.auth_events.contains(&create_event_id), + "The create event ID is implicitly part of the auth chain and should not be explicitly be in the auth_events" + ); + + let mut auth_events = self.auth_events.clone(); + auth_events.push(create_event_id); + Ok(auth_events) + } +} + +/// Validation helper for v4+ events that can have an optional room ID. +/// +/// Returns the validated room ID (which will be `None` for create events). +pub fn validate_optional_room_id( + room_id: Option<&Arc>, + common_fields: &'_ EventCommonFields, +) -> Result>, Error> { + let is_create_event = common_fields.type_state_key_tuple() == Some((M_ROOM_CREATE, "")); + + match (is_create_event, room_id) { + // For non-create events, room_id must be present. + (false, None) => bail!("non-create event must have a room ID"), + (false, Some(room_id)) => { + // We later derive the create event ID from the room ID by replacing + // the leading '!' with '$', so we require the room ID to start with + // '!'. + ensure!( + room_id.starts_with("!"), + "room_id must start with '!': {}", + room_id + ); + + Ok(Some(Arc::clone(room_id))) + } + + // For create events, room_id must be absent. + (true, Some(_)) => bail!("create event must not have a room ID"), + (true, None) => Ok(None), + } +} + +/// Room ID derivation helper for v4+ events, which can have an optional room +/// ID. +pub fn get_room_id_for_optional_room_id( + room_id: Option<&Arc>, + event_id: &str, + common_fields: &EventCommonFields, +) -> Result, Error> { + match validate_optional_room_id(room_id, common_fields)? { + Some(room_id) => Ok(room_id), + None => { + // This is the create event, where the room ID is derived from the + // event ID by replacing the leading '$' with '!'. + if !event_id.starts_with('$') { + bail!("Create event ID does not start with '$': {}", event_id); + } + + let mut room_id = String::with_capacity(event_id.len()); + room_id.push('!'); + room_id.push_str(&event_id[1..]); + + Ok(room_id.into()) + } + } +} diff --git a/rust/src/events/formats/vmsc4242.rs b/rust/src/events/formats/vmsc4242.rs new file mode 100644 index 00000000000..45d9a250688 --- /dev/null +++ b/rust/src/events/formats/vmsc4242.rs @@ -0,0 +1,94 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Event format for [MSC4242] (prev-state events). +//! +//! Adds `prev_state_events` and removes `auth_events` from the v4 layout +//! — auth chains are derived implicitly from the state DAG rather than +//! carried on each event. `room_id`, `prev_events` and the create-event +//! derivation rules carry over unchanged from v4 and are delegated to +//! [`EventFormatV4::validate`] via a shim that supplies an empty +//! explicit auth list. +//! +//! [MSC4242]: https://github.com/matrix-org/matrix-spec-proposals/pull/4242 + +use std::sync::Arc; + +use anyhow::bail; +use anyhow::Error; +use pyo3::exceptions::PyAssertionError; +use pyo3::PyResult; +use serde::{Deserialize, Serialize}; + +use crate::events::constants::event_type::M_ROOM_CREATE; +use crate::events::formats::v4::get_room_id_for_optional_room_id; +use crate::events::formats::EventCommonFields; +use crate::events::Event; +use crate::json::AllowMissing; + +/// Version-specific fields for the MSC4242 event format. +#[derive(Serialize, Deserialize)] +pub struct EventFormatVMSC4242 { + pub prev_state_events: Vec, + pub prev_events: Vec, + #[serde( + default, + with = "crate::json::allow_missing", + skip_serializing_if = "AllowMissing::is_absent" + )] + pub room_id: AllowMissing>, +} + +impl EventFormatVMSC4242 { + pub fn validate(&self, common_fields: &EventCommonFields) -> Result<(), Error> { + // Ensure that we don't have any `auth_events` or `event_id` fields + // set. + if common_fields.other_fields.contains_key("auth_events") { + bail!("MSC4242 events must not have explicit auth_events"); + } + if common_fields.other_fields.contains_key("event_id") { + bail!("MSC4242 events must not have an explicit event_id"); + } + + Ok(()) + } + + pub fn room_id( + &self, + event_id: &str, + common_fields: &EventCommonFields, + ) -> Result, Error> { + get_room_id_for_optional_room_id(self.room_id.as_ref_opt(), event_id, common_fields) + } + + pub fn auth_event_ids(&self, event: &Event) -> PyResult> { + // In the MSC4242 format, the auth events are calculated and stored in + // internal metadata. + let auth_event_ids = event.internal_metadata.get_calculated_auth_event_ids()?; + + // Catches cases where we accidentally call auth_event_ids() prior to calculating what they + // actually are. The exception being the m.room.create event which has no auth events. + if event.parsed_event.common_fields.type_state_key_tuple() != Some((M_ROOM_CREATE, "")) + && auth_event_ids.is_empty() + { + return Err(PyAssertionError::new_err(format!( + "auth_event_ids has not been calculated for event_id='{}'. This is most likely a Synapse programming error.", + event.event_id + ))); + } + + Ok(auth_event_ids) + } +} diff --git a/rust/src/events/internal_metadata.rs b/rust/src/events/internal_metadata.rs index 4084b8442d2..93f13f655ef 100644 --- a/rust/src/events/internal_metadata.rs +++ b/rust/src/events/internal_metadata.rs @@ -510,7 +510,7 @@ fn attr_err(val: Option, name: &str) -> PyResult { #[pymethods] impl EventInternalMetadata { #[new] - fn new(dict: &Bound<'_, PyDict>) -> PyResult { + pub fn new(dict: &Bound<'_, PyDict>) -> PyResult { let mut data = Vec::with_capacity(dict.len()); for (key, value) in dict.iter() { @@ -536,7 +536,10 @@ impl EventInternalMetadata { }) } - fn copy(&self) -> PyResult { + /// Create a deep copy of this `EventInternalMetadata` to allow modification + /// without affecting other references to the same metadata. This is needed + /// when we clone an event. + pub fn deep_copy(&self) -> PyResult { let guard = self.read_inner()?; Ok(EventInternalMetadata { inner: Arc::new(RwLock::new(guard.clone())), @@ -723,7 +726,7 @@ impl EventInternalMetadata { attr_err(self.read_inner()?.get_redacted(), "redacted") } #[setter] - fn set_redacted(&self, obj: bool) -> PyResult<()> { + pub fn set_redacted(&self, obj: bool) -> PyResult<()> { self.write_inner()?.set_redacted(obj); Ok(()) } @@ -742,7 +745,7 @@ impl EventInternalMetadata { /// The calculated auth event IDs, if it was set when the event was created. #[getter] - fn get_calculated_auth_event_ids(&self) -> PyResult> { + pub fn get_calculated_auth_event_ids(&self) -> PyResult> { let guard = self.read_inner()?; attr_err( guard.get_calculated_auth_event_ids().cloned(), diff --git a/rust/src/events/json_object.rs b/rust/src/events/json_object.rs index 0ab54e8dc56..bb4877d482f 100644 --- a/rust/src/events/json_object.rs +++ b/rust/src/events/json_object.rs @@ -193,6 +193,12 @@ impl JsonObject { } } +impl JsonObject { + pub fn get_field(&self, key: &str) -> Option<&serde_json::Value> { + self.object.get(key) + } +} + /// Helper class returned by `JsonObject.keys()` to act as a view into the keys /// of the object. /// diff --git a/rust/src/events/mod.rs b/rust/src/events/mod.rs index e60cdb70786..ad3f61e2fdf 100644 --- a/rust/src/events/mod.rs +++ b/rust/src/events/mod.rs @@ -18,18 +18,77 @@ * */ -//! Classes for representing Events. +//! Classes for representing Matrix events. +//! +//! # Overview +//! +//! A Matrix event has a JSON shape that varies by *room version*. The +//! per-room-version shape is captured in the [`formats`] module, where +//! [`FormattedEvent`] is a generic container parametrised by the +//! room-version-specific portion (`EventFormatV1`, `EventFormatV2V3`, +//! `EventFormatV4`, `EventFormatVMSC4242`). See [`formats`] for the layout +//! of the over-the-wire JSON and how the room-version-agnostic fields are +//! split from the version-specific ones. +//! +//! [`Event`] is the `pyclass` exposed to Python. It bundles a fully parsed +//! [`FormattedEvent`] (with the version-specific part type-erased as +//! [`formats::EventFormatEnum`]) together with the pieces of state that +//! live alongside the event JSON in Synapse: +//! +//! - `event_id` — either taken from the event JSON (format v1) or derived +//! from the canonical-JSON hash (v2+); computed once at construction +//! time and cached. +//! - `room_version` — a `'static` reference into the global room-version +//! table, used to drive format-dependent behaviour (e.g. where the +//! `redacts` field lives, which redaction rules apply). +//! - `internal_metadata` — Synapse-internal flags that are *not* part of +//! the federated event (outlier status, soft-failure, stream positions, +//! …). These come from a separate dict at construction time. +//! - `rejected_reason` — `None` for accepted events; otherwise a short +//! string describing why auth rejected the event. +//! +use std::sync::Arc; + +use anyhow::Error; use pyo3::{ - types::{PyAnyMethods, PyMapping, PyModule, PyModuleMethods}, - wrap_pyfunction, Bound, PyResult, Python, + exceptions::{PyAttributeError, PyKeyError, PyValueError}, + pyclass, pyfunction, pymethods, + types::{PyAnyMethods, PyDict, PyDictMethods, PyList, PyMapping, PyModule, PyModuleMethods}, + wrap_pyfunction, Bound, IntoPyObject, PyAny, PyResult, Python, }; +use pythonize::{depythonize, pythonize}; +use crate::events::{ + formats::{ + EventFormatEnum, EventFormatV1, EventFormatV2V3, EventFormatV4, EventFormatVMSC4242, + FormattedEvent, + }, + signatures::Signatures, + unsigned::Unsigned, + utils::redact, +}; +use crate::{ + duration::SynapseDuration, + events::{ + constants::event_field::{HASHES, MSC4354_STICKY, SIGNATURES, UNSIGNED}, + constants::membership_field::MEMBERSHIP, + constants::redaction_field::REDACTS, + constants::unsigned_field::{AGE, AGE_TS, REDACTED_BECAUSE}, + internal_metadata::EventInternalMetadata, + utils::calculate_event_id, + }, + room_versions::{EventFormatVersions, RoomVersion}, +}; + +pub mod constants; pub mod filter; -mod internal_metadata; -mod json_object; +pub mod formats; +pub mod internal_metadata; +pub mod json_object; pub mod signatures; pub mod unsigned; +pub mod utils; use json_object::JsonObject; @@ -39,14 +98,17 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> PyMapping::register::(py)?; let child_module = PyModule::new(py, "events")?; - child_module.add_class::()?; - child_module.add_class::()?; - child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; child_module.add_class::()?; child_module.add_class::()?; child_module.add_class::()?; child_module.add_class::()?; + child_module.add_class::()?; child_module.add_function(wrap_pyfunction!(filter::event_visible_to_server_py, m)?)?; + child_module.add_function(wrap_pyfunction!(redact_event_py, m)?)?; + child_module.add_function(wrap_pyfunction!(redact_event_dict, m)?)?; m.add_submodule(&child_module)?; @@ -58,3 +120,813 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> Ok(()) } + +/// The Rust-side representation of a Matrix event, exposed to Python. +/// +/// Wraps a parsed [`FormattedEvent`] together with the per-event state +/// that Synapse tracks outside the event JSON (event ID, internal +/// metadata, rejection reason, and a reference to the room version that +/// produced this event). See the module-level docs for the high-level +/// design. +#[pyclass(frozen, weakref)] +pub struct Event { + /// The parsed event JSON. + parsed_event: FormattedEvent, + + /// The event ID. For format v1 this is read directly from the JSON; + /// for v2+ it is computed from the canonical-JSON hash at + /// construction time and cached here. + event_id: Arc, + + /// The calculated room ID. + /// + /// For some room versions, this may be derived, e.g. for create events in + /// v4. + room_id: Arc, + + /// Synapse-internal per-event state that lives outside the federated + /// JSON (e.g. outlier flag, soft-failure, stream positions). + #[pyo3(get)] + internal_metadata: EventInternalMetadata, + + /// The room version this event was parsed for. + #[pyo3(get)] + room_version: &'static RoomVersion, + + /// `None` for accepted events; otherwise a short reason set by auth + /// when the event was rejected. + rejected_reason: Option>, +} + +#[pymethods] +impl Event { + #[new] + fn new_from_py<'a, 'py>( + py: Python<'py>, + event_dict: &'a Bound<'py, PyAny>, + room_version: &'a Bound<'py, PyAny>, + internal_metadata_dict: &'a Bound<'py, PyDict>, + rejected_reason: Option, + ) -> PyResult { + let room_version: &RoomVersion = { + let r = room_version.getattr("identifier")?; + let room_version_str = r.extract::<&str>()?; + room_version_str + .parse() + .map_err(|e| PyValueError::new_err(format!("Unsupported room version: {}", e)))? + }; + + let rejected_reason = rejected_reason.map(String::into_boxed_str); + + // Parse the event dict into a FormattedEvent, converting any failures to + // a `ValueError`. + let parsed_event = depythonize_event_dict(room_version, event_dict).map_err(|err| { + let new_err = PyValueError::new_err(format!( + "Failed to parse event for room version {}", + room_version + )); + new_err.set_cause(py, Some(err)); + new_err + })?; + + let internal_metadata = EventInternalMetadata::new(internal_metadata_dict)?; + + let event_id = match &*parsed_event.specific_fields { + EventFormatEnum::V1(format) => { + // V1/V2 events have the event_id in the event dict. + Arc::clone(&format.event_id) + } + _ => { + // Calculate the event ID by hashing the event JSON. This can + // fail if the event can't be serialized to canonical JSON (e.g. + // having out-of-range integers), which we report as + // `ValueError` as it indicates the event is invalid. + let event_value = serde_json::to_value(&parsed_event).map_err(|err| { + PyValueError::new_err(format!("Failed to serialize event: {}", err)) + })?; + calculate_event_id(&event_value, room_version) + .map_err(|err| { + PyValueError::new_err(format!("Failed to calculate event_id: {}", err)) + })? + .into() + } + }; + + let room_id = match &*parsed_event.specific_fields { + EventFormatEnum::V1(format) => Arc::clone(&format.room_id), + EventFormatEnum::V2V3(format) => Arc::clone(&format.room_id), + EventFormatEnum::V4(format) => format + .room_id(&event_id, &parsed_event.common_fields) + .map_err(|err| { + PyValueError::new_err(format!( + "Failed to calculate room_id for event {}: {}", + event_id, err + )) + })?, + EventFormatEnum::VMSC4242(format) => format + .room_id(&event_id, &parsed_event.common_fields) + .map_err(|err| { + PyValueError::new_err(format!( + "Failed to calculate room_id for event {}: {}", + event_id, err + )) + })?, + }; + + Ok(Self { + parsed_event, + + event_id, + room_id, + room_version, + rejected_reason, + internal_metadata, + }) + } + + /// Convert the event to a dictionary suitable for serialisation. + fn get_dict<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(pythonize(py, &self.parsed_event)?) + } + + /// Like `get_dict`, but serializes `unsigned` in a form suitable for + /// persistence. + fn get_dict_for_persistence<'py>(&self, py: Python<'py>) -> PyResult> { + let binding = self.get_dict(py)?; + let dict = binding.cast::()?; + + dict.set_item("unsigned", self.parsed_event.unsigned.for_persistence(py)?)?; + + Ok(binding) + } + + /// Like [`Event::get_dict`], but serializes `unsigned` in a form suitable + /// for sending over federation. + #[pyo3(signature = (time_now = None))] + fn get_pdu_json<'py>( + &self, + py: Python<'py>, + time_now: Option, + ) -> PyResult> { + let obj = self.get_dict(py)?; + let dict = obj.cast::()?; + + // Get or create the unsigned dict + if let Ok(Some(unsigned)) = dict.get_item(UNSIGNED) { + let unsigned = unsigned.cast::()?; + + if let Some(time_now) = time_now { + if let Ok(Some(age_ts)) = unsigned.get_item(AGE_TS) { + let age = time_now - age_ts.extract::()?; + unsigned.set_item(AGE, age)?; + unsigned.del_item(AGE_TS)?; + } + } + + // This may be a frozen event + unsigned.del_item(REDACTED_BECAUSE).ok(); + } + + Ok(obj) + } + + /// Like [`Event::get_dict`], except strips fields like `signatures`, + /// `hashes` and `unsigned` so that the result is suitable as a template for + /// creating new events. Used in make_{join,leave,knock} flows. + fn get_templated_pdu_json<'py>(&self, py: Python<'py>) -> PyResult> { + // Use get_dict but strip signatures, unsigned, and hashes — the + // joining/leaving/knocking server will re-sign and recalculate hashes. + let obj = self.get_dict(py)?; + let dict = obj.cast::()?; + dict.del_item(SIGNATURES).ok(); + dict.del_item(UNSIGNED).ok(); + dict.del_item(HASHES).ok(); + + Ok(obj) + } + + #[getter] + fn rejected_reason(&self) -> Option<&str> { + self.rejected_reason.as_deref() + } + + /// Returns the list of prev event IDs. The order matches the order + /// specified in the event, though there is no meaning to it. + fn prev_event_ids(&self) -> Vec { + match &*self.parsed_event.specific_fields { + EventFormatEnum::V1(format) => format.prev_event_ids(), + EventFormatEnum::V2V3(format) => format.prev_events.clone(), + EventFormatEnum::V4(format) => format.prev_events.clone(), + EventFormatEnum::VMSC4242(format) => format.prev_events.clone(), + } + } + + /// Returns the list of auth event IDs. The order matches the order + /// specified in the event, though there is no meaning to it. + fn auth_event_ids(&self) -> PyResult> { + match &*self.parsed_event.specific_fields { + EventFormatEnum::V1(format) => Ok(format.auth_event_ids()), + EventFormatEnum::V2V3(format) => Ok(format.auth_event_ids()), + EventFormatEnum::V4(format) => { + Ok(format.auth_event_ids(&self.parsed_event.common_fields)?) + } + EventFormatEnum::VMSC4242(format) => Ok(format.auth_event_ids(self)?), + } + } + + #[getter] + fn membership<'py>(&self, py: Python<'py>) -> PyResult> { + let content = self.content(); + let value = content.get_field(MEMBERSHIP); + match value { + Some(value) => Ok(pythonize(py, value)?), + None => Err(PyKeyError::new_err(MEMBERSHIP)), + } + } + + fn is_state(&self) -> bool { + self.parsed_event.common_fields.state_key.is_some() + } + + /// Get the state key of this event, or None if it's not a state event. + fn get_state_key(&self) -> Option<&str> { + self.parsed_event.common_fields.state_key.as_deref_opt() + } + + /// The EventFormatVersion implemented by this event. + #[getter] + fn format_version(&self) -> i32 { + self.room_version.event_format + } + + /// Returns a deep copy of this object, such that modifying the copy will + /// not affect the original. + fn deep_copy(&self) -> PyResult { + let internal_metadata = self.internal_metadata.deep_copy()?; + + let new_event = Event { + parsed_event: self.parsed_event.deep_copy(), + internal_metadata, + room_version: self.room_version, + rejected_reason: self.rejected_reason.clone(), + event_id: self.event_id.clone(), + room_id: self.room_id.clone(), + }; + Ok(new_event) + } + + /// If this event has the `msc4354_sticky` top-level field, returns a + /// `SynapseDuration` representing the sticky duration. Otherwise returns + /// `None`. + fn sticky_duration(&self) -> Option { + const MAX_DURATION: SynapseDuration = SynapseDuration::from_hours(1); + + let sticky_obj = self + .parsed_event + .common_fields + .other_fields + .get(MSC4354_STICKY); + + let sticky_obj = match sticky_obj { + Some(serde_json::Value::Object(obj)) => obj, + _ => return None, + }; + + // Check for a valid duration field. The MSC requires `duration_ms` to + // be a non-negative integer. If it's missing or invalid, we treat the + // event as non-sticky by returning `None`. + let duration_ms = sticky_obj.get("duration_ms")?.as_u64()?; + + let duration = SynapseDuration::from_milliseconds(duration_ms); + + let duration = std::cmp::min(duration, MAX_DURATION); + + Some(duration) + } + + // Below are the methods for interacting with the event as a mapping. + // + // These are rarely used, so we take the easy approach of re-serializing the + // event to a Python dict and then delegating to the standard dict methods. + // We can't remove these functions as third-party modules may rely on them. + + fn __contains__<'py>(&self, py: Python<'py>, key: &str) -> PyResult { + let dict = self.get_dict(py)?; + dict.contains(key) + } + + /// This is deprecated in favor of `get`, but we still need to support it + /// for backwards compatibility with modules. This is therefore not exposed + /// in the type stubs. + fn __getitem__<'py>(&self, py: Python<'py>, key: &str) -> PyResult> { + let dict = self.get_dict(py)?; + if dict.contains(key)? { + dict.get_item(key) + } else { + Err(PyKeyError::new_err(key.to_owned())) + } + } + + #[pyo3(signature = (key, default=None))] + fn get<'py>( + &self, + py: Python<'py>, + key: &str, + default: Option>, + ) -> PyResult> { + let dict = self.get_dict(py)?; + if dict.contains(key)? { + dict.get_item(key) + } else { + Ok(default.into_pyobject(py)?) + } + } + + fn items<'py>(&self, py: Python<'py>) -> PyResult> { + let dict = self.get_dict(py)?; + let dict = dict.cast::()?; + Ok(dict.items()) + } + + fn keys<'py>(&self, py: Python<'py>) -> PyResult> { + let dict = self.get_dict(py)?; + let dict = dict.cast::()?; + Ok(dict.keys()) + } + + // Below are the getters for the top-level fields on Matrix events. + + #[getter] + fn event_id(&self) -> &str { + &self.event_id + } + + #[getter] + fn room_id(&self) -> &str { + &self.room_id + } + + #[getter] + fn signatures(&self) -> Signatures { + self.parsed_event.signatures.clone() + } + + #[getter] + fn content(&self) -> JsonObject { + self.parsed_event.common_fields.content.clone() + } + + #[getter] + fn depth(&self) -> i64 { + self.parsed_event.common_fields.depth + } + + #[getter] + fn hashes<'py>(&self, py: Python<'py>) -> PyResult> { + let dict = PyDict::new(py); + for (key, value) in &self.parsed_event.common_fields.hashes { + dict.set_item(&**key, &**value)?; + } + Ok(dict) + } + + #[getter] + fn origin_server_ts(&self) -> i64 { + self.parsed_event.common_fields.origin_server_ts + } + + #[getter] + fn sender(&self) -> &str { + &self.parsed_event.common_fields.sender + } + + /// Deprecated alias for `sender`. Kept for backwards compatibility with + /// modules and tests that still read `event.user_id`. This is therefore not + /// exposed in the type stubs. + #[getter] + fn user_id(&self) -> &str { + &self.parsed_event.common_fields.sender + } + + #[getter(state_key)] + // We can't call this `state_key` because that would generate a + // `get_state_key` method which already exists. + fn state_key_attr(&self) -> PyResult<&str> { + let Some(state_key) = self.parsed_event.common_fields.state_key.as_deref_opt() else { + return Err(PyAttributeError::new_err("state_key")); + }; + Ok(state_key) + } + + #[getter] + fn r#type(&self) -> &str { + &self.parsed_event.common_fields.type_ + } + + #[getter] + fn unsigned(&self) -> Unsigned { + self.parsed_event.unsigned.clone() + } + + #[getter] + fn prev_state_events(&self) -> PyResult> { + // `prev_state_events` should only be called after validating the event + // is of a format that supports MSC4242, so we return an AttributeError + // for formats that don't support it. + match &*self.parsed_event.specific_fields { + EventFormatEnum::V1(_) | EventFormatEnum::V2V3(_) | EventFormatEnum::V4(_) => { + Err(PyAttributeError::new_err("prev_state_events")) + } + EventFormatEnum::VMSC4242(format) => Ok(format.prev_state_events.clone()), + } + } + + #[getter] + fn redacts<'py>(&self, py: Python<'py>) -> PyResult>> { + let common = &self.parsed_event.common_fields; + let value = if self.room_version.updated_redaction_rules { + common.content.get_field(REDACTS) + } else { + common.other_fields.get(REDACTS) + }; + value + .map(|v| pythonize(py, v).map_err(Into::into)) + .transpose() + } +} + +fn depythonize_event_dict( + room_version: &RoomVersion, + event_dict: &Bound<'_, PyAny>, +) -> PyResult { + let formatted_event: FormattedEvent = match room_version.event_format { + EventFormatVersions::ROOM_V1_V2 => { + let event_format: FormattedEvent = depythonize(event_dict)?; + + event_format.into() + } + EventFormatVersions::ROOM_V3 | EventFormatVersions::ROOM_V4_PLUS => { + let event_format: FormattedEvent = depythonize(event_dict)?; + event_format.into() + } + EventFormatVersions::ROOM_V11_HYDRA_PLUS => { + let event_format: FormattedEvent = depythonize(event_dict)?; + event_format.into() + } + EventFormatVersions::ROOM_VMSC4242 => { + let event_format: FormattedEvent = depythonize(event_dict)?; + event_format.into() + } + _ => { + return Err(PyValueError::new_err(format!( + "Unsupported room version: {}", + room_version + ))) + } + }; + + formatted_event.validate()?; + + Ok(formatted_event) +} + +/// Converts an event dict as [`serde_json::Value`] into a [`FormattedEvent`]. +fn event_dict_from_json_value( + room_version: &RoomVersion, + event_dict: serde_json::Value, +) -> Result { + let formatted_event: FormattedEvent = match room_version.event_format { + EventFormatVersions::ROOM_V1_V2 => { + let event_format: FormattedEvent = serde_json::from_value(event_dict)?; + + event_format.into() + } + EventFormatVersions::ROOM_V3 | EventFormatVersions::ROOM_V4_PLUS => { + let event_format: FormattedEvent = serde_json::from_value(event_dict)?; + event_format.into() + } + EventFormatVersions::ROOM_V11_HYDRA_PLUS => { + let event_format: FormattedEvent = serde_json::from_value(event_dict)?; + event_format.into() + } + EventFormatVersions::ROOM_VMSC4242 => { + let event_format: FormattedEvent = + serde_json::from_value(event_dict)?; + event_format.into() + } + _ => { + return Err(anyhow::anyhow!( + "Unsupported room version: {}", + room_version + )); + } + }; + + formatted_event.validate()?; + + Ok(formatted_event) +} + +/// Returns a pruned version of the given event, which removes all keys we don't +/// know about or think could potentially be dodgy. +/// +/// Returns the redacted event as a dict. +#[pyfunction(name = "redact_event")] +fn redact_event_py(event: &Event) -> PyResult { + let event_value = serde_json::to_value(&event.parsed_event).map_err(|err| { + PyValueError::new_err(format!("Failed to serialize event for redaction: {}", err)) + })?; + + let redacted_value = redact(&event_value, event.room_version)?; + let redacted_formatted_event = event_dict_from_json_value(event.room_version, redacted_value) + .map_err(|err| { + PyValueError::new_err(format!("Failed to deserialize redacted event: {}", err)) + })?; + + let redacted_event = Event { + parsed_event: redacted_formatted_event, + event_id: Arc::clone(&event.event_id), + room_id: Arc::clone(&event.room_id), + room_version: event.room_version, + rejected_reason: event.rejected_reason.clone(), + internal_metadata: event.internal_metadata.deep_copy()?, + }; + + // Mark event as redacted + redacted_event.internal_metadata.set_redacted(true)?; + + Ok(redacted_event) +} + +/// Returns a pruned version of the given event dict, which removes all keys we +/// don't know about or think could potentially be dodgy. +/// +/// Returns the redacted event as a dict. +#[pyfunction(name = "redact_event_dict")] +fn redact_event_dict<'py>( + py: Python<'py>, + room_version: &RoomVersion, + event_dict: &'py Bound<'py, PyAny>, +) -> PyResult> { + let event_value = depythonize(event_dict)?; + + let redacted = redact(&event_value, room_version)?; + + let redacted_py = pythonize(py, &redacted)?; + + Ok(redacted_py) +} + +#[cfg(test)] +mod tests { + + use super::*; + use crate::events::{ + constants::event_type::M_ROOM_CREATE, + formats::{EventFormatV1, EventFormatV2V3, EventFormatV4, EventFormatVMSC4242}, + }; + + #[test] + fn test_basic_v3_roundtrip() { + let json = r#"{"auth_events":[],"prev_events":[],"type":"m.room.create","sender":"@anon-20260225_142731-20:localhost:8800","content":{"room_version":"10","creator":"@anon-20260225_142731-20:localhost:8800"},"depth":1,"room_id":"!qVoJSympOqdUQRUfiC:localhost:8800","state_key":"","origin_server_ts":1772029657149,"hashes":{"sha256":"RIDkn4CrExGMOfRZlHl//1weAro5QC/q2D76YcyAUqk"},"signatures":{"localhost:8800":{"ed25519:a_GMSl":"GU7WmvI2Kd5kLrXKrWpRbUfEiVKGgH0sxQNEpBMMvgF3QhHN25AubVMmIClht5r/c+Iihb1xsq1j5Sw+RGfiDg"}},"unsigned":{"age_ts":1772029657149}}"#; + let event_value: serde_json::Value = serde_json::from_str(json).unwrap(); + + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + let parsed_value = serde_json::to_value(&event).unwrap(); + + // Check a couple of fields are as expected as a sanity check. + assert_eq!(&*event.common_fields.type_, M_ROOM_CREATE); + assert_eq!( + &*event.specific_fields.room_id, + "!qVoJSympOqdUQRUfiC:localhost:8800" + ); + + assert_eq!(event_value, parsed_value); + } + + #[test] + fn test_room_id_for_create_event_format_v4() { + let json = r#"{"auth_events":[],"prev_events":[],"type":"m.room.create","sender":"@erikj:jki.re","content":{"room_version":"12","predecessor":{"room_id":"!VuNGkDTdbMOOxSmuDa:jki.re"}},"depth":1,"state_key":"","origin_server_ts":1775568141481,"hashes":{"sha256":"qBX+glsKvogXFrvsEN0eh13pO2kpuE6o/b4yREPtOqw"},"signatures":{"jki.re":{"ed25519:auto":"n/4gHQRagk3r1r24L/7a+oaMMf9cysVfQRYdjpDZcf4ppkVym33rhTW18Vy4zMa1L5nsWLkxsBvbrRRDYUOhBQ"}},"unsigned":{"age_ts":1775568141481}}"#; + let event_value: serde_json::Value = serde_json::from_str(json).unwrap(); + + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + + let event_id = calculate_event_id(&event_value, &RoomVersion::V12).unwrap(); + + assert_eq!( + &*event + .specific_fields + .room_id(&event_id, &event.common_fields) + .unwrap(), + "!BeXKh925K_M46DwsuJFR0EyBpE1P7CFUDGuWW4xw55Y" + ); + } + + #[test] + fn test_basic_v1_roundtrip() { + let json = r#"{"auth_events":[["$auth1:localhost",{"sha256":"abc"}],["$auth2:localhost",{"sha256":"def"}]],"prev_events":[["$prev1:localhost",{"sha256":"ghi"}]],"type":"m.room.message","sender":"@user:localhost","content":{"body":"hello","msgtype":"m.text"},"depth":5,"room_id":"!room:localhost","event_id":"$event1:localhost","origin_server_ts":1234567890,"hashes":{"sha256":"base64hash"},"signatures":{"localhost":{"ed25519:key":"sig"}},"unsigned":{}}"#; + let event_value: serde_json::Value = serde_json::from_str(json).unwrap(); + + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + let parsed_value = serde_json::to_value(&event).unwrap(); + + // Check a few fields are as expected as a sanity check. + assert_eq!(&*event.common_fields.type_, "m.room.message"); + assert!(event.common_fields.state_key.is_absent()); + assert_eq!(&*event.specific_fields.room_id, "!room:localhost"); + assert_eq!(&*event.specific_fields.event_id, "$event1:localhost"); + + // Check auth/prev event extraction + let auth_ids = event.specific_fields.auth_event_ids(); + assert_eq!(auth_ids, vec!["$auth1:localhost", "$auth2:localhost"]); + + let prev_ids = event.specific_fields.prev_event_ids(); + assert_eq!(prev_ids, vec!["$prev1:localhost"]); + + assert_eq!(event_value, parsed_value); + } + + #[test] + fn test_basic_v4_roundtrip_with_room_id() { + // A regular (non-create) V4 event has an explicit room_id. + let json = r#"{"auth_events":["$auth1","$auth2"],"prev_events":["$prev1"],"type":"m.room.message","sender":"@user:localhost","content":{"body":"hello","msgtype":"m.text"},"depth":5,"room_id":"!room:localhost","origin_server_ts":1234567890,"hashes":{"sha256":"base64hash"},"signatures":{"localhost":{"ed25519:key":"sig"}},"unsigned":{}}"#; + let event_value: serde_json::Value = serde_json::from_str(json).unwrap(); + + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + let parsed_value = serde_json::to_value(&event).unwrap(); + + // Check a few fields are as expected as a sanity check. + assert_eq!(&*event.common_fields.type_, "m.room.message"); + assert_eq!( + event.specific_fields.room_id.as_deref_opt(), + Some("!room:localhost") + ); + assert_eq!( + event.specific_fields.auth_events, + vec!["$auth1".to_string(), "$auth2".to_string()] + ); + assert_eq!( + event.specific_fields.prev_events, + vec!["$prev1".to_string()] + ); + + assert_eq!(event_value, parsed_value); + } + + #[test] + fn test_basic_v4_roundtrip_create_event() { + // A V4 create event for a v12 room has no room_id field. + let json = r#"{"auth_events":[],"prev_events":[],"type":"m.room.create","sender":"@erikj:jki.re","content":{"room_version":"12"},"depth":1,"state_key":"","origin_server_ts":1775568141481,"hashes":{"sha256":"qBX+glsKvogXFrvsEN0eh13pO2kpuE6o/b4yREPtOqw"},"signatures":{"jki.re":{"ed25519:auto":"sig"}},"unsigned":{}}"#; + let event_value: serde_json::Value = serde_json::from_str(json).unwrap(); + + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + let parsed_value = serde_json::to_value(&event).unwrap(); + + // Check a few fields are as expected as a sanity check. + assert!(event.specific_fields.room_id.is_absent()); + assert_eq!(&*event.common_fields.type_, M_ROOM_CREATE); + + // Create events have no implicit auth events. + assert!(event + .specific_fields + .auth_event_ids(&event.common_fields) + .unwrap() + .is_empty()); + + assert_eq!(event_value, parsed_value); + } + + #[test] + fn test_v4_auth_event_ids_implicit_create() { + // Non-create events implicitly include the create event (derived from + // the room ID) in their auth chain. + let json = r#"{"auth_events":["$auth1"],"prev_events":["$prev1"],"type":"m.room.message","sender":"@user:localhost","content":{"body":"hi","msgtype":"m.text"},"depth":5,"room_id":"!BeXKh925K_M46DwsuJFR0EyBpE1P7CFUDGuWW4xw55Y","origin_server_ts":1234567890,"hashes":{"sha256":"h"},"signatures":{"localhost":{"ed25519:k":"s"}},"unsigned":{}}"#; + + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + + let auth_ids = event + .specific_fields + .auth_event_ids(&event.common_fields) + .unwrap(); + assert_eq!( + auth_ids, + vec![ + "$auth1".to_string(), + "$BeXKh925K_M46DwsuJFR0EyBpE1P7CFUDGuWW4xw55Y".to_string(), + ] + ); + } + + #[test] + fn test_v4_validate_rejects_missing_room_id_for_non_create() { + // A v12 non-create event without a room_id must fail validation. + let json = r#"{"auth_events":[],"prev_events":[],"type":"m.room.message","sender":"@u:l","content":{},"depth":2,"state_key":"","origin_server_ts":1,"hashes":{"sha256":"h"},"signatures":{"l":{"ed25519:k":"s"}},"unsigned":{}}"#; + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + assert!(event + .specific_fields + .validate(&event.common_fields) + .is_err()); + } + + #[test] + fn test_v4_validate_accepts_create_without_room_id() { + let json = r#"{"auth_events":[],"prev_events":[],"type":"m.room.create","sender":"@u:l","content":{"room_version":"12"},"depth":1,"state_key":"","origin_server_ts":1,"hashes":{"sha256":"h"},"signatures":{"l":{"ed25519:k":"s"}},"unsigned":{}}"#; + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + event + .specific_fields + .validate(&event.common_fields) + .unwrap(); + } + + #[test] + fn test_basic_vmsc4242_roundtrip() { + // VMSC4242 introduces a `prev_state_events` field on top of V4. + let json = r#"{"auth_events":["$auth1"],"prev_events":["$prev1"],"prev_state_events":["$pstate1","$pstate2"],"type":"m.room.member","sender":"@user:localhost","content":{"membership":"join"},"depth":5,"room_id":"!room:localhost","state_key":"@user:localhost","origin_server_ts":1234567890,"hashes":{"sha256":"h"},"signatures":{"localhost":{"ed25519:k":"s"}},"unsigned":{}}"#; + let event_value: serde_json::Value = serde_json::from_str(json).unwrap(); + + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + let parsed_value = serde_json::to_value(&event).unwrap(); + + assert_eq!( + event.specific_fields.prev_state_events, + vec!["$pstate1".to_string(), "$pstate2".to_string()] + ); + assert_eq!( + event.specific_fields.room_id.as_deref_opt(), + Some("!room:localhost") + ); + assert_eq!( + event.common_fields.state_key.as_deref_opt(), + Some("@user:localhost") + ); + + assert_eq!(event_value, parsed_value); + } + + #[test] + fn test_vmsc4242_room_id_for_create_event() { + let json = r#"{"auth_events":[],"prev_events":[],"prev_state_events":[],"type":"m.room.create","sender":"@erikj:jki.re","content":{"room_version":"12","predecessor":{"room_id":"!VuNGkDTdbMOOxSmuDa:jki.re"}},"depth":1,"state_key":"","origin_server_ts":1775568141481,"hashes":{"sha256":"qBX+glsKvogXFrvsEN0eh13pO2kpuE6o/b4yREPtOqw"},"signatures":{"jki.re":{"ed25519:auto":"n/4gHQRagk3r1r24L/7a+oaMMf9cysVfQRYdjpDZcf4ppkVym33rhTW18Vy4zMa1L5nsWLkxsBvbrRRDYUOhBQ"}},"unsigned":{"age_ts":1775568141481}}"#; + let event_value: serde_json::Value = serde_json::from_str(json).unwrap(); + + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + + // The event_id calculation is independent of the `prev_state_events` + // field not being present in V4, so the same event_id derivation works. + let event_id = calculate_event_id(&event_value, &RoomVersion::V12).unwrap(); + + assert_eq!( + &*event + .specific_fields + .room_id(&event_id, &event.common_fields) + .unwrap(), + "!BeXKh925K_M46DwsuJFR0EyBpE1P7CFUDGuWW4xw55Y" + ); + } + + #[test] + fn test_event_format_enum_untagged_roundtrip() { + // The untagged EventFormatEnum serialization/deserialization is + // driven by fields, so serializing any variant must match the + // original JSON exactly. + let v2v3_json = r#"{"auth_events":[],"prev_events":[],"type":"m.room.create","sender":"@a:b","content":{},"depth":1,"room_id":"!r:b","state_key":"","origin_server_ts":1,"hashes":{"sha256":"h"},"signatures":{"b":{"ed25519:k":"s"}},"unsigned":{}}"#; + let v2v3_value: serde_json::Value = serde_json::from_str(v2v3_json).unwrap(); + let v2v3_container: FormattedEvent = + serde_json::from_str(v2v3_json).unwrap(); + assert_eq!(serde_json::to_value(&v2v3_container).unwrap(), v2v3_value); + assert_eq!( + serde_json::to_value(v2v3_container.into_general()).unwrap(), + v2v3_value + ); + + let v4_json = r#"{"auth_events":[],"prev_events":[],"type":"m.room.create","sender":"@a:b","content":{"room_version":"12"},"depth":1,"state_key":"","origin_server_ts":1,"hashes":{"sha256":"h"},"signatures":{"b":{"ed25519:k":"s"}},"unsigned":{}}"#; + let v4_value: serde_json::Value = serde_json::from_str(v4_json).unwrap(); + let v4_container: FormattedEvent = serde_json::from_str(v4_json).unwrap(); + assert_eq!(serde_json::to_value(&v4_container).unwrap(), v4_value); + assert_eq!( + serde_json::to_value(v4_container.into_general()).unwrap(), + v4_value + ); + } + + #[test] + fn test_unknown_top_level_fields_preserved_roundtrip() { + // Extra top-level fields (e.g. unknown or experimental) are captured + // via `other_fields` and must round-trip losslessly. + let json = r#"{"auth_events":[],"prev_events":[],"type":"m.room.message","sender":"@a:b","content":{"body":"hi","msgtype":"m.text"},"depth":1,"room_id":"!r:b","origin_server_ts":1,"hashes":{"sha256":"h"},"signatures":{"b":{"ed25519:k":"s"}},"unsigned":{},"msc4354_sticky":{"duration_ms":5000},"some_unknown_field":"some_value"}"#; + let event_value: serde_json::Value = serde_json::from_str(json).unwrap(); + + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + let parsed_value = serde_json::to_value(&event).unwrap(); + + assert!(event + .common_fields + .other_fields + .contains_key(MSC4354_STICKY)); + assert!(event + .common_fields + .other_fields + .contains_key("some_unknown_field")); + + assert_eq!(event_value, parsed_value); + } +} diff --git a/rust/src/events/signatures.rs b/rust/src/events/signatures.rs index 0f2acd5c9bb..fa07dd056bf 100644 --- a/rust/src/events/signatures.rs +++ b/rust/src/events/signatures.rs @@ -30,12 +30,25 @@ use serde::{Deserialize, Serialize}; /// A class representing the signatures on an event. #[pyclass(frozen, skip_from_py_object)] -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] #[serde(transparent)] pub struct Signatures { inner: Arc>>>, } +impl Signatures { + /// Create a deep copy of this `Signatures` to allow modification without + /// affecting other references to the same signatures. This is needed when + /// we clone an event. + pub fn deep_copy(&self) -> Self { + let signatures = self.inner.read().expect("lock poisoned").clone(); // Deep copy the inner map + + Self { + inner: Arc::new(RwLock::new(signatures)), + } + } +} + #[pymethods] impl Signatures { #[new] diff --git a/rust/src/events/unsigned.rs b/rust/src/events/unsigned.rs index 5aa56812c03..0bb644ee909 100644 --- a/rust/src/events/unsigned.rs +++ b/rust/src/events/unsigned.rs @@ -101,6 +101,15 @@ impl Unsigned { .write() .map_err(|_| PyRuntimeError::new_err("Unsigned lock poisoned")) } + + /// Create a deep copy of this `Unsigned` to allow modification without + /// affecting other references to the same unsigned data. This is needed + /// when we clone an event. + pub fn deep_copy(&self) -> Self { + Self { + inner: Arc::new(RwLock::new(self.py_read().expect("lock poisoned").clone())), + } + } } #[pymethods] @@ -268,11 +277,11 @@ impl Unsigned { } } - fn for_persistence<'py>(&self, py: Python<'py>) -> PyResult> { + pub fn for_persistence<'py>(&self, py: Python<'py>) -> PyResult> { Ok(pythonize(py, &self.py_read()?.persisted_fields)?) } - fn for_event<'py>(&self, py: Python<'py>) -> PyResult> { + pub fn for_event<'py>(&self, py: Python<'py>) -> PyResult> { Ok(pythonize(py, &*self.py_read()?)?) } } diff --git a/rust/src/events/utils.rs b/rust/src/events/utils.rs new file mode 100644 index 00000000000..7d33c52b440 --- /dev/null +++ b/rust/src/events/utils.rs @@ -0,0 +1,1191 @@ +//! JSON +//! +//! Matrix event JSON utility functions. + +use std::collections::BTreeSet; + +use anyhow::Context as _; +use base64::Engine as _; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use super::constants::{ + aliases_field, create_field, + event_field::{ + AUTH_EVENTS, CONTENT, DEPTH, EVENT_ID, HASHES, MEMBERSHIP, ORIGIN, ORIGIN_SERVER_TS, + PREV_EVENTS, PREV_STATE, REPLACES_STATE, ROOM_ID, SENDER, SIGNATURES, STATE_KEY, TYPE, + UNSIGNED, + }, + event_type::{ + M_ROOM_ALIASES, M_ROOM_CREATE, M_ROOM_HISTORY_VISIBILITY, M_ROOM_JOIN_RULES, M_ROOM_MEMBER, + M_ROOM_POWER_LEVELS, M_ROOM_REDACTION, + }, + history_visibility_field, join_rules_field, membership_field, power_levels_field, + redaction_field, + unsigned_field::AGE_TS, +}; +use crate::{ + canonical_json::CanonicalizationOptions, events::constants::event_field::PREV_STATE_EVENTS, +}; +use crate::{ + events::constants::event_field::M_RELATES_TO, + room_versions::{EventFormatVersions, RoomVersion}, +}; + +/// Calculates the event_id of an event. +/// +/// The event_id is the `reference_hash` of the redacted event json, preceded by a `$`. +/// `calculate_event_id` can be used to determine the `event_id` for events in room versions V3+. +pub fn calculate_event_id(event: &Value, room_version: &RoomVersion) -> anyhow::Result> { + match room_version.event_format { + EventFormatVersions::ROOM_V1_V2 => { + anyhow::bail!( + "Attempted to calculate event_id using reference hash for room version v1/v2" + ); + } + EventFormatVersions::ROOM_V3 + | EventFormatVersions::ROOM_V4_PLUS + | EventFormatVersions::ROOM_V11_HYDRA_PLUS + | EventFormatVersions::ROOM_VMSC4242 => { + let reference_hash = compute_event_reference_hash(event, room_version)?; + + Ok(format!("${reference_hash}").into_boxed_str()) + } + _ => { + unimplemented!( + "Unknown event format version {}. This is a Synapse Programming error.", + room_version.event_format + ); + } + } +} + +/// Computes the event reference hash. This is the hash of the redacted event. +pub fn compute_event_reference_hash( + event: &Value, + room_version: &RoomVersion, +) -> anyhow::Result { + let mut redacted_value = redact(event, room_version)?; + + let redacted_value_mut = redacted_value + .as_object_mut() + .context("Failed getting `redacted_value` as mutable object")?; + + redacted_value_mut.remove(SIGNATURES); + redacted_value_mut.remove(UNSIGNED); + redacted_value_mut.remove(AGE_TS); + + let canonicalization_options = if room_version.strict_canonicaljson { + CanonicalizationOptions::strict() + } else { + CanonicalizationOptions::relaxed() + }; + + let json = + crate::canonical_json::to_string_canonical(&redacted_value_mut, canonicalization_options) + .map_err(|err| anyhow::anyhow!(err))?; + + let hash = Sha256::digest(json.as_bytes()); + + let base64_alphabet = if room_version.event_format == EventFormatVersions::ROOM_V3 { + base64::alphabet::STANDARD + } else { + base64::alphabet::URL_SAFE + }; + let base64_engine = base64::engine::GeneralPurpose::new( + &base64_alphabet, + base64::engine::general_purpose::NO_PAD, + ); + + Ok(base64_engine.encode(hash)) +} + +/// Attempts to redact the provided event, returning a copy of the redacted +/// event if successful. +/// +/// Events redacted with this function are meant to be sent over federation. +pub fn redact(event: &Value, room_version: &RoomVersion) -> anyhow::Result { + let mut allowed_keys = BTreeSet::from([ + (EVENT_ID), + (SENDER), + (ROOM_ID), + (HASHES), + (SIGNATURES), + (CONTENT), + (TYPE), + (STATE_KEY), + (DEPTH), + (PREV_EVENTS), + (ORIGIN_SERVER_TS), + ]); + + // Earlier room versions had additional allowed keys + if !room_version.updated_redaction_rules { + allowed_keys.extend([PREV_STATE, MEMBERSHIP, ORIGIN]); + } + + // Room versions with MSC4242 have `prev_state_events` instead of + // `auth_events`. + if room_version.msc4242_state_dags { + allowed_keys.insert(PREV_STATE_EVENTS); + } else { + allowed_keys.insert(AUTH_EVENTS); + } + + let event_type = event + .get(TYPE) + .with_context(|| format!("Missing {TYPE} field in json"))? + .as_str() + .with_context(|| format!("{TYPE} field is not a string"))?; + + let event_content = event + .get(CONTENT) + .with_context(|| format!("Missing {CONTENT} field in json"))?; + + let mut new_content = serde_json::json!({}); + let new_content_mut = new_content + .as_object_mut() + .context("Failed getting `new_content` as mutable object")?; + + let mut add_content_field = |field: &str| { + if let Some(existing_field) = event_content.get(field) { + new_content_mut.insert(field.to_string(), existing_field.clone()); + } + }; + + match event_type { + M_ROOM_MEMBER => { + add_content_field(membership_field::MEMBERSHIP); + if room_version.restricted_join_rule_fix { + add_content_field(membership_field::JOIN_AUTHORISED_VIA_USERS_SERVER); + } + if room_version.updated_redaction_rules { + // Preserve the signed field under third_party_invite. + if let Some(third_party_invite) = + event_content.get(membership_field::THIRD_PARTY_INVITE) + { + if third_party_invite.as_object().is_some() { + let mut new_third_party_invite = serde_json::json!({}); + if let Some(signed) = third_party_invite.get(membership_field::SIGNED) { + new_third_party_invite = + serde_json::json!({membership_field::SIGNED: signed.clone()}); + } + new_content_mut.insert( + membership_field::THIRD_PARTY_INVITE.to_string(), + new_third_party_invite, + ); + } + } + } + } + M_ROOM_CREATE => { + if room_version.updated_redaction_rules { + // MSC2176 rules state that create events cannot have their `content` redacted. + if let Some(event_content_object) = event_content.as_object() { + for (field, _value) in event_content_object { + add_content_field(field); + } + } + } + if !room_version.implicit_room_creator { + // Some room versions give meaning to `creator` + add_content_field(create_field::CREATOR); + } + if room_version.msc4291_room_ids_as_hashes { + // room_id is not allowed on the create event as it's derived from the event ID + allowed_keys.remove(ROOM_ID); + } + } + M_ROOM_JOIN_RULES => { + add_content_field(join_rules_field::JOIN_RULE); + if room_version.restricted_join_rule { + add_content_field(join_rules_field::ALLOW); + } + } + M_ROOM_POWER_LEVELS => { + add_content_field(power_levels_field::USERS); + add_content_field(power_levels_field::USERS_DEFAULT); + add_content_field(power_levels_field::EVENTS); + add_content_field(power_levels_field::EVENTS_DEFAULT); + add_content_field(power_levels_field::STATE_DEFAULT); + add_content_field(power_levels_field::BAN); + add_content_field(power_levels_field::KICK); + add_content_field(power_levels_field::REDACT); + if room_version.updated_redaction_rules { + add_content_field(power_levels_field::INVITE); + } + } + M_ROOM_ALIASES if room_version.special_case_aliases_auth => { + add_content_field(aliases_field::ALIASES); + } + M_ROOM_HISTORY_VISIBILITY => { + add_content_field(history_visibility_field::HISTORY_VISIBILITY) + } + M_ROOM_REDACTION if room_version.updated_redaction_rules => { + add_content_field(redaction_field::REDACTS); + } + _ => (), + }; + + let mut allowed_fields = serde_json::json!({}); + let allowed_fields_mut = allowed_fields + .as_object_mut() + .context("Failed getting `allowed_fields` as mutable object")?; + + for (k, v) in event + .as_object() + .context("Event is not a JSON object")? + .iter() + { + if allowed_keys.contains(&k.as_str()) { + allowed_fields_mut.insert(k.clone(), v.clone()); + } + } + + if room_version.msc3389_relation_redactions { + if let Some(relates_to) = event + .get(CONTENT) + .and_then(|content| content.get(M_RELATES_TO)) + { + if relates_to.is_object() { + let mut new_relates_to = serde_json::json!({}); + let new_relates_to_mut = new_relates_to + .as_object_mut() + .context("Failed getting `new_relates_to` as mutable object")?; + + for field in ["rel_type", "event_id"] { + if let Some(value) = relates_to.get(field) { + new_relates_to_mut.insert(field.to_string(), value.clone()); + } + } + + if !new_relates_to_mut.is_empty() { + new_content_mut.insert(M_RELATES_TO.to_string(), new_relates_to); + } + } + } + } + + allowed_fields_mut.insert(CONTENT.to_string(), new_content); + + // Copy over known good unsigned keys + let allowed_unsigned_keys = [AGE_TS, REPLACES_STATE]; + if let Some(unsigned) = event.get(UNSIGNED) { + let mut new_unsigned = serde_json::Map::new(); + for key in allowed_unsigned_keys { + if let Some(value) = unsigned.get(key) { + new_unsigned.insert(key.to_string(), value.clone()); + } + } + allowed_fields_mut.insert(UNSIGNED.to_string(), Value::Object(new_unsigned)); + } + + Ok(allowed_fields) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::{calculate_event_id, redact}; + use crate::room_versions::RoomVersion; + + #[test] + fn test_calculate_event_id() { + let original = json!( + { + "auth_events":[ + "$gbHO7IPUHybc7ULFnT7P0r3iWlZFHGmr6zBfEYCUyKw", + "$hy1eZFYcgNxMFNNBgCD5fyOzyWRcBkxfNcrUI5ZpZlE", + "$Nt_z68EwFfPqBeHjzEHGsp461Z4EfNEzR-KH5bOYdOY" + ], + "prev_events":[ + "$4FbpZrgPQTwoLD9H5y7jcikucCypUOn78mXhQX7WliY" + ], + "type":"m.room.message", + "room_id":"!DHmIIVvxFASSFgDGzr:localhost:8008", + "sender":"@tester_b:localhost:8008", + "content":{ + "msgtype":"m.text", + "body":"invited people can see history", + "m.mentions":{} + }, + "depth":24, + "origin":"localhost:8008", + "origin_server_ts":1731769874137_i64, + "hashes":{ + "sha256":"FoYV1w3TW/B2mVT0gX2/BZKpCwrrvGXqXFdUhN9LZYU" + }, + "signatures":{ + "localhost:8008":{ + "ed25519:a_phSE":"G8cfk/m97sndxMNrEZ2nMMSXkVeJE05G7if4JiVzAwGfD3TwnF/jfSHt2acWrpNqv/aEhZug3WLofc2id+rVBw" + } + }, + "unsigned":{ + "age_ts":1731769874137_i64 + } + } + ); + let redacted = json!( + { + "auth_events":[ + "$gbHO7IPUHybc7ULFnT7P0r3iWlZFHGmr6zBfEYCUyKw", + "$hy1eZFYcgNxMFNNBgCD5fyOzyWRcBkxfNcrUI5ZpZlE", + "$Nt_z68EwFfPqBeHjzEHGsp461Z4EfNEzR-KH5bOYdOY" + ], + "prev_events":[ + "$4FbpZrgPQTwoLD9H5y7jcikucCypUOn78mXhQX7WliY" + ], + "type":"m.room.message", + "room_id":"!DHmIIVvxFASSFgDGzr:localhost:8008", + "sender":"@tester_b:localhost:8008", + "content":{}, + "depth":24, + "origin":"localhost:8008", + "origin_server_ts":1731769874137_i64, + "hashes":{ + "sha256":"FoYV1w3TW/B2mVT0gX2/BZKpCwrrvGXqXFdUhN9LZYU" + }, + "signatures":{ + "localhost:8008":{ + "ed25519:a_phSE":"G8cfk/m97sndxMNrEZ2nMMSXkVeJE05G7if4JiVzAwGfD3TwnF/jfSHt2acWrpNqv/aEhZug3WLofc2id+rVBw" + } + } + } + ); + + let expected = "$zRz9jjiT9wZc3Hl9ij_74aCmTjqV3YMlj9sj3Uqxg6o"; + let expected_v3 = "$zRz9jjiT9wZc3Hl9ij/74aCmTjqV3YMlj9sj3Uqxg6o"; + + let original_event_id = calculate_event_id(&original, &RoomVersion::V10).unwrap(); + let redacted_event_id = calculate_event_id(&redacted, &RoomVersion::V10).unwrap(); + let _ = calculate_event_id(&original, &RoomVersion::V2).unwrap_err(); + let v3_event_id = calculate_event_id(&original, &RoomVersion::V3).unwrap(); + + assert_eq!(expected, &*original_event_id); + assert_eq!(expected, &*redacted_event_id); + assert_eq!(expected_v3, &*v3_event_id); + } + + #[test] + /// Tests to ensure events with overly large values for `depth` are handled appropriately. + /// This was added in room version 6 . + fn test_calculate_event_id_big_int_old_rooms() { + let original = json!( + { + "auth_events":[ + "$gbHO7IPUHybc7ULFnT7P0r3iWlZFHGmr6zBfEYCUyKw", + "$hy1eZFYcgNxMFNNBgCD5fyOzyWRcBkxfNcrUI5ZpZlE", + "$Nt_z68EwFfPqBeHjzEHGsp461Z4EfNEzR-KH5bOYdOY" + ], + "prev_events":[ + "$4FbpZrgPQTwoLD9H5y7jcikucCypUOn78mXhQX7WliY" + ], + "type":"m.room.message", + "room_id":"!DHmIIVvxFASSFgDGzr:localhost:8008", + "sender":"@tester_b:localhost:8008", + "content":{ + "msgtype":"m.text", + "body":"invited people can see history", + "m.mentions":{} + }, + // NOTE: use the biggest acceptable number + "depth":u64::MAX, + "origin":"localhost:8008", + "origin_server_ts":1731769874137_i64, + "hashes":{ + "sha256":"FoYV1w3TW/B2mVT0gX2/BZKpCwrrvGXqXFdUhN9LZYU" + }, + "signatures":{ + "localhost:8008":{ + "ed25519:a_phSE":"G8cfk/m97sndxMNrEZ2nMMSXkVeJE05G7if4JiVzAwGfD3TwnF/jfSHt2acWrpNqv/aEhZug3WLofc2id+rVBw" + } + }, + "unsigned":{ + "age_ts":1731769874137_i64 + } + } + ); + + // These should succeed. + let _event_id = calculate_event_id(&original, &RoomVersion::V3).unwrap(); + let _event_id = calculate_event_id(&original, &RoomVersion::V4).unwrap(); + let _event_id = calculate_event_id(&original, &RoomVersion::V5).unwrap(); + + // These should not succeed. + let versions = [ + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + RoomVersion::V11, + RoomVersion::V12, + ]; + for version in versions { + let _event_id = calculate_event_id(&original, &version).unwrap_err(); + } + } + + #[test] + /// Tests that calling `redact` on invalid event json that is missing the `type` property + /// fails. + /// The `type` field is required by all versions of the spec. Any json encountered where + /// this field is missing must be considered invalid. + fn test_redact_missing_type() { + let original = json!( + { + // "type": "missing_type" + "unknown_key":"unknown_value", + "auth_events":[ + "$gbHO7IPUHybc7ULFnT7P0r3iWlZFHGmr6zBfEYCUyKw", + "$hy1eZFYcgNxMFNNBgCD5fyOzyWRcBkxfNcrUI5ZpZlE", + "$Nt_z68EwFfPqBeHjzEHGsp461Z4EfNEzR-KH5bOYdOY" + ], + "prev_events":[ + "$4FbpZrgPQTwoLD9H5y7jcikucCypUOn78mXhQX7WliY" + ], + "room_id":"!DHmIIVvxFASSFgDGzr:localhost:8008", + "sender":"@tester_b:localhost:8008", + "content":{ + "msgtype":"m.text", + "body":"invited people can see history", + "m.mentions":{} + }, + "depth":24, + "origin":"localhost:8008", + "origin_server_ts":1731769874137_i64, + "hashes":{ + "sha256":"FoYV1w3TW/B2mVT0gX2/BZKpCwrrvGXqXFdUhN9LZYU" + }, + "signatures":{ + "localhost:8008":{ + "ed25519:a_phSE":"G8cfk/m97sndxMNrEZ2nMMSXkVeJE05G7if4JiVzAwGfD3TwnF/jfSHt2acWrpNqv/aEhZug3WLofc2id+rVBw" + } + }, + "unsigned":{ + "age_ts":1731769874137_i64 + } + } + ); + + let versions = [ + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + RoomVersion::V11, + RoomVersion::V12, + ]; + for version in versions { + let _ = redact(&original, &version).unwrap_err(); + } + } + + #[test] + /// Tests that calling `redact` on invalid event json that is missing the `content` property + /// fails. + /// The `content` field is required by all versions of the spec. Any json encountered where + /// this field is missing must be considered invalid. + fn test_redact_missing_content() { + let original = json!( + { + "unknown_key":"unknown_value", + "auth_events":[ + "$gbHO7IPUHybc7ULFnT7P0r3iWlZFHGmr6zBfEYCUyKw", + "$hy1eZFYcgNxMFNNBgCD5fyOzyWRcBkxfNcrUI5ZpZlE", + "$Nt_z68EwFfPqBeHjzEHGsp461Z4EfNEzR-KH5bOYdOY" + ], + "prev_events":[ + "$4FbpZrgPQTwoLD9H5y7jcikucCypUOn78mXhQX7WliY" + ], + "type":"m.room.message", + "room_id":"!DHmIIVvxFASSFgDGzr:localhost:8008", + "sender":"@tester_b:localhost:8008", + "depth":24, + "origin":"localhost:8008", + "origin_server_ts":1731769874137_i64, + "hashes":{ + "sha256":"FoYV1w3TW/B2mVT0gX2/BZKpCwrrvGXqXFdUhN9LZYU" + }, + "signatures":{ + "localhost:8008":{ + "ed25519:a_phSE":"G8cfk/m97sndxMNrEZ2nMMSXkVeJE05G7if4JiVzAwGfD3TwnF/jfSHt2acWrpNqv/aEhZug3WLofc2id+rVBw" + } + }, + "unsigned":{ + "age_ts":1731769874137_i64 + } + } + ); + + let versions = [ + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + RoomVersion::V11, + RoomVersion::V12, + ]; + for version in versions { + let _ = redact(&original, &version).unwrap_err(); + } + } + + #[test] + /// Tests redaction logic for `m.room.message` events against latest room versions. + /// This is only testing v10+ as it is rather cumbersome to create the proper json for all + /// rooms versions. + fn test_redact_m_room_message() { + let original = json!( + { + "unknown_key":"unknown_value", + "auth_events":[ + "$gbHO7IPUHybc7ULFnT7P0r3iWlZFHGmr6zBfEYCUyKw", + "$hy1eZFYcgNxMFNNBgCD5fyOzyWRcBkxfNcrUI5ZpZlE", + "$Nt_z68EwFfPqBeHjzEHGsp461Z4EfNEzR-KH5bOYdOY" + ], + "prev_events":[ + "$4FbpZrgPQTwoLD9H5y7jcikucCypUOn78mXhQX7WliY" + ], + "type":"m.room.message", + "room_id":"!DHmIIVvxFASSFgDGzr:localhost:8008", + "sender":"@tester_b:localhost:8008", + "content":{ + "msgtype":"m.text", + "body":"invited people can see history", + "m.mentions":{} + }, + "depth":24, + "origin":"localhost:8008", + "origin_server_ts":1731769874137_i64, + "hashes":{ + "sha256":"FoYV1w3TW/B2mVT0gX2/BZKpCwrrvGXqXFdUhN9LZYU" + }, + "signatures":{ + "localhost:8008":{ + "ed25519:a_phSE":"G8cfk/m97sndxMNrEZ2nMMSXkVeJE05G7if4JiVzAwGfD3TwnF/jfSHt2acWrpNqv/aEhZug3WLofc2id+rVBw" + } + }, + "unsigned":{ + "age_ts":1731769874137_i64 + } + } + ); + let expected = json!( + { + "auth_events":[ + "$gbHO7IPUHybc7ULFnT7P0r3iWlZFHGmr6zBfEYCUyKw", + "$hy1eZFYcgNxMFNNBgCD5fyOzyWRcBkxfNcrUI5ZpZlE", + "$Nt_z68EwFfPqBeHjzEHGsp461Z4EfNEzR-KH5bOYdOY" + ], + "prev_events":[ + "$4FbpZrgPQTwoLD9H5y7jcikucCypUOn78mXhQX7WliY" + ], + "type":"m.room.message", + "room_id":"!DHmIIVvxFASSFgDGzr:localhost:8008", + "sender":"@tester_b:localhost:8008", + "content":{}, + "depth":24, + "origin":"localhost:8008", + "origin_server_ts":1731769874137_i64, + "hashes":{ + "sha256":"FoYV1w3TW/B2mVT0gX2/BZKpCwrrvGXqXFdUhN9LZYU" + }, + "signatures":{ + "localhost:8008":{ + "ed25519:a_phSE":"G8cfk/m97sndxMNrEZ2nMMSXkVeJE05G7if4JiVzAwGfD3TwnF/jfSHt2acWrpNqv/aEhZug3WLofc2id+rVBw" + } + }, + "unsigned":{ + "age_ts":1731769874137_i64 + } + } + ); + let expected_v11 = json!( + { + "auth_events":[ + "$gbHO7IPUHybc7ULFnT7P0r3iWlZFHGmr6zBfEYCUyKw", + "$hy1eZFYcgNxMFNNBgCD5fyOzyWRcBkxfNcrUI5ZpZlE", + "$Nt_z68EwFfPqBeHjzEHGsp461Z4EfNEzR-KH5bOYdOY" + ], + "prev_events":[ + "$4FbpZrgPQTwoLD9H5y7jcikucCypUOn78mXhQX7WliY" + ], + "type":"m.room.message", + "room_id":"!DHmIIVvxFASSFgDGzr:localhost:8008", + "sender":"@tester_b:localhost:8008", + "content":{}, + "depth":24, + "origin_server_ts":1731769874137_i64, + "hashes":{ + "sha256":"FoYV1w3TW/B2mVT0gX2/BZKpCwrrvGXqXFdUhN9LZYU" + }, + "signatures":{ + "localhost:8008":{ + "ed25519:a_phSE":"G8cfk/m97sndxMNrEZ2nMMSXkVeJE05G7if4JiVzAwGfD3TwnF/jfSHt2acWrpNqv/aEhZug3WLofc2id+rVBw" + } + }, + "unsigned":{ + "age_ts":1731769874137_i64 + } + } + ); + + let redacted = redact(&original, &RoomVersion::V10).unwrap(); + + assert_eq!(expected, redacted); + + let versions = [RoomVersion::V11, RoomVersion::V12]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + + assert_eq!(expected_v11, redacted, "Room Version {version}"); + } + } + + #[test] + /// Tests redaction logic adheres to changes due to [super::use_updated_redaction_rules]. + /// Tests against all known room versions. + fn test_redact_updated_redaction_rules() { + let original = json!( + { + "type":"m.room.message", + "prev_state":{}, + "membership":{}, + "origin":"some_place", + "content":{}, + } + ); + + let expected_updated_redaction_rules = json!( + { + "type":"m.room.message", + "content":{}, + } + ); + let expected_pre_updated_redaction_rules = json!( + { + "type":"m.room.message", + "prev_state":{}, + "membership":{}, + "origin":"some_place", + "content":{}, + } + ); + + let versions = [ + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + ]; + for version in versions { + let redacted_1 = redact(&original, &version).unwrap(); + assert_eq!( + expected_pre_updated_redaction_rules, redacted_1, + "Room Version {version}" + ); + } + + let versions = [RoomVersion::V11, RoomVersion::V12]; + for version in versions { + let redacted_2 = redact(&original, &version).unwrap(); + assert_eq!( + expected_updated_redaction_rules, redacted_2, + "Room Version {version}" + ); + } + } + + #[test] + /// Tests redaction logic for `m.room.member` events against all known room versions. + fn test_redact_m_room_member() { + let original = json!( + { + "type":"m.room.member", + "content":{ + "unknown_key":"unknown_value", + "membership":"join", + "join_authorised_via_users_server":"server", + "third_party_invite":{ + "signed":{}, + }, + }, + } + ); + + let expected_pre_unrestricted_join_rule_fix = json!( + { + "type":"m.room.member", + "content":{ + "membership":"join", + }, + } + ); + + let expected_updated_redaction_rules = json!( + { + "type":"m.room.member", + "content":{ + "membership":"join", + "join_authorised_via_users_server":"server", + "third_party_invite":{ + "signed":{}, + }, + }, + } + ); + let expected_pre_updated_redaction_rules = json!( + { + "type":"m.room.member", + "content":{ + "membership":"join", + "join_authorised_via_users_server":"server", + }, + } + ); + + let versions = [ + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + ]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_pre_unrestricted_join_rule_fix, redacted, + "Room Version {version}" + ); + } + + let versions = [RoomVersion::V9, RoomVersion::V10]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_pre_updated_redaction_rules, redacted, + "Room Version {version}" + ); + } + + let versions = [RoomVersion::V11, RoomVersion::V12]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_updated_redaction_rules, redacted, + "Room Version {version}" + ); + } + } + + /// Tests redaction logic for `m.room.create` events against all known room versions. + #[test] + fn test_redact_m_room_create() { + let original = json!( + { + "type":"m.room.create", + "room_id": "!roomid", + "content":{ + "unknown_key":"unknown_value", + "other_key":"value", + "creator":"user", + }, + } + ); + + let expected_implicit_room_creator = json!( + { + "type":"m.room.create", + "room_id": "!roomid", + "content":{ + "unknown_key":"unknown_value", + "other_key":"value", + "creator":"user", + }, + } + ); + let expected_room_ids_as_hashes = json!( + { + "type":"m.room.create", + "content":{ + "unknown_key":"unknown_value", + "other_key":"value", + "creator":"user", + }, + } + ); + let expected_pre_implicit_room_creator = json!( + { + "type":"m.room.create", + "room_id": "!roomid", + "content":{ + "creator":"user", + }, + } + ); + + let versions = [ + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + ]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_pre_implicit_room_creator, redacted, + "Room Version {version}" + ); + } + + let redacted = redact(&original, &RoomVersion::V11).unwrap(); + assert_eq!(expected_implicit_room_creator, redacted); + + let versions = [RoomVersion::HYDRA_V11, RoomVersion::V12]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_room_ids_as_hashes, redacted, + "Room Version {version}" + ); + } + } + + #[test] + /// Tests redaction logic for `m.room.join_rules` events against all known room versions. + fn test_redact_m_room_join_rules() { + let original = json!( + { + "type":"m.room.join_rules", + "content":{ + "unknown_key":"unknown_value", + "join_rule":"invite", + "allow":"user", + }, + } + ); + + let expected_restricted_join_rule = json!( + { + "type":"m.room.join_rules", + "content":{ + "join_rule":"invite", + "allow":"user", + }, + } + ); + let expected_pre_restricted_join_rule = json!( + { + "type":"m.room.join_rules", + "content":{ + "join_rule":"invite", + }, + } + ); + + let versions = [ + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + ]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_pre_restricted_join_rule, redacted, + "Room Version {version}" + ); + } + + let versions = [ + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + RoomVersion::V11, + RoomVersion::V12, + ]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_restricted_join_rule, redacted, + "Room Version {version}" + ); + } + } + + #[test] + /// Tests redaction logic for `m.room.power_levels` events against all known room versions. + fn test_redact_m_room_power_levels() { + let original = json!( + { + "type":"m.room.power_levels", + "content":{ + "unknown_key":"unknown_value", + "users":{}, + "users_default":{}, + "events":{}, + "events_default":{}, + "state_default":{}, + "ban":{}, + "kick":{}, + "redact":{}, + "invite":{}, + }, + } + ); + + let expected_updated_redaction_rules = json!( + { + "type":"m.room.power_levels", + "content":{ + "users":{}, + "users_default":{}, + "events":{}, + "events_default":{}, + "state_default":{}, + "ban":{}, + "kick":{}, + "redact":{}, + "invite":{}, + }, + } + ); + let expected_pre_updated_redaction_rules = json!( + { + "type":"m.room.power_levels", + "content":{ + "users":{}, + "users_default":{}, + "events":{}, + "events_default":{}, + "state_default":{}, + "ban":{}, + "kick":{}, + "redact":{}, + }, + } + ); + + let versions = [ + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + ]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_pre_updated_redaction_rules, redacted, + "Room Version {version}" + ); + } + + let versions = [RoomVersion::V11, RoomVersion::V12]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_updated_redaction_rules, redacted, + "Room Version {version}" + ); + } + } + + #[test] + /// Tests redaction logic for `m.room.aliases` events against all known room versions. + fn test_redact_m_room_aliases() { + let original = json!( + { + "type":"m.room.aliases", + "content":{ + "unknown_key":"unknown_value", + "aliases":{}, + }, + } + ); + + let expected_special_case_aliases = json!( + { + "type":"m.room.aliases", + "content":{ + "aliases":{}, + }, + } + ); + let expected_post_special_case_aliases = json!( + { + "type":"m.room.aliases", + "content":{}, + } + ); + + let versions = [ + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + ]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_special_case_aliases, redacted, + "Room Version {version}" + ); + } + + let versions = [ + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + RoomVersion::V11, + RoomVersion::V12, + ]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_post_special_case_aliases, redacted, + "Room Version {version}" + ); + } + } + + #[test] + /// Tests redaction logic for `m.room.history_visibility` events against all known room versions. + fn test_redact_m_room_history_visibility() { + let original = json!( + { + "type":"m.room.history_visibility", + "content":{ + "unknown_key":"unknown_value", + "history_visibility":"visibility", + }, + } + ); + + let expected = json!( + { + "type":"m.room.history_visibility", + "content":{ + "history_visibility":"visibility", + }, + } + ); + + let versions = [ + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + RoomVersion::V11, + RoomVersion::V12, + ]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!(expected, redacted, "Room Version {version}"); + } + } + + #[test] + /// Tests redaction logic for `m.room.redaction` events against all known room versions. + fn test_redact_m_room_redaction() { + let original = json!( + { + "type":"m.room.redaction", + "content":{ + "unknown_key":"unknown_value", + "redacts":"event", + }, + } + ); + + let expected_updated_redaction_rules = json!( + { + "type":"m.room.redaction", + "content":{ + "redacts":"event", + }, + } + ); + let expected_pre_updated_redaction_rules = json!( + { + "type":"m.room.redaction", + "content":{}, + } + ); + + let versions = [ + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + ]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_pre_updated_redaction_rules, redacted, + "Room Version {version}" + ); + } + + let versions = [RoomVersion::V11, RoomVersion::V12]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_updated_redaction_rules, redacted, + "Room Version {version}" + ); + } + } +} diff --git a/rust/src/json.rs b/rust/src/json.rs new file mode 100644 index 00000000000..3e833c67072 --- /dev/null +++ b/rust/src/json.rs @@ -0,0 +1,218 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +/// A wrapper type that represents a value that may be missing. +/// +/// We can't necessarily use `Option` for this, as we want to distinguish +/// between a missing value and a value that is present but null (e.g. +/// `{"field": null}` vs `{}`). Serde by default treats missing fields as +/// `None`, so we need a custom type to capture this distinction. +/// +/// A plain `AllowMissing` is used for fields that are either present and of +/// type `T`, or absent. An `AllowMissing>` is used for fields that +/// are of type `T`, null, or absent. +/// +/// Note, to use this type correctly, the field **MUST** be annotated with: +/// +/// ```rust +/// #[serde( +/// default, +/// with = "crate::json::allow_missing", +/// skip_serializing_if = "AllowMissing::is_absent" +/// )] +/// ``` +/// +#[derive(Default, Debug, Clone)] +pub enum AllowMissing { + Some(T), + #[default] + Absent, +} + +impl AllowMissing { + /// Returns `true` if the value is present, even if it is null. + pub fn is_some(&self) -> bool { + matches!(self, AllowMissing::Some(_)) + } + + /// Returns `true` if the value is absent. + pub fn is_absent(&self) -> bool { + matches!(self, AllowMissing::Absent) + } + + /// Converts to `Option`. + /// + /// Useful for converting e.g. `AllowMissing` to `Option<&str>`. + pub fn as_deref_opt(&self) -> Option<&T::Target> + where + T: std::ops::Deref, + { + match self { + AllowMissing::Some(inner) => Some(inner.deref()), + AllowMissing::Absent => None, + } + } + + /// Converts to `Option<&T>`. + pub fn as_ref_opt(&self) -> Option<&T> { + match self { + AllowMissing::Some(inner) => Some(inner), + AllowMissing::Absent => None, + } + } +} + +/// A module that provides the serialization and deserialization logic for +/// `AllowMissing`. +pub mod allow_missing { + use serde::ser::Error as _; + + use super::AllowMissing; + + pub fn deserialize<'de, T, D>(deserializer: D) -> Result, D::Error> + where + T: serde::Deserialize<'de>, + D: serde::Deserializer<'de>, + { + Ok(AllowMissing::Some(T::deserialize(deserializer)?)) + } + + pub fn serialize(value: &AllowMissing, serializer: S) -> Result + where + T: serde::Serialize, + S: serde::Serializer, + { + match value { + AllowMissing::Some(inner) => inner.serialize(serializer), + // We should never attempt to serialize an `AllowMissing::Absent`, as we + // should have skipped it with `skip_serializing_if`. + AllowMissing::Absent => Err(S::Error::custom("cannot serialize AllowMissing::Absent")), + } + } +} + +#[cfg(test)] +mod tests { + use std::assert_matches; + + use serde::{Deserialize, Serialize}; + + use super::*; + + #[derive(Serialize, Deserialize)] + struct TestStruct { + #[serde( + default, + with = "crate::json::allow_missing", + skip_serializing_if = "AllowMissing::is_absent" + )] + value: AllowMissing, + } + + #[test] + fn test_deserialize() { + let json = r#"{"value":42}"#; + let deserialized: TestStruct = serde_json::from_str(json).unwrap(); + assert!(deserialized.value.is_some()); + assert_matches!(deserialized.value, AllowMissing::Some(42)); + + let json = r#"{}"#; + let deserialized: TestStruct = serde_json::from_str(json).unwrap(); + assert!(deserialized.value.is_absent()); + assert_matches!(deserialized.value, AllowMissing::Absent); + } + + #[test] + fn test_serialize() { + let value = TestStruct { + value: AllowMissing::Some(42), + }; + let serialized = serde_json::to_string(&value).unwrap(); + assert_eq!(serialized, r#"{"value":42}"#); + + let value = TestStruct { + value: AllowMissing::Absent, + }; + let serialized = serde_json::to_string(&value).unwrap(); + assert_eq!(serialized, r#"{}"#); + } + + /// Test that we get an error if we attempt to serialize an + /// `AllowMissing::Absent` without the skip_serializing_if annotation. + #[test] + fn test_serialize_absent_error() { + #[derive(Serialize)] + struct TestStructWithoutSkip { + #[serde(default, with = "crate::json::allow_missing")] + value: AllowMissing, + } + + let value = TestStructWithoutSkip { + value: AllowMissing::Absent, + }; + + let err = serde_json::to_string(&value).unwrap_err(); + assert_eq!(err.to_string(), "cannot serialize AllowMissing::Absent"); + } + + #[derive(Serialize, Deserialize)] + struct TestStructOption { + #[serde( + default, + with = "crate::json::allow_missing", + skip_serializing_if = "AllowMissing::is_absent" + )] + value: AllowMissing>, + } + + #[test] + fn test_serialize_option() { + let value = TestStructOption { + value: AllowMissing::Some(Some(42)), + }; + let serialized = serde_json::to_string(&value).unwrap(); + assert_eq!(serialized, r#"{"value":42}"#); + + let value = TestStructOption { + value: AllowMissing::Some(None), + }; + let serialized = serde_json::to_string(&value).unwrap(); + assert_eq!(serialized, r#"{"value":null}"#); + + let value = TestStructOption { + value: AllowMissing::Absent, + }; + let serialized = serde_json::to_string(&value).unwrap(); + assert_eq!(serialized, r#"{}"#); + } + + #[test] + fn test_deserialize_option() { + let json = r#"{"value":42}"#; + let deserialized: TestStructOption = serde_json::from_str(json).unwrap(); + assert!(deserialized.value.is_some()); + assert_matches!(deserialized.value, AllowMissing::Some(Some(42))); + + let json = r#"{"value":null}"#; + let deserialized: TestStructOption = serde_json::from_str(json).unwrap(); + assert!(deserialized.value.is_some()); + assert_matches!(deserialized.value, AllowMissing::Some(None)); + + let json = r#"{}"#; + let deserialized: TestStructOption = serde_json::from_str(json).unwrap(); + assert!(deserialized.value.is_absent()); + assert_matches!(deserialized.value, AllowMissing::Absent); + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index cc89862e4e7..8ed4e24b813 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -12,6 +12,7 @@ pub mod events; pub mod http; pub mod http_client; pub mod identifier; +pub mod json; pub mod matrix_const; pub mod msc4388_rendezvous; pub mod push; diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 823b6288e82..4f367408083 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -33,8 +33,9 @@ from synapse.api.errors import Codes, SynapseError from synapse.api.room_versions import RoomVersion from synapse.events import EventBase -from synapse.events.utils import prune_event, prune_event_dict +from synapse.events.utils import prune_event from synapse.logging.opentracing import trace +from synapse.synapse_rust.events import redact_event_dict from synapse.types import JsonDict, UserID logger = logging.getLogger(__name__) @@ -157,7 +158,7 @@ def compute_event_signature( Returns: a dictionary in the same format of an event's signatures field. """ - redact_json = prune_event_dict(room_version, event_dict) + redact_json = redact_event_dict(room_version, event_dict) redact_json.pop("age_ts", None) redact_json.pop("unsigned", None) if logger.isEnabledFor(logging.DEBUG): diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 36736b45592..a1633f881f7 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -46,9 +46,9 @@ ) from synapse.config.key import TrustedKeyServer from synapse.events import EventBase -from synapse.events.utils import prune_event_dict from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.storage.keys import FetchKeyResult +from synapse.synapse_rust.events import redact_event from synapse.types import JsonDict from synapse.util import unwrapFirstError from synapse.util.async_helpers import yieldable_gather_results @@ -136,7 +136,7 @@ def from_event( server_name, # We defer creating the redacted json object, as it uses a lot more # memory than the Event object itself. - lambda: prune_event_dict(event.room_version, event.get_pdu_json()), + lambda: redact_event(event).get_pdu_json(), minimum_valid_until_ms, key_ids=key_ids, ) diff --git a/synapse/event_auth.py b/synapse/event_auth.py index d9750651fa4..81a454b5442 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -66,6 +66,7 @@ from synapse.state import CREATE_KEY from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.types import ( + JsonMapping, MutableStateMap, StateKey, StateMap, @@ -856,6 +857,7 @@ def get_send_level( power level required to send this event. """ + power_levels_content: JsonMapping if power_levels_event: power_levels_content = power_levels_event.content else: diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index fcb169e0796..7abd91f0728 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -20,49 +20,37 @@ # # -import abc import collections.abc from typing import ( TYPE_CHECKING, Any, - Generic, - Iterable, - Literal, - TypeVar, + TypeAlias, Union, - overload, ) import attr -from typing_extensions import deprecated -from unpaddedbase64 import encode_base64 from synapse.api.constants import ( EventContentFields, EventTypes, RelationTypes, - StickyEvent, -) -from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions -from synapse.synapse_rust.events import ( - EventInternalMetadata, - JsonObject, - Signatures, - Unsigned, ) +from synapse.api.errors import Codes, SynapseError +from synapse.api.room_versions import RoomVersion, RoomVersions +from synapse.synapse_rust.events import Event from synapse.types import ( JsonDict, - JsonMapping, StateKey, - StrCollection, ) -from synapse.util.caches import intern_dict -from synapse.util.duration import Duration -from synapse.util.frozenutils import freeze if TYPE_CHECKING: from synapse.events.builder import EventBuilder +# The base class for events used to be called EventBase, but it was renamed to +# Event when we switched to using the Rust implementation. We keep the old name +# around for backwards compatibility. +EventBase: TypeAlias = Event + USE_FROZEN_DICTS = False """ @@ -70,639 +58,29 @@ bugs where we accidentally share e.g. signature dicts. However, converting a dict to frozen_dicts is expensive. -NOTE: This is overridden by the configuration by the Synapse worker apps, but -for the sake of tests, it is set here because it cannot be configured on the -homeserver object itself. - -FIXME: Because of how this option works (changing the underlying types), it causes -subtle downstream bugs that makes type comparisons brittle, tracked by -https://github.com/element-hq/synapse/issues/18117 +FIXME: Remove `USE_FROZEN_DICTS` and `use_frozen_dicts` config as this is no +longer used since we switched to using the Rust implementation, all events are +immutable already (and so don't benefit from freezing). """ -T = TypeVar("T") - - -# DictProperty (and DefaultDictProperty) require the classes they're used with to -# have a _dict property to pull properties from. -# -# TODO _DictPropertyInstance should not include EventBuilder but due to -# https://github.com/python/mypy/issues/5570 it thinks the DictProperty and -# DefaultDictProperty get applied to EventBuilder when it is in a Union with -# EventBase. This is the least invasive hack to get mypy to comply. -# -# Note that DictProperty/DefaultDictProperty cannot actually be used with -# EventBuilder as it lacks a _dict property. -_DictPropertyInstance = Union["EventBase", "EventBuilder"] - - -class DictProperty(Generic[T]): - """An object property which delegates to the `_dict` within its parent object.""" - - __slots__ = ["key"] - - def __init__(self, key: str): - self.key = key - - @overload - def __get__( - self, - instance: Literal[None], - owner: type[_DictPropertyInstance] | None = None, - ) -> "DictProperty": ... - - @overload - def __get__( - self, - instance: _DictPropertyInstance, - owner: type[_DictPropertyInstance] | None = None, - ) -> T: ... - - def __get__( - self, - instance: _DictPropertyInstance | None, - owner: type[_DictPropertyInstance] | None = None, - ) -> T | "DictProperty": - # if the property is accessed as a class property rather than an instance - # property, return the property itself rather than the value - if instance is None: - return self - try: - assert isinstance(instance, EventBase) - return instance._dict[self.key] - except KeyError as e1: - # We want this to look like a regular attribute error (mostly so that - # hasattr() works correctly), so we convert the KeyError into an - # AttributeError. - # - # To exclude the KeyError from the traceback, we explicitly - # 'raise from e1.__context__' (which is better than 'raise from None', - # because that would omit any *earlier* exceptions). - # - raise AttributeError( - "'%s' has no '%s' property" % (type(instance), self.key) - ) from e1.__context__ - - def __set__(self, instance: _DictPropertyInstance, v: T) -> None: - assert isinstance(instance, EventBase) - instance._dict[self.key] = v - - def __delete__(self, instance: _DictPropertyInstance) -> None: - assert isinstance(instance, EventBase) - try: - del instance._dict[self.key] - except KeyError as e1: - raise AttributeError( - "'%s' has no '%s' property" % (type(instance), self.key) - ) from e1.__context__ - - -class DefaultDictProperty(DictProperty, Generic[T]): - """An extension of DictProperty which provides a default if the property is - not present in the parent's _dict. - - Note that this means that hasattr() on the property always returns True. - """ - - __slots__ = ["default"] - - def __init__(self, key: str, default: T): - super().__init__(key) - self.default = default - - @overload - def __get__( - self, - instance: Literal[None], - owner: type[_DictPropertyInstance] | None = None, - ) -> "DefaultDictProperty": ... - - @overload - def __get__( - self, - instance: _DictPropertyInstance, - owner: type[_DictPropertyInstance] | None = None, - ) -> T: ... - - def __get__( - self, - instance: _DictPropertyInstance | None, - owner: type[_DictPropertyInstance] | None = None, - ) -> T | "DefaultDictProperty": - if instance is None: - return self - assert isinstance(instance, EventBase) - return instance._dict.get(self.key, self.default) - - -class EventBase(metaclass=abc.ABCMeta): - @property - @abc.abstractmethod - def format_version(self) -> int: - """The EventFormatVersion implemented by this event""" - ... - - def __init__( - self, - event_dict: JsonDict, - room_version: RoomVersion, - signatures: dict[str, dict[str, str]], - unsigned: JsonDict, - internal_metadata_dict: JsonDict, - rejected_reason: str | None, - ): - assert room_version.event_format == self.format_version - - if "content" in event_dict: - event_dict["content"] = JsonObject(event_dict["content"]) - - # We intern these strings because they turn up a lot (especially when - # caching). - event_dict = intern_dict(event_dict) - - if USE_FROZEN_DICTS: - frozen_dict = freeze(event_dict) - else: - frozen_dict = event_dict - - self.room_version = room_version - self.signatures = Signatures(signatures) - self.unsigned = Unsigned(unsigned) - self.rejected_reason = rejected_reason - - self._dict = frozen_dict - - self.internal_metadata = EventInternalMetadata(internal_metadata_dict) - - depth: DictProperty[int] = DictProperty("depth") - content: DictProperty[JsonMapping] = DictProperty("content") - hashes: DictProperty[dict[str, str]] = DictProperty("hashes") - origin_server_ts: DictProperty[int] = DictProperty("origin_server_ts") - sender: DictProperty[str] = DictProperty("sender") - # TODO state_key should be str | None. This is generally asserted in Synapse - # by calling is_state() first (which ensures it is not None), but it is hard (not possible?) - # to properly annotate that calling is_state() asserts that state_key exists - # and is non-None. It would be better to replace such direct references with - # get_state_key() (and a check for None). - state_key: DictProperty[str] = DictProperty("state_key") - type: DictProperty[str] = DictProperty("type") - - # This is a deprecated property, use `sender` instead. Only used by modules. - user_id: DictProperty[str] = DictProperty("sender") - - @property - def event_id(self) -> str: - raise NotImplementedError() - - @property - def room_id(self) -> str: - raise NotImplementedError() - - @property - def membership(self) -> str: - return self.content["membership"] - - @property - def redacts(self) -> str | None: - """MSC2176 moved the redacts field into the content.""" - if self.room_version.updated_redaction_rules: - return self.content.get("redacts") - return self.get("redacts") - - def is_state(self) -> bool: - return self.get_state_key() is not None - - def get_state_key(self) -> str | None: - """Get the state key of this event, or None if it's not a state event""" - return self._dict.get("state_key") - - def get_dict(self) -> JsonDict: - """Convert the event to a dictionary suitable for serialisation.""" - - d = dict(self._dict) - if "content" in d: - # Convert the content (which is a JsonObject) back to a dict. Json - # serialization should handle JsonObjects fine, but for sanities - # sake we want `get_dict()` and `get_pdu_json()` to return plain - # dicts. - d["content"] = dict(self.content) - d.update( - { - "signatures": self.signatures.as_dict(), - "unsigned": self.unsigned.for_event(), - } - ) - - return d - - def get_dict_for_persistence(self) -> JsonDict: - """Convert the event to a dictionary suitable for persistence.""" - d = dict(self._dict) - d.update( - { - "signatures": self.signatures.as_dict(), - "unsigned": self.unsigned.for_persistence(), - } - ) - - return d - - def get(self, key: str, default: Any | None = None) -> Any: - return self._dict.get(key, default) - - def get_internal_metadata_dict(self) -> JsonDict: - return self.internal_metadata.get_dict() - - def get_pdu_json(self, time_now: int | None = None) -> JsonDict: - pdu_json = self.get_dict() - - if time_now is not None and "age_ts" in pdu_json["unsigned"]: - age = time_now - pdu_json["unsigned"]["age_ts"] - pdu_json.setdefault("unsigned", {})["age"] = int(age) - del pdu_json["unsigned"]["age_ts"] - - # This may be a frozen event - pdu_json["unsigned"].pop("redacted_because", None) - - return pdu_json - - def get_templated_pdu_json(self) -> JsonDict: - """ - Return a JSON object suitable for a templated event, as used in the - make_{join,leave,knock} workflow. - """ - # By using _dict directly we don't pull in signatures/unsigned. - template_json = dict(self._dict) - # The hashes (similar to the signature) need to be recalculated by the - # joining/leaving/knocking server after (potentially) modifying the - # event. - template_json.pop("hashes") - - return template_json - - def __contains__(self, field: str) -> bool: - return field in self._dict - - def items(self) -> list[tuple[str, Any | None]]: - return list(self._dict.items()) - - def keys(self) -> Iterable[str]: - return self._dict.keys() - - def prev_event_ids(self) -> list[str]: - """Returns the list of prev event IDs. The order matches the order - specified in the event, though there is no meaning to it. - - Returns: - The list of event IDs of this event's prev_events - """ - return [e for e, _ in self._dict["prev_events"]] - - def auth_event_ids(self) -> StrCollection: - """Returns the list of auth event IDs. The order matches the order - specified in the event, though there is no meaning to it. - - Returns: - The list of event IDs of this event's auth_events - """ - return [e for e, _ in self._dict["auth_events"]] - - def freeze(self) -> None: - """'Freeze' the event dict, so it cannot be modified by accident""" - - # this will be a no-op if the event dict is already frozen. - self._dict = freeze(self._dict) - - def sticky_duration(self) -> Duration | None: - """ - Returns the effective sticky duration of this event, or None - if the event does not have a sticky duration. - (Sticky Events are a MSC4354 feature.) - - Clamps the sticky duration to the maximum allowed duration. - """ - sticky_obj = self.get_dict().get(StickyEvent.EVENT_FIELD_NAME, None) - if type(sticky_obj) is not dict: - return None - sticky_duration_ms = sticky_obj.get("duration_ms", None) - # MSC: Clamp to 0 and MAX_DURATION (1 hour) - # We use `type(...) is int` to avoid accepting bools as `isinstance(True, int)` - # (bool is a subclass of int) - if type(sticky_duration_ms) is int and sticky_duration_ms >= 0: - return min( - Duration(milliseconds=sticky_duration_ms), - StickyEvent.MAX_DURATION, - ) - return None - - def __str__(self) -> str: - return self.__repr__() - - def __repr__(self) -> str: - rejection = f"REJECTED={self.rejected_reason}, " if self.rejected_reason else "" - - conditional_membership_string = "" - if self.get("type") == EventTypes.Member: - conditional_membership_string = f"membership={self.membership}, " - - return ( - f"<{self.__class__.__name__} " - f"{rejection}" - f"event_id={self.event_id}, " - f"type={self.get('type')}, " - f"state_key={self.get('state_key')}, " - f"{conditional_membership_string}" - f"outlier={self.internal_metadata.is_outlier()}" - ">" - ) - - # Using `__getitem__` is deprecated. Only used by modules. - @deprecated("Use attribute access instead") - def __getitem__(self, field: str) -> Any | None: - return self._dict[field] - - -class FrozenEvent(EventBase): - format_version = EventFormatVersions.ROOM_V1_V2 # All events of this type are V1 - - def __init__( - self, - event_dict: JsonDict, - room_version: RoomVersion, - internal_metadata_dict: JsonDict | None = None, - rejected_reason: str | None = None, - ): - internal_metadata_dict = internal_metadata_dict or {} - - event_dict = dict(event_dict) - - # Signatures is a dict of dicts, and this is faster than doing a - # copy.deepcopy - signatures = { - name: dict(sigs.items()) - for name, sigs in event_dict.pop("signatures", {}).items() - } - - unsigned = event_dict.pop("unsigned", {}) - - self._event_id = event_dict["event_id"] - - super().__init__( - event_dict, - room_version=room_version, - signatures=signatures, - unsigned=unsigned, - internal_metadata_dict=internal_metadata_dict, - rejected_reason=rejected_reason, - ) - - @property - def event_id(self) -> str: - return self._event_id - - @property - def room_id(self) -> str: - return self._dict["room_id"] - - -class FrozenEventV2(EventBase): - format_version = EventFormatVersions.ROOM_V3 # All events of this type are V2 - - def __init__( - self, - event_dict: JsonDict, - room_version: RoomVersion, - internal_metadata_dict: JsonDict | None = None, - rejected_reason: str | None = None, - ): - internal_metadata_dict = internal_metadata_dict or {} - - event_dict = dict(event_dict) - - # Signatures is a dict of dicts, and this is faster than doing a - # copy.deepcopy - signatures = { - name: dict(sigs.items()) - for name, sigs in event_dict.pop("signatures", {}).items() - } - - assert "event_id" not in event_dict - - unsigned = event_dict.pop("unsigned", {}) - - self._event_id: str | None = None - - super().__init__( - event_dict, - room_version=room_version, - signatures=signatures, - unsigned=unsigned, - internal_metadata_dict=internal_metadata_dict, - rejected_reason=rejected_reason, - ) - - @property - def event_id(self) -> str: - # We have to import this here as otherwise we get an import loop which - # is hard to break. - from synapse.crypto.event_signing import compute_event_reference_hash - - if self._event_id: - return self._event_id - self._event_id = "$" + encode_base64(compute_event_reference_hash(self)[1]) - return self._event_id - - @property - def room_id(self) -> str: - return self._dict["room_id"] - - def prev_event_ids(self) -> list[str]: - """Returns the list of prev event IDs. The order matches the order - specified in the event, though there is no meaning to it. - - Returns: - The list of event IDs of this event's prev_events - """ - return self._dict["prev_events"] - - def auth_event_ids(self) -> StrCollection: - """Returns the list of auth event IDs. The order matches the order - specified in the event, though there is no meaning to it. - - Returns: - The list of event IDs of this event's auth_events - """ - return self._dict["auth_events"] - - -class FrozenEventV3(FrozenEventV2): - """FrozenEventV3, which differs from FrozenEventV2 only in the event_id format""" - - format_version = EventFormatVersions.ROOM_V4_PLUS # All events of this type are V3 - - @property - def event_id(self) -> str: - # We have to import this here as otherwise we get an import loop which - # is hard to break. - from synapse.crypto.event_signing import compute_event_reference_hash - - if self._event_id: - return self._event_id - self._event_id = "$" + encode_base64( - compute_event_reference_hash(self)[1], urlsafe=True - ) - return self._event_id - - -class FrozenEventV4(FrozenEventV3): - """FrozenEventV4 for MSC4291 room IDs are hashes""" - - format_version = EventFormatVersions.ROOM_V11_HYDRA_PLUS - - """Override the room_id for m.room.create events""" - - def __init__( - self, - event_dict: JsonDict, - room_version: RoomVersion, - internal_metadata_dict: JsonDict | None = None, - rejected_reason: str | None = None, - ): - super().__init__( - event_dict=event_dict, - room_version=room_version, - internal_metadata_dict=internal_metadata_dict, - rejected_reason=rejected_reason, - ) - self._room_id: str | None = None - - @property - def room_id(self) -> str: - # if we have calculated the room ID already, don't do it again. - if self._room_id: - return self._room_id - - is_create_event = self.type == EventTypes.Create and self.get_state_key() == "" - - # for non-create events: use the supplied value from the JSON, as per FrozenEventV3 - if not is_create_event: - self._room_id = self._dict["room_id"] - assert self._room_id is not None - return self._room_id - - # for create events: calculate the room ID - from synapse.crypto.event_signing import compute_event_reference_hash - - self._room_id = "!" + encode_base64( - compute_event_reference_hash(self)[1], urlsafe=True - ) - return self._room_id - - def auth_event_ids(self) -> StrCollection: - """Returns the list of auth event IDs. The order matches the order - specified in the event, though there is no meaning to it. - Returns: - The list of event IDs of this event's auth_events - Includes the creation event ID for convenience of all the codepaths - which expects the auth chain to include the creator ID, even though - it's explicitly not included on the wire. Excludes the create event - for the create event itself. - """ - create_event_id = "$" + self.room_id[1:] - assert create_event_id not in self._dict["auth_events"] - if self.type == EventTypes.Create and self.get_state_key() == "": - return self._dict["auth_events"] # should be [] - return [*self._dict["auth_events"], create_event_id] - - -class FrozenEventVMSC4242(FrozenEventV4): - """FrozenEventVMSC4242, which differs from FrozenEventV4 only in the addition of prev_state_events""" - - format_version = EventFormatVersions.ROOM_VMSC4242 - prev_state_events: DictProperty[StrCollection] = DictProperty("prev_state_events") - - def __init__( - self, - event_dict: JsonDict, - room_version: RoomVersion, - internal_metadata_dict: JsonDict | None = None, - rejected_reason: str | None = None, - ): - # Similar to how we assert event_id isn't in V2+ events, we do the same with auth_events. - # We don't expect `auth_events` in the wire format because we calculate it from prev_state_events. - assert "auth_events" not in event_dict - super().__init__( - event_dict=event_dict, - room_version=room_version, - internal_metadata_dict=internal_metadata_dict, - rejected_reason=rejected_reason, - ) - - def auth_event_ids(self) -> StrCollection: - """Returns the list of _calculated_ auth event IDs. - - Returns: - The list of event IDs of this event's auth events - """ - # Catches cases where we accidentally call auth_event_ids() prior to calculating what they - # actually are. The exception being the m.room.create event which has no auth events. - if self.type != EventTypes.Create: - assert len(self.internal_metadata.calculated_auth_event_ids) > 0 - return self.internal_metadata.calculated_auth_event_ids - - def __repr__(self) -> str: - rejection = f"REJECTED={self.rejected_reason}, " if self.rejected_reason else "" - - return ( - f"<{self.__class__.__name__} " - f"{rejection}" - f"event_id={self.event_id}, " - f"type={self.get('type')}, " - f"state_key={self.get('state_key')}, " - f"prev_events={self.get('prev_events')}, " - f"prev_state_events={self.get('prev_state_events')}, " - f"outlier={self.internal_metadata.is_outlier()}" - ">" - ) - - -def _event_type_from_format_version( - format_version: int, -) -> type[FrozenEvent | FrozenEventV2 | FrozenEventV3 | FrozenEventVMSC4242]: - """Returns the python type to use to construct an Event object for the - given event format version. - - Args: - format_version: The event format version - - Returns: - A type that can be initialized as per the initializer of `FrozenEvent` - """ - - if format_version == EventFormatVersions.ROOM_V1_V2: - return FrozenEvent - elif format_version == EventFormatVersions.ROOM_V3: - return FrozenEventV2 - elif format_version == EventFormatVersions.ROOM_V4_PLUS: - return FrozenEventV3 - elif format_version == EventFormatVersions.ROOM_VMSC4242: - return FrozenEventVMSC4242 - elif format_version == EventFormatVersions.ROOM_V11_HYDRA_PLUS: - return FrozenEventV4 - else: - raise Exception("No event format %r" % (format_version,)) - def make_event_from_dict( event_dict: JsonDict, room_version: RoomVersion = RoomVersions.V1, internal_metadata_dict: JsonDict | None = None, rejected_reason: str | None = None, -) -> EventBase: +) -> Event: """Construct an EventBase from the given event dict""" - event_type = _event_type_from_format_version(room_version.event_format) - return event_type( - event_dict, room_version, internal_metadata_dict or {}, rejected_reason - ) + + try: + return Event( + event_dict=event_dict, + room_version=room_version, + internal_metadata_dict=internal_metadata_dict or {}, + rejected_reason=rejected_reason, + ) + except ValueError: + raise SynapseError(400, "Invalid event dict", Codes.BAD_JSON) @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -716,7 +94,7 @@ class _EventRelation: aggregation_key: str | None -def relation_from_event(event: EventBase) -> _EventRelation | None: +def relation_from_event(event: Event) -> _EventRelation | None: """ Attempt to parse relation information an event. diff --git a/synapse/events/py_protocol.py b/synapse/events/py_protocol.py index d9ac8c066f5..a34170cd931 100644 --- a/synapse/events/py_protocol.py +++ b/synapse/events/py_protocol.py @@ -33,7 +33,6 @@ attributes. """ -import abc from typing import ( TYPE_CHECKING, Sequence, @@ -42,12 +41,14 @@ from typing_extensions import TypeIs from synapse.events import EventBase +from synapse.synapse_rust.events import Event +from synapse.types import StrCollection if TYPE_CHECKING: from synapse.events.snapshot import EventContext, EventPersistencePair -class _DisableIsInstance(abc.ABCMeta): +class _DisableIsInstance(type): """Metaclass which disables isinstance checks on classes which use it, by making isinstance() raise NotImplementedError. @@ -61,15 +62,38 @@ def __instancecheck__(cls, instance: object) -> bool: raise NotImplementedError("Instance cannot be used.") -class EventProtocol(EventBase, metaclass=_DisableIsInstance): - """Helper subclass that allows type narrowing for `EventBase` objects.""" +# We now define `EventProtocol` as a helper class for type narrowing. +# +# During type checking, we want the type narrowed event classes to still have +# all the fields as a normal `Event`, so we make `EventProtocol` a subclass of +# `Event`. +# +# However, at runtime we a) can't subclass `Event` because it's a Rust class, +# and b) don't want to allow `isinstance` checks against `EventProtocol` (as +# it's purely a type annotation helper, not a real class). So at runtime, we +# make `EventProtocol` a class with a metaclass that raises on `isinstance` +# checks. +if TYPE_CHECKING: + + class EventProtocol(Event): + """Helper subclass that allows type narrowing for `EventBase` objects.""" + +else: + + class EventProtocol(metaclass=_DisableIsInstance): + """Helper subclass that allows type narrowing for `EventBase` objects.""" + + def __new__(cls): + raise NotImplementedError( + f"{cls.__name__} cannot be instantiated as it is not a real class." + ) class MSC4242Event(EventProtocol): """Marker protocol for events in MSC4242 rooms. This allows us to narrow the type of events.""" - prev_state_events: list[str] + prev_state_events: StrCollection def supports_msc4242_state_dag(event: EventBase) -> TypeIs[MSC4242Event]: diff --git a/synapse/events/utils.py b/synapse/events/utils.py index adbede7f160..54f662796b5 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -39,18 +39,16 @@ CANONICALJSON_MAX_INT, CANONICALJSON_MIN_INT, MAX_PDU_SIZE, - EventContentFields, EventTypes, EventUnsignedContentFields, RelationTypes, ) from synapse.api.errors import Codes, SynapseError -from synapse.api.room_versions import RoomVersion from synapse.logging.opentracing import SynapseTags, set_tag, trace -from synapse.synapse_rust.events import Unsigned +from synapse.synapse_rust.events import Unsigned, redact_event from synapse.types import JsonDict, Requester -from . import EventBase, FrozenEventV2, StrippedStateEvent, make_event_from_dict +from . import EventBase, StrippedStateEvent if TYPE_CHECKING: from synapse.handlers.relations import BundledAggregations @@ -78,177 +76,18 @@ def prune_event(event: EventBase) -> EventBase: the user has specified, but we do want to keep necessary information like type, state_key etc. """ - pruned_event_dict = prune_event_dict(event.room_version, event.get_dict()) - - pruned_event = make_event_from_dict( - pruned_event_dict, event.room_version, event.internal_metadata.get_dict() - ) - - # Copy the bits of `internal_metadata` that aren't returned by `get_dict` - pruned_event.internal_metadata.stream_ordering = ( - event.internal_metadata.stream_ordering - ) - pruned_event.internal_metadata.instance_name = event.internal_metadata.instance_name - pruned_event.internal_metadata.outlier = event.internal_metadata.outlier - pruned_event.internal_metadata.redacted_by = event.internal_metadata.redacted_by - - # Mark the event as redacted - pruned_event.internal_metadata.redacted = True - - return pruned_event + return redact_event(event) def clone_event(event: EventBase) -> EventBase: """Take a copy of the event. - This is mostly useful because it does a *shallow* copy of the `unsigned` data, - which means it can then be updated without corrupting the in-memory cache. Note that - other properties of the event, such as `content`, are *not* (currently) copied here. - """ - # XXX: We rely on at least one of `event.get_dict()` and `make_event_from_dict()` - # making a copy of `unsigned`. Currently, both do, though I don't really know why. - # Still, as long as they do, there's not much point doing yet another copy here. - new_event = make_event_from_dict( - event.get_dict(), event.room_version, event.internal_metadata.get_dict() - ) - - # Starting FrozenEventV2, the event ID is an (expensive) hash of the event. This is - # lazily computed when we get the FrozenEventV2.event_id property, then cached in - # _event_id field. Later FrozenEvent formats all inherit from FrozenEventV2, so we - # can use the same logic here. - if isinstance(event, FrozenEventV2) and isinstance(new_event, FrozenEventV2): - # If we already pre-computed the event ID, use it. - new_event._event_id = event._event_id - - # Copy the bits of `internal_metadata` that aren't returned by `get_dict`. - new_event.internal_metadata.stream_ordering = ( - event.internal_metadata.stream_ordering - ) - new_event.internal_metadata.instance_name = event.internal_metadata.instance_name - new_event.internal_metadata.outlier = event.internal_metadata.outlier - new_event.internal_metadata.redacted_by = event.internal_metadata.redacted_by - - return new_event - - -def prune_event_dict(room_version: RoomVersion, event_dict: JsonDict) -> JsonDict: - """Redacts the event_dict in the same way as `prune_event`, except it - operates on dicts rather than event objects - - Returns: - A copy of the pruned event dict + Most fields of the event are immutable, however fields such as `unsigned`, + `signatures` and `internal_metadata` are mutable. Cloning the event allows + us to edit such fields without affecting the original event. """ - allowed_keys = [ - "event_id", - "sender", - "room_id", - "hashes", - "signatures", - "content", - "type", - "state_key", - "depth", - "prev_events", - "auth_events", - "origin_server_ts", - ] - - # Earlier room versions from had additional allowed keys. - if not room_version.updated_redaction_rules: - allowed_keys.extend(["prev_state", "membership", "origin"]) - # Custom room versions add new allowed keys and remove others - if room_version.msc4242_state_dags: - allowed_keys.extend(["prev_state_events"]) - allowed_keys.remove("auth_events") - - event_type = event_dict["type"] - - new_content = {} - - def add_fields(*fields: str) -> None: - for field in fields: - if field in event_dict["content"]: - new_content[field] = event_dict["content"][field] - - if event_type == EventTypes.Member: - add_fields("membership") - if room_version.restricted_join_rule_fix: - add_fields(EventContentFields.AUTHORISING_USER) - if room_version.updated_redaction_rules: - # Preserve the signed field under third_party_invite. - third_party_invite = event_dict["content"].get("third_party_invite") - if isinstance(third_party_invite, collections.abc.Mapping): - new_content["third_party_invite"] = {} - if "signed" in third_party_invite: - new_content["third_party_invite"]["signed"] = third_party_invite[ - "signed" - ] - - elif event_type == EventTypes.Create: - if room_version.updated_redaction_rules: - # MSC2176 rules state that create events cannot have their `content` redacted. - new_content = event_dict["content"] - if not room_version.implicit_room_creator: - # Some room versions give meaning to `creator` - add_fields("creator") - if room_version.msc4291_room_ids_as_hashes: - # room_id is not allowed on the create event as it's derived from the event ID - allowed_keys.remove("room_id") - - elif event_type == EventTypes.JoinRules: - add_fields("join_rule") - if room_version.restricted_join_rule: - add_fields("allow") - elif event_type == EventTypes.PowerLevels: - add_fields( - "users", - "users_default", - "events", - "events_default", - "state_default", - "ban", - "kick", - "redact", - ) - - if room_version.updated_redaction_rules: - add_fields("invite") - - elif event_type == EventTypes.Aliases and room_version.special_case_aliases_auth: - add_fields("aliases") - elif event_type == EventTypes.RoomHistoryVisibility: - add_fields("history_visibility") - elif event_type == EventTypes.Redaction and room_version.updated_redaction_rules: - add_fields("redacts") - - # Protect the rel_type and event_id fields under the m.relates_to field. - if room_version.msc3389_relation_redactions: - relates_to = event_dict["content"].get("m.relates_to") - if isinstance(relates_to, collections.abc.Mapping): - new_relates_to = {} - for field in ("rel_type", "event_id"): - if field in relates_to: - new_relates_to[field] = relates_to[field] - # Only include a non-empty relates_to field. - if new_relates_to: - new_content["m.relates_to"] = new_relates_to - - allowed_fields = {k: v for k, v in event_dict.items() if k in allowed_keys} - - allowed_fields["content"] = new_content - - unsigned: JsonDict = {} - allowed_fields["unsigned"] = unsigned - - event_unsigned = event_dict.get("unsigned", {}) - - if "age_ts" in event_unsigned: - unsigned["age_ts"] = event_unsigned["age_ts"] - if "replaces_state" in event_unsigned: - unsigned["replaces_state"] = event_unsigned["replaces_state"] - - return allowed_fields + return event.deep_copy() def _copy_field(src: JsonDict, dst: JsonDict, field: list[str]) -> None: diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 2b5ef5fbacc..5b334543256 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -21,7 +21,6 @@ # -import copy import itertools import logging from typing import ( @@ -1192,7 +1191,7 @@ async def _execute(pdu: EventBase) -> None: # NB: We *need* to copy to ensure that we don't have multiple # references being passed on, as that causes... issues. signed_state = [ - copy.copy(valid_pdus_map[p.event_id]) + valid_pdus_map[p.event_id].deep_copy() for p in state if p.event_id in valid_pdus_map ] @@ -1203,11 +1202,6 @@ async def _execute(pdu: EventBase) -> None: if p.event_id in valid_pdus_map ] - # NB: We *need* to copy to ensure that we don't have multiple - # references being passed on, as that causes... issues. - for s in signed_state: - s.internal_metadata = s.internal_metadata.copy() - # double-check that the auth chain doesn't include a different create event auth_chain_create_events = [ e.event_id diff --git a/synapse/module_api/callbacks/third_party_event_rules_callbacks.py b/synapse/module_api/callbacks/third_party_event_rules_callbacks.py index 65f5a6b1837..68053d65a31 100644 --- a/synapse/module_api/callbacks/third_party_event_rules_callbacks.py +++ b/synapse/module_api/callbacks/third_party_event_rules_callbacks.py @@ -283,11 +283,6 @@ async def check_event_allowed( events = await self.store.get_events(prev_state_ids.values()) state_events = {(ev.type, ev.state_key): ev for ev in events.values()} - # Ensure that the event is frozen, to make sure that the module is not tempted - # to try to modify it. Any attempt to modify it at this point will invalidate - # the hashes and signatures. - event.freeze() - for callback in self._check_event_allowed_callbacks: try: res, replacement_data = await delay_cancellation( diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index 3783211a927..f6693e09236 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -58,7 +58,14 @@ from synapse.rest.client.room import SerializeMessagesDeps, encode_messages_response from synapse.storage.databases.main.room import RoomSortOrder from synapse.streams.config import PaginationConfig -from synapse.types import JsonDict, RoomID, ScheduledTask, UserID, create_requester +from synapse.types import ( + JsonDict, + JsonMapping, + RoomID, + ScheduledTask, + UserID, + create_requester, +) from synapse.types.state import StateFilter if TYPE_CHECKING: @@ -682,6 +689,7 @@ async def on_POST( create_event = filtered_room_state[(EventTypes.Create, "")] power_levels = filtered_room_state.get((EventTypes.PowerLevels, "")) + pl_content: JsonMapping if power_levels is not None: # We pick the local user with the highest power. user_power = power_levels.content.get("users", {}) diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index 62e84f5ac5c..13f958d9983 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -36,7 +36,6 @@ Iterable, Sequence, TypeVar, - cast, ) import attr @@ -870,7 +869,7 @@ async def _process_state_dag_forward_extremities_and_state_delta( ) new_room_dag_fwd_extrems = await self._calculate_new_extremities( room_id, - cast(list[EventPersistencePair], event_contexts), + event_contexts, existing_room_dag_fwd_extrems, ) assert new_room_dag_fwd_extrems, ( @@ -889,7 +888,7 @@ async def _process_state_dag_forward_extremities_and_state_delta( ): (current_state, delta_ids, _) = await self._get_new_state_after_events( room_id, - cast(list[EventPersistencePair], event_contexts), + event_contexts, existing_state_dag_fwd_extrems, new_state_dag_fwd_extrems, # do not prune forward extremities in the state DAG @@ -923,7 +922,7 @@ async def _process_state_dag_forward_extremities_and_state_delta( # extremities. is_still_joined = await self._is_server_still_joined( room_id, - cast(list[EventPersistencePair], event_contexts), + event_contexts, delta, ) if not is_still_joined: @@ -1053,7 +1052,7 @@ async def _calculate_new_state_dag_extremities( async def _calculate_new_extremities( self, room_id: str, - event_contexts: list[EventPersistencePair], + event_contexts: Sequence[EventPersistencePair], latest_event_ids: AbstractSet[str], ) -> set[str]: """Calculates the new forward extremities for a room given events to @@ -1113,7 +1112,7 @@ async def _calculate_new_extremities( async def _get_new_state_after_events( self, room_id: str, - events_context: list[EventPersistencePair], + events_context: Sequence[EventPersistencePair], old_latest_event_ids: AbstractSet[str], new_latest_event_ids: set[str], should_prune: bool = True, @@ -1297,7 +1296,7 @@ async def _prune_extremities( new_latest_event_ids: set[str], resolved_state_group: int, event_id_to_state_group: dict[str, int], - events_context: list[EventPersistencePair], + events_context: Sequence[EventPersistencePair], ) -> set[str]: """See if we can prune any of the extremities after calculating the resolved state. @@ -1434,7 +1433,7 @@ async def _calculate_state_delta( async def _is_server_still_joined( self, room_id: str, - ev_ctx_rm: list[EventPersistencePair], + ev_ctx_rm: Sequence[EventPersistencePair], delta: DeltaState, ) -> bool: """Check if the server will still be joined after the given events have diff --git a/synapse/storage/databases/main/censor_events.py b/synapse/storage/databases/main/censor_events.py index a5ae4bf506a..92770b65906 100644 --- a/synapse/storage/databases/main/censor_events.py +++ b/synapse/storage/databases/main/censor_events.py @@ -22,7 +22,6 @@ import logging from typing import TYPE_CHECKING -from synapse.events.utils import prune_event_dict from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.storage._base import SQLBaseStore from synapse.storage.database import ( @@ -32,6 +31,7 @@ ) from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.databases.main.events_worker import EventsWorkerStore +from synapse.synapse_rust.events import redact_event from synapse.util.duration import Duration from synapse.util.json import json_encoder @@ -123,9 +123,7 @@ async def _censor_redactions(self) -> None: ): # Redaction was allowed pruned_json: str | None = json_encoder.encode( - prune_event_dict( - original_event.room_version, original_event.get_dict() - ) + redact_event(original_event).get_pdu_json() ) else: # Redaction wasn't allowed @@ -190,9 +188,7 @@ def delete_expired_event_txn(txn: LoggingTransaction) -> None: return # Prune the event's dict then convert it to JSON. - pruned_json = json_encoder.encode( - prune_event_dict(event.room_version, event.get_dict()) - ) + pruned_json = json_encoder.encode(redact_event(event).get_pdu_json()) # Update the event_json table to replace the event's JSON with the pruned # JSON. diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 5aab0067fc9..84b38f4bf2a 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -915,7 +915,9 @@ def calculate_chain_cover_index_for_events_txn( # instances as we'll potentially be pulling more events from the DB and # we don't need the overhead of fetching/parsing the full event JSON. event_to_types = {e.event_id: (e.type, e.state_key) for e in state_events} - event_to_auth_chain = {e.event_id: e.auth_event_ids() for e in state_events} + event_to_auth_chain: dict[str, StrCollection] = { + e.event_id: e.auth_event_ids() for e in state_events + } event_to_room_id = {e.event_id: e.room_id for e in state_events} return self._calculate_chain_cover_index( diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index c0d218398d9..9a11f9b9bb7 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -38,7 +38,6 @@ resign_event, ) from synapse.events import EventBase, make_event_from_dict -from synapse.events.utils import prune_event_dict from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause from synapse.storage.database import ( DatabasePool, @@ -63,6 +62,7 @@ from synapse.storage.databases.main.stream import StreamWorkerStore from synapse.storage.engines import PostgresEngine from synapse.storage.types import Cursor +from synapse.synapse_rust.events import redact_event from synapse.types import JsonDict, RoomStreamToken, StateMap, StrCollection from synapse.types.handlers import SLIDING_SYNC_DEFAULT_BUMP_EVENT_TYPES from synapse.types.state import StateFilter @@ -2831,7 +2831,7 @@ def _fetch_next_events_txn( # Verify the signature is genuinely from this key. We prune # first since signatures are computed over the redacted form. - pruned = prune_event_dict(event.room_version, event.get_pdu_json()) + pruned = redact_event(event).get_pdu_json() try: verify_signed_json(pruned, self.hs.hostname, old_verify_key) except SignatureVerifyException: diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 6f26bd17cec..4b372de1411 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -37,6 +37,7 @@ import attr from prometheus_client import Gauge +from typing_extensions import assert_never from twisted.internet import defer @@ -775,6 +776,14 @@ async def get_events_as_list( continue elif redact_behaviour == EventRedactBehaviour.redact: event = entry.redacted_event + elif redact_behaviour == EventRedactBehaviour.as_is: + # Allow event through as is + pass + else: + # We (should) have covered all possible values of + # redact_behaviour, so this is unreachable. + assert_never(redact_behaviour) + raise ValueError(f"Unknown redact_behaviour {redact_behaviour}") events.append(event) @@ -1507,12 +1516,17 @@ async def _fetch_event_ids_and_get_outstanding_redactions( ) continue - original_ev = make_event_from_dict( - event_dict=d, - room_version=room_version, - internal_metadata_dict=internal_metadata, - rejected_reason=rejected_reason, - ) + try: + original_ev = make_event_from_dict( + event_dict=d, + room_version=room_version, + internal_metadata_dict=internal_metadata, + rejected_reason=rejected_reason, + ) + except SynapseError: + logger.error("Unable to parse event from database: %s", event_id) + continue + original_ev.internal_metadata.stream_ordering = row.stream_ordering original_ev.internal_metadata.instance_name = row.instance_name original_ev.internal_metadata.outlier = row.outlier diff --git a/synapse/synapse_rust/events.pyi b/synapse/synapse_rust/events.pyi index 5b55d47f0d5..9a69438c1df 100644 --- a/synapse/synapse_rust/events.pyi +++ b/synapse/synapse_rust/events.pyi @@ -12,7 +12,9 @@ from typing import Any, Iterator, Mapping -from synapse.types import JsonDict, JsonMapping +from synapse.synapse_rust.room_versions import RoomVersion +from synapse.types import JsonDict, JsonMapping, StrSequence +from synapse.util.duration import Duration class EventInternalMetadata: def __init__(self, internal_metadata_dict: JsonDict): ... @@ -159,60 +161,60 @@ class Signatures: """A class representing the signatures on an event.""" def __init__(self, signatures: Mapping[str, Mapping[str, str]] | None = None): ... - def get_signature(self, server_name: str, key_id: str) -> str | None: ... - """Get the signature for the given server name and key ID, if it exists.""" + def get_signature(self, server_name: str, key_id: str) -> str | None: + """Get the signature for the given server name and key ID, if it exists.""" - def __getitem__(self, server_name: str) -> Mapping[str, str]: ... - """Get the signatures for the given server name. Raises KeyError if there - are no signatures for that server.""" + def __getitem__(self, server_name: str) -> Mapping[str, str]: + """Get the signatures for the given server name. Raises KeyError if there + are no signatures for that server.""" - def __contains__(self, server_name: Any) -> bool: ... - """Check if there are signatures for the given server name.""" + def __contains__(self, server_name: Any) -> bool: + """Check if there are signatures for the given server name.""" - def __len__(self) -> int: ... - """Return the number of servers that have signatures.""" + def __len__(self) -> int: + """Return the number of servers that have signatures.""" - def add_signature(self, server_name: str, key_id: str, signature: str) -> None: ... - """Add a signature for the given server name and key ID.""" + def add_signature(self, server_name: str, key_id: str, signature: str) -> None: + """Add a signature for the given server name and key ID.""" - def update(self, signatures: Mapping[str, Mapping[str, str]]) -> None: ... - """Update the signatures with the given signatures. + def update(self, signatures: Mapping[str, Mapping[str, str]]) -> None: + """Update the signatures with the given signatures. - Will overwrite all existing signatures for the server names provided. - """ + Will overwrite all existing signatures for the server names provided. + """ - def as_dict(self) -> dict[str, dict[str, str]]: ... - """Return a copy of the signatures as a dictionary.""" + def as_dict(self) -> dict[str, dict[str, str]]: + """Return a copy of the signatures as a dictionary.""" class Unsigned: """A class representing the unsigned data of an event.""" def __init__(self, unsigned_dict: JsonMapping): ... - def __getitem__(self, key: str) -> Any: ... - """Get the value for the given key. + def __getitem__(self, key: str) -> Any: + """Get the value for the given key. - Raises KeyError if the key is unset or not recognised.""" + Raises KeyError if the key is unset or not recognised.""" - def __setitem__(self, key: str, value: Any) -> None: ... - """Set the value for the given key. + def __setitem__(self, key: str, value: Any) -> None: + """Set the value for the given key. - Raises KeyError if the key is not recognised.""" + Raises KeyError if the key is not recognised.""" - def __delitem__(self, key: str) -> None: ... - """Delete the value for the given key. + def __delitem__(self, key: str) -> None: + """Delete the value for the given key. - Raises KeyError if the key is unset or not recognised.""" + Raises KeyError if the key is unset or not recognised.""" def __contains__(self, key: Any) -> bool: ... - def get(self, key: str, default: Any = None) -> Any: ... - """Get the value for the given key, or ``default`` if the key is unset.""" + def get(self, key: str, default: Any = None) -> Any: + """Get the value for the given key, or ``default`` if the key is unset.""" - def for_persistence(self) -> JsonDict: ... - """Return a dict of the fields that should be persisted to the database.""" + def for_persistence(self) -> JsonDict: + """Return a dict of the fields that should be persisted to the database.""" - def for_event(self) -> JsonDict: ... - """Return a dict of all unsigned fields, including those only kept in - memory, suitable for inclusion in an event.""" + def for_event(self) -> JsonDict: + """Return a dict of all unsigned fields, including those only kept in + memory, suitable for inclusion in an event.""" class JsonObject(Mapping[str, Any]): """Immutable JSON object mapping.""" @@ -222,3 +224,99 @@ class JsonObject(Mapping[str, Any]): def __getitem__(self, key: str) -> Any: ... def __iter__(self) -> Iterator[str]: ... def __eq__(self, other: object) -> bool: ... + +class Event: + """Represents a Matrix event.""" + + def __init__( + self, + event_dict: JsonDict, + room_version: RoomVersion, + internal_metadata_dict: JsonDict, + rejected_reason: str | None, + ) -> None: ... + def get_dict(self) -> JsonDict: + """Convert the event to a dictionary suitable for serialisation.""" + + def get_dict_for_persistence(self) -> JsonDict: + """Like ``get_dict``, but serializes ``unsigned`` in a form suitable for + persistence.""" + + def get_pdu_json(self, time_now: int | None = None) -> JsonDict: + """Like ``get_dict``, but serializes ``unsigned`` in a form suitable + for sending over federation.""" + + def get_templated_pdu_json(self) -> JsonDict: + """Like ``get_dict``, except strips fields like ``signatures``, + ``hashes`` and ``unsigned`` so that the result is suitable as a template for + creating new events. Used in make_{join,leave,knock} flows.""" + + @property + def event_id(self) -> str: ... + @property + def room_id(self) -> str: ... + @property + def signatures(self) -> Signatures: ... + @property + def content(self) -> JsonMapping: ... + @property + def depth(self) -> int: ... + @property + def hashes(self) -> dict[str, str]: ... + @property + def origin_server_ts(self) -> int: ... + @property + def sender(self) -> str: ... + @property + def state_key(self) -> str: ... + @property + def type(self) -> str: ... + @property + def unsigned(self) -> Unsigned: ... + @property + def internal_metadata(self) -> EventInternalMetadata: ... + @property + def rejected_reason(self) -> str | None: ... + @property + def room_version(self) -> RoomVersion: ... + @property + def format_version(self) -> int: + """The EventFormatVersion implemented by this event.""" + + @property + def membership(self) -> Any: ... + @property + def redacts(self) -> Any | None: ... + def prev_event_ids(self) -> StrSequence: + """Returns the list of prev event IDs.""" + + def auth_event_ids(self) -> StrSequence: + """Returns the list of auth event IDs""" + + def is_state(self) -> bool: ... + def get_state_key(self) -> str | None: + """Get the state key of this event, or None if it's not a state event.""" + def __contains__(self, key: str) -> bool: ... + def get(self, key: str, default: Any = None) -> Any: ... + def items(self) -> list[tuple[str, Any]]: ... + def keys(self) -> list[str]: ... + def deep_copy(self) -> "Event": + """Returns a deep copy of this object, such that modifying the copy will + not affect the original.""" + + def sticky_duration(self) -> Duration | None: + """If this event has the ``msc4354_sticky`` top-level field, returns a + ``SynapseDuration`` representing the sticky duration. Otherwise returns + ``None``.""" + +def redact_event(event: Event) -> Event: + """Returns a pruned version of the given event, which removes all keys we + don't know about or think could potentially be dodgy. + """ + +def redact_event_dict(room_version: RoomVersion, event_dict: JsonMapping) -> JsonDict: + """Returns a pruned version of the given event dict, which removes all keys + we don't know about or think could potentially be dodgy. + + Returns the redacted event as a dict. + """ diff --git a/tests/events/test_py_protocol.py b/tests/events/test_py_protocol.py index 306e3c17042..cfcc3648caf 100644 --- a/tests/events/test_py_protocol.py +++ b/tests/events/test_py_protocol.py @@ -16,7 +16,7 @@ from unittest.mock import Mock from synapse.api.room_versions import RoomVersion, RoomVersions -from synapse.events import EventBase, FrozenEvent, make_event_from_dict +from synapse.events import EventBase from synapse.events.py_protocol import ( EventProtocol, MSC4242Event, @@ -24,20 +24,20 @@ supports_msc4242_state_dag, ) +from tests.test_utils.event_builders import make_test_event from tests.unittest import TestCase def _make_event(room_version: RoomVersion) -> EventBase: """Helper to make an EventBase with the given room version.""" - event_dict = { - "content": {}, - "sender": "@user:example.com", - "type": "m.room.message", - "room_id": "!room:example.com", - } - if room_version.msc4242_state_dags: - event_dict["prev_state_events"] = [] - return make_event_from_dict(event_dict, room_version=room_version) + return make_test_event( + { + "sender": "@user:example.com", + "type": "m.room.message", + "room_id": "!room:example.com", + }, + room_version=room_version, + ) class TestMetaClass(TestCase): @@ -46,15 +46,14 @@ def test_is_instance(self) -> None: NotImplementedError, but that isinstance checks on EventBase and FrozenEvent still work as normal. """ - # EventBase and FrozenEvent should work as normal + # EventBase should work as normal self.assertFalse(isinstance(object(), EventBase)) - self.assertFalse(isinstance(object(), FrozenEvent)) with self.assertRaises(NotImplementedError): - isinstance(object(), EventProtocol) + isinstance(object(), MSC4242Event) with self.assertRaises(NotImplementedError): - isinstance(object(), MSC4242Event) + isinstance(object(), EventProtocol) class SupportsMSC4242StateDagTestCase(TestCase): diff --git a/tests/events/test_validator.py b/tests/events/test_validator.py index 3810fdb3da6..082ae04a4c7 100644 --- a/tests/events/test_validator.py +++ b/tests/events/test_validator.py @@ -19,10 +19,10 @@ class EventValidatorTestCase(HomeserverTestCase): - def test_validate_new_with_mentions_succeeds_even_when_frozen(self) -> None: + def test_validate_new_with_mentions_succeed(self) -> None: """ Test that `EventValidator.validate_new` accepts an event with valid `m.mentions` - content even when the event is frozen. + content. """ event = make_event_from_dict( { @@ -43,8 +43,5 @@ def test_validate_new_with_mentions_succeeds_even_when_frozen(self) -> None: }, room_version=RoomVersions.V9, ) - # Sanity check that the event is valid before freezing - EventValidator().validate_new(event, self.hs.config) - event.freeze() - # Event should still be valid after freezing + EventValidator().validate_new(event, self.hs.config) diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index 9e44b1dc1e0..736f251c277 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -36,7 +36,7 @@ from synapse.api.room_versions import RoomVersions from synapse.appservice import ApplicationService from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import EventBase, FrozenEventV3 +from synapse.events import EventBase, make_event_from_dict from synapse.federation.federation_client import SendJoinResult from synapse.federation.transport.client import ( StateRequestResponse, @@ -677,7 +677,7 @@ def _build_public_room(self) -> StateMap[EventBase]: self.REMOTE1_SERVER_SIGNATURE_KEY, ) - create_event = FrozenEventV3(create_event_dict, room_version, {}, None) + create_event = make_event_from_dict(create_event_dict, room_version) events.append(create_event) room_version = self.hs.config.server.default_room_version @@ -700,7 +700,7 @@ def _build_public_room(self) -> StateMap[EventBase]: self.hs.hostname, self.hs.signing_key, ) - join_event = FrozenEventV3(join_event_dict, room_version, {}, None) + join_event = make_event_from_dict(join_event_dict, room_version) events.append(join_event) # Then set the join rules to public @@ -722,7 +722,7 @@ def _build_public_room(self) -> StateMap[EventBase]: self.REMOTE1_SERVER_NAME, self.REMOTE1_SERVER_SIGNATURE_KEY, ) - join_rules_event = FrozenEventV3(join_rules_event_dict, room_version, {}, None) + join_rules_event = make_event_from_dict(join_rules_event_dict, room_version) events.append(join_rules_event) return {(event.type, event.state_key): event for event in events} @@ -733,7 +733,7 @@ def _build_signed_join_event( user: str, signing_key: SigningKey, state: StateMap[EventBase], - ) -> FrozenEventV3: + ) -> EventBase: """Build a join event for the local user, signed by the local server.""" latest_event = max(state.values(), key=lambda e: e.depth) @@ -759,7 +759,7 @@ def _build_signed_join_event( get_domain_from_id(user), signing_key, ) - return FrozenEventV3(join_event_dict, room_version, {}, None) + return make_event_from_dict(join_event_dict, room_version) @parameterized.expand([("not_pruned", False), ("pruned", True)]) @patch( diff --git a/tests/handlers/test_room_member.py b/tests/handlers/test_room_member.py index b550c2420b5..9cc7ebfa803 100644 --- a/tests/handlers/test_room_member.py +++ b/tests/handlers/test_room_member.py @@ -8,7 +8,7 @@ from synapse.api.constants import AccountDataTypes, EventTypes, Membership from synapse.api.errors import Codes, LimitExceededError, SynapseError from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import FrozenEventV3 +from synapse.events import Event from synapse.federation.federation_client import SendJoinResult from synapse.server import HomeServer from synapse.types import UserID, create_requester @@ -124,7 +124,7 @@ def test_remote_joins_contribute_to_rate_limit(self) -> None: create_event_source, self.hs.config.server.default_room_version, ) - create_event = FrozenEventV3( + create_event = Event( create_event_source, self.hs.config.server.default_room_version, {}, @@ -148,7 +148,7 @@ def test_remote_joins_contribute_to_rate_limit(self) -> None: self.hs.hostname, self.hs.signing_key, ) - join_event = FrozenEventV3( + join_event = Event( join_event_source, self.hs.config.server.default_room_version, {}, diff --git a/tests/handlers/test_room_policy.py b/tests/handlers/test_room_policy.py index 4f2188b8e71..c67ea9b0e02 100644 --- a/tests/handlers/test_room_policy.py +++ b/tests/handlers/test_room_policy.py @@ -27,7 +27,6 @@ from synapse.rest import admin from synapse.rest.client import filter, login, room, sync from synapse.server import HomeServer -from synapse.synapse_rust.events import Signatures from synapse.types import JsonDict, UserID from synapse.util.clock import Clock @@ -182,7 +181,7 @@ def _sign_with_random_key(self, server_name: str, event: EventBase) -> None: non_policyserver_key = signedjson.key.generate_signing_key( "non_policyserver_key" ) - event.signatures = Signatures( + event.signatures.update( compute_event_signature( event.room_version, event.get_dict(), diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py index 2ba5da3b95f..3114675052b 100644 --- a/tests/module_api/test_api.py +++ b/tests/module_api/test_api.py @@ -841,10 +841,10 @@ def test_event_deprecated_methods(self) -> None: create_event = state[(EventTypes.Create, "")] # `.user_id` is a deprecated alias for `.sender`. - self.assertEqual(create_event.user_id, user_id) + self.assertEqual(create_event.user_id, user_id) # type: ignore[attr-defined] # The event supports looking up keys via `__getitem__` although deprecated - self.assertEqual(create_event["room_id"], room_id) + self.assertEqual(create_event["room_id"], room_id) # type: ignore[index] class ModuleApiWorkerTestCase(BaseModuleApiTestCase, BaseMultiWorkerStreamTestCase): diff --git a/tests/replication/storage/test_events.py b/tests/replication/storage/test_events.py index b7b94482ef0..d7e6dfca832 100644 --- a/tests/replication/storage/test_events.py +++ b/tests/replication/storage/test_events.py @@ -272,6 +272,7 @@ def build_event( "origin_server_ts": self.event_id, "prev_events": prev_events, "auth_events": auth_events, + "hashes": {}, } if key is not None: event_dict["state_key"] = key diff --git a/tests/rest/client/test_third_party_rules.py b/tests/rest/client/test_third_party_rules.py index b4fa71ece8e..5eaa6f9fb28 100644 --- a/tests/rest/client/test_third_party_rules.py +++ b/tests/rest/client/test_third_party_rules.py @@ -236,7 +236,10 @@ def test_cannot_modify_event(self) -> None: async def check( ev: EventBase, state: StateMap[EventBase] ) -> tuple[bool, JsonDict | None]: - ev.content = {"x": "y"} + # Try and modify the content, this will fail because the event is + # immutable. (We therefore need the type ignore linter, as the + # linter will pick this bug up) + ev.content = {"x": "y"} # type: ignore[misc] return True, None self.hs.get_module_api_callbacks().third_party_event_rules._check_event_allowed_callbacks = [ diff --git a/tests/state/test_v2.py b/tests/state/test_v2.py index 8b3b919f441..a9924e28b18 100644 --- a/tests/state/test_v2.py +++ b/tests/state/test_v2.py @@ -97,9 +97,6 @@ def to_event(self, auth_events: list[str], prev_events: list[str]) -> EventBase: Args: auth_events: list of event_ids prev_events: list of event_ids - - Returns: - FrozenEvent """ global ORIGIN_SERVER_TS diff --git a/tests/storage/test_msc4242_state_dag.py b/tests/storage/test_msc4242_state_dag.py index 2150bc09960..52a165b9b24 100644 --- a/tests/storage/test_msc4242_state_dag.py +++ b/tests/storage/test_msc4242_state_dag.py @@ -19,13 +19,13 @@ from synapse.api.constants import EventTypes from synapse.api.errors import SynapseError from synapse.api.room_versions import RoomVersions -from synapse.events.py_protocol import MSC4242Event, supports_msc4242_state_dag +from synapse.events import EventBase +from synapse.events.py_protocol import MSC4242Event from synapse.events.snapshot import EventContext from synapse.rest.client import room from synapse.server import HomeServer from synapse.util.clock import Clock -from tests.test_utils.event_builders import make_test_event from tests.unittest import HomeserverTestCase, override_config @@ -154,21 +154,16 @@ def _make_event( prev_state_events: list[str], rejected: bool = False, ) -> tuple[MSC4242Event, EventContext]: - ev = make_test_event( - { - "prev_state_events": prev_state_events, - "content": { - "membership": "join", - }, - "sender": "@unimportant:info", - "state_key": "@unimportant:info", - "type": "m.room.member", - "room_id": self.room_id, - }, - room_version=RoomVersions.MSC4242v12, - ) - ev._event_id = id # type: ignore[attr-defined] - assert supports_msc4242_state_dag(ev) + # We use a mock here to allow us to set the `event_id`. + # + # FIXME: Having consistent human-readable event IDs in these tests is + # nice but the `Mock` is less than ideal. It would be better to use a + # real event but that is more complex to set up. + ev = Mock(spec=EventBase) + ev.event_id = id + ev.prev_state_events = prev_state_events + ev.state_key = "@unimportant:info" + ev.is_state.return_value = True ctx = Mock() ctx.rejected = rejected return ev, ctx diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index c3462457060..93e0b4a2b1b 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -26,7 +26,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.room_versions import RoomVersion, RoomVersions -from synapse.events import EventBase +from synapse.events import EventBase, make_event_from_dict from synapse.events.builder import EventBuilder from synapse.server import HomeServer from synapse.synapse_rust.events import EventInternalMetadata @@ -238,11 +238,16 @@ async def build( prev_event_ids=prev_event_ids, auth_event_ids=auth_event_ids ) - built_event._event_id = self._event_id # type: ignore[attr-defined] - built_event._dict["event_id"] = self._event_id - assert built_event.event_id == self._event_id + event_dict = built_event.get_dict() + event_dict["event_id"] = self._event_id + rebuilt_event = make_event_from_dict( + event_dict, + room_version=built_event.room_version, + internal_metadata_dict=built_event.internal_metadata.get_dict(), + ) + assert rebuilt_event.event_id == self._event_id - return built_event + return rebuilt_event @property def room_id(self) -> str: diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index d51fa1f8bad..ba3b3802cf4 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -35,7 +35,7 @@ ) from synapse.api.filtering import Filter from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import FrozenEventV3 +from synapse.events import Event from synapse.federation.federation_client import SendJoinResult from synapse.rest import admin from synapse.rest.client import login, room @@ -1385,7 +1385,7 @@ def test_remote_join(self) -> None: create_event_source, self.hs.config.server.default_room_version, ) - create_event = FrozenEventV3( + create_event = Event( create_event_source, self.hs.config.server.default_room_version, {}, @@ -1408,7 +1408,7 @@ def test_remote_join(self) -> None: creator_join_event_source, self.hs.config.server.default_room_version, ) - creator_join_event = FrozenEventV3( + creator_join_event = Event( creator_join_event_source, self.hs.config.server.default_room_version, {}, @@ -1433,7 +1433,7 @@ def test_remote_join(self) -> None: self.hs.hostname, self.hs.signing_key, ) - join_event = FrozenEventV3( + join_event = Event( join_event_source, self.hs.config.server.default_room_version, {}, diff --git a/tests/test_event_auth.py b/tests/test_event_auth.py index 4537186ee6d..eeba22728e1 100644 --- a/tests/test_event_auth.py +++ b/tests/test_event_auth.py @@ -93,8 +93,8 @@ def test_rejected_auth_events(self) -> None: RoomVersions.V9, creator, "public", + rejected_reason="stinky", ) - rejected_join_rules.rejected_reason = "stinky" auth_events.append(rejected_join_rules) event_store.add_event(rejected_join_rules) @@ -1180,7 +1180,10 @@ def _random_state_event( def _join_rules_event( - room_version: RoomVersion, sender: str, join_rule: str + room_version: RoomVersion, + sender: str, + join_rule: str, + rejected_reason: str | None = None, ) -> EventBase: return make_test_event( { @@ -1194,6 +1197,7 @@ def _join_rules_event( }, }, room_version=room_version, + rejected_reason=rejected_reason, ) diff --git a/tests/test_utils/event_builders.py b/tests/test_utils/event_builders.py index a8eb586c1f2..a5d686801d7 100644 --- a/tests/test_utils/event_builders.py +++ b/tests/test_utils/event_builders.py @@ -76,6 +76,20 @@ def make_test_event( **(event_dict or {}), **fields, } + + # For room versions where the create event's room_id is derived from its + # event ID (v11+ format), omit the default room_id on create events so each + # create event ends up with a distinct room_id. + # + # We can't do this in the `default_event_fields` as we don't know the event + # type at that point. + if ( + room_version.msc4291_room_ids_as_hashes + and merged["type"] == "m.room.create" + and merged["state_key"] == "" + ): + merged.pop("room_id", None) + return make_event_from_dict( merged, room_version=room_version, From 387dfabe3b445c71ef423a420d97e107d2fb30ab Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Jun 2026 12:34:36 +0100 Subject: [PATCH 64/92] Fix loading 'invalid' event from the database (#19816) Follow on from #19701. Some Synapse servers may have events in their database that don't pass the canonical JSON checks. This is bad, but we still want to be able to load them nonetheless. --- changelog.d/19816.misc | 1 + rust/src/events/utils.rs | 17 ++++++++--------- synapse/storage/databases/main/events_worker.py | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 changelog.d/19816.misc diff --git a/changelog.d/19816.misc b/changelog.d/19816.misc new file mode 100644 index 00000000000..4663e8b9611 --- /dev/null +++ b/changelog.d/19816.misc @@ -0,0 +1 @@ +Port the python Event classes to Rust. diff --git a/rust/src/events/utils.rs b/rust/src/events/utils.rs index 7d33c52b440..b58edc93528 100644 --- a/rust/src/events/utils.rs +++ b/rust/src/events/utils.rs @@ -75,15 +75,14 @@ pub fn compute_event_reference_hash( redacted_value_mut.remove(UNSIGNED); redacted_value_mut.remove(AGE_TS); - let canonicalization_options = if room_version.strict_canonicaljson { - CanonicalizationOptions::strict() - } else { - CanonicalizationOptions::relaxed() - }; - - let json = - crate::canonical_json::to_string_canonical(&redacted_value_mut, canonicalization_options) - .map_err(|err| anyhow::anyhow!(err))?; + // We use `CanonicalizationOptions::relaxed()` as we have some events that + // have already been accepted with int fields outside the valid range. We + // still want to be able to load them and calculate their event ID. + let json = crate::canonical_json::to_string_canonical( + &redacted_value_mut, + CanonicalizationOptions::relaxed(), + ) + .map_err(|err| anyhow::anyhow!(err))?; let hash = Sha256::digest(json.as_bytes()); diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 4b372de1411..65b7d251d95 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -1523,8 +1523,8 @@ async def _fetch_event_ids_and_get_outstanding_redactions( internal_metadata_dict=internal_metadata, rejected_reason=rejected_reason, ) - except SynapseError: - logger.error("Unable to parse event from database: %s", event_id) + except SynapseError as e: + logger.error("Unable to parse event from database %s: %s", event_id, e) continue original_ev.internal_metadata.stream_ordering = row.stream_ordering From 5bf825c0166079ff7d4afa9052dc2dddafdb6efa Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Jun 2026 16:57:55 +0100 Subject: [PATCH 65/92] Limit to-device EDU sizes (#19617) This is based on https://github.com/element-hq/synapse/pull/18416, which got reverted (#19614) due to it incorrectly rejecting to-device messages to users with many devices (and thus breaking message sending). Fix https://github.com/element-hq/synapse/issues/17035 A to-device message content looks like: ```jsonc { "@user:domain": {"device1": {...}, "device2": {...}}, ... } ``` The previous PR would split up into multiple EDUs, each with a subset of the users. However, if one user's entry was too large it would not further split it up and then error out. The main change in this PR is to allow splitting up a single user into multiple EDUs. Other changes: 1. Rename to `SOFT_MAX_EDU_SIZE` to indicate that we sometimes send EDUs with larger size than that, and its more a target than a hard limit. 2. Check early if any to-device message (to a specific device) is too large to send, even if we're not going to send it over federation. This ensures that we catch issues where clients try to send too large to-device. This still means that if a client send a large individual to-device message it will fail, but I don't believe we ever send such large to-device messages (normally they're in the range of a few KB). --- I ended up changing the implementation a bunch to make it easy to reuse the code to split up dictionaries. Instead of repeatedly splitting up the EDU until each bit fits into the size, we instead record the size of each entry in the dict and instead split up based on cumulative size. This means we call `encode_canonical_json` on each entry rather than once on the entire struct, but its not significantly slower to do so. -- cc @MatMaul @MadLittleMods --------- Co-authored-by: Mathieu Velten Co-authored-by: mcalinghee Co-authored-by: Eric Eastwood --- changelog.d/19617.bugfix | 1 + synapse/api/constants.py | 27 ++ .../sender/per_destination_queue.py | 13 +- synapse/handlers/devicemessage.py | 204 ++++++++++++-- synapse/storage/databases/main/deviceinbox.py | 47 +++- synapse/util/__init__.py | 108 +++++++ tests/handlers/test_appservice.py | 4 +- tests/rest/client/test_sendtodevice.py | 265 +++++++++++++++++- tests/util/test_split_dict.py | 201 +++++++++++++ 9 files changed, 832 insertions(+), 38 deletions(-) create mode 100644 changelog.d/19617.bugfix create mode 100644 tests/util/test_split_dict.py diff --git a/changelog.d/19617.bugfix b/changelog.d/19617.bugfix new file mode 100644 index 00000000000..71f181fed40 --- /dev/null +++ b/changelog.d/19617.bugfix @@ -0,0 +1 @@ +A long queue of to-device messages could prevent outgoing federation because of the size of the transaction, let's limit the to-device EDU size to a reasonable value. diff --git a/synapse/api/constants.py b/synapse/api/constants.py index eb9e6cc39b9..acac0573340 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -30,6 +30,33 @@ # the max size of a (canonical-json-encoded) event MAX_PDU_SIZE = 65536 +# This isn't spec'ed but is our own reasonable default to play nice with Synapse's +# `max_request_size`/`max_request_body_size`. We chose the same as `MAX_PDU_SIZE` as our +# `max_request_body_size` math is currently limited by 200 `MAX_PDU_SIZE` things. The +# spec for a `/federation/v1/send` request sets the limit at 100 EDU's and 50 PDU's +# which is below that 200 `MAX_PDU_SIZE` limit (`max_request_body_size`). +# +# Allowing oversized EDU's results in failed `/federation/v1/send` transactions (because +# the request overall can overrun the `max_request_body_size`) which are retried over +# and over and prevent other outbound federation traffic from happening. +# +# We may send EDU's that are larger than this, but we aim to avoid doing so. +SOFT_MAX_EDU_SIZE = 65536 + +# This is the maximum size of the content of a to-device message. This is not +# (yet) spec'ed but is our own reasonable default. We need to set a limit on the +# size of to-device message contents, as they get sent over federation and +# therefore need to fit inside transactions. +# +# https://github.com/matrix-org/matrix-spec/pull/2340 tracks adding this to the +# spec. +MAX_TO_DEVICE_CONTENT_SIZE = SOFT_MAX_EDU_SIZE + +# This is defined in the Matrix spec and enforced by the receiver. +MAX_EDUS_PER_TRANSACTION = 100 +# A transaction can contain up to 100 EDUs but synapse reserves 10 EDUs for other purposes +# like trickling out some device list updates. +NUMBER_OF_RESERVED_EDUS_PER_TRANSACTION = 10 # The maximum allowed size of an HTTP request. # Other than media uploads, the biggest request we expect to see is a fully-loaded diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index cdacf16d725..32f8630c9d1 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -30,7 +30,11 @@ from twisted.internet import defer -from synapse.api.constants import EduTypes +from synapse.api.constants import ( + MAX_EDUS_PER_TRANSACTION, + NUMBER_OF_RESERVED_EDUS_PER_TRANSACTION, + EduTypes, +) from synapse.api.errors import ( FederationDeniedError, HttpResponseException, @@ -51,9 +55,6 @@ if TYPE_CHECKING: import synapse.server -# This is defined in the Matrix spec and enforced by the receiver. -MAX_EDUS_PER_TRANSACTION = 100 - logger = logging.getLogger(__name__) @@ -798,7 +799,9 @@ async def __aenter__(self) -> tuple[list[EventBase], list[Edu]]: ( to_device_edus, device_stream_id, - ) = await self.queue._get_to_device_message_edus(edu_limit - 10) + ) = await self.queue._get_to_device_message_edus( + edu_limit - NUMBER_OF_RESERVED_EDUS_PER_TRANSACTION + ) if to_device_edus: self._device_stream_id = device_stream_id diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index 0ef14b31dad..3be65d91f3f 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -23,8 +23,16 @@ from http import HTTPStatus from typing import TYPE_CHECKING, Any -from synapse.api.constants import EduTypes, EventContentFields, ToDeviceEventTypes -from synapse.api.errors import Codes, SynapseError +from canonicaljson import encode_canonical_json + +from synapse.api.constants import ( + MAX_TO_DEVICE_CONTENT_SIZE, + SOFT_MAX_EDU_SIZE, + EduTypes, + EventContentFields, + ToDeviceEventTypes, +) +from synapse.api.errors import Codes, EventSizeError, SynapseError from synapse.api.ratelimiting import Ratelimiter from synapse.logging.context import run_in_background from synapse.logging.opentracing import ( @@ -34,8 +42,9 @@ set_tag, ) from synapse.types import JsonDict, Requester, StreamKeyType, UserID, get_domain_from_id +from synapse.util import split_dict_to_fit_to_size from synapse.util.json import json_encoder -from synapse.util.stringutils import random_string +from synapse.util.stringutils import random_string_insecure_fast if TYPE_CHECKING: from synapse.server import HomeServer @@ -222,6 +231,7 @@ async def send_device_message( set_tag(SynapseTags.TO_DEVICE_TYPE, message_type) set_tag(SynapseTags.TO_DEVICE_SENDER, sender_user_id) local_messages = {} + # Map from destination (server) -> recipient (user ID) -> device_id -> JSON message content remote_messages: dict[str, dict[str, dict[str, JsonDict]]] = {} for user_id, by_device in messages.items(): if not UserID.is_valid(user_id): @@ -233,6 +243,24 @@ async def send_device_message( # add an opentracing log entry for each message for device_id, message_content in by_device.items(): + # Check the size of each message, as if these are too large we + # can't send them over federation. + # + # We do this for all to-device messages, even those that aren't + # over federation, so as to more easily catch clients that are + # sending excessively large messages. + if ( + message_len := len(encode_canonical_json(message_content)) + ) > MAX_TO_DEVICE_CONTENT_SIZE: + # 413 is currently an unspecced response for `/sendToDevice` but is + # probably the best thing we can do. + # https://github.com/matrix-org/matrix-spec/pull/2340 tracks adding + # this to the spec + raise EventSizeError( + f"To-device message for {user_id}:{device_id} is too large to send ({message_len} > {MAX_TO_DEVICE_CONTENT_SIZE})", + unpersistable=True, + ) + log_kv( { "event": "send_to_device_message", @@ -277,28 +305,33 @@ async def send_device_message( destination = get_domain_from_id(user_id) remote_messages.setdefault(destination, {})[user_id] = by_device - context = get_active_span_text_map() - - remote_edu_contents = {} - for destination, messages in remote_messages.items(): - # The EDU contains a "message_id" property which is used for - # idempotence. Make up a random one. - message_id = random_string(16) - log_kv({"destination": destination, "message_id": message_id}) - - remote_edu_contents[destination] = { - "messages": messages, - "sender": sender_user_id, - "type": message_type, - "message_id": message_id, - "org.matrix.opentracing_context": json_encoder.encode(context), - } - - # Add messages to the database. + # Add local messages to the database. # Retrieve the stream id of the last-processed to-device message. - last_stream_id = await self.store.add_messages_to_device_inbox( - local_messages, remote_edu_contents + last_stream_id = ( + await self.store.add_local_messages_from_client_to_device_inbox( + local_messages + ) ) + for destination, messages in remote_messages.items(): + split_edus = split_device_messages_into_edus( + sender_user_id, message_type, messages + ) + for edu in split_edus: + edu["org.matrix.opentracing_context"] = json_encoder.encode( + get_active_span_text_map() + ) + # Add remote messages to the database. + last_stream_id = ( + await self.store.add_remote_messages_from_client_to_device_inbox( + {destination: edu} + ) + ) + log_kv( + { + "destination": destination, + "message_id": edu["message_id"], + } + ) # Notify listeners that there are new to-device messages to process, # handing them the latest stream id. @@ -397,3 +430,128 @@ async def get_events_for_dehydrated_device( "events": messages, "next_batch": f"d{stream_id}", } + + +def split_device_messages_into_edus( + sender_user_id: str, + message_type: str, + messages_by_user_then_device: dict[str, dict[str, JsonDict]], +) -> list[JsonDict]: + """ + This function takes many to-device messages and fits/splits them into several EDUs + as necessary. We split the messages up as the overall request can overrun the + `max_request_body_size` and prevent outbound federation traffic because of the size + of the transaction (cf. `SOFT_MAX_EDU_SIZE`). + + Args: + sender_user_id: The user that is sending the to-device messages. + message_type: The type of to-device messages that are being sent. + messages_by_user_then_device: Dictionary of recipient user_id to recipient device_id to message. + + Returns: + Bin-packed list of EDU JSON content for the given to_device messages + """ + split_edus_content: list[JsonDict] = [] + + # The header size of the full EDU. + base_edu_size = _EMPTY_EDU_SIZE + len(sender_user_id) + len(message_type) + + # First split up the top-level dict of user_id to device messages. + for subset_messages, estimated_size in split_dict_to_fit_to_size( + messages_by_user_then_device, + soft_max_size=SOFT_MAX_EDU_SIZE, + wrapping_object_size=base_edu_size, + ): + # The returned subset might be larger than the soft max size if it + # contains a single entry that is larger than the soft max size. + if estimated_size <= SOFT_MAX_EDU_SIZE: + # This message fits in a single EDU, add it as is. + content = create_new_to_device_edu_content( + sender_user_id, message_type, subset_messages + ) + split_edus_content.append(content) + logger.debug( + "Created EDU with %d recipients from %s (message_id=%s), (total EDUs so far: %d)", + len(subset_messages), + sender_user_id, + content["message_id"], + len(split_edus_content), + ) + else: + # This message doesn't fit in a single EDU. We split the message up + # further by device. + # + # Note: `subset` should only have a single entry in it. + for recipient, messages_by_device in subset_messages.items(): + # The header size of the EDU for this recipient, which includes + # the size of the recipient user ID and the wrapping structure + # for the device messages. + subset_base_size = len( + encode_canonical_json( + _create_new_to_device_edu( + sender_user_id, message_type, {recipient: {}} + ) + ) + ) + + for subset_messages, _estimated_size in split_dict_to_fit_to_size( + messages_by_device, + soft_max_size=SOFT_MAX_EDU_SIZE, + wrapping_object_size=subset_base_size, + ): + # Again, the returned subset might be larger than the soft + # max size, but we can't split it any further so we have to + # add it as is. + content = create_new_to_device_edu_content( + sender_user_id, message_type, {recipient: subset_messages} + ) + split_edus_content.append(content) + logger.debug( + "Created EDU with %d recipients from %s (message_id=%s), (total EDUs so far: %d)", + len(subset_messages), + sender_user_id, + content["message_id"], + len(split_edus_content), + ) + + return split_edus_content + + +def create_new_to_device_edu_content( + sender_user_id: str, + message_type: str, + messages_by_user_then_device: dict[str, dict[str, JsonDict]], +) -> JsonDict: + """ + Create a new `m.direct_to_device` EDU `content` object with a unique message ID. + """ + # The EDU contains a "message_id" property which is used for + # idempotence. Make up a random one. + message_id = random_string_insecure_fast(16) + content = { + "sender": sender_user_id, + "type": message_type, + "message_id": message_id, + "messages": messages_by_user_then_device, + } + return content + + +def _create_new_to_device_edu( + sender_user_id: str, + message_type: str, + messages_by_user_then_device: dict[str, dict[str, JsonDict]], +) -> dict[str, Any]: + """Create a new `m.direct_to_device` EDU, returning the full EDU dict.""" + return { + "edu_type": EduTypes.DIRECT_TO_DEVICE, + "content": create_new_to_device_edu_content( + sender_user_id, message_type, messages_by_user_then_device + ), + } + + +# The size of an empty EDU with no messages, which we use as the base size when +# packing messages into EDUs. The size of the sender and message type must be +# added to this when calculating the size of an EDU. +_EMPTY_EDU_SIZE = len(encode_canonical_json(_create_new_to_device_edu("", "", {}))) diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index a3806fd1123..f147cbe7281 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -739,18 +739,15 @@ def get_all_new_device_messages_txn( ) @trace - async def add_messages_to_device_inbox( + async def add_local_messages_from_client_to_device_inbox( self, local_messages_by_user_then_device: dict[str, dict[str, JsonDict]], - remote_messages_by_destination: dict[str, JsonDict], ) -> int: - """Used to send messages from this server. + """Queue local device messages that will be sent to devices of local users. Args: local_messages_by_user_then_device: Dictionary of recipient user_id to recipient device_id to message. - remote_messages_by_destination: - Dictionary of destination server_name to the EDU JSON to send. Returns: The new stream_id. @@ -766,6 +763,39 @@ def add_messages_txn( txn, stream_id, local_messages_by_user_then_device ) + async with self._to_device_msg_id_gen.get_next() as stream_id: + now_ms = self.clock.time_msec() + await self.db_pool.runInteraction( + "add_local_messages_from_client_to_device_inbox", + add_messages_txn, + now_ms, + stream_id, + ) + for user_id in local_messages_by_user_then_device.keys(): + self._device_inbox_stream_cache.entity_has_changed(user_id, stream_id) + + return self._to_device_msg_id_gen.get_current_token() + + @trace + async def add_remote_messages_from_client_to_device_inbox( + self, + remote_messages_by_destination: dict[str, JsonDict], + ) -> int: + """Queue device messages that will be sent to remote servers. + + Args: + remote_messages_by_destination: + Dictionary of destination server_name to the EDU JSON to send. + + Returns: + The new stream_id. + """ + + assert self._can_write_to_device + + def add_messages_txn( + txn: LoggingTransaction, now_ms: int, stream_id: int + ) -> None: # Add the remote messages to the federation outbox. # We'll send them to a remote server when we next send a # federation transaction to that destination. @@ -824,10 +854,11 @@ def add_messages_txn( async with self._to_device_msg_id_gen.get_next() as stream_id: now_ms = self.clock.time_msec() await self.db_pool.runInteraction( - "add_messages_to_device_inbox", add_messages_txn, now_ms, stream_id + "add_remote_messages_from_client_to_device_inbox", + add_messages_txn, + now_ms, + stream_id, ) - for user_id in local_messages_by_user_then_device.keys(): - self._device_inbox_stream_cache.entity_has_changed(user_id, stream_id) for destination in remote_messages_by_destination.keys(): self._device_federation_outbox_stream_cache.entity_has_changed( destination, stream_id diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index f2898de0b8e..977a662d7a5 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -24,6 +24,7 @@ import os import typing from typing import ( + Any, Iterator, Mapping, Sequence, @@ -31,11 +32,14 @@ ) import attr +from canonicaljson import encode_canonical_json from matrix_common.versionstring import get_distribution_version_string from twisted.internet import defer from twisted.python.failure import Failure +from synapse.types import JsonDict + if typing.TYPE_CHECKING: pass @@ -165,3 +169,107 @@ def clear(self) -> None: self._underlying_map = {} self._mutable_map.clear() self._deletions.clear() + + +@attr.s(slots=True, auto_attribs=True) +class _DictSplitterState: + """State for splitting a dict into multiple dicts, c.f. + `split_dict_to_fit_to_size`.""" + + subset: dict[str, Any] + """A subset of the original dict.""" + + estimated_size: int + """Estimated size of the JSON encoding of the current payload, including any + wrapping structure.""" + + +def split_dict_to_fit_to_size( + original_dict: dict[str, Any], + *, + soft_max_size: int, + wrapping_object_size: int = 2, +) -> Iterator[tuple[dict[str, JsonDict], int]]: + """Splits a dict up into a list of dicts, each of which is small enough to + fit into the given size when encoded as JSON. Every entry in the original + dict is in exactly one of the resulting dicts. + + The `wrapping_object_size` can be used if the resulting dicts are going to + be wrapped in some additional JSON structure, to account for the additional + size of that structure. The default assumes no wrapping, and just accounts + for the two curly braces of the dict itself. + + Note that if an individual entry in the original dict is larger than + `soft_max_size` then this will emit a dict containing just that entry, which + will be larger than `soft_max_size` when encoded as JSON. + + Args: + original_dict: The dict to split. + soft_max_size: The maximum size of each dict when encoded as JSON. + wrapping_object_size: The estimated size of the JSON encoding of the + payload when empty. + + Returns: + An iterator of (dict, size) pairs, where dict is a subset of the + original dict and size is the estimated size of the JSON encoding of + that dict, including any wrapping structure. + """ + + if not original_dict: + return + + # Check if the whole dict fits within the size limit. If it does, we can + # skip the splitting logic and just return the original dict. + full_size = _len_with_wrapping_object(original_dict, wrapping_object_size) + if full_size <= soft_max_size: + yield (original_dict, full_size) + return + + # The current payload being built up. We keep track of the estimated size of + # the JSON encoding of this payload so that we can decide when to start a + # new one. + current_payload = _DictSplitterState(subset={}, estimated_size=wrapping_object_size) + + for key, payload in original_dict.items(): + current_payload.subset[key] = payload + current_size = _len_with_wrapping_object( + current_payload.subset, wrapping_object_size + ) + + if current_size > soft_max_size: + # We've exceeded the size limit, so we need to start a new payload. We pop + # the current entry from the payload and yield the previous payload, then + # start a new payload with just the current entry. + if len(current_payload.subset) > 1: + current_payload.subset.pop(key) + yield current_payload.subset, current_payload.estimated_size + + current_payload = _DictSplitterState( + subset={}, + estimated_size=wrapping_object_size, + ) + + # Recalculate the current size with just the current entry. + current_size = _len_with_wrapping_object( + {key: payload}, wrapping_object_size + ) + + current_payload.subset[key] = payload + current_payload.estimated_size = current_size + + if current_payload.subset: + # yield the final payload if it's non-empty + yield current_payload.subset, current_payload.estimated_size + + +def _len_with_wrapping_object(payload: Any, wrapping_object_size: int) -> int: + """Helper function to calculate the size of a payload when encoded as JSON, + including any wrapping structure.""" + return ( + len(encode_canonical_json(payload)) + + wrapping_object_size + # account for the curly braces of the dict itself, which are + # included in the size of the subset but not in the size of the + # payload + - 2 + ) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 7f67d0ca5e4..08a4ab624ae 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -856,7 +856,9 @@ def test_application_services_receive_bursts_of_to_device(self) -> None: # Seed the device_inbox table with our fake messages self.get_success( - self.hs.get_datastores().main.add_messages_to_device_inbox(messages, {}) + self.hs.get_datastores().main.add_local_messages_from_client_to_device_inbox( + messages + ) ) # Now have local_user send a final to-device message to exclusive_as_user. All unsent diff --git a/tests/rest/client/test_sendtodevice.py b/tests/rest/client/test_sendtodevice.py index 56533d85f58..c06c7312f21 100644 --- a/tests/rest/client/test_sendtodevice.py +++ b/tests/rest/client/test_sendtodevice.py @@ -18,9 +18,26 @@ # [This file includes modifications made by New Vector Limited] # # -from synapse.api.constants import EduTypes +from unittest.mock import AsyncMock, Mock + +from canonicaljson import encode_canonical_json + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.api.constants import ( + MAX_EDUS_PER_TRANSACTION, + SOFT_MAX_EDU_SIZE, + EduTypes, +) +from synapse.api.errors import Codes +from synapse.handlers.devicemessage import ( + _create_new_to_device_edu, +) from synapse.rest import admin from synapse.rest.client import login, sendtodevice, sync +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util.clock import Clock from tests.unittest import HomeserverTestCase, override_config @@ -41,6 +58,20 @@ class SendToDeviceTestCase(HomeserverTestCase): sync.register_servlets, ] + def default_config(self) -> JsonDict: + config = super().default_config() + config["federation_sender_instances"] = None + return config + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + self.federation_transport_client = Mock(spec=["send_transaction"]) + self.federation_transport_client.send_transaction = AsyncMock() + hs = self.setup_test_homeserver( + federation_transport_client=self.federation_transport_client, + ) + + return hs + def test_user_to_user(self) -> None: """A to-device message from one user to another should get delivered""" @@ -91,6 +122,238 @@ def test_user_to_user(self) -> None: self.assertEqual(channel.code, 200, channel.result) self.assertEqual(channel.json_body.get("to_device", {}).get("events", []), []) + def test_large_remote_todevice(self) -> None: + """A to-device message needs to fit in the EDU size limit""" + _ = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + + # Create a message that is over the `SOFT_MAX_EDU_SIZE` + test_msg = {"foo": "a" * SOFT_MAX_EDU_SIZE} + channel = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.test/12345", + content={"messages": {"@remote_user:secondserver": {"device": test_msg}}}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 413, channel.result) + self.assertEqual(Codes.TOO_LARGE, channel.json_body["errcode"]) + + def test_edu_large_messages_splitting(self) -> None: + """ + Test that a bunch of to-device messages are split over multiple EDUs if they are + collectively too large to fit into a single EDU + """ + mock_send_transaction: AsyncMock = ( + self.federation_transport_client.send_transaction + ) + mock_send_transaction.return_value = {} + + sender = self.hs.get_federation_sender() + + _ = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + destination = "secondserver" + messages = {} + + # 2 messages, each just big enough to fit into their own EDU + for i in range(2): + messages[f"@remote_user{i}:" + destination] = { + "device": {"foo": "a" * (SOFT_MAX_EDU_SIZE - 1000)} + } + + channel = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.test/1234567", + content={"messages": messages}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + self.get_success(sender.send_device_messages([destination])) + + number_of_edus_sent = 0 + for call in mock_send_transaction.call_args_list: + number_of_edus_sent += len(call[0][1]()["edus"]) + + self.assertEqual(number_of_edus_sent, 2) + + def test_edu_large_messages_splitting_one_user(self) -> None: + """ + Test that a bunch of to-device messages for the same user are split over multiple EDUs if they are + collectively too large to fit into a single EDU + """ + mock_send_transaction: AsyncMock = ( + self.federation_transport_client.send_transaction + ) + mock_send_transaction.return_value = {} + + sender = self.hs.get_federation_sender() + + _ = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + destination = "secondserver" + messages = {} + + # 2 messages, each just big enough to fit into their own EDU + messages["@remote_user:" + destination] = { + "device1": {"foo": "a" * (SOFT_MAX_EDU_SIZE - 1000)}, + "device2": {"foo": "a" * (SOFT_MAX_EDU_SIZE - 1000)}, + } + + channel = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.test/12345678", + content={"messages": messages}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + self.get_success(sender.send_device_messages([destination])) + + number_of_edus_sent = 0 + for call in mock_send_transaction.call_args_list: + number_of_edus_sent += len(call[0][1]()["edus"]) + + self.assertEqual(number_of_edus_sent, 2) + + def test_edu_large_messages_not_splitting_one_user(self) -> None: + """Test that a couple of to-device messages for the same user that just + collectively fit in a single EDU do not get split into multiple EDUs. + + This tests that we don't over-split messages into multiple EDUs when we + don't need to. + """ + + mock_send_transaction: AsyncMock = ( + self.federation_transport_client.send_transaction + ) + mock_send_transaction.return_value = {} + + sender = self.hs.get_federation_sender() + + user_id = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + destination = "secondserver" + remote_user = "@remote_user:" + destination + messages = {} + + # We create two messages such that the combined size of the messages is + # just under the `SOFT_MAX_EDU_SIZE` limit, so they should be able to + # fit in a single EDU together, but not separately. + wrapper_size = len( + encode_canonical_json(_create_new_to_device_edu(user_id, "m.test", {})) + ) + max_size_of_message = ( + SOFT_MAX_EDU_SIZE + - wrapper_size + # Size of the device content wrapper, `{"remote_user": …}` + - (len(remote_user) + 5) + ) + messages[remote_user] = { + # The constants are the size of each individual message + "d1": {"a": "a" * (max_size_of_message - 13 - 16)}, + "d2": {"b": "b"}, + } + + # The full EDU size should now be just shy of the `SOFT_MAX_EDU_SIZE` limit + full_edu = encode_canonical_json( + _create_new_to_device_edu(user_id, "m.test", messages) + ) + self.assertEqual(len(full_edu), SOFT_MAX_EDU_SIZE - 1) + + # Now send the messages, and check they are sent in a single EDU. + channel = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.test/12345678", + content={"messages": messages}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + self.get_success(sender.send_device_messages([destination])) + + number_of_edus_sent = 0 + for call in mock_send_transaction.call_args_list: + number_of_edus_sent += len(call[0][1]()["edus"]) + + self.assertEqual(number_of_edus_sent, 1) + + def test_edu_small_messages_not_splitting(self) -> None: + """ + Test that a couple of small messages do not get split into multiple EDUs + """ + mock_send_transaction: AsyncMock = ( + self.federation_transport_client.send_transaction + ) + mock_send_transaction.return_value = {} + + sender = self.hs.get_federation_sender() + + _ = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + destination = "secondserver" + messages = {} + + # 2 small messages that should fit in a single EDU + for i in range(2): + messages[f"@remote_user{i}:" + destination] = {"device": {"foo": "a" * 100}} + + channel = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.test/123456", + content={"messages": messages}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + self.get_success(sender.send_device_messages([destination])) + + number_of_edus_sent = 0 + for call in mock_send_transaction.call_args_list: + number_of_edus_sent += len(call[0][1]()["edus"]) + + self.assertEqual(number_of_edus_sent, 1) + + def test_transaction_splitting(self) -> None: + """Test that a bunch of to-device messages are split into multiple transactions if there are too many EDUs""" + mock_send_transaction: AsyncMock = ( + self.federation_transport_client.send_transaction + ) + mock_send_transaction.return_value = {} + + sender = self.hs.get_federation_sender() + + _ = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + destination = "secondserver" + messages = {} + + number_of_edus_to_send = MAX_EDUS_PER_TRANSACTION + 1 + + for i in range(number_of_edus_to_send): + messages[f"@remote_user{i}:" + destination] = { + "device": {"foo": "a" * (SOFT_MAX_EDU_SIZE - 1000)} + } + + channel = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.test/12345678", + content={"messages": messages}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + self.get_success(sender.send_device_messages([destination])) + + # At least 2 transactions should be sent since we are over the EDU limit per transaction + self.assertGreaterEqual(mock_send_transaction.call_count, 2) + + number_of_edus_sent = 0 + for call in mock_send_transaction.call_args_list: + number_of_edus_sent += len(call[0][1]()["edus"]) + + self.assertEqual(number_of_edus_sent, number_of_edus_to_send) + @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}}) def test_local_room_key_request(self) -> None: """m.room_key_request has special-casing; test from local user""" diff --git a/tests/util/test_split_dict.py b/tests/util/test_split_dict.py new file mode 100644 index 00000000000..3abba4d3d4f --- /dev/null +++ b/tests/util/test_split_dict.py @@ -0,0 +1,201 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + + +from canonicaljson import encode_canonical_json + +from synapse.util import split_dict_to_fit_to_size + +from tests.unittest import TestCase + + +class SplitDictTestCase(TestCase): + def test_empty(self) -> None: + "Test that an empty dict yields no payloads" + + self.assertEqual( + list( + split_dict_to_fit_to_size({}, soft_max_size=10, wrapping_object_size=0) + ), + [], + ) + + def test_no_splitting(self) -> None: + "Test that a dict that fits within the size limit is yielded as a single payload" + + original_dict = {"a": {"key": "value"}, "b": {"key": "value"}} + + # Set the soft max size to be the size of the original dict, so it + # should fit + soft_max_size = len(encode_canonical_json(original_dict)) + + self.assertEqual( + list( + split_dict_to_fit_to_size( + original_dict, + soft_max_size=soft_max_size, + ) + ), + [(original_dict, soft_max_size)], + ) + + def test_no_splitting_with_wrapping_size(self) -> None: + "Test that the wrapping size is taken into account when deciding whether to split" + + wrapping = {"key": "value", "payload": {}} + original_dict = {"a": {"key": "value"}, "b": {"key": "value"}} + wrapping_object_size = len(encode_canonical_json(wrapping)) + + # Set the soft max size to the size of the expected final output. + soft_max_size = len( + encode_canonical_json({"key": "value", "payload": original_dict}) + ) + + self.assertEqual( + list( + split_dict_to_fit_to_size( + original_dict, + soft_max_size=soft_max_size, + wrapping_object_size=wrapping_object_size, + ) + ), + [(original_dict, soft_max_size)], + ) + + def test_splitting(self) -> None: + "Test that a dict that exceeds the size limit is split into multiple payloads" + + original_dict = { + "a": {"key": "value"}, + "b": {"key": "value"}, + "c": {"key": "value"}, + } + + # Set the soft max size to be the size of a single key-value pair, so + # it should split into three payloads. + soft_max_size = len(encode_canonical_json({"a": {"key": "value"}})) + + self.assertEqual( + list( + split_dict_to_fit_to_size( + original_dict, + soft_max_size=soft_max_size, + ) + ), + [ + ({"a": {"key": "value"}}, soft_max_size), + ({"b": {"key": "value"}}, soft_max_size), + ({"c": {"key": "value"}}, soft_max_size), + ], + ) + + def test_splitting_with_wrapping_size(self) -> None: + "Test that the wrapping size is taken into account when splitting" + + wrapping = {"key": "value", "payload": {}} + original_dict = { + "a": {"key": "value"}, + "b": {"key": "value"}, + "c": {"key": "value"}, + } + wrapping_object_size = len(encode_canonical_json(wrapping)) + # Set the soft max size to be the size of a single key-value pair plus + # the wrapping size, so it should split into three payloads. + soft_max_size = ( + len(encode_canonical_json({"a": {"key": "value"}})) + + wrapping_object_size + - 2 + ) + + self.assertEqual( + list( + split_dict_to_fit_to_size( + original_dict, + soft_max_size=soft_max_size, + wrapping_object_size=wrapping_object_size, + ) + ), + [ + ({"a": {"key": "value"}}, soft_max_size), + ({"b": {"key": "value"}}, soft_max_size), + ({"c": {"key": "value"}}, soft_max_size), + ], + ) + + def test_oversized_entry(self) -> None: + """Test that if a single entry exceeds the size limit, it is still + yielded as a single payload""" + + original_dict = { + "a": {"key": "value"}, + "b": {"key": "value"}, + "c": {"key": "value"}, + } + + # Set the soft max size to be smaller than the size of a single + # key-value pair, so each entry exceeds the limit. + soft_max_size = len(encode_canonical_json({"a": {"key": "value"}})) - 1 + + self.assertEqual( + list( + split_dict_to_fit_to_size( + original_dict, + soft_max_size=soft_max_size, + ) + ), + [ + ( + {"a": {"key": "value"}}, + len(encode_canonical_json({"a": {"key": "value"}})), + ), + ( + {"b": {"key": "value"}}, + len(encode_canonical_json({"b": {"key": "value"}})), + ), + ( + {"c": {"key": "value"}}, + len(encode_canonical_json({"c": {"key": "value"}})), + ), + ], + ) + + def test_different_sized_entries(self) -> None: + """Test that entries of different sizes are split correctly""" + + original_dict = { + "a": "X" * 5, # size 13 + "b": "X" * 10, # size 18 + "c": "X" * 5, # size 13 + } + + soft_max_size = 30 + + self.assertEqual( + list( + split_dict_to_fit_to_size( + original_dict, + soft_max_size=soft_max_size, + ) + ), + [ + ( + {"a": "X" * 5, "b": "X" * 10}, + len(encode_canonical_json({"a": "X" * 5, "b": "X" * 10})), + ), + ( + {"c": "X" * 5}, + len(encode_canonical_json({"c": "X" * 5})), + ), + ], + ) From 0de805ab9ad59e4bc1672d8d8aa3ec5dd4f826fb Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 2 Jun 2026 12:40:29 -0500 Subject: [PATCH 66/92] expand on docstrings and add some comments --- synapse/storage/databases/main/user_directory.py | 8 ++++++++ synapse/storage/engines/_base.py | 5 ++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py index 7531fcf8e61..2f08638ff2e 100644 --- a/synapse/storage/databases/main/user_directory.py +++ b/synapse/storage/databases/main/user_directory.py @@ -700,6 +700,10 @@ def _update_profiles_in_user_dir_txn( # server name if isinstance(self.database_engine, Psycopg2Engine): + # `execute_values()` is the performative go-to with psycopg2. It differs + # from psycopg(v3+) below in that the `template` needs to be separated + # from the query itself so that the value variables can be bound and + # expanded before being inserted into the final query. template = """ ( %s, @@ -733,6 +737,10 @@ def _update_profiles_in_user_dir_txn( fetch=False, ) elif isinstance(self.database_engine, PsycopgEngine): + # Unlike psycopg2, psycopg(v3+) can not use `execute_values()` at this + # time. However, `executemany()` on psycopg uses it's internal pipeline + # mode to be much faster than an iterative `execute()`, especially on + # writes. sql = """ INSERT INTO user_directory_search(user_id, vector) VALUES diff --git a/synapse/storage/engines/_base.py b/synapse/storage/engines/_base.py index ea958173673..7ae5609fef3 100644 --- a/synapse/storage/engines/_base.py +++ b/synapse/storage/engines/_base.py @@ -130,7 +130,10 @@ def attempt_to_set_isolation_level( ) -> None: """Attempt to set the connections isolation level. - Note: This has no effect on SQLite3, as transactions are SERIALIZABLE by default. + Note: + * This has no effect on SQLite3, as transactions are SERIALIZABLE by default. + * On Postgres, an isolation_level of None restores the default from the + attribute `default_isolation_level` """ ... From 69526539f5bd1a87793ee7e33fd95e54a48d38b9 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 2 Jun 2026 12:41:20 -0500 Subject: [PATCH 67/92] switch from autocommit attribute to property setter to work around __setattr__() missing --- synapse/storage/engines/psycopg.py | 7 +------ tests/storage/test_base.py | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/synapse/storage/engines/psycopg.py b/synapse/storage/engines/psycopg.py index d4ff2325324..25c8ff1a2c6 100644 --- a/synapse/storage/engines/psycopg.py +++ b/synapse/storage/engines/psycopg.py @@ -20,7 +20,6 @@ import psycopg.sql from twisted.enterprise.adbapi import Connection as TxConnection - from synapse.storage.engines import PostgresEngine from synapse.storage.engines._base import IsolationLevel @@ -89,11 +88,7 @@ def in_transaction(self, conn: psycopg.Connection) -> bool: def attempt_to_set_autocommit( self, conn: psycopg.Connection, autocommit: bool ) -> None: - # Sometimes this gets called with a Twisted connection instead, unwrap - # it because it doesn't support __setattr__. - if isinstance(conn, TxConnection): - conn = conn._connection - conn.autocommit = autocommit + conn.set_autocommit(autocommit) def attempt_to_set_isolation_level( self, conn: psycopg.Connection, isolation_level: IsolationLevel | None = None diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index c19c8c7591c..adad9d5f170 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -53,7 +53,7 @@ def setUp(self) -> None: ] if USE_POSTGRES_FOR_TESTS == "psycopg": - conn_attributes += ["autocommit"] + conn_attributes += ["set_autocommit"] # copy returns a context manager which needs assertions. self.mock_copy = Mock(spec_set=["write_row", "rows"]) From e14c5a30f0cc7bcec770b3a0f16a91233d8e0cac Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 2 Jun 2026 12:41:57 -0500 Subject: [PATCH 68/92] add assert in create_index_sqlite() to make sure it is not being erroneously used by psycopg --- synapse/storage/background_updates.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index f40480e5d9d..bd87f5620a6 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -36,7 +36,7 @@ import attr from pydantic import BaseModel -from synapse.storage.engines import PostgresEngine +from synapse.storage.engines import PostgresEngine, Sqlite3Engine from synapse.storage.types import Connection, Cursor from synapse.types import JsonDict, StrCollection from synapse.util.clock import Clock @@ -834,6 +834,8 @@ def create_index_psql(conn: "LoggingDatabaseConnection") -> None: conn.engine.attempt_to_set_autocommit(conn.conn, False) def create_index_sqlite(conn: "LoggingDatabaseConnection") -> None: + assert isinstance(self.db_pool.engine, Sqlite3Engine) + # Sqlite doesn't support concurrent creation of indexes. # # We assume that sqlite doesn't give us invalid indices; however From 858452076548030f87266ac5fda054327c698049 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 2 Jun 2026 12:42:53 -0500 Subject: [PATCH 69/92] remove use of one_line_sql to split the verb out of, and add maxsplit=1 for a potential performance micro-enhancement --- synapse/storage/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 598e45d6664..0ab5cb7108f 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -579,7 +579,7 @@ def _do_execute( secs = time.time() - start sql_logger.debug("[SQL time] {%s} %f sec", self.name, secs) sql_query_timer.labels( - verb=one_line_sql.split()[0], **{SERVER_NAME_LABEL: self.server_name} + verb=sql.split(maxsplit=1)[0], **{SERVER_NAME_LABEL: self.server_name} ).observe(secs) def close(self) -> None: From 0a3cd96996054d6d88c1faa49108cd266f99ad57 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 2 Jun 2026 12:51:19 -0500 Subject: [PATCH 70/92] remove dead commented-out code --- synapse/storage/engines/psycopg.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/storage/engines/psycopg.py b/synapse/storage/engines/psycopg.py index 25c8ff1a2c6..fd281b22410 100644 --- a/synapse/storage/engines/psycopg.py +++ b/synapse/storage/engines/psycopg.py @@ -69,9 +69,6 @@ def set_statement_timeout( cursor.execute(query_str.as_string()) def convert_param_style(self, sql: str) -> str: - # if isinstance(sql, psycopg.sql.Composed): - # return sql - return sql.replace("?", "%s") def is_deadlock(self, error: Exception) -> bool: From 1331918d11687e6054d9e82ee28db663d97f0e91 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 2 Jun 2026 13:00:43 -0500 Subject: [PATCH 71/92] more comments --- synapse/storage/database.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 0ab5cb7108f..202ea5abc1c 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -417,6 +417,10 @@ def execute_batch(self, sql: str, args: Iterable[Iterable[Any]]) -> None: """ if isinstance(self.database_engine, Psycopg2Engine): + # This is narrowed to the only be used by psycopg2 as it is imported from + # that module. Internally, `execute_batch()` just injects a given grouping + # of parameters into the query and then appends multiple of those queries + # together with a `;`, leading to large reduction of 'round-tripping'. from psycopg2.extras import execute_batch # TODO: is it safe for values to be Iterable[Iterable[Any]] here? @@ -433,6 +437,9 @@ def execute_batch(self, sql: str, args: Iterable[Iterable[Any]]) -> None: # suggests that the outer collection may be iterable, but # https://docs.python.org/3/library/sqlite3.html?highlight=sqlite3#how-to-use-placeholders-to-bind-values-in-sql-queries # suggests that the inner collection should be a sequence or dict. + # In the case of psycopg v3+ usage, `executemany()` uses a postgres + # optimization called pipelining to vastly speed up processing of the query + # when there are many args. self.executemany(sql, args) def execute_values( From 595bd8db7acdcc209e66504f7b8b798a48897b62 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 2 Jun 2026 13:24:05 -0500 Subject: [PATCH 72/92] lint to remove unused dependency --- synapse/storage/engines/psycopg.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/engines/psycopg.py b/synapse/storage/engines/psycopg.py index fd281b22410..483052aeb46 100644 --- a/synapse/storage/engines/psycopg.py +++ b/synapse/storage/engines/psycopg.py @@ -19,7 +19,6 @@ import psycopg.errors import psycopg.sql -from twisted.enterprise.adbapi import Connection as TxConnection from synapse.storage.engines import PostgresEngine from synapse.storage.engines._base import IsolationLevel From e39c61c2629c288d87313c111395de3e96c3eb63 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 3 Jun 2026 08:50:08 -0500 Subject: [PATCH 73/92] add comment about why using the "c" extra and not the other implementations --- pyproject.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 598a2d55651..516360b9c61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,6 +127,13 @@ postgres = [ "psycopg2cffi>=2.8;platform_python_implementation == 'PyPy'", "psycopg2cffi-compat==1.1;platform_python_implementation == 'PyPy'", ] +# The "pure python" implementation of psycopg(v3+) does not play nicely with the +# complement docker image because of differences in the required debian package `libpq`. +# Using the "binary" extra to inject that correct lib prevented the debian packaging of +# Synapse to fail(see test workflow 'poetry_lockfile' where all packages must have +# sdists and not just wheels). The "c" extra is a source distribution, but requires a +# compilation step during build. +# # v3.2.8 was selected here as minimum to support the libpq5 shipped with debian # bookworm. This libpq5 includes support for postgres 18 and psycopg will fail to build # below this selected version. From e8e5a421803c6ffb14ad01df95121cf0c2da1648 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Jun 2026 14:52:05 +0100 Subject: [PATCH 74/92] Fix parsing events that have large integers (#19819) Follow on from https://github.com/element-hq/synapse/pull/19701. Unfortunately serde has a bug when using `#[serde(flatten)]` with `arbitrary-precision` feature when handling integers that fit in a i128 when doing `serde_json::from_value`. See https://github.com/serde-rs/serde/issues/2230. The `depythonize` hits the same issue. To fix this we make it so we only parse events from strings and not values. --- changelog.d/19819.misc | 1 + rust/src/events/formats/mod.rs | 7 ++ rust/src/events/mod.rs | 80 ++++++++-------------- rust/src/events/unsigned.rs | 13 ++-- synapse/events/__init__.py | 25 ++++++- synapse/synapse_rust/events.pyi | 4 +- tests/events/test_event_parsing.py | 102 ++++++++++++++++++++++++++++ tests/handlers/test_room_member.py | 6 +- tests/storage/test_stream.py | 8 +-- tests/synapse_rust/test_unsigned.py | 13 +++- 10 files changed, 189 insertions(+), 70 deletions(-) create mode 100644 changelog.d/19819.misc create mode 100644 tests/events/test_event_parsing.py diff --git a/changelog.d/19819.misc b/changelog.d/19819.misc new file mode 100644 index 00000000000..4663e8b9611 --- /dev/null +++ b/changelog.d/19819.misc @@ -0,0 +1 @@ +Port the python Event classes to Rust. diff --git a/rust/src/events/formats/mod.rs b/rust/src/events/formats/mod.rs index 10b16a5436c..86023add17f 100644 --- a/rust/src/events/formats/mod.rs +++ b/rust/src/events/formats/mod.rs @@ -98,6 +98,13 @@ pub use vmsc4242::EventFormatVMSC4242; /// fields as they are mutable (and must be deep-copied if the event is cloned). /// `common_fields` and `specific_fields` are both `#[serde(flatten)]`ed so that /// the serialised JSON is a single flat object matching the Matrix spec. +/// +/// Note, deserialization of this struct must not be done from +/// [`serde_json::Value`] nor [`pythonize::depythonize`], due to a bug with +/// `#[serde(flatten)]` combined with the `arbitrary_precision` feature. +/// Instead, deserialize directly from a JSON string with +/// `serde_json::from_str`. See https://github.com/serde-rs/serde/issues/2230 +/// for details. #[derive(Serialize, Deserialize)] pub struct FormattedEvent> { /// The event's signatures. diff --git a/rust/src/events/mod.rs b/rust/src/events/mod.rs index ad3f61e2fdf..a4fe15c522b 100644 --- a/rust/src/events/mod.rs +++ b/rust/src/events/mod.rs @@ -160,10 +160,15 @@ pub struct Event { #[pymethods] impl Event { + /// Construct an Event from a JSON string, room version, and internal + /// metadata dict. + /// + /// We do no accept a Python dict directly because of the issues with + /// depythonize and large integers (see [`FormattedEvent`] for details). #[new] fn new_from_py<'a, 'py>( py: Python<'py>, - event_dict: &'a Bound<'py, PyAny>, + event_json: &str, room_version: &'a Bound<'py, PyAny>, internal_metadata_dict: &'a Bound<'py, PyDict>, rejected_reason: Option, @@ -178,14 +183,14 @@ impl Event { let rejected_reason = rejected_reason.map(String::into_boxed_str); - // Parse the event dict into a FormattedEvent, converting any failures to + // Parse the event JSON into a FormattedEvent, converting any failures to // a `ValueError`. - let parsed_event = depythonize_event_dict(room_version, event_dict).map_err(|err| { + let parsed_event = event_dict_from_json_str(room_version, event_json).map_err(|err| { let new_err = PyValueError::new_err(format!( - "Failed to parse event for room version {}", - room_version + "Failed to parse event for room version {}: {}", + room_version, err )); - new_err.set_cause(py, Some(err)); + new_err.set_cause(py, Some(PyValueError::new_err(err.to_string()))); new_err })?; @@ -555,63 +560,27 @@ impl Event { } } -fn depythonize_event_dict( +/// Parses a JSON string into a [`FormattedEvent`] for the given room version. +fn event_dict_from_json_str( room_version: &RoomVersion, - event_dict: &Bound<'_, PyAny>, -) -> PyResult { - let formatted_event: FormattedEvent = match room_version.event_format { - EventFormatVersions::ROOM_V1_V2 => { - let event_format: FormattedEvent = depythonize(event_dict)?; - - event_format.into() - } - EventFormatVersions::ROOM_V3 | EventFormatVersions::ROOM_V4_PLUS => { - let event_format: FormattedEvent = depythonize(event_dict)?; - event_format.into() - } - EventFormatVersions::ROOM_V11_HYDRA_PLUS => { - let event_format: FormattedEvent = depythonize(event_dict)?; - event_format.into() - } - EventFormatVersions::ROOM_VMSC4242 => { - let event_format: FormattedEvent = depythonize(event_dict)?; - event_format.into() - } - _ => { - return Err(PyValueError::new_err(format!( - "Unsupported room version: {}", - room_version - ))) - } - }; - - formatted_event.validate()?; - - Ok(formatted_event) -} - -/// Converts an event dict as [`serde_json::Value`] into a [`FormattedEvent`]. -fn event_dict_from_json_value( - room_version: &RoomVersion, - event_dict: serde_json::Value, + event_json: &str, ) -> Result { let formatted_event: FormattedEvent = match room_version.event_format { EventFormatVersions::ROOM_V1_V2 => { - let event_format: FormattedEvent = serde_json::from_value(event_dict)?; - + let event_format: FormattedEvent = serde_json::from_str(event_json)?; event_format.into() } EventFormatVersions::ROOM_V3 | EventFormatVersions::ROOM_V4_PLUS => { - let event_format: FormattedEvent = serde_json::from_value(event_dict)?; + let event_format: FormattedEvent = serde_json::from_str(event_json)?; event_format.into() } EventFormatVersions::ROOM_V11_HYDRA_PLUS => { - let event_format: FormattedEvent = serde_json::from_value(event_dict)?; + let event_format: FormattedEvent = serde_json::from_str(event_json)?; event_format.into() } EventFormatVersions::ROOM_VMSC4242 => { let event_format: FormattedEvent = - serde_json::from_value(event_dict)?; + serde_json::from_str(event_json)?; event_format.into() } _ => { @@ -638,10 +607,17 @@ fn redact_event_py(event: &Event) -> PyResult { })?; let redacted_value = redact(&event_value, event.room_version)?; - let redacted_formatted_event = event_dict_from_json_value(event.room_version, redacted_value) - .map_err(|err| { - PyValueError::new_err(format!("Failed to deserialize redacted event: {}", err)) + + // We can't convert from a value into [`Event`] directly, so we round-trip + // through JSON. See [`FormattedEvent`] for details on why we can't go + // directly through Python dicts. + let redacted_event_json = serde_json::to_string(&redacted_value).map_err(|err| { + PyValueError::new_err(format!("Failed to serialize redacted event: {}", err)) })?; + let redacted_formatted_event = + event_dict_from_json_str(event.room_version, &redacted_event_json).map_err(|err| { + PyValueError::new_err(format!("Failed to deserialize redacted event: {}", err)) + })?; let redacted_event = Event { parsed_event: redacted_formatted_event, diff --git a/rust/src/events/unsigned.rs b/rust/src/events/unsigned.rs index 0bb644ee909..931c412325b 100644 --- a/rust/src/events/unsigned.rs +++ b/rust/src/events/unsigned.rs @@ -16,9 +16,9 @@ use std::sync::{Arc, RwLock, RwLockReadGuard}; use pyo3::{ - exceptions::{PyKeyError, PyRuntimeError, PyTypeError}, + exceptions::{PyKeyError, PyRuntimeError, PyTypeError, PyValueError}, pyclass, pymethods, - types::{PyAnyMethods, PyList, PyListMethods, PyMapping}, + types::{PyAnyMethods, PyList, PyListMethods}, Bound, IntoPyObjectExt, PyAny, PyResult, Python, }; use pythonize::{depythonize, pythonize}; @@ -114,9 +114,14 @@ impl Unsigned { #[pymethods] impl Unsigned { + /// Create a new `Unsigned` from a JSON string. + /// + /// We do no accept a Python dict directly because of the issues with + /// depythonize and large integers (see [`FormattedEvent`] for details). #[new] - fn py_new(unsigned: Bound<'_, PyMapping>) -> PyResult { - let inner = depythonize(&unsigned)?; + fn py_new(unsigned_json: &str) -> PyResult { + let inner = serde_json::from_str(unsigned_json) + .map_err(|err| PyValueError::new_err(format!("Failed to parse unsigned: {}", err)))?; Ok(Self { inner: Arc::new(RwLock::new(inner)), diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 7abd91f0728..b01e786550a 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -29,6 +29,7 @@ ) import attr +from canonicaljson import encode_canonical_json from synapse.api.constants import ( EventContentFields, @@ -72,15 +73,35 @@ def make_event_from_dict( ) -> Event: """Construct an EventBase from the given event dict""" + # Event constructor only takes JSON string, see Event constructor for + # details. + event_json = encode_canonical_json(event_dict).decode("utf-8") + + return make_event_from_json( + event_json=event_json, + room_version=room_version, + internal_metadata_dict=internal_metadata_dict, + rejected_reason=rejected_reason, + ) + + +def make_event_from_json( + event_json: str, + room_version: RoomVersion = RoomVersions.V1, + internal_metadata_dict: JsonDict | None = None, + rejected_reason: str | None = None, +) -> Event: + """Construct an EventBase from the given event JSON string""" + try: return Event( - event_dict=event_dict, + event_json=event_json, room_version=room_version, internal_metadata_dict=internal_metadata_dict or {}, rejected_reason=rejected_reason, ) except ValueError: - raise SynapseError(400, "Invalid event dict", Codes.BAD_JSON) + raise SynapseError(400, "Invalid event JSON", Codes.BAD_JSON) @attr.s(slots=True, frozen=True, auto_attribs=True) diff --git a/synapse/synapse_rust/events.pyi b/synapse/synapse_rust/events.pyi index 9a69438c1df..f84eeb55d65 100644 --- a/synapse/synapse_rust/events.pyi +++ b/synapse/synapse_rust/events.pyi @@ -189,7 +189,7 @@ class Signatures: class Unsigned: """A class representing the unsigned data of an event.""" - def __init__(self, unsigned_dict: JsonMapping): ... + def __init__(self, unsigned_json: str): ... def __getitem__(self, key: str) -> Any: """Get the value for the given key. @@ -230,7 +230,7 @@ class Event: def __init__( self, - event_dict: JsonDict, + event_json: str, room_version: RoomVersion, internal_metadata_dict: JsonDict, rejected_reason: str | None, diff --git a/tests/events/test_event_parsing.py b/tests/events/test_event_parsing.py new file mode 100644 index 00000000000..9d68cfabd6f --- /dev/null +++ b/tests/events/test_event_parsing.py @@ -0,0 +1,102 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + + +from typing import Any + +from parameterized import parameterized_class + +from synapse.api.room_versions import RoomVersions +from synapse.events import make_event_from_dict +from synapse.synapse_rust.events import redact_event +from synapse.types import JsonDict + +from tests.test_utils.event_injection import EventTypes +from tests.unittest import TestCase + + +def create_minimal_event_dict(**fields: Any) -> JsonDict: + """Create a minimal event dict that will parse correctly.""" + return { + "type": EventTypes.Message, + "content": {}, + "room_id": "!room:id", + "sender": "@user:id", + "event_id": "$event:id", + "origin_server_ts": 0, + "auth_events": [], + "prev_events": [], + "hashes": {}, + "signatures": {}, + "depth": 0, + **fields, + } + + +@parameterized_class( + [ + {"test_int": 2**7 - 1}, + {"test_int": 2**15 - 1}, + {"test_int": 2**31 - 1}, + {"test_int": 2**63 - 1}, + {"test_int": 2**127 - 1}, + {"test_int": 2**200}, + ] +) +class LargeIntTestCase(TestCase): + """Test that we can handle various sized integers in events. + + This is a regression test where we had issues handling integers that fit in + a Rust `i128`. + """ + + test_int: int + """The integer to test with. This will be set by the parameterized_class decorator.""" + + def test_very_large_int_in_event_content(self) -> None: + """Test that we can handle integers in the event content, which is a + JsonObject and thus can contain arbitrary JSON.""" + + event_dict = create_minimal_event_dict(content={"some_field": self.test_int}) + + event = make_event_from_dict(event_dict, RoomVersions.V1) + + self.assertEqual(event.content["some_field"], self.test_int) + + def test_large_int_in_unsigned(self) -> None: + """Test that we can handle integers in the unsigned data, which is an + Unsigned and thus can contain arbitrary JSON.""" + + event_dict = create_minimal_event_dict( + unsigned={"prev_content": {"some_field": self.test_int}} + ) + + event = make_event_from_dict(event_dict, RoomVersions.V1) + + self.assertEqual(event.unsigned["prev_content"]["some_field"], self.test_int) + + def test_large_int_redacted(self) -> None: + """Test that redact events that have an unsigned field with a large + integer in a protected field""" + + event_dict = create_minimal_event_dict( + type=EventTypes.PowerLevels, + state_key="", + content={"users": {"@user:id": self.test_int}}, + ) + + event = make_event_from_dict(event_dict, RoomVersions.V1) + + redacted_event = redact_event(event) + + self.assertEqual(redacted_event.content["users"]["@user:id"], self.test_int) diff --git a/tests/handlers/test_room_member.py b/tests/handlers/test_room_member.py index 9cc7ebfa803..d5b95e4ef6b 100644 --- a/tests/handlers/test_room_member.py +++ b/tests/handlers/test_room_member.py @@ -8,7 +8,7 @@ from synapse.api.constants import AccountDataTypes, EventTypes, Membership from synapse.api.errors import Codes, LimitExceededError, SynapseError from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import Event +from synapse.events import make_event_from_dict from synapse.federation.federation_client import SendJoinResult from synapse.server import HomeServer from synapse.types import UserID, create_requester @@ -124,7 +124,7 @@ def test_remote_joins_contribute_to_rate_limit(self) -> None: create_event_source, self.hs.config.server.default_room_version, ) - create_event = Event( + create_event = make_event_from_dict( create_event_source, self.hs.config.server.default_room_version, {}, @@ -148,7 +148,7 @@ def test_remote_joins_contribute_to_rate_limit(self) -> None: self.hs.hostname, self.hs.signing_key, ) - join_event = Event( + join_event = make_event_from_dict( join_event_source, self.hs.config.server.default_room_version, {}, diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index ba3b3802cf4..de127e3971d 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -35,7 +35,7 @@ ) from synapse.api.filtering import Filter from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import Event +from synapse.events import make_event_from_dict from synapse.federation.federation_client import SendJoinResult from synapse.rest import admin from synapse.rest.client import login, room @@ -1385,7 +1385,7 @@ def test_remote_join(self) -> None: create_event_source, self.hs.config.server.default_room_version, ) - create_event = Event( + create_event = make_event_from_dict( create_event_source, self.hs.config.server.default_room_version, {}, @@ -1408,7 +1408,7 @@ def test_remote_join(self) -> None: creator_join_event_source, self.hs.config.server.default_room_version, ) - creator_join_event = Event( + creator_join_event = make_event_from_dict( creator_join_event_source, self.hs.config.server.default_room_version, {}, @@ -1433,7 +1433,7 @@ def test_remote_join(self) -> None: self.hs.hostname, self.hs.signing_key, ) - join_event = Event( + join_event = make_event_from_dict( join_event_source, self.hs.config.server.default_room_version, {}, diff --git a/tests/synapse_rust/test_unsigned.py b/tests/synapse_rust/test_unsigned.py index 5193188f387..6c1bf2238b5 100644 --- a/tests/synapse_rust/test_unsigned.py +++ b/tests/synapse_rust/test_unsigned.py @@ -11,15 +11,22 @@ # See the GNU Affero General Public License for more details: # . +from canonicaljson import encode_canonical_json + from synapse.synapse_rust.events import Unsigned +from synapse.types import JsonDict from tests import unittest +def _make_unsigned(d: JsonDict) -> Unsigned: + return Unsigned(encode_canonical_json(d).decode("utf-8")) + + class UnsignedTestCase(unittest.TestCase): def test_prev_content(self) -> None: """Test that the prev_content field is correctly exposed as a JsonObject.""" - unsigned = Unsigned({"prev_content": {"key1": "value1", "key2": 42}}) + unsigned = _make_unsigned({"prev_content": {"key1": "value1", "key2": 42}}) self.assert_dict(unsigned["prev_content"], {"key1": "value1", "key2": 42}) @@ -32,7 +39,7 @@ def test_large_age_ts(self) -> None: than the maximum rust native integer size.""" large_int = 2**200 - unsigned = Unsigned({"age_ts": large_int}) + unsigned = _make_unsigned({"age_ts": large_int}) self.assertEqual(unsigned["age_ts"], large_int) @@ -44,7 +51,7 @@ def test_large_integer_in_prev_content(self) -> None: JSON.""" large_int = 2**200 - unsigned = Unsigned({"prev_content": {"some_field": large_int}}) + unsigned = _make_unsigned({"prev_content": {"some_field": large_int}}) self.assertEqual(unsigned["prev_content"]["some_field"], large_int) self.assert_dict( From b8cacf75372c5cda46b815d519d190c1c16839e1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Jun 2026 15:19:27 +0100 Subject: [PATCH 75/92] Fix `Event.__repr__` (#19817) Follow on from #19701 This (more or less) matches what we had before. Otherwise we just get a default `` --------- Co-authored-by: Eric Eastwood --- changelog.d/19817.misc | 1 + rust/src/events/internal_metadata.rs | 2 +- rust/src/events/mod.rs | 47 ++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19817.misc diff --git a/changelog.d/19817.misc b/changelog.d/19817.misc new file mode 100644 index 00000000000..4663e8b9611 --- /dev/null +++ b/changelog.d/19817.misc @@ -0,0 +1 @@ +Port the python Event classes to Rust. diff --git a/rust/src/events/internal_metadata.rs b/rust/src/events/internal_metadata.rs index 93f13f655ef..d61bf0d48c8 100644 --- a/rust/src/events/internal_metadata.rs +++ b/rust/src/events/internal_metadata.rs @@ -563,7 +563,7 @@ impl EventInternalMetadata { Ok(dict.into()) } - fn is_outlier(&self) -> PyResult { + pub fn is_outlier(&self) -> PyResult { Ok(self.read_inner()?.is_outlier()) } diff --git a/rust/src/events/mod.rs b/rust/src/events/mod.rs index a4fe15c522b..83900e14baa 100644 --- a/rust/src/events/mod.rs +++ b/rust/src/events/mod.rs @@ -60,6 +60,7 @@ use pyo3::{ use pythonize::{depythonize, pythonize}; use crate::events::{ + constants::event_type::M_ROOM_MEMBER, formats::{ EventFormatEnum, EventFormatV1, EventFormatV2V3, EventFormatV4, EventFormatVMSC4242, FormattedEvent, @@ -409,6 +410,52 @@ impl Event { Some(duration) } + fn __str__(&self) -> PyResult { + self.__repr__() + } + + /// We want to include some representative fields in the repr for + /// debugging purposes. + /// + /// The end result will look something like: + /// `` + fn __repr__(&self) -> PyResult { + let mut fields: Vec<(&str, &str)> = Vec::with_capacity(6); + + if let Some(reason) = self.rejected_reason.as_deref() { + fields.push(("REJECTED", reason)); + }; + + fields.push(("event_id", &self.event_id)); + fields.push(("type", self.r#type())); + + if let Some(state_key) = self.get_state_key() { + fields.push(("state_key", state_key)); + }; + + // Include the membership field for membership events. + if self.r#type() == M_ROOM_MEMBER && self.is_state() { + let content = &self.parsed_event.common_fields.content; + let membership_field = content.get_field(MEMBERSHIP).and_then(|m| m.as_str()); + + if let Some(membership_str) = membership_field { + fields.push(("membership", membership_str)); + } + } + + if self.internal_metadata.is_outlier()? { + fields.push(("outlier", "true")); + } + + let fields_str = fields + .into_iter() + .map(|(name, value)| format!("{}={}", name, value)) + .collect::>() + .join(", "); + + Ok(format!("", fields_str)) + } + // Below are the methods for interacting with the event as a mapping. // // These are rarely used, so we take the easy approach of re-serializing the From 7afece6261250be5a3840ccb025031e7f595e01d Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 3 Jun 2026 10:27:55 -0500 Subject: [PATCH 76/92] add assert_never() statement at the end of execute_values(), to prevent accidents with database engines --- synapse/storage/database.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 202ea5abc1c..bff590e129b 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -44,7 +44,7 @@ import attr from prometheus_client import Counter, Histogram -from typing_extensions import Concatenate, ParamSpec +from typing_extensions import Concatenate, ParamSpec, assert_never from twisted.enterprise import adbapi from twisted.internet.interfaces import IReactorCore @@ -476,7 +476,7 @@ def execute_values( sql, values, ) - else: + elif isinstance(self.database_engine, PsycopgEngine): # We use fetch = False to mean a writable query. You *might* be able # to morph that into a COPY (...) FROM STDIN, but it isn't worth the # effort for the few places we set fetch = False. @@ -499,6 +499,9 @@ def f( # Flatten the values. return self._do_execute(f, sql, list(itertools.chain.from_iterable(values))) + # mypy does not like that self.database_engine is not a Never type + assert_never(self.database_engine) # type: ignore[arg-type] + def copy_write( self, sql: str, args: Iterable[Any], values: Iterable[Iterable[Any]] ) -> None: From 4305c480cac5f098f26c166dd0f27fa334671420 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 3 Jun 2026 10:30:23 -0500 Subject: [PATCH 77/92] assert that inner values are the same length --- synapse/storage/database.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index bff590e129b..8430dbc89d5 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -37,6 +37,7 @@ Literal, Mapping, Sequence, + Sized, TypeVar, cast, overload, @@ -483,7 +484,19 @@ def execute_values( assert fetch is True # execute_values requires a single replacement, but we need to expand it - # for COPY. This assumes all inner sequences are the same length. + # for COPY. These inner sequences must be the same length. + assertion_length = 0 + for _inner_value in values: + # Check for the Sized class here, to verify that this particular + # iterable can use len(). In the future, switch the `values` argument to + # use this as a Collection instead of an Iterable, which allows all the + # types wanted while excluding Generators and this assertion can be + # removed. + assert isinstance(_inner_value, Sized) + if not assertion_length: + assertion_length = len(_inner_value) + assert assertion_length == len(_inner_value) + value_str = "(" + ", ".join("?" for _ in next(iter(values))) + ")" sql = sql.replace("?", ", ".join(value_str for _ in values)) From 8618482059ab69bacc903db319633ee8114a7d5b Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 3 Jun 2026 10:30:40 -0500 Subject: [PATCH 78/92] add assertion that template is not used yet and comment why --- synapse/storage/database.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 8430dbc89d5..88b93791ac8 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -483,6 +483,11 @@ def execute_values( # effort for the few places we set fetch = False. assert fetch is True + # For the moment, no code paths in Synapse use `template` with `fetch=True` + # but the opposite is true. Prevent this mistake until the code can handle + # templates as well. + assert template is None + # execute_values requires a single replacement, but we need to expand it # for COPY. These inner sequences must be the same length. assertion_length = 0 From 9f2ce523614844954aee5dcb0d35664661ab1fc1 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 3 Jun 2026 10:44:39 -0500 Subject: [PATCH 79/92] Document how to see Rust build failure output when using `poetry install` (#19818) Knowledge from @reivilibre in [`#element-backend-internal:matrix.org`](https://matrix.to/#/!SGNQGPGUwtcPBUotTL:matrix.org/$2WUfCdA02wZw-6-jhw3QbLQ44BmKJrqLuZ6wjz2r7hk?via=jki.re&via=element.io&via=matrix.org) on 2025-12-15. Spawning from me making Rust changes but nothing useful was printed until I added `-v`, ```shell $ poetry install --extras all Installing dependencies from lock file Package operations: 0 installs, 1 update, 0 removals - Updating pyjwt (2.11.0 -> 2.12.0) Installing the current project: matrix-synapse (1.154.0rc1) Failed to install /home/eric/Documents/github/element/synapse ``` I also see `poetry run maturin develop` suggested but I'd prefer not to need to install `maturin` as yet another system tool to manage myself. --- changelog.d/19818.doc | 1 + docs/development/contributing_guide.md | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19818.doc diff --git a/changelog.d/19818.doc b/changelog.d/19818.doc new file mode 100644 index 00000000000..c6aca252571 --- /dev/null +++ b/changelog.d/19818.doc @@ -0,0 +1 @@ +Document how to see Rust build failure output when using `poetry install`. diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md index ab506b5f05a..f967dda6f29 100644 --- a/docs/development/contributing_guide.md +++ b/docs/development/contributing_guide.md @@ -153,8 +153,14 @@ writing new pages, please to check that your contributions render correctly. The docs are written in [GitHub-Flavoured Markdown](https://guides.github.com/features/mastering-markdown/). +## Making changes to the Rust code + When changes are made to any Rust code then you must call either `poetry install` -or `maturin develop` (if installed) to rebuild the Rust code. Using [`maturin`](https://github.com/PyO3/maturin) +or `maturin develop` (if installed) to rebuild the Rust code. + +You can use `poetry install -v` or `-vv` to get more info about build failures. + +Using [`maturin`](https://github.com/PyO3/maturin) is quicker than `poetry install`, so is recommended when making frequent changes to the Rust code. From b706c1d8e79963bae93c8b67190e3f564751777b Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 3 Jun 2026 12:38:59 -0500 Subject: [PATCH 80/92] try and summarize why using COPY TO as a replacement for execute_values() --- synapse/storage/database.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 88b93791ac8..a290375d39e 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -502,6 +502,28 @@ def execute_values( assertion_length = len(_inner_value) assert assertion_length == len(_inner_value) + # To avoid having to port several psycopg2 utilities that are built into its + # Cursor class(mogrify, for example) and import execute_values() from it's + # 'extras' module, use a different mechanism that facilitates performance. + # + # The COPY Postgres-only verb allows for a bulk import and export of data. + # However building this query for use with VALUES is somewhat convoluted. + # + # For exporting of data, which would be the equivalent of a SELECT query, + # and given a simple query of the sort: + # SELECT * FROM table, (VALUES ?) AS ld(id) WHERE table.id = ld.id + # and VALUES being an example of sequential numbers, 1-5 representing the 5 + # rows to retrieve, the "VALUES ?" clause needs to be expanded to + # VALUES (?), (?), (?), (?), (?) + # Then, the values themselves have to be flattened in a similar fashion + # [(1, 2, 3, 4, 5)] + # Similarly, disregarding that the WHERE clause will no longer hold true, if + # the count VALUES to be passed were 3, then VALUES clause would expand to: + # VALUES (?, ?, ?), (?, ?, ?), (?, ?, ?), (?, ?, ?), (?, ?, ?) + # and for berevity assume that all passed values were actually the same, the + # values would be flattened to look like: + # [(1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5)] + value_str = "(" + ", ".join("?" for _ in next(iter(values))) + ")" sql = sql.replace("?", ", ".join(value_str for _ in values)) From 003febe399a94c99c2da6d6184cb7aa2d6f6f938 Mon Sep 17 00:00:00 2001 From: Noah Markert Date: Thu, 4 Jun 2026 00:36:58 +0200 Subject: [PATCH 81/92] Add tests to ensure that email notification links are sanitized (#19741) Fix #2860 Also cleans up comments around plans to define `ALLOWED_SCHEMES` as we can rely on Bleach's `ALLOWED_PROTOCOLS` defaults (`http`, `https` and `mailto`). --- changelog.d/19741.misc | 1 + synapse/push/mailer.py | 4 --- tests/push/test_email.py | 67 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 changelog.d/19741.misc diff --git a/changelog.d/19741.misc b/changelog.d/19741.misc new file mode 100644 index 00000000000..55347570d8b --- /dev/null +++ b/changelog.d/19741.misc @@ -0,0 +1 @@ +Added tests to ensure that email notification links are sanitized. Contributed by Noah Markert. diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 1ebbc6d4f39..c1cd070d8e8 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -111,8 +111,6 @@ # would make sense if we did "img": ["src"], } -# When bleach release a version with this option, we can specify schemes -# ALLOWED_SCHEMES = ["http", "https", "ftp", "mailto"] class Mailer: @@ -971,8 +969,6 @@ def safe_markup(raw_html: str) -> Markup: raw_html, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRS, - # bleach master has this, but it isn't released yet - # protocols=ALLOWED_SCHEMES, strip=True, ) ) diff --git a/tests/push/test_email.py b/tests/push/test_email.py index d3822b8643d..b1dcd38704f 100644 --- a/tests/push/test_email.py +++ b/tests/push/test_email.py @@ -379,6 +379,73 @@ def test_room_notifications_include_avatar(self) -> None: self.assertIn("_matrix/media/v1/thumbnail/DUMMY_MEDIA_ID", html) + @parameterized.expand( + [ + ( + "https allowed", + 'test click me', + 'test click me', + ), + ( + "http allowed", + 'test click me', + 'test click me', + ), + ( + "mailto allowed", + 'test click me', + 'test click me', + ), + ( + "javascript disallowed", + 'test click me', + "test click me", + ), + ( + "data disallowed", + 'test click me', + "test click me", + ), + ], + ) + def test_link_schemes_sanitized( + self, + _test_label: str, + input: str, + expected: str, + ) -> None: + # Create a room + room = self.helper.create_room_as(self.user_id, tok=self.access_token) + + self.helper.invite( + room=room, src=self.user_id, tok=self.access_token, targ=self.others[0].id + ) + self.helper.join(room=room, user=self.others[0].id, tok=self.others[0].token) + + self.helper.send_event( + room, + type="m.room.message", + content={ + "msgtype": "m.text", + "format": "org.matrix.custom.html", + "formatted_body": input, + "body": "test click me", # Plain-text Fallback + }, + tok=self.others[0].token, + ) + + args, kwargs = self._check_for_mail() + msg: bytes = args[5] + html_message = email.message_from_bytes(msg).get_payload(i=1) + assert isinstance(html_message, email.message.Message) + + # Extract the `bytes` from the html Message object, and decode to a `str`. + html = html_message.get_payload(decode=True) + assert isinstance(html, bytes) + html = html.decode() + + self.assertIn(expected, html) + def test_empty_room(self) -> None: """All users leaving a room shouldn't cause the pusher to break.""" # Create a simple room with two users From 102fb6e8ae44536f6d46b0819000f2fac338103b Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Thu, 4 Jun 2026 14:16:30 +0100 Subject: [PATCH 82/92] 1.154.0 --- CHANGES.md | 4 ++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index c3ed13ecb51..e210f8f1d52 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +# Synapse 1.154.0 (2026-06-04) + +No significant changes since 1.154.0rc1. + # Synapse 1.154.0rc1 (2026-05-27) ## Features diff --git a/debian/changelog b/debian/changelog index 4c3999413b6..7750ca563c2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.154.0) stable; urgency=medium + + * New Synapse release 1.154.0. + + -- Synapse Packaging team Thu, 04 Jun 2026 14:16:23 +0100 + matrix-synapse-py3 (1.154.0~rc1) stable; urgency=medium * New Synapse release 1.154.0rc1. diff --git a/pyproject.toml b/pyproject.toml index 76ccbbe946a..89bace10a2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.154.0rc1" +version = "1.154.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 678e7116b1f06d74d3883fb02eebba991105c92c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 4 Jun 2026 16:13:29 +0100 Subject: [PATCH 83/92] Reduce log spam from to-device stream logging (#19821) Followup to https://github.com/element-hq/synapse/pull/19801: I only meant for this logging to happen on the instance that is doing the persisting. --- changelog.d/19821.misc | 1 + synapse/storage/util/id_generators.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19821.misc diff --git a/changelog.d/19821.misc b/changelog.d/19821.misc new file mode 100644 index 00000000000..56ec0f7ddca --- /dev/null +++ b/changelog.d/19821.misc @@ -0,0 +1 @@ +Add more logging to the to-device message replication stream. diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index 1e053be6aff..c9c339b235c 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -828,9 +828,11 @@ def _add_persisted_position(self, new_id: int) -> None: # do. break - # Hacky debug logging to attempt to trace https://github.com/element-hq/synapse/issues/19795 + # Hacky debug logging to attempt to trace https://github.com/element-hq/synapse/issues/19795. + # If this is the to-device stream, and we are a writer for that stream, log some stats if ( issue9533_logger.isEnabledFor(logging.DEBUG) + and our_current_position > 0 and self._stream_name == "to_device" ): issue9533_logger.debug( From 7f45a1ce2e5e6d47b2a6db87a0dc75cbe1da4c90 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 5 Jun 2026 16:36:18 +0100 Subject: [PATCH 84/92] Document that the SQLite version included in Ubuntu LTS, aside from ESM-only versions, is included in our support policy. (#19823) The reason for querying this support was wanting support for SQLite's JSON operators, which are currently not present in the SQLite version found in Ubuntu's oldest supported LTS. The JSON operators were used in some of the sticky events work (related: #19452). Our ruling was that we should support Ubuntu oldest LTS equally to Debian oldstable, so support the oldest of the two versions from those. That makes some kind of sense as it would be difficult to do otherwise without dropping support for that version of Ubuntu altogether, given if we kept publishing packages intended for use with Postgres, there's a risk that an innocent sysadmin would update their SQLite deployment without realising that it is no longer supported. This was [discussed months ago (private)](https://docs.google.com/document/d/12RZKPk3a4__JUSH9wYHODo9rRyKzsHg6BSCAcmqmbOU/edit?tab=t.0#bookmark=id.fcdvoc88dy5s) and at [private](https://docs.google.com/document/d/12RZKPk3a4__JUSH9wYHODo9rRyKzsHg6BSCAcmqmbOU/edit?tab=t.0#bookmark=id.u48ivjge4qpt). --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19823.doc | 1 + docs/deprecation_policy.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19823.doc diff --git a/changelog.d/19823.doc b/changelog.d/19823.doc new file mode 100644 index 00000000000..5bbbdd5ccf2 --- /dev/null +++ b/changelog.d/19823.doc @@ -0,0 +1 @@ +Document that the SQLite version included in Ubuntu LTS, aside from ESM-only versions, is included in our support policy. \ No newline at end of file diff --git a/docs/deprecation_policy.md b/docs/deprecation_policy.md index 06c724d3489..2a4301b4ee0 100644 --- a/docs/deprecation_policy.md +++ b/docs/deprecation_policy.md @@ -22,7 +22,7 @@ people building from source should ensure they can fetch recent versions of Rust The oldest supported version of SQLite is the version [provided](https://packages.debian.org/oldstable/libsqlite3-0) by -[Debian oldstable](https://wiki.debian.org/DebianOldStable). +[Debian oldstable](https://wiki.debian.org/DebianOldStable) or the oldest maintenance/security-supported [Ubuntu LTS](https://endoflife.date/ubuntu) (Ubuntu versions with only Expanded Security Maintenance are not included). ### Context From 8c9b1ff877dc8f2155b7d0a89c5193f74c4cde21 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 8 Jun 2026 11:42:57 +0100 Subject: [PATCH 85/92] Reintroduce #19714 - Send a SSS response immediately if the config has changed (#19792) Reintroduces #19714, after being reverted in #19784. Fix https://github.com/element-hq/synapse/issues/18844 Fix https://github.com/element-hq/synapse/issues/19783 Fix https://github.com/element-hq/synapse/issues/18880 This PR also adds a fix so that we don't always return immediately when using the e2ee extension. --------- Co-authored-by: Benjamin Bouvier Co-authored-by: Eric Eastwood Co-authored-by: Eric Eastwood --- changelog.d/19734.bugfix | 1 + changelog.d/19792.bugfix | 1 + synapse/handlers/sliding_sync/__init__.py | 59 ++++--- synapse/handlers/sliding_sync/room_lists.py | 12 +- synapse/types/handlers/sliding_sync.py | 64 +++++-- .../sliding_sync/test_extension_e2ee.py | 167 ++++++++++++++++++ .../sliding_sync/test_room_subscriptions.py | 119 +++++++++++++ .../sliding_sync/test_rooms_required_state.py | 81 ++++++++- .../client/sliding_sync/test_sliding_sync.py | 36 +++- 9 files changed, 491 insertions(+), 49 deletions(-) create mode 100644 changelog.d/19734.bugfix create mode 100644 changelog.d/19792.bugfix diff --git a/changelog.d/19734.bugfix b/changelog.d/19734.bugfix new file mode 100644 index 00000000000..01af7d9ab83 --- /dev/null +++ b/changelog.d/19734.bugfix @@ -0,0 +1 @@ +Update Sliding Sync to return a new response immediately if a room subscription have changed and produced a new response. diff --git a/changelog.d/19792.bugfix b/changelog.d/19792.bugfix new file mode 100644 index 00000000000..01af7d9ab83 --- /dev/null +++ b/changelog.d/19792.bugfix @@ -0,0 +1 @@ +Update Sliding Sync to return a new response immediately if a room subscription have changed and produced a new response. diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index 4d6287b147c..e7f0e7340be 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -184,34 +184,45 @@ async def wait_for_sync_for_user( timeout_ms -= after_wait_ts - before_wait_ts timeout_ms = max(timeout_ms, 0) - # We're going to respond immediately if the timeout is 0 or if this is an - # initial sync (without a `from_token`) so we can avoid calling - # `notifier.wait_for_events()`. - if timeout_ms == 0 or from_token is None: - now_token = self.event_sources.get_current_token() - result = await self.current_sync_for_user( + # Compute a response immediately. We always need to do this before + # waiting for new data (unlike in /v3/sync), as the request config might + # have changed (e.g. new room subscriptions, etc). + now_token = self.event_sources.get_current_token() + result = await self.current_sync_for_user( + sync_config, + from_token=from_token, + to_token=now_token, + ) + + # Return immediately if we have a result, the timeout is 0, or this is + # an initial sync. + if result or timeout_ms == 0 or from_token is None: + return result, did_wait + + # Otherwise, we wait for something to happen and report it to the user. + async def current_sync_callback( + before_token: StreamToken, after_token: StreamToken + ) -> SlidingSyncResult: + return await self.current_sync_for_user( sync_config, from_token=from_token, - to_token=now_token, + to_token=after_token, ) - else: - # Otherwise, we wait for something to happen and report it to the user. - async def current_sync_callback( - before_token: StreamToken, after_token: StreamToken - ) -> SlidingSyncResult: - return await self.current_sync_for_user( - sync_config, - from_token=from_token, - to_token=after_token, - ) - result = await self.notifier.wait_for_events( - sync_config.user.to_string(), - timeout_ms, - current_sync_callback, - from_token=from_token.stream_token, - ) - did_wait = True + result = await self.notifier.wait_for_events( + sync_config.user.to_string(), + timeout_ms, + current_sync_callback, + # We *wait* from `now_token` as we have already computed the sync + # response up to `now_token` above, so as a minor optimization, we + # can wait for something new to arrive after `now_token`. + # + # We still generate the sync response using `from_token` in the + # callback above though, as to generate the correct response it + # needs to know the "real" `from_token`. + from_token=now_token, + ) + did_wait = True return result, did_wait diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index 8969d915836..216ef3b0710 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -852,11 +852,15 @@ async def _filter_relevant_rooms_to_send( previous_connection_state.room_configs.get(room_id) ) if prev_room_sync_config is not None: - # Always include rooms whose timeline limit has increased. - # (see the "XXX: Odd behavior" described below) + # Always include rooms whose effective config has + # expanded. This covers timeline-limit increases and + # required-state additions introduced by room + # subscriptions overriding list-derived params. if ( - prev_room_sync_config.timeline_limit - < room_config.timeline_limit + prev_room_sync_config.combine_room_sync_config( + room_config + ) + != prev_room_sync_config ): rooms_should_send.add(room_id) continue diff --git a/synapse/types/handlers/sliding_sync.py b/synapse/types/handlers/sliding_sync.py index 1a84bf1ff8f..e73533d296d 100644 --- a/synapse/types/handlers/sliding_sync.py +++ b/synapse/types/handlers/sliding_sync.py @@ -203,6 +203,9 @@ class StrippedHero: highlight_count: int def __bool__(self) -> bool: + """Are there any updates that should be returned immediately to + the client? + """ return ( # If this is the first time the client is seeing the room, we should not filter it out # under any circumstance. @@ -270,6 +273,8 @@ class ToDeviceExtension: events: Sequence[JsonMapping] def __bool__(self) -> bool: + """Are there any updates that should be returned immediately to + the client?""" return bool(self.events) @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -294,23 +299,37 @@ class E2eeExtension: device_unused_fallback_key_types: Sequence[str] def __bool__(self) -> bool: - # Note that "signed_curve25519" is always returned in key count responses - # regardless of whether we uploaded any keys for it. This is necessary until + """Are there any updates that should be returned immediately to + the client?""" + # Note that "signed_curve25519" is always returned in key count + # responses regardless of whether we uploaded any keys for it. + # This is necessary until # https://github.com/matrix-org/matrix-doc/issues/3298 is fixed. # # Also related: # https://github.com/element-hq/element-android/issues/3725 and # https://github.com/matrix-org/synapse/issues/10456 - default_otk = self.device_one_time_keys_count.get("signed_curve25519") - more_than_default_otk = len(self.device_one_time_keys_count) > 1 or ( - default_otk is not None and default_otk > 0 - ) - - return bool( - more_than_default_otk - or self.device_list_updates - or self.device_unused_fallback_key_types - ) + # + # This is why we don't incorporate `device_one_time_keys_count` + # (or `device_unused_fallback_key_types`) into the `__bool__` + # check. + # + # FIXME: Ideally we'd detect if either of those fields have + # changed since the last sync, but we do not currently track + # such state. + # + # Note that the client will receive these fields eventually when + # we respond to the sync request (usually sync timeouts are set + # to ~30s), we just won't immediately respond (even if there are + # changes). This delay is acceptable for clients, as a) these + # fields do not trigger UI (and so don't affect user perceivable + # latency) and b) are handled in the background by the clients + # anyway. The only risk being that one-time keys could be exhausted + # before the client knows about adding some more. But for example, + # if the client is syncing with a timeout of 30s, the window of + # staleness is so small for this not to matter. + + return bool(self.device_list_updates) @attr.s(slots=True, frozen=True, auto_attribs=True) class AccountDataExtension: @@ -327,6 +346,8 @@ class AccountDataExtension: account_data_by_room_map: Mapping[str, Mapping[str, JsonMapping]] def __bool__(self) -> bool: + """Are there any updates that should be returned immediately to + the client?""" return bool( self.global_account_data_map or self.account_data_by_room_map ) @@ -343,6 +364,8 @@ class ReceiptsExtension: room_id_to_receipt_map: Mapping[str, JsonMapping] def __bool__(self) -> bool: + """Are there any updates that should be returned immediately to + the client?""" return bool(self.room_id_to_receipt_map) @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -357,6 +380,8 @@ class TypingExtension: room_id_to_typing_map: Mapping[str, JsonMapping] def __bool__(self) -> bool: + """Are there any updates that should be returned immediately to + the client?""" return bool(self.room_id_to_typing_map) @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -391,6 +416,8 @@ class ThreadUnsubscription: prev_batch: ThreadSubscriptionsToken | None def __bool__(self) -> bool: + """Are there any updates that should be returned immediately to + the client?""" return ( bool(self.subscribed) or bool(self.unsubscribed) @@ -405,6 +432,8 @@ def __bool__(self) -> bool: thread_subscriptions: ThreadSubscriptionsExtension | None = None def __bool__(self) -> bool: + """Are there any updates that should be returned immediately to + the client?""" return bool( self.to_device or self.e2ee @@ -420,9 +449,14 @@ def __bool__(self) -> bool: extensions: Extensions def __bool__(self) -> bool: - """Make the result appear empty if there are no updates. This is used - to tell if the notifier needs to wait for more events when polling for - events. + """Are there any updates that should be returned immediately to + the client? + + This is used to determine if a sliding sync response should be returned + immediately or if the notifier needs to wait for further updates, and + thus MUST return false if there is no new data since the last sync. This + is subtly different than just checking if any of the fields are set, + since some fields are always included (like `bump_stamp`). """ # We don't include `self.lists` here, as a) `lists` is always non-empty even if # there are no changes, and b) since we're sorting rooms by `stream_ordering` of diff --git a/tests/rest/client/sliding_sync/test_extension_e2ee.py b/tests/rest/client/sliding_sync/test_extension_e2ee.py index 4a5e4070382..2707e4d98dc 100644 --- a/tests/rest/client/sliding_sync/test_extension_e2ee.py +++ b/tests/rest/client/sliding_sync/test_extension_e2ee.py @@ -290,6 +290,173 @@ def test_wait_for_new_data_timeout(self) -> None: [], ) + def test_wait_for_new_data_timeout_with_otks(self) -> None: + """ + Test that an incremental Sliding Sync with the e2ee extension enabled + does not return immediately when the user has uploaded one-time keys + (i.e. `device_one_time_keys_count` contains entries beyond the default + `signed_curve25519: 0`). + """ + test_device_id = "TESTDEVICE" + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass", device_id=test_device_id) + + # Upload one-time keys for the user/device so that + # `device_one_time_keys_count` is non-default in the response. + self.get_success( + self.e2e_keys_handler.upload_keys_for_user( + user1_id, + test_device_id, + { + "one_time_keys": { + "alg1:k1": "key1", + "alg2:k2": {"key": "key2", "signatures": {"k1": "sig1"}}, + } + }, + ) + ) + + sync_body = { + "lists": {}, + "extensions": { + "e2ee": { + "enabled": True, + } + }, + } + _, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Make an incremental Sliding Sync request with a timeout + channel = self.make_request( + "POST", + self.sync_endpoint + f"?timeout=10000&pos={from_token}", + content=sync_body, + access_token=user1_tok, + await_result=False, + ) + # Block for 5 seconds to make sure we are `notifier.wait_for_events(...)` + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=5000) + + # Wake-up `notifier.wait_for_events(...)` that will cause us to test + # `SlidingSyncResult.__bool__` for new results. The non-default + # `device_one_time_keys_count` must not be considered new data. + self._bump_notifier_wait_for_events( + user1_id, wake_stream_key=StreamKeyType.ACCOUNT_DATA + ) + + # Block for a little bit more to ensure we don't see any new results. + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=4000) + # Wait for the sync to complete (wait for the rest of the 10 second timeout, + # 5000 + 4000 + 1200 > 10000) + channel.await_result(timeout_ms=1200) + self.assertEqual(channel.code, 200, channel.json_body) + + # Device lists are present for incremental syncs but empty because no device changes + self.assertEqual( + channel.json_body["extensions"]["e2ee"] + .get("device_lists", {}) + .get("changed"), + [], + ) + self.assertEqual( + channel.json_body["extensions"]["e2ee"].get("device_lists", {}).get("left"), + [], + ) + + # The one-time key counts are present, but they should not have caused + # the sync to return early. + self.assertEqual( + channel.json_body["extensions"]["e2ee"]["device_one_time_keys_count"], + { + "alg1": 1, + "alg2": 1, + "signed_curve25519": 0, + }, + ) + self.assertEqual( + channel.json_body["extensions"]["e2ee"]["device_unused_fallback_key_types"], + [], + ) + + def test_wait_for_new_data_timeout_with_fallback_keys(self) -> None: + """ + Test that an incremental Sliding Sync with the e2ee extension enabled + does not return immediately when the user has uploaded fallback keys + (i.e. `device_unused_fallback_key_types` is non-empty). + """ + test_device_id = "TESTDEVICE" + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass", device_id=test_device_id) + + # Upload a fallback key for the user/device so that + # `device_unused_fallback_key_types` is non-empty in the response. + self.get_success( + self.e2e_keys_handler.upload_keys_for_user( + user1_id, + test_device_id, + {"fallback_keys": {"alg1:k1": "fallback_key1"}}, + ) + ) + + sync_body = { + "lists": {}, + "extensions": { + "e2ee": { + "enabled": True, + } + }, + } + _, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Make an incremental Sliding Sync request with a timeout + channel = self.make_request( + "POST", + self.sync_endpoint + f"?timeout=10000&pos={from_token}", + content=sync_body, + access_token=user1_tok, + await_result=False, + ) + # Block for 5 seconds to make sure we are `notifier.wait_for_events(...)` + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=5000) + + # Wake-up `notifier.wait_for_events(...)` that will cause us to test + # `SlidingSyncResult.__bool__` for new results. The non-empty + # `device_unused_fallback_key_types` must not be considered new data. + self._bump_notifier_wait_for_events( + user1_id, wake_stream_key=StreamKeyType.ACCOUNT_DATA + ) + + # Block for a little bit more to ensure we don't see any new results. + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=4000) + + # Wait for the sync to complete (wait for the rest of the 10 second timeout, + # 5000 + 4000 + 1200 > 10000) + channel.await_result(timeout_ms=1200) + self.assertEqual(channel.code, 200, channel.json_body) + + # Device lists are present for incremental syncs but empty because no device changes + self.assertEqual( + channel.json_body["extensions"]["e2ee"] + .get("device_lists", {}) + .get("changed"), + [], + ) + self.assertEqual( + channel.json_body["extensions"]["e2ee"].get("device_lists", {}).get("left"), + [], + ) + + # The unused fallback key types are present, but they should not have + # caused the sync to return early. + self.assertEqual( + channel.json_body["extensions"]["e2ee"]["device_unused_fallback_key_types"], + ["alg1"], + ) + def test_device_lists(self) -> None: """ Test that device list updates are included in the response diff --git a/tests/rest/client/sliding_sync/test_room_subscriptions.py b/tests/rest/client/sliding_sync/test_room_subscriptions.py index 811478f1ba7..d970af367d3 100644 --- a/tests/rest/client/sliding_sync/test_room_subscriptions.py +++ b/tests/rest/client/sliding_sync/test_room_subscriptions.py @@ -22,6 +22,7 @@ from synapse.api.constants import EventTypes, HistoryVisibility from synapse.rest.client import login, room, sync from synapse.server import HomeServer +from synapse.types import JsonDict from synapse.util.clock import Clock from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase @@ -126,6 +127,124 @@ def test_room_subscriptions_with_join_membership(self) -> None: response_body["rooms"][room_id1], ) + def test_room_subscription_required_state_expansion_returns_immediately( + self, + ) -> None: + """ + Test that adding a room subscription with stronger params than the list causes an + incremental long-poll to return immediately, even without new stream activity. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + room_id1 = self.helper.create_room_as(user1_id, tok=user1_tok) + + sync_body: JsonDict = { + "lists": { + "foo-list": { + "ranges": [[0, 0]], + "required_state": [], + "timeline_limit": 0, + } + }, + "conn_id": "conn_id", + } + _, from_token = self.do_sync(sync_body, tok=user1_tok) + + sync_body["room_subscriptions"] = { + room_id1: { + "required_state": [ + [EventTypes.Create, ""], + ], + "timeline_limit": 0, + } + } + + channel = self.make_request( + "POST", + self.sync_endpoint + f"?timeout=10000&pos={from_token}", + content=sync_body, + access_token=user1_tok, + await_result=False, + ) + channel.await_result(timeout_ms=3000) + self.assertEqual(channel.code, 200, channel.json_body) + + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + + room_response = channel.json_body["rooms"][room_id1] + self.assertNotIn("initial", room_response) + self._assertRequiredStateIncludes( + room_response["required_state"], + { + state_map[(EventTypes.Create, "")], + }, + exact=True, + ) + + def test_room_subscription_required_state_change_returns_immediately(self) -> None: + """ + Test that expanding an existing room subscription's required state causes an + incremental long-poll to return immediately, even without new stream activity. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + room_id1 = self.helper.create_room_as( + user1_id, tok=user1_tok, extra_content={"name": "Foo"} + ) + + sync_body: JsonDict = { + "room_subscriptions": { + room_id1: { + "required_state": [ + [EventTypes.Create, ""], + ], + "timeline_limit": 0, + } + }, + "conn_id": "conn_id", + } + response_body, from_token = self.do_sync(sync_body, tok=user1_tok) + + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + state_map[(EventTypes.Create, "")], + }, + exact=True, + ) + + sync_body["room_subscriptions"][room_id1]["required_state"] = [ + [EventTypes.Create, ""], + [EventTypes.Name, ""], + ] + + channel = self.make_request( + "POST", + self.sync_endpoint + f"?timeout=10000&pos={from_token}", + content=sync_body, + access_token=user1_tok, + await_result=False, + ) + channel.await_result(timeout_ms=3000) + self.assertEqual(channel.code, 200, channel.json_body) + + room_response = channel.json_body["rooms"][room_id1] + self.assertNotIn("initial", room_response) + self._assertRequiredStateIncludes( + room_response["required_state"], + { + state_map[(EventTypes.Name, "")], + }, + exact=True, + ) + def test_room_subscriptions_with_leave_membership(self) -> None: """ Test `room_subscriptions` with a leave room should give us timeline and state diff --git a/tests/rest/client/sliding_sync/test_rooms_required_state.py b/tests/rest/client/sliding_sync/test_rooms_required_state.py index 586b127f8a1..901f22a35d1 100644 --- a/tests/rest/client/sliding_sync/test_rooms_required_state.py +++ b/tests/rest/client/sliding_sync/test_rooms_required_state.py @@ -25,8 +25,10 @@ from synapse.server import HomeServer from synapse.storage.databases.main.events import DeltaState, SlidingSyncTableChanges from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase +from tests.server import TimedOutException from tests.test_utils.event_injection import mark_event_as_partial_state logger = logging.getLogger(__name__) @@ -1924,7 +1926,12 @@ def test_rooms_required_state_expand(self) -> None: def test_rooms_required_state_expand_retract_expand(self) -> None: """Test that when expanding, retracting and then expanding the required - state, we get the changes that happened.""" + state, we get the changes that happened. + + Also see `test_changing_required_state_returns_immediately`, which tests + that the sync stream is woken up immediately when changing the required + state, and not just on the next change to the room. + """ user1_id = self.register_user("user1", "pass") user1_tok = self.login(user1_id, "pass") @@ -2245,3 +2252,75 @@ def test_lazy_loading_room_members_state_reset_non_limited_timeline(self) -> Non response_body["rooms"][room_id]["required_state"][0]["event_id"], first_event_id, ) + + def test_changing_required_state_returns_immediately(self) -> None: + """Test that if we change the `required_state`, then we return immediately + with the new `required_state`.""" + + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + room_id1 = self.helper.create_room_as(user1_id, tok=user1_tok) + + # Make an initial sync request with no required state + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [], + "timeline_limit": 0, + } + } + } + response_body, from_token = self.do_sync(sync_body, tok=user1_tok) + + # We should see no required state + self.assertIsNone(response_body["rooms"][room_id1].get("required_state")) + + # Get the state_map before we change the state as this is the final state we + # expect to see when we update the required state. + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + + # There is no new data, and so making another sync request will block. + channel = self.make_sync_request( + sync_body, + since=from_token, + tok=user1_tok, + timeout=Duration(seconds=10), + await_result=False, + ) + + # Request will block for 10 seconds as there no updates. + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=9500) + + # Wait for the request to actually finish. (We do this to ensure log + # contexts don't leak between tests). + channel.await_result(timeout_ms=1000) + + # Now update the Sliding Sync requests to include a `required_state` + # event, and make another sync request. + sync_body["lists"]["foo-list"]["required_state"] = [ + [EventTypes.Create, ""], + ] + + channel = self.make_sync_request( + sync_body, + since=from_token, + tok=user1_tok, + timeout=Duration(seconds=10), + await_result=False, + ) + + # We should see the new `required_state` immediately without waiting + channel.await_result(timeout_ms=0) + response_body = channel.json_body + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + state_map[(EventTypes.Create, "")], + }, + exact=True, + ) diff --git a/tests/rest/client/sliding_sync/test_sliding_sync.py b/tests/rest/client/sliding_sync/test_sliding_sync.py index ebf41cd87c3..fc7d6a279c3 100644 --- a/tests/rest/client/sliding_sync/test_sliding_sync.py +++ b/tests/rest/client/sliding_sync/test_sliding_sync.py @@ -12,6 +12,7 @@ # . # import logging +import urllib.parse from typing import Any, Iterable, Literal from unittest.mock import AsyncMock @@ -43,6 +44,7 @@ StreamToken, ) from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.stringutils import random_string from tests import unittest @@ -82,7 +84,13 @@ def default_config(self) -> JsonDict: return config def make_sync_request( - self, sync_body: JsonDict, *, since: str | None = None, tok: str + self, + sync_body: JsonDict, + *, + since: str | None = None, + tok: str, + timeout: Duration | None = None, + await_result: bool = True, ) -> FakeChannel: """Make a sliding sync request with given body. @@ -90,25 +98,40 @@ def make_sync_request( sync_body: The full request body to use since: Optional since token tok: Access token to use - + timeout_ms: Optional timeout in milliseconds to use for the request. + await_result: Whether to block and wait for the result before returning. Returns: A tuple of the response body and the `pos` field. """ sync_path = self.sync_endpoint + + query_params: dict[str, Any] = {} if since: - sync_path += f"?pos={since}" + query_params["pos"] = since + if timeout is not None: + query_params["timeout"] = timeout.as_millis() + + if query_params: + query_str = urllib.parse.urlencode(query_params) + sync_path += f"?{query_str}" channel = self.make_request( method="POST", path=sync_path, content=sync_body, access_token=tok, + await_result=await_result, ) return channel def do_sync( - self, sync_body: JsonDict, *, since: str | None = None, tok: str + self, + sync_body: JsonDict, + *, + since: str | None = None, + tok: str, + timeout: Duration | None = None, ) -> tuple[JsonDict, str]: """Do a sliding sync request with given body. @@ -118,11 +141,14 @@ def do_sync( sync_body: The full request body to use since: Optional since token tok: Access token to use + timeout: Optional timeout to use for the request. Returns: A tuple of the response body and the `pos` field. """ - channel = self.make_sync_request(sync_body, since=since, tok=tok) + channel = self.make_sync_request( + sync_body, since=since, tok=tok, timeout=timeout + ) self.assertEqual(channel.code, 200, channel.json_body) return channel.json_body, channel.json_body["pos"] From ac21bf08f380a0a053fc709eaab22da83a88f06d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Jun 2026 10:26:04 +0100 Subject: [PATCH 86/92] Port `Requester` class to Rust. (#19828) This is in prep for converting the event serialization to Rust. This is a fairly mechanical port, except that we store the appservice ID rather than the appservice object. This avoids us having to store a `Py<..>` (or port the appservice object over). --- changelog.d/19828.misc | 1 + rust/src/lib.rs | 2 + rust/src/types/mod.rs | 340 ++++++++++++++++++++++++ synapse/api/auth/base.py | 6 +- synapse/api/auth/internal.py | 4 +- synapse/api/auth/mas.py | 4 +- synapse/api/auth/msc3861_delegated.py | 4 +- synapse/api/auth_blocking.py | 2 +- synapse/api/ratelimiting.py | 7 +- synapse/events/utils.py | 2 +- synapse/handlers/admin.py | 2 +- synapse/handlers/directory.py | 6 +- synapse/handlers/message.py | 22 +- synapse/handlers/room_member.py | 4 +- synapse/replication/http/membership.py | 8 +- synapse/replication/http/send_events.py | 4 +- synapse/rest/client/account.py | 2 +- synapse/rest/client/appservice_ping.py | 17 +- synapse/rest/client/devices.py | 6 +- synapse/rest/client/directory.py | 15 +- synapse/rest/client/keys.py | 2 +- synapse/rest/client/login.py | 6 +- synapse/rest/client/room.py | 4 +- synapse/rest/client/transactions.py | 4 +- synapse/rest/media/create_resource.py | 8 +- synapse/synapse_rust/types.pyi | 68 +++++ synapse/types/__init__.py | 80 +----- tests/api/test_auth.py | 12 +- tests/api/test_ratelimiting.py | 6 + tests/rest/client/test_transactions.py | 2 +- 30 files changed, 517 insertions(+), 133 deletions(-) create mode 100644 changelog.d/19828.misc create mode 100644 rust/src/types/mod.rs create mode 100644 synapse/synapse_rust/types.pyi diff --git a/changelog.d/19828.misc b/changelog.d/19828.misc new file mode 100644 index 00000000000..1ed01eba2ea --- /dev/null +++ b/changelog.d/19828.misc @@ -0,0 +1 @@ +Port `Requester` class to Rust. diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 8ed4e24b813..bf96422cbb0 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -19,6 +19,7 @@ pub mod push; pub mod rendezvous; pub mod room_versions; pub mod segmenter; +pub mod types; lazy_static! { static ref LOGGING_HANDLE: ResetHandle = pyo3_log::init(); @@ -71,6 +72,7 @@ fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { msc4388_rendezvous::register_module(py, m)?; segmenter::register_module(py, m)?; room_versions::register_module(py, m)?; + types::register_module(py, m)?; Ok(()) } diff --git a/rust/src/types/mod.rs b/rust/src/types/mod.rs new file mode 100644 index 00000000000..ffb19a83a2e --- /dev/null +++ b/rust/src/types/mod.rs @@ -0,0 +1,340 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Rust implementations of types from `synapse.types`. + +use std::collections::HashSet; + +use once_cell::sync::OnceCell; +use pyo3::prelude::*; +use pyo3::{ + exceptions::PyKeyError, + types::{PyDict, PyDictMethods, PyList}, +}; + +/// A reference to the `synapse.types.UserID` class. +static USER_ID_CLASS: OnceCell> = OnceCell::new(); + +/// Access to the `synapse.types.UserID` class. +fn user_id_class(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + Ok(USER_ID_CLASS + .get_or_try_init(|| -> PyResult<_> { + Ok(py.import("synapse.types")?.getattr("UserID")?.unbind()) + })? + .bind(py)) +} + +/// Represents the user making a request. +#[pyclass(frozen, skip_from_py_object, get_all, eq)] +#[derive(Debug, PartialEq, Eq)] +pub struct Requester { + /// The ID of the user making the request, in string form (see + /// [`Self::user`] for accessing the parsed `UserID`). + user_id: String, + /// The ID of the access token used for this request, or None for + /// appservices, guests, and tokens generated by the admin API + access_token_id: Option, + /// True if the user making this request is a guest + is_guest: bool, + /// Any scopes associated with the access token used for this request, or an + /// empty set if no token or a non-oauth token was used + scope: HashSet, + /// True if the user making this request is shadow banned + shadow_banned: bool, + /// The device_id which was set at authentication time, or None for + /// appservices, guests, and tokens generated by the admin API + device_id: Option, + /// The ID of the AS requesting on behalf of the user, or None. + app_service_id: Option, + /// The entity that authenticated when making the request. + /// + /// This is different to the `user_id` when an admin user or the server is + /// "puppeting" the user. + authenticated_entity: String, +} + +#[pymethods] +impl Requester { + #[new] + #[pyo3(signature = ( + user, + access_token_id, + is_guest, + scope, + shadow_banned, + device_id, + app_service_id, + authenticated_entity, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + user: &Bound<'_, PyAny>, + access_token_id: Option, + is_guest: &Bound<'_, PyAny>, + scope: HashSet, + shadow_banned: &Bound<'_, PyAny>, + device_id: Option, + app_service_id: Option, + authenticated_entity: String, + ) -> PyResult { + // The `user` argument should be a `UserID`, which has a `to_string` for + // getting the string form. + let user_id = user.call_method0("to_string")?.extract::()?; + + // The `is_guest` and `shadow_banned` arguments are expected to be + // Python bools, but unfortunately Synapse often passes them as truthy + // values (mainly due to reading from SQLite, which returns 0/1 for + // bools). + let is_guest = is_guest.is_truthy()?; + let shadow_banned = shadow_banned.is_truthy()?; + + Ok(Requester { + user_id, + access_token_id, + is_guest, + scope, + shadow_banned, + device_id, + app_service_id, + authenticated_entity, + }) + } + + /// The user making the request, as a Python `UserID`. + #[getter] + fn user<'py>(&self, py: Python<'py>) -> PyResult> { + user_id_class(py)?.call_method1("from_string", (&self.user_id,)) + } + + /// Converts self to a type that can be serialized as JSON, and then + /// deserialized by [`Self::deserialize`] + fn serialize<'py>(&self, py: Python<'py>) -> PyResult> { + let dict = PyDict::new(py); + dict.set_item("user_id", &self.user_id)?; + dict.set_item("access_token_id", self.access_token_id)?; + dict.set_item("is_guest", self.is_guest)?; + dict.set_item("scope", PyList::new(py, &self.scope)?)?; + dict.set_item("shadow_banned", self.shadow_banned)?; + dict.set_item("device_id", self.device_id.as_deref())?; + // NB: the wire key is "app_server_id" (server, not service). Changing + // this is non-trivial as it would break replication during a rolling + // upgrade. + dict.set_item("app_server_id", self.app_service_id.as_deref())?; + dict.set_item("authenticated_entity", &self.authenticated_entity)?; + Ok(dict) + } + + /// Converts a dict that was produced by [`Self::serialize`] back into a + /// [`Requester`]. + #[staticmethod] + fn deserialize(py: Python<'_>, input: &Bound<'_, PyAny>) -> PyResult { + let user_id = input.get_item("user_id")?.extract::()?; + let access_token_id = input + .get_item("access_token_id")? + .extract::>()?; + let is_guest = input.get_item("is_guest")?.is_truthy()?; + + // `serialize` stores the scope as a list, so extract it as a `Vec` + // (which accepts any sequence) and collect into a set. For backwards + // compatibility, "scope" is optional and defaults to an empty set if + // not present. + let scope = match input.get_item("scope") { + Ok(scope) => scope.extract::>()?.into_iter().collect(), + Err(err) if err.is_instance_of::(py) => HashSet::new(), + Err(err) => return Err(err), + }; + + let shadow_banned = input.get_item("shadow_banned")?.is_truthy()?; + let device_id = input.get_item("device_id")?.extract::>()?; + + // The wire key is "app_server_id", not "app_service_id" — see `serialize`. + let app_service_id = input + .get_item("app_server_id")? + .extract::>()?; + + let authenticated_entity = input + .get_item("authenticated_entity")? + .extract::()?; + + Ok(Requester { + user_id, + access_token_id, + is_guest, + scope, + shadow_banned, + device_id, + app_service_id, + authenticated_entity, + }) + } + + fn __repr__(&self) -> String { + format!( + "Requester(user_id={}, access_token_id={:?}, is_guest={}, scope={:?}, \ + shadow_banned={}, device_id={:?}, app_service_id={:?}, authenticated_entity={})", + self.user_id, + self.access_token_id, + self.is_guest, + self.scope, + self.shadow_banned, + self.device_id, + self.app_service_id, + self.authenticated_entity, + ) + } +} + +/// Called when registering modules with python. +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child_module = PyModule::new(py, "types")?; + child_module.add_class::()?; + + m.add_submodule(&child_module)?; + + // We need to manually add the module to sys.modules to make `from + // synapse.synapse_rust.types import Requester` work. + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.types", child_module)?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A `Requester` with every field populated, for use as a test fixture. + fn sample_requester() -> Requester { + Requester { + user_id: "@alice:example.com".to_string(), + access_token_id: Some(42), + is_guest: false, + scope: HashSet::from(["urn:matrix:client:api:*".to_string()]), + shadow_banned: false, + device_id: Some("ABCDEFG".to_string()), + app_service_id: None, + authenticated_entity: "@alice:example.com".to_string(), + } + } + + #[test] + fn test_serialize() { + Python::initialize(); + Python::attach(|py| -> Result<(), PyErr> { + let requester = sample_requester(); + let dict = requester.serialize(py)?; + let dict = dict.as_any(); + + assert_eq!( + dict.get_item("user_id")?.extract::()?, + "@alice:example.com" + ); + assert_eq!(dict.get_item("access_token_id")?.extract::()?, 42); + assert!(!dict.get_item("is_guest")?.extract::()?); + assert_eq!( + dict.get_item("scope")?.extract::>()?, + vec!["urn:matrix:client:api:*".to_string()] + ); + assert!(!dict.get_item("shadow_banned")?.extract::()?); + assert_eq!( + dict.get_item("device_id")?.extract::>()?, + Some("ABCDEFG".to_string()) + ); + assert_eq!( + dict.get_item("authenticated_entity")?.extract::()?, + "@alice:example.com" + ); + + // The `app_service_id` field is serialized under the wire key + // "app_server_id" (server, not service), and there must be no + // "app_service_id" key. + assert!(dict.get_item("app_server_id")?.is_none()); + assert!(!dict.contains("app_service_id")?); + + Ok(()) + }) + .unwrap(); + } + + #[test] + fn test_serialize_deserialize_round_trip() { + Python::initialize(); + Python::attach(|py| { + // Use a requester that exercises the optional fields and the + // `app_service_id` -> "app_server_id" wire-key mapping. + let requester = Requester { + user_id: "@bob:example.com".to_string(), + access_token_id: None, + is_guest: true, + scope: HashSet::from(["a".to_string(), "b".to_string()]), + shadow_banned: true, + device_id: None, + app_service_id: Some("my_appservice".to_string()), + authenticated_entity: "@admin:example.com".to_string(), + }; + + let dict = requester.serialize(py).unwrap(); + let deserialized = Requester::deserialize(py, dict.as_any()).unwrap(); + let deserialized = Bound::new(py, deserialized).unwrap(); + + assert_eq!(&requester, deserialized.get()); + }); + } + + #[test] + fn test_deserialize_defaults_scope_when_missing() { + Python::initialize(); + Python::attach(|py| { + // An older serialized form may omit "scope"; it should default to + // an empty set rather than erroring. + let dict = PyDict::new(py); + dict.set_item("user_id", "@alice:example.com").unwrap(); + dict.set_item("access_token_id", py.None()).unwrap(); + dict.set_item("is_guest", false).unwrap(); + dict.set_item("shadow_banned", false).unwrap(); + dict.set_item("device_id", py.None()).unwrap(); + dict.set_item("app_server_id", py.None()).unwrap(); + dict.set_item("authenticated_entity", "@alice:example.com") + .unwrap(); + + let requester = Requester::deserialize(py, dict.as_any()).unwrap(); + assert!(requester.scope.is_empty()); + }); + } + + #[test] + fn test_deserialize_coerces_truthy_bools() { + Python::initialize(); + Python::attach(|py| { + // SQLite returns 0/1 for booleans, so non-bool truthy values must + // be coerced for `is_guest` and `shadow_banned`. + let dict = PyDict::new(py); + dict.set_item("user_id", "@alice:example.com").unwrap(); + dict.set_item("access_token_id", py.None()).unwrap(); + dict.set_item("is_guest", 1).unwrap(); + dict.set_item("scope", PyList::empty(py)).unwrap(); + dict.set_item("shadow_banned", 0).unwrap(); + dict.set_item("device_id", py.None()).unwrap(); + dict.set_item("app_server_id", py.None()).unwrap(); + dict.set_item("authenticated_entity", "@alice:example.com") + .unwrap(); + + let requester = Requester::deserialize(py, dict.as_any()).unwrap(); + assert!(requester.is_guest); + assert!(!requester.shadow_banned); + }); + } +} diff --git a/synapse/api/auth/base.py b/synapse/api/auth/base.py index ff876b9d222..14e76b0cffe 100644 --- a/synapse/api/auth/base.py +++ b/synapse/api/auth/base.py @@ -371,7 +371,9 @@ async def _record_request( """ ip_addr = request.get_client_ip_if_available() - if ip_addr and (not requester.app_service or self._track_appservice_user_ips): + if ip_addr and ( + not requester.app_service_id or self._track_appservice_user_ips + ): user_agent = get_request_user_agent(request) access_token = self.get_access_token_from_request(request) @@ -381,7 +383,7 @@ async def _record_request( # table during the transition recorded_device_id = ( "dummy-device" - if requester.device_id is None and requester.app_service is not None + if requester.device_id is None and requester.app_service_id is not None else requester.device_id ) await self.store.insert_client_ip( diff --git a/synapse/api/auth/internal.py b/synapse/api/auth/internal.py index b33384c13fa..28c0491140a 100644 --- a/synapse/api/auth/internal.py +++ b/synapse/api/auth/internal.py @@ -106,8 +106,8 @@ async def get_user_by_req( parent_span.set_tag("user_id", requester.user.to_string()) if requester.device_id is not None: parent_span.set_tag("device_id", requester.device_id) - if requester.app_service is not None: - parent_span.set_tag("appservice_id", requester.app_service.id) + if requester.app_service_id is not None: + parent_span.set_tag("appservice_id", requester.app_service_id) return requester async def get_user_by_req_experimental_feature( diff --git a/synapse/api/auth/mas.py b/synapse/api/auth/mas.py index 95c6d62a9d7..ed0427d6f3f 100644 --- a/synapse/api/auth/mas.py +++ b/synapse/api/auth/mas.py @@ -311,8 +311,8 @@ async def get_user_by_req( parent_span.set_tag("user_id", requester.user.to_string()) if requester.device_id is not None: parent_span.set_tag("device_id", requester.device_id) - if requester.app_service is not None: - parent_span.set_tag("appservice_id", requester.app_service.id) + if requester.app_service_id is not None: + parent_span.set_tag("appservice_id", requester.app_service_id) return requester async def get_user_by_access_token( diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index 27ab4af8056..3b37f398755 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -420,8 +420,8 @@ async def get_user_by_req( parent_span.set_tag("user_id", requester.user.to_string()) if requester.device_id is not None: parent_span.set_tag("device_id", requester.device_id) - if requester.app_service is not None: - parent_span.set_tag("appservice_id", requester.app_service.id) + if requester.app_service_id is not None: + parent_span.set_tag("appservice_id", requester.app_service_id) return requester async def _wrapped_get_user_by_req( diff --git a/synapse/api/auth_blocking.py b/synapse/api/auth_blocking.py index 3ed47b20c49..87918e15dca 100644 --- a/synapse/api/auth_blocking.py +++ b/synapse/api/auth_blocking.py @@ -88,7 +88,7 @@ async def check_auth_blocking( # We never block the server from doing actions on behalf of # users. return - if requester.app_service and not self._track_appservice_user_ips: + if requester.app_service_id and not self._track_appservice_user_ips: # If we're authenticated as an appservice then we only block # auth if `track_appservice_user_ips` is set, as that option # implicitly means that application services are part of MAU diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py index d6cc3d26b5f..b17fcd1ef33 100644 --- a/synapse/api/ratelimiting.py +++ b/synapse/api/ratelimiting.py @@ -163,7 +163,12 @@ async def can_do_action( if requester: # Disable rate limiting of users belonging to any AS that is configured # not to be rate limited in its registration file (rate_limited: true|false). - if requester.app_service and not requester.app_service.is_rate_limited(): + app_service = ( + self.store.get_app_service_by_id(requester.app_service_id) + if requester.app_service_id + else None + ) + if app_service and not app_service.is_rate_limited(): return True, -1.0 # Check if ratelimiting has been disabled for the user. diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 54f662796b5..8ce795052a5 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -419,7 +419,7 @@ def _serialize_event( and event_token_id == config.requester.access_token_id ) or config.requester.is_guest - or config.requester.app_service + or config.requester.app_service_id ): d["unsigned"]["transaction_id"] = txn_id diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index 0b1251f9e3e..ada799d96aa 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -427,7 +427,7 @@ async def _redact_all_events( r = task.params.get("requester") assert r is not None - admin = Requester.deserialize(self._store, r) + admin = Requester.deserialize(r) user_id = task.params.get("user_id") assert user_id is not None diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index bdb7cba64bd..91e7854f39a 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -132,7 +132,11 @@ async def create_association( Codes.INVALID_PARAM, ) - service = requester.app_service + service = ( + self.store.get_app_service_by_id(requester.app_service_id) + if requester.app_service_id + else None + ) if service: if not service.is_room_alias_in_namespace(room_alias_str): raise SynapseError( diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index bade219ddf9..ed04547d427 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -343,7 +343,7 @@ async def get_joined_members(self, requester: Requester, room_id: str) -> dict: Returns: A dict of user_id to profile info """ - if not requester.app_service: + if not requester.app_service_id: # We check AS auth after fetching the room membership, as it # requires us to pull out all joined members anyway. membership, _ = await self.auth.check_user_in_room_or_world_readable( @@ -365,12 +365,14 @@ async def get_joined_members(self, requester: Requester, room_id: str) -> dict: # If this is an AS, double check that they are allowed to see the members. # This can either be because the AS user is in the room or because there # is a user in the room that the AS is "interested in" - if ( - requester.app_service - and requester.user.to_string() not in users_with_profile - ): + app_service = ( + self.store.get_app_service_by_id(requester.app_service_id) + if requester.app_service_id + else None + ) + if app_service and requester.user.to_string() not in users_with_profile: for uid in users_with_profile: - if requester.app_service.is_interested_in_user(uid): + if app_service.is_interested_in_user(uid): break else: # Loop fell through, AS has no interested users in room @@ -846,7 +848,7 @@ async def assert_accepted_privacy_policy(self, requester: Requester) -> None: return # exempt AS users from needing consent - if requester.app_service is not None: + if requester.app_service_id is not None: return user_id = requester.authenticated_entity @@ -1425,8 +1427,10 @@ async def create_new_client_event( else: context = await self.state.calculate_context_info(event) - if requester: - context.app_service = requester.app_service + if requester and requester.app_service_id: + context.app_service = self.store.get_app_service_by_id( + requester.app_service_id + ) res, new_content = await self._third_party_event_rules.check_event_allowed( event, context diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 8a19dba5ee5..5152d0b522e 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -661,8 +661,8 @@ async def update_membership( key = (room_id,) as_id = object() - if requester.app_service: - as_id = requester.app_service.id + if requester.app_service_id: + as_id = requester.app_service_id # We first linearise by the application service (to try to limit concurrent joins # by application services), and then by room ID. diff --git a/synapse/replication/http/membership.py b/synapse/replication/http/membership.py index 8a6c9717201..80dd9b7da72 100644 --- a/synapse/replication/http/membership.py +++ b/synapse/replication/http/membership.py @@ -89,7 +89,7 @@ async def _handle_request( # type: ignore[override] remote_room_hosts = content["remote_room_hosts"] event_content = content["content"] - requester = Requester.deserialize(self.store, content["requester"]) + requester = Requester.deserialize(content["requester"]) request.requester = requester logger.info("remote_join: %s into room: %s", user_id, room_id) @@ -153,7 +153,7 @@ async def _handle_request( # type: ignore[override] remote_room_hosts = content["remote_room_hosts"] event_content = content["content"] - requester = Requester.deserialize(self.store, content["requester"]) + requester = Requester.deserialize(content["requester"]) request.requester = requester logger.debug("remote_knock: %s on room: %s", user_id, room_id) @@ -219,7 +219,7 @@ async def _handle_request( # type: ignore[override] txn_id = content["txn_id"] event_content = content["content"] - requester = Requester.deserialize(self.store, content["requester"]) + requester = Requester.deserialize(content["requester"]) request.requester = requester # hopefully we're now on the master, so this won't recurse! @@ -283,7 +283,7 @@ async def _handle_request( # type: ignore[override] txn_id = content["txn_id"] event_content = content["content"] - requester = Requester.deserialize(self.store, content["requester"]) + requester = Requester.deserialize(content["requester"]) request.requester = requester # hopefully we're now on the master, so this won't recurse! diff --git a/synapse/replication/http/send_events.py b/synapse/replication/http/send_events.py index b020a0fe7c6..e0cfb0d17cf 100644 --- a/synapse/replication/http/send_events.py +++ b/synapse/replication/http/send_events.py @@ -141,9 +141,7 @@ async def _handle_request( # type: ignore[override] ) event.internal_metadata.outlier = event_payload["outlier"] - requester = Requester.deserialize( - self.store, event_payload["requester"] - ) + requester = Requester.deserialize(event_payload["requester"]) context = EventContext.deserialize( self._storage_controllers, event_payload["context"] ) diff --git a/synapse/rest/client/account.py b/synapse/rest/client/account.py index 3cb1e09f44c..d1e404f0dc9 100644 --- a/synapse/rest/client/account.py +++ b/synapse/rest/client/account.py @@ -305,7 +305,7 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: # allow ASes to deactivate their own users: # ASes don't need user-interactive auth - if not requester.app_service: + if not requester.app_service_id: await self.auth_handler.validate_user_via_ui_auth( requester, request, diff --git a/synapse/rest/client/appservice_ping.py b/synapse/rest/client/appservice_ping.py index 7e2ac15783a..2c6ad5bcf0d 100644 --- a/synapse/rest/client/appservice_ping.py +++ b/synapse/rest/client/appservice_ping.py @@ -55,25 +55,32 @@ def __init__(self, hs: "HomeServer"): self.as_api = hs.get_application_service_api() self.scheduler = hs.get_application_service_scheduler() self.auth = hs.get_auth() + self.store = hs.get_datastores().main async def on_POST( self, request: SynapseRequest, appservice_id: str ) -> tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request) - if not requester.app_service: + app_service = ( + self.store.get_app_service_by_id(requester.app_service_id) + if requester.app_service_id + else None + ) + + if not app_service: raise SynapseError( HTTPStatus.FORBIDDEN, "Only application services can use the /appservice/ping endpoint", Codes.FORBIDDEN, ) - elif requester.app_service.id != appservice_id: + elif app_service.id != appservice_id: raise SynapseError( HTTPStatus.FORBIDDEN, "Mismatching application service ID in path", Codes.FORBIDDEN, ) - elif not requester.app_service.url: + elif not app_service.url: raise SynapseError( HTTPStatus.BAD_REQUEST, "The application service does not have a URL set", @@ -85,11 +92,11 @@ async def on_POST( start = time.monotonic() try: - await self.as_api.ping(requester.app_service, txn_id) + await self.as_api.ping(app_service, txn_id) # We got a OK response, so if the AS needs to be recovered then lets recover it now. # This sets off a task in the background and so is safe to execute and forget. - self.scheduler.txn_ctrl.force_retry(requester.app_service) + self.scheduler.txn_ctrl.force_retry(app_service) except RequestTimedOutError as e: raise SynapseError( HTTPStatus.GATEWAY_TIMEOUT, diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index 4b84131d326..0231ed374d7 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -105,7 +105,7 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: else: raise e - if requester.app_service: + if requester.app_service_id: # MSC4190 can skip UIA for this endpoint pass else: @@ -177,7 +177,7 @@ async def on_DELETE( else: raise - if requester.app_service: + if requester.app_service_id: # MSC4190 allows appservices to delete devices through this endpoint without UIA # It's also allowed with MSC3861 enabled pass @@ -212,7 +212,7 @@ async def on_PUT( body = parse_and_validate_json_object_from_request(request, self.PutBody) # MSC4190 allows appservices to create devices through this endpoint - if requester.app_service: + if requester.app_service_id: created = await self.device_handler.upsert_device( user_id=requester.user.to_string(), device_id=device_id, diff --git a/synapse/rest/client/directory.py b/synapse/rest/client/directory.py index 0b334f9b0bd..04941b25f33 100644 --- a/synapse/rest/client/directory.py +++ b/synapse/rest/client/directory.py @@ -110,14 +110,19 @@ async def on_DELETE( room_alias_obj = RoomAlias.from_string(room_alias) requester = await self.auth.get_user_by_req(request) - if requester.app_service: + app_service = ( + self.store.get_app_service_by_id(requester.app_service_id) + if requester.app_service_id + else None + ) + if app_service: await self.directory_handler.delete_appservice_association( - requester.app_service, room_alias_obj + app_service, room_alias_obj ) logger.info( "Application service at %s deleted alias %s", - requester.app_service.url, + app_service.url, room_alias_obj.to_string(), ) @@ -199,13 +204,13 @@ async def _edit( visibility: Literal["public", "private"], ) -> tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request) - if not requester.app_service: + if not requester.app_service_id: raise AuthError( 403, "Only appservices can edit the appservice published room list" ) await self.directory_handler.edit_published_appservice_room_list( - requester.app_service.id, network_id, room_id, visibility + requester.app_service_id, network_id, room_id, visibility ) return 200, {} diff --git a/synapse/rest/client/keys.py b/synapse/rest/client/keys.py index 2c65a55ea17..463c87d92b1 100644 --- a/synapse/rest/client/keys.py +++ b/synapse/rest/client/keys.py @@ -535,7 +535,7 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: # setup, and that is allowed without UIA, per MSC3967. # If yes, then we need to authenticate the change. # MSC4190 can skip UIA for replacing cross-signing keys as well. - if is_cross_signing_setup and not requester.app_service: + if is_cross_signing_setup and not requester.app_service_id: # With MSC3861, UIA is not possible. Instead, the auth service has to # explicitly mark the master key as replaceable. if self.hs.config.mas.enabled: diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index a49b27d009f..cfdc97b6645 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -203,7 +203,11 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, LoginResponse]: try: if login_submission["type"] == LoginRestServlet.APPSERVICE_TYPE: requester = await self.auth.get_user_by_req(request) - appservice = requester.app_service + appservice = ( + self._main_store.get_app_service_by_id(requester.app_service_id) + if requester.app_service_id + else None + ) if appservice is None: raise InvalidClientTokenError( diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 48ef42c7d6f..6e00197b6de 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -336,7 +336,7 @@ async def on_PUT( ) origin_server_ts = None - if requester.app_service: + if requester.app_service_id: origin_server_ts = parse_integer(request, "ts") sticky_duration_ms: int | None = None @@ -435,7 +435,7 @@ async def _do( content = parse_json_object_from_request(request) origin_server_ts = None - if requester.app_service: + if requester.app_service_id: origin_server_ts = parse_integer(request, "ts") sticky_duration_ms: int | None = None diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py index 43c7b6f9938..0046b31b490 100644 --- a/synapse/rest/client/transactions.py +++ b/synapse/rest/client/transactions.py @@ -82,8 +82,8 @@ def _get_transaction_key(self, request: IRequest, requester: Requester) -> Hasha assert requester.user is not None, "Guest requester must have a user ID set" return (path, "guest", requester.user) - elif requester.app_service is not None: - return (path, "appservice", requester.app_service.id) + elif requester.app_service_id is not None: + return (path, "appservice", requester.app_service_id) # Use the user ID and device ID as the transaction key. elif requester.device_id: diff --git a/synapse/rest/media/create_resource.py b/synapse/rest/media/create_resource.py index 8bef1f4c080..1b6b001b450 100644 --- a/synapse/rest/media/create_resource.py +++ b/synapse/rest/media/create_resource.py @@ -43,6 +43,7 @@ def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"): super().__init__() self.media_repo = media_repo + self.store = hs.get_datastores().main self.clock = hs.get_clock() self.auth = hs.get_auth() self.max_pending_media_uploads = hs.config.media.max_pending_media_uploads @@ -60,7 +61,12 @@ async def on_POST(self, request: SynapseRequest) -> None: # If the create media requests for the user are over the limit, drop them. await self._create_media_rate_limiter.ratelimit(requester) - if not requester.app_service or requester.app_service.is_rate_limited(): + app_service = ( + self.store.get_app_service_by_id(requester.app_service_id) + if requester.app_service_id + else None + ) + if not app_service or app_service.is_rate_limited(): ( reached_pending_limit, first_expiration_ts, diff --git a/synapse/synapse_rust/types.pyi b/synapse/synapse_rust/types.pyi new file mode 100644 index 00000000000..36a030fb57a --- /dev/null +++ b/synapse/synapse_rust/types.pyi @@ -0,0 +1,68 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from synapse.types import JsonDict, UserID + +class Requester: + def __init__( + self, + user: UserID, + access_token_id: int | None, + is_guest: bool, + scope: set[str], + shadow_banned: bool, + device_id: str | None, + app_service_id: str | None, + authenticated_entity: str, + ) -> None: ... + @property + def user_id(self) -> str: + """The ID of the user making the request, in string form (see `user` + for the parsed UserID)""" + + @property + def user(self) -> UserID: + """The user making the request""" + @property + def access_token_id(self) -> int | None: + """The ID of the access token used for this request, or + None for appservices, guests, and tokens generated by the admin API""" + @property + def is_guest(self) -> bool: + """True if the user making this request is a guest user""" + @property + def scope(self) -> set[str]: + """Any scopes associated with the access token used for this request, or + an empty set if no token or a non-oauth token was used""" + @property + def shadow_banned(self) -> bool: + """True if the user making this request has been shadow-banned.""" + @property + def device_id(self) -> str | None: + """The device_id which was set at authentication time, or + None for appservices, guests, and tokens generated by the admin API""" + @property + def app_service_id(self) -> str | None: + """The ID of the AS requesting on behalf of the user, or None.""" + @property + def authenticated_entity(self) -> str: + """The entity that authenticated when making the request. + + This is different to the user_id when an admin user or the server is + "puppeting" the user.""" + def serialize(self) -> JsonDict: + """Converts self to a type that can be serialized as JSON, and then + deserialized by `deserialize`""" + @staticmethod + def deserialize(input: JsonDict) -> Requester: + """Converts a dict that was produced by `serialize` back into a + Requester.""" diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 8537a63bdec..f2b1c3e42ba 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -61,6 +61,7 @@ ) from synapse.api.errors import Codes, SynapseError +from synapse.synapse_rust.types import Requester from synapse.util.cancellation import cancellable from synapse.util.stringutils import parse_and_validate_server_name @@ -70,7 +71,6 @@ from synapse.appservice.api import ApplicationService from synapse.events import EventBase from synapse.storage.databases.main import DataStore, PurgeEventsStore - from synapse.storage.databases.main.appservice import ApplicationServiceWorkerStore from synapse.storage.util.id_generators import MultiWriterIdGenerator @@ -138,82 +138,6 @@ class ISynapseReactor( """The interfaces necessary for Synapse to function.""" -@attr.s(frozen=True, slots=True, auto_attribs=True) -class Requester: - """ - Represents the user making a request - - Attributes: - user: id of the user making the request - access_token_id: *ID* of the access token used for this request, or - None for appservices, guests, and tokens generated by the admin API - is_guest: True if the user making this request is a guest user - shadow_banned: True if the user making this request has been shadow-banned. - device_id: device_id which was set at authentication time, or - None for appservices, guests, and tokens generated by the admin API - app_service: the AS requesting on behalf of the user - authenticated_entity: The entity that authenticated when making the request. - This is different to the user_id when an admin user or the server is - "puppeting" the user. - """ - - user: "UserID" - access_token_id: int | None - is_guest: bool - scope: set[str] - shadow_banned: bool - device_id: str | None - app_service: Optional["ApplicationService"] - authenticated_entity: str - - def serialize(self) -> dict[str, Any]: - """Converts self to a type that can be serialized as JSON, and then - deserialized by `deserialize` - - Returns: - dict - """ - return { - "user_id": self.user.to_string(), - "access_token_id": self.access_token_id, - "is_guest": self.is_guest, - "scope": list(self.scope), - "shadow_banned": self.shadow_banned, - "device_id": self.device_id, - "app_server_id": self.app_service.id if self.app_service else None, - "authenticated_entity": self.authenticated_entity, - } - - @staticmethod - def deserialize( - store: "ApplicationServiceWorkerStore", input: dict[str, Any] - ) -> "Requester": - """Converts a dict that was produced by `serialize` back into a - Requester. - - Args: - store: Used to convert AS ID to AS object - input: A dict produced by `serialize` - - Returns: - Requester - """ - appservice = None - if input["app_server_id"]: - appservice = store.get_app_service_by_id(input["app_server_id"]) - - return Requester( - user=UserID.from_string(input["user_id"]), - access_token_id=input["access_token_id"], - is_guest=input["is_guest"], - scope=set(input.get("scope", [])), - shadow_banned=input["shadow_banned"], - device_id=input["device_id"], - app_service=appservice, - authenticated_entity=input["authenticated_entity"], - ) - - def create_requester( user_id: Union[str, "UserID"], access_token_id: int | None = None, @@ -258,7 +182,7 @@ def create_requester( scope, shadow_banned, device_id, - app_service, + app_service.id if app_service else None, authenticated_entity, ) diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index f7905ced7e8..e7ee7eec91a 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -108,6 +108,7 @@ def test_get_user_by_req_user_missing_token(self) -> None: def test_get_user_by_req_appservice_valid_token(self) -> None: app_service = Mock( + id="as_id", token="foobar", url="a_url", sender=self.test_user_id, @@ -132,6 +133,7 @@ def test_get_user_by_req_appservice_valid_token_good_ip(self) -> None: sender=self.test_user_id.to_string(), ip_range_whitelist=IPSet(["192.168.0.0/16"]), ) + app_service.id = "as_id" self.store.get_app_service_by_token = Mock(return_value=app_service) self.store.get_user_by_access_token = AsyncMock(return_value=None) @@ -151,6 +153,7 @@ def test_get_user_by_req_appservice_valid_token_bad_ip(self) -> None: sender=self.test_user_id, ip_range_whitelist=IPSet(["192.168.0.0/16"]), ) + app_service.id = "as_id" self.store.get_app_service_by_token = Mock(return_value=app_service) self.store.get_user_by_access_token = AsyncMock(return_value=None) @@ -179,6 +182,7 @@ def test_get_user_by_req_appservice_bad_token(self) -> None: def test_get_user_by_req_appservice_missing_token(self) -> None: app_service = Mock(token="foobar", url="a_url", sender=self.test_user_id) + app_service.id = "as_id" self.store.get_app_service_by_token = Mock(return_value=app_service) self.store.get_user_by_access_token = AsyncMock(return_value=None) @@ -199,6 +203,7 @@ def test_get_user_by_req_appservice_valid_token_valid_user_id(self) -> None: ip_range_whitelist=None, ) app_service.is_interested_in_user = Mock(return_value=True) + app_service.id = "as_id" self.store.get_app_service_by_token = Mock(return_value=app_service) class FakeUserInfo: @@ -226,6 +231,7 @@ def test_get_user_by_req_appservice_valid_token_bad_user_id(self) -> None: ip_range_whitelist=None, ) app_service.is_interested_in_user = Mock(return_value=False) + app_service.id = "as_id" self.store.get_app_service_by_token = Mock(return_value=app_service) self.store.get_user_by_access_token = AsyncMock(return_value=None) @@ -251,6 +257,7 @@ def test_get_user_by_req_appservice_valid_token_valid_device_id(self) -> None: ip_range_whitelist=None, ) app_service.is_interested_in_user = Mock(return_value=True) + app_service.id = "as_id" self.store.get_app_service_by_token = Mock(return_value=app_service) # This just needs to return a truth-y value. self.store.get_user_by_id = AsyncMock(return_value={"is_guest": False}) @@ -285,6 +292,7 @@ def test_get_user_by_req_appservice_valid_token_invalid_device_id(self) -> None: ip_range_whitelist=None, ) app_service.is_interested_in_user = Mock(return_value=True) + app_service.id = "as_id" self.store.get_app_service_by_token = Mock(return_value=app_service) # This just needs to return a truth-y value. self.store.get_user_by_id = AsyncMock(return_value={"is_guest": False}) @@ -457,7 +465,7 @@ def test_blocking_mau__appservice_requester_allowed_when_not_tracking_ips( is_guest=False, scope=set(), shadow_banned=False, - app_service=appservice, + app_service_id=appservice.id, authenticated_entity="@appservice:server", ) self.get_success(self.auth_blocking.check_auth_blocking(requester=requester)) @@ -488,7 +496,7 @@ def test_blocking_mau__appservice_requester_disallowed_when_tracking_ips( is_guest=False, scope=set(), shadow_banned=False, - app_service=appservice, + app_service_id=appservice.id, authenticated_entity="@appservice:server", ) self.get_failure( diff --git a/tests/api/test_ratelimiting.py b/tests/api/test_ratelimiting.py index 0ef537841df..3237c6d4122 100644 --- a/tests/api/test_ratelimiting.py +++ b/tests/api/test_ratelimiting.py @@ -40,6 +40,9 @@ def test_allowed_appservice_ratelimited_via_can_requester_do_action(self) -> Non rate_limited=True, sender=UserID.from_string("@as:example.com"), ) + # The ratelimiter now resolves the AS via get_app_service_by_id, so the + # appservice must be in the store's cache for the lookup to hit. + self.hs.get_datastores().main.services_cache.append(appservice) as_requester = create_requester("@user:example.com", app_service=appservice) limiter = Ratelimiter( @@ -76,6 +79,9 @@ def test_allowed_appservice_via_can_requester_do_action(self) -> None: rate_limited=False, sender=UserID.from_string("@as:example.com"), ) + # The ratelimiter now resolves the AS via get_app_service_by_id, so the + # appservice must be in the store's cache for the lookup to hit. + self.hs.get_datastores().main.services_cache.append(appservice) as_requester = create_requester("@user:example.com", app_service=appservice) limiter = Ratelimiter( diff --git a/tests/rest/client/test_transactions.py b/tests/rest/client/test_transactions.py index 31586a451ff..d62f9dd4a52 100644 --- a/tests/rest/client/test_transactions.py +++ b/tests/rest/client/test_transactions.py @@ -52,7 +52,7 @@ def setUp(self) -> None: self.mock_request = Mock() self.mock_request.path = b"/foo/bar" self.mock_requester = Mock() - self.mock_requester.app_service = None + self.mock_requester.app_service_id = None self.mock_requester.is_guest = False self.mock_requester.access_token_id = 1234 From edadf5768d9a39887cce99c95783fa6a26fd5439 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 9 Jun 2026 14:57:14 +0100 Subject: [PATCH 87/92] Fix the `/capabilities` endpoint returning a 500 error on non-media workers when MSC4452: Preview URL capabilities API is enabled. (#19839) Fixes: #19825 Introduced in: #19715 Always populate `url_preview_enabled` so `/capabilities` can expose it Needed so this line can be happy: https://github.com/element-hq/synapse/blob/106ed3623d434891fe1ac50aacc851e9804404fe/synapse/rest/client/capabilities.py#L82 --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19839.bugfix | 1 + synapse/config/repository.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19839.bugfix diff --git a/changelog.d/19839.bugfix b/changelog.d/19839.bugfix new file mode 100644 index 00000000000..74606ce6f17 --- /dev/null +++ b/changelog.d/19839.bugfix @@ -0,0 +1 @@ +Fix the `/capabilities` endpoint returning a 500 error on non-media workers when [MSC4452: Preview URL capabilities API](https://github.com/matrix-org/matrix-spec-proposals/pull/4452) is enabled. \ No newline at end of file diff --git a/synapse/config/repository.py b/synapse/config/repository.py index cb50d0dc1de..373e518ddc9 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -139,6 +139,10 @@ class ContentRepositoryConfig(Config): section = "media" def read_config(self, config: JsonDict, **kwargs: Any) -> None: + # We need to set this configuration flag even if this worker + # is not a media repo worker, as it's exposed in `/capabilities` + self.url_preview_enabled = bool(config.get("url_preview_enabled", False)) + # Only enable the media repo if either the media repo is enabled or the # current worker app is the media repo. if ( @@ -242,7 +246,6 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.thumbnail_requirements = parse_thumbnail_requirements( config.get("thumbnail_sizes", DEFAULT_THUMBNAIL_SIZES) ) - self.url_preview_enabled = bool(config.get("url_preview_enabled", False)) if self.url_preview_enabled: check_requirements("url-preview") From f53f1044494c6bd002889d29df35cf2f168860c3 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 9 Jun 2026 14:59:48 +0100 Subject: [PATCH 88/92] 1.155.0rc1 --- CHANGES.md | 22 ++++++++++++++++++++++ changelog.d/19617.bugfix | 1 - changelog.d/19701.misc | 1 - changelog.d/19730.bugfix | 1 - changelog.d/19734.bugfix | 1 - changelog.d/19741.misc | 1 - changelog.d/19775.misc | 1 - changelog.d/19792.bugfix | 1 - changelog.d/19801.misc | 1 - changelog.d/19816.misc | 1 - changelog.d/19817.misc | 1 - changelog.d/19818.doc | 1 - changelog.d/19819.misc | 1 - changelog.d/19821.misc | 1 - changelog.d/19823.doc | 1 - changelog.d/19828.misc | 1 - changelog.d/19839.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- schema/synapse-config.schema.yaml | 2 +- 20 files changed, 30 insertions(+), 18 deletions(-) delete mode 100644 changelog.d/19617.bugfix delete mode 100644 changelog.d/19701.misc delete mode 100644 changelog.d/19730.bugfix delete mode 100644 changelog.d/19734.bugfix delete mode 100644 changelog.d/19741.misc delete mode 100644 changelog.d/19775.misc delete mode 100644 changelog.d/19792.bugfix delete mode 100644 changelog.d/19801.misc delete mode 100644 changelog.d/19816.misc delete mode 100644 changelog.d/19817.misc delete mode 100644 changelog.d/19818.doc delete mode 100644 changelog.d/19819.misc delete mode 100644 changelog.d/19821.misc delete mode 100644 changelog.d/19823.doc delete mode 100644 changelog.d/19828.misc delete mode 100644 changelog.d/19839.bugfix diff --git a/CHANGES.md b/CHANGES.md index e210f8f1d52..13e25709cd8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,25 @@ +# Synapse 1.155.0rc1 (2026-06-09) + +## Bugfixes + +- Limit the to-device EDU size to a reasonable value to mitigate long queues of to-device messages preventing outgoing federation because of the size of the transaction. ([\#19617](https://github.com/element-hq/synapse/issues/19617)) +- Work around bug that sometimes breaks joining restricted rooms that require a remote join. Contributed by @tulir @ Beeper. ([\#19730](https://github.com/element-hq/synapse/issues/19730)) +- Update Sliding Sync to return a new response immediately if a room subscription has changed and produced a new response. ([\#19734](https://github.com/element-hq/synapse/issues/19734), [\#19792](https://github.com/element-hq/synapse/issues/19792)) +- Fix the `/capabilities` endpoint returning a 500 error on non-media workers when [MSC4452: Preview URL capabilities API](https://github.com/matrix-org/matrix-spec-proposals/pull/4452) is enabled. ([\#19839](https://github.com/element-hq/synapse/issues/19839)) + +## Improved Documentation + +- Document how to see Rust build failure output when using `poetry install`. ([\#19818](https://github.com/element-hq/synapse/issues/19818)) +- Document that the SQLite version included in Ubuntu LTS, aside from ESM-only versions, is included in our support policy. ([\#19823](https://github.com/element-hq/synapse/issues/19823)) + +## Internal Changes + +- Port the Python Event classes to Rust. ([\#19701](https://github.com/element-hq/synapse/issues/19701), [\#19816](https://github.com/element-hq/synapse/issues/19816), [\#19817](https://github.com/element-hq/synapse/issues/19817), [\#19819](https://github.com/element-hq/synapse/issues/19819)) +- Added tests to ensure that email notification links are sanitized. Contributed by Noah Markert. ([\#19741](https://github.com/element-hq/synapse/issues/19741)) +- Add `GcpJsonFormatter` logging formatter for use with Google Cloud Logging and GKE deployments. ([\#19775](https://github.com/element-hq/synapse/issues/19775)) +- Add more logging to the to-device message replication stream. ([\#19801](https://github.com/element-hq/synapse/issues/19801), [\#19821](https://github.com/element-hq/synapse/issues/19821)) +- Port `Requester` class to Rust. ([\#19828](https://github.com/element-hq/synapse/issues/19828)) + # Synapse 1.154.0 (2026-06-04) No significant changes since 1.154.0rc1. diff --git a/changelog.d/19617.bugfix b/changelog.d/19617.bugfix deleted file mode 100644 index 71f181fed40..00000000000 --- a/changelog.d/19617.bugfix +++ /dev/null @@ -1 +0,0 @@ -A long queue of to-device messages could prevent outgoing federation because of the size of the transaction, let's limit the to-device EDU size to a reasonable value. diff --git a/changelog.d/19701.misc b/changelog.d/19701.misc deleted file mode 100644 index 4663e8b9611..00000000000 --- a/changelog.d/19701.misc +++ /dev/null @@ -1 +0,0 @@ -Port the python Event classes to Rust. diff --git a/changelog.d/19730.bugfix b/changelog.d/19730.bugfix deleted file mode 100644 index 074405d9442..00000000000 --- a/changelog.d/19730.bugfix +++ /dev/null @@ -1 +0,0 @@ -Work around bug that sometimes breaks joining restricted rooms that require a remote join. Contributed by @tulir @ Beeper. diff --git a/changelog.d/19734.bugfix b/changelog.d/19734.bugfix deleted file mode 100644 index 01af7d9ab83..00000000000 --- a/changelog.d/19734.bugfix +++ /dev/null @@ -1 +0,0 @@ -Update Sliding Sync to return a new response immediately if a room subscription have changed and produced a new response. diff --git a/changelog.d/19741.misc b/changelog.d/19741.misc deleted file mode 100644 index 55347570d8b..00000000000 --- a/changelog.d/19741.misc +++ /dev/null @@ -1 +0,0 @@ -Added tests to ensure that email notification links are sanitized. Contributed by Noah Markert. diff --git a/changelog.d/19775.misc b/changelog.d/19775.misc deleted file mode 100644 index 3ff0fd34b9c..00000000000 --- a/changelog.d/19775.misc +++ /dev/null @@ -1 +0,0 @@ -Add `GcpJsonFormatter` logging formatter for use with Google Cloud Logging and GKE deployments. \ No newline at end of file diff --git a/changelog.d/19792.bugfix b/changelog.d/19792.bugfix deleted file mode 100644 index 01af7d9ab83..00000000000 --- a/changelog.d/19792.bugfix +++ /dev/null @@ -1 +0,0 @@ -Update Sliding Sync to return a new response immediately if a room subscription have changed and produced a new response. diff --git a/changelog.d/19801.misc b/changelog.d/19801.misc deleted file mode 100644 index 56ec0f7ddca..00000000000 --- a/changelog.d/19801.misc +++ /dev/null @@ -1 +0,0 @@ -Add more logging to the to-device message replication stream. diff --git a/changelog.d/19816.misc b/changelog.d/19816.misc deleted file mode 100644 index 4663e8b9611..00000000000 --- a/changelog.d/19816.misc +++ /dev/null @@ -1 +0,0 @@ -Port the python Event classes to Rust. diff --git a/changelog.d/19817.misc b/changelog.d/19817.misc deleted file mode 100644 index 4663e8b9611..00000000000 --- a/changelog.d/19817.misc +++ /dev/null @@ -1 +0,0 @@ -Port the python Event classes to Rust. diff --git a/changelog.d/19818.doc b/changelog.d/19818.doc deleted file mode 100644 index c6aca252571..00000000000 --- a/changelog.d/19818.doc +++ /dev/null @@ -1 +0,0 @@ -Document how to see Rust build failure output when using `poetry install`. diff --git a/changelog.d/19819.misc b/changelog.d/19819.misc deleted file mode 100644 index 4663e8b9611..00000000000 --- a/changelog.d/19819.misc +++ /dev/null @@ -1 +0,0 @@ -Port the python Event classes to Rust. diff --git a/changelog.d/19821.misc b/changelog.d/19821.misc deleted file mode 100644 index 56ec0f7ddca..00000000000 --- a/changelog.d/19821.misc +++ /dev/null @@ -1 +0,0 @@ -Add more logging to the to-device message replication stream. diff --git a/changelog.d/19823.doc b/changelog.d/19823.doc deleted file mode 100644 index 5bbbdd5ccf2..00000000000 --- a/changelog.d/19823.doc +++ /dev/null @@ -1 +0,0 @@ -Document that the SQLite version included in Ubuntu LTS, aside from ESM-only versions, is included in our support policy. \ No newline at end of file diff --git a/changelog.d/19828.misc b/changelog.d/19828.misc deleted file mode 100644 index 1ed01eba2ea..00000000000 --- a/changelog.d/19828.misc +++ /dev/null @@ -1 +0,0 @@ -Port `Requester` class to Rust. diff --git a/changelog.d/19839.bugfix b/changelog.d/19839.bugfix deleted file mode 100644 index 74606ce6f17..00000000000 --- a/changelog.d/19839.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix the `/capabilities` endpoint returning a 500 error on non-media workers when [MSC4452: Preview URL capabilities API](https://github.com/matrix-org/matrix-spec-proposals/pull/4452) is enabled. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index 7750ca563c2..617bc736112 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.155.0~rc1) stable; urgency=medium + + * New Synapse release 1.155.0rc1. + + -- Synapse Packaging team Tue, 09 Jun 2026 14:58:03 +0100 + matrix-synapse-py3 (1.154.0) stable; urgency=medium * New Synapse release 1.154.0. diff --git a/pyproject.toml b/pyproject.toml index 89bace10a2d..3500b4f5167 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.154.0" +version = "1.155.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index dc57cfeea50..1a2caddbfda 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/schema/synapse/v1.154/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.155/synapse-config.schema.json type: object properties: modules: From ff034c1f623bc769f1f3bca801592dd42697ea66 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Jun 2026 17:33:21 +0100 Subject: [PATCH 89/92] Fix flakey test failing with "twisted.protocols.amp.TooLong" (#19832) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was all done through claude. ## Fix flaky `test_edu_large_messages_not_splitting_one_user` (`TooLong` under `trial -jN`) ### Problem The "Build .deb packages" CI step intermittently failed with: ``` twisted.protocols.amp.TooLong ...in tests.rest.client.test_sendtodevice.SendToDeviceTestCase.test_edu_large_messages_not_splitting_one_user ``` The deb build runs the suite with `twisted.trial -j2`. In that mode, worker log events are shipped to the manager process over Twisted's AMP protocol, which encodes each value with a 2-byte length prefix — so any single log line of **64 KiB or more** raises `TooLong`. ### Root cause Not a bug in the to-device/EDU logic, and **not** debug logging enabled by the build (it runs at the default `ERROR` level). It's a **log-level leak between tests sharing a `-j2` worker**: - `tests/logging/test_loggers.py::ExplicitlyConfiguredLoggerTestCase` calls `root_logger.setLevel(logging.DEBUG)` directly and never restores it (no `setUp`/`tearDown`/`addCleanup`). - When that test runs before `test_edu_large_messages_not_splitting_one_user` **in the same worker process**, the root logger is left at `DEBUG`. - That test deliberately builds an EDU of exactly `SOFT_MAX_EDU_SIZE - 1` (65 535) bytes. Storing it triggers `synapse/storage/database.py`'s `[SQL values]` DEBUG log, which dumps the full query params — producing a **65 708-byte** line that overflows AMP's cap. It looks "flaky" purely because of `-j2` scheduling: whether the two tests land on the same worker, and in what order. ### Fix Three commits: 1. **Restore the root logger level in `ExplicitlyConfiguredLoggerTestCase`** — `addCleanup` to put the level back. Fixes the root cause. 2. **Truncate oversized log lines in the test log handler** (`ToTwistedHandler.emit`) — caps lines at 1000 chars so no debug line can break `trial -jN`, regardless of which query is logged (defense in depth). 3. **Truncate values in `[SQL values]` debug logging** — caps the logged param repr at 1000 chars (guarded by `isEnabledFor(DEBUG)` to keep the hot path lazy). Keeps production debug logs sane too. ### Testing - The reproduction (`trial -j2 tests.logging.test_loggers `) went from **~3/8 failing** to **10/10 passing**. - Confirmed the root level is restored after the logging tests, and that the `[SQL values]` line is now capped (~50 KB with a `[truncated]` marker, was 65 708). - `ruff` + `mypy` clean; `tests.logging.test_loggers` and `tests.rest.client.test_sendtodevice` pass (14/14). --------- Co-authored-by: Claude Opus 4.8 Co-authored-by: Eric Eastwood --- changelog.d/19832.misc | 1 + synapse/storage/database.py | 15 ++++++++++++++- tests/logging/test_loggers.py | 13 +++++++++++++ tests/test_utils/logging_setup.py | 13 +++++++++++++ 4 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19832.misc diff --git a/changelog.d/19832.misc b/changelog.d/19832.misc new file mode 100644 index 00000000000..23c63fa10c9 --- /dev/null +++ b/changelog.d/19832.misc @@ -0,0 +1 @@ +Fix a flaky test (`twisted.protocols.amp.TooLong` error under `trial -jN`) caused by an oversized debug log line. diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 6e38b55686b..39c0631d50a 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -77,6 +77,13 @@ transaction_logger = logging.getLogger("synapse.storage.txn") perf_logger = logging.getLogger("synapse.storage.TIME") +# The maximum length of the repr of a query's values that we log at DEBUG. Some +# queries carry large payloads (e.g. to-device messages), and logging them in +# full is not useful. Truncating also keeps the line well under AMP's 64KiB +# per-value limit, which would otherwise break `trial -jN` test runs (c.f. +# https://github.com/twisted/twisted/issues/12482). +MAX_SQL_VALUE_LOG_LENGTH = 1000 + sql_scheduling_timer = Histogram( "synapse_storage_schedule_time", "sec", labelnames=[SERVER_NAME_LABEL] ) @@ -502,7 +509,13 @@ def _do_execute( sql = self.database_engine.convert_param_style(sql) if args: try: - sql_logger.debug("[SQL values] {%s} %r", self.name, args[0]) + if sql_logger.isEnabledFor(logging.DEBUG): + value_repr = repr(args[0]) + if len(value_repr) > MAX_SQL_VALUE_LOG_LENGTH: + value_repr = ( + value_repr[:MAX_SQL_VALUE_LOG_LENGTH] + "... [truncated]" + ) + sql_logger.debug("[SQL values] {%s} %s", self.name, value_repr) except Exception: # Don't let logging failures stop SQL from working pass diff --git a/tests/logging/test_loggers.py b/tests/logging/test_loggers.py index 9a9bf14376d..815746b2ddb 100644 --- a/tests/logging/test_loggers.py +++ b/tests/logging/test_loggers.py @@ -21,6 +21,19 @@ class ExplicitlyConfiguredLoggerTestCase(TestCase): + def setUp(self) -> None: + super().setUp() + + # These tests mutate the root logger's level (see the various + # `root_logger.setLevel(...)` calls below). Make sure we restore the + # original level afterwards, otherwise we leak a raised log level into + # the other tests running in the same worker process. A raised level can + # cause oversized debug log lines which break `trial -jN` (worker log + # events are shipped to the manager over trials's AMP, which caps each + # value at 64KiB, c.f. https://github.com/twisted/twisted/issues/12482). + root_logger = logging.getLogger() + self.addCleanup(root_logger.setLevel, root_logger.level) + def _create_explicitly_configured_logger(self) -> logging.Logger: original_logger_class = logging.getLoggerClass() logging.setLoggerClass(ExplicitlyConfiguredLogger) diff --git a/tests/test_utils/logging_setup.py b/tests/test_utils/logging_setup.py index d58222a9f6b..bfae36436f4 100644 --- a/tests/test_utils/logging_setup.py +++ b/tests/test_utils/logging_setup.py @@ -25,6 +25,14 @@ from synapse.logging.context import LoggingContextFilter from synapse.synapse_rust import reset_logging_config +# The maximum length of a single log line. Extremely long log lines aren't +# useful, and keeping them short also stays well under the limit that would +# otherwise break parallel test runs: under `trial -jN` worker log events are +# shipped to the manager process over trial's AMP, which raises `TooLong` for +# any value of 64KiB or more (e.g. a DEBUG dump of a large SQL query's values) +# (c.f. https://github.com/twisted/twisted/issues/12482). +_MAX_LOG_LINE_LENGTH = 60_000 + class ToTwistedHandler(logging.Handler): """logging handler which sends the logs to the twisted log""" @@ -33,6 +41,11 @@ class ToTwistedHandler(logging.Handler): def emit(self, record: logging.LogRecord) -> None: log_entry = self.format(record) + if len(log_entry) > _MAX_LOG_LINE_LENGTH: + log_entry = ( + log_entry[:_MAX_LOG_LINE_LENGTH] + + "... [truncated overly long log line]" + ) log_level = record.levelname.lower().replace("warning", "warn") self.tx_log.emit( twisted.logger.LogLevel.levelWithName(log_level), "{entry}", entry=log_entry From bfaca98e575dc14a7204fb32be23f5249ad4f942 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Thu, 11 Jun 2026 17:36:38 +0100 Subject: [PATCH 90/92] Fix admin API user endpoint documentation examples to use JSON booleans (#19847) I noticed that the documentation doesn't match reality. The values from the database are cast to bools at: - https://github.com/element-hq/synapse/blob/ff034c1f623bc769f1f3bca801592dd42697ea66/synapse/storage/databases/main/registration.py#L424-L439 - https://github.com/element-hq/synapse/blob/ff034c1f623bc769f1f3bca801592dd42697ea66/synapse/storage/databases/main/__init__.py#L330-L344 --- changelog.d/19847.doc | 1 + docs/admin_api/user_admin_api.md | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 12 deletions(-) create mode 100644 changelog.d/19847.doc diff --git a/changelog.d/19847.doc b/changelog.d/19847.doc new file mode 100644 index 00000000000..225b03d7b30 --- /dev/null +++ b/changelog.d/19847.doc @@ -0,0 +1 @@ +Fixed the Admin API user endpoint documentation examples to use JSON booleans (true/false) instead of numeric (0/1) values. diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index 14f86fa9768..9e2b7562086 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -34,11 +34,11 @@ It returns a JSON body like the following: } ], "avatar_url": "", // can be null if not set - "is_guest": 0, - "admin": 0, - "deactivated": 0, + "is_guest": false, + "admin": false, + "deactivated": false, "erased": false, - "shadow_banned": 0, + "shadow_banned": false, "creation_ts": 1560432506, "last_seen_ts": 1732919539393, "appservice_id": null, @@ -183,24 +183,24 @@ A response body like the following is returned: "users": [ { "name": "", - "is_guest": 0, - "admin": 0, + "is_guest": false, + "admin": false, "user_type": null, - "deactivated": 0, + "deactivated": false, "erased": false, - "shadow_banned": 0, + "shadow_banned": false, "displayname": "", "avatar_url": null, "creation_ts": 1560432668000, "locked": false }, { "name": "", - "is_guest": 0, - "admin": 1, + "is_guest": false, + "admin": true, "user_type": null, - "deactivated": 0, + "deactivated": false, "erased": false, - "shadow_banned": 0, + "shadow_banned": false, "displayname": "", "avatar_url": "", "creation_ts": 1561550621000, From 673461a388443ad354db7dfa8e51525cdc703a2f Mon Sep 17 00:00:00 2001 From: Jason Little Date: Thu, 11 Jun 2026 12:30:49 -0500 Subject: [PATCH 91/92] try and comment why simple_insert_many_txn() uses COPY FROM as it's psycopg preference --- synapse/storage/database.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index a290375d39e..9527b2a6752 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -1349,6 +1349,11 @@ def simple_insert_many_txn( txn.execute_values(sql, values, fetch=False) elif isinstance(txn.database_engine, PsycopgEngine): + # `execute_values` is not available for insertions on psycopg at this time. + # However, Postgres allows for bulk insertion of data with its COPY command + # which is very performant. This should be on-par with `execute_values` from + # psycopg2. The available alternative would be `executemany` which uses + # Postgres' pipeline mode but is approximately an order of magnitude slower. sql = "COPY %s (%s) FROM STDIN" % ( table, ", ".join(k for k in keys), From 94e9f0cd6cb65ee4e9bec63614935a089cccc542 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Fri, 12 Jun 2026 07:52:58 -0500 Subject: [PATCH 92/92] add enhanced docstring to new copy_write() helper on LoggingTransaction --- synapse/storage/database.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 30599acd5a1..58501819a56 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -552,7 +552,25 @@ def f( def copy_write( self, sql: str, args: Iterable[Any], values: Iterable[Iterable[Any]] ) -> None: - """Corresponds to a PostgreSQL COPY (...) FROM STDIN call.""" + """ + Corresponds to a PostgreSQL COPY (...) FROM STDIN call using psycopg v3 + attributes and helpers. + + Note that COPY commands do not have an INSERT INTO or similar, merely the + tablename and the columns in the order of insertion. Assumes no return values. + Do not use for DELETE, UPSERTs or when needing a RETURNING value + + Instead of + INSERT INTO table (user_id, email, locked) VALUES ("alice", "a@here.com", False) + + it would be + COPY table (user_id, email, locked) FROM STDIN + + The 'values' parameter input should not change, a Collection of Iterables. + Details on formatting are handled by psycopg. + + The most performant way to insert data, especially at scale. + """ assert isinstance(self.database_engine, PsycopgEngine) def f(