From 65ddc7118ab556dd2dbfa47ae9175e8fa67d4566 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 May 2026 17:33:26 +0000 Subject: [PATCH 01/26] Bump go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp from 1.34.0 to 1.43.0 in /complement (#19673) Signed-off-by: dependabot[bot] --- complement/go.mod | 24 +++++++------- complement/go.sum | 84 +++++++++++++++++++++++------------------------ 2 files changed, 54 insertions(+), 54 deletions(-) diff --git a/complement/go.mod b/complement/go.mod index d1ccf6b496..5ebf19bbf6 100644 --- a/complement/go.mod +++ b/complement/go.mod @@ -1,8 +1,6 @@ module github.com/element-hq/synapse -go 1.24.1 - -toolchain go1.24.4 +go 1.25.0 require ( github.com/matrix-org/complement v0.0.0-20251120181401-44111a2a8a9d @@ -16,7 +14,7 @@ require ( github.com/hashicorp/go-set/v3 v3.0.0 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect github.com/oleiade/lane/v2 v2.0.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.34.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect gotest.tools/v3 v3.4.0 // indirect ) @@ -47,13 +45,15 @@ require ( github.com/tidwall/sjson v1.2.5 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect - go.opentelemetry.io/otel v1.41.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 // indirect - go.opentelemetry.io/otel/metric v1.41.0 // indirect - go.opentelemetry.io/otel/trace v1.41.0 // indirect - go.opentelemetry.io/proto/otlp v1.7.0 // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/sys v0.38.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect + golang.org/x/crypto v0.49.0 // indirect + golang.org/x/net v0.52.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.42.0 // indirect golang.org/x/time v0.11.0 // indirect - google.golang.org/protobuf v1.36.6 // indirect + golang.org/x/tools v0.42.0 // indirect ) diff --git a/complement/go.sum b/complement/go.sum index f82487882b..25c6922669 100644 --- a/complement/go.sum +++ b/complement/go.sum @@ -2,8 +2,8 @@ github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEK github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= @@ -38,8 +38,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/go-set/v3 v3.0.0 h1:CaJBQvQCOWoftrBcDt7Nwgo0kdpmrKxar/x2o6pV9JA= @@ -101,55 +101,55 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= -go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9HsPl42NIXFIFSUSSs0fiqra0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0/go.mod h1:90PoxvaEB5n6AOdZvi+yWJQoE95U8Dhhw2bSyRqnTD0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.34.0 h1:BEj3SPM81McUZHYjRS5pEgNgnmzGJ5tRpU5krWnV8Bs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.34.0/go.mod h1:9cKLGBDzI/F3NoHLQGm4ZrYdIHsvGt6ej6hUowxY0J4= -go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= -go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= -go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= -go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= -go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= -go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= -go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= -go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= -go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= -go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= -golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= -golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -157,20 +157,20 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= -golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a h1:SGktgSolFCo75dnHJF2yMvnns6jCmHFJ0vE4Vn2JKvQ= -google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a/go.mod h1:a77HrdMjoeKbnd2jmgcWdaS++ZLZAEq3orIOAEIKiVw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= -google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0= From 0a6aed8df273d599ca477364c0279960a9d9cecc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 May 2026 17:35:45 +0000 Subject: [PATCH 02/26] Bump rand from 0.9.2 to 0.9.4 (#19687) Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8285c7cf38..761e99b3e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -952,9 +952,9 @@ checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha", "rand_core", From a6b8a19ac72e17211777455b817ff1e23855bf31 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 28 May 2026 20:44:17 +0300 Subject: [PATCH 03/26] Add workaround for missing events in `_should_perform_remote_join` (#19730) --- changelog.d/19730.bugfix | 1 + synapse/handlers/room_member.py | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 changelog.d/19730.bugfix diff --git a/changelog.d/19730.bugfix b/changelog.d/19730.bugfix new file mode 100644 index 0000000000..074405d944 --- /dev/null +++ b/changelog.d/19730.bugfix @@ -0,0 +1 @@ +Work around bug that sometimes breaks joining restricted rooms that require a remote join. Contributed by @tulir @ Beeper. diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 236c8ca03c..8a19dba5ee 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -1355,7 +1355,15 @@ async def _should_perform_remote_join( current_state = { state_key: event_map[event_id] for state_key, event_id in state_before_join.items() + # TODO figure out why events present in state_before_join are sometimes not found in event_map + # See https://github.com/element-hq/synapse/issues/19465 + if event_id in event_map } + if len(current_state) < len(state_before_join): + logger.warning( + "Some events from state_before_join were not found in event_map: %s", + set(state_before_join.values()) - set(event_map.keys()), + ) servers_that_can_issue_invite = get_servers_from_users( get_users_which_can_issue_invite(current_state) ) From c45096e7e82de5c550c169253b81a4fc01f203e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 May 2026 09:00:39 +0000 Subject: [PATCH 04/26] Bump sigstore/cosign-installer from 4.1.1 to 4.1.2 in the minor-and-patches group (#19803) Signed-off-by: dependabot[bot] --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c451e49301..e343d950b3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -186,7 +186,7 @@ jobs: uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Install Cosign - uses: sigstore/cosign-installer@cad07c2e89fa2edd6e2d7bab4c1aa38e53f76003 # v4.1.1 + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - name: Calculate docker image tag uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 From 306d8b23bd405802ef668e99420d37f6f8a2ecec Mon Sep 17 00:00:00 2001 From: shcherbak Date: Fri, 29 May 2026 13:57:33 +0300 Subject: [PATCH 05/26] Feat/gcp json formatter (#19775) Co-authored-by: Andrew Morgan --- changelog.d/19775.misc | 1 + docs/structured_logging.md | 41 ++++++++++++++++ synapse/logging/__init__.py | 8 +++- synapse/logging/_terse_json.py | 29 ++++++++++++ tests/logging/test_terse_json.py | 80 +++++++++++++++++++++++++++++++- 5 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 changelog.d/19775.misc diff --git a/changelog.d/19775.misc b/changelog.d/19775.misc new file mode 100644 index 0000000000..3ff0fd34b9 --- /dev/null +++ b/changelog.d/19775.misc @@ -0,0 +1 @@ +Add `GcpJsonFormatter` logging formatter for use with Google Cloud Logging and GKE deployments. \ No newline at end of file diff --git a/docs/structured_logging.md b/docs/structured_logging.md index 761d6466dd..c6916670aa 100644 --- a/docs/structured_logging.md +++ b/docs/structured_logging.md @@ -78,3 +78,44 @@ loggers: The above logging config will set Synapse as 'INFO' logging level by default, with the SQL layer at 'WARNING', and will log JSON formatted messages to a remote endpoint at 10.1.2.3:9999. + +## Google Cloud Logging (GKE) + +When running Synapse on GKE, use `synapse.logging.GcpJsonFormatter`. It outputs +JSON to stdout with a `severity` field that Google Cloud Logging maps to the +correct per-entry severity. Without this, GKE assigns `ERROR` to everything +written to stderr regardless of the actual Python log level. + +Example output: + +```json +{"severity":"INFO","message":"Processed request: 3.481sec 200 GET /sync","logger":"synapse.access.http.8008","time":"2026-05-12T13:40:37.829Z"} +``` + +Configuration: + +```yaml +version: 1 +disable_existing_loggers: false + +formatters: + gcp_json: + class: synapse.logging.GcpJsonFormatter + +handlers: + console: + class: logging.StreamHandler + formatter: gcp_json + stream: ext://sys.stdout + +loggers: + synapse.storage.SQL: + level: WARNING + twisted: + handlers: [console] + propagate: false + +root: + level: INFO + handlers: [console] +``` diff --git a/synapse/logging/__init__.py b/synapse/logging/__init__.py index 15b92d7ef3..4058561e5e 100644 --- a/synapse/logging/__init__.py +++ b/synapse/logging/__init__.py @@ -22,10 +22,14 @@ import logging from synapse.logging._remote import RemoteHandler -from synapse.logging._terse_json import JsonFormatter, TerseJsonFormatter +from synapse.logging._terse_json import ( + GcpJsonFormatter, + JsonFormatter, + TerseJsonFormatter, +) # These are imported to allow for nicer logging configuration files. -__all__ = ["RemoteHandler", "JsonFormatter", "TerseJsonFormatter"] +__all__ = ["RemoteHandler", "JsonFormatter", "TerseJsonFormatter", "GcpJsonFormatter"] # Debug logger for https://github.com/matrix-org/synapse/issues/9533 etc issue9533_logger = logging.getLogger("synapse.9533_debug") diff --git a/synapse/logging/_terse_json.py b/synapse/logging/_terse_json.py index d9ff70b252..afa3288809 100644 --- a/synapse/logging/_terse_json.py +++ b/synapse/logging/_terse_json.py @@ -25,6 +25,7 @@ import json import logging +from datetime import datetime, timezone _encoder = json.JSONEncoder(ensure_ascii=False, separators=(",", ":")) @@ -93,3 +94,31 @@ def format(self, record: logging.LogRecord) -> str: } return self._format(record, event) + + +class GcpJsonFormatter(logging.Formatter): + """JSON formatter compatible with Google Cloud Logging structured logging. + + Outputs `severity` (not `level`) so GCL correctly maps each log record to + the right severity instead of inheriting ERROR from stderr. + """ + + def format(self, record: logging.LogRecord) -> str: + msg = record.getMessage() + if record.exc_info: + if not record.exc_text: + record.exc_text = self.formatException(record.exc_info) + if record.exc_text: + msg = f"{msg}\n{record.exc_text}" + + event = { + "severity": record.levelname, + "message": msg, + "logger": record.name, + "time": datetime.fromtimestamp(record.created, tz=timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%S.%f" + )[:-3] + + "Z", + } + + return _encoder.encode(event) diff --git a/tests/logging/test_terse_json.py b/tests/logging/test_terse_json.py index a857737ddf..433408fee6 100644 --- a/tests/logging/test_terse_json.py +++ b/tests/logging/test_terse_json.py @@ -28,7 +28,11 @@ from twisted.web.server import Request from synapse.http.site import SynapseRequest -from synapse.logging._terse_json import JsonFormatter, TerseJsonFormatter +from synapse.logging._terse_json import ( + GcpJsonFormatter, + JsonFormatter, + TerseJsonFormatter, +) from synapse.logging.context import LoggingContext, LoggingContextFilter from synapse.types import JsonDict @@ -251,3 +255,77 @@ def test_with_exception(self) -> None: self.assertEqual(log["log"], "Hello there, wally!") self.assertEqual(log["exc_type"], "ValueError") self.assertEqual(log["exc_value"], "That's wrong, you wally!") + + +class GcpJsonFormatterTestCase(LoggerCleanupMixin, TestCase): + def setUp(self) -> None: + self.output = StringIO() + + def get_log_line(self) -> JsonDict: + data = self.output.getvalue() + logs = data.splitlines() + self.assertEqual(len(logs), 1) + self.assertEqual(data.count("\n"), 1) + return json.loads(logs[0]) + + def test_gcp_json_output(self) -> None: + """ + GcpJsonFormatter produces exactly the four fields GCL expects. + """ + handler = logging.StreamHandler(self.output) + handler.setFormatter(GcpJsonFormatter()) + logger = self.get_logger(handler) + + logger.info("Hello there, %s!", "wally") + + log = self.get_log_line() + + self.assertIncludes( + log.keys(), {"severity", "message", "logger", "time"}, exact=True + ) + self.assertEqual(log["message"], "Hello there, wally!") + self.assertEqual(log["severity"], "INFO") + self.assertTrue(log["time"].endswith("Z")) + + def test_severity_levels(self) -> None: + """ + Python log levels are mapped to their GCL severity equivalents. + """ + cases = [ + (logging.DEBUG, "DEBUG"), + (logging.INFO, "INFO"), + (logging.WARNING, "WARNING"), + (logging.ERROR, "ERROR"), + (logging.CRITICAL, "CRITICAL"), + ] + for level, expected_severity in cases: + self.output = StringIO() + handler = logging.StreamHandler(self.output) + handler.setFormatter(GcpJsonFormatter()) + logger = self.get_logger(handler) + logger.setLevel(level) + logger.log(level, "test") + log = self.get_log_line() + self.assertEqual(log["severity"], expected_severity, f"level={level}") + + def test_gcp_json_with_exception(self) -> None: + """ + Exception info is appended to the message field, not separate keys. + """ + handler = logging.StreamHandler(self.output) + handler.setFormatter(GcpJsonFormatter()) + logger = self.get_logger(handler) + + try: + raise ValueError("That's wrong, you wally!") + except ValueError: + logger.exception("Hello there, %s!", "wally") + + log = self.get_log_line() + + self.assertIncludes( + log.keys(), {"severity", "message", "logger", "time"}, exact=True + ) + self.assertIn("Hello there, wally!", log["message"]) + self.assertIn("ValueError", log["message"]) + self.assertIn("That's wrong, you wally!", log["message"]) From 2aef6c33a842fe7f50e0a8492a0396412aed5115 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 1 Jun 2026 11:34:12 +0100 Subject: [PATCH 06/26] Add logging to help diagnose missing to-device messages (#19801) Some attempts to debug https://github.com/element-hq/synapse/issues/19795. --------- Co-authored-by: Eric Eastwood Co-authored-by: Eric Eastwood --- changelog.d/19801.misc | 1 + synapse/replication/tcp/resource.py | 102 +++++++++--------- synapse/storage/databases/main/deviceinbox.py | 101 ++++++++++++----- synapse/storage/util/id_generators.py | 26 ++++- 4 files changed, 151 insertions(+), 79 deletions(-) create mode 100644 changelog.d/19801.misc diff --git a/changelog.d/19801.misc b/changelog.d/19801.misc new file mode 100644 index 0000000000..56ec0f7ddc --- /dev/null +++ b/changelog.d/19801.misc @@ -0,0 +1 @@ +Add more logging to the to-device message replication stream. diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index 36dd39ed67..95364778cc 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -182,13 +182,10 @@ async def _run_notifier_loop(self) -> None: self.command_handler.will_announce_positions() for stream in all_streams: - self.command_handler.send_command( - PositionCommand( - stream.NAME, - self._instance_name, - stream.last_token, - stream.last_token, - ) + self._send_position_command( + stream_name=stream.NAME, + prev_token=stream.last_token, + new_token=stream.last_token, ) for stream in all_streams: @@ -205,9 +202,9 @@ async def _run_notifier_loop(self) -> None: last_token = stream.last_token logger.debug( - "Getting stream: %s: %s -> %s", + "Getting stream updates for %s: %s -> %s", stream.NAME, - stream.last_token, + last_token, stream.current_token(self._instance_name), ) try: @@ -217,26 +214,7 @@ async def _run_notifier_loop(self) -> None: logger.info("Failed to handle stream %s", stream.NAME) raise - logger.debug( - "Sending %d updates", - len(updates), - ) - - if updates: - logger.info( - "Streaming: %s -> %s (limited: %s, updates: %s, max token: %s)", - stream.NAME, - updates[-1][0], - limited, - len(updates), - current_token, - ) - stream_updates_counter.labels( - stream_name=stream.NAME, - **{SERVER_NAME_LABEL: self.server_name}, - ).inc(len(updates)) - - else: + if not updates: # The token has advanced but there is no data to # send, so we send a `POSITION` to inform other # workers of the updated position. @@ -266,21 +244,28 @@ async def _run_notifier_loop(self) -> None: # POSITION with last token of X+1, which will # cause them to check if there were any missing # updates between X and X+1. - logger.info( - "Sending position: %s -> %s", - stream.NAME, - current_token, - ) - self.command_handler.send_command( - PositionCommand( - stream.NAME, - self._instance_name, - last_token, - current_token, - ) + self._send_position_command( + stream_name=stream.NAME, + prev_token=last_token, + new_token=current_token, ) continue + logger.info( + "Sending update for %s: %s -> %s (limited: %s, updates: %s, max token: %s)", + stream.NAME, + last_token, + updates[-1][0], + limited, + len(updates), + current_token, + ) + + stream_updates_counter.labels( + stream_name=stream.NAME, + **{SERVER_NAME_LABEL: self.server_name}, + ).inc(len(updates)) + # Some streams return multiple rows with the same stream IDs, # we need to make sure they get sent out in batches. We do # this by setting the current token to all but the last of @@ -300,18 +285,10 @@ async def _run_notifier_loop(self) -> None: # token, in which case we want to send out a `POSITION` # to tell other workers the actual current position. if updates[-1][0] < current_token: - logger.info( - "Sending position: %s -> %s", - stream.NAME, - current_token, - ) - self.command_handler.send_command( - PositionCommand( - stream.NAME, - self._instance_name, - updates[-1][0], - current_token, - ) + self._send_position_command( + stream_name=stream.NAME, + prev_token=updates[-1][0], + new_token=current_token, ) logger.debug("No more pending updates, breaking poke loop") @@ -319,6 +296,25 @@ async def _run_notifier_loop(self) -> None: self.pending_updates = False self.is_looping = False + def _send_position_command( + self, *, stream_name: str, prev_token: int, new_token: int + ) -> None: + """Send a POSITION command over replication""" + logger.info( + "Sending position for %s: %s -> %s", + stream_name, + prev_token, + new_token, + ) + self.command_handler.send_command( + PositionCommand( + stream_name, + self._instance_name, + prev_token, + new_token, + ) + ) + def _batch_updates( updates: list[tuple[Token, StreamRow]], diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index fc61f46c1c..a3806fd112 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -897,10 +897,20 @@ def _add_messages_to_local_device_inbox_txn( ) -> None: assert self._can_write_to_device - local_by_user_then_device = {} + # A map from user id, to device id, to a pair of (serialized message, msgid). + local_by_user_then_device: dict[str, dict[str, tuple[str, str]]] = {} + for user_id, messages_by_device in messages_by_user_then_device.items(): - messages_json_for_user = {} + # Mesages to send to this specific user. A map + # from device id, to a pair of (serialized message, msgid). + messages_json_for_user: dict[str, tuple[str, str]] = {} + devices = list(messages_by_device.keys()) + if not devices: + # No to-device messages for this user. (For example, someone has + # hit `/sendToDevice` with an empty {device: message} dict.) + continue + if len(devices) == 1 and devices[0] == "*": # Handle wildcard device_ids. # We exclude hidden devices (such as cross-signing keys) here as they are @@ -912,15 +922,28 @@ def _add_messages_to_local_device_inbox_txn( retcol="device_id", ) - message_json = json_encoder.encode(messages_by_device["*"]) + # Don't bother to serialize if there are no devices for this user + if not devices: + if issue9533_logger.isEnabledFor(logging.DEBUG): + msgid = _get_msgid_for_message(messages_by_device["*"]) + issue9533_logger.debug( + "Dropping wildcard to-device message for user %s with no devices (msgid %s)", + user_id, + msgid, + ) + continue + + message_json, msgid = _serialize_to_device_message( + user_id=user_id, device_id="*", msg=messages_by_device["*"] + ) for device_id in devices: # Add the message for all devices for this user on this # server. - messages_json_for_user[device_id] = message_json + messages_json_for_user[device_id] = (message_json, msgid) else: - if not devices: - continue - + # Query the database to determine which of the target devices actually + # exist. + # # We exclude hidden devices (such as cross-signing keys) here as they are # not expected to receive to-device messages. rows = cast( @@ -938,19 +961,25 @@ def _add_messages_to_local_device_inbox_txn( for (device_id,) in rows: # Only insert into the local inbox if the device exists on # this server - with start_active_span("serialise_to_device_message"): - msg = messages_by_device[device_id] - set_tag(SynapseTags.TO_DEVICE_TYPE, msg["type"]) - set_tag(SynapseTags.TO_DEVICE_SENDER, msg["sender"]) - set_tag(SynapseTags.TO_DEVICE_RECIPIENT, user_id) - set_tag(SynapseTags.TO_DEVICE_RECIPIENT_DEVICE, device_id) - set_tag( - SynapseTags.TO_DEVICE_MSGID, - msg["content"].get(EventContentFields.TO_DEVICE_MSGID), - ) - message_json = json_encoder.encode(msg) + msg = messages_by_device[device_id] + message_json, msgid = _serialize_to_device_message( + user_id=user_id, device_id=device_id, msg=msg + ) + messages_json_for_user[device_id] = (message_json, msgid) - messages_json_for_user[device_id] = message_json + if issue9533_logger.isEnabledFor(logging.DEBUG): + # Log any messages we are dropping + unmapped_devices = ( + messages_by_device.keys() - messages_json_for_user.keys() + ) + if unmapped_devices: + issue9533_logger.debug( + "Dropping to-device messages for unknown devices: %s", + [ + f"{user_id}/{device_id} (msgid {_get_msgid_for_message(messages_by_device[device_id])})" + for device_id in unmapped_devices + ], + ) if messages_json_for_user: local_by_user_then_device[user_id] = messages_json_for_user @@ -965,22 +994,21 @@ def _add_messages_to_local_device_inbox_txn( values=[ (user_id, device_id, stream_id, message_json, self._instance_name) for user_id, messages_by_device in local_by_user_then_device.items() - for device_id, message_json in messages_by_device.items() + for device_id, (message_json, _msgid) in messages_by_device.items() ], ) if issue9533_logger.isEnabledFor(logging.DEBUG): issue9533_logger.debug( - "Stored to-device messages with stream_id %i: %s", + "Storing to-device messages with stream_id %i: %s", stream_id, [ - f"{user_id}/{device_id} (msgid " - f"{msg['content'].get(EventContentFields.TO_DEVICE_MSGID)})" + f"{user_id}/{device_id} (msgid {msgid})" for ( user_id, messages_by_device, - ) in messages_by_user_then_device.items() - for (device_id, msg) in messages_by_device.items() + ) in local_by_user_then_device.items() + for (device_id, (_msg, msgid)) in messages_by_device.items() ], ) @@ -1066,6 +1094,29 @@ def get_devices_with_messages_txn( return results +def _serialize_to_device_message( + *, user_id: str, device_id: str, msg: JsonDict +) -> tuple[str, str]: + """Serialiize a to-device message, ready to add to the device_inbox table. + + Returns a tuple (message_json, msgid). + """ + with start_active_span("serialise_to_device_message"): + msgid = _get_msgid_for_message(msg) + set_tag(SynapseTags.TO_DEVICE_TYPE, msg["type"]) + set_tag(SynapseTags.TO_DEVICE_SENDER, msg["sender"]) + set_tag(SynapseTags.TO_DEVICE_RECIPIENT, user_id) + set_tag(SynapseTags.TO_DEVICE_RECIPIENT_DEVICE, device_id) + set_tag(SynapseTags.TO_DEVICE_MSGID, msgid) + message_json = json_encoder.encode(msg) + return message_json, msgid + + +def _get_msgid_for_message(msg: JsonDict) -> str: + """Extract the message ID from a to-device message.""" + return str(msg["content"].get(EventContentFields.TO_DEVICE_MSGID, "")) + + class DeviceInboxBackgroundUpdateStore(SQLBaseStore): DEVICE_INBOX_STREAM_ID = "device_inbox_stream_drop" REMOVE_DEAD_DEVICES_FROM_INBOX = "remove_dead_devices_from_device_inbox" diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index a1afe6d2ae..1e053be6af 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -38,6 +38,7 @@ import attr from sortedcontainers import SortedList, SortedSet +from synapse.logging import issue9533_logger from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.database import ( DatabasePool, @@ -774,6 +775,14 @@ def _add_persisted_position(self, new_id: int) -> None: # We move the current min position up if the minimum current positions # of all instances is higher (since by definition all positions less # that that have been persisted). + # + # If we are one of several writers, then we don't need to factor our own + # `_current_position` into `_persisted_upto_position` unless we have unfinished + # writes (since we know that any future write that happens locally will have + # a higher stream ID than any of the other writers' current positions). In other + # words, when we have no outstanding writes, then the new `_persisted_upto_position` + # can be the minimum of all *other* writers' current positions, + # our_current_position = self._current_positions.get(self._instance_name, 0) min_curr = min( ( @@ -783,7 +792,6 @@ def _add_persisted_position(self, new_id: int) -> None: ), default=our_current_position, ) - if our_current_position and (self._unfinished_ids or self._in_flight_fetches): min_curr = min(min_curr, our_current_position) @@ -820,6 +828,22 @@ def _add_persisted_position(self, new_id: int) -> None: # do. break + # Hacky debug logging to attempt to trace https://github.com/element-hq/synapse/issues/19795 + if ( + issue9533_logger.isEnabledFor(logging.DEBUG) + and self._stream_name == "to_device" + ): + issue9533_logger.debug( + "stream_id=%i now persisted for stream=%s; _current_positions=%s _unfinished_ids=%s, _known_persisted_positions=%s _persisted_upto_position=%i min_curr=%i", + new_id, + self._stream_name, + self._current_positions, + self._unfinished_ids, + self._known_persisted_positions, + self._persisted_upto_position, + min_curr, + ) + def _update_stream_positions_table_txn(self, txn: Cursor) -> None: """Update the `stream_positions` table with newly persisted position.""" From 71e07d4c75e1dad297bc505b249acd556bf5b253 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:38:08 +0200 Subject: [PATCH 07/26] Bump hashicorp/vault-action from 3.4.0 to 4.0.0 (#19804) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [hashicorp/vault-action](https://github.com/hashicorp/vault-action) from 3.4.0 to 4.0.0.
Release notes

Sourced from hashicorp/vault-action's releases.

v4.0.0

4.0.0 (May 12, 2026)

Improvements:

  • Bump node runtime from node20 to node24 GH-604
  • Fix leading slash in secret paths causing HTTP 400 errors (e.g. /cubbyhole/test → v1/cubbyhole/test instead of v1//cubbyhole/test)
  • bump jsrsasign from 11.1.0 to 11.1.3
  • bump body-parser from 1.20.3 to 1.20.5
  • bump qs from 6.13.0 to 6.15.1
  • bump http-errors from 2.0.0 to 2.0.1
  • bump minimatch from 3.1.2 to 3.1.5
  • bump underscore from 1.13.4 to 1.13.8
Changelog

Sourced from hashicorp/vault-action's changelog.

4.0.0 (May 12, 2026)

Improvements:

  • Bump node runtime from node20 to node24 GH-604
  • Fix leading slash in secret paths causing HTTP 400 errors (e.g. /cubbyhole/test → v1/cubbyhole/test instead of v1//cubbyhole/test)
  • bump jsrsasign from 11.1.0 to 11.1.3
  • bump body-parser from 1.20.3 to 1.20.5
  • bump qs from 6.13.0 to 6.15.1
  • bump http-errors from 2.0.0 to 2.0.1
  • bump minimatch from 3.1.2 to 3.1.5
  • bump underscore from 1.13.4 to 1.13.8

3.4.0 (June 13, 2025)

Bugs:

Improvements:

3.3.0 (March 3, 2025)

Features:

  • Wildcard secret imports can use ** to retain case of exported env keys GH-545

3.2.0 (March 3, 2025)

Improvements:

  • Add retry for jwt auth login to fix intermittent login failures GH-574

3.1.0 (January 9, 2025)

Improvements:

  • fix wildcard handling when field contains dot GH-542
  • bump body-parser from 1.20.0 to 1.20.3
  • bump braces from 3.0.2 to 3.0.3
  • bump cross-spawn from 7.0.3 to 7.0.6
  • bump micromatch from 4.0.5 to 4.0.8

Features:

  • secretId is no longer required for approle to support advanced use cases like machine login when bind_secret_id is false. GH-522
  • Use pki configuration to generate certificates from Vault GH-564

3.0.0 (February 15, 2024)

... (truncated)

Commits
  • 892a268 Update copywrite headers for v.4.0.0 release (#607)
  • a7ffa26 Prepare for release v4.0.0 (#606)
  • a049f01 [COMPLIANCE] Add/Update Copyright Headers (#605)
  • 95977a3 Adding team-vault-consumption as CODEOWNERS (#600)
  • 7e48e56 Upgrade Node.js to 24 and update dependencies (#604)
  • 79632e3 [COMPLIANCE] Add Copyright and License Headers (Batch 1 of 1) (#589)
  • 734c523 README.md: Removing jwtGithubAudience default (#590)
  • 2c58270 [Compliance] - PR Template Changes Required (#586)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=hashicorp/vault-action&package-manager=github_actions&previous-version=3.4.0&new-version=4.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e343d950b3..801d369c48 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -67,7 +67,7 @@ jobs: - name: Get team registry token id: import-secrets - uses: hashicorp/vault-action@4c06c5ccf5c0761b6029f56cfb1dcf5565918a3b # v3.4.0 + uses: hashicorp/vault-action@892a26828f195e65540a40b4768ae4571f51ebfc # v4.0.0 with: url: https://vault.infra.ci.i.element.dev role: ${{ steps.vault-jwt-role.outputs.role_name }} @@ -164,7 +164,7 @@ jobs: - name: Get team registry token id: import-secrets - uses: hashicorp/vault-action@4c06c5ccf5c0761b6029f56cfb1dcf5565918a3b # v3.4.0 + uses: hashicorp/vault-action@892a26828f195e65540a40b4768ae4571f51ebfc # v4.0.0 with: url: https://vault.infra.ci.i.element.dev role: ${{ steps.vault-jwt-role.outputs.role_name }} From 6c3ba2205bcc78187dcbdee25b3950c6be92da83 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:45:41 +0200 Subject: [PATCH 08/26] Bump idna from 3.11 to 3.15 (#19790) Bumps [idna](https://github.com/kjd/idna) from 3.11 to 3.15.
Changelog

Sourced from idna's changelog.

3.15 (2026-05-12)

  • Enforce DNS-length cap on individual labels early in check_label, short-circuiting contextual-rule processing for oversized input while staying compatible with UTS 46 usage.
  • Tidy core helpers: hoist bidi category sets to module-level frozensets (avoiding per-codepoint list construction), simplify length checks, and reuse the shared _unicode_dots_re from idna.core in the codec module.
  • Use raise ... from err for proper exception chaining and switch internal string formatting to f-strings.
  • Allow flit_core 4.x in the build backend.
  • Expand the ruff lint set (flake8-bugbear, flake8-simplify, pyupgrade, perflint) and apply the surfaced fixes; pin lint CI to Python 3.14.
  • Add Dependabot configuration for GitHub Actions.
  • Convert README and HISTORY from reStructuredText to Markdown.
  • Reference CVE-2026-45409 for the 3.14 advisory in place of the initial GHSA identifier.

Thanks to Felix Yan, Stan Ulbrych, and metsw24-max for contributions to this release.

3.14 (2026-05-10)

  • Removed opportunity to process long inputs into quadratic time by rejecting oversize inputs up-front. Closes a bypass of the CVE-2024-3651 mitigation. [CVE-2026-45409]

Thanks to Stan Ulbrych for reporting the issue.

3.13 (2026-04-22)

  • Correct classification error for codepoint U+A7F1

3.12 (2026-04-21)

  • Update to Unicode 17.0.0.
  • Issue a deprecation warning for the transitional argument.
  • Added lazy-loading to provide some performance improvements.
  • Removed vestiges of code related to Python 2 support, including segmentation of data structures specific to Jython.

Thanks to Rodrigo Nogueira for contributions to this release.

Commits
  • af30a09 Release 3.15
  • 30314d4 Pre-release 3.15rc0
  • 05d4b21 Merge pull request #237 from kjd/convert-docs-to-markdown
  • 2987fdb Convert README and HISTORY from reStructuredText to Markdown
  • 59fa800 Merge pull request #236 from kjd/dependabot/github_actions/actions-f3e34333ea
  • def6983 Merge branch 'master' into dependabot/github_actions/actions-f3e34333ea
  • bbd8004 Merge pull request #234 from StanFromIreland/patch-1
  • edd07c0 Bump github/codeql-action from 3.35.2 to 4.35.2 in the actions group
  • 5557db0 Merge branch 'master' into patch-1
  • f11746c Merge pull request #235 from StanFromIreland/patch-2
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=idna&package-manager=pip&previous-version=3.11&new-version=3.15)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/element-hq/synapse/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 6093b13c02..4afbaad2c1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -752,18 +752,18 @@ test = ["coverage[toml]", "pretend", "pytest", "pytest-cov"] [[package]] name = "idna" -version = "3.11" +version = "3.15" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.8" groups = ["main", "dev"] files = [ - {file = "idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea"}, - {file = "idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902"}, + {file = "idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8"}, + {file = "idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc"}, ] [package.extras] -all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] +all = ["mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] [[package]] name = "ijson" From 9e2a076144601e07f253cb335f2b0190417b57cb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Jun 2026 11:05:38 +0100 Subject: [PATCH 09/26] Port Event class to Rust (#19701) Ports the event class to Rust. The main difference here are: 1. There is now a single event class 2. We now validate a lot more at event construction time than we previously did (we basically checked nothing before). This required some changes to the tests, including https://github.com/matrix-org/sytest/pull/1423 Reviewable commit-by-commit. ### Overview of Event Rust structure The format of the event struct in Rust is quite different than that in Python. The top-level looks like: ```rust pub struct Event { /// The parsed event JSON. fields: FormattedEvent, /// The event ID. For format v1 this is read directly from the JSON; /// for v2+ it is computed from the canonical-JSON hash at /// construction time and cached here. event_id: Arc, /// Synapse-internal per-event state that lives outside the federated /// JSON (e.g. outlier flag, soft-failure, stream positions). #[pyo3(get)] internal_metadata: EventInternalMetadata, /// The room version this event was parsed for. #[pyo3(get)] room_version: &'static RoomVersion, /// `None` for accepted events; otherwise a short reason set by auth /// when the event was rejected. rejected_reason: Option>, } ``` which includes the actual parsed event in `FormattedEvent`, plus the rest of the event metadata. ```rust pub struct FormattedEvent> { #[serde(default)] pub signatures: Signatures, #[serde(default)] pub unsigned: Unsigned, #[serde(flatten)] pub specific_fields: E, #[serde(flatten)] pub common_fields: Arc, } ``` The struct is further split into the common fields, format specific fields, plus the signatures and unsigned. We split out the signature and unsigned fields as they are mutable, so when we clone the event we can still share the common and specific fields and only copy signature and unsigned. The `specific_fields` are the fields that depend on the format version. They can either be a specific format (e.g. `E = EventFormatV1`) or a type-erased enum `EventFormatEnum` that is across all room versions: ```rust pub enum EventFormatEnum { V1(EventFormatV1), V2V3(EventFormatV2V3), V4(EventFormatV4), VMSC4242(EventFormatVMSC4242), } ``` For example: ```rust /// Shared flat-list encoding of `auth_events` and `prev_events`, reused /// by every format from v2/v3 onwards. #[derive(Serialize, Deserialize)] pub struct SimpleAuthPrevEvents { pub auth_events: Vec, pub prev_events: Vec, } /// Version-specific fields for room versions 3-10. #[derive(Serialize, Deserialize)] pub struct EventFormatV2V3 { pub room_id: Box, #[serde(flatten)] pub auth_prev_events: SimpleAuthPrevEvents, } ``` ### Dev notes As discussed in [`#element-backend-internal:matrix.org`](https://matrix.to/#/!SGNQGPGUwtcPBUotTL:matrix.org/$3gTjDO440GbAz57cXcCawwiyFLiD0crrarvS1uhzKOY?via=jki.re&via=element.io&via=matrix.org) --------- Co-authored-by: Eric Eastwood --- changelog.d/19701.misc | 1 + rust/src/duration.rs | 32 +- rust/src/events/constants.rs | 143 ++ rust/src/events/formats/mod.rs | 268 ++++ rust/src/events/formats/v1.rs | 54 + rust/src/events/formats/v2v3.rs | 60 + rust/src/events/formats/v4.rs | 156 +++ rust/src/events/formats/vmsc4242.rs | 94 ++ rust/src/events/internal_metadata.rs | 11 +- rust/src/events/json_object.rs | 6 + rust/src/events/mod.rs | 888 +++++++++++- rust/src/events/signatures.rs | 15 +- rust/src/events/unsigned.rs | 13 +- rust/src/events/utils.rs | 1191 +++++++++++++++++ rust/src/json.rs | 218 +++ rust/src/lib.rs | 1 + synapse/crypto/event_signing.py | 5 +- synapse/crypto/keyring.py | 4 +- synapse/event_auth.py | 2 + synapse/events/__init__.py | 670 +--------- synapse/events/py_protocol.py | 34 +- synapse/events/utils.py | 175 +-- synapse/federation/federation_client.py | 8 +- .../third_party_event_rules_callbacks.py | 5 - synapse/rest/admin/rooms.py | 10 +- synapse/storage/controllers/persist_events.py | 15 +- .../storage/databases/main/censor_events.py | 10 +- synapse/storage/databases/main/events.py | 4 +- .../databases/main/events_bg_updates.py | 4 +- .../storage/databases/main/events_worker.py | 26 +- synapse/synapse_rust/events.pyi | 166 ++- tests/events/test_py_protocol.py | 27 +- tests/events/test_validator.py | 9 +- tests/handlers/test_device.py | 12 +- tests/handlers/test_room_member.py | 6 +- tests/handlers/test_room_policy.py | 3 +- tests/module_api/test_api.py | 4 +- tests/replication/storage/test_events.py | 1 + tests/rest/client/test_third_party_rules.py | 5 +- tests/state/test_v2.py | 3 - tests/storage/test_msc4242_state_dag.py | 29 +- tests/storage/test_redaction.py | 15 +- tests/storage/test_stream.py | 8 +- tests/test_event_auth.py | 8 +- tests/test_utils/event_builders.py | 14 + 45 files changed, 3454 insertions(+), 979 deletions(-) create mode 100644 changelog.d/19701.misc create mode 100644 rust/src/events/constants.rs create mode 100644 rust/src/events/formats/mod.rs create mode 100644 rust/src/events/formats/v1.rs create mode 100644 rust/src/events/formats/v2v3.rs create mode 100644 rust/src/events/formats/v4.rs create mode 100644 rust/src/events/formats/vmsc4242.rs create mode 100644 rust/src/events/utils.rs create mode 100644 rust/src/json.rs diff --git a/changelog.d/19701.misc b/changelog.d/19701.misc new file mode 100644 index 0000000000..4663e8b961 --- /dev/null +++ b/changelog.d/19701.misc @@ -0,0 +1 @@ +Port the python Event classes to Rust. diff --git a/rust/src/duration.rs b/rust/src/duration.rs index a3dbe919b2..6c2e2653d1 100644 --- a/rust/src/duration.rs +++ b/rust/src/duration.rs @@ -29,19 +29,41 @@ fn duration_module(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { } /// Mirrors the `synapse.util.duration.Duration` Python class. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct SynapseDuration { - microseconds: u64, + milliseconds: u64, } impl SynapseDuration { - /// For now we only need to create durations from milliseconds. - pub fn from_milliseconds(milliseconds: u64) -> Self { + /// Creates a `SynapseDuration` from a number of milliseconds. + pub const fn from_milliseconds(milliseconds: u64) -> Self { + Self { milliseconds } + } + + /// Creates a `SynapseDuration` from a number of hours. + pub const fn from_hours(hours: u32) -> Self { + // We take a u32 here so that we know the multiplication won't overflow. + // We could instead panic, but that is unstable in a const context (for + // the current MSRV 1.82). Self { - microseconds: milliseconds * 1_000, + milliseconds: (hours as u64) * 3_600_000, } } } +impl<'py> IntoPyObject<'py> for SynapseDuration { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let duration_module = duration_module(py)?; + let kwargs = [("milliseconds", self.milliseconds)].into_py_dict(py)?; + let duration_instance = duration_module.call_method("Duration", (), Some(&kwargs))?; + Ok(duration_instance.into_bound()) + } +} + impl<'py> IntoPyObject<'py> for &SynapseDuration { type Target = PyAny; type Output = Bound<'py, Self::Target>; @@ -49,7 +71,7 @@ impl<'py> IntoPyObject<'py> for &SynapseDuration { fn into_pyobject(self, py: Python<'py>) -> Result { let duration_module = duration_module(py)?; - let kwargs = [("microseconds", self.microseconds)].into_py_dict(py)?; + let kwargs = [("milliseconds", self.milliseconds)].into_py_dict(py)?; let duration_instance = duration_module.call_method("Duration", (), Some(&kwargs))?; Ok(duration_instance.into_bound()) } diff --git a/rust/src/events/constants.rs b/rust/src/events/constants.rs new file mode 100644 index 0000000000..811794d48c --- /dev/null +++ b/rust/src/events/constants.rs @@ -0,0 +1,143 @@ +//! Matrix Events +//! +//! Contains types and utilities for working with Matrix events. + +/// Maximum size of a PDU +pub const MAX_PDU_SIZE_BYTES: usize = 65_536; + +/// Event Types +pub mod event_type { + /// Event type: m.room.member + pub const M_ROOM_MEMBER: &str = "m.room.member"; + /// Event type: m.room.create + pub const M_ROOM_CREATE: &str = "m.room.create"; + /// Event type: m.room.join_rules + pub const M_ROOM_JOIN_RULES: &str = "m.room.join_rules"; + /// Event type: m.room.power_levels + pub const M_ROOM_POWER_LEVELS: &str = "m.room.power_levels"; + /// Event type: m.room.aliases + pub const M_ROOM_ALIASES: &str = "m.room.aliases"; + /// Event type: m.room.history_visibility + pub const M_ROOM_HISTORY_VISIBILITY: &str = "m.room.history_visibility"; + /// Event type: m.room.redaction + pub const M_ROOM_REDACTION: &str = "m.room.redaction"; +} + +/// Event Fields +pub mod event_field { + /// Event field: auth_events + pub const AUTH_EVENTS: &str = "auth_events"; + /// Event field: content + pub const CONTENT: &str = "content"; + /// Event field: depth + pub const DEPTH: &str = "depth"; + /// Event field: hashes + pub const HASHES: &str = "hashes"; + /// Event field: origin_server_ts + pub const ORIGIN_SERVER_TS: &str = "origin_server_ts"; + /// Event field: prev_events + pub const PREV_EVENTS: &str = "prev_events"; + /// Event field: room_id + pub const ROOM_ID: &str = "room_id"; + /// Event field: sender + pub const SENDER: &str = "sender"; + /// Event field: signatures + pub const SIGNATURES: &str = "signatures"; + /// Event field: state_key + pub const STATE_KEY: &str = "state_key"; + /// Event field: type + pub const TYPE: &str = "type"; + /// Event field: unsigned + pub const UNSIGNED: &str = "unsigned"; + /// Event field: event_id + pub const EVENT_ID: &str = "event_id"; + /// Event field: origin + pub const ORIGIN: &str = "origin"; + /// Event field: prev_state + pub const PREV_STATE: &str = "prev_state"; + /// Event field: membership + pub const MEMBERSHIP: &str = "membership"; + /// Event field: replaces_state + pub const REPLACES_STATE: &str = "replaces_state"; + /// Event field: msc4354_sticky + pub const MSC4354_STICKY: &str = "msc4354_sticky"; + // Event field: prev_state_events + pub const PREV_STATE_EVENTS: &str = "prev_state_events"; + // Event field: m.relates_to + pub const M_RELATES_TO: &str = "m.relates_to"; +} + +pub mod unsigned_field { + /// Unsigned field: age + pub const AGE: &str = "age"; + /// Unsigned field: age_ts + pub const AGE_TS: &str = "age_ts"; + /// Unsigned field: redacted_because + pub const REDACTED_BECAUSE: &str = "redacted_because"; +} + +/// Membership Event Fields +pub mod membership_field { + /// Membership event field: membership + pub const MEMBERSHIP: &str = "membership"; + /// Membership event field: join_authorised_via_users_server + pub const JOIN_AUTHORISED_VIA_USERS_SERVER: &str = "join_authorised_via_users_server"; + /// Membership event field: third_party_invite + pub const THIRD_PARTY_INVITE: &str = "third_party_invite"; + /// Membership event field: signed + pub const SIGNED: &str = "signed"; +} + +/// Create Event Fields +pub mod create_field { + /// Create event field: creator + pub const CREATOR: &str = "creator"; +} + +/// Join Rules Event Fields +pub mod join_rules_field { + /// Join Rules event field: join_rule + pub const JOIN_RULE: &str = "join_rule"; + /// Join Rules event field: allow + pub const ALLOW: &str = "allow"; +} + +/// Power Levels Event Fields +pub mod power_levels_field { + /// Power Levels event field: users + pub const USERS: &str = "users"; + /// Power Levels event field: users_default + pub const USERS_DEFAULT: &str = "users_default"; + /// Power Levels event field: events + pub const EVENTS: &str = "events"; + /// Power Levels event field: events_default + pub const EVENTS_DEFAULT: &str = "events_default"; + /// Power Levels event field: state_default + pub const STATE_DEFAULT: &str = "state_default"; + /// Power Levels event field: ban + pub const BAN: &str = "ban"; + /// Power Levels event field: kick + pub const KICK: &str = "kick"; + /// Power Levels event field: redact + pub const REDACT: &str = "redact"; + /// Power Levels event field: invite + pub const INVITE: &str = "invite"; +} + +/// Aliases Event Fields +pub mod aliases_field { + /// Aliases event field: aliases + pub const ALIASES: &str = "aliases"; +} + +/// History Visibility Event Fields +pub mod history_visibility_field { + /// History Visibility event field: history_visibility + pub const HISTORY_VISIBILITY: &str = "history_visibility"; +} + +/// Redaction Event Fields +pub mod redaction_field { + /// Redacts event field: redacts + pub const REDACTS: &str = "redacts"; +} diff --git a/rust/src/events/formats/mod.rs b/rust/src/events/formats/mod.rs new file mode 100644 index 0000000000..10b16a5436 --- /dev/null +++ b/rust/src/events/formats/mod.rs @@ -0,0 +1,268 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Over-the-wire representations of Matrix events, parameterised by event +//! format version (i.e. the structure we parse the event JSON into). +//! +//! # Design +//! +//! The shape of an event's JSON varies with the room version, but there are +//! many common fields across all of them — `type`, `sender`, `content`, etc. +//! +//! We model this with a single [`FormattedEvent`] container that is generic +//! over the format-specific tail `E`. Serde `#[serde(flatten)]` merges the +//! common and specific halves into a single JSON object over the wire, while +//! keeping them as distinct structs in Rust. This lets version-agnostic code +//! (field getters, the `unsigned` accessor, …) read [`EventCommonFields`] +//! directly, and only the small amount of version-aware logic (auth-event +//! derivation, room-ID lookup, validation) needs to match on the format. +//! +//! The default `E` parameter is the type-erased [`EventFormatEnum`], which can +//! contain any known format. The [`FormattedEvent::into_general`] method allows +//! converting from a specific format to the general enum. +//! +//! The `signatures` and `unsigned` fields are kept distinct from the +//! common/specific as they allow mutation. When copying an event they need to +//! be deep-copied, but the common/specific fields (which are immutable) can be +//! shared. +//! +//! # Format variants +//! +//! Different room versions have different over-the-wire formats, which is +//! tracked by [`crate::room_versions::RoomVersion::event_format`] field. +//! +//! Each format struct owns only its version-specific fields and any +//! validation/derivation logic; the rest lives in [`EventCommonFields`]. The +//! [`EventFormatEnum`] sum type erases the generic parameter when an `Event` +//! needs to be stored alongside others of unknown room version. +//! +//! Note that any fields not recognised by the format-specific struct or by the +//! common fields are captured into [`EventCommonFields::other_fields`] and +//! round-tripped losslessly. This is useful for capturing optional fields that +//! don't need to be parsed up front. Generally, optional fields should be +//! handled via `other_fields`, as this saves space when they are not present. +//! +//! # Serialization and deserialization +//! +//! Deserializing a Matrix Event from JSON is done by specifying the expected +//! format struct (e.g. [`FormattedEvent`]), which enforces the +//! invariants of that format at parse time. This can then be converted into the +//! version-agnostic [`FormattedEvent`] with the +//! [`FormattedEvent::into_general`] method, which erases the format-specific +//! type but keeps the parsed fields intact. +//! +//! Serializing a [`FormattedEvent`] produces the correct Matrix Event JSON +//! shape for the format variant it contains. + +use std::{collections::HashMap, sync::Arc}; + +use anyhow::Error; +use serde::{Deserialize, Serialize}; + +use crate::{ + events::{json_object::JsonObject, signatures::Signatures, unsigned::Unsigned}, + json::AllowMissing, +}; + +mod v1; +mod v2v3; +mod v4; +mod vmsc4242; + +pub use v1::EventFormatV1; +pub use v2v3::EventFormatV2V3; +pub use v4::EventFormatV4; +pub use vmsc4242::EventFormatVMSC4242; + +/// A parsed Matrix event in its over-the-wire layout. +/// +/// `E` is the format-specific tail. Code that deserialises a known +/// room version picks a concrete `E` (e.g. `FormattedEvent`); +/// the default `Arc` is used once the event has been +/// boxed into the version-agnostic [`Event`](crate::events::Event) +/// pyclass. +/// +/// The `signatures` and `unsigned` fields are kept separate from the other +/// fields as they are mutable (and must be deep-copied if the event is cloned). +/// `common_fields` and `specific_fields` are both `#[serde(flatten)]`ed so that +/// the serialised JSON is a single flat object matching the Matrix spec. +#[derive(Serialize, Deserialize)] +pub struct FormattedEvent> { + /// The event's signatures. + /// + /// Kept separate from common/specific fields as this this is a mutable + /// field. + #[serde(default)] + pub signatures: Signatures, + + /// The event's unsigned data. + /// + /// Kept separate from common/specific fields as this this is a mutable + /// field. + #[serde(default)] + pub unsigned: Unsigned, + + /// The format-specific fields of the event. This is an immutable field. + #[serde(flatten)] + pub specific_fields: E, + + /// The fields common to all event formats. This is an immutable field. + #[serde(flatten)] + pub common_fields: Arc, +} + +impl FormattedEvent { + /// Creates a deep copy of this event, allowing the signatures and unsigned + /// to be mutated without affecting the original. + /// + /// The common and specific fields are shared between the copy and the + /// original, as they are immutable. + pub fn deep_copy(&self) -> FormattedEvent { + FormattedEvent { + signatures: self.signatures.deep_copy(), + unsigned: self.unsigned.deep_copy(), + // These fields can safely be shared among all of the copies as they + // are immutable (they're behind an Arc and so you can't get a + // mutable reference and they have no interior mutability) and these + // write protections extend into Python land as well (i.e. you can't + // accidentally do the wrong thing and mutate) + specific_fields: Arc::clone(&self.specific_fields), + common_fields: Arc::clone(&self.common_fields), + } + } + + pub fn validate(&self) -> Result<(), Error> { + match &*self.specific_fields { + EventFormatEnum::V1(format) => format.validate(&self.common_fields), + EventFormatEnum::V2V3(format) => format.validate(&self.common_fields), + EventFormatEnum::V4(format) => format.validate(&self.common_fields), + EventFormatEnum::VMSC4242(format) => format.validate(&self.common_fields), + } + } +} + +impl FormattedEvent +where + E: Into, +{ + /// Transforms a container of a specific event format into a container of + /// the enum type. + pub fn into_general(self) -> FormattedEvent { + let format: Arc = Arc::new(self.specific_fields.into()); + FormattedEvent { + signatures: self.signatures, + unsigned: self.unsigned, + specific_fields: format, + common_fields: self.common_fields, + } + } +} + +impl From> for FormattedEvent +where + E: Into, +{ + fn from(container: FormattedEvent) -> Self { + container.into_general() + } +} + +/// Fields that appear in every supported event format. +/// +/// Anything not recognised by the format-specific tail or by the fields +/// named here is captured into `other_fields` so events round-trip +/// losslessly even when they carry experimental or future-version +/// keys. +#[derive(Serialize, Deserialize)] +pub struct EventCommonFields { + pub content: JsonObject, + pub depth: i64, + pub hashes: HashMap, Box>, + pub origin_server_ts: i64, + pub sender: Box, + #[serde( + default, + with = "crate::json::allow_missing", + skip_serializing_if = "AllowMissing::is_absent" + )] + pub state_key: AllowMissing>, + + /// The `type` field of the event (we use `type_` in Rust to avoid the + /// reserved keyword). + #[serde(rename = "type")] + pub type_: Box, + + /// All other fields that are not required/parsed by the specific/common + /// fields. This allows us to round-trip events that contain extra fields. + /// + /// Generally, optional fields should be handled via `other_fields`, as this + /// saves space when they are not present. However, that does mean we don't + /// do any type-checking until they get used. + #[serde(flatten)] + pub other_fields: HashMap, serde_json::Value>, +} + +impl EventCommonFields { + /// Helper method to check if the event is a state event and return the + /// tuple of `(type, state_key)` if so. + fn type_state_key_tuple(&self) -> Option<(&str, &str)> { + if let AllowMissing::Some(state_key) = &self.state_key { + Some((&self.type_, state_key)) + } else { + None + } + } +} + +/// Type-erased version-specific tail. +/// +/// Used as the default `E` parameter on [`FormattedEvent`] so the +/// pyclass [`Event`](crate::events::Event) can hold any room version +/// behind a single type. The enum is `#[serde(untagged)]` because the +/// discriminator (the room version) lives outside the JSON; in +/// practice the only direction this is serialised in is `Event -> +/// JSON`, where the chosen variant alone determines the shape. +#[derive(Serialize)] +#[serde(untagged)] +pub enum EventFormatEnum { + V1(EventFormatV1), + V2V3(EventFormatV2V3), + V4(EventFormatV4), + VMSC4242(EventFormatVMSC4242), +} + +impl From for EventFormatEnum { + fn from(format: EventFormatV1) -> Self { + EventFormatEnum::V1(format) + } +} + +impl From for EventFormatEnum { + fn from(format: EventFormatV2V3) -> Self { + EventFormatEnum::V2V3(format) + } +} + +impl From for EventFormatEnum { + fn from(format: EventFormatV4) -> Self { + EventFormatEnum::V4(format) + } +} + +impl From for EventFormatEnum { + fn from(format: EventFormatVMSC4242) -> Self { + EventFormatEnum::VMSC4242(format) + } +} diff --git a/rust/src/events/formats/v1.rs b/rust/src/events/formats/v1.rs new file mode 100644 index 0000000000..09caabc597 --- /dev/null +++ b/rust/src/events/formats/v1.rs @@ -0,0 +1,54 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Event format v1 (room versions 1 and 2). +//! +//! Distinguishing features compared to later formats: +//! +//! - `auth_events` and `prev_events` are `[event_id, hashes]` pairs +//! rather than flat lists of IDs. +//! - `event_id` is carried explicitly in the event JSON, rather than +//! being derived from the canonical-JSON hash. +//! - `room_id` is always present. + +use std::{collections::HashMap, sync::Arc}; + +use anyhow::Error; +use serde::{Deserialize, Serialize}; + +use crate::events::formats::EventCommonFields; + +/// Version-specific fields for room versions 1 and 2. +#[derive(Serialize, Deserialize)] +pub struct EventFormatV1 { + pub auth_events: Vec<(String, HashMap)>, + pub prev_events: Vec<(String, HashMap)>, + pub room_id: Arc, + pub event_id: Arc, +} + +impl EventFormatV1 { + pub fn validate(&self, _common_fields: &EventCommonFields) -> Result<(), Error> { + Ok(()) + } + + pub fn auth_event_ids(&self) -> Vec { + self.auth_events.iter().map(|(id, _)| id.clone()).collect() + } + + pub fn prev_event_ids(&self) -> Vec { + self.prev_events.iter().map(|(id, _)| id.clone()).collect() + } +} diff --git a/rust/src/events/formats/v2v3.rs b/rust/src/events/formats/v2v3.rs new file mode 100644 index 0000000000..a62c68a4e4 --- /dev/null +++ b/rust/src/events/formats/v2v3.rs @@ -0,0 +1,60 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Event format v2/v3 (room versions 3 through 10). +//! +//! Differences from v1: +//! +//! - `auth_events` and `prev_events` are flat `Vec` lists of event IDs +//! rather than `[id, hashes]` pairs. +//! - `event_id` is no longer in the event JSON; it is derived from the +//! canonical-JSON hash at parse time. +//! +//! Note that the difference between event format v2 and v3 is purely in the +//! base64 encoding of the event ID, so the same struct can be used for both +//! formats. +//! +//! [`SimpleAuthPrevEvents`] is shared with [`v4`](super::v4) since the +//! flat-list encoding carries forward unchanged. + +use std::sync::Arc; + +use anyhow::{bail, Error}; +use serde::{Deserialize, Serialize}; + +use crate::events::formats::EventCommonFields; + +/// Version-specific fields for room versions 3-10. +#[derive(Serialize, Deserialize)] +pub struct EventFormatV2V3 { + pub room_id: Arc, + pub auth_events: Vec, + pub prev_events: Vec, +} + +impl EventFormatV2V3 { + pub fn validate(&self, common_fields: &EventCommonFields) -> Result<(), Error> { + // Ensure that we don't have an event_id set. + if common_fields.other_fields.contains_key("event_id") { + bail!("v2/v3 events must not have an explicit event_id"); + } + + Ok(()) + } + + pub fn auth_event_ids(&self) -> Vec { + self.auth_events.clone() + } +} diff --git a/rust/src/events/formats/v4.rs b/rust/src/events/formats/v4.rs new file mode 100644 index 0000000000..22a1a677f2 --- /dev/null +++ b/rust/src/events/formats/v4.rs @@ -0,0 +1,156 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Event format v4 (room version 11). +//! +//! The main change from v2/v3 is that `room_id` becomes optional: an +//! `m.room.create` event no longer carries an explicit room ID, and the +//! room ID is instead *derived* from the create event's ID by replacing +//! the leading `$` with `!`. Conversely, every non-create event still +//! has an explicit `room_id`, and the create event is implicitly +//! included in the auth chain of every non-create event (so it does not +//! need to appear in `auth_events`). +//! +//! [`EventFormatV4::validate`] enforces these invariants at parse time; +//! [`EventFormatV4::room_id`] and [`EventFormatV4::auth_event_ids`] +//! expose the derived values to callers. + +use std::sync::Arc; + +use anyhow::{bail, ensure, Error}; +use serde::{Deserialize, Serialize}; + +use crate::{ + events::{constants::event_type::M_ROOM_CREATE, formats::EventCommonFields}, + json::AllowMissing, +}; + +/// Version-specific fields for room version 11. +#[derive(Serialize, Deserialize)] +pub struct EventFormatV4 { + #[serde( + default, + with = "crate::json::allow_missing", + skip_serializing_if = "AllowMissing::is_absent" + )] + pub room_id: AllowMissing>, + pub auth_events: Vec, + pub prev_events: Vec, +} + +impl EventFormatV4 { + pub fn validate(&self, common_fields: &EventCommonFields) -> Result<(), Error> { + // Ensure that we don't have an event_id set. + if common_fields.other_fields.contains_key("event_id") { + bail!("v4 events must not have an explicit event_id"); + } + + Ok(()) + } + + pub fn room_id( + &self, + event_id: &str, + common_fields: &EventCommonFields, + ) -> Result, Error> { + get_room_id_for_optional_room_id(self.room_id.as_ref_opt(), event_id, common_fields) + } + + pub fn auth_event_ids(&self, common_fields: &EventCommonFields) -> Result, Error> { + let is_create = common_fields.type_state_key_tuple() == Some((M_ROOM_CREATE, "")); + + if is_create { + // The create event itself has no implicit auth events. + return Ok(self.auth_events.clone()); + } + + // For non-create events, the create event is implicitly part of + // the auth chain. Derive its event ID from the room ID by + // replacing the leading '!' with '$'. + let room_id = self + .room_id + .as_deref_opt() + .ok_or_else(|| anyhow::anyhow!("non-create event has no room_id"))?; + + let mut create_event_id = String::with_capacity(room_id.len()); + create_event_id.push('$'); + create_event_id.push_str(&room_id[1..]); + + ensure!( + !self.auth_events.contains(&create_event_id), + "The create event ID is implicitly part of the auth chain and should not be explicitly be in the auth_events" + ); + + let mut auth_events = self.auth_events.clone(); + auth_events.push(create_event_id); + Ok(auth_events) + } +} + +/// Validation helper for v4+ events that can have an optional room ID. +/// +/// Returns the validated room ID (which will be `None` for create events). +pub fn validate_optional_room_id( + room_id: Option<&Arc>, + common_fields: &'_ EventCommonFields, +) -> Result>, Error> { + let is_create_event = common_fields.type_state_key_tuple() == Some((M_ROOM_CREATE, "")); + + match (is_create_event, room_id) { + // For non-create events, room_id must be present. + (false, None) => bail!("non-create event must have a room ID"), + (false, Some(room_id)) => { + // We later derive the create event ID from the room ID by replacing + // the leading '!' with '$', so we require the room ID to start with + // '!'. + ensure!( + room_id.starts_with("!"), + "room_id must start with '!': {}", + room_id + ); + + Ok(Some(Arc::clone(room_id))) + } + + // For create events, room_id must be absent. + (true, Some(_)) => bail!("create event must not have a room ID"), + (true, None) => Ok(None), + } +} + +/// Room ID derivation helper for v4+ events, which can have an optional room +/// ID. +pub fn get_room_id_for_optional_room_id( + room_id: Option<&Arc>, + event_id: &str, + common_fields: &EventCommonFields, +) -> Result, Error> { + match validate_optional_room_id(room_id, common_fields)? { + Some(room_id) => Ok(room_id), + None => { + // This is the create event, where the room ID is derived from the + // event ID by replacing the leading '$' with '!'. + if !event_id.starts_with('$') { + bail!("Create event ID does not start with '$': {}", event_id); + } + + let mut room_id = String::with_capacity(event_id.len()); + room_id.push('!'); + room_id.push_str(&event_id[1..]); + + Ok(room_id.into()) + } + } +} diff --git a/rust/src/events/formats/vmsc4242.rs b/rust/src/events/formats/vmsc4242.rs new file mode 100644 index 0000000000..45d9a25068 --- /dev/null +++ b/rust/src/events/formats/vmsc4242.rs @@ -0,0 +1,94 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Event format for [MSC4242] (prev-state events). +//! +//! Adds `prev_state_events` and removes `auth_events` from the v4 layout +//! — auth chains are derived implicitly from the state DAG rather than +//! carried on each event. `room_id`, `prev_events` and the create-event +//! derivation rules carry over unchanged from v4 and are delegated to +//! [`EventFormatV4::validate`] via a shim that supplies an empty +//! explicit auth list. +//! +//! [MSC4242]: https://github.com/matrix-org/matrix-spec-proposals/pull/4242 + +use std::sync::Arc; + +use anyhow::bail; +use anyhow::Error; +use pyo3::exceptions::PyAssertionError; +use pyo3::PyResult; +use serde::{Deserialize, Serialize}; + +use crate::events::constants::event_type::M_ROOM_CREATE; +use crate::events::formats::v4::get_room_id_for_optional_room_id; +use crate::events::formats::EventCommonFields; +use crate::events::Event; +use crate::json::AllowMissing; + +/// Version-specific fields for the MSC4242 event format. +#[derive(Serialize, Deserialize)] +pub struct EventFormatVMSC4242 { + pub prev_state_events: Vec, + pub prev_events: Vec, + #[serde( + default, + with = "crate::json::allow_missing", + skip_serializing_if = "AllowMissing::is_absent" + )] + pub room_id: AllowMissing>, +} + +impl EventFormatVMSC4242 { + pub fn validate(&self, common_fields: &EventCommonFields) -> Result<(), Error> { + // Ensure that we don't have any `auth_events` or `event_id` fields + // set. + if common_fields.other_fields.contains_key("auth_events") { + bail!("MSC4242 events must not have explicit auth_events"); + } + if common_fields.other_fields.contains_key("event_id") { + bail!("MSC4242 events must not have an explicit event_id"); + } + + Ok(()) + } + + pub fn room_id( + &self, + event_id: &str, + common_fields: &EventCommonFields, + ) -> Result, Error> { + get_room_id_for_optional_room_id(self.room_id.as_ref_opt(), event_id, common_fields) + } + + pub fn auth_event_ids(&self, event: &Event) -> PyResult> { + // In the MSC4242 format, the auth events are calculated and stored in + // internal metadata. + let auth_event_ids = event.internal_metadata.get_calculated_auth_event_ids()?; + + // Catches cases where we accidentally call auth_event_ids() prior to calculating what they + // actually are. The exception being the m.room.create event which has no auth events. + if event.parsed_event.common_fields.type_state_key_tuple() != Some((M_ROOM_CREATE, "")) + && auth_event_ids.is_empty() + { + return Err(PyAssertionError::new_err(format!( + "auth_event_ids has not been calculated for event_id='{}'. This is most likely a Synapse programming error.", + event.event_id + ))); + } + + Ok(auth_event_ids) + } +} diff --git a/rust/src/events/internal_metadata.rs b/rust/src/events/internal_metadata.rs index 4084b8442d..93f13f655e 100644 --- a/rust/src/events/internal_metadata.rs +++ b/rust/src/events/internal_metadata.rs @@ -510,7 +510,7 @@ fn attr_err(val: Option, name: &str) -> PyResult { #[pymethods] impl EventInternalMetadata { #[new] - fn new(dict: &Bound<'_, PyDict>) -> PyResult { + pub fn new(dict: &Bound<'_, PyDict>) -> PyResult { let mut data = Vec::with_capacity(dict.len()); for (key, value) in dict.iter() { @@ -536,7 +536,10 @@ impl EventInternalMetadata { }) } - fn copy(&self) -> PyResult { + /// Create a deep copy of this `EventInternalMetadata` to allow modification + /// without affecting other references to the same metadata. This is needed + /// when we clone an event. + pub fn deep_copy(&self) -> PyResult { let guard = self.read_inner()?; Ok(EventInternalMetadata { inner: Arc::new(RwLock::new(guard.clone())), @@ -723,7 +726,7 @@ impl EventInternalMetadata { attr_err(self.read_inner()?.get_redacted(), "redacted") } #[setter] - fn set_redacted(&self, obj: bool) -> PyResult<()> { + pub fn set_redacted(&self, obj: bool) -> PyResult<()> { self.write_inner()?.set_redacted(obj); Ok(()) } @@ -742,7 +745,7 @@ impl EventInternalMetadata { /// The calculated auth event IDs, if it was set when the event was created. #[getter] - fn get_calculated_auth_event_ids(&self) -> PyResult> { + pub fn get_calculated_auth_event_ids(&self) -> PyResult> { let guard = self.read_inner()?; attr_err( guard.get_calculated_auth_event_ids().cloned(), diff --git a/rust/src/events/json_object.rs b/rust/src/events/json_object.rs index 0ab54e8dc5..bb4877d482 100644 --- a/rust/src/events/json_object.rs +++ b/rust/src/events/json_object.rs @@ -193,6 +193,12 @@ impl JsonObject { } } +impl JsonObject { + pub fn get_field(&self, key: &str) -> Option<&serde_json::Value> { + self.object.get(key) + } +} + /// Helper class returned by `JsonObject.keys()` to act as a view into the keys /// of the object. /// diff --git a/rust/src/events/mod.rs b/rust/src/events/mod.rs index e60cdb7078..ad3f61e2fd 100644 --- a/rust/src/events/mod.rs +++ b/rust/src/events/mod.rs @@ -18,18 +18,77 @@ * */ -//! Classes for representing Events. +//! Classes for representing Matrix events. +//! +//! # Overview +//! +//! A Matrix event has a JSON shape that varies by *room version*. The +//! per-room-version shape is captured in the [`formats`] module, where +//! [`FormattedEvent`] is a generic container parametrised by the +//! room-version-specific portion (`EventFormatV1`, `EventFormatV2V3`, +//! `EventFormatV4`, `EventFormatVMSC4242`). See [`formats`] for the layout +//! of the over-the-wire JSON and how the room-version-agnostic fields are +//! split from the version-specific ones. +//! +//! [`Event`] is the `pyclass` exposed to Python. It bundles a fully parsed +//! [`FormattedEvent`] (with the version-specific part type-erased as +//! [`formats::EventFormatEnum`]) together with the pieces of state that +//! live alongside the event JSON in Synapse: +//! +//! - `event_id` — either taken from the event JSON (format v1) or derived +//! from the canonical-JSON hash (v2+); computed once at construction +//! time and cached. +//! - `room_version` — a `'static` reference into the global room-version +//! table, used to drive format-dependent behaviour (e.g. where the +//! `redacts` field lives, which redaction rules apply). +//! - `internal_metadata` — Synapse-internal flags that are *not* part of +//! the federated event (outlier status, soft-failure, stream positions, +//! …). These come from a separate dict at construction time. +//! - `rejected_reason` — `None` for accepted events; otherwise a short +//! string describing why auth rejected the event. +//! +use std::sync::Arc; + +use anyhow::Error; use pyo3::{ - types::{PyAnyMethods, PyMapping, PyModule, PyModuleMethods}, - wrap_pyfunction, Bound, PyResult, Python, + exceptions::{PyAttributeError, PyKeyError, PyValueError}, + pyclass, pyfunction, pymethods, + types::{PyAnyMethods, PyDict, PyDictMethods, PyList, PyMapping, PyModule, PyModuleMethods}, + wrap_pyfunction, Bound, IntoPyObject, PyAny, PyResult, Python, }; +use pythonize::{depythonize, pythonize}; +use crate::events::{ + formats::{ + EventFormatEnum, EventFormatV1, EventFormatV2V3, EventFormatV4, EventFormatVMSC4242, + FormattedEvent, + }, + signatures::Signatures, + unsigned::Unsigned, + utils::redact, +}; +use crate::{ + duration::SynapseDuration, + events::{ + constants::event_field::{HASHES, MSC4354_STICKY, SIGNATURES, UNSIGNED}, + constants::membership_field::MEMBERSHIP, + constants::redaction_field::REDACTS, + constants::unsigned_field::{AGE, AGE_TS, REDACTED_BECAUSE}, + internal_metadata::EventInternalMetadata, + utils::calculate_event_id, + }, + room_versions::{EventFormatVersions, RoomVersion}, +}; + +pub mod constants; pub mod filter; -mod internal_metadata; -mod json_object; +pub mod formats; +pub mod internal_metadata; +pub mod json_object; pub mod signatures; pub mod unsigned; +pub mod utils; use json_object::JsonObject; @@ -39,14 +98,17 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> PyMapping::register::(py)?; let child_module = PyModule::new(py, "events")?; - child_module.add_class::()?; - child_module.add_class::()?; - child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; child_module.add_class::()?; child_module.add_class::()?; child_module.add_class::()?; child_module.add_class::()?; + child_module.add_class::()?; child_module.add_function(wrap_pyfunction!(filter::event_visible_to_server_py, m)?)?; + child_module.add_function(wrap_pyfunction!(redact_event_py, m)?)?; + child_module.add_function(wrap_pyfunction!(redact_event_dict, m)?)?; m.add_submodule(&child_module)?; @@ -58,3 +120,813 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> Ok(()) } + +/// The Rust-side representation of a Matrix event, exposed to Python. +/// +/// Wraps a parsed [`FormattedEvent`] together with the per-event state +/// that Synapse tracks outside the event JSON (event ID, internal +/// metadata, rejection reason, and a reference to the room version that +/// produced this event). See the module-level docs for the high-level +/// design. +#[pyclass(frozen, weakref)] +pub struct Event { + /// The parsed event JSON. + parsed_event: FormattedEvent, + + /// The event ID. For format v1 this is read directly from the JSON; + /// for v2+ it is computed from the canonical-JSON hash at + /// construction time and cached here. + event_id: Arc, + + /// The calculated room ID. + /// + /// For some room versions, this may be derived, e.g. for create events in + /// v4. + room_id: Arc, + + /// Synapse-internal per-event state that lives outside the federated + /// JSON (e.g. outlier flag, soft-failure, stream positions). + #[pyo3(get)] + internal_metadata: EventInternalMetadata, + + /// The room version this event was parsed for. + #[pyo3(get)] + room_version: &'static RoomVersion, + + /// `None` for accepted events; otherwise a short reason set by auth + /// when the event was rejected. + rejected_reason: Option>, +} + +#[pymethods] +impl Event { + #[new] + fn new_from_py<'a, 'py>( + py: Python<'py>, + event_dict: &'a Bound<'py, PyAny>, + room_version: &'a Bound<'py, PyAny>, + internal_metadata_dict: &'a Bound<'py, PyDict>, + rejected_reason: Option, + ) -> PyResult { + let room_version: &RoomVersion = { + let r = room_version.getattr("identifier")?; + let room_version_str = r.extract::<&str>()?; + room_version_str + .parse() + .map_err(|e| PyValueError::new_err(format!("Unsupported room version: {}", e)))? + }; + + let rejected_reason = rejected_reason.map(String::into_boxed_str); + + // Parse the event dict into a FormattedEvent, converting any failures to + // a `ValueError`. + let parsed_event = depythonize_event_dict(room_version, event_dict).map_err(|err| { + let new_err = PyValueError::new_err(format!( + "Failed to parse event for room version {}", + room_version + )); + new_err.set_cause(py, Some(err)); + new_err + })?; + + let internal_metadata = EventInternalMetadata::new(internal_metadata_dict)?; + + let event_id = match &*parsed_event.specific_fields { + EventFormatEnum::V1(format) => { + // V1/V2 events have the event_id in the event dict. + Arc::clone(&format.event_id) + } + _ => { + // Calculate the event ID by hashing the event JSON. This can + // fail if the event can't be serialized to canonical JSON (e.g. + // having out-of-range integers), which we report as + // `ValueError` as it indicates the event is invalid. + let event_value = serde_json::to_value(&parsed_event).map_err(|err| { + PyValueError::new_err(format!("Failed to serialize event: {}", err)) + })?; + calculate_event_id(&event_value, room_version) + .map_err(|err| { + PyValueError::new_err(format!("Failed to calculate event_id: {}", err)) + })? + .into() + } + }; + + let room_id = match &*parsed_event.specific_fields { + EventFormatEnum::V1(format) => Arc::clone(&format.room_id), + EventFormatEnum::V2V3(format) => Arc::clone(&format.room_id), + EventFormatEnum::V4(format) => format + .room_id(&event_id, &parsed_event.common_fields) + .map_err(|err| { + PyValueError::new_err(format!( + "Failed to calculate room_id for event {}: {}", + event_id, err + )) + })?, + EventFormatEnum::VMSC4242(format) => format + .room_id(&event_id, &parsed_event.common_fields) + .map_err(|err| { + PyValueError::new_err(format!( + "Failed to calculate room_id for event {}: {}", + event_id, err + )) + })?, + }; + + Ok(Self { + parsed_event, + + event_id, + room_id, + room_version, + rejected_reason, + internal_metadata, + }) + } + + /// Convert the event to a dictionary suitable for serialisation. + fn get_dict<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(pythonize(py, &self.parsed_event)?) + } + + /// Like `get_dict`, but serializes `unsigned` in a form suitable for + /// persistence. + fn get_dict_for_persistence<'py>(&self, py: Python<'py>) -> PyResult> { + let binding = self.get_dict(py)?; + let dict = binding.cast::()?; + + dict.set_item("unsigned", self.parsed_event.unsigned.for_persistence(py)?)?; + + Ok(binding) + } + + /// Like [`Event::get_dict`], but serializes `unsigned` in a form suitable + /// for sending over federation. + #[pyo3(signature = (time_now = None))] + fn get_pdu_json<'py>( + &self, + py: Python<'py>, + time_now: Option, + ) -> PyResult> { + let obj = self.get_dict(py)?; + let dict = obj.cast::()?; + + // Get or create the unsigned dict + if let Ok(Some(unsigned)) = dict.get_item(UNSIGNED) { + let unsigned = unsigned.cast::()?; + + if let Some(time_now) = time_now { + if let Ok(Some(age_ts)) = unsigned.get_item(AGE_TS) { + let age = time_now - age_ts.extract::()?; + unsigned.set_item(AGE, age)?; + unsigned.del_item(AGE_TS)?; + } + } + + // This may be a frozen event + unsigned.del_item(REDACTED_BECAUSE).ok(); + } + + Ok(obj) + } + + /// Like [`Event::get_dict`], except strips fields like `signatures`, + /// `hashes` and `unsigned` so that the result is suitable as a template for + /// creating new events. Used in make_{join,leave,knock} flows. + fn get_templated_pdu_json<'py>(&self, py: Python<'py>) -> PyResult> { + // Use get_dict but strip signatures, unsigned, and hashes — the + // joining/leaving/knocking server will re-sign and recalculate hashes. + let obj = self.get_dict(py)?; + let dict = obj.cast::()?; + dict.del_item(SIGNATURES).ok(); + dict.del_item(UNSIGNED).ok(); + dict.del_item(HASHES).ok(); + + Ok(obj) + } + + #[getter] + fn rejected_reason(&self) -> Option<&str> { + self.rejected_reason.as_deref() + } + + /// Returns the list of prev event IDs. The order matches the order + /// specified in the event, though there is no meaning to it. + fn prev_event_ids(&self) -> Vec { + match &*self.parsed_event.specific_fields { + EventFormatEnum::V1(format) => format.prev_event_ids(), + EventFormatEnum::V2V3(format) => format.prev_events.clone(), + EventFormatEnum::V4(format) => format.prev_events.clone(), + EventFormatEnum::VMSC4242(format) => format.prev_events.clone(), + } + } + + /// Returns the list of auth event IDs. The order matches the order + /// specified in the event, though there is no meaning to it. + fn auth_event_ids(&self) -> PyResult> { + match &*self.parsed_event.specific_fields { + EventFormatEnum::V1(format) => Ok(format.auth_event_ids()), + EventFormatEnum::V2V3(format) => Ok(format.auth_event_ids()), + EventFormatEnum::V4(format) => { + Ok(format.auth_event_ids(&self.parsed_event.common_fields)?) + } + EventFormatEnum::VMSC4242(format) => Ok(format.auth_event_ids(self)?), + } + } + + #[getter] + fn membership<'py>(&self, py: Python<'py>) -> PyResult> { + let content = self.content(); + let value = content.get_field(MEMBERSHIP); + match value { + Some(value) => Ok(pythonize(py, value)?), + None => Err(PyKeyError::new_err(MEMBERSHIP)), + } + } + + fn is_state(&self) -> bool { + self.parsed_event.common_fields.state_key.is_some() + } + + /// Get the state key of this event, or None if it's not a state event. + fn get_state_key(&self) -> Option<&str> { + self.parsed_event.common_fields.state_key.as_deref_opt() + } + + /// The EventFormatVersion implemented by this event. + #[getter] + fn format_version(&self) -> i32 { + self.room_version.event_format + } + + /// Returns a deep copy of this object, such that modifying the copy will + /// not affect the original. + fn deep_copy(&self) -> PyResult { + let internal_metadata = self.internal_metadata.deep_copy()?; + + let new_event = Event { + parsed_event: self.parsed_event.deep_copy(), + internal_metadata, + room_version: self.room_version, + rejected_reason: self.rejected_reason.clone(), + event_id: self.event_id.clone(), + room_id: self.room_id.clone(), + }; + Ok(new_event) + } + + /// If this event has the `msc4354_sticky` top-level field, returns a + /// `SynapseDuration` representing the sticky duration. Otherwise returns + /// `None`. + fn sticky_duration(&self) -> Option { + const MAX_DURATION: SynapseDuration = SynapseDuration::from_hours(1); + + let sticky_obj = self + .parsed_event + .common_fields + .other_fields + .get(MSC4354_STICKY); + + let sticky_obj = match sticky_obj { + Some(serde_json::Value::Object(obj)) => obj, + _ => return None, + }; + + // Check for a valid duration field. The MSC requires `duration_ms` to + // be a non-negative integer. If it's missing or invalid, we treat the + // event as non-sticky by returning `None`. + let duration_ms = sticky_obj.get("duration_ms")?.as_u64()?; + + let duration = SynapseDuration::from_milliseconds(duration_ms); + + let duration = std::cmp::min(duration, MAX_DURATION); + + Some(duration) + } + + // Below are the methods for interacting with the event as a mapping. + // + // These are rarely used, so we take the easy approach of re-serializing the + // event to a Python dict and then delegating to the standard dict methods. + // We can't remove these functions as third-party modules may rely on them. + + fn __contains__<'py>(&self, py: Python<'py>, key: &str) -> PyResult { + let dict = self.get_dict(py)?; + dict.contains(key) + } + + /// This is deprecated in favor of `get`, but we still need to support it + /// for backwards compatibility with modules. This is therefore not exposed + /// in the type stubs. + fn __getitem__<'py>(&self, py: Python<'py>, key: &str) -> PyResult> { + let dict = self.get_dict(py)?; + if dict.contains(key)? { + dict.get_item(key) + } else { + Err(PyKeyError::new_err(key.to_owned())) + } + } + + #[pyo3(signature = (key, default=None))] + fn get<'py>( + &self, + py: Python<'py>, + key: &str, + default: Option>, + ) -> PyResult> { + let dict = self.get_dict(py)?; + if dict.contains(key)? { + dict.get_item(key) + } else { + Ok(default.into_pyobject(py)?) + } + } + + fn items<'py>(&self, py: Python<'py>) -> PyResult> { + let dict = self.get_dict(py)?; + let dict = dict.cast::()?; + Ok(dict.items()) + } + + fn keys<'py>(&self, py: Python<'py>) -> PyResult> { + let dict = self.get_dict(py)?; + let dict = dict.cast::()?; + Ok(dict.keys()) + } + + // Below are the getters for the top-level fields on Matrix events. + + #[getter] + fn event_id(&self) -> &str { + &self.event_id + } + + #[getter] + fn room_id(&self) -> &str { + &self.room_id + } + + #[getter] + fn signatures(&self) -> Signatures { + self.parsed_event.signatures.clone() + } + + #[getter] + fn content(&self) -> JsonObject { + self.parsed_event.common_fields.content.clone() + } + + #[getter] + fn depth(&self) -> i64 { + self.parsed_event.common_fields.depth + } + + #[getter] + fn hashes<'py>(&self, py: Python<'py>) -> PyResult> { + let dict = PyDict::new(py); + for (key, value) in &self.parsed_event.common_fields.hashes { + dict.set_item(&**key, &**value)?; + } + Ok(dict) + } + + #[getter] + fn origin_server_ts(&self) -> i64 { + self.parsed_event.common_fields.origin_server_ts + } + + #[getter] + fn sender(&self) -> &str { + &self.parsed_event.common_fields.sender + } + + /// Deprecated alias for `sender`. Kept for backwards compatibility with + /// modules and tests that still read `event.user_id`. This is therefore not + /// exposed in the type stubs. + #[getter] + fn user_id(&self) -> &str { + &self.parsed_event.common_fields.sender + } + + #[getter(state_key)] + // We can't call this `state_key` because that would generate a + // `get_state_key` method which already exists. + fn state_key_attr(&self) -> PyResult<&str> { + let Some(state_key) = self.parsed_event.common_fields.state_key.as_deref_opt() else { + return Err(PyAttributeError::new_err("state_key")); + }; + Ok(state_key) + } + + #[getter] + fn r#type(&self) -> &str { + &self.parsed_event.common_fields.type_ + } + + #[getter] + fn unsigned(&self) -> Unsigned { + self.parsed_event.unsigned.clone() + } + + #[getter] + fn prev_state_events(&self) -> PyResult> { + // `prev_state_events` should only be called after validating the event + // is of a format that supports MSC4242, so we return an AttributeError + // for formats that don't support it. + match &*self.parsed_event.specific_fields { + EventFormatEnum::V1(_) | EventFormatEnum::V2V3(_) | EventFormatEnum::V4(_) => { + Err(PyAttributeError::new_err("prev_state_events")) + } + EventFormatEnum::VMSC4242(format) => Ok(format.prev_state_events.clone()), + } + } + + #[getter] + fn redacts<'py>(&self, py: Python<'py>) -> PyResult>> { + let common = &self.parsed_event.common_fields; + let value = if self.room_version.updated_redaction_rules { + common.content.get_field(REDACTS) + } else { + common.other_fields.get(REDACTS) + }; + value + .map(|v| pythonize(py, v).map_err(Into::into)) + .transpose() + } +} + +fn depythonize_event_dict( + room_version: &RoomVersion, + event_dict: &Bound<'_, PyAny>, +) -> PyResult { + let formatted_event: FormattedEvent = match room_version.event_format { + EventFormatVersions::ROOM_V1_V2 => { + let event_format: FormattedEvent = depythonize(event_dict)?; + + event_format.into() + } + EventFormatVersions::ROOM_V3 | EventFormatVersions::ROOM_V4_PLUS => { + let event_format: FormattedEvent = depythonize(event_dict)?; + event_format.into() + } + EventFormatVersions::ROOM_V11_HYDRA_PLUS => { + let event_format: FormattedEvent = depythonize(event_dict)?; + event_format.into() + } + EventFormatVersions::ROOM_VMSC4242 => { + let event_format: FormattedEvent = depythonize(event_dict)?; + event_format.into() + } + _ => { + return Err(PyValueError::new_err(format!( + "Unsupported room version: {}", + room_version + ))) + } + }; + + formatted_event.validate()?; + + Ok(formatted_event) +} + +/// Converts an event dict as [`serde_json::Value`] into a [`FormattedEvent`]. +fn event_dict_from_json_value( + room_version: &RoomVersion, + event_dict: serde_json::Value, +) -> Result { + let formatted_event: FormattedEvent = match room_version.event_format { + EventFormatVersions::ROOM_V1_V2 => { + let event_format: FormattedEvent = serde_json::from_value(event_dict)?; + + event_format.into() + } + EventFormatVersions::ROOM_V3 | EventFormatVersions::ROOM_V4_PLUS => { + let event_format: FormattedEvent = serde_json::from_value(event_dict)?; + event_format.into() + } + EventFormatVersions::ROOM_V11_HYDRA_PLUS => { + let event_format: FormattedEvent = serde_json::from_value(event_dict)?; + event_format.into() + } + EventFormatVersions::ROOM_VMSC4242 => { + let event_format: FormattedEvent = + serde_json::from_value(event_dict)?; + event_format.into() + } + _ => { + return Err(anyhow::anyhow!( + "Unsupported room version: {}", + room_version + )); + } + }; + + formatted_event.validate()?; + + Ok(formatted_event) +} + +/// Returns a pruned version of the given event, which removes all keys we don't +/// know about or think could potentially be dodgy. +/// +/// Returns the redacted event as a dict. +#[pyfunction(name = "redact_event")] +fn redact_event_py(event: &Event) -> PyResult { + let event_value = serde_json::to_value(&event.parsed_event).map_err(|err| { + PyValueError::new_err(format!("Failed to serialize event for redaction: {}", err)) + })?; + + let redacted_value = redact(&event_value, event.room_version)?; + let redacted_formatted_event = event_dict_from_json_value(event.room_version, redacted_value) + .map_err(|err| { + PyValueError::new_err(format!("Failed to deserialize redacted event: {}", err)) + })?; + + let redacted_event = Event { + parsed_event: redacted_formatted_event, + event_id: Arc::clone(&event.event_id), + room_id: Arc::clone(&event.room_id), + room_version: event.room_version, + rejected_reason: event.rejected_reason.clone(), + internal_metadata: event.internal_metadata.deep_copy()?, + }; + + // Mark event as redacted + redacted_event.internal_metadata.set_redacted(true)?; + + Ok(redacted_event) +} + +/// Returns a pruned version of the given event dict, which removes all keys we +/// don't know about or think could potentially be dodgy. +/// +/// Returns the redacted event as a dict. +#[pyfunction(name = "redact_event_dict")] +fn redact_event_dict<'py>( + py: Python<'py>, + room_version: &RoomVersion, + event_dict: &'py Bound<'py, PyAny>, +) -> PyResult> { + let event_value = depythonize(event_dict)?; + + let redacted = redact(&event_value, room_version)?; + + let redacted_py = pythonize(py, &redacted)?; + + Ok(redacted_py) +} + +#[cfg(test)] +mod tests { + + use super::*; + use crate::events::{ + constants::event_type::M_ROOM_CREATE, + formats::{EventFormatV1, EventFormatV2V3, EventFormatV4, EventFormatVMSC4242}, + }; + + #[test] + fn test_basic_v3_roundtrip() { + let json = r#"{"auth_events":[],"prev_events":[],"type":"m.room.create","sender":"@anon-20260225_142731-20:localhost:8800","content":{"room_version":"10","creator":"@anon-20260225_142731-20:localhost:8800"},"depth":1,"room_id":"!qVoJSympOqdUQRUfiC:localhost:8800","state_key":"","origin_server_ts":1772029657149,"hashes":{"sha256":"RIDkn4CrExGMOfRZlHl//1weAro5QC/q2D76YcyAUqk"},"signatures":{"localhost:8800":{"ed25519:a_GMSl":"GU7WmvI2Kd5kLrXKrWpRbUfEiVKGgH0sxQNEpBMMvgF3QhHN25AubVMmIClht5r/c+Iihb1xsq1j5Sw+RGfiDg"}},"unsigned":{"age_ts":1772029657149}}"#; + let event_value: serde_json::Value = serde_json::from_str(json).unwrap(); + + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + let parsed_value = serde_json::to_value(&event).unwrap(); + + // Check a couple of fields are as expected as a sanity check. + assert_eq!(&*event.common_fields.type_, M_ROOM_CREATE); + assert_eq!( + &*event.specific_fields.room_id, + "!qVoJSympOqdUQRUfiC:localhost:8800" + ); + + assert_eq!(event_value, parsed_value); + } + + #[test] + fn test_room_id_for_create_event_format_v4() { + let json = r#"{"auth_events":[],"prev_events":[],"type":"m.room.create","sender":"@erikj:jki.re","content":{"room_version":"12","predecessor":{"room_id":"!VuNGkDTdbMOOxSmuDa:jki.re"}},"depth":1,"state_key":"","origin_server_ts":1775568141481,"hashes":{"sha256":"qBX+glsKvogXFrvsEN0eh13pO2kpuE6o/b4yREPtOqw"},"signatures":{"jki.re":{"ed25519:auto":"n/4gHQRagk3r1r24L/7a+oaMMf9cysVfQRYdjpDZcf4ppkVym33rhTW18Vy4zMa1L5nsWLkxsBvbrRRDYUOhBQ"}},"unsigned":{"age_ts":1775568141481}}"#; + let event_value: serde_json::Value = serde_json::from_str(json).unwrap(); + + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + + let event_id = calculate_event_id(&event_value, &RoomVersion::V12).unwrap(); + + assert_eq!( + &*event + .specific_fields + .room_id(&event_id, &event.common_fields) + .unwrap(), + "!BeXKh925K_M46DwsuJFR0EyBpE1P7CFUDGuWW4xw55Y" + ); + } + + #[test] + fn test_basic_v1_roundtrip() { + let json = r#"{"auth_events":[["$auth1:localhost",{"sha256":"abc"}],["$auth2:localhost",{"sha256":"def"}]],"prev_events":[["$prev1:localhost",{"sha256":"ghi"}]],"type":"m.room.message","sender":"@user:localhost","content":{"body":"hello","msgtype":"m.text"},"depth":5,"room_id":"!room:localhost","event_id":"$event1:localhost","origin_server_ts":1234567890,"hashes":{"sha256":"base64hash"},"signatures":{"localhost":{"ed25519:key":"sig"}},"unsigned":{}}"#; + let event_value: serde_json::Value = serde_json::from_str(json).unwrap(); + + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + let parsed_value = serde_json::to_value(&event).unwrap(); + + // Check a few fields are as expected as a sanity check. + assert_eq!(&*event.common_fields.type_, "m.room.message"); + assert!(event.common_fields.state_key.is_absent()); + assert_eq!(&*event.specific_fields.room_id, "!room:localhost"); + assert_eq!(&*event.specific_fields.event_id, "$event1:localhost"); + + // Check auth/prev event extraction + let auth_ids = event.specific_fields.auth_event_ids(); + assert_eq!(auth_ids, vec!["$auth1:localhost", "$auth2:localhost"]); + + let prev_ids = event.specific_fields.prev_event_ids(); + assert_eq!(prev_ids, vec!["$prev1:localhost"]); + + assert_eq!(event_value, parsed_value); + } + + #[test] + fn test_basic_v4_roundtrip_with_room_id() { + // A regular (non-create) V4 event has an explicit room_id. + let json = r#"{"auth_events":["$auth1","$auth2"],"prev_events":["$prev1"],"type":"m.room.message","sender":"@user:localhost","content":{"body":"hello","msgtype":"m.text"},"depth":5,"room_id":"!room:localhost","origin_server_ts":1234567890,"hashes":{"sha256":"base64hash"},"signatures":{"localhost":{"ed25519:key":"sig"}},"unsigned":{}}"#; + let event_value: serde_json::Value = serde_json::from_str(json).unwrap(); + + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + let parsed_value = serde_json::to_value(&event).unwrap(); + + // Check a few fields are as expected as a sanity check. + assert_eq!(&*event.common_fields.type_, "m.room.message"); + assert_eq!( + event.specific_fields.room_id.as_deref_opt(), + Some("!room:localhost") + ); + assert_eq!( + event.specific_fields.auth_events, + vec!["$auth1".to_string(), "$auth2".to_string()] + ); + assert_eq!( + event.specific_fields.prev_events, + vec!["$prev1".to_string()] + ); + + assert_eq!(event_value, parsed_value); + } + + #[test] + fn test_basic_v4_roundtrip_create_event() { + // A V4 create event for a v12 room has no room_id field. + let json = r#"{"auth_events":[],"prev_events":[],"type":"m.room.create","sender":"@erikj:jki.re","content":{"room_version":"12"},"depth":1,"state_key":"","origin_server_ts":1775568141481,"hashes":{"sha256":"qBX+glsKvogXFrvsEN0eh13pO2kpuE6o/b4yREPtOqw"},"signatures":{"jki.re":{"ed25519:auto":"sig"}},"unsigned":{}}"#; + let event_value: serde_json::Value = serde_json::from_str(json).unwrap(); + + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + let parsed_value = serde_json::to_value(&event).unwrap(); + + // Check a few fields are as expected as a sanity check. + assert!(event.specific_fields.room_id.is_absent()); + assert_eq!(&*event.common_fields.type_, M_ROOM_CREATE); + + // Create events have no implicit auth events. + assert!(event + .specific_fields + .auth_event_ids(&event.common_fields) + .unwrap() + .is_empty()); + + assert_eq!(event_value, parsed_value); + } + + #[test] + fn test_v4_auth_event_ids_implicit_create() { + // Non-create events implicitly include the create event (derived from + // the room ID) in their auth chain. + let json = r#"{"auth_events":["$auth1"],"prev_events":["$prev1"],"type":"m.room.message","sender":"@user:localhost","content":{"body":"hi","msgtype":"m.text"},"depth":5,"room_id":"!BeXKh925K_M46DwsuJFR0EyBpE1P7CFUDGuWW4xw55Y","origin_server_ts":1234567890,"hashes":{"sha256":"h"},"signatures":{"localhost":{"ed25519:k":"s"}},"unsigned":{}}"#; + + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + + let auth_ids = event + .specific_fields + .auth_event_ids(&event.common_fields) + .unwrap(); + assert_eq!( + auth_ids, + vec![ + "$auth1".to_string(), + "$BeXKh925K_M46DwsuJFR0EyBpE1P7CFUDGuWW4xw55Y".to_string(), + ] + ); + } + + #[test] + fn test_v4_validate_rejects_missing_room_id_for_non_create() { + // A v12 non-create event without a room_id must fail validation. + let json = r#"{"auth_events":[],"prev_events":[],"type":"m.room.message","sender":"@u:l","content":{},"depth":2,"state_key":"","origin_server_ts":1,"hashes":{"sha256":"h"},"signatures":{"l":{"ed25519:k":"s"}},"unsigned":{}}"#; + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + assert!(event + .specific_fields + .validate(&event.common_fields) + .is_err()); + } + + #[test] + fn test_v4_validate_accepts_create_without_room_id() { + let json = r#"{"auth_events":[],"prev_events":[],"type":"m.room.create","sender":"@u:l","content":{"room_version":"12"},"depth":1,"state_key":"","origin_server_ts":1,"hashes":{"sha256":"h"},"signatures":{"l":{"ed25519:k":"s"}},"unsigned":{}}"#; + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + event + .specific_fields + .validate(&event.common_fields) + .unwrap(); + } + + #[test] + fn test_basic_vmsc4242_roundtrip() { + // VMSC4242 introduces a `prev_state_events` field on top of V4. + let json = r#"{"auth_events":["$auth1"],"prev_events":["$prev1"],"prev_state_events":["$pstate1","$pstate2"],"type":"m.room.member","sender":"@user:localhost","content":{"membership":"join"},"depth":5,"room_id":"!room:localhost","state_key":"@user:localhost","origin_server_ts":1234567890,"hashes":{"sha256":"h"},"signatures":{"localhost":{"ed25519:k":"s"}},"unsigned":{}}"#; + let event_value: serde_json::Value = serde_json::from_str(json).unwrap(); + + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + let parsed_value = serde_json::to_value(&event).unwrap(); + + assert_eq!( + event.specific_fields.prev_state_events, + vec!["$pstate1".to_string(), "$pstate2".to_string()] + ); + assert_eq!( + event.specific_fields.room_id.as_deref_opt(), + Some("!room:localhost") + ); + assert_eq!( + event.common_fields.state_key.as_deref_opt(), + Some("@user:localhost") + ); + + assert_eq!(event_value, parsed_value); + } + + #[test] + fn test_vmsc4242_room_id_for_create_event() { + let json = r#"{"auth_events":[],"prev_events":[],"prev_state_events":[],"type":"m.room.create","sender":"@erikj:jki.re","content":{"room_version":"12","predecessor":{"room_id":"!VuNGkDTdbMOOxSmuDa:jki.re"}},"depth":1,"state_key":"","origin_server_ts":1775568141481,"hashes":{"sha256":"qBX+glsKvogXFrvsEN0eh13pO2kpuE6o/b4yREPtOqw"},"signatures":{"jki.re":{"ed25519:auto":"n/4gHQRagk3r1r24L/7a+oaMMf9cysVfQRYdjpDZcf4ppkVym33rhTW18Vy4zMa1L5nsWLkxsBvbrRRDYUOhBQ"}},"unsigned":{"age_ts":1775568141481}}"#; + let event_value: serde_json::Value = serde_json::from_str(json).unwrap(); + + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + + // The event_id calculation is independent of the `prev_state_events` + // field not being present in V4, so the same event_id derivation works. + let event_id = calculate_event_id(&event_value, &RoomVersion::V12).unwrap(); + + assert_eq!( + &*event + .specific_fields + .room_id(&event_id, &event.common_fields) + .unwrap(), + "!BeXKh925K_M46DwsuJFR0EyBpE1P7CFUDGuWW4xw55Y" + ); + } + + #[test] + fn test_event_format_enum_untagged_roundtrip() { + // The untagged EventFormatEnum serialization/deserialization is + // driven by fields, so serializing any variant must match the + // original JSON exactly. + let v2v3_json = r#"{"auth_events":[],"prev_events":[],"type":"m.room.create","sender":"@a:b","content":{},"depth":1,"room_id":"!r:b","state_key":"","origin_server_ts":1,"hashes":{"sha256":"h"},"signatures":{"b":{"ed25519:k":"s"}},"unsigned":{}}"#; + let v2v3_value: serde_json::Value = serde_json::from_str(v2v3_json).unwrap(); + let v2v3_container: FormattedEvent = + serde_json::from_str(v2v3_json).unwrap(); + assert_eq!(serde_json::to_value(&v2v3_container).unwrap(), v2v3_value); + assert_eq!( + serde_json::to_value(v2v3_container.into_general()).unwrap(), + v2v3_value + ); + + let v4_json = r#"{"auth_events":[],"prev_events":[],"type":"m.room.create","sender":"@a:b","content":{"room_version":"12"},"depth":1,"state_key":"","origin_server_ts":1,"hashes":{"sha256":"h"},"signatures":{"b":{"ed25519:k":"s"}},"unsigned":{}}"#; + let v4_value: serde_json::Value = serde_json::from_str(v4_json).unwrap(); + let v4_container: FormattedEvent = serde_json::from_str(v4_json).unwrap(); + assert_eq!(serde_json::to_value(&v4_container).unwrap(), v4_value); + assert_eq!( + serde_json::to_value(v4_container.into_general()).unwrap(), + v4_value + ); + } + + #[test] + fn test_unknown_top_level_fields_preserved_roundtrip() { + // Extra top-level fields (e.g. unknown or experimental) are captured + // via `other_fields` and must round-trip losslessly. + let json = r#"{"auth_events":[],"prev_events":[],"type":"m.room.message","sender":"@a:b","content":{"body":"hi","msgtype":"m.text"},"depth":1,"room_id":"!r:b","origin_server_ts":1,"hashes":{"sha256":"h"},"signatures":{"b":{"ed25519:k":"s"}},"unsigned":{},"msc4354_sticky":{"duration_ms":5000},"some_unknown_field":"some_value"}"#; + let event_value: serde_json::Value = serde_json::from_str(json).unwrap(); + + let event: FormattedEvent = serde_json::from_str(json).unwrap(); + let parsed_value = serde_json::to_value(&event).unwrap(); + + assert!(event + .common_fields + .other_fields + .contains_key(MSC4354_STICKY)); + assert!(event + .common_fields + .other_fields + .contains_key("some_unknown_field")); + + assert_eq!(event_value, parsed_value); + } +} diff --git a/rust/src/events/signatures.rs b/rust/src/events/signatures.rs index 0f2acd5c9b..fa07dd056b 100644 --- a/rust/src/events/signatures.rs +++ b/rust/src/events/signatures.rs @@ -30,12 +30,25 @@ use serde::{Deserialize, Serialize}; /// A class representing the signatures on an event. #[pyclass(frozen, skip_from_py_object)] -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] #[serde(transparent)] pub struct Signatures { inner: Arc>>>, } +impl Signatures { + /// Create a deep copy of this `Signatures` to allow modification without + /// affecting other references to the same signatures. This is needed when + /// we clone an event. + pub fn deep_copy(&self) -> Self { + let signatures = self.inner.read().expect("lock poisoned").clone(); // Deep copy the inner map + + Self { + inner: Arc::new(RwLock::new(signatures)), + } + } +} + #[pymethods] impl Signatures { #[new] diff --git a/rust/src/events/unsigned.rs b/rust/src/events/unsigned.rs index 5aa56812c0..0bb644ee90 100644 --- a/rust/src/events/unsigned.rs +++ b/rust/src/events/unsigned.rs @@ -101,6 +101,15 @@ impl Unsigned { .write() .map_err(|_| PyRuntimeError::new_err("Unsigned lock poisoned")) } + + /// Create a deep copy of this `Unsigned` to allow modification without + /// affecting other references to the same unsigned data. This is needed + /// when we clone an event. + pub fn deep_copy(&self) -> Self { + Self { + inner: Arc::new(RwLock::new(self.py_read().expect("lock poisoned").clone())), + } + } } #[pymethods] @@ -268,11 +277,11 @@ impl Unsigned { } } - fn for_persistence<'py>(&self, py: Python<'py>) -> PyResult> { + pub fn for_persistence<'py>(&self, py: Python<'py>) -> PyResult> { Ok(pythonize(py, &self.py_read()?.persisted_fields)?) } - fn for_event<'py>(&self, py: Python<'py>) -> PyResult> { + pub fn for_event<'py>(&self, py: Python<'py>) -> PyResult> { Ok(pythonize(py, &*self.py_read()?)?) } } diff --git a/rust/src/events/utils.rs b/rust/src/events/utils.rs new file mode 100644 index 0000000000..7d33c52b44 --- /dev/null +++ b/rust/src/events/utils.rs @@ -0,0 +1,1191 @@ +//! JSON +//! +//! Matrix event JSON utility functions. + +use std::collections::BTreeSet; + +use anyhow::Context as _; +use base64::Engine as _; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use super::constants::{ + aliases_field, create_field, + event_field::{ + AUTH_EVENTS, CONTENT, DEPTH, EVENT_ID, HASHES, MEMBERSHIP, ORIGIN, ORIGIN_SERVER_TS, + PREV_EVENTS, PREV_STATE, REPLACES_STATE, ROOM_ID, SENDER, SIGNATURES, STATE_KEY, TYPE, + UNSIGNED, + }, + event_type::{ + M_ROOM_ALIASES, M_ROOM_CREATE, M_ROOM_HISTORY_VISIBILITY, M_ROOM_JOIN_RULES, M_ROOM_MEMBER, + M_ROOM_POWER_LEVELS, M_ROOM_REDACTION, + }, + history_visibility_field, join_rules_field, membership_field, power_levels_field, + redaction_field, + unsigned_field::AGE_TS, +}; +use crate::{ + canonical_json::CanonicalizationOptions, events::constants::event_field::PREV_STATE_EVENTS, +}; +use crate::{ + events::constants::event_field::M_RELATES_TO, + room_versions::{EventFormatVersions, RoomVersion}, +}; + +/// Calculates the event_id of an event. +/// +/// The event_id is the `reference_hash` of the redacted event json, preceded by a `$`. +/// `calculate_event_id` can be used to determine the `event_id` for events in room versions V3+. +pub fn calculate_event_id(event: &Value, room_version: &RoomVersion) -> anyhow::Result> { + match room_version.event_format { + EventFormatVersions::ROOM_V1_V2 => { + anyhow::bail!( + "Attempted to calculate event_id using reference hash for room version v1/v2" + ); + } + EventFormatVersions::ROOM_V3 + | EventFormatVersions::ROOM_V4_PLUS + | EventFormatVersions::ROOM_V11_HYDRA_PLUS + | EventFormatVersions::ROOM_VMSC4242 => { + let reference_hash = compute_event_reference_hash(event, room_version)?; + + Ok(format!("${reference_hash}").into_boxed_str()) + } + _ => { + unimplemented!( + "Unknown event format version {}. This is a Synapse Programming error.", + room_version.event_format + ); + } + } +} + +/// Computes the event reference hash. This is the hash of the redacted event. +pub fn compute_event_reference_hash( + event: &Value, + room_version: &RoomVersion, +) -> anyhow::Result { + let mut redacted_value = redact(event, room_version)?; + + let redacted_value_mut = redacted_value + .as_object_mut() + .context("Failed getting `redacted_value` as mutable object")?; + + redacted_value_mut.remove(SIGNATURES); + redacted_value_mut.remove(UNSIGNED); + redacted_value_mut.remove(AGE_TS); + + let canonicalization_options = if room_version.strict_canonicaljson { + CanonicalizationOptions::strict() + } else { + CanonicalizationOptions::relaxed() + }; + + let json = + crate::canonical_json::to_string_canonical(&redacted_value_mut, canonicalization_options) + .map_err(|err| anyhow::anyhow!(err))?; + + let hash = Sha256::digest(json.as_bytes()); + + let base64_alphabet = if room_version.event_format == EventFormatVersions::ROOM_V3 { + base64::alphabet::STANDARD + } else { + base64::alphabet::URL_SAFE + }; + let base64_engine = base64::engine::GeneralPurpose::new( + &base64_alphabet, + base64::engine::general_purpose::NO_PAD, + ); + + Ok(base64_engine.encode(hash)) +} + +/// Attempts to redact the provided event, returning a copy of the redacted +/// event if successful. +/// +/// Events redacted with this function are meant to be sent over federation. +pub fn redact(event: &Value, room_version: &RoomVersion) -> anyhow::Result { + let mut allowed_keys = BTreeSet::from([ + (EVENT_ID), + (SENDER), + (ROOM_ID), + (HASHES), + (SIGNATURES), + (CONTENT), + (TYPE), + (STATE_KEY), + (DEPTH), + (PREV_EVENTS), + (ORIGIN_SERVER_TS), + ]); + + // Earlier room versions had additional allowed keys + if !room_version.updated_redaction_rules { + allowed_keys.extend([PREV_STATE, MEMBERSHIP, ORIGIN]); + } + + // Room versions with MSC4242 have `prev_state_events` instead of + // `auth_events`. + if room_version.msc4242_state_dags { + allowed_keys.insert(PREV_STATE_EVENTS); + } else { + allowed_keys.insert(AUTH_EVENTS); + } + + let event_type = event + .get(TYPE) + .with_context(|| format!("Missing {TYPE} field in json"))? + .as_str() + .with_context(|| format!("{TYPE} field is not a string"))?; + + let event_content = event + .get(CONTENT) + .with_context(|| format!("Missing {CONTENT} field in json"))?; + + let mut new_content = serde_json::json!({}); + let new_content_mut = new_content + .as_object_mut() + .context("Failed getting `new_content` as mutable object")?; + + let mut add_content_field = |field: &str| { + if let Some(existing_field) = event_content.get(field) { + new_content_mut.insert(field.to_string(), existing_field.clone()); + } + }; + + match event_type { + M_ROOM_MEMBER => { + add_content_field(membership_field::MEMBERSHIP); + if room_version.restricted_join_rule_fix { + add_content_field(membership_field::JOIN_AUTHORISED_VIA_USERS_SERVER); + } + if room_version.updated_redaction_rules { + // Preserve the signed field under third_party_invite. + if let Some(third_party_invite) = + event_content.get(membership_field::THIRD_PARTY_INVITE) + { + if third_party_invite.as_object().is_some() { + let mut new_third_party_invite = serde_json::json!({}); + if let Some(signed) = third_party_invite.get(membership_field::SIGNED) { + new_third_party_invite = + serde_json::json!({membership_field::SIGNED: signed.clone()}); + } + new_content_mut.insert( + membership_field::THIRD_PARTY_INVITE.to_string(), + new_third_party_invite, + ); + } + } + } + } + M_ROOM_CREATE => { + if room_version.updated_redaction_rules { + // MSC2176 rules state that create events cannot have their `content` redacted. + if let Some(event_content_object) = event_content.as_object() { + for (field, _value) in event_content_object { + add_content_field(field); + } + } + } + if !room_version.implicit_room_creator { + // Some room versions give meaning to `creator` + add_content_field(create_field::CREATOR); + } + if room_version.msc4291_room_ids_as_hashes { + // room_id is not allowed on the create event as it's derived from the event ID + allowed_keys.remove(ROOM_ID); + } + } + M_ROOM_JOIN_RULES => { + add_content_field(join_rules_field::JOIN_RULE); + if room_version.restricted_join_rule { + add_content_field(join_rules_field::ALLOW); + } + } + M_ROOM_POWER_LEVELS => { + add_content_field(power_levels_field::USERS); + add_content_field(power_levels_field::USERS_DEFAULT); + add_content_field(power_levels_field::EVENTS); + add_content_field(power_levels_field::EVENTS_DEFAULT); + add_content_field(power_levels_field::STATE_DEFAULT); + add_content_field(power_levels_field::BAN); + add_content_field(power_levels_field::KICK); + add_content_field(power_levels_field::REDACT); + if room_version.updated_redaction_rules { + add_content_field(power_levels_field::INVITE); + } + } + M_ROOM_ALIASES if room_version.special_case_aliases_auth => { + add_content_field(aliases_field::ALIASES); + } + M_ROOM_HISTORY_VISIBILITY => { + add_content_field(history_visibility_field::HISTORY_VISIBILITY) + } + M_ROOM_REDACTION if room_version.updated_redaction_rules => { + add_content_field(redaction_field::REDACTS); + } + _ => (), + }; + + let mut allowed_fields = serde_json::json!({}); + let allowed_fields_mut = allowed_fields + .as_object_mut() + .context("Failed getting `allowed_fields` as mutable object")?; + + for (k, v) in event + .as_object() + .context("Event is not a JSON object")? + .iter() + { + if allowed_keys.contains(&k.as_str()) { + allowed_fields_mut.insert(k.clone(), v.clone()); + } + } + + if room_version.msc3389_relation_redactions { + if let Some(relates_to) = event + .get(CONTENT) + .and_then(|content| content.get(M_RELATES_TO)) + { + if relates_to.is_object() { + let mut new_relates_to = serde_json::json!({}); + let new_relates_to_mut = new_relates_to + .as_object_mut() + .context("Failed getting `new_relates_to` as mutable object")?; + + for field in ["rel_type", "event_id"] { + if let Some(value) = relates_to.get(field) { + new_relates_to_mut.insert(field.to_string(), value.clone()); + } + } + + if !new_relates_to_mut.is_empty() { + new_content_mut.insert(M_RELATES_TO.to_string(), new_relates_to); + } + } + } + } + + allowed_fields_mut.insert(CONTENT.to_string(), new_content); + + // Copy over known good unsigned keys + let allowed_unsigned_keys = [AGE_TS, REPLACES_STATE]; + if let Some(unsigned) = event.get(UNSIGNED) { + let mut new_unsigned = serde_json::Map::new(); + for key in allowed_unsigned_keys { + if let Some(value) = unsigned.get(key) { + new_unsigned.insert(key.to_string(), value.clone()); + } + } + allowed_fields_mut.insert(UNSIGNED.to_string(), Value::Object(new_unsigned)); + } + + Ok(allowed_fields) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::{calculate_event_id, redact}; + use crate::room_versions::RoomVersion; + + #[test] + fn test_calculate_event_id() { + let original = json!( + { + "auth_events":[ + "$gbHO7IPUHybc7ULFnT7P0r3iWlZFHGmr6zBfEYCUyKw", + "$hy1eZFYcgNxMFNNBgCD5fyOzyWRcBkxfNcrUI5ZpZlE", + "$Nt_z68EwFfPqBeHjzEHGsp461Z4EfNEzR-KH5bOYdOY" + ], + "prev_events":[ + "$4FbpZrgPQTwoLD9H5y7jcikucCypUOn78mXhQX7WliY" + ], + "type":"m.room.message", + "room_id":"!DHmIIVvxFASSFgDGzr:localhost:8008", + "sender":"@tester_b:localhost:8008", + "content":{ + "msgtype":"m.text", + "body":"invited people can see history", + "m.mentions":{} + }, + "depth":24, + "origin":"localhost:8008", + "origin_server_ts":1731769874137_i64, + "hashes":{ + "sha256":"FoYV1w3TW/B2mVT0gX2/BZKpCwrrvGXqXFdUhN9LZYU" + }, + "signatures":{ + "localhost:8008":{ + "ed25519:a_phSE":"G8cfk/m97sndxMNrEZ2nMMSXkVeJE05G7if4JiVzAwGfD3TwnF/jfSHt2acWrpNqv/aEhZug3WLofc2id+rVBw" + } + }, + "unsigned":{ + "age_ts":1731769874137_i64 + } + } + ); + let redacted = json!( + { + "auth_events":[ + "$gbHO7IPUHybc7ULFnT7P0r3iWlZFHGmr6zBfEYCUyKw", + "$hy1eZFYcgNxMFNNBgCD5fyOzyWRcBkxfNcrUI5ZpZlE", + "$Nt_z68EwFfPqBeHjzEHGsp461Z4EfNEzR-KH5bOYdOY" + ], + "prev_events":[ + "$4FbpZrgPQTwoLD9H5y7jcikucCypUOn78mXhQX7WliY" + ], + "type":"m.room.message", + "room_id":"!DHmIIVvxFASSFgDGzr:localhost:8008", + "sender":"@tester_b:localhost:8008", + "content":{}, + "depth":24, + "origin":"localhost:8008", + "origin_server_ts":1731769874137_i64, + "hashes":{ + "sha256":"FoYV1w3TW/B2mVT0gX2/BZKpCwrrvGXqXFdUhN9LZYU" + }, + "signatures":{ + "localhost:8008":{ + "ed25519:a_phSE":"G8cfk/m97sndxMNrEZ2nMMSXkVeJE05G7if4JiVzAwGfD3TwnF/jfSHt2acWrpNqv/aEhZug3WLofc2id+rVBw" + } + } + } + ); + + let expected = "$zRz9jjiT9wZc3Hl9ij_74aCmTjqV3YMlj9sj3Uqxg6o"; + let expected_v3 = "$zRz9jjiT9wZc3Hl9ij/74aCmTjqV3YMlj9sj3Uqxg6o"; + + let original_event_id = calculate_event_id(&original, &RoomVersion::V10).unwrap(); + let redacted_event_id = calculate_event_id(&redacted, &RoomVersion::V10).unwrap(); + let _ = calculate_event_id(&original, &RoomVersion::V2).unwrap_err(); + let v3_event_id = calculate_event_id(&original, &RoomVersion::V3).unwrap(); + + assert_eq!(expected, &*original_event_id); + assert_eq!(expected, &*redacted_event_id); + assert_eq!(expected_v3, &*v3_event_id); + } + + #[test] + /// Tests to ensure events with overly large values for `depth` are handled appropriately. + /// This was added in room version 6 . + fn test_calculate_event_id_big_int_old_rooms() { + let original = json!( + { + "auth_events":[ + "$gbHO7IPUHybc7ULFnT7P0r3iWlZFHGmr6zBfEYCUyKw", + "$hy1eZFYcgNxMFNNBgCD5fyOzyWRcBkxfNcrUI5ZpZlE", + "$Nt_z68EwFfPqBeHjzEHGsp461Z4EfNEzR-KH5bOYdOY" + ], + "prev_events":[ + "$4FbpZrgPQTwoLD9H5y7jcikucCypUOn78mXhQX7WliY" + ], + "type":"m.room.message", + "room_id":"!DHmIIVvxFASSFgDGzr:localhost:8008", + "sender":"@tester_b:localhost:8008", + "content":{ + "msgtype":"m.text", + "body":"invited people can see history", + "m.mentions":{} + }, + // NOTE: use the biggest acceptable number + "depth":u64::MAX, + "origin":"localhost:8008", + "origin_server_ts":1731769874137_i64, + "hashes":{ + "sha256":"FoYV1w3TW/B2mVT0gX2/BZKpCwrrvGXqXFdUhN9LZYU" + }, + "signatures":{ + "localhost:8008":{ + "ed25519:a_phSE":"G8cfk/m97sndxMNrEZ2nMMSXkVeJE05G7if4JiVzAwGfD3TwnF/jfSHt2acWrpNqv/aEhZug3WLofc2id+rVBw" + } + }, + "unsigned":{ + "age_ts":1731769874137_i64 + } + } + ); + + // These should succeed. + let _event_id = calculate_event_id(&original, &RoomVersion::V3).unwrap(); + let _event_id = calculate_event_id(&original, &RoomVersion::V4).unwrap(); + let _event_id = calculate_event_id(&original, &RoomVersion::V5).unwrap(); + + // These should not succeed. + let versions = [ + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + RoomVersion::V11, + RoomVersion::V12, + ]; + for version in versions { + let _event_id = calculate_event_id(&original, &version).unwrap_err(); + } + } + + #[test] + /// Tests that calling `redact` on invalid event json that is missing the `type` property + /// fails. + /// The `type` field is required by all versions of the spec. Any json encountered where + /// this field is missing must be considered invalid. + fn test_redact_missing_type() { + let original = json!( + { + // "type": "missing_type" + "unknown_key":"unknown_value", + "auth_events":[ + "$gbHO7IPUHybc7ULFnT7P0r3iWlZFHGmr6zBfEYCUyKw", + "$hy1eZFYcgNxMFNNBgCD5fyOzyWRcBkxfNcrUI5ZpZlE", + "$Nt_z68EwFfPqBeHjzEHGsp461Z4EfNEzR-KH5bOYdOY" + ], + "prev_events":[ + "$4FbpZrgPQTwoLD9H5y7jcikucCypUOn78mXhQX7WliY" + ], + "room_id":"!DHmIIVvxFASSFgDGzr:localhost:8008", + "sender":"@tester_b:localhost:8008", + "content":{ + "msgtype":"m.text", + "body":"invited people can see history", + "m.mentions":{} + }, + "depth":24, + "origin":"localhost:8008", + "origin_server_ts":1731769874137_i64, + "hashes":{ + "sha256":"FoYV1w3TW/B2mVT0gX2/BZKpCwrrvGXqXFdUhN9LZYU" + }, + "signatures":{ + "localhost:8008":{ + "ed25519:a_phSE":"G8cfk/m97sndxMNrEZ2nMMSXkVeJE05G7if4JiVzAwGfD3TwnF/jfSHt2acWrpNqv/aEhZug3WLofc2id+rVBw" + } + }, + "unsigned":{ + "age_ts":1731769874137_i64 + } + } + ); + + let versions = [ + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + RoomVersion::V11, + RoomVersion::V12, + ]; + for version in versions { + let _ = redact(&original, &version).unwrap_err(); + } + } + + #[test] + /// Tests that calling `redact` on invalid event json that is missing the `content` property + /// fails. + /// The `content` field is required by all versions of the spec. Any json encountered where + /// this field is missing must be considered invalid. + fn test_redact_missing_content() { + let original = json!( + { + "unknown_key":"unknown_value", + "auth_events":[ + "$gbHO7IPUHybc7ULFnT7P0r3iWlZFHGmr6zBfEYCUyKw", + "$hy1eZFYcgNxMFNNBgCD5fyOzyWRcBkxfNcrUI5ZpZlE", + "$Nt_z68EwFfPqBeHjzEHGsp461Z4EfNEzR-KH5bOYdOY" + ], + "prev_events":[ + "$4FbpZrgPQTwoLD9H5y7jcikucCypUOn78mXhQX7WliY" + ], + "type":"m.room.message", + "room_id":"!DHmIIVvxFASSFgDGzr:localhost:8008", + "sender":"@tester_b:localhost:8008", + "depth":24, + "origin":"localhost:8008", + "origin_server_ts":1731769874137_i64, + "hashes":{ + "sha256":"FoYV1w3TW/B2mVT0gX2/BZKpCwrrvGXqXFdUhN9LZYU" + }, + "signatures":{ + "localhost:8008":{ + "ed25519:a_phSE":"G8cfk/m97sndxMNrEZ2nMMSXkVeJE05G7if4JiVzAwGfD3TwnF/jfSHt2acWrpNqv/aEhZug3WLofc2id+rVBw" + } + }, + "unsigned":{ + "age_ts":1731769874137_i64 + } + } + ); + + let versions = [ + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + RoomVersion::V11, + RoomVersion::V12, + ]; + for version in versions { + let _ = redact(&original, &version).unwrap_err(); + } + } + + #[test] + /// Tests redaction logic for `m.room.message` events against latest room versions. + /// This is only testing v10+ as it is rather cumbersome to create the proper json for all + /// rooms versions. + fn test_redact_m_room_message() { + let original = json!( + { + "unknown_key":"unknown_value", + "auth_events":[ + "$gbHO7IPUHybc7ULFnT7P0r3iWlZFHGmr6zBfEYCUyKw", + "$hy1eZFYcgNxMFNNBgCD5fyOzyWRcBkxfNcrUI5ZpZlE", + "$Nt_z68EwFfPqBeHjzEHGsp461Z4EfNEzR-KH5bOYdOY" + ], + "prev_events":[ + "$4FbpZrgPQTwoLD9H5y7jcikucCypUOn78mXhQX7WliY" + ], + "type":"m.room.message", + "room_id":"!DHmIIVvxFASSFgDGzr:localhost:8008", + "sender":"@tester_b:localhost:8008", + "content":{ + "msgtype":"m.text", + "body":"invited people can see history", + "m.mentions":{} + }, + "depth":24, + "origin":"localhost:8008", + "origin_server_ts":1731769874137_i64, + "hashes":{ + "sha256":"FoYV1w3TW/B2mVT0gX2/BZKpCwrrvGXqXFdUhN9LZYU" + }, + "signatures":{ + "localhost:8008":{ + "ed25519:a_phSE":"G8cfk/m97sndxMNrEZ2nMMSXkVeJE05G7if4JiVzAwGfD3TwnF/jfSHt2acWrpNqv/aEhZug3WLofc2id+rVBw" + } + }, + "unsigned":{ + "age_ts":1731769874137_i64 + } + } + ); + let expected = json!( + { + "auth_events":[ + "$gbHO7IPUHybc7ULFnT7P0r3iWlZFHGmr6zBfEYCUyKw", + "$hy1eZFYcgNxMFNNBgCD5fyOzyWRcBkxfNcrUI5ZpZlE", + "$Nt_z68EwFfPqBeHjzEHGsp461Z4EfNEzR-KH5bOYdOY" + ], + "prev_events":[ + "$4FbpZrgPQTwoLD9H5y7jcikucCypUOn78mXhQX7WliY" + ], + "type":"m.room.message", + "room_id":"!DHmIIVvxFASSFgDGzr:localhost:8008", + "sender":"@tester_b:localhost:8008", + "content":{}, + "depth":24, + "origin":"localhost:8008", + "origin_server_ts":1731769874137_i64, + "hashes":{ + "sha256":"FoYV1w3TW/B2mVT0gX2/BZKpCwrrvGXqXFdUhN9LZYU" + }, + "signatures":{ + "localhost:8008":{ + "ed25519:a_phSE":"G8cfk/m97sndxMNrEZ2nMMSXkVeJE05G7if4JiVzAwGfD3TwnF/jfSHt2acWrpNqv/aEhZug3WLofc2id+rVBw" + } + }, + "unsigned":{ + "age_ts":1731769874137_i64 + } + } + ); + let expected_v11 = json!( + { + "auth_events":[ + "$gbHO7IPUHybc7ULFnT7P0r3iWlZFHGmr6zBfEYCUyKw", + "$hy1eZFYcgNxMFNNBgCD5fyOzyWRcBkxfNcrUI5ZpZlE", + "$Nt_z68EwFfPqBeHjzEHGsp461Z4EfNEzR-KH5bOYdOY" + ], + "prev_events":[ + "$4FbpZrgPQTwoLD9H5y7jcikucCypUOn78mXhQX7WliY" + ], + "type":"m.room.message", + "room_id":"!DHmIIVvxFASSFgDGzr:localhost:8008", + "sender":"@tester_b:localhost:8008", + "content":{}, + "depth":24, + "origin_server_ts":1731769874137_i64, + "hashes":{ + "sha256":"FoYV1w3TW/B2mVT0gX2/BZKpCwrrvGXqXFdUhN9LZYU" + }, + "signatures":{ + "localhost:8008":{ + "ed25519:a_phSE":"G8cfk/m97sndxMNrEZ2nMMSXkVeJE05G7if4JiVzAwGfD3TwnF/jfSHt2acWrpNqv/aEhZug3WLofc2id+rVBw" + } + }, + "unsigned":{ + "age_ts":1731769874137_i64 + } + } + ); + + let redacted = redact(&original, &RoomVersion::V10).unwrap(); + + assert_eq!(expected, redacted); + + let versions = [RoomVersion::V11, RoomVersion::V12]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + + assert_eq!(expected_v11, redacted, "Room Version {version}"); + } + } + + #[test] + /// Tests redaction logic adheres to changes due to [super::use_updated_redaction_rules]. + /// Tests against all known room versions. + fn test_redact_updated_redaction_rules() { + let original = json!( + { + "type":"m.room.message", + "prev_state":{}, + "membership":{}, + "origin":"some_place", + "content":{}, + } + ); + + let expected_updated_redaction_rules = json!( + { + "type":"m.room.message", + "content":{}, + } + ); + let expected_pre_updated_redaction_rules = json!( + { + "type":"m.room.message", + "prev_state":{}, + "membership":{}, + "origin":"some_place", + "content":{}, + } + ); + + let versions = [ + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + ]; + for version in versions { + let redacted_1 = redact(&original, &version).unwrap(); + assert_eq!( + expected_pre_updated_redaction_rules, redacted_1, + "Room Version {version}" + ); + } + + let versions = [RoomVersion::V11, RoomVersion::V12]; + for version in versions { + let redacted_2 = redact(&original, &version).unwrap(); + assert_eq!( + expected_updated_redaction_rules, redacted_2, + "Room Version {version}" + ); + } + } + + #[test] + /// Tests redaction logic for `m.room.member` events against all known room versions. + fn test_redact_m_room_member() { + let original = json!( + { + "type":"m.room.member", + "content":{ + "unknown_key":"unknown_value", + "membership":"join", + "join_authorised_via_users_server":"server", + "third_party_invite":{ + "signed":{}, + }, + }, + } + ); + + let expected_pre_unrestricted_join_rule_fix = json!( + { + "type":"m.room.member", + "content":{ + "membership":"join", + }, + } + ); + + let expected_updated_redaction_rules = json!( + { + "type":"m.room.member", + "content":{ + "membership":"join", + "join_authorised_via_users_server":"server", + "third_party_invite":{ + "signed":{}, + }, + }, + } + ); + let expected_pre_updated_redaction_rules = json!( + { + "type":"m.room.member", + "content":{ + "membership":"join", + "join_authorised_via_users_server":"server", + }, + } + ); + + let versions = [ + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + ]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_pre_unrestricted_join_rule_fix, redacted, + "Room Version {version}" + ); + } + + let versions = [RoomVersion::V9, RoomVersion::V10]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_pre_updated_redaction_rules, redacted, + "Room Version {version}" + ); + } + + let versions = [RoomVersion::V11, RoomVersion::V12]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_updated_redaction_rules, redacted, + "Room Version {version}" + ); + } + } + + /// Tests redaction logic for `m.room.create` events against all known room versions. + #[test] + fn test_redact_m_room_create() { + let original = json!( + { + "type":"m.room.create", + "room_id": "!roomid", + "content":{ + "unknown_key":"unknown_value", + "other_key":"value", + "creator":"user", + }, + } + ); + + let expected_implicit_room_creator = json!( + { + "type":"m.room.create", + "room_id": "!roomid", + "content":{ + "unknown_key":"unknown_value", + "other_key":"value", + "creator":"user", + }, + } + ); + let expected_room_ids_as_hashes = json!( + { + "type":"m.room.create", + "content":{ + "unknown_key":"unknown_value", + "other_key":"value", + "creator":"user", + }, + } + ); + let expected_pre_implicit_room_creator = json!( + { + "type":"m.room.create", + "room_id": "!roomid", + "content":{ + "creator":"user", + }, + } + ); + + let versions = [ + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + ]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_pre_implicit_room_creator, redacted, + "Room Version {version}" + ); + } + + let redacted = redact(&original, &RoomVersion::V11).unwrap(); + assert_eq!(expected_implicit_room_creator, redacted); + + let versions = [RoomVersion::HYDRA_V11, RoomVersion::V12]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_room_ids_as_hashes, redacted, + "Room Version {version}" + ); + } + } + + #[test] + /// Tests redaction logic for `m.room.join_rules` events against all known room versions. + fn test_redact_m_room_join_rules() { + let original = json!( + { + "type":"m.room.join_rules", + "content":{ + "unknown_key":"unknown_value", + "join_rule":"invite", + "allow":"user", + }, + } + ); + + let expected_restricted_join_rule = json!( + { + "type":"m.room.join_rules", + "content":{ + "join_rule":"invite", + "allow":"user", + }, + } + ); + let expected_pre_restricted_join_rule = json!( + { + "type":"m.room.join_rules", + "content":{ + "join_rule":"invite", + }, + } + ); + + let versions = [ + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + ]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_pre_restricted_join_rule, redacted, + "Room Version {version}" + ); + } + + let versions = [ + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + RoomVersion::V11, + RoomVersion::V12, + ]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_restricted_join_rule, redacted, + "Room Version {version}" + ); + } + } + + #[test] + /// Tests redaction logic for `m.room.power_levels` events against all known room versions. + fn test_redact_m_room_power_levels() { + let original = json!( + { + "type":"m.room.power_levels", + "content":{ + "unknown_key":"unknown_value", + "users":{}, + "users_default":{}, + "events":{}, + "events_default":{}, + "state_default":{}, + "ban":{}, + "kick":{}, + "redact":{}, + "invite":{}, + }, + } + ); + + let expected_updated_redaction_rules = json!( + { + "type":"m.room.power_levels", + "content":{ + "users":{}, + "users_default":{}, + "events":{}, + "events_default":{}, + "state_default":{}, + "ban":{}, + "kick":{}, + "redact":{}, + "invite":{}, + }, + } + ); + let expected_pre_updated_redaction_rules = json!( + { + "type":"m.room.power_levels", + "content":{ + "users":{}, + "users_default":{}, + "events":{}, + "events_default":{}, + "state_default":{}, + "ban":{}, + "kick":{}, + "redact":{}, + }, + } + ); + + let versions = [ + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + ]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_pre_updated_redaction_rules, redacted, + "Room Version {version}" + ); + } + + let versions = [RoomVersion::V11, RoomVersion::V12]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_updated_redaction_rules, redacted, + "Room Version {version}" + ); + } + } + + #[test] + /// Tests redaction logic for `m.room.aliases` events against all known room versions. + fn test_redact_m_room_aliases() { + let original = json!( + { + "type":"m.room.aliases", + "content":{ + "unknown_key":"unknown_value", + "aliases":{}, + }, + } + ); + + let expected_special_case_aliases = json!( + { + "type":"m.room.aliases", + "content":{ + "aliases":{}, + }, + } + ); + let expected_post_special_case_aliases = json!( + { + "type":"m.room.aliases", + "content":{}, + } + ); + + let versions = [ + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + ]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_special_case_aliases, redacted, + "Room Version {version}" + ); + } + + let versions = [ + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + RoomVersion::V11, + RoomVersion::V12, + ]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_post_special_case_aliases, redacted, + "Room Version {version}" + ); + } + } + + #[test] + /// Tests redaction logic for `m.room.history_visibility` events against all known room versions. + fn test_redact_m_room_history_visibility() { + let original = json!( + { + "type":"m.room.history_visibility", + "content":{ + "unknown_key":"unknown_value", + "history_visibility":"visibility", + }, + } + ); + + let expected = json!( + { + "type":"m.room.history_visibility", + "content":{ + "history_visibility":"visibility", + }, + } + ); + + let versions = [ + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + RoomVersion::V11, + RoomVersion::V12, + ]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!(expected, redacted, "Room Version {version}"); + } + } + + #[test] + /// Tests redaction logic for `m.room.redaction` events against all known room versions. + fn test_redact_m_room_redaction() { + let original = json!( + { + "type":"m.room.redaction", + "content":{ + "unknown_key":"unknown_value", + "redacts":"event", + }, + } + ); + + let expected_updated_redaction_rules = json!( + { + "type":"m.room.redaction", + "content":{ + "redacts":"event", + }, + } + ); + let expected_pre_updated_redaction_rules = json!( + { + "type":"m.room.redaction", + "content":{}, + } + ); + + let versions = [ + RoomVersion::V1, + RoomVersion::V2, + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, + RoomVersion::V6, + RoomVersion::V7, + RoomVersion::V8, + RoomVersion::V9, + RoomVersion::V10, + ]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_pre_updated_redaction_rules, redacted, + "Room Version {version}" + ); + } + + let versions = [RoomVersion::V11, RoomVersion::V12]; + for version in versions { + let redacted = redact(&original, &version).unwrap(); + assert_eq!( + expected_updated_redaction_rules, redacted, + "Room Version {version}" + ); + } + } +} diff --git a/rust/src/json.rs b/rust/src/json.rs new file mode 100644 index 0000000000..3e833c6707 --- /dev/null +++ b/rust/src/json.rs @@ -0,0 +1,218 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +/// A wrapper type that represents a value that may be missing. +/// +/// We can't necessarily use `Option` for this, as we want to distinguish +/// between a missing value and a value that is present but null (e.g. +/// `{"field": null}` vs `{}`). Serde by default treats missing fields as +/// `None`, so we need a custom type to capture this distinction. +/// +/// A plain `AllowMissing` is used for fields that are either present and of +/// type `T`, or absent. An `AllowMissing>` is used for fields that +/// are of type `T`, null, or absent. +/// +/// Note, to use this type correctly, the field **MUST** be annotated with: +/// +/// ```rust +/// #[serde( +/// default, +/// with = "crate::json::allow_missing", +/// skip_serializing_if = "AllowMissing::is_absent" +/// )] +/// ``` +/// +#[derive(Default, Debug, Clone)] +pub enum AllowMissing { + Some(T), + #[default] + Absent, +} + +impl AllowMissing { + /// Returns `true` if the value is present, even if it is null. + pub fn is_some(&self) -> bool { + matches!(self, AllowMissing::Some(_)) + } + + /// Returns `true` if the value is absent. + pub fn is_absent(&self) -> bool { + matches!(self, AllowMissing::Absent) + } + + /// Converts to `Option`. + /// + /// Useful for converting e.g. `AllowMissing` to `Option<&str>`. + pub fn as_deref_opt(&self) -> Option<&T::Target> + where + T: std::ops::Deref, + { + match self { + AllowMissing::Some(inner) => Some(inner.deref()), + AllowMissing::Absent => None, + } + } + + /// Converts to `Option<&T>`. + pub fn as_ref_opt(&self) -> Option<&T> { + match self { + AllowMissing::Some(inner) => Some(inner), + AllowMissing::Absent => None, + } + } +} + +/// A module that provides the serialization and deserialization logic for +/// `AllowMissing`. +pub mod allow_missing { + use serde::ser::Error as _; + + use super::AllowMissing; + + pub fn deserialize<'de, T, D>(deserializer: D) -> Result, D::Error> + where + T: serde::Deserialize<'de>, + D: serde::Deserializer<'de>, + { + Ok(AllowMissing::Some(T::deserialize(deserializer)?)) + } + + pub fn serialize(value: &AllowMissing, serializer: S) -> Result + where + T: serde::Serialize, + S: serde::Serializer, + { + match value { + AllowMissing::Some(inner) => inner.serialize(serializer), + // We should never attempt to serialize an `AllowMissing::Absent`, as we + // should have skipped it with `skip_serializing_if`. + AllowMissing::Absent => Err(S::Error::custom("cannot serialize AllowMissing::Absent")), + } + } +} + +#[cfg(test)] +mod tests { + use std::assert_matches; + + use serde::{Deserialize, Serialize}; + + use super::*; + + #[derive(Serialize, Deserialize)] + struct TestStruct { + #[serde( + default, + with = "crate::json::allow_missing", + skip_serializing_if = "AllowMissing::is_absent" + )] + value: AllowMissing, + } + + #[test] + fn test_deserialize() { + let json = r#"{"value":42}"#; + let deserialized: TestStruct = serde_json::from_str(json).unwrap(); + assert!(deserialized.value.is_some()); + assert_matches!(deserialized.value, AllowMissing::Some(42)); + + let json = r#"{}"#; + let deserialized: TestStruct = serde_json::from_str(json).unwrap(); + assert!(deserialized.value.is_absent()); + assert_matches!(deserialized.value, AllowMissing::Absent); + } + + #[test] + fn test_serialize() { + let value = TestStruct { + value: AllowMissing::Some(42), + }; + let serialized = serde_json::to_string(&value).unwrap(); + assert_eq!(serialized, r#"{"value":42}"#); + + let value = TestStruct { + value: AllowMissing::Absent, + }; + let serialized = serde_json::to_string(&value).unwrap(); + assert_eq!(serialized, r#"{}"#); + } + + /// Test that we get an error if we attempt to serialize an + /// `AllowMissing::Absent` without the skip_serializing_if annotation. + #[test] + fn test_serialize_absent_error() { + #[derive(Serialize)] + struct TestStructWithoutSkip { + #[serde(default, with = "crate::json::allow_missing")] + value: AllowMissing, + } + + let value = TestStructWithoutSkip { + value: AllowMissing::Absent, + }; + + let err = serde_json::to_string(&value).unwrap_err(); + assert_eq!(err.to_string(), "cannot serialize AllowMissing::Absent"); + } + + #[derive(Serialize, Deserialize)] + struct TestStructOption { + #[serde( + default, + with = "crate::json::allow_missing", + skip_serializing_if = "AllowMissing::is_absent" + )] + value: AllowMissing>, + } + + #[test] + fn test_serialize_option() { + let value = TestStructOption { + value: AllowMissing::Some(Some(42)), + }; + let serialized = serde_json::to_string(&value).unwrap(); + assert_eq!(serialized, r#"{"value":42}"#); + + let value = TestStructOption { + value: AllowMissing::Some(None), + }; + let serialized = serde_json::to_string(&value).unwrap(); + assert_eq!(serialized, r#"{"value":null}"#); + + let value = TestStructOption { + value: AllowMissing::Absent, + }; + let serialized = serde_json::to_string(&value).unwrap(); + assert_eq!(serialized, r#"{}"#); + } + + #[test] + fn test_deserialize_option() { + let json = r#"{"value":42}"#; + let deserialized: TestStructOption = serde_json::from_str(json).unwrap(); + assert!(deserialized.value.is_some()); + assert_matches!(deserialized.value, AllowMissing::Some(Some(42))); + + let json = r#"{"value":null}"#; + let deserialized: TestStructOption = serde_json::from_str(json).unwrap(); + assert!(deserialized.value.is_some()); + assert_matches!(deserialized.value, AllowMissing::Some(None)); + + let json = r#"{}"#; + let deserialized: TestStructOption = serde_json::from_str(json).unwrap(); + assert!(deserialized.value.is_absent()); + assert_matches!(deserialized.value, AllowMissing::Absent); + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index cc89862e4e..8ed4e24b81 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -12,6 +12,7 @@ pub mod events; pub mod http; pub mod http_client; pub mod identifier; +pub mod json; pub mod matrix_const; pub mod msc4388_rendezvous; pub mod push; diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 823b6288e8..4f36740808 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -33,8 +33,9 @@ from synapse.api.errors import Codes, SynapseError from synapse.api.room_versions import RoomVersion from synapse.events import EventBase -from synapse.events.utils import prune_event, prune_event_dict +from synapse.events.utils import prune_event from synapse.logging.opentracing import trace +from synapse.synapse_rust.events import redact_event_dict from synapse.types import JsonDict, UserID logger = logging.getLogger(__name__) @@ -157,7 +158,7 @@ def compute_event_signature( Returns: a dictionary in the same format of an event's signatures field. """ - redact_json = prune_event_dict(room_version, event_dict) + redact_json = redact_event_dict(room_version, event_dict) redact_json.pop("age_ts", None) redact_json.pop("unsigned", None) if logger.isEnabledFor(logging.DEBUG): diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 36736b4559..a1633f881f 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -46,9 +46,9 @@ ) from synapse.config.key import TrustedKeyServer from synapse.events import EventBase -from synapse.events.utils import prune_event_dict from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.storage.keys import FetchKeyResult +from synapse.synapse_rust.events import redact_event from synapse.types import JsonDict from synapse.util import unwrapFirstError from synapse.util.async_helpers import yieldable_gather_results @@ -136,7 +136,7 @@ def from_event( server_name, # We defer creating the redacted json object, as it uses a lot more # memory than the Event object itself. - lambda: prune_event_dict(event.room_version, event.get_pdu_json()), + lambda: redact_event(event).get_pdu_json(), minimum_valid_until_ms, key_ids=key_ids, ) diff --git a/synapse/event_auth.py b/synapse/event_auth.py index d9750651fa..81a454b544 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -66,6 +66,7 @@ from synapse.state import CREATE_KEY from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.types import ( + JsonMapping, MutableStateMap, StateKey, StateMap, @@ -856,6 +857,7 @@ def get_send_level( power level required to send this event. """ + power_levels_content: JsonMapping if power_levels_event: power_levels_content = power_levels_event.content else: diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index fcb169e079..7abd91f072 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -20,49 +20,37 @@ # # -import abc import collections.abc from typing import ( TYPE_CHECKING, Any, - Generic, - Iterable, - Literal, - TypeVar, + TypeAlias, Union, - overload, ) import attr -from typing_extensions import deprecated -from unpaddedbase64 import encode_base64 from synapse.api.constants import ( EventContentFields, EventTypes, RelationTypes, - StickyEvent, -) -from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions -from synapse.synapse_rust.events import ( - EventInternalMetadata, - JsonObject, - Signatures, - Unsigned, ) +from synapse.api.errors import Codes, SynapseError +from synapse.api.room_versions import RoomVersion, RoomVersions +from synapse.synapse_rust.events import Event from synapse.types import ( JsonDict, - JsonMapping, StateKey, - StrCollection, ) -from synapse.util.caches import intern_dict -from synapse.util.duration import Duration -from synapse.util.frozenutils import freeze if TYPE_CHECKING: from synapse.events.builder import EventBuilder +# The base class for events used to be called EventBase, but it was renamed to +# Event when we switched to using the Rust implementation. We keep the old name +# around for backwards compatibility. +EventBase: TypeAlias = Event + USE_FROZEN_DICTS = False """ @@ -70,639 +58,29 @@ bugs where we accidentally share e.g. signature dicts. However, converting a dict to frozen_dicts is expensive. -NOTE: This is overridden by the configuration by the Synapse worker apps, but -for the sake of tests, it is set here because it cannot be configured on the -homeserver object itself. - -FIXME: Because of how this option works (changing the underlying types), it causes -subtle downstream bugs that makes type comparisons brittle, tracked by -https://github.com/element-hq/synapse/issues/18117 +FIXME: Remove `USE_FROZEN_DICTS` and `use_frozen_dicts` config as this is no +longer used since we switched to using the Rust implementation, all events are +immutable already (and so don't benefit from freezing). """ -T = TypeVar("T") - - -# DictProperty (and DefaultDictProperty) require the classes they're used with to -# have a _dict property to pull properties from. -# -# TODO _DictPropertyInstance should not include EventBuilder but due to -# https://github.com/python/mypy/issues/5570 it thinks the DictProperty and -# DefaultDictProperty get applied to EventBuilder when it is in a Union with -# EventBase. This is the least invasive hack to get mypy to comply. -# -# Note that DictProperty/DefaultDictProperty cannot actually be used with -# EventBuilder as it lacks a _dict property. -_DictPropertyInstance = Union["EventBase", "EventBuilder"] - - -class DictProperty(Generic[T]): - """An object property which delegates to the `_dict` within its parent object.""" - - __slots__ = ["key"] - - def __init__(self, key: str): - self.key = key - - @overload - def __get__( - self, - instance: Literal[None], - owner: type[_DictPropertyInstance] | None = None, - ) -> "DictProperty": ... - - @overload - def __get__( - self, - instance: _DictPropertyInstance, - owner: type[_DictPropertyInstance] | None = None, - ) -> T: ... - - def __get__( - self, - instance: _DictPropertyInstance | None, - owner: type[_DictPropertyInstance] | None = None, - ) -> T | "DictProperty": - # if the property is accessed as a class property rather than an instance - # property, return the property itself rather than the value - if instance is None: - return self - try: - assert isinstance(instance, EventBase) - return instance._dict[self.key] - except KeyError as e1: - # We want this to look like a regular attribute error (mostly so that - # hasattr() works correctly), so we convert the KeyError into an - # AttributeError. - # - # To exclude the KeyError from the traceback, we explicitly - # 'raise from e1.__context__' (which is better than 'raise from None', - # because that would omit any *earlier* exceptions). - # - raise AttributeError( - "'%s' has no '%s' property" % (type(instance), self.key) - ) from e1.__context__ - - def __set__(self, instance: _DictPropertyInstance, v: T) -> None: - assert isinstance(instance, EventBase) - instance._dict[self.key] = v - - def __delete__(self, instance: _DictPropertyInstance) -> None: - assert isinstance(instance, EventBase) - try: - del instance._dict[self.key] - except KeyError as e1: - raise AttributeError( - "'%s' has no '%s' property" % (type(instance), self.key) - ) from e1.__context__ - - -class DefaultDictProperty(DictProperty, Generic[T]): - """An extension of DictProperty which provides a default if the property is - not present in the parent's _dict. - - Note that this means that hasattr() on the property always returns True. - """ - - __slots__ = ["default"] - - def __init__(self, key: str, default: T): - super().__init__(key) - self.default = default - - @overload - def __get__( - self, - instance: Literal[None], - owner: type[_DictPropertyInstance] | None = None, - ) -> "DefaultDictProperty": ... - - @overload - def __get__( - self, - instance: _DictPropertyInstance, - owner: type[_DictPropertyInstance] | None = None, - ) -> T: ... - - def __get__( - self, - instance: _DictPropertyInstance | None, - owner: type[_DictPropertyInstance] | None = None, - ) -> T | "DefaultDictProperty": - if instance is None: - return self - assert isinstance(instance, EventBase) - return instance._dict.get(self.key, self.default) - - -class EventBase(metaclass=abc.ABCMeta): - @property - @abc.abstractmethod - def format_version(self) -> int: - """The EventFormatVersion implemented by this event""" - ... - - def __init__( - self, - event_dict: JsonDict, - room_version: RoomVersion, - signatures: dict[str, dict[str, str]], - unsigned: JsonDict, - internal_metadata_dict: JsonDict, - rejected_reason: str | None, - ): - assert room_version.event_format == self.format_version - - if "content" in event_dict: - event_dict["content"] = JsonObject(event_dict["content"]) - - # We intern these strings because they turn up a lot (especially when - # caching). - event_dict = intern_dict(event_dict) - - if USE_FROZEN_DICTS: - frozen_dict = freeze(event_dict) - else: - frozen_dict = event_dict - - self.room_version = room_version - self.signatures = Signatures(signatures) - self.unsigned = Unsigned(unsigned) - self.rejected_reason = rejected_reason - - self._dict = frozen_dict - - self.internal_metadata = EventInternalMetadata(internal_metadata_dict) - - depth: DictProperty[int] = DictProperty("depth") - content: DictProperty[JsonMapping] = DictProperty("content") - hashes: DictProperty[dict[str, str]] = DictProperty("hashes") - origin_server_ts: DictProperty[int] = DictProperty("origin_server_ts") - sender: DictProperty[str] = DictProperty("sender") - # TODO state_key should be str | None. This is generally asserted in Synapse - # by calling is_state() first (which ensures it is not None), but it is hard (not possible?) - # to properly annotate that calling is_state() asserts that state_key exists - # and is non-None. It would be better to replace such direct references with - # get_state_key() (and a check for None). - state_key: DictProperty[str] = DictProperty("state_key") - type: DictProperty[str] = DictProperty("type") - - # This is a deprecated property, use `sender` instead. Only used by modules. - user_id: DictProperty[str] = DictProperty("sender") - - @property - def event_id(self) -> str: - raise NotImplementedError() - - @property - def room_id(self) -> str: - raise NotImplementedError() - - @property - def membership(self) -> str: - return self.content["membership"] - - @property - def redacts(self) -> str | None: - """MSC2176 moved the redacts field into the content.""" - if self.room_version.updated_redaction_rules: - return self.content.get("redacts") - return self.get("redacts") - - def is_state(self) -> bool: - return self.get_state_key() is not None - - def get_state_key(self) -> str | None: - """Get the state key of this event, or None if it's not a state event""" - return self._dict.get("state_key") - - def get_dict(self) -> JsonDict: - """Convert the event to a dictionary suitable for serialisation.""" - - d = dict(self._dict) - if "content" in d: - # Convert the content (which is a JsonObject) back to a dict. Json - # serialization should handle JsonObjects fine, but for sanities - # sake we want `get_dict()` and `get_pdu_json()` to return plain - # dicts. - d["content"] = dict(self.content) - d.update( - { - "signatures": self.signatures.as_dict(), - "unsigned": self.unsigned.for_event(), - } - ) - - return d - - def get_dict_for_persistence(self) -> JsonDict: - """Convert the event to a dictionary suitable for persistence.""" - d = dict(self._dict) - d.update( - { - "signatures": self.signatures.as_dict(), - "unsigned": self.unsigned.for_persistence(), - } - ) - - return d - - def get(self, key: str, default: Any | None = None) -> Any: - return self._dict.get(key, default) - - def get_internal_metadata_dict(self) -> JsonDict: - return self.internal_metadata.get_dict() - - def get_pdu_json(self, time_now: int | None = None) -> JsonDict: - pdu_json = self.get_dict() - - if time_now is not None and "age_ts" in pdu_json["unsigned"]: - age = time_now - pdu_json["unsigned"]["age_ts"] - pdu_json.setdefault("unsigned", {})["age"] = int(age) - del pdu_json["unsigned"]["age_ts"] - - # This may be a frozen event - pdu_json["unsigned"].pop("redacted_because", None) - - return pdu_json - - def get_templated_pdu_json(self) -> JsonDict: - """ - Return a JSON object suitable for a templated event, as used in the - make_{join,leave,knock} workflow. - """ - # By using _dict directly we don't pull in signatures/unsigned. - template_json = dict(self._dict) - # The hashes (similar to the signature) need to be recalculated by the - # joining/leaving/knocking server after (potentially) modifying the - # event. - template_json.pop("hashes") - - return template_json - - def __contains__(self, field: str) -> bool: - return field in self._dict - - def items(self) -> list[tuple[str, Any | None]]: - return list(self._dict.items()) - - def keys(self) -> Iterable[str]: - return self._dict.keys() - - def prev_event_ids(self) -> list[str]: - """Returns the list of prev event IDs. The order matches the order - specified in the event, though there is no meaning to it. - - Returns: - The list of event IDs of this event's prev_events - """ - return [e for e, _ in self._dict["prev_events"]] - - def auth_event_ids(self) -> StrCollection: - """Returns the list of auth event IDs. The order matches the order - specified in the event, though there is no meaning to it. - - Returns: - The list of event IDs of this event's auth_events - """ - return [e for e, _ in self._dict["auth_events"]] - - def freeze(self) -> None: - """'Freeze' the event dict, so it cannot be modified by accident""" - - # this will be a no-op if the event dict is already frozen. - self._dict = freeze(self._dict) - - def sticky_duration(self) -> Duration | None: - """ - Returns the effective sticky duration of this event, or None - if the event does not have a sticky duration. - (Sticky Events are a MSC4354 feature.) - - Clamps the sticky duration to the maximum allowed duration. - """ - sticky_obj = self.get_dict().get(StickyEvent.EVENT_FIELD_NAME, None) - if type(sticky_obj) is not dict: - return None - sticky_duration_ms = sticky_obj.get("duration_ms", None) - # MSC: Clamp to 0 and MAX_DURATION (1 hour) - # We use `type(...) is int` to avoid accepting bools as `isinstance(True, int)` - # (bool is a subclass of int) - if type(sticky_duration_ms) is int and sticky_duration_ms >= 0: - return min( - Duration(milliseconds=sticky_duration_ms), - StickyEvent.MAX_DURATION, - ) - return None - - def __str__(self) -> str: - return self.__repr__() - - def __repr__(self) -> str: - rejection = f"REJECTED={self.rejected_reason}, " if self.rejected_reason else "" - - conditional_membership_string = "" - if self.get("type") == EventTypes.Member: - conditional_membership_string = f"membership={self.membership}, " - - return ( - f"<{self.__class__.__name__} " - f"{rejection}" - f"event_id={self.event_id}, " - f"type={self.get('type')}, " - f"state_key={self.get('state_key')}, " - f"{conditional_membership_string}" - f"outlier={self.internal_metadata.is_outlier()}" - ">" - ) - - # Using `__getitem__` is deprecated. Only used by modules. - @deprecated("Use attribute access instead") - def __getitem__(self, field: str) -> Any | None: - return self._dict[field] - - -class FrozenEvent(EventBase): - format_version = EventFormatVersions.ROOM_V1_V2 # All events of this type are V1 - - def __init__( - self, - event_dict: JsonDict, - room_version: RoomVersion, - internal_metadata_dict: JsonDict | None = None, - rejected_reason: str | None = None, - ): - internal_metadata_dict = internal_metadata_dict or {} - - event_dict = dict(event_dict) - - # Signatures is a dict of dicts, and this is faster than doing a - # copy.deepcopy - signatures = { - name: dict(sigs.items()) - for name, sigs in event_dict.pop("signatures", {}).items() - } - - unsigned = event_dict.pop("unsigned", {}) - - self._event_id = event_dict["event_id"] - - super().__init__( - event_dict, - room_version=room_version, - signatures=signatures, - unsigned=unsigned, - internal_metadata_dict=internal_metadata_dict, - rejected_reason=rejected_reason, - ) - - @property - def event_id(self) -> str: - return self._event_id - - @property - def room_id(self) -> str: - return self._dict["room_id"] - - -class FrozenEventV2(EventBase): - format_version = EventFormatVersions.ROOM_V3 # All events of this type are V2 - - def __init__( - self, - event_dict: JsonDict, - room_version: RoomVersion, - internal_metadata_dict: JsonDict | None = None, - rejected_reason: str | None = None, - ): - internal_metadata_dict = internal_metadata_dict or {} - - event_dict = dict(event_dict) - - # Signatures is a dict of dicts, and this is faster than doing a - # copy.deepcopy - signatures = { - name: dict(sigs.items()) - for name, sigs in event_dict.pop("signatures", {}).items() - } - - assert "event_id" not in event_dict - - unsigned = event_dict.pop("unsigned", {}) - - self._event_id: str | None = None - - super().__init__( - event_dict, - room_version=room_version, - signatures=signatures, - unsigned=unsigned, - internal_metadata_dict=internal_metadata_dict, - rejected_reason=rejected_reason, - ) - - @property - def event_id(self) -> str: - # We have to import this here as otherwise we get an import loop which - # is hard to break. - from synapse.crypto.event_signing import compute_event_reference_hash - - if self._event_id: - return self._event_id - self._event_id = "$" + encode_base64(compute_event_reference_hash(self)[1]) - return self._event_id - - @property - def room_id(self) -> str: - return self._dict["room_id"] - - def prev_event_ids(self) -> list[str]: - """Returns the list of prev event IDs. The order matches the order - specified in the event, though there is no meaning to it. - - Returns: - The list of event IDs of this event's prev_events - """ - return self._dict["prev_events"] - - def auth_event_ids(self) -> StrCollection: - """Returns the list of auth event IDs. The order matches the order - specified in the event, though there is no meaning to it. - - Returns: - The list of event IDs of this event's auth_events - """ - return self._dict["auth_events"] - - -class FrozenEventV3(FrozenEventV2): - """FrozenEventV3, which differs from FrozenEventV2 only in the event_id format""" - - format_version = EventFormatVersions.ROOM_V4_PLUS # All events of this type are V3 - - @property - def event_id(self) -> str: - # We have to import this here as otherwise we get an import loop which - # is hard to break. - from synapse.crypto.event_signing import compute_event_reference_hash - - if self._event_id: - return self._event_id - self._event_id = "$" + encode_base64( - compute_event_reference_hash(self)[1], urlsafe=True - ) - return self._event_id - - -class FrozenEventV4(FrozenEventV3): - """FrozenEventV4 for MSC4291 room IDs are hashes""" - - format_version = EventFormatVersions.ROOM_V11_HYDRA_PLUS - - """Override the room_id for m.room.create events""" - - def __init__( - self, - event_dict: JsonDict, - room_version: RoomVersion, - internal_metadata_dict: JsonDict | None = None, - rejected_reason: str | None = None, - ): - super().__init__( - event_dict=event_dict, - room_version=room_version, - internal_metadata_dict=internal_metadata_dict, - rejected_reason=rejected_reason, - ) - self._room_id: str | None = None - - @property - def room_id(self) -> str: - # if we have calculated the room ID already, don't do it again. - if self._room_id: - return self._room_id - - is_create_event = self.type == EventTypes.Create and self.get_state_key() == "" - - # for non-create events: use the supplied value from the JSON, as per FrozenEventV3 - if not is_create_event: - self._room_id = self._dict["room_id"] - assert self._room_id is not None - return self._room_id - - # for create events: calculate the room ID - from synapse.crypto.event_signing import compute_event_reference_hash - - self._room_id = "!" + encode_base64( - compute_event_reference_hash(self)[1], urlsafe=True - ) - return self._room_id - - def auth_event_ids(self) -> StrCollection: - """Returns the list of auth event IDs. The order matches the order - specified in the event, though there is no meaning to it. - Returns: - The list of event IDs of this event's auth_events - Includes the creation event ID for convenience of all the codepaths - which expects the auth chain to include the creator ID, even though - it's explicitly not included on the wire. Excludes the create event - for the create event itself. - """ - create_event_id = "$" + self.room_id[1:] - assert create_event_id not in self._dict["auth_events"] - if self.type == EventTypes.Create and self.get_state_key() == "": - return self._dict["auth_events"] # should be [] - return [*self._dict["auth_events"], create_event_id] - - -class FrozenEventVMSC4242(FrozenEventV4): - """FrozenEventVMSC4242, which differs from FrozenEventV4 only in the addition of prev_state_events""" - - format_version = EventFormatVersions.ROOM_VMSC4242 - prev_state_events: DictProperty[StrCollection] = DictProperty("prev_state_events") - - def __init__( - self, - event_dict: JsonDict, - room_version: RoomVersion, - internal_metadata_dict: JsonDict | None = None, - rejected_reason: str | None = None, - ): - # Similar to how we assert event_id isn't in V2+ events, we do the same with auth_events. - # We don't expect `auth_events` in the wire format because we calculate it from prev_state_events. - assert "auth_events" not in event_dict - super().__init__( - event_dict=event_dict, - room_version=room_version, - internal_metadata_dict=internal_metadata_dict, - rejected_reason=rejected_reason, - ) - - def auth_event_ids(self) -> StrCollection: - """Returns the list of _calculated_ auth event IDs. - - Returns: - The list of event IDs of this event's auth events - """ - # Catches cases where we accidentally call auth_event_ids() prior to calculating what they - # actually are. The exception being the m.room.create event which has no auth events. - if self.type != EventTypes.Create: - assert len(self.internal_metadata.calculated_auth_event_ids) > 0 - return self.internal_metadata.calculated_auth_event_ids - - def __repr__(self) -> str: - rejection = f"REJECTED={self.rejected_reason}, " if self.rejected_reason else "" - - return ( - f"<{self.__class__.__name__} " - f"{rejection}" - f"event_id={self.event_id}, " - f"type={self.get('type')}, " - f"state_key={self.get('state_key')}, " - f"prev_events={self.get('prev_events')}, " - f"prev_state_events={self.get('prev_state_events')}, " - f"outlier={self.internal_metadata.is_outlier()}" - ">" - ) - - -def _event_type_from_format_version( - format_version: int, -) -> type[FrozenEvent | FrozenEventV2 | FrozenEventV3 | FrozenEventVMSC4242]: - """Returns the python type to use to construct an Event object for the - given event format version. - - Args: - format_version: The event format version - - Returns: - A type that can be initialized as per the initializer of `FrozenEvent` - """ - - if format_version == EventFormatVersions.ROOM_V1_V2: - return FrozenEvent - elif format_version == EventFormatVersions.ROOM_V3: - return FrozenEventV2 - elif format_version == EventFormatVersions.ROOM_V4_PLUS: - return FrozenEventV3 - elif format_version == EventFormatVersions.ROOM_VMSC4242: - return FrozenEventVMSC4242 - elif format_version == EventFormatVersions.ROOM_V11_HYDRA_PLUS: - return FrozenEventV4 - else: - raise Exception("No event format %r" % (format_version,)) - def make_event_from_dict( event_dict: JsonDict, room_version: RoomVersion = RoomVersions.V1, internal_metadata_dict: JsonDict | None = None, rejected_reason: str | None = None, -) -> EventBase: +) -> Event: """Construct an EventBase from the given event dict""" - event_type = _event_type_from_format_version(room_version.event_format) - return event_type( - event_dict, room_version, internal_metadata_dict or {}, rejected_reason - ) + + try: + return Event( + event_dict=event_dict, + room_version=room_version, + internal_metadata_dict=internal_metadata_dict or {}, + rejected_reason=rejected_reason, + ) + except ValueError: + raise SynapseError(400, "Invalid event dict", Codes.BAD_JSON) @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -716,7 +94,7 @@ class _EventRelation: aggregation_key: str | None -def relation_from_event(event: EventBase) -> _EventRelation | None: +def relation_from_event(event: Event) -> _EventRelation | None: """ Attempt to parse relation information an event. diff --git a/synapse/events/py_protocol.py b/synapse/events/py_protocol.py index d9ac8c066f..a34170cd93 100644 --- a/synapse/events/py_protocol.py +++ b/synapse/events/py_protocol.py @@ -33,7 +33,6 @@ attributes. """ -import abc from typing import ( TYPE_CHECKING, Sequence, @@ -42,12 +41,14 @@ from typing_extensions import TypeIs from synapse.events import EventBase +from synapse.synapse_rust.events import Event +from synapse.types import StrCollection if TYPE_CHECKING: from synapse.events.snapshot import EventContext, EventPersistencePair -class _DisableIsInstance(abc.ABCMeta): +class _DisableIsInstance(type): """Metaclass which disables isinstance checks on classes which use it, by making isinstance() raise NotImplementedError. @@ -61,15 +62,38 @@ def __instancecheck__(cls, instance: object) -> bool: raise NotImplementedError("Instance cannot be used.") -class EventProtocol(EventBase, metaclass=_DisableIsInstance): - """Helper subclass that allows type narrowing for `EventBase` objects.""" +# We now define `EventProtocol` as a helper class for type narrowing. +# +# During type checking, we want the type narrowed event classes to still have +# all the fields as a normal `Event`, so we make `EventProtocol` a subclass of +# `Event`. +# +# However, at runtime we a) can't subclass `Event` because it's a Rust class, +# and b) don't want to allow `isinstance` checks against `EventProtocol` (as +# it's purely a type annotation helper, not a real class). So at runtime, we +# make `EventProtocol` a class with a metaclass that raises on `isinstance` +# checks. +if TYPE_CHECKING: + + class EventProtocol(Event): + """Helper subclass that allows type narrowing for `EventBase` objects.""" + +else: + + class EventProtocol(metaclass=_DisableIsInstance): + """Helper subclass that allows type narrowing for `EventBase` objects.""" + + def __new__(cls): + raise NotImplementedError( + f"{cls.__name__} cannot be instantiated as it is not a real class." + ) class MSC4242Event(EventProtocol): """Marker protocol for events in MSC4242 rooms. This allows us to narrow the type of events.""" - prev_state_events: list[str] + prev_state_events: StrCollection def supports_msc4242_state_dag(event: EventBase) -> TypeIs[MSC4242Event]: diff --git a/synapse/events/utils.py b/synapse/events/utils.py index adbede7f16..54f662796b 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -39,18 +39,16 @@ CANONICALJSON_MAX_INT, CANONICALJSON_MIN_INT, MAX_PDU_SIZE, - EventContentFields, EventTypes, EventUnsignedContentFields, RelationTypes, ) from synapse.api.errors import Codes, SynapseError -from synapse.api.room_versions import RoomVersion from synapse.logging.opentracing import SynapseTags, set_tag, trace -from synapse.synapse_rust.events import Unsigned +from synapse.synapse_rust.events import Unsigned, redact_event from synapse.types import JsonDict, Requester -from . import EventBase, FrozenEventV2, StrippedStateEvent, make_event_from_dict +from . import EventBase, StrippedStateEvent if TYPE_CHECKING: from synapse.handlers.relations import BundledAggregations @@ -78,177 +76,18 @@ def prune_event(event: EventBase) -> EventBase: the user has specified, but we do want to keep necessary information like type, state_key etc. """ - pruned_event_dict = prune_event_dict(event.room_version, event.get_dict()) - - pruned_event = make_event_from_dict( - pruned_event_dict, event.room_version, event.internal_metadata.get_dict() - ) - - # Copy the bits of `internal_metadata` that aren't returned by `get_dict` - pruned_event.internal_metadata.stream_ordering = ( - event.internal_metadata.stream_ordering - ) - pruned_event.internal_metadata.instance_name = event.internal_metadata.instance_name - pruned_event.internal_metadata.outlier = event.internal_metadata.outlier - pruned_event.internal_metadata.redacted_by = event.internal_metadata.redacted_by - - # Mark the event as redacted - pruned_event.internal_metadata.redacted = True - - return pruned_event + return redact_event(event) def clone_event(event: EventBase) -> EventBase: """Take a copy of the event. - This is mostly useful because it does a *shallow* copy of the `unsigned` data, - which means it can then be updated without corrupting the in-memory cache. Note that - other properties of the event, such as `content`, are *not* (currently) copied here. - """ - # XXX: We rely on at least one of `event.get_dict()` and `make_event_from_dict()` - # making a copy of `unsigned`. Currently, both do, though I don't really know why. - # Still, as long as they do, there's not much point doing yet another copy here. - new_event = make_event_from_dict( - event.get_dict(), event.room_version, event.internal_metadata.get_dict() - ) - - # Starting FrozenEventV2, the event ID is an (expensive) hash of the event. This is - # lazily computed when we get the FrozenEventV2.event_id property, then cached in - # _event_id field. Later FrozenEvent formats all inherit from FrozenEventV2, so we - # can use the same logic here. - if isinstance(event, FrozenEventV2) and isinstance(new_event, FrozenEventV2): - # If we already pre-computed the event ID, use it. - new_event._event_id = event._event_id - - # Copy the bits of `internal_metadata` that aren't returned by `get_dict`. - new_event.internal_metadata.stream_ordering = ( - event.internal_metadata.stream_ordering - ) - new_event.internal_metadata.instance_name = event.internal_metadata.instance_name - new_event.internal_metadata.outlier = event.internal_metadata.outlier - new_event.internal_metadata.redacted_by = event.internal_metadata.redacted_by - - return new_event - - -def prune_event_dict(room_version: RoomVersion, event_dict: JsonDict) -> JsonDict: - """Redacts the event_dict in the same way as `prune_event`, except it - operates on dicts rather than event objects - - Returns: - A copy of the pruned event dict + Most fields of the event are immutable, however fields such as `unsigned`, + `signatures` and `internal_metadata` are mutable. Cloning the event allows + us to edit such fields without affecting the original event. """ - allowed_keys = [ - "event_id", - "sender", - "room_id", - "hashes", - "signatures", - "content", - "type", - "state_key", - "depth", - "prev_events", - "auth_events", - "origin_server_ts", - ] - - # Earlier room versions from had additional allowed keys. - if not room_version.updated_redaction_rules: - allowed_keys.extend(["prev_state", "membership", "origin"]) - # Custom room versions add new allowed keys and remove others - if room_version.msc4242_state_dags: - allowed_keys.extend(["prev_state_events"]) - allowed_keys.remove("auth_events") - - event_type = event_dict["type"] - - new_content = {} - - def add_fields(*fields: str) -> None: - for field in fields: - if field in event_dict["content"]: - new_content[field] = event_dict["content"][field] - - if event_type == EventTypes.Member: - add_fields("membership") - if room_version.restricted_join_rule_fix: - add_fields(EventContentFields.AUTHORISING_USER) - if room_version.updated_redaction_rules: - # Preserve the signed field under third_party_invite. - third_party_invite = event_dict["content"].get("third_party_invite") - if isinstance(third_party_invite, collections.abc.Mapping): - new_content["third_party_invite"] = {} - if "signed" in third_party_invite: - new_content["third_party_invite"]["signed"] = third_party_invite[ - "signed" - ] - - elif event_type == EventTypes.Create: - if room_version.updated_redaction_rules: - # MSC2176 rules state that create events cannot have their `content` redacted. - new_content = event_dict["content"] - if not room_version.implicit_room_creator: - # Some room versions give meaning to `creator` - add_fields("creator") - if room_version.msc4291_room_ids_as_hashes: - # room_id is not allowed on the create event as it's derived from the event ID - allowed_keys.remove("room_id") - - elif event_type == EventTypes.JoinRules: - add_fields("join_rule") - if room_version.restricted_join_rule: - add_fields("allow") - elif event_type == EventTypes.PowerLevels: - add_fields( - "users", - "users_default", - "events", - "events_default", - "state_default", - "ban", - "kick", - "redact", - ) - - if room_version.updated_redaction_rules: - add_fields("invite") - - elif event_type == EventTypes.Aliases and room_version.special_case_aliases_auth: - add_fields("aliases") - elif event_type == EventTypes.RoomHistoryVisibility: - add_fields("history_visibility") - elif event_type == EventTypes.Redaction and room_version.updated_redaction_rules: - add_fields("redacts") - - # Protect the rel_type and event_id fields under the m.relates_to field. - if room_version.msc3389_relation_redactions: - relates_to = event_dict["content"].get("m.relates_to") - if isinstance(relates_to, collections.abc.Mapping): - new_relates_to = {} - for field in ("rel_type", "event_id"): - if field in relates_to: - new_relates_to[field] = relates_to[field] - # Only include a non-empty relates_to field. - if new_relates_to: - new_content["m.relates_to"] = new_relates_to - - allowed_fields = {k: v for k, v in event_dict.items() if k in allowed_keys} - - allowed_fields["content"] = new_content - - unsigned: JsonDict = {} - allowed_fields["unsigned"] = unsigned - - event_unsigned = event_dict.get("unsigned", {}) - - if "age_ts" in event_unsigned: - unsigned["age_ts"] = event_unsigned["age_ts"] - if "replaces_state" in event_unsigned: - unsigned["replaces_state"] = event_unsigned["replaces_state"] - - return allowed_fields + return event.deep_copy() def _copy_field(src: JsonDict, dst: JsonDict, field: list[str]) -> None: diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 2b5ef5fbac..5b33454325 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -21,7 +21,6 @@ # -import copy import itertools import logging from typing import ( @@ -1192,7 +1191,7 @@ async def _execute(pdu: EventBase) -> None: # NB: We *need* to copy to ensure that we don't have multiple # references being passed on, as that causes... issues. signed_state = [ - copy.copy(valid_pdus_map[p.event_id]) + valid_pdus_map[p.event_id].deep_copy() for p in state if p.event_id in valid_pdus_map ] @@ -1203,11 +1202,6 @@ async def _execute(pdu: EventBase) -> None: if p.event_id in valid_pdus_map ] - # NB: We *need* to copy to ensure that we don't have multiple - # references being passed on, as that causes... issues. - for s in signed_state: - s.internal_metadata = s.internal_metadata.copy() - # double-check that the auth chain doesn't include a different create event auth_chain_create_events = [ e.event_id diff --git a/synapse/module_api/callbacks/third_party_event_rules_callbacks.py b/synapse/module_api/callbacks/third_party_event_rules_callbacks.py index 65f5a6b183..68053d65a3 100644 --- a/synapse/module_api/callbacks/third_party_event_rules_callbacks.py +++ b/synapse/module_api/callbacks/third_party_event_rules_callbacks.py @@ -283,11 +283,6 @@ async def check_event_allowed( events = await self.store.get_events(prev_state_ids.values()) state_events = {(ev.type, ev.state_key): ev for ev in events.values()} - # Ensure that the event is frozen, to make sure that the module is not tempted - # to try to modify it. Any attempt to modify it at this point will invalidate - # the hashes and signatures. - event.freeze() - for callback in self._check_event_allowed_callbacks: try: res, replacement_data = await delay_cancellation( diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index 3783211a92..f6693e0923 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -58,7 +58,14 @@ from synapse.rest.client.room import SerializeMessagesDeps, encode_messages_response from synapse.storage.databases.main.room import RoomSortOrder from synapse.streams.config import PaginationConfig -from synapse.types import JsonDict, RoomID, ScheduledTask, UserID, create_requester +from synapse.types import ( + JsonDict, + JsonMapping, + RoomID, + ScheduledTask, + UserID, + create_requester, +) from synapse.types.state import StateFilter if TYPE_CHECKING: @@ -682,6 +689,7 @@ async def on_POST( create_event = filtered_room_state[(EventTypes.Create, "")] power_levels = filtered_room_state.get((EventTypes.PowerLevels, "")) + pl_content: JsonMapping if power_levels is not None: # We pick the local user with the highest power. user_power = power_levels.content.get("users", {}) diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index 62e84f5ac5..13f958d998 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -36,7 +36,6 @@ Iterable, Sequence, TypeVar, - cast, ) import attr @@ -870,7 +869,7 @@ async def _process_state_dag_forward_extremities_and_state_delta( ) new_room_dag_fwd_extrems = await self._calculate_new_extremities( room_id, - cast(list[EventPersistencePair], event_contexts), + event_contexts, existing_room_dag_fwd_extrems, ) assert new_room_dag_fwd_extrems, ( @@ -889,7 +888,7 @@ async def _process_state_dag_forward_extremities_and_state_delta( ): (current_state, delta_ids, _) = await self._get_new_state_after_events( room_id, - cast(list[EventPersistencePair], event_contexts), + event_contexts, existing_state_dag_fwd_extrems, new_state_dag_fwd_extrems, # do not prune forward extremities in the state DAG @@ -923,7 +922,7 @@ async def _process_state_dag_forward_extremities_and_state_delta( # extremities. is_still_joined = await self._is_server_still_joined( room_id, - cast(list[EventPersistencePair], event_contexts), + event_contexts, delta, ) if not is_still_joined: @@ -1053,7 +1052,7 @@ async def _calculate_new_state_dag_extremities( async def _calculate_new_extremities( self, room_id: str, - event_contexts: list[EventPersistencePair], + event_contexts: Sequence[EventPersistencePair], latest_event_ids: AbstractSet[str], ) -> set[str]: """Calculates the new forward extremities for a room given events to @@ -1113,7 +1112,7 @@ async def _calculate_new_extremities( async def _get_new_state_after_events( self, room_id: str, - events_context: list[EventPersistencePair], + events_context: Sequence[EventPersistencePair], old_latest_event_ids: AbstractSet[str], new_latest_event_ids: set[str], should_prune: bool = True, @@ -1297,7 +1296,7 @@ async def _prune_extremities( new_latest_event_ids: set[str], resolved_state_group: int, event_id_to_state_group: dict[str, int], - events_context: list[EventPersistencePair], + events_context: Sequence[EventPersistencePair], ) -> set[str]: """See if we can prune any of the extremities after calculating the resolved state. @@ -1434,7 +1433,7 @@ async def _calculate_state_delta( async def _is_server_still_joined( self, room_id: str, - ev_ctx_rm: list[EventPersistencePair], + ev_ctx_rm: Sequence[EventPersistencePair], delta: DeltaState, ) -> bool: """Check if the server will still be joined after the given events have diff --git a/synapse/storage/databases/main/censor_events.py b/synapse/storage/databases/main/censor_events.py index a5ae4bf506..92770b6590 100644 --- a/synapse/storage/databases/main/censor_events.py +++ b/synapse/storage/databases/main/censor_events.py @@ -22,7 +22,6 @@ import logging from typing import TYPE_CHECKING -from synapse.events.utils import prune_event_dict from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.storage._base import SQLBaseStore from synapse.storage.database import ( @@ -32,6 +31,7 @@ ) from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.databases.main.events_worker import EventsWorkerStore +from synapse.synapse_rust.events import redact_event from synapse.util.duration import Duration from synapse.util.json import json_encoder @@ -123,9 +123,7 @@ async def _censor_redactions(self) -> None: ): # Redaction was allowed pruned_json: str | None = json_encoder.encode( - prune_event_dict( - original_event.room_version, original_event.get_dict() - ) + redact_event(original_event).get_pdu_json() ) else: # Redaction wasn't allowed @@ -190,9 +188,7 @@ def delete_expired_event_txn(txn: LoggingTransaction) -> None: return # Prune the event's dict then convert it to JSON. - pruned_json = json_encoder.encode( - prune_event_dict(event.room_version, event.get_dict()) - ) + pruned_json = json_encoder.encode(redact_event(event).get_pdu_json()) # Update the event_json table to replace the event's JSON with the pruned # JSON. diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 5aab0067fc..84b38f4bf2 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -915,7 +915,9 @@ def calculate_chain_cover_index_for_events_txn( # instances as we'll potentially be pulling more events from the DB and # we don't need the overhead of fetching/parsing the full event JSON. event_to_types = {e.event_id: (e.type, e.state_key) for e in state_events} - event_to_auth_chain = {e.event_id: e.auth_event_ids() for e in state_events} + event_to_auth_chain: dict[str, StrCollection] = { + e.event_id: e.auth_event_ids() for e in state_events + } event_to_room_id = {e.event_id: e.room_id for e in state_events} return self._calculate_chain_cover_index( diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index c0d218398d..9a11f9b9bb 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -38,7 +38,6 @@ resign_event, ) from synapse.events import EventBase, make_event_from_dict -from synapse.events.utils import prune_event_dict from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause from synapse.storage.database import ( DatabasePool, @@ -63,6 +62,7 @@ from synapse.storage.databases.main.stream import StreamWorkerStore from synapse.storage.engines import PostgresEngine from synapse.storage.types import Cursor +from synapse.synapse_rust.events import redact_event from synapse.types import JsonDict, RoomStreamToken, StateMap, StrCollection from synapse.types.handlers import SLIDING_SYNC_DEFAULT_BUMP_EVENT_TYPES from synapse.types.state import StateFilter @@ -2831,7 +2831,7 @@ def _fetch_next_events_txn( # Verify the signature is genuinely from this key. We prune # first since signatures are computed over the redacted form. - pruned = prune_event_dict(event.room_version, event.get_pdu_json()) + pruned = redact_event(event).get_pdu_json() try: verify_signed_json(pruned, self.hs.hostname, old_verify_key) except SignatureVerifyException: diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 6f26bd17ce..4b372de141 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -37,6 +37,7 @@ import attr from prometheus_client import Gauge +from typing_extensions import assert_never from twisted.internet import defer @@ -775,6 +776,14 @@ async def get_events_as_list( continue elif redact_behaviour == EventRedactBehaviour.redact: event = entry.redacted_event + elif redact_behaviour == EventRedactBehaviour.as_is: + # Allow event through as is + pass + else: + # We (should) have covered all possible values of + # redact_behaviour, so this is unreachable. + assert_never(redact_behaviour) + raise ValueError(f"Unknown redact_behaviour {redact_behaviour}") events.append(event) @@ -1507,12 +1516,17 @@ async def _fetch_event_ids_and_get_outstanding_redactions( ) continue - original_ev = make_event_from_dict( - event_dict=d, - room_version=room_version, - internal_metadata_dict=internal_metadata, - rejected_reason=rejected_reason, - ) + try: + original_ev = make_event_from_dict( + event_dict=d, + room_version=room_version, + internal_metadata_dict=internal_metadata, + rejected_reason=rejected_reason, + ) + except SynapseError: + logger.error("Unable to parse event from database: %s", event_id) + continue + original_ev.internal_metadata.stream_ordering = row.stream_ordering original_ev.internal_metadata.instance_name = row.instance_name original_ev.internal_metadata.outlier = row.outlier diff --git a/synapse/synapse_rust/events.pyi b/synapse/synapse_rust/events.pyi index 5b55d47f0d..9a69438c1d 100644 --- a/synapse/synapse_rust/events.pyi +++ b/synapse/synapse_rust/events.pyi @@ -12,7 +12,9 @@ from typing import Any, Iterator, Mapping -from synapse.types import JsonDict, JsonMapping +from synapse.synapse_rust.room_versions import RoomVersion +from synapse.types import JsonDict, JsonMapping, StrSequence +from synapse.util.duration import Duration class EventInternalMetadata: def __init__(self, internal_metadata_dict: JsonDict): ... @@ -159,60 +161,60 @@ class Signatures: """A class representing the signatures on an event.""" def __init__(self, signatures: Mapping[str, Mapping[str, str]] | None = None): ... - def get_signature(self, server_name: str, key_id: str) -> str | None: ... - """Get the signature for the given server name and key ID, if it exists.""" + def get_signature(self, server_name: str, key_id: str) -> str | None: + """Get the signature for the given server name and key ID, if it exists.""" - def __getitem__(self, server_name: str) -> Mapping[str, str]: ... - """Get the signatures for the given server name. Raises KeyError if there - are no signatures for that server.""" + def __getitem__(self, server_name: str) -> Mapping[str, str]: + """Get the signatures for the given server name. Raises KeyError if there + are no signatures for that server.""" - def __contains__(self, server_name: Any) -> bool: ... - """Check if there are signatures for the given server name.""" + def __contains__(self, server_name: Any) -> bool: + """Check if there are signatures for the given server name.""" - def __len__(self) -> int: ... - """Return the number of servers that have signatures.""" + def __len__(self) -> int: + """Return the number of servers that have signatures.""" - def add_signature(self, server_name: str, key_id: str, signature: str) -> None: ... - """Add a signature for the given server name and key ID.""" + def add_signature(self, server_name: str, key_id: str, signature: str) -> None: + """Add a signature for the given server name and key ID.""" - def update(self, signatures: Mapping[str, Mapping[str, str]]) -> None: ... - """Update the signatures with the given signatures. + def update(self, signatures: Mapping[str, Mapping[str, str]]) -> None: + """Update the signatures with the given signatures. - Will overwrite all existing signatures for the server names provided. - """ + Will overwrite all existing signatures for the server names provided. + """ - def as_dict(self) -> dict[str, dict[str, str]]: ... - """Return a copy of the signatures as a dictionary.""" + def as_dict(self) -> dict[str, dict[str, str]]: + """Return a copy of the signatures as a dictionary.""" class Unsigned: """A class representing the unsigned data of an event.""" def __init__(self, unsigned_dict: JsonMapping): ... - def __getitem__(self, key: str) -> Any: ... - """Get the value for the given key. + def __getitem__(self, key: str) -> Any: + """Get the value for the given key. - Raises KeyError if the key is unset or not recognised.""" + Raises KeyError if the key is unset or not recognised.""" - def __setitem__(self, key: str, value: Any) -> None: ... - """Set the value for the given key. + def __setitem__(self, key: str, value: Any) -> None: + """Set the value for the given key. - Raises KeyError if the key is not recognised.""" + Raises KeyError if the key is not recognised.""" - def __delitem__(self, key: str) -> None: ... - """Delete the value for the given key. + def __delitem__(self, key: str) -> None: + """Delete the value for the given key. - Raises KeyError if the key is unset or not recognised.""" + Raises KeyError if the key is unset or not recognised.""" def __contains__(self, key: Any) -> bool: ... - def get(self, key: str, default: Any = None) -> Any: ... - """Get the value for the given key, or ``default`` if the key is unset.""" + def get(self, key: str, default: Any = None) -> Any: + """Get the value for the given key, or ``default`` if the key is unset.""" - def for_persistence(self) -> JsonDict: ... - """Return a dict of the fields that should be persisted to the database.""" + def for_persistence(self) -> JsonDict: + """Return a dict of the fields that should be persisted to the database.""" - def for_event(self) -> JsonDict: ... - """Return a dict of all unsigned fields, including those only kept in - memory, suitable for inclusion in an event.""" + def for_event(self) -> JsonDict: + """Return a dict of all unsigned fields, including those only kept in + memory, suitable for inclusion in an event.""" class JsonObject(Mapping[str, Any]): """Immutable JSON object mapping.""" @@ -222,3 +224,99 @@ class JsonObject(Mapping[str, Any]): def __getitem__(self, key: str) -> Any: ... def __iter__(self) -> Iterator[str]: ... def __eq__(self, other: object) -> bool: ... + +class Event: + """Represents a Matrix event.""" + + def __init__( + self, + event_dict: JsonDict, + room_version: RoomVersion, + internal_metadata_dict: JsonDict, + rejected_reason: str | None, + ) -> None: ... + def get_dict(self) -> JsonDict: + """Convert the event to a dictionary suitable for serialisation.""" + + def get_dict_for_persistence(self) -> JsonDict: + """Like ``get_dict``, but serializes ``unsigned`` in a form suitable for + persistence.""" + + def get_pdu_json(self, time_now: int | None = None) -> JsonDict: + """Like ``get_dict``, but serializes ``unsigned`` in a form suitable + for sending over federation.""" + + def get_templated_pdu_json(self) -> JsonDict: + """Like ``get_dict``, except strips fields like ``signatures``, + ``hashes`` and ``unsigned`` so that the result is suitable as a template for + creating new events. Used in make_{join,leave,knock} flows.""" + + @property + def event_id(self) -> str: ... + @property + def room_id(self) -> str: ... + @property + def signatures(self) -> Signatures: ... + @property + def content(self) -> JsonMapping: ... + @property + def depth(self) -> int: ... + @property + def hashes(self) -> dict[str, str]: ... + @property + def origin_server_ts(self) -> int: ... + @property + def sender(self) -> str: ... + @property + def state_key(self) -> str: ... + @property + def type(self) -> str: ... + @property + def unsigned(self) -> Unsigned: ... + @property + def internal_metadata(self) -> EventInternalMetadata: ... + @property + def rejected_reason(self) -> str | None: ... + @property + def room_version(self) -> RoomVersion: ... + @property + def format_version(self) -> int: + """The EventFormatVersion implemented by this event.""" + + @property + def membership(self) -> Any: ... + @property + def redacts(self) -> Any | None: ... + def prev_event_ids(self) -> StrSequence: + """Returns the list of prev event IDs.""" + + def auth_event_ids(self) -> StrSequence: + """Returns the list of auth event IDs""" + + def is_state(self) -> bool: ... + def get_state_key(self) -> str | None: + """Get the state key of this event, or None if it's not a state event.""" + def __contains__(self, key: str) -> bool: ... + def get(self, key: str, default: Any = None) -> Any: ... + def items(self) -> list[tuple[str, Any]]: ... + def keys(self) -> list[str]: ... + def deep_copy(self) -> "Event": + """Returns a deep copy of this object, such that modifying the copy will + not affect the original.""" + + def sticky_duration(self) -> Duration | None: + """If this event has the ``msc4354_sticky`` top-level field, returns a + ``SynapseDuration`` representing the sticky duration. Otherwise returns + ``None``.""" + +def redact_event(event: Event) -> Event: + """Returns a pruned version of the given event, which removes all keys we + don't know about or think could potentially be dodgy. + """ + +def redact_event_dict(room_version: RoomVersion, event_dict: JsonMapping) -> JsonDict: + """Returns a pruned version of the given event dict, which removes all keys + we don't know about or think could potentially be dodgy. + + Returns the redacted event as a dict. + """ diff --git a/tests/events/test_py_protocol.py b/tests/events/test_py_protocol.py index 306e3c1704..cfcc3648ca 100644 --- a/tests/events/test_py_protocol.py +++ b/tests/events/test_py_protocol.py @@ -16,7 +16,7 @@ from unittest.mock import Mock from synapse.api.room_versions import RoomVersion, RoomVersions -from synapse.events import EventBase, FrozenEvent, make_event_from_dict +from synapse.events import EventBase from synapse.events.py_protocol import ( EventProtocol, MSC4242Event, @@ -24,20 +24,20 @@ supports_msc4242_state_dag, ) +from tests.test_utils.event_builders import make_test_event from tests.unittest import TestCase def _make_event(room_version: RoomVersion) -> EventBase: """Helper to make an EventBase with the given room version.""" - event_dict = { - "content": {}, - "sender": "@user:example.com", - "type": "m.room.message", - "room_id": "!room:example.com", - } - if room_version.msc4242_state_dags: - event_dict["prev_state_events"] = [] - return make_event_from_dict(event_dict, room_version=room_version) + return make_test_event( + { + "sender": "@user:example.com", + "type": "m.room.message", + "room_id": "!room:example.com", + }, + room_version=room_version, + ) class TestMetaClass(TestCase): @@ -46,15 +46,14 @@ def test_is_instance(self) -> None: NotImplementedError, but that isinstance checks on EventBase and FrozenEvent still work as normal. """ - # EventBase and FrozenEvent should work as normal + # EventBase should work as normal self.assertFalse(isinstance(object(), EventBase)) - self.assertFalse(isinstance(object(), FrozenEvent)) with self.assertRaises(NotImplementedError): - isinstance(object(), EventProtocol) + isinstance(object(), MSC4242Event) with self.assertRaises(NotImplementedError): - isinstance(object(), MSC4242Event) + isinstance(object(), EventProtocol) class SupportsMSC4242StateDagTestCase(TestCase): diff --git a/tests/events/test_validator.py b/tests/events/test_validator.py index 3810fdb3da..082ae04a4c 100644 --- a/tests/events/test_validator.py +++ b/tests/events/test_validator.py @@ -19,10 +19,10 @@ class EventValidatorTestCase(HomeserverTestCase): - def test_validate_new_with_mentions_succeeds_even_when_frozen(self) -> None: + def test_validate_new_with_mentions_succeed(self) -> None: """ Test that `EventValidator.validate_new` accepts an event with valid `m.mentions` - content even when the event is frozen. + content. """ event = make_event_from_dict( { @@ -43,8 +43,5 @@ def test_validate_new_with_mentions_succeeds_even_when_frozen(self) -> None: }, room_version=RoomVersions.V9, ) - # Sanity check that the event is valid before freezing - EventValidator().validate_new(event, self.hs.config) - event.freeze() - # Event should still be valid after freezing + EventValidator().validate_new(event, self.hs.config) diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index 9e44b1dc1e..736f251c27 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -36,7 +36,7 @@ from synapse.api.room_versions import RoomVersions from synapse.appservice import ApplicationService from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import EventBase, FrozenEventV3 +from synapse.events import EventBase, make_event_from_dict from synapse.federation.federation_client import SendJoinResult from synapse.federation.transport.client import ( StateRequestResponse, @@ -677,7 +677,7 @@ def _build_public_room(self) -> StateMap[EventBase]: self.REMOTE1_SERVER_SIGNATURE_KEY, ) - create_event = FrozenEventV3(create_event_dict, room_version, {}, None) + create_event = make_event_from_dict(create_event_dict, room_version) events.append(create_event) room_version = self.hs.config.server.default_room_version @@ -700,7 +700,7 @@ def _build_public_room(self) -> StateMap[EventBase]: self.hs.hostname, self.hs.signing_key, ) - join_event = FrozenEventV3(join_event_dict, room_version, {}, None) + join_event = make_event_from_dict(join_event_dict, room_version) events.append(join_event) # Then set the join rules to public @@ -722,7 +722,7 @@ def _build_public_room(self) -> StateMap[EventBase]: self.REMOTE1_SERVER_NAME, self.REMOTE1_SERVER_SIGNATURE_KEY, ) - join_rules_event = FrozenEventV3(join_rules_event_dict, room_version, {}, None) + join_rules_event = make_event_from_dict(join_rules_event_dict, room_version) events.append(join_rules_event) return {(event.type, event.state_key): event for event in events} @@ -733,7 +733,7 @@ def _build_signed_join_event( user: str, signing_key: SigningKey, state: StateMap[EventBase], - ) -> FrozenEventV3: + ) -> EventBase: """Build a join event for the local user, signed by the local server.""" latest_event = max(state.values(), key=lambda e: e.depth) @@ -759,7 +759,7 @@ def _build_signed_join_event( get_domain_from_id(user), signing_key, ) - return FrozenEventV3(join_event_dict, room_version, {}, None) + return make_event_from_dict(join_event_dict, room_version) @parameterized.expand([("not_pruned", False), ("pruned", True)]) @patch( diff --git a/tests/handlers/test_room_member.py b/tests/handlers/test_room_member.py index b550c2420b..9cc7ebfa80 100644 --- a/tests/handlers/test_room_member.py +++ b/tests/handlers/test_room_member.py @@ -8,7 +8,7 @@ from synapse.api.constants import AccountDataTypes, EventTypes, Membership from synapse.api.errors import Codes, LimitExceededError, SynapseError from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import FrozenEventV3 +from synapse.events import Event from synapse.federation.federation_client import SendJoinResult from synapse.server import HomeServer from synapse.types import UserID, create_requester @@ -124,7 +124,7 @@ def test_remote_joins_contribute_to_rate_limit(self) -> None: create_event_source, self.hs.config.server.default_room_version, ) - create_event = FrozenEventV3( + create_event = Event( create_event_source, self.hs.config.server.default_room_version, {}, @@ -148,7 +148,7 @@ def test_remote_joins_contribute_to_rate_limit(self) -> None: self.hs.hostname, self.hs.signing_key, ) - join_event = FrozenEventV3( + join_event = Event( join_event_source, self.hs.config.server.default_room_version, {}, diff --git a/tests/handlers/test_room_policy.py b/tests/handlers/test_room_policy.py index 4f2188b8e7..c67ea9b0e0 100644 --- a/tests/handlers/test_room_policy.py +++ b/tests/handlers/test_room_policy.py @@ -27,7 +27,6 @@ from synapse.rest import admin from synapse.rest.client import filter, login, room, sync from synapse.server import HomeServer -from synapse.synapse_rust.events import Signatures from synapse.types import JsonDict, UserID from synapse.util.clock import Clock @@ -182,7 +181,7 @@ def _sign_with_random_key(self, server_name: str, event: EventBase) -> None: non_policyserver_key = signedjson.key.generate_signing_key( "non_policyserver_key" ) - event.signatures = Signatures( + event.signatures.update( compute_event_signature( event.room_version, event.get_dict(), diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py index 2ba5da3b95..3114675052 100644 --- a/tests/module_api/test_api.py +++ b/tests/module_api/test_api.py @@ -841,10 +841,10 @@ def test_event_deprecated_methods(self) -> None: create_event = state[(EventTypes.Create, "")] # `.user_id` is a deprecated alias for `.sender`. - self.assertEqual(create_event.user_id, user_id) + self.assertEqual(create_event.user_id, user_id) # type: ignore[attr-defined] # The event supports looking up keys via `__getitem__` although deprecated - self.assertEqual(create_event["room_id"], room_id) + self.assertEqual(create_event["room_id"], room_id) # type: ignore[index] class ModuleApiWorkerTestCase(BaseModuleApiTestCase, BaseMultiWorkerStreamTestCase): diff --git a/tests/replication/storage/test_events.py b/tests/replication/storage/test_events.py index b7b94482ef..d7e6dfca83 100644 --- a/tests/replication/storage/test_events.py +++ b/tests/replication/storage/test_events.py @@ -272,6 +272,7 @@ def build_event( "origin_server_ts": self.event_id, "prev_events": prev_events, "auth_events": auth_events, + "hashes": {}, } if key is not None: event_dict["state_key"] = key diff --git a/tests/rest/client/test_third_party_rules.py b/tests/rest/client/test_third_party_rules.py index b4fa71ece8..5eaa6f9fb2 100644 --- a/tests/rest/client/test_third_party_rules.py +++ b/tests/rest/client/test_third_party_rules.py @@ -236,7 +236,10 @@ def test_cannot_modify_event(self) -> None: async def check( ev: EventBase, state: StateMap[EventBase] ) -> tuple[bool, JsonDict | None]: - ev.content = {"x": "y"} + # Try and modify the content, this will fail because the event is + # immutable. (We therefore need the type ignore linter, as the + # linter will pick this bug up) + ev.content = {"x": "y"} # type: ignore[misc] return True, None self.hs.get_module_api_callbacks().third_party_event_rules._check_event_allowed_callbacks = [ diff --git a/tests/state/test_v2.py b/tests/state/test_v2.py index 8b3b919f44..a9924e28b1 100644 --- a/tests/state/test_v2.py +++ b/tests/state/test_v2.py @@ -97,9 +97,6 @@ def to_event(self, auth_events: list[str], prev_events: list[str]) -> EventBase: Args: auth_events: list of event_ids prev_events: list of event_ids - - Returns: - FrozenEvent """ global ORIGIN_SERVER_TS diff --git a/tests/storage/test_msc4242_state_dag.py b/tests/storage/test_msc4242_state_dag.py index 2150bc0996..52a165b9b2 100644 --- a/tests/storage/test_msc4242_state_dag.py +++ b/tests/storage/test_msc4242_state_dag.py @@ -19,13 +19,13 @@ from synapse.api.constants import EventTypes from synapse.api.errors import SynapseError from synapse.api.room_versions import RoomVersions -from synapse.events.py_protocol import MSC4242Event, supports_msc4242_state_dag +from synapse.events import EventBase +from synapse.events.py_protocol import MSC4242Event from synapse.events.snapshot import EventContext from synapse.rest.client import room from synapse.server import HomeServer from synapse.util.clock import Clock -from tests.test_utils.event_builders import make_test_event from tests.unittest import HomeserverTestCase, override_config @@ -154,21 +154,16 @@ def _make_event( prev_state_events: list[str], rejected: bool = False, ) -> tuple[MSC4242Event, EventContext]: - ev = make_test_event( - { - "prev_state_events": prev_state_events, - "content": { - "membership": "join", - }, - "sender": "@unimportant:info", - "state_key": "@unimportant:info", - "type": "m.room.member", - "room_id": self.room_id, - }, - room_version=RoomVersions.MSC4242v12, - ) - ev._event_id = id # type: ignore[attr-defined] - assert supports_msc4242_state_dag(ev) + # We use a mock here to allow us to set the `event_id`. + # + # FIXME: Having consistent human-readable event IDs in these tests is + # nice but the `Mock` is less than ideal. It would be better to use a + # real event but that is more complex to set up. + ev = Mock(spec=EventBase) + ev.event_id = id + ev.prev_state_events = prev_state_events + ev.state_key = "@unimportant:info" + ev.is_state.return_value = True ctx = Mock() ctx.rejected = rejected return ev, ctx diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index c346245706..93e0b4a2b1 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -26,7 +26,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.room_versions import RoomVersion, RoomVersions -from synapse.events import EventBase +from synapse.events import EventBase, make_event_from_dict from synapse.events.builder import EventBuilder from synapse.server import HomeServer from synapse.synapse_rust.events import EventInternalMetadata @@ -238,11 +238,16 @@ async def build( prev_event_ids=prev_event_ids, auth_event_ids=auth_event_ids ) - built_event._event_id = self._event_id # type: ignore[attr-defined] - built_event._dict["event_id"] = self._event_id - assert built_event.event_id == self._event_id + event_dict = built_event.get_dict() + event_dict["event_id"] = self._event_id + rebuilt_event = make_event_from_dict( + event_dict, + room_version=built_event.room_version, + internal_metadata_dict=built_event.internal_metadata.get_dict(), + ) + assert rebuilt_event.event_id == self._event_id - return built_event + return rebuilt_event @property def room_id(self) -> str: diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index d51fa1f8ba..ba3b3802cf 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -35,7 +35,7 @@ ) from synapse.api.filtering import Filter from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import FrozenEventV3 +from synapse.events import Event from synapse.federation.federation_client import SendJoinResult from synapse.rest import admin from synapse.rest.client import login, room @@ -1385,7 +1385,7 @@ def test_remote_join(self) -> None: create_event_source, self.hs.config.server.default_room_version, ) - create_event = FrozenEventV3( + create_event = Event( create_event_source, self.hs.config.server.default_room_version, {}, @@ -1408,7 +1408,7 @@ def test_remote_join(self) -> None: creator_join_event_source, self.hs.config.server.default_room_version, ) - creator_join_event = FrozenEventV3( + creator_join_event = Event( creator_join_event_source, self.hs.config.server.default_room_version, {}, @@ -1433,7 +1433,7 @@ def test_remote_join(self) -> None: self.hs.hostname, self.hs.signing_key, ) - join_event = FrozenEventV3( + join_event = Event( join_event_source, self.hs.config.server.default_room_version, {}, diff --git a/tests/test_event_auth.py b/tests/test_event_auth.py index 4537186ee6..eeba22728e 100644 --- a/tests/test_event_auth.py +++ b/tests/test_event_auth.py @@ -93,8 +93,8 @@ def test_rejected_auth_events(self) -> None: RoomVersions.V9, creator, "public", + rejected_reason="stinky", ) - rejected_join_rules.rejected_reason = "stinky" auth_events.append(rejected_join_rules) event_store.add_event(rejected_join_rules) @@ -1180,7 +1180,10 @@ def _random_state_event( def _join_rules_event( - room_version: RoomVersion, sender: str, join_rule: str + room_version: RoomVersion, + sender: str, + join_rule: str, + rejected_reason: str | None = None, ) -> EventBase: return make_test_event( { @@ -1194,6 +1197,7 @@ def _join_rules_event( }, }, room_version=room_version, + rejected_reason=rejected_reason, ) diff --git a/tests/test_utils/event_builders.py b/tests/test_utils/event_builders.py index a8eb586c1f..a5d686801d 100644 --- a/tests/test_utils/event_builders.py +++ b/tests/test_utils/event_builders.py @@ -76,6 +76,20 @@ def make_test_event( **(event_dict or {}), **fields, } + + # For room versions where the create event's room_id is derived from its + # event ID (v11+ format), omit the default room_id on create events so each + # create event ends up with a distinct room_id. + # + # We can't do this in the `default_event_fields` as we don't know the event + # type at that point. + if ( + room_version.msc4291_room_ids_as_hashes + and merged["type"] == "m.room.create" + and merged["state_key"] == "" + ): + merged.pop("room_id", None) + return make_event_from_dict( merged, room_version=room_version, From 387dfabe3b445c71ef423a420d97e107d2fb30ab Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Jun 2026 12:34:36 +0100 Subject: [PATCH 10/26] Fix loading 'invalid' event from the database (#19816) Follow on from #19701. Some Synapse servers may have events in their database that don't pass the canonical JSON checks. This is bad, but we still want to be able to load them nonetheless. --- changelog.d/19816.misc | 1 + rust/src/events/utils.rs | 17 ++++++++--------- synapse/storage/databases/main/events_worker.py | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 changelog.d/19816.misc diff --git a/changelog.d/19816.misc b/changelog.d/19816.misc new file mode 100644 index 0000000000..4663e8b961 --- /dev/null +++ b/changelog.d/19816.misc @@ -0,0 +1 @@ +Port the python Event classes to Rust. diff --git a/rust/src/events/utils.rs b/rust/src/events/utils.rs index 7d33c52b44..b58edc9352 100644 --- a/rust/src/events/utils.rs +++ b/rust/src/events/utils.rs @@ -75,15 +75,14 @@ pub fn compute_event_reference_hash( redacted_value_mut.remove(UNSIGNED); redacted_value_mut.remove(AGE_TS); - let canonicalization_options = if room_version.strict_canonicaljson { - CanonicalizationOptions::strict() - } else { - CanonicalizationOptions::relaxed() - }; - - let json = - crate::canonical_json::to_string_canonical(&redacted_value_mut, canonicalization_options) - .map_err(|err| anyhow::anyhow!(err))?; + // We use `CanonicalizationOptions::relaxed()` as we have some events that + // have already been accepted with int fields outside the valid range. We + // still want to be able to load them and calculate their event ID. + let json = crate::canonical_json::to_string_canonical( + &redacted_value_mut, + CanonicalizationOptions::relaxed(), + ) + .map_err(|err| anyhow::anyhow!(err))?; let hash = Sha256::digest(json.as_bytes()); diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 4b372de141..65b7d251d9 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -1523,8 +1523,8 @@ async def _fetch_event_ids_and_get_outstanding_redactions( internal_metadata_dict=internal_metadata, rejected_reason=rejected_reason, ) - except SynapseError: - logger.error("Unable to parse event from database: %s", event_id) + except SynapseError as e: + logger.error("Unable to parse event from database %s: %s", event_id, e) continue original_ev.internal_metadata.stream_ordering = row.stream_ordering From 5bf825c0166079ff7d4afa9052dc2dddafdb6efa Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Jun 2026 16:57:55 +0100 Subject: [PATCH 11/26] Limit to-device EDU sizes (#19617) This is based on https://github.com/element-hq/synapse/pull/18416, which got reverted (#19614) due to it incorrectly rejecting to-device messages to users with many devices (and thus breaking message sending). Fix https://github.com/element-hq/synapse/issues/17035 A to-device message content looks like: ```jsonc { "@user:domain": {"device1": {...}, "device2": {...}}, ... } ``` The previous PR would split up into multiple EDUs, each with a subset of the users. However, if one user's entry was too large it would not further split it up and then error out. The main change in this PR is to allow splitting up a single user into multiple EDUs. Other changes: 1. Rename to `SOFT_MAX_EDU_SIZE` to indicate that we sometimes send EDUs with larger size than that, and its more a target than a hard limit. 2. Check early if any to-device message (to a specific device) is too large to send, even if we're not going to send it over federation. This ensures that we catch issues where clients try to send too large to-device. This still means that if a client send a large individual to-device message it will fail, but I don't believe we ever send such large to-device messages (normally they're in the range of a few KB). --- I ended up changing the implementation a bunch to make it easy to reuse the code to split up dictionaries. Instead of repeatedly splitting up the EDU until each bit fits into the size, we instead record the size of each entry in the dict and instead split up based on cumulative size. This means we call `encode_canonical_json` on each entry rather than once on the entire struct, but its not significantly slower to do so. -- cc @MatMaul @MadLittleMods --------- Co-authored-by: Mathieu Velten Co-authored-by: mcalinghee Co-authored-by: Eric Eastwood --- changelog.d/19617.bugfix | 1 + synapse/api/constants.py | 27 ++ .../sender/per_destination_queue.py | 13 +- synapse/handlers/devicemessage.py | 204 ++++++++++++-- synapse/storage/databases/main/deviceinbox.py | 47 +++- synapse/util/__init__.py | 108 +++++++ tests/handlers/test_appservice.py | 4 +- tests/rest/client/test_sendtodevice.py | 265 +++++++++++++++++- tests/util/test_split_dict.py | 201 +++++++++++++ 9 files changed, 832 insertions(+), 38 deletions(-) create mode 100644 changelog.d/19617.bugfix create mode 100644 tests/util/test_split_dict.py diff --git a/changelog.d/19617.bugfix b/changelog.d/19617.bugfix new file mode 100644 index 0000000000..71f181fed4 --- /dev/null +++ b/changelog.d/19617.bugfix @@ -0,0 +1 @@ +A long queue of to-device messages could prevent outgoing federation because of the size of the transaction, let's limit the to-device EDU size to a reasonable value. diff --git a/synapse/api/constants.py b/synapse/api/constants.py index eb9e6cc39b..acac057334 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -30,6 +30,33 @@ # the max size of a (canonical-json-encoded) event MAX_PDU_SIZE = 65536 +# This isn't spec'ed but is our own reasonable default to play nice with Synapse's +# `max_request_size`/`max_request_body_size`. We chose the same as `MAX_PDU_SIZE` as our +# `max_request_body_size` math is currently limited by 200 `MAX_PDU_SIZE` things. The +# spec for a `/federation/v1/send` request sets the limit at 100 EDU's and 50 PDU's +# which is below that 200 `MAX_PDU_SIZE` limit (`max_request_body_size`). +# +# Allowing oversized EDU's results in failed `/federation/v1/send` transactions (because +# the request overall can overrun the `max_request_body_size`) which are retried over +# and over and prevent other outbound federation traffic from happening. +# +# We may send EDU's that are larger than this, but we aim to avoid doing so. +SOFT_MAX_EDU_SIZE = 65536 + +# This is the maximum size of the content of a to-device message. This is not +# (yet) spec'ed but is our own reasonable default. We need to set a limit on the +# size of to-device message contents, as they get sent over federation and +# therefore need to fit inside transactions. +# +# https://github.com/matrix-org/matrix-spec/pull/2340 tracks adding this to the +# spec. +MAX_TO_DEVICE_CONTENT_SIZE = SOFT_MAX_EDU_SIZE + +# This is defined in the Matrix spec and enforced by the receiver. +MAX_EDUS_PER_TRANSACTION = 100 +# A transaction can contain up to 100 EDUs but synapse reserves 10 EDUs for other purposes +# like trickling out some device list updates. +NUMBER_OF_RESERVED_EDUS_PER_TRANSACTION = 10 # The maximum allowed size of an HTTP request. # Other than media uploads, the biggest request we expect to see is a fully-loaded diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index cdacf16d72..32f8630c9d 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -30,7 +30,11 @@ from twisted.internet import defer -from synapse.api.constants import EduTypes +from synapse.api.constants import ( + MAX_EDUS_PER_TRANSACTION, + NUMBER_OF_RESERVED_EDUS_PER_TRANSACTION, + EduTypes, +) from synapse.api.errors import ( FederationDeniedError, HttpResponseException, @@ -51,9 +55,6 @@ if TYPE_CHECKING: import synapse.server -# This is defined in the Matrix spec and enforced by the receiver. -MAX_EDUS_PER_TRANSACTION = 100 - logger = logging.getLogger(__name__) @@ -798,7 +799,9 @@ async def __aenter__(self) -> tuple[list[EventBase], list[Edu]]: ( to_device_edus, device_stream_id, - ) = await self.queue._get_to_device_message_edus(edu_limit - 10) + ) = await self.queue._get_to_device_message_edus( + edu_limit - NUMBER_OF_RESERVED_EDUS_PER_TRANSACTION + ) if to_device_edus: self._device_stream_id = device_stream_id diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index 0ef14b31da..3be65d91f3 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -23,8 +23,16 @@ from http import HTTPStatus from typing import TYPE_CHECKING, Any -from synapse.api.constants import EduTypes, EventContentFields, ToDeviceEventTypes -from synapse.api.errors import Codes, SynapseError +from canonicaljson import encode_canonical_json + +from synapse.api.constants import ( + MAX_TO_DEVICE_CONTENT_SIZE, + SOFT_MAX_EDU_SIZE, + EduTypes, + EventContentFields, + ToDeviceEventTypes, +) +from synapse.api.errors import Codes, EventSizeError, SynapseError from synapse.api.ratelimiting import Ratelimiter from synapse.logging.context import run_in_background from synapse.logging.opentracing import ( @@ -34,8 +42,9 @@ set_tag, ) from synapse.types import JsonDict, Requester, StreamKeyType, UserID, get_domain_from_id +from synapse.util import split_dict_to_fit_to_size from synapse.util.json import json_encoder -from synapse.util.stringutils import random_string +from synapse.util.stringutils import random_string_insecure_fast if TYPE_CHECKING: from synapse.server import HomeServer @@ -222,6 +231,7 @@ async def send_device_message( set_tag(SynapseTags.TO_DEVICE_TYPE, message_type) set_tag(SynapseTags.TO_DEVICE_SENDER, sender_user_id) local_messages = {} + # Map from destination (server) -> recipient (user ID) -> device_id -> JSON message content remote_messages: dict[str, dict[str, dict[str, JsonDict]]] = {} for user_id, by_device in messages.items(): if not UserID.is_valid(user_id): @@ -233,6 +243,24 @@ async def send_device_message( # add an opentracing log entry for each message for device_id, message_content in by_device.items(): + # Check the size of each message, as if these are too large we + # can't send them over federation. + # + # We do this for all to-device messages, even those that aren't + # over federation, so as to more easily catch clients that are + # sending excessively large messages. + if ( + message_len := len(encode_canonical_json(message_content)) + ) > MAX_TO_DEVICE_CONTENT_SIZE: + # 413 is currently an unspecced response for `/sendToDevice` but is + # probably the best thing we can do. + # https://github.com/matrix-org/matrix-spec/pull/2340 tracks adding + # this to the spec + raise EventSizeError( + f"To-device message for {user_id}:{device_id} is too large to send ({message_len} > {MAX_TO_DEVICE_CONTENT_SIZE})", + unpersistable=True, + ) + log_kv( { "event": "send_to_device_message", @@ -277,28 +305,33 @@ async def send_device_message( destination = get_domain_from_id(user_id) remote_messages.setdefault(destination, {})[user_id] = by_device - context = get_active_span_text_map() - - remote_edu_contents = {} - for destination, messages in remote_messages.items(): - # The EDU contains a "message_id" property which is used for - # idempotence. Make up a random one. - message_id = random_string(16) - log_kv({"destination": destination, "message_id": message_id}) - - remote_edu_contents[destination] = { - "messages": messages, - "sender": sender_user_id, - "type": message_type, - "message_id": message_id, - "org.matrix.opentracing_context": json_encoder.encode(context), - } - - # Add messages to the database. + # Add local messages to the database. # Retrieve the stream id of the last-processed to-device message. - last_stream_id = await self.store.add_messages_to_device_inbox( - local_messages, remote_edu_contents + last_stream_id = ( + await self.store.add_local_messages_from_client_to_device_inbox( + local_messages + ) ) + for destination, messages in remote_messages.items(): + split_edus = split_device_messages_into_edus( + sender_user_id, message_type, messages + ) + for edu in split_edus: + edu["org.matrix.opentracing_context"] = json_encoder.encode( + get_active_span_text_map() + ) + # Add remote messages to the database. + last_stream_id = ( + await self.store.add_remote_messages_from_client_to_device_inbox( + {destination: edu} + ) + ) + log_kv( + { + "destination": destination, + "message_id": edu["message_id"], + } + ) # Notify listeners that there are new to-device messages to process, # handing them the latest stream id. @@ -397,3 +430,128 @@ async def get_events_for_dehydrated_device( "events": messages, "next_batch": f"d{stream_id}", } + + +def split_device_messages_into_edus( + sender_user_id: str, + message_type: str, + messages_by_user_then_device: dict[str, dict[str, JsonDict]], +) -> list[JsonDict]: + """ + This function takes many to-device messages and fits/splits them into several EDUs + as necessary. We split the messages up as the overall request can overrun the + `max_request_body_size` and prevent outbound federation traffic because of the size + of the transaction (cf. `SOFT_MAX_EDU_SIZE`). + + Args: + sender_user_id: The user that is sending the to-device messages. + message_type: The type of to-device messages that are being sent. + messages_by_user_then_device: Dictionary of recipient user_id to recipient device_id to message. + + Returns: + Bin-packed list of EDU JSON content for the given to_device messages + """ + split_edus_content: list[JsonDict] = [] + + # The header size of the full EDU. + base_edu_size = _EMPTY_EDU_SIZE + len(sender_user_id) + len(message_type) + + # First split up the top-level dict of user_id to device messages. + for subset_messages, estimated_size in split_dict_to_fit_to_size( + messages_by_user_then_device, + soft_max_size=SOFT_MAX_EDU_SIZE, + wrapping_object_size=base_edu_size, + ): + # The returned subset might be larger than the soft max size if it + # contains a single entry that is larger than the soft max size. + if estimated_size <= SOFT_MAX_EDU_SIZE: + # This message fits in a single EDU, add it as is. + content = create_new_to_device_edu_content( + sender_user_id, message_type, subset_messages + ) + split_edus_content.append(content) + logger.debug( + "Created EDU with %d recipients from %s (message_id=%s), (total EDUs so far: %d)", + len(subset_messages), + sender_user_id, + content["message_id"], + len(split_edus_content), + ) + else: + # This message doesn't fit in a single EDU. We split the message up + # further by device. + # + # Note: `subset` should only have a single entry in it. + for recipient, messages_by_device in subset_messages.items(): + # The header size of the EDU for this recipient, which includes + # the size of the recipient user ID and the wrapping structure + # for the device messages. + subset_base_size = len( + encode_canonical_json( + _create_new_to_device_edu( + sender_user_id, message_type, {recipient: {}} + ) + ) + ) + + for subset_messages, _estimated_size in split_dict_to_fit_to_size( + messages_by_device, + soft_max_size=SOFT_MAX_EDU_SIZE, + wrapping_object_size=subset_base_size, + ): + # Again, the returned subset might be larger than the soft + # max size, but we can't split it any further so we have to + # add it as is. + content = create_new_to_device_edu_content( + sender_user_id, message_type, {recipient: subset_messages} + ) + split_edus_content.append(content) + logger.debug( + "Created EDU with %d recipients from %s (message_id=%s), (total EDUs so far: %d)", + len(subset_messages), + sender_user_id, + content["message_id"], + len(split_edus_content), + ) + + return split_edus_content + + +def create_new_to_device_edu_content( + sender_user_id: str, + message_type: str, + messages_by_user_then_device: dict[str, dict[str, JsonDict]], +) -> JsonDict: + """ + Create a new `m.direct_to_device` EDU `content` object with a unique message ID. + """ + # The EDU contains a "message_id" property which is used for + # idempotence. Make up a random one. + message_id = random_string_insecure_fast(16) + content = { + "sender": sender_user_id, + "type": message_type, + "message_id": message_id, + "messages": messages_by_user_then_device, + } + return content + + +def _create_new_to_device_edu( + sender_user_id: str, + message_type: str, + messages_by_user_then_device: dict[str, dict[str, JsonDict]], +) -> dict[str, Any]: + """Create a new `m.direct_to_device` EDU, returning the full EDU dict.""" + return { + "edu_type": EduTypes.DIRECT_TO_DEVICE, + "content": create_new_to_device_edu_content( + sender_user_id, message_type, messages_by_user_then_device + ), + } + + +# The size of an empty EDU with no messages, which we use as the base size when +# packing messages into EDUs. The size of the sender and message type must be +# added to this when calculating the size of an EDU. +_EMPTY_EDU_SIZE = len(encode_canonical_json(_create_new_to_device_edu("", "", {}))) diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index a3806fd112..f147cbe728 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -739,18 +739,15 @@ def get_all_new_device_messages_txn( ) @trace - async def add_messages_to_device_inbox( + async def add_local_messages_from_client_to_device_inbox( self, local_messages_by_user_then_device: dict[str, dict[str, JsonDict]], - remote_messages_by_destination: dict[str, JsonDict], ) -> int: - """Used to send messages from this server. + """Queue local device messages that will be sent to devices of local users. Args: local_messages_by_user_then_device: Dictionary of recipient user_id to recipient device_id to message. - remote_messages_by_destination: - Dictionary of destination server_name to the EDU JSON to send. Returns: The new stream_id. @@ -766,6 +763,39 @@ def add_messages_txn( txn, stream_id, local_messages_by_user_then_device ) + async with self._to_device_msg_id_gen.get_next() as stream_id: + now_ms = self.clock.time_msec() + await self.db_pool.runInteraction( + "add_local_messages_from_client_to_device_inbox", + add_messages_txn, + now_ms, + stream_id, + ) + for user_id in local_messages_by_user_then_device.keys(): + self._device_inbox_stream_cache.entity_has_changed(user_id, stream_id) + + return self._to_device_msg_id_gen.get_current_token() + + @trace + async def add_remote_messages_from_client_to_device_inbox( + self, + remote_messages_by_destination: dict[str, JsonDict], + ) -> int: + """Queue device messages that will be sent to remote servers. + + Args: + remote_messages_by_destination: + Dictionary of destination server_name to the EDU JSON to send. + + Returns: + The new stream_id. + """ + + assert self._can_write_to_device + + def add_messages_txn( + txn: LoggingTransaction, now_ms: int, stream_id: int + ) -> None: # Add the remote messages to the federation outbox. # We'll send them to a remote server when we next send a # federation transaction to that destination. @@ -824,10 +854,11 @@ def add_messages_txn( async with self._to_device_msg_id_gen.get_next() as stream_id: now_ms = self.clock.time_msec() await self.db_pool.runInteraction( - "add_messages_to_device_inbox", add_messages_txn, now_ms, stream_id + "add_remote_messages_from_client_to_device_inbox", + add_messages_txn, + now_ms, + stream_id, ) - for user_id in local_messages_by_user_then_device.keys(): - self._device_inbox_stream_cache.entity_has_changed(user_id, stream_id) for destination in remote_messages_by_destination.keys(): self._device_federation_outbox_stream_cache.entity_has_changed( destination, stream_id diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index f2898de0b8..977a662d7a 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -24,6 +24,7 @@ import os import typing from typing import ( + Any, Iterator, Mapping, Sequence, @@ -31,11 +32,14 @@ ) import attr +from canonicaljson import encode_canonical_json from matrix_common.versionstring import get_distribution_version_string from twisted.internet import defer from twisted.python.failure import Failure +from synapse.types import JsonDict + if typing.TYPE_CHECKING: pass @@ -165,3 +169,107 @@ def clear(self) -> None: self._underlying_map = {} self._mutable_map.clear() self._deletions.clear() + + +@attr.s(slots=True, auto_attribs=True) +class _DictSplitterState: + """State for splitting a dict into multiple dicts, c.f. + `split_dict_to_fit_to_size`.""" + + subset: dict[str, Any] + """A subset of the original dict.""" + + estimated_size: int + """Estimated size of the JSON encoding of the current payload, including any + wrapping structure.""" + + +def split_dict_to_fit_to_size( + original_dict: dict[str, Any], + *, + soft_max_size: int, + wrapping_object_size: int = 2, +) -> Iterator[tuple[dict[str, JsonDict], int]]: + """Splits a dict up into a list of dicts, each of which is small enough to + fit into the given size when encoded as JSON. Every entry in the original + dict is in exactly one of the resulting dicts. + + The `wrapping_object_size` can be used if the resulting dicts are going to + be wrapped in some additional JSON structure, to account for the additional + size of that structure. The default assumes no wrapping, and just accounts + for the two curly braces of the dict itself. + + Note that if an individual entry in the original dict is larger than + `soft_max_size` then this will emit a dict containing just that entry, which + will be larger than `soft_max_size` when encoded as JSON. + + Args: + original_dict: The dict to split. + soft_max_size: The maximum size of each dict when encoded as JSON. + wrapping_object_size: The estimated size of the JSON encoding of the + payload when empty. + + Returns: + An iterator of (dict, size) pairs, where dict is a subset of the + original dict and size is the estimated size of the JSON encoding of + that dict, including any wrapping structure. + """ + + if not original_dict: + return + + # Check if the whole dict fits within the size limit. If it does, we can + # skip the splitting logic and just return the original dict. + full_size = _len_with_wrapping_object(original_dict, wrapping_object_size) + if full_size <= soft_max_size: + yield (original_dict, full_size) + return + + # The current payload being built up. We keep track of the estimated size of + # the JSON encoding of this payload so that we can decide when to start a + # new one. + current_payload = _DictSplitterState(subset={}, estimated_size=wrapping_object_size) + + for key, payload in original_dict.items(): + current_payload.subset[key] = payload + current_size = _len_with_wrapping_object( + current_payload.subset, wrapping_object_size + ) + + if current_size > soft_max_size: + # We've exceeded the size limit, so we need to start a new payload. We pop + # the current entry from the payload and yield the previous payload, then + # start a new payload with just the current entry. + if len(current_payload.subset) > 1: + current_payload.subset.pop(key) + yield current_payload.subset, current_payload.estimated_size + + current_payload = _DictSplitterState( + subset={}, + estimated_size=wrapping_object_size, + ) + + # Recalculate the current size with just the current entry. + current_size = _len_with_wrapping_object( + {key: payload}, wrapping_object_size + ) + + current_payload.subset[key] = payload + current_payload.estimated_size = current_size + + if current_payload.subset: + # yield the final payload if it's non-empty + yield current_payload.subset, current_payload.estimated_size + + +def _len_with_wrapping_object(payload: Any, wrapping_object_size: int) -> int: + """Helper function to calculate the size of a payload when encoded as JSON, + including any wrapping structure.""" + return ( + len(encode_canonical_json(payload)) + + wrapping_object_size + # account for the curly braces of the dict itself, which are + # included in the size of the subset but not in the size of the + # payload + - 2 + ) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 7f67d0ca5e..08a4ab624a 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -856,7 +856,9 @@ def test_application_services_receive_bursts_of_to_device(self) -> None: # Seed the device_inbox table with our fake messages self.get_success( - self.hs.get_datastores().main.add_messages_to_device_inbox(messages, {}) + self.hs.get_datastores().main.add_local_messages_from_client_to_device_inbox( + messages + ) ) # Now have local_user send a final to-device message to exclusive_as_user. All unsent diff --git a/tests/rest/client/test_sendtodevice.py b/tests/rest/client/test_sendtodevice.py index 56533d85f5..c06c7312f2 100644 --- a/tests/rest/client/test_sendtodevice.py +++ b/tests/rest/client/test_sendtodevice.py @@ -18,9 +18,26 @@ # [This file includes modifications made by New Vector Limited] # # -from synapse.api.constants import EduTypes +from unittest.mock import AsyncMock, Mock + +from canonicaljson import encode_canonical_json + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.api.constants import ( + MAX_EDUS_PER_TRANSACTION, + SOFT_MAX_EDU_SIZE, + EduTypes, +) +from synapse.api.errors import Codes +from synapse.handlers.devicemessage import ( + _create_new_to_device_edu, +) from synapse.rest import admin from synapse.rest.client import login, sendtodevice, sync +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util.clock import Clock from tests.unittest import HomeserverTestCase, override_config @@ -41,6 +58,20 @@ class SendToDeviceTestCase(HomeserverTestCase): sync.register_servlets, ] + def default_config(self) -> JsonDict: + config = super().default_config() + config["federation_sender_instances"] = None + return config + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + self.federation_transport_client = Mock(spec=["send_transaction"]) + self.federation_transport_client.send_transaction = AsyncMock() + hs = self.setup_test_homeserver( + federation_transport_client=self.federation_transport_client, + ) + + return hs + def test_user_to_user(self) -> None: """A to-device message from one user to another should get delivered""" @@ -91,6 +122,238 @@ def test_user_to_user(self) -> None: self.assertEqual(channel.code, 200, channel.result) self.assertEqual(channel.json_body.get("to_device", {}).get("events", []), []) + def test_large_remote_todevice(self) -> None: + """A to-device message needs to fit in the EDU size limit""" + _ = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + + # Create a message that is over the `SOFT_MAX_EDU_SIZE` + test_msg = {"foo": "a" * SOFT_MAX_EDU_SIZE} + channel = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.test/12345", + content={"messages": {"@remote_user:secondserver": {"device": test_msg}}}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 413, channel.result) + self.assertEqual(Codes.TOO_LARGE, channel.json_body["errcode"]) + + def test_edu_large_messages_splitting(self) -> None: + """ + Test that a bunch of to-device messages are split over multiple EDUs if they are + collectively too large to fit into a single EDU + """ + mock_send_transaction: AsyncMock = ( + self.federation_transport_client.send_transaction + ) + mock_send_transaction.return_value = {} + + sender = self.hs.get_federation_sender() + + _ = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + destination = "secondserver" + messages = {} + + # 2 messages, each just big enough to fit into their own EDU + for i in range(2): + messages[f"@remote_user{i}:" + destination] = { + "device": {"foo": "a" * (SOFT_MAX_EDU_SIZE - 1000)} + } + + channel = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.test/1234567", + content={"messages": messages}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + self.get_success(sender.send_device_messages([destination])) + + number_of_edus_sent = 0 + for call in mock_send_transaction.call_args_list: + number_of_edus_sent += len(call[0][1]()["edus"]) + + self.assertEqual(number_of_edus_sent, 2) + + def test_edu_large_messages_splitting_one_user(self) -> None: + """ + Test that a bunch of to-device messages for the same user are split over multiple EDUs if they are + collectively too large to fit into a single EDU + """ + mock_send_transaction: AsyncMock = ( + self.federation_transport_client.send_transaction + ) + mock_send_transaction.return_value = {} + + sender = self.hs.get_federation_sender() + + _ = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + destination = "secondserver" + messages = {} + + # 2 messages, each just big enough to fit into their own EDU + messages["@remote_user:" + destination] = { + "device1": {"foo": "a" * (SOFT_MAX_EDU_SIZE - 1000)}, + "device2": {"foo": "a" * (SOFT_MAX_EDU_SIZE - 1000)}, + } + + channel = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.test/12345678", + content={"messages": messages}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + self.get_success(sender.send_device_messages([destination])) + + number_of_edus_sent = 0 + for call in mock_send_transaction.call_args_list: + number_of_edus_sent += len(call[0][1]()["edus"]) + + self.assertEqual(number_of_edus_sent, 2) + + def test_edu_large_messages_not_splitting_one_user(self) -> None: + """Test that a couple of to-device messages for the same user that just + collectively fit in a single EDU do not get split into multiple EDUs. + + This tests that we don't over-split messages into multiple EDUs when we + don't need to. + """ + + mock_send_transaction: AsyncMock = ( + self.federation_transport_client.send_transaction + ) + mock_send_transaction.return_value = {} + + sender = self.hs.get_federation_sender() + + user_id = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + destination = "secondserver" + remote_user = "@remote_user:" + destination + messages = {} + + # We create two messages such that the combined size of the messages is + # just under the `SOFT_MAX_EDU_SIZE` limit, so they should be able to + # fit in a single EDU together, but not separately. + wrapper_size = len( + encode_canonical_json(_create_new_to_device_edu(user_id, "m.test", {})) + ) + max_size_of_message = ( + SOFT_MAX_EDU_SIZE + - wrapper_size + # Size of the device content wrapper, `{"remote_user": …}` + - (len(remote_user) + 5) + ) + messages[remote_user] = { + # The constants are the size of each individual message + "d1": {"a": "a" * (max_size_of_message - 13 - 16)}, + "d2": {"b": "b"}, + } + + # The full EDU size should now be just shy of the `SOFT_MAX_EDU_SIZE` limit + full_edu = encode_canonical_json( + _create_new_to_device_edu(user_id, "m.test", messages) + ) + self.assertEqual(len(full_edu), SOFT_MAX_EDU_SIZE - 1) + + # Now send the messages, and check they are sent in a single EDU. + channel = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.test/12345678", + content={"messages": messages}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + self.get_success(sender.send_device_messages([destination])) + + number_of_edus_sent = 0 + for call in mock_send_transaction.call_args_list: + number_of_edus_sent += len(call[0][1]()["edus"]) + + self.assertEqual(number_of_edus_sent, 1) + + def test_edu_small_messages_not_splitting(self) -> None: + """ + Test that a couple of small messages do not get split into multiple EDUs + """ + mock_send_transaction: AsyncMock = ( + self.federation_transport_client.send_transaction + ) + mock_send_transaction.return_value = {} + + sender = self.hs.get_federation_sender() + + _ = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + destination = "secondserver" + messages = {} + + # 2 small messages that should fit in a single EDU + for i in range(2): + messages[f"@remote_user{i}:" + destination] = {"device": {"foo": "a" * 100}} + + channel = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.test/123456", + content={"messages": messages}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + self.get_success(sender.send_device_messages([destination])) + + number_of_edus_sent = 0 + for call in mock_send_transaction.call_args_list: + number_of_edus_sent += len(call[0][1]()["edus"]) + + self.assertEqual(number_of_edus_sent, 1) + + def test_transaction_splitting(self) -> None: + """Test that a bunch of to-device messages are split into multiple transactions if there are too many EDUs""" + mock_send_transaction: AsyncMock = ( + self.federation_transport_client.send_transaction + ) + mock_send_transaction.return_value = {} + + sender = self.hs.get_federation_sender() + + _ = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + destination = "secondserver" + messages = {} + + number_of_edus_to_send = MAX_EDUS_PER_TRANSACTION + 1 + + for i in range(number_of_edus_to_send): + messages[f"@remote_user{i}:" + destination] = { + "device": {"foo": "a" * (SOFT_MAX_EDU_SIZE - 1000)} + } + + channel = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.test/12345678", + content={"messages": messages}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + self.get_success(sender.send_device_messages([destination])) + + # At least 2 transactions should be sent since we are over the EDU limit per transaction + self.assertGreaterEqual(mock_send_transaction.call_count, 2) + + number_of_edus_sent = 0 + for call in mock_send_transaction.call_args_list: + number_of_edus_sent += len(call[0][1]()["edus"]) + + self.assertEqual(number_of_edus_sent, number_of_edus_to_send) + @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}}) def test_local_room_key_request(self) -> None: """m.room_key_request has special-casing; test from local user""" diff --git a/tests/util/test_split_dict.py b/tests/util/test_split_dict.py new file mode 100644 index 0000000000..3abba4d3d4 --- /dev/null +++ b/tests/util/test_split_dict.py @@ -0,0 +1,201 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + + +from canonicaljson import encode_canonical_json + +from synapse.util import split_dict_to_fit_to_size + +from tests.unittest import TestCase + + +class SplitDictTestCase(TestCase): + def test_empty(self) -> None: + "Test that an empty dict yields no payloads" + + self.assertEqual( + list( + split_dict_to_fit_to_size({}, soft_max_size=10, wrapping_object_size=0) + ), + [], + ) + + def test_no_splitting(self) -> None: + "Test that a dict that fits within the size limit is yielded as a single payload" + + original_dict = {"a": {"key": "value"}, "b": {"key": "value"}} + + # Set the soft max size to be the size of the original dict, so it + # should fit + soft_max_size = len(encode_canonical_json(original_dict)) + + self.assertEqual( + list( + split_dict_to_fit_to_size( + original_dict, + soft_max_size=soft_max_size, + ) + ), + [(original_dict, soft_max_size)], + ) + + def test_no_splitting_with_wrapping_size(self) -> None: + "Test that the wrapping size is taken into account when deciding whether to split" + + wrapping = {"key": "value", "payload": {}} + original_dict = {"a": {"key": "value"}, "b": {"key": "value"}} + wrapping_object_size = len(encode_canonical_json(wrapping)) + + # Set the soft max size to the size of the expected final output. + soft_max_size = len( + encode_canonical_json({"key": "value", "payload": original_dict}) + ) + + self.assertEqual( + list( + split_dict_to_fit_to_size( + original_dict, + soft_max_size=soft_max_size, + wrapping_object_size=wrapping_object_size, + ) + ), + [(original_dict, soft_max_size)], + ) + + def test_splitting(self) -> None: + "Test that a dict that exceeds the size limit is split into multiple payloads" + + original_dict = { + "a": {"key": "value"}, + "b": {"key": "value"}, + "c": {"key": "value"}, + } + + # Set the soft max size to be the size of a single key-value pair, so + # it should split into three payloads. + soft_max_size = len(encode_canonical_json({"a": {"key": "value"}})) + + self.assertEqual( + list( + split_dict_to_fit_to_size( + original_dict, + soft_max_size=soft_max_size, + ) + ), + [ + ({"a": {"key": "value"}}, soft_max_size), + ({"b": {"key": "value"}}, soft_max_size), + ({"c": {"key": "value"}}, soft_max_size), + ], + ) + + def test_splitting_with_wrapping_size(self) -> None: + "Test that the wrapping size is taken into account when splitting" + + wrapping = {"key": "value", "payload": {}} + original_dict = { + "a": {"key": "value"}, + "b": {"key": "value"}, + "c": {"key": "value"}, + } + wrapping_object_size = len(encode_canonical_json(wrapping)) + # Set the soft max size to be the size of a single key-value pair plus + # the wrapping size, so it should split into three payloads. + soft_max_size = ( + len(encode_canonical_json({"a": {"key": "value"}})) + + wrapping_object_size + - 2 + ) + + self.assertEqual( + list( + split_dict_to_fit_to_size( + original_dict, + soft_max_size=soft_max_size, + wrapping_object_size=wrapping_object_size, + ) + ), + [ + ({"a": {"key": "value"}}, soft_max_size), + ({"b": {"key": "value"}}, soft_max_size), + ({"c": {"key": "value"}}, soft_max_size), + ], + ) + + def test_oversized_entry(self) -> None: + """Test that if a single entry exceeds the size limit, it is still + yielded as a single payload""" + + original_dict = { + "a": {"key": "value"}, + "b": {"key": "value"}, + "c": {"key": "value"}, + } + + # Set the soft max size to be smaller than the size of a single + # key-value pair, so each entry exceeds the limit. + soft_max_size = len(encode_canonical_json({"a": {"key": "value"}})) - 1 + + self.assertEqual( + list( + split_dict_to_fit_to_size( + original_dict, + soft_max_size=soft_max_size, + ) + ), + [ + ( + {"a": {"key": "value"}}, + len(encode_canonical_json({"a": {"key": "value"}})), + ), + ( + {"b": {"key": "value"}}, + len(encode_canonical_json({"b": {"key": "value"}})), + ), + ( + {"c": {"key": "value"}}, + len(encode_canonical_json({"c": {"key": "value"}})), + ), + ], + ) + + def test_different_sized_entries(self) -> None: + """Test that entries of different sizes are split correctly""" + + original_dict = { + "a": "X" * 5, # size 13 + "b": "X" * 10, # size 18 + "c": "X" * 5, # size 13 + } + + soft_max_size = 30 + + self.assertEqual( + list( + split_dict_to_fit_to_size( + original_dict, + soft_max_size=soft_max_size, + ) + ), + [ + ( + {"a": "X" * 5, "b": "X" * 10}, + len(encode_canonical_json({"a": "X" * 5, "b": "X" * 10})), + ), + ( + {"c": "X" * 5}, + len(encode_canonical_json({"c": "X" * 5})), + ), + ], + ) From e8e5a421803c6ffb14ad01df95121cf0c2da1648 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Jun 2026 14:52:05 +0100 Subject: [PATCH 12/26] Fix parsing events that have large integers (#19819) Follow on from https://github.com/element-hq/synapse/pull/19701. Unfortunately serde has a bug when using `#[serde(flatten)]` with `arbitrary-precision` feature when handling integers that fit in a i128 when doing `serde_json::from_value`. See https://github.com/serde-rs/serde/issues/2230. The `depythonize` hits the same issue. To fix this we make it so we only parse events from strings and not values. --- changelog.d/19819.misc | 1 + rust/src/events/formats/mod.rs | 7 ++ rust/src/events/mod.rs | 80 ++++++++-------------- rust/src/events/unsigned.rs | 13 ++-- synapse/events/__init__.py | 25 ++++++- synapse/synapse_rust/events.pyi | 4 +- tests/events/test_event_parsing.py | 102 ++++++++++++++++++++++++++++ tests/handlers/test_room_member.py | 6 +- tests/storage/test_stream.py | 8 +-- tests/synapse_rust/test_unsigned.py | 13 +++- 10 files changed, 189 insertions(+), 70 deletions(-) create mode 100644 changelog.d/19819.misc create mode 100644 tests/events/test_event_parsing.py diff --git a/changelog.d/19819.misc b/changelog.d/19819.misc new file mode 100644 index 0000000000..4663e8b961 --- /dev/null +++ b/changelog.d/19819.misc @@ -0,0 +1 @@ +Port the python Event classes to Rust. diff --git a/rust/src/events/formats/mod.rs b/rust/src/events/formats/mod.rs index 10b16a5436..86023add17 100644 --- a/rust/src/events/formats/mod.rs +++ b/rust/src/events/formats/mod.rs @@ -98,6 +98,13 @@ pub use vmsc4242::EventFormatVMSC4242; /// fields as they are mutable (and must be deep-copied if the event is cloned). /// `common_fields` and `specific_fields` are both `#[serde(flatten)]`ed so that /// the serialised JSON is a single flat object matching the Matrix spec. +/// +/// Note, deserialization of this struct must not be done from +/// [`serde_json::Value`] nor [`pythonize::depythonize`], due to a bug with +/// `#[serde(flatten)]` combined with the `arbitrary_precision` feature. +/// Instead, deserialize directly from a JSON string with +/// `serde_json::from_str`. See https://github.com/serde-rs/serde/issues/2230 +/// for details. #[derive(Serialize, Deserialize)] pub struct FormattedEvent> { /// The event's signatures. diff --git a/rust/src/events/mod.rs b/rust/src/events/mod.rs index ad3f61e2fd..a4fe15c522 100644 --- a/rust/src/events/mod.rs +++ b/rust/src/events/mod.rs @@ -160,10 +160,15 @@ pub struct Event { #[pymethods] impl Event { + /// Construct an Event from a JSON string, room version, and internal + /// metadata dict. + /// + /// We do no accept a Python dict directly because of the issues with + /// depythonize and large integers (see [`FormattedEvent`] for details). #[new] fn new_from_py<'a, 'py>( py: Python<'py>, - event_dict: &'a Bound<'py, PyAny>, + event_json: &str, room_version: &'a Bound<'py, PyAny>, internal_metadata_dict: &'a Bound<'py, PyDict>, rejected_reason: Option, @@ -178,14 +183,14 @@ impl Event { let rejected_reason = rejected_reason.map(String::into_boxed_str); - // Parse the event dict into a FormattedEvent, converting any failures to + // Parse the event JSON into a FormattedEvent, converting any failures to // a `ValueError`. - let parsed_event = depythonize_event_dict(room_version, event_dict).map_err(|err| { + let parsed_event = event_dict_from_json_str(room_version, event_json).map_err(|err| { let new_err = PyValueError::new_err(format!( - "Failed to parse event for room version {}", - room_version + "Failed to parse event for room version {}: {}", + room_version, err )); - new_err.set_cause(py, Some(err)); + new_err.set_cause(py, Some(PyValueError::new_err(err.to_string()))); new_err })?; @@ -555,63 +560,27 @@ impl Event { } } -fn depythonize_event_dict( +/// Parses a JSON string into a [`FormattedEvent`] for the given room version. +fn event_dict_from_json_str( room_version: &RoomVersion, - event_dict: &Bound<'_, PyAny>, -) -> PyResult { - let formatted_event: FormattedEvent = match room_version.event_format { - EventFormatVersions::ROOM_V1_V2 => { - let event_format: FormattedEvent = depythonize(event_dict)?; - - event_format.into() - } - EventFormatVersions::ROOM_V3 | EventFormatVersions::ROOM_V4_PLUS => { - let event_format: FormattedEvent = depythonize(event_dict)?; - event_format.into() - } - EventFormatVersions::ROOM_V11_HYDRA_PLUS => { - let event_format: FormattedEvent = depythonize(event_dict)?; - event_format.into() - } - EventFormatVersions::ROOM_VMSC4242 => { - let event_format: FormattedEvent = depythonize(event_dict)?; - event_format.into() - } - _ => { - return Err(PyValueError::new_err(format!( - "Unsupported room version: {}", - room_version - ))) - } - }; - - formatted_event.validate()?; - - Ok(formatted_event) -} - -/// Converts an event dict as [`serde_json::Value`] into a [`FormattedEvent`]. -fn event_dict_from_json_value( - room_version: &RoomVersion, - event_dict: serde_json::Value, + event_json: &str, ) -> Result { let formatted_event: FormattedEvent = match room_version.event_format { EventFormatVersions::ROOM_V1_V2 => { - let event_format: FormattedEvent = serde_json::from_value(event_dict)?; - + let event_format: FormattedEvent = serde_json::from_str(event_json)?; event_format.into() } EventFormatVersions::ROOM_V3 | EventFormatVersions::ROOM_V4_PLUS => { - let event_format: FormattedEvent = serde_json::from_value(event_dict)?; + let event_format: FormattedEvent = serde_json::from_str(event_json)?; event_format.into() } EventFormatVersions::ROOM_V11_HYDRA_PLUS => { - let event_format: FormattedEvent = serde_json::from_value(event_dict)?; + let event_format: FormattedEvent = serde_json::from_str(event_json)?; event_format.into() } EventFormatVersions::ROOM_VMSC4242 => { let event_format: FormattedEvent = - serde_json::from_value(event_dict)?; + serde_json::from_str(event_json)?; event_format.into() } _ => { @@ -638,10 +607,17 @@ fn redact_event_py(event: &Event) -> PyResult { })?; let redacted_value = redact(&event_value, event.room_version)?; - let redacted_formatted_event = event_dict_from_json_value(event.room_version, redacted_value) - .map_err(|err| { - PyValueError::new_err(format!("Failed to deserialize redacted event: {}", err)) + + // We can't convert from a value into [`Event`] directly, so we round-trip + // through JSON. See [`FormattedEvent`] for details on why we can't go + // directly through Python dicts. + let redacted_event_json = serde_json::to_string(&redacted_value).map_err(|err| { + PyValueError::new_err(format!("Failed to serialize redacted event: {}", err)) })?; + let redacted_formatted_event = + event_dict_from_json_str(event.room_version, &redacted_event_json).map_err(|err| { + PyValueError::new_err(format!("Failed to deserialize redacted event: {}", err)) + })?; let redacted_event = Event { parsed_event: redacted_formatted_event, diff --git a/rust/src/events/unsigned.rs b/rust/src/events/unsigned.rs index 0bb644ee90..931c412325 100644 --- a/rust/src/events/unsigned.rs +++ b/rust/src/events/unsigned.rs @@ -16,9 +16,9 @@ use std::sync::{Arc, RwLock, RwLockReadGuard}; use pyo3::{ - exceptions::{PyKeyError, PyRuntimeError, PyTypeError}, + exceptions::{PyKeyError, PyRuntimeError, PyTypeError, PyValueError}, pyclass, pymethods, - types::{PyAnyMethods, PyList, PyListMethods, PyMapping}, + types::{PyAnyMethods, PyList, PyListMethods}, Bound, IntoPyObjectExt, PyAny, PyResult, Python, }; use pythonize::{depythonize, pythonize}; @@ -114,9 +114,14 @@ impl Unsigned { #[pymethods] impl Unsigned { + /// Create a new `Unsigned` from a JSON string. + /// + /// We do no accept a Python dict directly because of the issues with + /// depythonize and large integers (see [`FormattedEvent`] for details). #[new] - fn py_new(unsigned: Bound<'_, PyMapping>) -> PyResult { - let inner = depythonize(&unsigned)?; + fn py_new(unsigned_json: &str) -> PyResult { + let inner = serde_json::from_str(unsigned_json) + .map_err(|err| PyValueError::new_err(format!("Failed to parse unsigned: {}", err)))?; Ok(Self { inner: Arc::new(RwLock::new(inner)), diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 7abd91f072..b01e786550 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -29,6 +29,7 @@ ) import attr +from canonicaljson import encode_canonical_json from synapse.api.constants import ( EventContentFields, @@ -72,15 +73,35 @@ def make_event_from_dict( ) -> Event: """Construct an EventBase from the given event dict""" + # Event constructor only takes JSON string, see Event constructor for + # details. + event_json = encode_canonical_json(event_dict).decode("utf-8") + + return make_event_from_json( + event_json=event_json, + room_version=room_version, + internal_metadata_dict=internal_metadata_dict, + rejected_reason=rejected_reason, + ) + + +def make_event_from_json( + event_json: str, + room_version: RoomVersion = RoomVersions.V1, + internal_metadata_dict: JsonDict | None = None, + rejected_reason: str | None = None, +) -> Event: + """Construct an EventBase from the given event JSON string""" + try: return Event( - event_dict=event_dict, + event_json=event_json, room_version=room_version, internal_metadata_dict=internal_metadata_dict or {}, rejected_reason=rejected_reason, ) except ValueError: - raise SynapseError(400, "Invalid event dict", Codes.BAD_JSON) + raise SynapseError(400, "Invalid event JSON", Codes.BAD_JSON) @attr.s(slots=True, frozen=True, auto_attribs=True) diff --git a/synapse/synapse_rust/events.pyi b/synapse/synapse_rust/events.pyi index 9a69438c1d..f84eeb55d6 100644 --- a/synapse/synapse_rust/events.pyi +++ b/synapse/synapse_rust/events.pyi @@ -189,7 +189,7 @@ class Signatures: class Unsigned: """A class representing the unsigned data of an event.""" - def __init__(self, unsigned_dict: JsonMapping): ... + def __init__(self, unsigned_json: str): ... def __getitem__(self, key: str) -> Any: """Get the value for the given key. @@ -230,7 +230,7 @@ class Event: def __init__( self, - event_dict: JsonDict, + event_json: str, room_version: RoomVersion, internal_metadata_dict: JsonDict, rejected_reason: str | None, diff --git a/tests/events/test_event_parsing.py b/tests/events/test_event_parsing.py new file mode 100644 index 0000000000..9d68cfabd6 --- /dev/null +++ b/tests/events/test_event_parsing.py @@ -0,0 +1,102 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + + +from typing import Any + +from parameterized import parameterized_class + +from synapse.api.room_versions import RoomVersions +from synapse.events import make_event_from_dict +from synapse.synapse_rust.events import redact_event +from synapse.types import JsonDict + +from tests.test_utils.event_injection import EventTypes +from tests.unittest import TestCase + + +def create_minimal_event_dict(**fields: Any) -> JsonDict: + """Create a minimal event dict that will parse correctly.""" + return { + "type": EventTypes.Message, + "content": {}, + "room_id": "!room:id", + "sender": "@user:id", + "event_id": "$event:id", + "origin_server_ts": 0, + "auth_events": [], + "prev_events": [], + "hashes": {}, + "signatures": {}, + "depth": 0, + **fields, + } + + +@parameterized_class( + [ + {"test_int": 2**7 - 1}, + {"test_int": 2**15 - 1}, + {"test_int": 2**31 - 1}, + {"test_int": 2**63 - 1}, + {"test_int": 2**127 - 1}, + {"test_int": 2**200}, + ] +) +class LargeIntTestCase(TestCase): + """Test that we can handle various sized integers in events. + + This is a regression test where we had issues handling integers that fit in + a Rust `i128`. + """ + + test_int: int + """The integer to test with. This will be set by the parameterized_class decorator.""" + + def test_very_large_int_in_event_content(self) -> None: + """Test that we can handle integers in the event content, which is a + JsonObject and thus can contain arbitrary JSON.""" + + event_dict = create_minimal_event_dict(content={"some_field": self.test_int}) + + event = make_event_from_dict(event_dict, RoomVersions.V1) + + self.assertEqual(event.content["some_field"], self.test_int) + + def test_large_int_in_unsigned(self) -> None: + """Test that we can handle integers in the unsigned data, which is an + Unsigned and thus can contain arbitrary JSON.""" + + event_dict = create_minimal_event_dict( + unsigned={"prev_content": {"some_field": self.test_int}} + ) + + event = make_event_from_dict(event_dict, RoomVersions.V1) + + self.assertEqual(event.unsigned["prev_content"]["some_field"], self.test_int) + + def test_large_int_redacted(self) -> None: + """Test that redact events that have an unsigned field with a large + integer in a protected field""" + + event_dict = create_minimal_event_dict( + type=EventTypes.PowerLevels, + state_key="", + content={"users": {"@user:id": self.test_int}}, + ) + + event = make_event_from_dict(event_dict, RoomVersions.V1) + + redacted_event = redact_event(event) + + self.assertEqual(redacted_event.content["users"]["@user:id"], self.test_int) diff --git a/tests/handlers/test_room_member.py b/tests/handlers/test_room_member.py index 9cc7ebfa80..d5b95e4ef6 100644 --- a/tests/handlers/test_room_member.py +++ b/tests/handlers/test_room_member.py @@ -8,7 +8,7 @@ from synapse.api.constants import AccountDataTypes, EventTypes, Membership from synapse.api.errors import Codes, LimitExceededError, SynapseError from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import Event +from synapse.events import make_event_from_dict from synapse.federation.federation_client import SendJoinResult from synapse.server import HomeServer from synapse.types import UserID, create_requester @@ -124,7 +124,7 @@ def test_remote_joins_contribute_to_rate_limit(self) -> None: create_event_source, self.hs.config.server.default_room_version, ) - create_event = Event( + create_event = make_event_from_dict( create_event_source, self.hs.config.server.default_room_version, {}, @@ -148,7 +148,7 @@ def test_remote_joins_contribute_to_rate_limit(self) -> None: self.hs.hostname, self.hs.signing_key, ) - join_event = Event( + join_event = make_event_from_dict( join_event_source, self.hs.config.server.default_room_version, {}, diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index ba3b3802cf..de127e3971 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -35,7 +35,7 @@ ) from synapse.api.filtering import Filter from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import Event +from synapse.events import make_event_from_dict from synapse.federation.federation_client import SendJoinResult from synapse.rest import admin from synapse.rest.client import login, room @@ -1385,7 +1385,7 @@ def test_remote_join(self) -> None: create_event_source, self.hs.config.server.default_room_version, ) - create_event = Event( + create_event = make_event_from_dict( create_event_source, self.hs.config.server.default_room_version, {}, @@ -1408,7 +1408,7 @@ def test_remote_join(self) -> None: creator_join_event_source, self.hs.config.server.default_room_version, ) - creator_join_event = Event( + creator_join_event = make_event_from_dict( creator_join_event_source, self.hs.config.server.default_room_version, {}, @@ -1433,7 +1433,7 @@ def test_remote_join(self) -> None: self.hs.hostname, self.hs.signing_key, ) - join_event = Event( + join_event = make_event_from_dict( join_event_source, self.hs.config.server.default_room_version, {}, diff --git a/tests/synapse_rust/test_unsigned.py b/tests/synapse_rust/test_unsigned.py index 5193188f38..6c1bf2238b 100644 --- a/tests/synapse_rust/test_unsigned.py +++ b/tests/synapse_rust/test_unsigned.py @@ -11,15 +11,22 @@ # See the GNU Affero General Public License for more details: # . +from canonicaljson import encode_canonical_json + from synapse.synapse_rust.events import Unsigned +from synapse.types import JsonDict from tests import unittest +def _make_unsigned(d: JsonDict) -> Unsigned: + return Unsigned(encode_canonical_json(d).decode("utf-8")) + + class UnsignedTestCase(unittest.TestCase): def test_prev_content(self) -> None: """Test that the prev_content field is correctly exposed as a JsonObject.""" - unsigned = Unsigned({"prev_content": {"key1": "value1", "key2": 42}}) + unsigned = _make_unsigned({"prev_content": {"key1": "value1", "key2": 42}}) self.assert_dict(unsigned["prev_content"], {"key1": "value1", "key2": 42}) @@ -32,7 +39,7 @@ def test_large_age_ts(self) -> None: than the maximum rust native integer size.""" large_int = 2**200 - unsigned = Unsigned({"age_ts": large_int}) + unsigned = _make_unsigned({"age_ts": large_int}) self.assertEqual(unsigned["age_ts"], large_int) @@ -44,7 +51,7 @@ def test_large_integer_in_prev_content(self) -> None: JSON.""" large_int = 2**200 - unsigned = Unsigned({"prev_content": {"some_field": large_int}}) + unsigned = _make_unsigned({"prev_content": {"some_field": large_int}}) self.assertEqual(unsigned["prev_content"]["some_field"], large_int) self.assert_dict( From b8cacf75372c5cda46b815d519d190c1c16839e1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Jun 2026 15:19:27 +0100 Subject: [PATCH 13/26] Fix `Event.__repr__` (#19817) Follow on from #19701 This (more or less) matches what we had before. Otherwise we just get a default `` --------- Co-authored-by: Eric Eastwood --- changelog.d/19817.misc | 1 + rust/src/events/internal_metadata.rs | 2 +- rust/src/events/mod.rs | 47 ++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19817.misc diff --git a/changelog.d/19817.misc b/changelog.d/19817.misc new file mode 100644 index 0000000000..4663e8b961 --- /dev/null +++ b/changelog.d/19817.misc @@ -0,0 +1 @@ +Port the python Event classes to Rust. diff --git a/rust/src/events/internal_metadata.rs b/rust/src/events/internal_metadata.rs index 93f13f655e..d61bf0d48c 100644 --- a/rust/src/events/internal_metadata.rs +++ b/rust/src/events/internal_metadata.rs @@ -563,7 +563,7 @@ impl EventInternalMetadata { Ok(dict.into()) } - fn is_outlier(&self) -> PyResult { + pub fn is_outlier(&self) -> PyResult { Ok(self.read_inner()?.is_outlier()) } diff --git a/rust/src/events/mod.rs b/rust/src/events/mod.rs index a4fe15c522..83900e14ba 100644 --- a/rust/src/events/mod.rs +++ b/rust/src/events/mod.rs @@ -60,6 +60,7 @@ use pyo3::{ use pythonize::{depythonize, pythonize}; use crate::events::{ + constants::event_type::M_ROOM_MEMBER, formats::{ EventFormatEnum, EventFormatV1, EventFormatV2V3, EventFormatV4, EventFormatVMSC4242, FormattedEvent, @@ -409,6 +410,52 @@ impl Event { Some(duration) } + fn __str__(&self) -> PyResult { + self.__repr__() + } + + /// We want to include some representative fields in the repr for + /// debugging purposes. + /// + /// The end result will look something like: + /// `` + fn __repr__(&self) -> PyResult { + let mut fields: Vec<(&str, &str)> = Vec::with_capacity(6); + + if let Some(reason) = self.rejected_reason.as_deref() { + fields.push(("REJECTED", reason)); + }; + + fields.push(("event_id", &self.event_id)); + fields.push(("type", self.r#type())); + + if let Some(state_key) = self.get_state_key() { + fields.push(("state_key", state_key)); + }; + + // Include the membership field for membership events. + if self.r#type() == M_ROOM_MEMBER && self.is_state() { + let content = &self.parsed_event.common_fields.content; + let membership_field = content.get_field(MEMBERSHIP).and_then(|m| m.as_str()); + + if let Some(membership_str) = membership_field { + fields.push(("membership", membership_str)); + } + } + + if self.internal_metadata.is_outlier()? { + fields.push(("outlier", "true")); + } + + let fields_str = fields + .into_iter() + .map(|(name, value)| format!("{}={}", name, value)) + .collect::>() + .join(", "); + + Ok(format!("", fields_str)) + } + // Below are the methods for interacting with the event as a mapping. // // These are rarely used, so we take the easy approach of re-serializing the From 9f2ce523614844954aee5dcb0d35664661ab1fc1 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 3 Jun 2026 10:44:39 -0500 Subject: [PATCH 14/26] Document how to see Rust build failure output when using `poetry install` (#19818) Knowledge from @reivilibre in [`#element-backend-internal:matrix.org`](https://matrix.to/#/!SGNQGPGUwtcPBUotTL:matrix.org/$2WUfCdA02wZw-6-jhw3QbLQ44BmKJrqLuZ6wjz2r7hk?via=jki.re&via=element.io&via=matrix.org) on 2025-12-15. Spawning from me making Rust changes but nothing useful was printed until I added `-v`, ```shell $ poetry install --extras all Installing dependencies from lock file Package operations: 0 installs, 1 update, 0 removals - Updating pyjwt (2.11.0 -> 2.12.0) Installing the current project: matrix-synapse (1.154.0rc1) Failed to install /home/eric/Documents/github/element/synapse ``` I also see `poetry run maturin develop` suggested but I'd prefer not to need to install `maturin` as yet another system tool to manage myself. --- changelog.d/19818.doc | 1 + docs/development/contributing_guide.md | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19818.doc diff --git a/changelog.d/19818.doc b/changelog.d/19818.doc new file mode 100644 index 0000000000..c6aca25257 --- /dev/null +++ b/changelog.d/19818.doc @@ -0,0 +1 @@ +Document how to see Rust build failure output when using `poetry install`. diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md index ab506b5f05..f967dda6f2 100644 --- a/docs/development/contributing_guide.md +++ b/docs/development/contributing_guide.md @@ -153,8 +153,14 @@ writing new pages, please to check that your contributions render correctly. The docs are written in [GitHub-Flavoured Markdown](https://guides.github.com/features/mastering-markdown/). +## Making changes to the Rust code + When changes are made to any Rust code then you must call either `poetry install` -or `maturin develop` (if installed) to rebuild the Rust code. Using [`maturin`](https://github.com/PyO3/maturin) +or `maturin develop` (if installed) to rebuild the Rust code. + +You can use `poetry install -v` or `-vv` to get more info about build failures. + +Using [`maturin`](https://github.com/PyO3/maturin) is quicker than `poetry install`, so is recommended when making frequent changes to the Rust code. From 003febe399a94c99c2da6d6184cb7aa2d6f6f938 Mon Sep 17 00:00:00 2001 From: Noah Markert Date: Thu, 4 Jun 2026 00:36:58 +0200 Subject: [PATCH 15/26] Add tests to ensure that email notification links are sanitized (#19741) Fix #2860 Also cleans up comments around plans to define `ALLOWED_SCHEMES` as we can rely on Bleach's `ALLOWED_PROTOCOLS` defaults (`http`, `https` and `mailto`). --- changelog.d/19741.misc | 1 + synapse/push/mailer.py | 4 --- tests/push/test_email.py | 67 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 changelog.d/19741.misc diff --git a/changelog.d/19741.misc b/changelog.d/19741.misc new file mode 100644 index 0000000000..55347570d8 --- /dev/null +++ b/changelog.d/19741.misc @@ -0,0 +1 @@ +Added tests to ensure that email notification links are sanitized. Contributed by Noah Markert. diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 1ebbc6d4f3..c1cd070d8e 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -111,8 +111,6 @@ # would make sense if we did "img": ["src"], } -# When bleach release a version with this option, we can specify schemes -# ALLOWED_SCHEMES = ["http", "https", "ftp", "mailto"] class Mailer: @@ -971,8 +969,6 @@ def safe_markup(raw_html: str) -> Markup: raw_html, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRS, - # bleach master has this, but it isn't released yet - # protocols=ALLOWED_SCHEMES, strip=True, ) ) diff --git a/tests/push/test_email.py b/tests/push/test_email.py index d3822b8643..b1dcd38704 100644 --- a/tests/push/test_email.py +++ b/tests/push/test_email.py @@ -379,6 +379,73 @@ def test_room_notifications_include_avatar(self) -> None: self.assertIn("_matrix/media/v1/thumbnail/DUMMY_MEDIA_ID", html) + @parameterized.expand( + [ + ( + "https allowed", + 'test click me', + 'test click me', + ), + ( + "http allowed", + 'test click me', + 'test click me', + ), + ( + "mailto allowed", + 'test click me', + 'test click me', + ), + ( + "javascript disallowed", + 'test click me', + "test click me", + ), + ( + "data disallowed", + 'test click me', + "test click me", + ), + ], + ) + def test_link_schemes_sanitized( + self, + _test_label: str, + input: str, + expected: str, + ) -> None: + # Create a room + room = self.helper.create_room_as(self.user_id, tok=self.access_token) + + self.helper.invite( + room=room, src=self.user_id, tok=self.access_token, targ=self.others[0].id + ) + self.helper.join(room=room, user=self.others[0].id, tok=self.others[0].token) + + self.helper.send_event( + room, + type="m.room.message", + content={ + "msgtype": "m.text", + "format": "org.matrix.custom.html", + "formatted_body": input, + "body": "test click me", # Plain-text Fallback + }, + tok=self.others[0].token, + ) + + args, kwargs = self._check_for_mail() + msg: bytes = args[5] + html_message = email.message_from_bytes(msg).get_payload(i=1) + assert isinstance(html_message, email.message.Message) + + # Extract the `bytes` from the html Message object, and decode to a `str`. + html = html_message.get_payload(decode=True) + assert isinstance(html, bytes) + html = html.decode() + + self.assertIn(expected, html) + def test_empty_room(self) -> None: """All users leaving a room shouldn't cause the pusher to break.""" # Create a simple room with two users From 678e7116b1f06d74d3883fb02eebba991105c92c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 4 Jun 2026 16:13:29 +0100 Subject: [PATCH 16/26] Reduce log spam from to-device stream logging (#19821) Followup to https://github.com/element-hq/synapse/pull/19801: I only meant for this logging to happen on the instance that is doing the persisting. --- changelog.d/19821.misc | 1 + synapse/storage/util/id_generators.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19821.misc diff --git a/changelog.d/19821.misc b/changelog.d/19821.misc new file mode 100644 index 0000000000..56ec0f7ddc --- /dev/null +++ b/changelog.d/19821.misc @@ -0,0 +1 @@ +Add more logging to the to-device message replication stream. diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index 1e053be6af..c9c339b235 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -828,9 +828,11 @@ def _add_persisted_position(self, new_id: int) -> None: # do. break - # Hacky debug logging to attempt to trace https://github.com/element-hq/synapse/issues/19795 + # Hacky debug logging to attempt to trace https://github.com/element-hq/synapse/issues/19795. + # If this is the to-device stream, and we are a writer for that stream, log some stats if ( issue9533_logger.isEnabledFor(logging.DEBUG) + and our_current_position > 0 and self._stream_name == "to_device" ): issue9533_logger.debug( From 7f45a1ce2e5e6d47b2a6db87a0dc75cbe1da4c90 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Fri, 5 Jun 2026 16:36:18 +0100 Subject: [PATCH 17/26] Document that the SQLite version included in Ubuntu LTS, aside from ESM-only versions, is included in our support policy. (#19823) The reason for querying this support was wanting support for SQLite's JSON operators, which are currently not present in the SQLite version found in Ubuntu's oldest supported LTS. The JSON operators were used in some of the sticky events work (related: #19452). Our ruling was that we should support Ubuntu oldest LTS equally to Debian oldstable, so support the oldest of the two versions from those. That makes some kind of sense as it would be difficult to do otherwise without dropping support for that version of Ubuntu altogether, given if we kept publishing packages intended for use with Postgres, there's a risk that an innocent sysadmin would update their SQLite deployment without realising that it is no longer supported. This was [discussed months ago (private)](https://docs.google.com/document/d/12RZKPk3a4__JUSH9wYHODo9rRyKzsHg6BSCAcmqmbOU/edit?tab=t.0#bookmark=id.fcdvoc88dy5s) and at [private](https://docs.google.com/document/d/12RZKPk3a4__JUSH9wYHODo9rRyKzsHg6BSCAcmqmbOU/edit?tab=t.0#bookmark=id.u48ivjge4qpt). --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19823.doc | 1 + docs/deprecation_policy.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19823.doc diff --git a/changelog.d/19823.doc b/changelog.d/19823.doc new file mode 100644 index 0000000000..5bbbdd5ccf --- /dev/null +++ b/changelog.d/19823.doc @@ -0,0 +1 @@ +Document that the SQLite version included in Ubuntu LTS, aside from ESM-only versions, is included in our support policy. \ No newline at end of file diff --git a/docs/deprecation_policy.md b/docs/deprecation_policy.md index 06c724d348..2a4301b4ee 100644 --- a/docs/deprecation_policy.md +++ b/docs/deprecation_policy.md @@ -22,7 +22,7 @@ people building from source should ensure they can fetch recent versions of Rust The oldest supported version of SQLite is the version [provided](https://packages.debian.org/oldstable/libsqlite3-0) by -[Debian oldstable](https://wiki.debian.org/DebianOldStable). +[Debian oldstable](https://wiki.debian.org/DebianOldStable) or the oldest maintenance/security-supported [Ubuntu LTS](https://endoflife.date/ubuntu) (Ubuntu versions with only Expanded Security Maintenance are not included). ### Context From 8c9b1ff877dc8f2155b7d0a89c5193f74c4cde21 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 8 Jun 2026 11:42:57 +0100 Subject: [PATCH 18/26] Reintroduce #19714 - Send a SSS response immediately if the config has changed (#19792) Reintroduces #19714, after being reverted in #19784. Fix https://github.com/element-hq/synapse/issues/18844 Fix https://github.com/element-hq/synapse/issues/19783 Fix https://github.com/element-hq/synapse/issues/18880 This PR also adds a fix so that we don't always return immediately when using the e2ee extension. --------- Co-authored-by: Benjamin Bouvier Co-authored-by: Eric Eastwood Co-authored-by: Eric Eastwood --- changelog.d/19734.bugfix | 1 + changelog.d/19792.bugfix | 1 + synapse/handlers/sliding_sync/__init__.py | 59 ++++--- synapse/handlers/sliding_sync/room_lists.py | 12 +- synapse/types/handlers/sliding_sync.py | 64 +++++-- .../sliding_sync/test_extension_e2ee.py | 167 ++++++++++++++++++ .../sliding_sync/test_room_subscriptions.py | 119 +++++++++++++ .../sliding_sync/test_rooms_required_state.py | 81 ++++++++- .../client/sliding_sync/test_sliding_sync.py | 36 +++- 9 files changed, 491 insertions(+), 49 deletions(-) create mode 100644 changelog.d/19734.bugfix create mode 100644 changelog.d/19792.bugfix diff --git a/changelog.d/19734.bugfix b/changelog.d/19734.bugfix new file mode 100644 index 0000000000..01af7d9ab8 --- /dev/null +++ b/changelog.d/19734.bugfix @@ -0,0 +1 @@ +Update Sliding Sync to return a new response immediately if a room subscription have changed and produced a new response. diff --git a/changelog.d/19792.bugfix b/changelog.d/19792.bugfix new file mode 100644 index 0000000000..01af7d9ab8 --- /dev/null +++ b/changelog.d/19792.bugfix @@ -0,0 +1 @@ +Update Sliding Sync to return a new response immediately if a room subscription have changed and produced a new response. diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index 4d6287b147..e7f0e7340b 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -184,34 +184,45 @@ async def wait_for_sync_for_user( timeout_ms -= after_wait_ts - before_wait_ts timeout_ms = max(timeout_ms, 0) - # We're going to respond immediately if the timeout is 0 or if this is an - # initial sync (without a `from_token`) so we can avoid calling - # `notifier.wait_for_events()`. - if timeout_ms == 0 or from_token is None: - now_token = self.event_sources.get_current_token() - result = await self.current_sync_for_user( + # Compute a response immediately. We always need to do this before + # waiting for new data (unlike in /v3/sync), as the request config might + # have changed (e.g. new room subscriptions, etc). + now_token = self.event_sources.get_current_token() + result = await self.current_sync_for_user( + sync_config, + from_token=from_token, + to_token=now_token, + ) + + # Return immediately if we have a result, the timeout is 0, or this is + # an initial sync. + if result or timeout_ms == 0 or from_token is None: + return result, did_wait + + # Otherwise, we wait for something to happen and report it to the user. + async def current_sync_callback( + before_token: StreamToken, after_token: StreamToken + ) -> SlidingSyncResult: + return await self.current_sync_for_user( sync_config, from_token=from_token, - to_token=now_token, + to_token=after_token, ) - else: - # Otherwise, we wait for something to happen and report it to the user. - async def current_sync_callback( - before_token: StreamToken, after_token: StreamToken - ) -> SlidingSyncResult: - return await self.current_sync_for_user( - sync_config, - from_token=from_token, - to_token=after_token, - ) - result = await self.notifier.wait_for_events( - sync_config.user.to_string(), - timeout_ms, - current_sync_callback, - from_token=from_token.stream_token, - ) - did_wait = True + result = await self.notifier.wait_for_events( + sync_config.user.to_string(), + timeout_ms, + current_sync_callback, + # We *wait* from `now_token` as we have already computed the sync + # response up to `now_token` above, so as a minor optimization, we + # can wait for something new to arrive after `now_token`. + # + # We still generate the sync response using `from_token` in the + # callback above though, as to generate the correct response it + # needs to know the "real" `from_token`. + from_token=now_token, + ) + did_wait = True return result, did_wait diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index 8969d91583..216ef3b071 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -852,11 +852,15 @@ async def _filter_relevant_rooms_to_send( previous_connection_state.room_configs.get(room_id) ) if prev_room_sync_config is not None: - # Always include rooms whose timeline limit has increased. - # (see the "XXX: Odd behavior" described below) + # Always include rooms whose effective config has + # expanded. This covers timeline-limit increases and + # required-state additions introduced by room + # subscriptions overriding list-derived params. if ( - prev_room_sync_config.timeline_limit - < room_config.timeline_limit + prev_room_sync_config.combine_room_sync_config( + room_config + ) + != prev_room_sync_config ): rooms_should_send.add(room_id) continue diff --git a/synapse/types/handlers/sliding_sync.py b/synapse/types/handlers/sliding_sync.py index 1a84bf1ff8..e73533d296 100644 --- a/synapse/types/handlers/sliding_sync.py +++ b/synapse/types/handlers/sliding_sync.py @@ -203,6 +203,9 @@ class StrippedHero: highlight_count: int def __bool__(self) -> bool: + """Are there any updates that should be returned immediately to + the client? + """ return ( # If this is the first time the client is seeing the room, we should not filter it out # under any circumstance. @@ -270,6 +273,8 @@ class ToDeviceExtension: events: Sequence[JsonMapping] def __bool__(self) -> bool: + """Are there any updates that should be returned immediately to + the client?""" return bool(self.events) @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -294,23 +299,37 @@ class E2eeExtension: device_unused_fallback_key_types: Sequence[str] def __bool__(self) -> bool: - # Note that "signed_curve25519" is always returned in key count responses - # regardless of whether we uploaded any keys for it. This is necessary until + """Are there any updates that should be returned immediately to + the client?""" + # Note that "signed_curve25519" is always returned in key count + # responses regardless of whether we uploaded any keys for it. + # This is necessary until # https://github.com/matrix-org/matrix-doc/issues/3298 is fixed. # # Also related: # https://github.com/element-hq/element-android/issues/3725 and # https://github.com/matrix-org/synapse/issues/10456 - default_otk = self.device_one_time_keys_count.get("signed_curve25519") - more_than_default_otk = len(self.device_one_time_keys_count) > 1 or ( - default_otk is not None and default_otk > 0 - ) - - return bool( - more_than_default_otk - or self.device_list_updates - or self.device_unused_fallback_key_types - ) + # + # This is why we don't incorporate `device_one_time_keys_count` + # (or `device_unused_fallback_key_types`) into the `__bool__` + # check. + # + # FIXME: Ideally we'd detect if either of those fields have + # changed since the last sync, but we do not currently track + # such state. + # + # Note that the client will receive these fields eventually when + # we respond to the sync request (usually sync timeouts are set + # to ~30s), we just won't immediately respond (even if there are + # changes). This delay is acceptable for clients, as a) these + # fields do not trigger UI (and so don't affect user perceivable + # latency) and b) are handled in the background by the clients + # anyway. The only risk being that one-time keys could be exhausted + # before the client knows about adding some more. But for example, + # if the client is syncing with a timeout of 30s, the window of + # staleness is so small for this not to matter. + + return bool(self.device_list_updates) @attr.s(slots=True, frozen=True, auto_attribs=True) class AccountDataExtension: @@ -327,6 +346,8 @@ class AccountDataExtension: account_data_by_room_map: Mapping[str, Mapping[str, JsonMapping]] def __bool__(self) -> bool: + """Are there any updates that should be returned immediately to + the client?""" return bool( self.global_account_data_map or self.account_data_by_room_map ) @@ -343,6 +364,8 @@ class ReceiptsExtension: room_id_to_receipt_map: Mapping[str, JsonMapping] def __bool__(self) -> bool: + """Are there any updates that should be returned immediately to + the client?""" return bool(self.room_id_to_receipt_map) @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -357,6 +380,8 @@ class TypingExtension: room_id_to_typing_map: Mapping[str, JsonMapping] def __bool__(self) -> bool: + """Are there any updates that should be returned immediately to + the client?""" return bool(self.room_id_to_typing_map) @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -391,6 +416,8 @@ class ThreadUnsubscription: prev_batch: ThreadSubscriptionsToken | None def __bool__(self) -> bool: + """Are there any updates that should be returned immediately to + the client?""" return ( bool(self.subscribed) or bool(self.unsubscribed) @@ -405,6 +432,8 @@ def __bool__(self) -> bool: thread_subscriptions: ThreadSubscriptionsExtension | None = None def __bool__(self) -> bool: + """Are there any updates that should be returned immediately to + the client?""" return bool( self.to_device or self.e2ee @@ -420,9 +449,14 @@ def __bool__(self) -> bool: extensions: Extensions def __bool__(self) -> bool: - """Make the result appear empty if there are no updates. This is used - to tell if the notifier needs to wait for more events when polling for - events. + """Are there any updates that should be returned immediately to + the client? + + This is used to determine if a sliding sync response should be returned + immediately or if the notifier needs to wait for further updates, and + thus MUST return false if there is no new data since the last sync. This + is subtly different than just checking if any of the fields are set, + since some fields are always included (like `bump_stamp`). """ # We don't include `self.lists` here, as a) `lists` is always non-empty even if # there are no changes, and b) since we're sorting rooms by `stream_ordering` of diff --git a/tests/rest/client/sliding_sync/test_extension_e2ee.py b/tests/rest/client/sliding_sync/test_extension_e2ee.py index 4a5e407038..2707e4d98d 100644 --- a/tests/rest/client/sliding_sync/test_extension_e2ee.py +++ b/tests/rest/client/sliding_sync/test_extension_e2ee.py @@ -290,6 +290,173 @@ def test_wait_for_new_data_timeout(self) -> None: [], ) + def test_wait_for_new_data_timeout_with_otks(self) -> None: + """ + Test that an incremental Sliding Sync with the e2ee extension enabled + does not return immediately when the user has uploaded one-time keys + (i.e. `device_one_time_keys_count` contains entries beyond the default + `signed_curve25519: 0`). + """ + test_device_id = "TESTDEVICE" + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass", device_id=test_device_id) + + # Upload one-time keys for the user/device so that + # `device_one_time_keys_count` is non-default in the response. + self.get_success( + self.e2e_keys_handler.upload_keys_for_user( + user1_id, + test_device_id, + { + "one_time_keys": { + "alg1:k1": "key1", + "alg2:k2": {"key": "key2", "signatures": {"k1": "sig1"}}, + } + }, + ) + ) + + sync_body = { + "lists": {}, + "extensions": { + "e2ee": { + "enabled": True, + } + }, + } + _, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Make an incremental Sliding Sync request with a timeout + channel = self.make_request( + "POST", + self.sync_endpoint + f"?timeout=10000&pos={from_token}", + content=sync_body, + access_token=user1_tok, + await_result=False, + ) + # Block for 5 seconds to make sure we are `notifier.wait_for_events(...)` + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=5000) + + # Wake-up `notifier.wait_for_events(...)` that will cause us to test + # `SlidingSyncResult.__bool__` for new results. The non-default + # `device_one_time_keys_count` must not be considered new data. + self._bump_notifier_wait_for_events( + user1_id, wake_stream_key=StreamKeyType.ACCOUNT_DATA + ) + + # Block for a little bit more to ensure we don't see any new results. + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=4000) + # Wait for the sync to complete (wait for the rest of the 10 second timeout, + # 5000 + 4000 + 1200 > 10000) + channel.await_result(timeout_ms=1200) + self.assertEqual(channel.code, 200, channel.json_body) + + # Device lists are present for incremental syncs but empty because no device changes + self.assertEqual( + channel.json_body["extensions"]["e2ee"] + .get("device_lists", {}) + .get("changed"), + [], + ) + self.assertEqual( + channel.json_body["extensions"]["e2ee"].get("device_lists", {}).get("left"), + [], + ) + + # The one-time key counts are present, but they should not have caused + # the sync to return early. + self.assertEqual( + channel.json_body["extensions"]["e2ee"]["device_one_time_keys_count"], + { + "alg1": 1, + "alg2": 1, + "signed_curve25519": 0, + }, + ) + self.assertEqual( + channel.json_body["extensions"]["e2ee"]["device_unused_fallback_key_types"], + [], + ) + + def test_wait_for_new_data_timeout_with_fallback_keys(self) -> None: + """ + Test that an incremental Sliding Sync with the e2ee extension enabled + does not return immediately when the user has uploaded fallback keys + (i.e. `device_unused_fallback_key_types` is non-empty). + """ + test_device_id = "TESTDEVICE" + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass", device_id=test_device_id) + + # Upload a fallback key for the user/device so that + # `device_unused_fallback_key_types` is non-empty in the response. + self.get_success( + self.e2e_keys_handler.upload_keys_for_user( + user1_id, + test_device_id, + {"fallback_keys": {"alg1:k1": "fallback_key1"}}, + ) + ) + + sync_body = { + "lists": {}, + "extensions": { + "e2ee": { + "enabled": True, + } + }, + } + _, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Make an incremental Sliding Sync request with a timeout + channel = self.make_request( + "POST", + self.sync_endpoint + f"?timeout=10000&pos={from_token}", + content=sync_body, + access_token=user1_tok, + await_result=False, + ) + # Block for 5 seconds to make sure we are `notifier.wait_for_events(...)` + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=5000) + + # Wake-up `notifier.wait_for_events(...)` that will cause us to test + # `SlidingSyncResult.__bool__` for new results. The non-empty + # `device_unused_fallback_key_types` must not be considered new data. + self._bump_notifier_wait_for_events( + user1_id, wake_stream_key=StreamKeyType.ACCOUNT_DATA + ) + + # Block for a little bit more to ensure we don't see any new results. + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=4000) + + # Wait for the sync to complete (wait for the rest of the 10 second timeout, + # 5000 + 4000 + 1200 > 10000) + channel.await_result(timeout_ms=1200) + self.assertEqual(channel.code, 200, channel.json_body) + + # Device lists are present for incremental syncs but empty because no device changes + self.assertEqual( + channel.json_body["extensions"]["e2ee"] + .get("device_lists", {}) + .get("changed"), + [], + ) + self.assertEqual( + channel.json_body["extensions"]["e2ee"].get("device_lists", {}).get("left"), + [], + ) + + # The unused fallback key types are present, but they should not have + # caused the sync to return early. + self.assertEqual( + channel.json_body["extensions"]["e2ee"]["device_unused_fallback_key_types"], + ["alg1"], + ) + def test_device_lists(self) -> None: """ Test that device list updates are included in the response diff --git a/tests/rest/client/sliding_sync/test_room_subscriptions.py b/tests/rest/client/sliding_sync/test_room_subscriptions.py index 811478f1ba..d970af367d 100644 --- a/tests/rest/client/sliding_sync/test_room_subscriptions.py +++ b/tests/rest/client/sliding_sync/test_room_subscriptions.py @@ -22,6 +22,7 @@ from synapse.api.constants import EventTypes, HistoryVisibility from synapse.rest.client import login, room, sync from synapse.server import HomeServer +from synapse.types import JsonDict from synapse.util.clock import Clock from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase @@ -126,6 +127,124 @@ def test_room_subscriptions_with_join_membership(self) -> None: response_body["rooms"][room_id1], ) + def test_room_subscription_required_state_expansion_returns_immediately( + self, + ) -> None: + """ + Test that adding a room subscription with stronger params than the list causes an + incremental long-poll to return immediately, even without new stream activity. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + room_id1 = self.helper.create_room_as(user1_id, tok=user1_tok) + + sync_body: JsonDict = { + "lists": { + "foo-list": { + "ranges": [[0, 0]], + "required_state": [], + "timeline_limit": 0, + } + }, + "conn_id": "conn_id", + } + _, from_token = self.do_sync(sync_body, tok=user1_tok) + + sync_body["room_subscriptions"] = { + room_id1: { + "required_state": [ + [EventTypes.Create, ""], + ], + "timeline_limit": 0, + } + } + + channel = self.make_request( + "POST", + self.sync_endpoint + f"?timeout=10000&pos={from_token}", + content=sync_body, + access_token=user1_tok, + await_result=False, + ) + channel.await_result(timeout_ms=3000) + self.assertEqual(channel.code, 200, channel.json_body) + + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + + room_response = channel.json_body["rooms"][room_id1] + self.assertNotIn("initial", room_response) + self._assertRequiredStateIncludes( + room_response["required_state"], + { + state_map[(EventTypes.Create, "")], + }, + exact=True, + ) + + def test_room_subscription_required_state_change_returns_immediately(self) -> None: + """ + Test that expanding an existing room subscription's required state causes an + incremental long-poll to return immediately, even without new stream activity. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + room_id1 = self.helper.create_room_as( + user1_id, tok=user1_tok, extra_content={"name": "Foo"} + ) + + sync_body: JsonDict = { + "room_subscriptions": { + room_id1: { + "required_state": [ + [EventTypes.Create, ""], + ], + "timeline_limit": 0, + } + }, + "conn_id": "conn_id", + } + response_body, from_token = self.do_sync(sync_body, tok=user1_tok) + + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + state_map[(EventTypes.Create, "")], + }, + exact=True, + ) + + sync_body["room_subscriptions"][room_id1]["required_state"] = [ + [EventTypes.Create, ""], + [EventTypes.Name, ""], + ] + + channel = self.make_request( + "POST", + self.sync_endpoint + f"?timeout=10000&pos={from_token}", + content=sync_body, + access_token=user1_tok, + await_result=False, + ) + channel.await_result(timeout_ms=3000) + self.assertEqual(channel.code, 200, channel.json_body) + + room_response = channel.json_body["rooms"][room_id1] + self.assertNotIn("initial", room_response) + self._assertRequiredStateIncludes( + room_response["required_state"], + { + state_map[(EventTypes.Name, "")], + }, + exact=True, + ) + def test_room_subscriptions_with_leave_membership(self) -> None: """ Test `room_subscriptions` with a leave room should give us timeline and state diff --git a/tests/rest/client/sliding_sync/test_rooms_required_state.py b/tests/rest/client/sliding_sync/test_rooms_required_state.py index 586b127f8a..901f22a35d 100644 --- a/tests/rest/client/sliding_sync/test_rooms_required_state.py +++ b/tests/rest/client/sliding_sync/test_rooms_required_state.py @@ -25,8 +25,10 @@ from synapse.server import HomeServer from synapse.storage.databases.main.events import DeltaState, SlidingSyncTableChanges from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase +from tests.server import TimedOutException from tests.test_utils.event_injection import mark_event_as_partial_state logger = logging.getLogger(__name__) @@ -1924,7 +1926,12 @@ def test_rooms_required_state_expand(self) -> None: def test_rooms_required_state_expand_retract_expand(self) -> None: """Test that when expanding, retracting and then expanding the required - state, we get the changes that happened.""" + state, we get the changes that happened. + + Also see `test_changing_required_state_returns_immediately`, which tests + that the sync stream is woken up immediately when changing the required + state, and not just on the next change to the room. + """ user1_id = self.register_user("user1", "pass") user1_tok = self.login(user1_id, "pass") @@ -2245,3 +2252,75 @@ def test_lazy_loading_room_members_state_reset_non_limited_timeline(self) -> Non response_body["rooms"][room_id]["required_state"][0]["event_id"], first_event_id, ) + + def test_changing_required_state_returns_immediately(self) -> None: + """Test that if we change the `required_state`, then we return immediately + with the new `required_state`.""" + + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + room_id1 = self.helper.create_room_as(user1_id, tok=user1_tok) + + # Make an initial sync request with no required state + sync_body = { + "lists": { + "foo-list": { + "ranges": [[0, 1]], + "required_state": [], + "timeline_limit": 0, + } + } + } + response_body, from_token = self.do_sync(sync_body, tok=user1_tok) + + # We should see no required state + self.assertIsNone(response_body["rooms"][room_id1].get("required_state")) + + # Get the state_map before we change the state as this is the final state we + # expect to see when we update the required state. + state_map = self.get_success( + self.storage_controllers.state.get_current_state(room_id1) + ) + + # There is no new data, and so making another sync request will block. + channel = self.make_sync_request( + sync_body, + since=from_token, + tok=user1_tok, + timeout=Duration(seconds=10), + await_result=False, + ) + + # Request will block for 10 seconds as there no updates. + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=9500) + + # Wait for the request to actually finish. (We do this to ensure log + # contexts don't leak between tests). + channel.await_result(timeout_ms=1000) + + # Now update the Sliding Sync requests to include a `required_state` + # event, and make another sync request. + sync_body["lists"]["foo-list"]["required_state"] = [ + [EventTypes.Create, ""], + ] + + channel = self.make_sync_request( + sync_body, + since=from_token, + tok=user1_tok, + timeout=Duration(seconds=10), + await_result=False, + ) + + # We should see the new `required_state` immediately without waiting + channel.await_result(timeout_ms=0) + response_body = channel.json_body + self._assertRequiredStateIncludes( + response_body["rooms"][room_id1]["required_state"], + { + state_map[(EventTypes.Create, "")], + }, + exact=True, + ) diff --git a/tests/rest/client/sliding_sync/test_sliding_sync.py b/tests/rest/client/sliding_sync/test_sliding_sync.py index ebf41cd87c..fc7d6a279c 100644 --- a/tests/rest/client/sliding_sync/test_sliding_sync.py +++ b/tests/rest/client/sliding_sync/test_sliding_sync.py @@ -12,6 +12,7 @@ # . # import logging +import urllib.parse from typing import Any, Iterable, Literal from unittest.mock import AsyncMock @@ -43,6 +44,7 @@ StreamToken, ) from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.stringutils import random_string from tests import unittest @@ -82,7 +84,13 @@ def default_config(self) -> JsonDict: return config def make_sync_request( - self, sync_body: JsonDict, *, since: str | None = None, tok: str + self, + sync_body: JsonDict, + *, + since: str | None = None, + tok: str, + timeout: Duration | None = None, + await_result: bool = True, ) -> FakeChannel: """Make a sliding sync request with given body. @@ -90,25 +98,40 @@ def make_sync_request( sync_body: The full request body to use since: Optional since token tok: Access token to use - + timeout_ms: Optional timeout in milliseconds to use for the request. + await_result: Whether to block and wait for the result before returning. Returns: A tuple of the response body and the `pos` field. """ sync_path = self.sync_endpoint + + query_params: dict[str, Any] = {} if since: - sync_path += f"?pos={since}" + query_params["pos"] = since + if timeout is not None: + query_params["timeout"] = timeout.as_millis() + + if query_params: + query_str = urllib.parse.urlencode(query_params) + sync_path += f"?{query_str}" channel = self.make_request( method="POST", path=sync_path, content=sync_body, access_token=tok, + await_result=await_result, ) return channel def do_sync( - self, sync_body: JsonDict, *, since: str | None = None, tok: str + self, + sync_body: JsonDict, + *, + since: str | None = None, + tok: str, + timeout: Duration | None = None, ) -> tuple[JsonDict, str]: """Do a sliding sync request with given body. @@ -118,11 +141,14 @@ def do_sync( sync_body: The full request body to use since: Optional since token tok: Access token to use + timeout: Optional timeout to use for the request. Returns: A tuple of the response body and the `pos` field. """ - channel = self.make_sync_request(sync_body, since=since, tok=tok) + channel = self.make_sync_request( + sync_body, since=since, tok=tok, timeout=timeout + ) self.assertEqual(channel.code, 200, channel.json_body) return channel.json_body, channel.json_body["pos"] From ac21bf08f380a0a053fc709eaab22da83a88f06d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Jun 2026 10:26:04 +0100 Subject: [PATCH 19/26] Port `Requester` class to Rust. (#19828) This is in prep for converting the event serialization to Rust. This is a fairly mechanical port, except that we store the appservice ID rather than the appservice object. This avoids us having to store a `Py<..>` (or port the appservice object over). --- changelog.d/19828.misc | 1 + rust/src/lib.rs | 2 + rust/src/types/mod.rs | 340 ++++++++++++++++++++++++ synapse/api/auth/base.py | 6 +- synapse/api/auth/internal.py | 4 +- synapse/api/auth/mas.py | 4 +- synapse/api/auth/msc3861_delegated.py | 4 +- synapse/api/auth_blocking.py | 2 +- synapse/api/ratelimiting.py | 7 +- synapse/events/utils.py | 2 +- synapse/handlers/admin.py | 2 +- synapse/handlers/directory.py | 6 +- synapse/handlers/message.py | 22 +- synapse/handlers/room_member.py | 4 +- synapse/replication/http/membership.py | 8 +- synapse/replication/http/send_events.py | 4 +- synapse/rest/client/account.py | 2 +- synapse/rest/client/appservice_ping.py | 17 +- synapse/rest/client/devices.py | 6 +- synapse/rest/client/directory.py | 15 +- synapse/rest/client/keys.py | 2 +- synapse/rest/client/login.py | 6 +- synapse/rest/client/room.py | 4 +- synapse/rest/client/transactions.py | 4 +- synapse/rest/media/create_resource.py | 8 +- synapse/synapse_rust/types.pyi | 68 +++++ synapse/types/__init__.py | 80 +----- tests/api/test_auth.py | 12 +- tests/api/test_ratelimiting.py | 6 + tests/rest/client/test_transactions.py | 2 +- 30 files changed, 517 insertions(+), 133 deletions(-) create mode 100644 changelog.d/19828.misc create mode 100644 rust/src/types/mod.rs create mode 100644 synapse/synapse_rust/types.pyi diff --git a/changelog.d/19828.misc b/changelog.d/19828.misc new file mode 100644 index 0000000000..1ed01eba2e --- /dev/null +++ b/changelog.d/19828.misc @@ -0,0 +1 @@ +Port `Requester` class to Rust. diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 8ed4e24b81..bf96422cbb 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -19,6 +19,7 @@ pub mod push; pub mod rendezvous; pub mod room_versions; pub mod segmenter; +pub mod types; lazy_static! { static ref LOGGING_HANDLE: ResetHandle = pyo3_log::init(); @@ -71,6 +72,7 @@ fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { msc4388_rendezvous::register_module(py, m)?; segmenter::register_module(py, m)?; room_versions::register_module(py, m)?; + types::register_module(py, m)?; Ok(()) } diff --git a/rust/src/types/mod.rs b/rust/src/types/mod.rs new file mode 100644 index 0000000000..ffb19a83a2 --- /dev/null +++ b/rust/src/types/mod.rs @@ -0,0 +1,340 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Rust implementations of types from `synapse.types`. + +use std::collections::HashSet; + +use once_cell::sync::OnceCell; +use pyo3::prelude::*; +use pyo3::{ + exceptions::PyKeyError, + types::{PyDict, PyDictMethods, PyList}, +}; + +/// A reference to the `synapse.types.UserID` class. +static USER_ID_CLASS: OnceCell> = OnceCell::new(); + +/// Access to the `synapse.types.UserID` class. +fn user_id_class(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + Ok(USER_ID_CLASS + .get_or_try_init(|| -> PyResult<_> { + Ok(py.import("synapse.types")?.getattr("UserID")?.unbind()) + })? + .bind(py)) +} + +/// Represents the user making a request. +#[pyclass(frozen, skip_from_py_object, get_all, eq)] +#[derive(Debug, PartialEq, Eq)] +pub struct Requester { + /// The ID of the user making the request, in string form (see + /// [`Self::user`] for accessing the parsed `UserID`). + user_id: String, + /// The ID of the access token used for this request, or None for + /// appservices, guests, and tokens generated by the admin API + access_token_id: Option, + /// True if the user making this request is a guest + is_guest: bool, + /// Any scopes associated with the access token used for this request, or an + /// empty set if no token or a non-oauth token was used + scope: HashSet, + /// True if the user making this request is shadow banned + shadow_banned: bool, + /// The device_id which was set at authentication time, or None for + /// appservices, guests, and tokens generated by the admin API + device_id: Option, + /// The ID of the AS requesting on behalf of the user, or None. + app_service_id: Option, + /// The entity that authenticated when making the request. + /// + /// This is different to the `user_id` when an admin user or the server is + /// "puppeting" the user. + authenticated_entity: String, +} + +#[pymethods] +impl Requester { + #[new] + #[pyo3(signature = ( + user, + access_token_id, + is_guest, + scope, + shadow_banned, + device_id, + app_service_id, + authenticated_entity, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + user: &Bound<'_, PyAny>, + access_token_id: Option, + is_guest: &Bound<'_, PyAny>, + scope: HashSet, + shadow_banned: &Bound<'_, PyAny>, + device_id: Option, + app_service_id: Option, + authenticated_entity: String, + ) -> PyResult { + // The `user` argument should be a `UserID`, which has a `to_string` for + // getting the string form. + let user_id = user.call_method0("to_string")?.extract::()?; + + // The `is_guest` and `shadow_banned` arguments are expected to be + // Python bools, but unfortunately Synapse often passes them as truthy + // values (mainly due to reading from SQLite, which returns 0/1 for + // bools). + let is_guest = is_guest.is_truthy()?; + let shadow_banned = shadow_banned.is_truthy()?; + + Ok(Requester { + user_id, + access_token_id, + is_guest, + scope, + shadow_banned, + device_id, + app_service_id, + authenticated_entity, + }) + } + + /// The user making the request, as a Python `UserID`. + #[getter] + fn user<'py>(&self, py: Python<'py>) -> PyResult> { + user_id_class(py)?.call_method1("from_string", (&self.user_id,)) + } + + /// Converts self to a type that can be serialized as JSON, and then + /// deserialized by [`Self::deserialize`] + fn serialize<'py>(&self, py: Python<'py>) -> PyResult> { + let dict = PyDict::new(py); + dict.set_item("user_id", &self.user_id)?; + dict.set_item("access_token_id", self.access_token_id)?; + dict.set_item("is_guest", self.is_guest)?; + dict.set_item("scope", PyList::new(py, &self.scope)?)?; + dict.set_item("shadow_banned", self.shadow_banned)?; + dict.set_item("device_id", self.device_id.as_deref())?; + // NB: the wire key is "app_server_id" (server, not service). Changing + // this is non-trivial as it would break replication during a rolling + // upgrade. + dict.set_item("app_server_id", self.app_service_id.as_deref())?; + dict.set_item("authenticated_entity", &self.authenticated_entity)?; + Ok(dict) + } + + /// Converts a dict that was produced by [`Self::serialize`] back into a + /// [`Requester`]. + #[staticmethod] + fn deserialize(py: Python<'_>, input: &Bound<'_, PyAny>) -> PyResult { + let user_id = input.get_item("user_id")?.extract::()?; + let access_token_id = input + .get_item("access_token_id")? + .extract::>()?; + let is_guest = input.get_item("is_guest")?.is_truthy()?; + + // `serialize` stores the scope as a list, so extract it as a `Vec` + // (which accepts any sequence) and collect into a set. For backwards + // compatibility, "scope" is optional and defaults to an empty set if + // not present. + let scope = match input.get_item("scope") { + Ok(scope) => scope.extract::>()?.into_iter().collect(), + Err(err) if err.is_instance_of::(py) => HashSet::new(), + Err(err) => return Err(err), + }; + + let shadow_banned = input.get_item("shadow_banned")?.is_truthy()?; + let device_id = input.get_item("device_id")?.extract::>()?; + + // The wire key is "app_server_id", not "app_service_id" — see `serialize`. + let app_service_id = input + .get_item("app_server_id")? + .extract::>()?; + + let authenticated_entity = input + .get_item("authenticated_entity")? + .extract::()?; + + Ok(Requester { + user_id, + access_token_id, + is_guest, + scope, + shadow_banned, + device_id, + app_service_id, + authenticated_entity, + }) + } + + fn __repr__(&self) -> String { + format!( + "Requester(user_id={}, access_token_id={:?}, is_guest={}, scope={:?}, \ + shadow_banned={}, device_id={:?}, app_service_id={:?}, authenticated_entity={})", + self.user_id, + self.access_token_id, + self.is_guest, + self.scope, + self.shadow_banned, + self.device_id, + self.app_service_id, + self.authenticated_entity, + ) + } +} + +/// Called when registering modules with python. +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child_module = PyModule::new(py, "types")?; + child_module.add_class::()?; + + m.add_submodule(&child_module)?; + + // We need to manually add the module to sys.modules to make `from + // synapse.synapse_rust.types import Requester` work. + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.types", child_module)?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A `Requester` with every field populated, for use as a test fixture. + fn sample_requester() -> Requester { + Requester { + user_id: "@alice:example.com".to_string(), + access_token_id: Some(42), + is_guest: false, + scope: HashSet::from(["urn:matrix:client:api:*".to_string()]), + shadow_banned: false, + device_id: Some("ABCDEFG".to_string()), + app_service_id: None, + authenticated_entity: "@alice:example.com".to_string(), + } + } + + #[test] + fn test_serialize() { + Python::initialize(); + Python::attach(|py| -> Result<(), PyErr> { + let requester = sample_requester(); + let dict = requester.serialize(py)?; + let dict = dict.as_any(); + + assert_eq!( + dict.get_item("user_id")?.extract::()?, + "@alice:example.com" + ); + assert_eq!(dict.get_item("access_token_id")?.extract::()?, 42); + assert!(!dict.get_item("is_guest")?.extract::()?); + assert_eq!( + dict.get_item("scope")?.extract::>()?, + vec!["urn:matrix:client:api:*".to_string()] + ); + assert!(!dict.get_item("shadow_banned")?.extract::()?); + assert_eq!( + dict.get_item("device_id")?.extract::>()?, + Some("ABCDEFG".to_string()) + ); + assert_eq!( + dict.get_item("authenticated_entity")?.extract::()?, + "@alice:example.com" + ); + + // The `app_service_id` field is serialized under the wire key + // "app_server_id" (server, not service), and there must be no + // "app_service_id" key. + assert!(dict.get_item("app_server_id")?.is_none()); + assert!(!dict.contains("app_service_id")?); + + Ok(()) + }) + .unwrap(); + } + + #[test] + fn test_serialize_deserialize_round_trip() { + Python::initialize(); + Python::attach(|py| { + // Use a requester that exercises the optional fields and the + // `app_service_id` -> "app_server_id" wire-key mapping. + let requester = Requester { + user_id: "@bob:example.com".to_string(), + access_token_id: None, + is_guest: true, + scope: HashSet::from(["a".to_string(), "b".to_string()]), + shadow_banned: true, + device_id: None, + app_service_id: Some("my_appservice".to_string()), + authenticated_entity: "@admin:example.com".to_string(), + }; + + let dict = requester.serialize(py).unwrap(); + let deserialized = Requester::deserialize(py, dict.as_any()).unwrap(); + let deserialized = Bound::new(py, deserialized).unwrap(); + + assert_eq!(&requester, deserialized.get()); + }); + } + + #[test] + fn test_deserialize_defaults_scope_when_missing() { + Python::initialize(); + Python::attach(|py| { + // An older serialized form may omit "scope"; it should default to + // an empty set rather than erroring. + let dict = PyDict::new(py); + dict.set_item("user_id", "@alice:example.com").unwrap(); + dict.set_item("access_token_id", py.None()).unwrap(); + dict.set_item("is_guest", false).unwrap(); + dict.set_item("shadow_banned", false).unwrap(); + dict.set_item("device_id", py.None()).unwrap(); + dict.set_item("app_server_id", py.None()).unwrap(); + dict.set_item("authenticated_entity", "@alice:example.com") + .unwrap(); + + let requester = Requester::deserialize(py, dict.as_any()).unwrap(); + assert!(requester.scope.is_empty()); + }); + } + + #[test] + fn test_deserialize_coerces_truthy_bools() { + Python::initialize(); + Python::attach(|py| { + // SQLite returns 0/1 for booleans, so non-bool truthy values must + // be coerced for `is_guest` and `shadow_banned`. + let dict = PyDict::new(py); + dict.set_item("user_id", "@alice:example.com").unwrap(); + dict.set_item("access_token_id", py.None()).unwrap(); + dict.set_item("is_guest", 1).unwrap(); + dict.set_item("scope", PyList::empty(py)).unwrap(); + dict.set_item("shadow_banned", 0).unwrap(); + dict.set_item("device_id", py.None()).unwrap(); + dict.set_item("app_server_id", py.None()).unwrap(); + dict.set_item("authenticated_entity", "@alice:example.com") + .unwrap(); + + let requester = Requester::deserialize(py, dict.as_any()).unwrap(); + assert!(requester.is_guest); + assert!(!requester.shadow_banned); + }); + } +} diff --git a/synapse/api/auth/base.py b/synapse/api/auth/base.py index ff876b9d22..14e76b0cff 100644 --- a/synapse/api/auth/base.py +++ b/synapse/api/auth/base.py @@ -371,7 +371,9 @@ async def _record_request( """ ip_addr = request.get_client_ip_if_available() - if ip_addr and (not requester.app_service or self._track_appservice_user_ips): + if ip_addr and ( + not requester.app_service_id or self._track_appservice_user_ips + ): user_agent = get_request_user_agent(request) access_token = self.get_access_token_from_request(request) @@ -381,7 +383,7 @@ async def _record_request( # table during the transition recorded_device_id = ( "dummy-device" - if requester.device_id is None and requester.app_service is not None + if requester.device_id is None and requester.app_service_id is not None else requester.device_id ) await self.store.insert_client_ip( diff --git a/synapse/api/auth/internal.py b/synapse/api/auth/internal.py index b33384c13f..28c0491140 100644 --- a/synapse/api/auth/internal.py +++ b/synapse/api/auth/internal.py @@ -106,8 +106,8 @@ async def get_user_by_req( parent_span.set_tag("user_id", requester.user.to_string()) if requester.device_id is not None: parent_span.set_tag("device_id", requester.device_id) - if requester.app_service is not None: - parent_span.set_tag("appservice_id", requester.app_service.id) + if requester.app_service_id is not None: + parent_span.set_tag("appservice_id", requester.app_service_id) return requester async def get_user_by_req_experimental_feature( diff --git a/synapse/api/auth/mas.py b/synapse/api/auth/mas.py index 95c6d62a9d..ed0427d6f3 100644 --- a/synapse/api/auth/mas.py +++ b/synapse/api/auth/mas.py @@ -311,8 +311,8 @@ async def get_user_by_req( parent_span.set_tag("user_id", requester.user.to_string()) if requester.device_id is not None: parent_span.set_tag("device_id", requester.device_id) - if requester.app_service is not None: - parent_span.set_tag("appservice_id", requester.app_service.id) + if requester.app_service_id is not None: + parent_span.set_tag("appservice_id", requester.app_service_id) return requester async def get_user_by_access_token( diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index 27ab4af805..3b37f39875 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -420,8 +420,8 @@ async def get_user_by_req( parent_span.set_tag("user_id", requester.user.to_string()) if requester.device_id is not None: parent_span.set_tag("device_id", requester.device_id) - if requester.app_service is not None: - parent_span.set_tag("appservice_id", requester.app_service.id) + if requester.app_service_id is not None: + parent_span.set_tag("appservice_id", requester.app_service_id) return requester async def _wrapped_get_user_by_req( diff --git a/synapse/api/auth_blocking.py b/synapse/api/auth_blocking.py index 3ed47b20c4..87918e15dc 100644 --- a/synapse/api/auth_blocking.py +++ b/synapse/api/auth_blocking.py @@ -88,7 +88,7 @@ async def check_auth_blocking( # We never block the server from doing actions on behalf of # users. return - if requester.app_service and not self._track_appservice_user_ips: + if requester.app_service_id and not self._track_appservice_user_ips: # If we're authenticated as an appservice then we only block # auth if `track_appservice_user_ips` is set, as that option # implicitly means that application services are part of MAU diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py index d6cc3d26b5..b17fcd1ef3 100644 --- a/synapse/api/ratelimiting.py +++ b/synapse/api/ratelimiting.py @@ -163,7 +163,12 @@ async def can_do_action( if requester: # Disable rate limiting of users belonging to any AS that is configured # not to be rate limited in its registration file (rate_limited: true|false). - if requester.app_service and not requester.app_service.is_rate_limited(): + app_service = ( + self.store.get_app_service_by_id(requester.app_service_id) + if requester.app_service_id + else None + ) + if app_service and not app_service.is_rate_limited(): return True, -1.0 # Check if ratelimiting has been disabled for the user. diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 54f662796b..8ce795052a 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -419,7 +419,7 @@ def _serialize_event( and event_token_id == config.requester.access_token_id ) or config.requester.is_guest - or config.requester.app_service + or config.requester.app_service_id ): d["unsigned"]["transaction_id"] = txn_id diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index 0b1251f9e3..ada799d96a 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -427,7 +427,7 @@ async def _redact_all_events( r = task.params.get("requester") assert r is not None - admin = Requester.deserialize(self._store, r) + admin = Requester.deserialize(r) user_id = task.params.get("user_id") assert user_id is not None diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index bdb7cba64b..91e7854f39 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -132,7 +132,11 @@ async def create_association( Codes.INVALID_PARAM, ) - service = requester.app_service + service = ( + self.store.get_app_service_by_id(requester.app_service_id) + if requester.app_service_id + else None + ) if service: if not service.is_room_alias_in_namespace(room_alias_str): raise SynapseError( diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index bade219ddf..ed04547d42 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -343,7 +343,7 @@ async def get_joined_members(self, requester: Requester, room_id: str) -> dict: Returns: A dict of user_id to profile info """ - if not requester.app_service: + if not requester.app_service_id: # We check AS auth after fetching the room membership, as it # requires us to pull out all joined members anyway. membership, _ = await self.auth.check_user_in_room_or_world_readable( @@ -365,12 +365,14 @@ async def get_joined_members(self, requester: Requester, room_id: str) -> dict: # If this is an AS, double check that they are allowed to see the members. # This can either be because the AS user is in the room or because there # is a user in the room that the AS is "interested in" - if ( - requester.app_service - and requester.user.to_string() not in users_with_profile - ): + app_service = ( + self.store.get_app_service_by_id(requester.app_service_id) + if requester.app_service_id + else None + ) + if app_service and requester.user.to_string() not in users_with_profile: for uid in users_with_profile: - if requester.app_service.is_interested_in_user(uid): + if app_service.is_interested_in_user(uid): break else: # Loop fell through, AS has no interested users in room @@ -846,7 +848,7 @@ async def assert_accepted_privacy_policy(self, requester: Requester) -> None: return # exempt AS users from needing consent - if requester.app_service is not None: + if requester.app_service_id is not None: return user_id = requester.authenticated_entity @@ -1425,8 +1427,10 @@ async def create_new_client_event( else: context = await self.state.calculate_context_info(event) - if requester: - context.app_service = requester.app_service + if requester and requester.app_service_id: + context.app_service = self.store.get_app_service_by_id( + requester.app_service_id + ) res, new_content = await self._third_party_event_rules.check_event_allowed( event, context diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 8a19dba5ee..5152d0b522 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -661,8 +661,8 @@ async def update_membership( key = (room_id,) as_id = object() - if requester.app_service: - as_id = requester.app_service.id + if requester.app_service_id: + as_id = requester.app_service_id # We first linearise by the application service (to try to limit concurrent joins # by application services), and then by room ID. diff --git a/synapse/replication/http/membership.py b/synapse/replication/http/membership.py index 8a6c971720..80dd9b7da7 100644 --- a/synapse/replication/http/membership.py +++ b/synapse/replication/http/membership.py @@ -89,7 +89,7 @@ async def _handle_request( # type: ignore[override] remote_room_hosts = content["remote_room_hosts"] event_content = content["content"] - requester = Requester.deserialize(self.store, content["requester"]) + requester = Requester.deserialize(content["requester"]) request.requester = requester logger.info("remote_join: %s into room: %s", user_id, room_id) @@ -153,7 +153,7 @@ async def _handle_request( # type: ignore[override] remote_room_hosts = content["remote_room_hosts"] event_content = content["content"] - requester = Requester.deserialize(self.store, content["requester"]) + requester = Requester.deserialize(content["requester"]) request.requester = requester logger.debug("remote_knock: %s on room: %s", user_id, room_id) @@ -219,7 +219,7 @@ async def _handle_request( # type: ignore[override] txn_id = content["txn_id"] event_content = content["content"] - requester = Requester.deserialize(self.store, content["requester"]) + requester = Requester.deserialize(content["requester"]) request.requester = requester # hopefully we're now on the master, so this won't recurse! @@ -283,7 +283,7 @@ async def _handle_request( # type: ignore[override] txn_id = content["txn_id"] event_content = content["content"] - requester = Requester.deserialize(self.store, content["requester"]) + requester = Requester.deserialize(content["requester"]) request.requester = requester # hopefully we're now on the master, so this won't recurse! diff --git a/synapse/replication/http/send_events.py b/synapse/replication/http/send_events.py index b020a0fe7c..e0cfb0d17c 100644 --- a/synapse/replication/http/send_events.py +++ b/synapse/replication/http/send_events.py @@ -141,9 +141,7 @@ async def _handle_request( # type: ignore[override] ) event.internal_metadata.outlier = event_payload["outlier"] - requester = Requester.deserialize( - self.store, event_payload["requester"] - ) + requester = Requester.deserialize(event_payload["requester"]) context = EventContext.deserialize( self._storage_controllers, event_payload["context"] ) diff --git a/synapse/rest/client/account.py b/synapse/rest/client/account.py index 3cb1e09f44..d1e404f0dc 100644 --- a/synapse/rest/client/account.py +++ b/synapse/rest/client/account.py @@ -305,7 +305,7 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: # allow ASes to deactivate their own users: # ASes don't need user-interactive auth - if not requester.app_service: + if not requester.app_service_id: await self.auth_handler.validate_user_via_ui_auth( requester, request, diff --git a/synapse/rest/client/appservice_ping.py b/synapse/rest/client/appservice_ping.py index 7e2ac15783..2c6ad5bcf0 100644 --- a/synapse/rest/client/appservice_ping.py +++ b/synapse/rest/client/appservice_ping.py @@ -55,25 +55,32 @@ def __init__(self, hs: "HomeServer"): self.as_api = hs.get_application_service_api() self.scheduler = hs.get_application_service_scheduler() self.auth = hs.get_auth() + self.store = hs.get_datastores().main async def on_POST( self, request: SynapseRequest, appservice_id: str ) -> tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request) - if not requester.app_service: + app_service = ( + self.store.get_app_service_by_id(requester.app_service_id) + if requester.app_service_id + else None + ) + + if not app_service: raise SynapseError( HTTPStatus.FORBIDDEN, "Only application services can use the /appservice/ping endpoint", Codes.FORBIDDEN, ) - elif requester.app_service.id != appservice_id: + elif app_service.id != appservice_id: raise SynapseError( HTTPStatus.FORBIDDEN, "Mismatching application service ID in path", Codes.FORBIDDEN, ) - elif not requester.app_service.url: + elif not app_service.url: raise SynapseError( HTTPStatus.BAD_REQUEST, "The application service does not have a URL set", @@ -85,11 +92,11 @@ async def on_POST( start = time.monotonic() try: - await self.as_api.ping(requester.app_service, txn_id) + await self.as_api.ping(app_service, txn_id) # We got a OK response, so if the AS needs to be recovered then lets recover it now. # This sets off a task in the background and so is safe to execute and forget. - self.scheduler.txn_ctrl.force_retry(requester.app_service) + self.scheduler.txn_ctrl.force_retry(app_service) except RequestTimedOutError as e: raise SynapseError( HTTPStatus.GATEWAY_TIMEOUT, diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index 4b84131d32..0231ed374d 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -105,7 +105,7 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: else: raise e - if requester.app_service: + if requester.app_service_id: # MSC4190 can skip UIA for this endpoint pass else: @@ -177,7 +177,7 @@ async def on_DELETE( else: raise - if requester.app_service: + if requester.app_service_id: # MSC4190 allows appservices to delete devices through this endpoint without UIA # It's also allowed with MSC3861 enabled pass @@ -212,7 +212,7 @@ async def on_PUT( body = parse_and_validate_json_object_from_request(request, self.PutBody) # MSC4190 allows appservices to create devices through this endpoint - if requester.app_service: + if requester.app_service_id: created = await self.device_handler.upsert_device( user_id=requester.user.to_string(), device_id=device_id, diff --git a/synapse/rest/client/directory.py b/synapse/rest/client/directory.py index 0b334f9b0b..04941b25f3 100644 --- a/synapse/rest/client/directory.py +++ b/synapse/rest/client/directory.py @@ -110,14 +110,19 @@ async def on_DELETE( room_alias_obj = RoomAlias.from_string(room_alias) requester = await self.auth.get_user_by_req(request) - if requester.app_service: + app_service = ( + self.store.get_app_service_by_id(requester.app_service_id) + if requester.app_service_id + else None + ) + if app_service: await self.directory_handler.delete_appservice_association( - requester.app_service, room_alias_obj + app_service, room_alias_obj ) logger.info( "Application service at %s deleted alias %s", - requester.app_service.url, + app_service.url, room_alias_obj.to_string(), ) @@ -199,13 +204,13 @@ async def _edit( visibility: Literal["public", "private"], ) -> tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request) - if not requester.app_service: + if not requester.app_service_id: raise AuthError( 403, "Only appservices can edit the appservice published room list" ) await self.directory_handler.edit_published_appservice_room_list( - requester.app_service.id, network_id, room_id, visibility + requester.app_service_id, network_id, room_id, visibility ) return 200, {} diff --git a/synapse/rest/client/keys.py b/synapse/rest/client/keys.py index 2c65a55ea1..463c87d92b 100644 --- a/synapse/rest/client/keys.py +++ b/synapse/rest/client/keys.py @@ -535,7 +535,7 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: # setup, and that is allowed without UIA, per MSC3967. # If yes, then we need to authenticate the change. # MSC4190 can skip UIA for replacing cross-signing keys as well. - if is_cross_signing_setup and not requester.app_service: + if is_cross_signing_setup and not requester.app_service_id: # With MSC3861, UIA is not possible. Instead, the auth service has to # explicitly mark the master key as replaceable. if self.hs.config.mas.enabled: diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index a49b27d009..cfdc97b664 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -203,7 +203,11 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, LoginResponse]: try: if login_submission["type"] == LoginRestServlet.APPSERVICE_TYPE: requester = await self.auth.get_user_by_req(request) - appservice = requester.app_service + appservice = ( + self._main_store.get_app_service_by_id(requester.app_service_id) + if requester.app_service_id + else None + ) if appservice is None: raise InvalidClientTokenError( diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 48ef42c7d6..6e00197b6d 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -336,7 +336,7 @@ async def on_PUT( ) origin_server_ts = None - if requester.app_service: + if requester.app_service_id: origin_server_ts = parse_integer(request, "ts") sticky_duration_ms: int | None = None @@ -435,7 +435,7 @@ async def _do( content = parse_json_object_from_request(request) origin_server_ts = None - if requester.app_service: + if requester.app_service_id: origin_server_ts = parse_integer(request, "ts") sticky_duration_ms: int | None = None diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py index 43c7b6f993..0046b31b49 100644 --- a/synapse/rest/client/transactions.py +++ b/synapse/rest/client/transactions.py @@ -82,8 +82,8 @@ def _get_transaction_key(self, request: IRequest, requester: Requester) -> Hasha assert requester.user is not None, "Guest requester must have a user ID set" return (path, "guest", requester.user) - elif requester.app_service is not None: - return (path, "appservice", requester.app_service.id) + elif requester.app_service_id is not None: + return (path, "appservice", requester.app_service_id) # Use the user ID and device ID as the transaction key. elif requester.device_id: diff --git a/synapse/rest/media/create_resource.py b/synapse/rest/media/create_resource.py index 8bef1f4c08..1b6b001b45 100644 --- a/synapse/rest/media/create_resource.py +++ b/synapse/rest/media/create_resource.py @@ -43,6 +43,7 @@ def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"): super().__init__() self.media_repo = media_repo + self.store = hs.get_datastores().main self.clock = hs.get_clock() self.auth = hs.get_auth() self.max_pending_media_uploads = hs.config.media.max_pending_media_uploads @@ -60,7 +61,12 @@ async def on_POST(self, request: SynapseRequest) -> None: # If the create media requests for the user are over the limit, drop them. await self._create_media_rate_limiter.ratelimit(requester) - if not requester.app_service or requester.app_service.is_rate_limited(): + app_service = ( + self.store.get_app_service_by_id(requester.app_service_id) + if requester.app_service_id + else None + ) + if not app_service or app_service.is_rate_limited(): ( reached_pending_limit, first_expiration_ts, diff --git a/synapse/synapse_rust/types.pyi b/synapse/synapse_rust/types.pyi new file mode 100644 index 0000000000..36a030fb57 --- /dev/null +++ b/synapse/synapse_rust/types.pyi @@ -0,0 +1,68 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from synapse.types import JsonDict, UserID + +class Requester: + def __init__( + self, + user: UserID, + access_token_id: int | None, + is_guest: bool, + scope: set[str], + shadow_banned: bool, + device_id: str | None, + app_service_id: str | None, + authenticated_entity: str, + ) -> None: ... + @property + def user_id(self) -> str: + """The ID of the user making the request, in string form (see `user` + for the parsed UserID)""" + + @property + def user(self) -> UserID: + """The user making the request""" + @property + def access_token_id(self) -> int | None: + """The ID of the access token used for this request, or + None for appservices, guests, and tokens generated by the admin API""" + @property + def is_guest(self) -> bool: + """True if the user making this request is a guest user""" + @property + def scope(self) -> set[str]: + """Any scopes associated with the access token used for this request, or + an empty set if no token or a non-oauth token was used""" + @property + def shadow_banned(self) -> bool: + """True if the user making this request has been shadow-banned.""" + @property + def device_id(self) -> str | None: + """The device_id which was set at authentication time, or + None for appservices, guests, and tokens generated by the admin API""" + @property + def app_service_id(self) -> str | None: + """The ID of the AS requesting on behalf of the user, or None.""" + @property + def authenticated_entity(self) -> str: + """The entity that authenticated when making the request. + + This is different to the user_id when an admin user or the server is + "puppeting" the user.""" + def serialize(self) -> JsonDict: + """Converts self to a type that can be serialized as JSON, and then + deserialized by `deserialize`""" + @staticmethod + def deserialize(input: JsonDict) -> Requester: + """Converts a dict that was produced by `serialize` back into a + Requester.""" diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 8537a63bde..f2b1c3e42b 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -61,6 +61,7 @@ ) from synapse.api.errors import Codes, SynapseError +from synapse.synapse_rust.types import Requester from synapse.util.cancellation import cancellable from synapse.util.stringutils import parse_and_validate_server_name @@ -70,7 +71,6 @@ from synapse.appservice.api import ApplicationService from synapse.events import EventBase from synapse.storage.databases.main import DataStore, PurgeEventsStore - from synapse.storage.databases.main.appservice import ApplicationServiceWorkerStore from synapse.storage.util.id_generators import MultiWriterIdGenerator @@ -138,82 +138,6 @@ class ISynapseReactor( """The interfaces necessary for Synapse to function.""" -@attr.s(frozen=True, slots=True, auto_attribs=True) -class Requester: - """ - Represents the user making a request - - Attributes: - user: id of the user making the request - access_token_id: *ID* of the access token used for this request, or - None for appservices, guests, and tokens generated by the admin API - is_guest: True if the user making this request is a guest user - shadow_banned: True if the user making this request has been shadow-banned. - device_id: device_id which was set at authentication time, or - None for appservices, guests, and tokens generated by the admin API - app_service: the AS requesting on behalf of the user - authenticated_entity: The entity that authenticated when making the request. - This is different to the user_id when an admin user or the server is - "puppeting" the user. - """ - - user: "UserID" - access_token_id: int | None - is_guest: bool - scope: set[str] - shadow_banned: bool - device_id: str | None - app_service: Optional["ApplicationService"] - authenticated_entity: str - - def serialize(self) -> dict[str, Any]: - """Converts self to a type that can be serialized as JSON, and then - deserialized by `deserialize` - - Returns: - dict - """ - return { - "user_id": self.user.to_string(), - "access_token_id": self.access_token_id, - "is_guest": self.is_guest, - "scope": list(self.scope), - "shadow_banned": self.shadow_banned, - "device_id": self.device_id, - "app_server_id": self.app_service.id if self.app_service else None, - "authenticated_entity": self.authenticated_entity, - } - - @staticmethod - def deserialize( - store: "ApplicationServiceWorkerStore", input: dict[str, Any] - ) -> "Requester": - """Converts a dict that was produced by `serialize` back into a - Requester. - - Args: - store: Used to convert AS ID to AS object - input: A dict produced by `serialize` - - Returns: - Requester - """ - appservice = None - if input["app_server_id"]: - appservice = store.get_app_service_by_id(input["app_server_id"]) - - return Requester( - user=UserID.from_string(input["user_id"]), - access_token_id=input["access_token_id"], - is_guest=input["is_guest"], - scope=set(input.get("scope", [])), - shadow_banned=input["shadow_banned"], - device_id=input["device_id"], - app_service=appservice, - authenticated_entity=input["authenticated_entity"], - ) - - def create_requester( user_id: Union[str, "UserID"], access_token_id: int | None = None, @@ -258,7 +182,7 @@ def create_requester( scope, shadow_banned, device_id, - app_service, + app_service.id if app_service else None, authenticated_entity, ) diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index f7905ced7e..e7ee7eec91 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -108,6 +108,7 @@ def test_get_user_by_req_user_missing_token(self) -> None: def test_get_user_by_req_appservice_valid_token(self) -> None: app_service = Mock( + id="as_id", token="foobar", url="a_url", sender=self.test_user_id, @@ -132,6 +133,7 @@ def test_get_user_by_req_appservice_valid_token_good_ip(self) -> None: sender=self.test_user_id.to_string(), ip_range_whitelist=IPSet(["192.168.0.0/16"]), ) + app_service.id = "as_id" self.store.get_app_service_by_token = Mock(return_value=app_service) self.store.get_user_by_access_token = AsyncMock(return_value=None) @@ -151,6 +153,7 @@ def test_get_user_by_req_appservice_valid_token_bad_ip(self) -> None: sender=self.test_user_id, ip_range_whitelist=IPSet(["192.168.0.0/16"]), ) + app_service.id = "as_id" self.store.get_app_service_by_token = Mock(return_value=app_service) self.store.get_user_by_access_token = AsyncMock(return_value=None) @@ -179,6 +182,7 @@ def test_get_user_by_req_appservice_bad_token(self) -> None: def test_get_user_by_req_appservice_missing_token(self) -> None: app_service = Mock(token="foobar", url="a_url", sender=self.test_user_id) + app_service.id = "as_id" self.store.get_app_service_by_token = Mock(return_value=app_service) self.store.get_user_by_access_token = AsyncMock(return_value=None) @@ -199,6 +203,7 @@ def test_get_user_by_req_appservice_valid_token_valid_user_id(self) -> None: ip_range_whitelist=None, ) app_service.is_interested_in_user = Mock(return_value=True) + app_service.id = "as_id" self.store.get_app_service_by_token = Mock(return_value=app_service) class FakeUserInfo: @@ -226,6 +231,7 @@ def test_get_user_by_req_appservice_valid_token_bad_user_id(self) -> None: ip_range_whitelist=None, ) app_service.is_interested_in_user = Mock(return_value=False) + app_service.id = "as_id" self.store.get_app_service_by_token = Mock(return_value=app_service) self.store.get_user_by_access_token = AsyncMock(return_value=None) @@ -251,6 +257,7 @@ def test_get_user_by_req_appservice_valid_token_valid_device_id(self) -> None: ip_range_whitelist=None, ) app_service.is_interested_in_user = Mock(return_value=True) + app_service.id = "as_id" self.store.get_app_service_by_token = Mock(return_value=app_service) # This just needs to return a truth-y value. self.store.get_user_by_id = AsyncMock(return_value={"is_guest": False}) @@ -285,6 +292,7 @@ def test_get_user_by_req_appservice_valid_token_invalid_device_id(self) -> None: ip_range_whitelist=None, ) app_service.is_interested_in_user = Mock(return_value=True) + app_service.id = "as_id" self.store.get_app_service_by_token = Mock(return_value=app_service) # This just needs to return a truth-y value. self.store.get_user_by_id = AsyncMock(return_value={"is_guest": False}) @@ -457,7 +465,7 @@ def test_blocking_mau__appservice_requester_allowed_when_not_tracking_ips( is_guest=False, scope=set(), shadow_banned=False, - app_service=appservice, + app_service_id=appservice.id, authenticated_entity="@appservice:server", ) self.get_success(self.auth_blocking.check_auth_blocking(requester=requester)) @@ -488,7 +496,7 @@ def test_blocking_mau__appservice_requester_disallowed_when_tracking_ips( is_guest=False, scope=set(), shadow_banned=False, - app_service=appservice, + app_service_id=appservice.id, authenticated_entity="@appservice:server", ) self.get_failure( diff --git a/tests/api/test_ratelimiting.py b/tests/api/test_ratelimiting.py index 0ef537841d..3237c6d412 100644 --- a/tests/api/test_ratelimiting.py +++ b/tests/api/test_ratelimiting.py @@ -40,6 +40,9 @@ def test_allowed_appservice_ratelimited_via_can_requester_do_action(self) -> Non rate_limited=True, sender=UserID.from_string("@as:example.com"), ) + # The ratelimiter now resolves the AS via get_app_service_by_id, so the + # appservice must be in the store's cache for the lookup to hit. + self.hs.get_datastores().main.services_cache.append(appservice) as_requester = create_requester("@user:example.com", app_service=appservice) limiter = Ratelimiter( @@ -76,6 +79,9 @@ def test_allowed_appservice_via_can_requester_do_action(self) -> None: rate_limited=False, sender=UserID.from_string("@as:example.com"), ) + # The ratelimiter now resolves the AS via get_app_service_by_id, so the + # appservice must be in the store's cache for the lookup to hit. + self.hs.get_datastores().main.services_cache.append(appservice) as_requester = create_requester("@user:example.com", app_service=appservice) limiter = Ratelimiter( diff --git a/tests/rest/client/test_transactions.py b/tests/rest/client/test_transactions.py index 31586a451f..d62f9dd4a5 100644 --- a/tests/rest/client/test_transactions.py +++ b/tests/rest/client/test_transactions.py @@ -52,7 +52,7 @@ def setUp(self) -> None: self.mock_request = Mock() self.mock_request.path = b"/foo/bar" self.mock_requester = Mock() - self.mock_requester.app_service = None + self.mock_requester.app_service_id = None self.mock_requester.is_guest = False self.mock_requester.access_token_id = 1234 From edadf5768d9a39887cce99c95783fa6a26fd5439 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 9 Jun 2026 14:57:14 +0100 Subject: [PATCH 20/26] Fix the `/capabilities` endpoint returning a 500 error on non-media workers when MSC4452: Preview URL capabilities API is enabled. (#19839) Fixes: #19825 Introduced in: #19715 Always populate `url_preview_enabled` so `/capabilities` can expose it Needed so this line can be happy: https://github.com/element-hq/synapse/blob/106ed3623d434891fe1ac50aacc851e9804404fe/synapse/rest/client/capabilities.py#L82 --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19839.bugfix | 1 + synapse/config/repository.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19839.bugfix diff --git a/changelog.d/19839.bugfix b/changelog.d/19839.bugfix new file mode 100644 index 0000000000..74606ce6f1 --- /dev/null +++ b/changelog.d/19839.bugfix @@ -0,0 +1 @@ +Fix the `/capabilities` endpoint returning a 500 error on non-media workers when [MSC4452: Preview URL capabilities API](https://github.com/matrix-org/matrix-spec-proposals/pull/4452) is enabled. \ No newline at end of file diff --git a/synapse/config/repository.py b/synapse/config/repository.py index cb50d0dc1d..373e518ddc 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -139,6 +139,10 @@ class ContentRepositoryConfig(Config): section = "media" def read_config(self, config: JsonDict, **kwargs: Any) -> None: + # We need to set this configuration flag even if this worker + # is not a media repo worker, as it's exposed in `/capabilities` + self.url_preview_enabled = bool(config.get("url_preview_enabled", False)) + # Only enable the media repo if either the media repo is enabled or the # current worker app is the media repo. if ( @@ -242,7 +246,6 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.thumbnail_requirements = parse_thumbnail_requirements( config.get("thumbnail_sizes", DEFAULT_THUMBNAIL_SIZES) ) - self.url_preview_enabled = bool(config.get("url_preview_enabled", False)) if self.url_preview_enabled: check_requirements("url-preview") From f53f1044494c6bd002889d29df35cf2f168860c3 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 9 Jun 2026 14:59:48 +0100 Subject: [PATCH 21/26] 1.155.0rc1 --- CHANGES.md | 22 ++++++++++++++++++++++ changelog.d/19617.bugfix | 1 - changelog.d/19701.misc | 1 - changelog.d/19730.bugfix | 1 - changelog.d/19734.bugfix | 1 - changelog.d/19741.misc | 1 - changelog.d/19775.misc | 1 - changelog.d/19792.bugfix | 1 - changelog.d/19801.misc | 1 - changelog.d/19816.misc | 1 - changelog.d/19817.misc | 1 - changelog.d/19818.doc | 1 - changelog.d/19819.misc | 1 - changelog.d/19821.misc | 1 - changelog.d/19823.doc | 1 - changelog.d/19828.misc | 1 - changelog.d/19839.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- schema/synapse-config.schema.yaml | 2 +- 20 files changed, 30 insertions(+), 18 deletions(-) delete mode 100644 changelog.d/19617.bugfix delete mode 100644 changelog.d/19701.misc delete mode 100644 changelog.d/19730.bugfix delete mode 100644 changelog.d/19734.bugfix delete mode 100644 changelog.d/19741.misc delete mode 100644 changelog.d/19775.misc delete mode 100644 changelog.d/19792.bugfix delete mode 100644 changelog.d/19801.misc delete mode 100644 changelog.d/19816.misc delete mode 100644 changelog.d/19817.misc delete mode 100644 changelog.d/19818.doc delete mode 100644 changelog.d/19819.misc delete mode 100644 changelog.d/19821.misc delete mode 100644 changelog.d/19823.doc delete mode 100644 changelog.d/19828.misc delete mode 100644 changelog.d/19839.bugfix diff --git a/CHANGES.md b/CHANGES.md index e210f8f1d5..13e25709cd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,25 @@ +# Synapse 1.155.0rc1 (2026-06-09) + +## Bugfixes + +- Limit the to-device EDU size to a reasonable value to mitigate long queues of to-device messages preventing outgoing federation because of the size of the transaction. ([\#19617](https://github.com/element-hq/synapse/issues/19617)) +- Work around bug that sometimes breaks joining restricted rooms that require a remote join. Contributed by @tulir @ Beeper. ([\#19730](https://github.com/element-hq/synapse/issues/19730)) +- Update Sliding Sync to return a new response immediately if a room subscription has changed and produced a new response. ([\#19734](https://github.com/element-hq/synapse/issues/19734), [\#19792](https://github.com/element-hq/synapse/issues/19792)) +- Fix the `/capabilities` endpoint returning a 500 error on non-media workers when [MSC4452: Preview URL capabilities API](https://github.com/matrix-org/matrix-spec-proposals/pull/4452) is enabled. ([\#19839](https://github.com/element-hq/synapse/issues/19839)) + +## Improved Documentation + +- Document how to see Rust build failure output when using `poetry install`. ([\#19818](https://github.com/element-hq/synapse/issues/19818)) +- Document that the SQLite version included in Ubuntu LTS, aside from ESM-only versions, is included in our support policy. ([\#19823](https://github.com/element-hq/synapse/issues/19823)) + +## Internal Changes + +- Port the Python Event classes to Rust. ([\#19701](https://github.com/element-hq/synapse/issues/19701), [\#19816](https://github.com/element-hq/synapse/issues/19816), [\#19817](https://github.com/element-hq/synapse/issues/19817), [\#19819](https://github.com/element-hq/synapse/issues/19819)) +- Added tests to ensure that email notification links are sanitized. Contributed by Noah Markert. ([\#19741](https://github.com/element-hq/synapse/issues/19741)) +- Add `GcpJsonFormatter` logging formatter for use with Google Cloud Logging and GKE deployments. ([\#19775](https://github.com/element-hq/synapse/issues/19775)) +- Add more logging to the to-device message replication stream. ([\#19801](https://github.com/element-hq/synapse/issues/19801), [\#19821](https://github.com/element-hq/synapse/issues/19821)) +- Port `Requester` class to Rust. ([\#19828](https://github.com/element-hq/synapse/issues/19828)) + # Synapse 1.154.0 (2026-06-04) No significant changes since 1.154.0rc1. diff --git a/changelog.d/19617.bugfix b/changelog.d/19617.bugfix deleted file mode 100644 index 71f181fed4..0000000000 --- a/changelog.d/19617.bugfix +++ /dev/null @@ -1 +0,0 @@ -A long queue of to-device messages could prevent outgoing federation because of the size of the transaction, let's limit the to-device EDU size to a reasonable value. diff --git a/changelog.d/19701.misc b/changelog.d/19701.misc deleted file mode 100644 index 4663e8b961..0000000000 --- a/changelog.d/19701.misc +++ /dev/null @@ -1 +0,0 @@ -Port the python Event classes to Rust. diff --git a/changelog.d/19730.bugfix b/changelog.d/19730.bugfix deleted file mode 100644 index 074405d944..0000000000 --- a/changelog.d/19730.bugfix +++ /dev/null @@ -1 +0,0 @@ -Work around bug that sometimes breaks joining restricted rooms that require a remote join. Contributed by @tulir @ Beeper. diff --git a/changelog.d/19734.bugfix b/changelog.d/19734.bugfix deleted file mode 100644 index 01af7d9ab8..0000000000 --- a/changelog.d/19734.bugfix +++ /dev/null @@ -1 +0,0 @@ -Update Sliding Sync to return a new response immediately if a room subscription have changed and produced a new response. diff --git a/changelog.d/19741.misc b/changelog.d/19741.misc deleted file mode 100644 index 55347570d8..0000000000 --- a/changelog.d/19741.misc +++ /dev/null @@ -1 +0,0 @@ -Added tests to ensure that email notification links are sanitized. Contributed by Noah Markert. diff --git a/changelog.d/19775.misc b/changelog.d/19775.misc deleted file mode 100644 index 3ff0fd34b9..0000000000 --- a/changelog.d/19775.misc +++ /dev/null @@ -1 +0,0 @@ -Add `GcpJsonFormatter` logging formatter for use with Google Cloud Logging and GKE deployments. \ No newline at end of file diff --git a/changelog.d/19792.bugfix b/changelog.d/19792.bugfix deleted file mode 100644 index 01af7d9ab8..0000000000 --- a/changelog.d/19792.bugfix +++ /dev/null @@ -1 +0,0 @@ -Update Sliding Sync to return a new response immediately if a room subscription have changed and produced a new response. diff --git a/changelog.d/19801.misc b/changelog.d/19801.misc deleted file mode 100644 index 56ec0f7ddc..0000000000 --- a/changelog.d/19801.misc +++ /dev/null @@ -1 +0,0 @@ -Add more logging to the to-device message replication stream. diff --git a/changelog.d/19816.misc b/changelog.d/19816.misc deleted file mode 100644 index 4663e8b961..0000000000 --- a/changelog.d/19816.misc +++ /dev/null @@ -1 +0,0 @@ -Port the python Event classes to Rust. diff --git a/changelog.d/19817.misc b/changelog.d/19817.misc deleted file mode 100644 index 4663e8b961..0000000000 --- a/changelog.d/19817.misc +++ /dev/null @@ -1 +0,0 @@ -Port the python Event classes to Rust. diff --git a/changelog.d/19818.doc b/changelog.d/19818.doc deleted file mode 100644 index c6aca25257..0000000000 --- a/changelog.d/19818.doc +++ /dev/null @@ -1 +0,0 @@ -Document how to see Rust build failure output when using `poetry install`. diff --git a/changelog.d/19819.misc b/changelog.d/19819.misc deleted file mode 100644 index 4663e8b961..0000000000 --- a/changelog.d/19819.misc +++ /dev/null @@ -1 +0,0 @@ -Port the python Event classes to Rust. diff --git a/changelog.d/19821.misc b/changelog.d/19821.misc deleted file mode 100644 index 56ec0f7ddc..0000000000 --- a/changelog.d/19821.misc +++ /dev/null @@ -1 +0,0 @@ -Add more logging to the to-device message replication stream. diff --git a/changelog.d/19823.doc b/changelog.d/19823.doc deleted file mode 100644 index 5bbbdd5ccf..0000000000 --- a/changelog.d/19823.doc +++ /dev/null @@ -1 +0,0 @@ -Document that the SQLite version included in Ubuntu LTS, aside from ESM-only versions, is included in our support policy. \ No newline at end of file diff --git a/changelog.d/19828.misc b/changelog.d/19828.misc deleted file mode 100644 index 1ed01eba2e..0000000000 --- a/changelog.d/19828.misc +++ /dev/null @@ -1 +0,0 @@ -Port `Requester` class to Rust. diff --git a/changelog.d/19839.bugfix b/changelog.d/19839.bugfix deleted file mode 100644 index 74606ce6f1..0000000000 --- a/changelog.d/19839.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix the `/capabilities` endpoint returning a 500 error on non-media workers when [MSC4452: Preview URL capabilities API](https://github.com/matrix-org/matrix-spec-proposals/pull/4452) is enabled. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index 7750ca563c..617bc73611 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.155.0~rc1) stable; urgency=medium + + * New Synapse release 1.155.0rc1. + + -- Synapse Packaging team Tue, 09 Jun 2026 14:58:03 +0100 + matrix-synapse-py3 (1.154.0) stable; urgency=medium * New Synapse release 1.154.0. diff --git a/pyproject.toml b/pyproject.toml index 89bace10a2..3500b4f516 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.154.0" +version = "1.155.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index dc57cfeea5..1a2caddbfd 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/schema/synapse/v1.154/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.155/synapse-config.schema.json type: object properties: modules: From c3351eede074eff5b2d14fc956647a5187fe2934 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 10 Jun 2026 10:56:08 +0100 Subject: [PATCH 22/26] When building releases, don't cancel Debian package builds when one of them fails. (#19842) When building v1.155.0rc1, flaky deb builds combined with fail-fast behaviour meant that I had to press the retry button 5 times to get a full set. Without fail-fast, I suspect 1 or 2 retries would have done the job. --------- Signed-off-by: Olivier 'reivilibre Co-authored-by: Eric Eastwood --- .github/workflows/release-artifacts.yml | 6 ++++++ changelog.d/19842.misc | 1 + 2 files changed, 7 insertions(+) create mode 100644 changelog.d/19842.misc diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 41a7cb5611..8be377bded 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -50,6 +50,12 @@ jobs: name: "Build .deb packages" runs-on: ubuntu-latest strategy: + # Prevent all Debian package builds from being cancelled + # when one of them fails. Especially helpful when things are + # flaky during a release. + # + # `fail-fast` defaults to true and applies to the entire test matrix + fail-fast: false matrix: distro: ${{ fromJson(needs.get-distros.outputs.distros) }} diff --git a/changelog.d/19842.misc b/changelog.d/19842.misc new file mode 100644 index 0000000000..b0fe5c072e --- /dev/null +++ b/changelog.d/19842.misc @@ -0,0 +1 @@ +When building releases, don't cancel Debian package builds when one of them fails. From 3fad636880891a6120e57d6a181535ef0a3d22a4 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 16 Jun 2026 14:48:25 +0100 Subject: [PATCH 23/26] 1.155.0 --- CHANGES.md | 11 +++++++++++ changelog.d/19842.misc | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 18 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/19842.misc diff --git a/CHANGES.md b/CHANGES.md index 13e25709cd..70a9b60e4f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,14 @@ +# Synapse 1.155.0 (2026-06-16) + +## End of Life of Debian 12 Bookworm + +The next version of Synapse will not include Debian packages for Debian 12 Bookworm +as it reached end of life on the 10th of June 2026. + +## Internal Changes + +- When building releases, don't cancel Debian package builds when one of them fails. ([\#19842](https://github.com/element-hq/synapse/issues/19842)) + # Synapse 1.155.0rc1 (2026-06-09) ## Bugfixes diff --git a/changelog.d/19842.misc b/changelog.d/19842.misc deleted file mode 100644 index b0fe5c072e..0000000000 --- a/changelog.d/19842.misc +++ /dev/null @@ -1 +0,0 @@ -When building releases, don't cancel Debian package builds when one of them fails. diff --git a/debian/changelog b/debian/changelog index 617bc73611..609b22438a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.155.0) stable; urgency=medium + + * New Synapse release 1.155.0. + + -- Synapse Packaging team Tue, 16 Jun 2026 14:44:04 +0100 + matrix-synapse-py3 (1.155.0~rc1) stable; urgency=medium * New Synapse release 1.155.0rc1. diff --git a/pyproject.toml b/pyproject.toml index 3500b4f516..fa70791f7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.155.0rc1" +version = "1.155.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From 0b8abb80155b9bf31c20c49140ea4d5213ab2750 Mon Sep 17 00:00:00 2001 From: FrenchGithubUser Date: Thu, 25 Jun 2026 16:29:26 +0200 Subject: [PATCH 24/26] fix: revert frozen-event workaround superseded by Rust Event port upstream v1.155 ported the Event class to Rust, removing freeze() and unfreeze() from the Python API. Our workaround used a pattern solely to guarantee event.unfreeze() always ran. That pattern is not necessary anymore. Revert to upstream's state. --- .../callbacks/third_party_event_rules_callbacks.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/synapse/module_api/callbacks/third_party_event_rules_callbacks.py b/synapse/module_api/callbacks/third_party_event_rules_callbacks.py index 9333c763e6..6f951c4b72 100644 --- a/synapse/module_api/callbacks/third_party_event_rules_callbacks.py +++ b/synapse/module_api/callbacks/third_party_event_rules_callbacks.py @@ -314,17 +314,11 @@ async def check_event_allowed( # Return if the event shouldn't be allowed or if the module came up with a # replacement dict for the event. if res is False: - ret = (False, None) - break + return res, None elif isinstance(replacement_data, dict): - ret = (True, replacement_data) - break + return True, replacement_data - # We can unfreeze the event unconditionally here, since incoming events should never have been frozen. - # This is necessary, because other parts of the code expect unfrozen events currently. - event.unfreeze() - - return ret + return True, None async def on_create_room( self, requester: Requester, config: dict, is_requester_admin: bool From d87ed0c703280cdb8cde9a95c1728bd6f17f0e60 Mon Sep 17 00:00:00 2001 From: FrenchGithubUser Date: Thu, 25 Jun 2026 16:53:17 +0200 Subject: [PATCH 25/26] chore: ran documentation generation script after bad formatting --- docs/usage/configuration/config_documentation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 548687a39e..76aac5110b 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -3822,7 +3822,7 @@ This setting has the following sub-options: Defaults to `null`. -* `update_profile_information` (boolean): Use this setting to keep a user's profile fields in sync with information from the identity provider. Fields are checked on every SSO login, and are updated if necessary. Note that enabling this option will override user profile information, regardless of whether users have opted-out of syncing that information when first signing in. Fields that will be synced: +* `update_profile_information` (boolean): Use this setting to keep a user's profile fields in sync with information from the identity provider. Fields are checked on every SSO login, and are updated if necessary. Note that enabling this option will override user profile information, regardless of whether users have opted-out of syncing that information when first signing in. Fields that will be synced: * displayname * picture - only if Synapse media repository is running in the main process (i.e. not workerized) and media is stored locally Defaults to `false`. From 74995010088420b74d2fe32be6f7efd42046430c Mon Sep 17 00:00:00 2001 From: FrenchGithubUser Date: Tue, 30 Jun 2026 14:53:33 +0200 Subject: [PATCH 26/26] Update changelog --- CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index ae172c8aa9..848cf020d7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,6 +9,10 @@ as it reached end of life on the 10th of June 2026. - When building releases, don't cancel Debian package builds when one of them fails. ([\#19842](https://github.com/element-hq/synapse/issues/19842)) +## Famedly additions for v1.155.0_1 + +- add destination to incoming federation responses metrics (Jason Little) + # Synapse 1.155.0rc1 (2026-06-09) ## Bugfixes