diff --git a/Cargo.lock b/Cargo.lock index 43e5c1cc787..a04e1f97fde 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4397,7 +4397,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.12.1" -source = "git+https://github.com/ruma/ruma?rev=b1cb83544faafaef92be56c53cd98af4c51da858#b1cb83544faafaef92be56c53cd98af4c51da858" +source = "git+https://github.com/ruma/ruma?rev=25b70024889340633fd0e1f856349ddd5b5a687a#25b70024889340633fd0e1f856349ddd5b5a687a" dependencies = [ "assign", "js_int", @@ -4413,7 +4413,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.20.1" -source = "git+https://github.com/ruma/ruma?rev=b1cb83544faafaef92be56c53cd98af4c51da858#b1cb83544faafaef92be56c53cd98af4c51da858" +source = "git+https://github.com/ruma/ruma?rev=25b70024889340633fd0e1f856349ddd5b5a687a#25b70024889340633fd0e1f856349ddd5b5a687a" dependencies = [ "as_variant", "assign", @@ -4436,7 +4436,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.15.1" -source = "git+https://github.com/ruma/ruma?rev=b1cb83544faafaef92be56c53cd98af4c51da858#b1cb83544faafaef92be56c53cd98af4c51da858" +source = "git+https://github.com/ruma/ruma?rev=25b70024889340633fd0e1f856349ddd5b5a687a#25b70024889340633fd0e1f856349ddd5b5a687a" dependencies = [ "as_variant", "base64", @@ -4468,7 +4468,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.30.1" -source = "git+https://github.com/ruma/ruma?rev=b1cb83544faafaef92be56c53cd98af4c51da858#b1cb83544faafaef92be56c53cd98af4c51da858" +source = "git+https://github.com/ruma/ruma?rev=25b70024889340633fd0e1f856349ddd5b5a687a#25b70024889340633fd0e1f856349ddd5b5a687a" dependencies = [ "as_variant", "indexmap", @@ -4493,7 +4493,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.11.0" -source = "git+https://github.com/ruma/ruma?rev=b1cb83544faafaef92be56c53cd98af4c51da858#b1cb83544faafaef92be56c53cd98af4c51da858" +source = "git+https://github.com/ruma/ruma?rev=25b70024889340633fd0e1f856349ddd5b5a687a#25b70024889340633fd0e1f856349ddd5b5a687a" dependencies = [ "http", "js_int", @@ -4507,7 +4507,7 @@ dependencies = [ [[package]] name = "ruma-html" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=b1cb83544faafaef92be56c53cd98af4c51da858#b1cb83544faafaef92be56c53cd98af4c51da858" +source = "git+https://github.com/ruma/ruma?rev=25b70024889340633fd0e1f856349ddd5b5a687a#25b70024889340633fd0e1f856349ddd5b5a687a" dependencies = [ "as_variant", "html5ever", @@ -4519,7 +4519,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.10.1" -source = "git+https://github.com/ruma/ruma?rev=b1cb83544faafaef92be56c53cd98af4c51da858#b1cb83544faafaef92be56c53cd98af4c51da858" +source = "git+https://github.com/ruma/ruma?rev=25b70024889340633fd0e1f856349ddd5b5a687a#25b70024889340633fd0e1f856349ddd5b5a687a" dependencies = [ "js_int", "thiserror 2.0.11", @@ -4528,7 +4528,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.15.1" -source = "git+https://github.com/ruma/ruma?rev=b1cb83544faafaef92be56c53cd98af4c51da858#b1cb83544faafaef92be56c53cd98af4c51da858" +source = "git+https://github.com/ruma/ruma?rev=25b70024889340633fd0e1f856349ddd5b5a687a#25b70024889340633fd0e1f856349ddd5b5a687a" dependencies = [ "cfg-if", "proc-macro-crate", diff --git a/Cargo.toml b/Cargo.toml index ed28f21910a..e3cd86d954f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,7 @@ reqwest = { version = "0.12.12", default-features = false } rmp-serde = "1.3.0" # Be careful to use commits from the https://github.com/ruma/ruma/tree/ruma-0.12 # branch until a proper release with breaking changes happens. -ruma = { git = "https://github.com/ruma/ruma", rev = "b1cb83544faafaef92be56c53cd98af4c51da858", features = [ +ruma = { git = "https://github.com/ruma/ruma", rev = "25b70024889340633fd0e1f856349ddd5b5a687a", features = [ "client-api-c", "compat-upload-signatures", "compat-user-id", @@ -74,8 +74,9 @@ ruma = { git = "https://github.com/ruma/ruma", rev = "b1cb83544faafaef92be56c53c "unstable-msc4075", "unstable-msc4140", "unstable-msc4171", + "unstable-msc4274" ] } -ruma-common = { git = "https://github.com/ruma/ruma", rev = "b1cb83544faafaef92be56c53cd98af4c51da858" } +ruma-common = { git = "https://github.com/ruma/ruma", rev = "25b70024889340633fd0e1f856349ddd5b5a687a" } serde = "1.0.217" serde_html_form = "0.2.7" serde_json = "1.0.138" diff --git a/bindings/matrix-sdk-ffi/src/event.rs b/bindings/matrix-sdk-ffi/src/event.rs index 188c75e9ce4..63ded95a56d 100644 --- a/bindings/matrix-sdk-ffi/src/event.rs +++ b/bindings/matrix-sdk-ffi/src/event.rs @@ -365,6 +365,7 @@ pub enum RoomMessageEventMessageType { Audio, Emote, File, + Gallery, Image, Location, Notice, @@ -381,6 +382,7 @@ impl From for RoomMessageEventMessageType { RumaMessageType::Audio { .. } => Self::Audio, RumaMessageType::Emote { .. } => Self::Emote, RumaMessageType::File { .. } => Self::File, + RumaMessageType::Gallery { .. } => Self::Gallery, RumaMessageType::Image { .. } => Self::Image, RumaMessageType::Location { .. } => Self::Location, RumaMessageType::Notice { .. } => Self::Notice, diff --git a/bindings/matrix-sdk-ffi/src/ruma.rs b/bindings/matrix-sdk-ffi/src/ruma.rs index 2bc395be8ff..3a675e95a70 100644 --- a/bindings/matrix-sdk-ffi/src/ruma.rs +++ b/bindings/matrix-sdk-ffi/src/ruma.rs @@ -28,7 +28,8 @@ use ruma::{ AudioMessageEventContent as RumaAudioMessageEventContent, EmoteMessageEventContent as RumaEmoteMessageEventContent, FileInfo as RumaFileInfo, FileMessageEventContent as RumaFileMessageEventContent, - FormattedBody as RumaFormattedBody, + FormattedBody as RumaFormattedBody, GalleryItemType as RumaGalleryItemType, + GalleryMessageEventContent as RumaGalleryMessageEventContent, ImageMessageEventContent as RumaImageMessageEventContent, LocationMessageEventContent as RumaLocationMessageEventContent, MessageType as RumaMessageType, @@ -313,6 +314,7 @@ pub enum MessageType { Audio { content: AudioMessageContent }, Video { content: VideoMessageContent }, File { content: FileMessageContent }, + Gallery { content: GalleryMessageContent }, Notice { content: NoticeMessageContent }, Text { content: TextMessageContent }, Location { content: LocationContent }, @@ -382,6 +384,18 @@ impl TryFrom for RumaMessageType { event_content.filename = filename; Self::File(event_content) } + MessageType::Gallery { content } => { + let event_content = RumaGalleryMessageEventContent::new( + content.body, + content.formatted.map(Into::into), + content + .itemtypes + .into_iter() + .map(TryInto::try_into) + .collect::, _>>()?, + ); + Self::Gallery(event_content) + } MessageType::Notice { content } => { Self::Notice(assign!(RumaNoticeMessageEventContent::plain(content.body), { formatted: content.formatted.map(Into::into), @@ -452,6 +466,17 @@ impl TryFrom for MessageType { info: c.info.as_deref().map(TryInto::try_into).transpose()?, }, }, + RumaMessageType::Gallery(c) => MessageType::Gallery { + content: GalleryMessageContent { + body: c.body, + formatted: c.formatted.as_ref().map(Into::into), + itemtypes: c + .itemtypes + .into_iter() + .map(TryInto::try_into) + .collect::, _>>()?, + }, + }, RumaMessageType::Notice(c) => MessageType::Notice { content: NoticeMessageContent { body: c.body.clone(), @@ -489,6 +514,105 @@ impl TryFrom for MessageType { } } +impl TryFrom for RumaGalleryItemType { + type Error = ClientError; + + fn try_from(value: GalleryItemType) -> Result { + Ok(match value { + GalleryItemType::Image { content } => { + let (body, filename) = get_body_and_filename(content.filename, content.caption); + let mut event_content = + RumaImageMessageEventContent::new(body, (*content.source).clone().into()) + .info(content.info.map(Into::into).map(Box::new)); + event_content.formatted = content.formatted_caption.map(Into::into); + event_content.filename = filename; + Self::Image(event_content) + } + GalleryItemType::Audio { content } => { + let (body, filename) = get_body_and_filename(content.filename, content.caption); + let mut event_content = + RumaAudioMessageEventContent::new(body, (*content.source).clone().into()) + .info(content.info.map(Into::into).map(Box::new)); + event_content.formatted = content.formatted_caption.map(Into::into); + event_content.filename = filename; + Self::Audio(event_content) + } + GalleryItemType::Video { content } => { + let (body, filename) = get_body_and_filename(content.filename, content.caption); + let mut event_content = + RumaVideoMessageEventContent::new(body, (*content.source).clone().into()) + .info(content.info.map(Into::into).map(Box::new)); + event_content.formatted = content.formatted_caption.map(Into::into); + event_content.filename = filename; + Self::Video(event_content) + } + GalleryItemType::File { content } => { + let (body, filename) = get_body_and_filename(content.filename, content.caption); + let mut event_content = + RumaFileMessageEventContent::new(body, (*content.source).clone().into()) + .info(content.info.map(Into::into).map(Box::new)); + event_content.formatted = content.formatted_caption.map(Into::into); + event_content.filename = filename; + Self::File(event_content) + } + GalleryItemType::Other { itemtype, body } => { + Self::new(&itemtype, body, JsonObject::default())? + } + }) + } +} + +impl TryFrom for GalleryItemType { + type Error = ClientError; + + fn try_from(value: RumaGalleryItemType) -> Result { + Ok(match value { + RumaGalleryItemType::Image(c) => GalleryItemType::Image { + content: ImageMessageContent { + filename: c.filename().to_owned(), + caption: c.caption().map(ToString::to_string), + formatted_caption: c.formatted_caption().map(Into::into), + source: Arc::new(c.source.try_into()?), + info: c.info.as_deref().map(TryInto::try_into).transpose()?, + }, + }, + RumaGalleryItemType::Audio(c) => GalleryItemType::Audio { + content: AudioMessageContent { + filename: c.filename().to_owned(), + caption: c.caption().map(ToString::to_string), + formatted_caption: c.formatted_caption().map(Into::into), + source: Arc::new(c.source.try_into()?), + info: c.info.as_deref().map(Into::into), + audio: c.audio.map(Into::into), + voice: c.voice.map(Into::into), + }, + }, + RumaGalleryItemType::Video(c) => GalleryItemType::Video { + content: VideoMessageContent { + filename: c.filename().to_owned(), + caption: c.caption().map(ToString::to_string), + formatted_caption: c.formatted_caption().map(Into::into), + source: Arc::new(c.source.try_into()?), + info: c.info.as_deref().map(TryInto::try_into).transpose()?, + }, + }, + RumaGalleryItemType::File(c) => GalleryItemType::File { + content: FileMessageContent { + filename: c.filename().to_owned(), + caption: c.caption().map(ToString::to_string), + formatted_caption: c.formatted_caption().map(Into::into), + source: Arc::new(c.source.try_into()?), + info: c.info.as_deref().map(TryInto::try_into).transpose()?, + }, + }, + _ => GalleryItemType::Other { + itemtype: value.itemtype().to_owned(), + body: value.body().to_owned(), + }, + }) + } +} + #[derive(Clone, uniffi::Enum)] pub enum NotifyType { Ring, @@ -561,6 +685,22 @@ pub struct FileMessageContent { pub info: Option, } +#[derive(Clone, uniffi::Record)] +pub struct GalleryMessageContent { + pub body: String, + pub formatted: Option, + pub itemtypes: Vec, +} + +#[derive(Clone, uniffi::Enum)] +pub enum GalleryItemType { + Image { content: ImageMessageContent }, + Audio { content: AudioMessageContent }, + Video { content: VideoMessageContent }, + File { content: FileMessageContent }, + Other { itemtype: String, body: String }, +} + #[derive(Clone, uniffi::Record)] pub struct ImageInfo { pub height: Option, @@ -808,6 +948,11 @@ pub struct FormattedBody { pub body: String, } +#[uniffi::export] +pub fn formatted_body_from_html(body: String) -> FormattedBody { + FormattedBody::from(&RumaFormattedBody::html(body)) +} + impl From for RumaFormattedBody { fn from(f: FormattedBody) -> Self { Self { diff --git a/bindings/matrix-sdk-ffi/src/timeline/mod.rs b/bindings/matrix-sdk-ffi/src/timeline/mod.rs index 312d567c6e4..8542094051c 100644 --- a/bindings/matrix-sdk-ffi/src/timeline/mod.rs +++ b/bindings/matrix-sdk-ffi/src/timeline/mod.rs @@ -23,7 +23,7 @@ use futures_util::{pin_mut, StreamExt as _}; use matrix_sdk::{ attachment::{ AttachmentConfig, AttachmentInfo, BaseAudioInfo, BaseFileInfo, BaseImageInfo, - BaseVideoInfo, Thumbnail, + BaseVideoInfo, GalleryConfig, Thumbnail, }, deserialized_responses::{ShieldState as SdkShieldState, ShieldStateCode}, event_cache::RoomPaginationStatus, @@ -210,6 +210,173 @@ pub struct UploadParameters { use_send_queue: bool, } +#[derive(uniffi::Record)] +pub struct GalleryUploadParameters { + /// Optional non-formatted caption, for clients that support it. + caption: Option, + /// Optional HTML-formatted caption, for clients that support it. + formatted_caption: Option, + // Optional intentional mentions to be sent with the gallery. + mentions: Option, +} + +#[derive(uniffi::Enum)] +pub enum GalleryItemInfo { + Audio(AudioGalleryItemInfo), + File(FileGalleryItemInfo), + Image(ImageGalleryItemInfo), + Video(VideoGalleryItemInfo), +} + +impl GalleryItemInfo { + fn mimetype(&self) -> &Option { + match self { + GalleryItemInfo::Audio(info) => &info.audio_info.mimetype, + GalleryItemInfo::File(info) => &info.file_info.mimetype, + GalleryItemInfo::Image(info) => &info.image_info.mimetype, + GalleryItemInfo::Video(info) => &info.video_info.mimetype, + } + } + + fn filename(&self) -> &String { + match self { + GalleryItemInfo::Audio(info) => &info.filename, + GalleryItemInfo::File(info) => &info.filename, + GalleryItemInfo::Image(info) => &info.filename, + GalleryItemInfo::Video(info) => &info.filename, + } + } + + fn caption(&self) -> &Option { + match self { + GalleryItemInfo::Audio(info) => &info.caption, + GalleryItemInfo::File(info) => &info.caption, + GalleryItemInfo::Image(info) => &info.caption, + GalleryItemInfo::Video(info) => &info.caption, + } + } + + fn formatted_caption(&self) -> &Option { + match self { + GalleryItemInfo::Audio(info) => &info.formatted_caption, + GalleryItemInfo::File(info) => &info.formatted_caption, + GalleryItemInfo::Image(info) => &info.formatted_caption, + GalleryItemInfo::Video(info) => &info.formatted_caption, + } + } + + fn attachment_info(&self) -> Result { + match self { + GalleryItemInfo::Audio(info) => Ok(AttachmentInfo::Audio( + BaseAudioInfo::try_from(&info.audio_info) + .map_err(|_| RoomError::InvalidAttachmentData)?, + )), + GalleryItemInfo::File(info) => Ok(AttachmentInfo::File( + BaseFileInfo::try_from(&info.file_info) + .map_err(|_| RoomError::InvalidAttachmentData)?, + )), + GalleryItemInfo::Image(info) => Ok(AttachmentInfo::Image( + BaseImageInfo::try_from(&info.image_info) + .map_err(|_| RoomError::InvalidAttachmentData)?, + )), + GalleryItemInfo::Video(info) => Ok(AttachmentInfo::Video( + BaseVideoInfo::try_from(&info.video_info) + .map_err(|_| RoomError::InvalidAttachmentData)?, + )), + } + } + + fn thumbnail(&self) -> Result, RoomError> { + match self { + GalleryItemInfo::Audio(..) => Ok(None), + GalleryItemInfo::File(..) => Ok(None), + GalleryItemInfo::Image(info) => build_thumbnail_info( + info.thumbnail_path.clone(), + info.image_info.thumbnail_info.clone(), + ), + GalleryItemInfo::Video(info) => build_thumbnail_info( + info.thumbnail_path.clone(), + info.video_info.thumbnail_info.clone(), + ), + } + } +} + +impl TryInto for GalleryItemInfo { + type Error = RoomError; + + fn try_into( + self, + ) -> std::result::Result { + let mime_str = self.mimetype().as_ref().ok_or(RoomError::InvalidAttachmentMimeType)?; + let mime_type = + mime_str.parse::().map_err(|_| RoomError::InvalidAttachmentMimeType)?; + Ok(matrix_sdk_ui::timeline::GalleryItemInfo { + source: self.filename().into(), + attachment_info: self.attachment_info()?, + caption: self.caption().clone(), + formatted_caption: self + .formatted_caption() + .clone() + .map(ruma::events::room::message::FormattedBody::from), + mime_type, + thumbnail: self.thumbnail()?, + }) + } +} + +#[derive(uniffi::Record)] +pub struct AudioGalleryItemInfo { + /// Audio metadata + pub audio_info: AudioInfo, + /// Filename + pub filename: String, + /// Caption + pub caption: Option, + /// Rich caption + pub formatted_caption: Option, +} + +#[derive(uniffi::Record)] +pub struct FileGalleryItemInfo { + /// File metadata + pub file_info: FileInfo, + /// Filename + pub filename: String, + /// Caption + pub caption: Option, + /// Rich caption + pub formatted_caption: Option, +} + +#[derive(uniffi::Record)] +pub struct ImageGalleryItemInfo { + /// Image metadata + pub image_info: ImageInfo, + /// Filename + pub filename: String, + /// Caption + pub caption: Option, + /// Rich caption + pub formatted_caption: Option, + /// Path to the thumbnail + pub thumbnail_path: Option, +} + +#[derive(uniffi::Record)] +pub struct VideoGalleryItemInfo { + /// Video metadata + pub video_info: VideoInfo, + /// Filename + pub filename: String, + /// Caption + pub caption: Option, + /// Rich caption + pub formatted_caption: Option, + /// Path to the thumbnail + pub thumbnail_path: Option, +} + #[matrix_sdk_ffi_macros::export] impl Timeline { pub async fn add_listener(&self, listener: Box) -> Arc { @@ -404,6 +571,46 @@ impl Timeline { self.send_attachment(params, attachment_info, file_info.mimetype, progress_watcher, None) } + pub fn send_gallery( + self: Arc, + params: GalleryUploadParameters, + item_infos: Vec, + progress_watcher: Option>, + ) -> Result, RoomError> { + let mapped_infos = item_infos + .into_iter() + .map(TryInto::try_into) + .collect::, _>>()?; + + let formatted_caption = formatted_body_from( + params.caption.as_deref(), + params.formatted_caption.map(Into::into), + ); + + let gallery_config = GalleryConfig::new() + .caption(params.caption) + .formatted_caption(formatted_caption) + .mentions(params.mentions.map(Into::into)); + + let handle = SendGalleryJoinHandle::new(get_runtime_handle().spawn(async move { + let request = self.inner.send_gallery(mapped_infos, gallery_config).use_send_queue(); + + if let Some(progress_watcher) = progress_watcher { + let mut subscriber = request.subscribe_to_send_progress(); + get_runtime_handle().spawn(async move { + while let Some(progress) = subscriber.next().await { + progress_watcher.transmission_progress(progress.into()); + } + }); + } + + request.await.map_err(|_| RoomError::FailedSendingAttachment)?; + Ok(()) + })); + + Ok(handle) + } + pub async fn create_poll( self: Arc, question: String, @@ -1251,6 +1458,49 @@ impl SendAttachmentJoinHandle { } } +#[derive(uniffi::Object)] +pub struct SendGalleryJoinHandle { + join_hdl: Arc>>>, + abort_hdl: AbortHandle, +} + +impl SendGalleryJoinHandle { + fn new(join_hdl: JoinHandle>) -> Arc { + let abort_hdl = join_hdl.abort_handle(); + let join_hdl = Arc::new(Mutex::new(join_hdl)); + Arc::new(Self { join_hdl, abort_hdl }) + } +} + +#[matrix_sdk_ffi_macros::export] +impl SendGalleryJoinHandle { + /// Wait until the gallery has been sent. + /// + /// If the sending had been cancelled, will return immediately. + pub async fn join(&self) -> Result<(), RoomError> { + let handle = self.join_hdl.clone(); + let mut locked_handle = handle.lock().await; + let join_result = (&mut *locked_handle).await; + match join_result { + Ok(res) => res, + Err(err) => { + if err.is_cancelled() { + return Ok(()); + } + error!("task panicked! resuming panic from here."); + panic::resume_unwind(err.into_panic()); + } + } + } + + /// Cancel the current sending task. + /// + /// A subsequent call to [`Self::join`] will return immediately. + pub fn cancel(&self) { + self.abort_hdl.abort(); + } +} + /// A [`TimelineItem`](super::TimelineItem) that doesn't correspond to an event. #[derive(uniffi::Enum)] pub enum VirtualTimelineItem { diff --git a/crates/matrix-sdk-base/src/store/mod.rs b/crates/matrix-sdk-base/src/store/mod.rs index 57c04a35900..6b45822f74e 100644 --- a/crates/matrix-sdk-base/src/store/mod.rs +++ b/crates/matrix-sdk-base/src/store/mod.rs @@ -75,8 +75,9 @@ pub use self::{ memory_store::MemoryStore, send_queue::{ ChildTransactionId, DependentQueuedRequest, DependentQueuedRequestKind, - FinishUploadThumbnailInfo, QueueWedgeError, QueuedRequest, QueuedRequestKind, - SentMediaInfo, SentRequestKey, SerializableEventContent, + FinishGalleryItemInfo, FinishUploadThumbnailInfo, QueueWedgeError, QueuedRequest, + QueuedRequestKind, SentGalleryMediaInfo, SentMediaInfo, SentRequestKey, + SerializableEventContent, }, traits::{ ComposerDraft, ComposerDraftType, DynStateStore, IntoStateStore, ServerCapabilities, diff --git a/crates/matrix-sdk-base/src/store/send_queue.rs b/crates/matrix-sdk-base/src/store/send_queue.rs index 8c87b6c5ebb..4ebff68ec4d 100644 --- a/crates/matrix-sdk-base/src/store/send_queue.rs +++ b/crates/matrix-sdk-base/src/store/send_queue.rs @@ -104,6 +104,31 @@ pub enum QueuedRequestKind { /// To which media event transaction does this upload relate? related_to: OwnedTransactionId, }, + + /// Content to upload on the media server. + /// + /// The bytes must be stored in the media cache, and are identified by the + /// cache key. + GalleryMediaUpload { + /// Content type of the media to be uploaded. + /// + /// Stored as a `String` because `Mime` which we'd really want to use + /// here, is not serializable. Oh well. + content_type: String, + + /// The cache key used to retrieve the media's bytes in the event cache + /// store. + cache_key: MediaRequestParameters, + + /// An optional media source for a thumbnail already uploaded. + thumbnail_source: Option, + + /// To which media event transaction does this upload relate? + related_to: OwnedTransactionId, + + /// Media sources for previously uploaded gallery items. + sent_media_infos: Vec, + }, } impl From for QueuedRequestKind { @@ -244,6 +269,30 @@ pub enum DependentQueuedRequestKind { /// Information about the thumbnail, if present. thumbnail_info: Option, }, + + /// Upload a gallery file or thumbnail. + UploadGalleryFileOrThumbnail { + /// Content type for the file or thumbnail. + content_type: String, + + /// Media request necessary to retrieve the file or thumbnail. + cache_key: MediaRequestParameters, + + /// To which gallery transaction id does this upload relate to? + related_to: OwnedTransactionId, + + /// Whether the request depends on a thumbnail or a file upload. + depends_on_thumbnail: bool, + }, + + /// Finish a gallery upload. + FinishGallery { + /// Local echo for the event (containing the local MXC URIs). + local_echo: RoomMessageEventContent, + + /// Metadata about the gallery items + item_infos: Vec, + }, } /// Detailed record about a thumbnail used when finishing a media upload. @@ -263,6 +312,15 @@ pub struct FinishUploadThumbnailInfo { pub height: Option, } +#[derive(Clone, Debug, Serialize, Deserialize)] +/// Metadata for a gallery item on FinishGallery. +pub struct FinishGalleryItemInfo { + /// Transaction id for the file upload. + pub file_upload: OwnedTransactionId, + /// Information about the thumbnail, if present. + pub thumbnail_info: Option, +} + /// A transaction id identifying a [`DependentQueuedRequest`] rather than its /// parent [`QueuedRequest`]. /// @@ -324,6 +382,26 @@ pub struct SentMediaInfo { pub thumbnail: Option, } +/// Information about a media (and its thumbnail) that have been sent to an +/// homeserver and are part of a media gallery. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SentGalleryMediaInfo { + /// File that was uploaded by this request. + /// + /// If the request related to a thumbnail upload, this contains the + /// thumbnail media source. + pub file: MediaSource, + + /// Optional thumbnail previously uploaded, when uploading a file. + /// + /// When uploading a thumbnail, this is set to `None`. + pub thumbnail: Option, + + /// Accumulated list of infos for previously uploaded media + /// in the same gallery. + pub accumulated: Vec, +} + /// A unique key (identifier) indicating that a transaction has been /// successfully sent to the server. /// @@ -335,6 +413,10 @@ pub enum SentRequestKey { /// The parent transaction returned an uploaded resource URL. Media(SentMediaInfo), + + /// The parent transaction returned an uploaded resource URL + /// and accumulated previously uploaded resource URLs. + GalleryMedia(SentGalleryMediaInfo), } impl SentRequestKey { @@ -348,6 +430,12 @@ impl SentRequestKey { pub fn into_media(self) -> Option { as_variant!(self, Self::Media) } + + /// Converts the current parent key into information about a sent gallery, + /// if possible. + pub fn into_gallery_media(self) -> Option { + as_variant!(self, Self::GalleryMedia) + } } /// A request to be sent, depending on a [`QueuedRequest`] to be sent first. @@ -388,7 +476,8 @@ impl DependentQueuedRequest { DependentQueuedRequestKind::EditEvent { .. } | DependentQueuedRequestKind::RedactEvent | DependentQueuedRequestKind::ReactEvent { .. } - | DependentQueuedRequestKind::UploadFileWithThumbnail { .. } => { + | DependentQueuedRequestKind::UploadFileWithThumbnail { .. } + | DependentQueuedRequestKind::UploadGalleryFileOrThumbnail { .. } => { // These are all aggregated events, or non-visible items (file upload producing // a new MXC ID). false @@ -397,6 +486,10 @@ impl DependentQueuedRequest { // This one graduates into a new media event. true } + DependentQueuedRequestKind::FinishGallery { .. } => { + // This one graduates into a new gallery event. + true + } } } } diff --git a/crates/matrix-sdk-ui/src/timeline/controller/mod.rs b/crates/matrix-sdk-ui/src/timeline/controller/mod.rs index ee0a4ea9d20..3e422a1a9fc 100644 --- a/crates/matrix-sdk-ui/src/timeline/controller/mod.rs +++ b/crates/matrix-sdk-ui/src/timeline/controller/mod.rs @@ -229,6 +229,7 @@ pub fn default_event_filter(event: &AnySyncTimelineEvent, room_version: &RoomVer MessageType::Audio(_) | MessageType::Emote(_) | MessageType::File(_) + | MessageType::Gallery(_) | MessageType::Image(_) | MessageType::Location(_) | MessageType::Notice(_) diff --git a/crates/matrix-sdk-ui/src/timeline/event_item/mod.rs b/crates/matrix-sdk-ui/src/timeline/event_item/mod.rs index 5c5295654b8..048d1dd5f7f 100644 --- a/crates/matrix-sdk-ui/src/timeline/event_item/mod.rs +++ b/crates/matrix-sdk-ui/src/timeline/event_item/mod.rs @@ -363,6 +363,7 @@ impl EventTimelineItem { | MessageType::Emote(_) | MessageType::Audio(_) | MessageType::File(_) + | MessageType::Gallery(_) | MessageType::Image(_) | MessageType::Video(_) ) @@ -612,6 +613,7 @@ impl EventTimelineItem { MessageType::Text(text) => Some(text.body.as_str()), MessageType::Audio(audio) => audio.caption(), MessageType::File(file) => file.caption(), + MessageType::Gallery(gallery) => Some(gallery.body.as_str()), MessageType::Image(image) => image.caption(), MessageType::Video(video) => video.caption(), _ => None, diff --git a/crates/matrix-sdk-ui/src/timeline/futures.rs b/crates/matrix-sdk-ui/src/timeline/futures.rs index bba9799a60a..3fd239769e5 100644 --- a/crates/matrix-sdk-ui/src/timeline/futures.rs +++ b/crates/matrix-sdk-ui/src/timeline/futures.rs @@ -1,12 +1,15 @@ use std::future::IntoFuture; use eyeball::SharedObservable; -use matrix_sdk::{attachment::AttachmentConfig, TransmissionProgress}; +use matrix_sdk::{ + attachment::{AttachmentConfig, GalleryConfig}, + TransmissionProgress, +}; use matrix_sdk_base::boxed_into_future; use mime::Mime; use tracing::{Instrument as _, Span}; -use super::{AttachmentSource, Error, Timeline}; +use super::{AttachmentSource, Error, GalleryItemInfo, Timeline}; pub struct SendAttachment<'a> { timeline: &'a Timeline, @@ -92,3 +95,75 @@ impl<'a> IntoFuture for SendAttachment<'a> { Box::pin(fut.instrument(tracing_span)) } } + +pub struct SendGallery<'a> { + timeline: &'a Timeline, + item_infos: Vec, + config: GalleryConfig, + tracing_span: Span, + pub(crate) send_progress: SharedObservable, + use_send_queue: bool, +} + +impl<'a> SendGallery<'a> { + pub(crate) fn new( + timeline: &'a Timeline, + item_infos: Vec, + config: GalleryConfig, + ) -> Self { + Self { + timeline, + item_infos, + config, + tracing_span: Span::current(), + send_progress: Default::default(), + use_send_queue: false, + } + } + + /// (Experimental) Uses the send queue to upload this media. + /// + /// This uses the send queue to upload the medias, and as such it provides + /// local echoes for the uploaded media too, not blocking the sending + /// request. + /// + /// This will be the default in future versions, when the feature work will + /// be done there. + pub fn use_send_queue(self) -> Self { + Self { use_send_queue: true, ..self } + } + + /// Get a subscriber to observe the progress of sending the request body. + #[cfg(not(target_arch = "wasm32"))] + pub fn subscribe_to_send_progress(&self) -> eyeball::Subscriber { + self.send_progress.subscribe() + } +} + +impl<'a> IntoFuture for SendGallery<'a> { + type Output = Result<(), Error>; + boxed_into_future!(extra_bounds: 'a); + + fn into_future(self) -> Self::IntoFuture { + let Self { timeline, item_infos, config, tracing_span, use_send_queue, .. } = self; + + let fut = async move { + let mapped_infos = item_infos + .into_iter() + .map(TryInto::try_into) + .collect::, Error>>()?; + + if use_send_queue { + let send_queue = timeline.room().send_queue(); + let fut = send_queue.send_gallery(mapped_infos, config); + fut.await.map_err(|_| Error::FailedSendingAttachment)?; + } else { + todo!("implement queue-less sending for gallery events"); + } + + Ok(()) + }; + + Box::pin(fut.instrument(tracing_span)) + } +} diff --git a/crates/matrix-sdk-ui/src/timeline/mod.rs b/crates/matrix-sdk-ui/src/timeline/mod.rs index 03c698ce4ba..59ffc96e579 100644 --- a/crates/matrix-sdk-ui/src/timeline/mod.rs +++ b/crates/matrix-sdk-ui/src/timeline/mod.rs @@ -21,10 +21,11 @@ use std::{fs, path::PathBuf, sync::Arc}; use algorithms::rfind_event_by_item_id; use event_item::{extract_room_msg_edit_content, TimelineItemHandle}; use eyeball_im::VectorDiff; +use futures::SendGallery; use futures_core::Stream; use imbl::Vector; use matrix_sdk::{ - attachment::AttachmentConfig, + attachment::{AttachmentConfig, AttachmentInfo, GalleryConfig, Thumbnail}, event_cache::{EventCacheDropHandles, RoomEventCache}, event_handler::EventHandlerHandle, executor::JoinHandle, @@ -43,8 +44,8 @@ use ruma::{ room::{ encrypted::Relation as EncryptedRelation, message::{ - AddMentions, ForwardThread, OriginalRoomMessageEvent, Relation, ReplyWithinThread, - RoomMessageEventContentWithoutRelation, + AddMentions, FormattedBody, ForwardThread, OriginalRoomMessageEvent, Relation, + ReplyWithinThread, RoomMessageEventContentWithoutRelation, }, pinned_events::RoomPinnedEventsEventContent, }, @@ -628,6 +629,16 @@ impl Timeline { SendAttachment::new(self, source.into(), mime_type, config) } + /// Sends a gallery to the room. + #[instrument(skip_all)] + pub fn send_gallery( + &self, + item_infos: Vec, + config: GalleryConfig, + ) -> SendGallery<'_> { + SendGallery::new(self, item_infos, config) + } + /// Redact an event given its [`TimelineEventItemId`] and an optional /// reason. pub async fn redact( @@ -979,3 +990,36 @@ where Self::File(value.into()) } } + +#[derive(Debug)] +pub struct GalleryItemInfo { + /// The item's attachment source + pub source: AttachmentSource, + /// The item's attachment info + pub attachment_info: AttachmentInfo, + /// The caption + pub caption: Option, + /// The formatted caption + pub formatted_caption: Option, + /// The item's thumbnail + pub thumbnail: Option, + /// The item's mimetype + pub mime_type: Mime, +} + +impl TryInto for GalleryItemInfo { + type Error = Error; + + fn try_into(self) -> std::result::Result { + let (data, filename) = self.source.try_into_bytes_and_filename()?; + Ok(matrix_sdk::attachment::GalleryItemInfo { + content_type: self.mime_type, + data, + filename, + attachment_info: self.attachment_info, + caption: self.caption, + formatted_caption: self.formatted_caption, + thumbnail: self.thumbnail, + }) + } +} diff --git a/crates/matrix-sdk/src/attachment.rs b/crates/matrix-sdk/src/attachment.rs index c39f00e30a7..67292595e5f 100644 --- a/crates/matrix-sdk/src/attachment.rs +++ b/crates/matrix-sdk/src/attachment.rs @@ -263,3 +263,81 @@ impl AttachmentConfig { self } } + +/// Configuration for sending a gallery. +#[derive(Debug, Default)] +pub struct GalleryConfig { + pub(crate) txn_id: Option, + pub(crate) caption: Option, + pub(crate) formatted_caption: Option, + pub(crate) mentions: Option, +} + +impl GalleryConfig { + /// Create a new empty `GalleryConfig`. + pub fn new() -> Self { + Self::default() + } + + /// Set the transaction ID to send. + /// + /// # Arguments + /// + /// * `txn_id` - A unique ID that can be attached to a `MessageEvent` held + /// in its unsigned field as `transaction_id`. If not given, one is + /// created for the message. + #[must_use] + pub fn txn_id(mut self, txn_id: &TransactionId) -> Self { + self.txn_id = Some(txn_id.to_owned()); + self + } + + /// Set the optional caption + /// + /// # Arguments + /// + /// * `caption` - The optional caption + pub fn caption(mut self, caption: Option) -> Self { + self.caption = caption; + self + } + + /// Set the optional formatted caption + /// + /// # Arguments + /// + /// * `formatted_caption` - The optional formatted caption + pub fn formatted_caption(mut self, formatted_caption: Option) -> Self { + self.formatted_caption = formatted_caption; + self + } + + /// Set the mentions of the message. + /// + /// # Arguments + /// + /// * `mentions` - The mentions of the message + pub fn mentions(mut self, mentions: Option) -> Self { + self.mentions = mentions; + self + } +} + +#[derive(Debug)] +/// Metadata for a gallery item +pub struct GalleryItemInfo { + /// The filename + pub filename: String, + /// The mime type + pub content_type: mime::Mime, + /// The binary data + pub data: Vec, + /// The attachment info + pub attachment_info: AttachmentInfo, + /// The caption + pub caption: Option, + /// The formatted caption + pub formatted_caption: Option, + /// The thumbnail + pub thumbnail: Option, +} diff --git a/crates/matrix-sdk/src/room/edit.rs b/crates/matrix-sdk/src/room/edit.rs index e5095431293..d1a678a0bc4 100644 --- a/crates/matrix-sdk/src/room/edit.rs +++ b/crates/matrix-sdk/src/room/edit.rs @@ -298,6 +298,11 @@ pub(crate) fn update_media_caption( event.formatted = formatted_caption; true } + MessageType::Gallery(event) => { + event.body = caption.unwrap_or_default(); + event.formatted = formatted_caption; + true + } MessageType::Image(event) => { set_caption!(event, caption); event.formatted = formatted_caption; diff --git a/crates/matrix-sdk/src/room/mod.rs b/crates/matrix-sdk/src/room/mod.rs index 73fb55a50e8..86e002dc338 100644 --- a/crates/matrix-sdk/src/room/mod.rs +++ b/crates/matrix-sdk/src/room/mod.rs @@ -97,7 +97,8 @@ use ruma::{ member::{MembershipChange, SyncRoomMemberEvent}, message::{ AudioInfo, AudioMessageEventContent, FileInfo, FileMessageEventContent, - FormattedBody, ImageMessageEventContent, MessageType, RoomMessageEventContent, + FormattedBody, GalleryItemType, GalleryMessageEventContent, + ImageMessageEventContent, MessageType, RoomMessageEventContent, UnstableAudioDetailsContentBlock, UnstableVoiceContentBlock, VideoInfo, VideoMessageEventContent, }, @@ -2225,6 +2226,118 @@ impl Room { content } + /// Creates the inner [`GalleryItemType`] for an already-uploaded media file + /// provided by its source. + #[allow(clippy::too_many_arguments)] + pub(crate) fn make_gallery_item_type( + &self, + content_type: &Mime, + filename: String, + source: MediaSource, + caption: Option, + formatted_caption: Option, + info: Option, + thumbnail: Option<(MediaSource, Box)>, + ) -> GalleryItemType { + // If caption is set, use it as body, and filename as the file name; otherwise, + // body is the filename, and the filename is not set. + // https://github.com/matrix-org/matrix-spec-proposals/blob/main/proposals/2530-body-as-caption.md + let (body, filename) = match caption { + Some(caption) => (caption, Some(filename)), + None => (filename, None), + }; + + let (thumbnail_source, thumbnail_info) = thumbnail.unzip(); + + match content_type.type_() { + mime::IMAGE => { + let info = assign!(info.map(ImageInfo::from).unwrap_or_default(), { + mimetype: Some(content_type.as_ref().to_owned()), + thumbnail_source, + thumbnail_info + }); + let content = assign!(ImageMessageEventContent::new(body, source), { + info: Some(Box::new(info)), + formatted: formatted_caption, + filename + }); + GalleryItemType::Image(content) + } + + mime::AUDIO => { + let mut content = assign!(AudioMessageEventContent::new(body, source), { + formatted: formatted_caption, + filename + }); + + if let Some(AttachmentInfo::Voice { audio_info, waveform: Some(waveform_vec) }) = + &info + { + if let Some(duration) = audio_info.duration { + let waveform = waveform_vec.iter().map(|v| (*v).into()).collect(); + content.audio = + Some(UnstableAudioDetailsContentBlock::new(duration, waveform)); + } + content.voice = Some(UnstableVoiceContentBlock::new()); + } + + let mut audio_info = info.map(AudioInfo::from).unwrap_or_default(); + audio_info.mimetype = Some(content_type.as_ref().to_owned()); + let content = content.info(Box::new(audio_info)); + + GalleryItemType::Audio(content) + } + + mime::VIDEO => { + let info = assign!(info.map(VideoInfo::from).unwrap_or_default(), { + mimetype: Some(content_type.as_ref().to_owned()), + thumbnail_source, + thumbnail_info + }); + let content = assign!(VideoMessageEventContent::new(body, source), { + info: Some(Box::new(info)), + formatted: formatted_caption, + filename + }); + GalleryItemType::Video(content) + } + + _ => { + let info = assign!(info.map(FileInfo::from).unwrap_or_default(), { + mimetype: Some(content_type.as_ref().to_owned()), + thumbnail_source, + thumbnail_info + }); + let content = assign!(FileMessageEventContent::new(body, source), { + info: Some(Box::new(info)), + formatted: formatted_caption, + filename, + }); + GalleryItemType::File(content) + } + } + } + + /// Creates the [`RoomMessageEventContent`] based on the message type and + /// mentions. + pub(crate) fn make_gallery_event( + caption: Option, + formatted_caption: Option, + itemtypes: Vec, + mentions: Option, + ) -> RoomMessageEventContent { + let msg_type = MessageType::Gallery(GalleryMessageEventContent::new( + caption.unwrap_or_default(), + formatted_caption, + itemtypes, + )); + let mut content = RoomMessageEventContent::new(msg_type); + if let Some(mentions) = mentions { + content = content.add_mentions(mentions); + } + content + } + /// Update the power levels of a select set of users of this room. /// /// Issue a `power_levels` state event request to the server, changing the diff --git a/crates/matrix-sdk/src/send_queue/mod.rs b/crates/matrix-sdk/src/send_queue/mod.rs index 1caa9190ed9..8e1bb58c64c 100644 --- a/crates/matrix-sdk/src/send_queue/mod.rs +++ b/crates/matrix-sdk/src/send_queue/mod.rs @@ -137,14 +137,14 @@ use std::{ }, }; -use as_variant::as_variant; use matrix_sdk_base::{ event_cache::store::EventCacheStoreError, media::MediaRequestParameters, store::{ ChildTransactionId, DependentQueuedRequest, DependentQueuedRequestKind, - FinishUploadThumbnailInfo, QueueWedgeError, QueuedRequest, QueuedRequestKind, - SentMediaInfo, SentRequestKey, SerializableEventContent, + FinishGalleryItemInfo, FinishUploadThumbnailInfo, QueueWedgeError, QueuedRequest, + QueuedRequestKind, SentGalleryMediaInfo, SentMediaInfo, SentRequestKey, + SerializableEventContent, }, store_locks::LockStoreError, RoomState, StoreError, @@ -453,7 +453,7 @@ impl RoomSendQueue { let send_handle = SendHandle { room: self.clone(), transaction_id: transaction_id.clone(), - media_handles: None, + media_handles: vec![], created_at, }; @@ -566,7 +566,13 @@ impl RoomSendQueue { let txn_id = queued_request.transaction_id.clone(); trace!(txn_id = %txn_id, "received a request to send!"); - let related_txn_id = as_variant!(&queued_request.kind, QueuedRequestKind::MediaUpload { related_to, .. } => related_to.clone()); + let related_txn_id = match &queued_request.kind { + QueuedRequestKind::MediaUpload { related_to, .. } => Some(related_to.clone()), + QueuedRequestKind::GalleryMediaUpload { related_to, .. } => { + Some(related_to.clone()) + } + _ => None, + }; let Some(room) = room.get() else { if is_dropping.load(Ordering::SeqCst) { @@ -593,6 +599,13 @@ impl RoomSendQueue { file: media_info.file, }); } + + SentRequestKey::GalleryMedia(gallery_info) => { + let _ = updates.send(RoomSendQueueUpdate::UploadedMedia { + related_to: related_txn_id.as_ref().unwrap_or(&txn_id).clone(), + file: gallery_info.file, + }); + } }, Err(err) => { @@ -778,6 +791,95 @@ impl RoomSendQueue { } } } + + QueuedRequestKind::GalleryMediaUpload { + content_type, + cache_key, + thumbnail_source, + related_to: relates_to, + sent_media_infos, + } => { + trace!(%relates_to, "uploading media related to gallery"); + + let fut = async move { + let mime = Mime::from_str(&content_type).map_err(|_| { + crate::Error::SendQueueWedgeError(QueueWedgeError::InvalidMimeType { + mime_type: content_type.clone(), + }) + })?; + + let data = room + .client() + .event_cache_store() + .lock() + .await? + .get_media_content(&cache_key) + .await? + .ok_or(crate::Error::SendQueueWedgeError( + QueueWedgeError::MissingMediaContent, + ))?; + + #[cfg(feature = "e2e-encryption")] + let media_source = if room.latest_encryption_state().await?.is_encrypted() { + trace!("upload will be encrypted (encrypted room)"); + let mut cursor = std::io::Cursor::new(data); + let encrypted_file = room + .client() + .upload_encrypted_file(&mime, &mut cursor) + .with_request_config(RequestConfig::short_retry()) + .await?; + MediaSource::Encrypted(Box::new(encrypted_file)) + } else { + trace!("upload will be in clear text (room without encryption)"); + let request_config = RequestConfig::short_retry() + .timeout(Media::reasonable_upload_timeout(&data)); + let res = + room.client().media().upload(&mime, data, Some(request_config)).await?; + MediaSource::Plain(res.content_uri) + }; + + #[cfg(not(feature = "e2e-encryption"))] + let media_source = { + let request_config = RequestConfig::short_retry() + .timeout(Media::reasonable_upload_timeout(&data)); + let res = + room.client().media().upload(&mime, data, Some(request_config)).await?; + MediaSource::Plain(res.content_uri) + }; + + let uri = match &media_source { + MediaSource::Plain(uri) => uri, + MediaSource::Encrypted(encrypted_file) => &encrypted_file.url, + }; + trace!(%relates_to, mxc_uri = %uri, "media successfully uploaded"); + + Ok(SentRequestKey::GalleryMedia(SentGalleryMediaInfo { + file: media_source, + thumbnail: thumbnail_source, + accumulated: sent_media_infos, + })) + }; + + let wait_for_cancel = async move { + if let Some(rx) = cancel_upload_rx { + rx.await + } else { + std::future::pending().await + } + }; + + tokio::select! { + biased; + + _ = wait_for_cancel => { + Ok(None) + } + + res = fut => { + res.map(Some) + } + } + } } } @@ -985,13 +1087,16 @@ impl QueueStorage { guard.client()?.store().load_send_queue_requests(&self.room_id).await?; if let Some(request) = queued_requests.iter().find(|queued| !queued.is_wedged()) { - let (cancel_upload_tx, cancel_upload_rx) = - if matches!(request.kind, QueuedRequestKind::MediaUpload { .. }) { - let (tx, rx) = oneshot::channel(); - (Some(tx), Some(rx)) - } else { - Default::default() - }; + let (cancel_upload_tx, cancel_upload_rx) = if matches!( + request.kind, + QueuedRequestKind::MediaUpload { .. } + | QueuedRequestKind::GalleryMediaUpload { .. } + ) { + let (tx, rx) = oneshot::channel(); + (Some(tx), Some(rx)) + } else { + Default::default() + }; let prev = guard.being_sent.replace(BeingSentInfo { transaction_id: request.transaction_id.clone(), @@ -1270,6 +1375,198 @@ impl QueueStorage { Ok(()) } + /// Push requests (and dependents) to upload a gallery. + /// + /// See the module-level description for details of the whole processus. + #[allow(clippy::too_many_arguments)] + async fn push_gallery( + &self, + event: RoomMessageEventContent, + send_event_txn: OwnedTransactionId, + created_at: MilliSecondsSinceUnixEpoch, + item_queue_infos: Vec, + ) -> Result<(), RoomSendQueueStorageError> { + let guard = self.store.lock().await; + let client = guard.client()?; + let store = client.store(); + + let mut last_upload_file_txn: Option = None; + + let mut finish_item_infos = Vec::::new(); + + if let Some((first, rest)) = item_queue_infos.split_first() { + let GalleryItemQueueInfo { + content_type, + upload_file_txn, + file_media_request, + thumbnail, + } = first; + + let thumbnail_info = + if let Some((thumbnail_info, thumbnail_media_request, thumbnail_content_type)) = + thumbnail + { + let upload_thumbnail_txn = thumbnail_info.txn.clone(); + + // Save the thumbnail upload request. + store + .save_send_queue_request( + &self.room_id, + upload_thumbnail_txn.clone(), + created_at, + QueuedRequestKind::GalleryMediaUpload { + content_type: thumbnail_content_type.to_string(), + cache_key: thumbnail_media_request.clone(), + thumbnail_source: None, // the thumbnail has no thumbnails :) + related_to: send_event_txn.clone(), + sent_media_infos: vec![], + }, + Self::LOW_PRIORITY, + ) + .await?; + + // Save the file upload request as a dependent request of the thumbnail upload. + store + .save_dependent_queued_request( + &self.room_id, + &upload_thumbnail_txn, + upload_file_txn.clone().into(), + created_at, + DependentQueuedRequestKind::UploadGalleryFileOrThumbnail { + content_type: content_type.to_string(), + cache_key: file_media_request.clone(), + related_to: send_event_txn.clone(), + depends_on_thumbnail: true, + }, + ) + .await?; + + Some(thumbnail_info) + } else { + // Save the file upload as its own request, not a dependent one. + store + .save_send_queue_request( + &self.room_id, + upload_file_txn.clone(), + created_at, + QueuedRequestKind::GalleryMediaUpload { + content_type: content_type.to_string(), + cache_key: file_media_request.clone(), + thumbnail_source: None, + related_to: send_event_txn.clone(), + sent_media_infos: vec![], + }, + Self::LOW_PRIORITY, + ) + .await?; + + None + }; + + finish_item_infos.push(FinishGalleryItemInfo { + file_upload: upload_file_txn.clone(), + thumbnail_info: thumbnail_info.cloned(), + }); + + last_upload_file_txn = Some(upload_file_txn.clone()); + + for item_queue_info in rest { + let GalleryItemQueueInfo { + content_type, + upload_file_txn, + file_media_request, + thumbnail, + } = item_queue_info; + + let thumbnail_info = if let Some(( + thumbnail_info, + thumbnail_media_request, + thumbnail_content_type, + )) = thumbnail + { + let upload_thumbnail_txn = thumbnail_info.txn.clone(); + + // Save the thumbnail upload request as a dependent request of the last file + // upload. + store + .save_dependent_queued_request( + &self.room_id, + &last_upload_file_txn.unwrap(), + upload_thumbnail_txn.clone().into(), + created_at, + DependentQueuedRequestKind::UploadGalleryFileOrThumbnail { + content_type: thumbnail_content_type.to_string(), + cache_key: thumbnail_media_request.clone(), + related_to: send_event_txn.clone(), + depends_on_thumbnail: false, + }, + ) + .await?; + + // Save the file upload request as a dependent request of the thumbnail upload. + store + .save_dependent_queued_request( + &self.room_id, + &upload_thumbnail_txn, + upload_file_txn.clone().into(), + created_at, + DependentQueuedRequestKind::UploadGalleryFileOrThumbnail { + content_type: content_type.to_string(), + cache_key: file_media_request.clone(), + related_to: send_event_txn.clone(), + depends_on_thumbnail: true, + }, + ) + .await?; + + Some(thumbnail_info) + } else { + // Save the file upload as a dependent request of the last file upload. + store + .save_dependent_queued_request( + &self.room_id, + &last_upload_file_txn.unwrap(), + upload_file_txn.clone().into(), + created_at, + DependentQueuedRequestKind::UploadGalleryFileOrThumbnail { + content_type: content_type.to_string(), + cache_key: file_media_request.clone(), + related_to: send_event_txn.clone(), + depends_on_thumbnail: false, + }, + ) + .await?; + + None + }; + + finish_item_infos.push(FinishGalleryItemInfo { + file_upload: upload_file_txn.clone(), + thumbnail_info: thumbnail_info.cloned(), + }); + + last_upload_file_txn = Some(upload_file_txn.clone()); + } + } + + // Push the request for the event itself as a dependent request of the last file + // upload. + store + .save_dependent_queued_request( + &self.room_id, + &last_upload_file_txn.unwrap(), + send_event_txn.into(), + created_at, + DependentQueuedRequestKind::FinishGallery { + local_echo: event, + item_infos: finish_item_infos, + }, + ) + .await?; + + Ok(()) + } + /// Reacts to the given local echo of an event. #[instrument(skip(self))] async fn react( @@ -1334,13 +1631,14 @@ impl QueueStorage { send_handle: SendHandle { room: room.clone(), transaction_id: queued.transaction_id, - media_handles: None, + media_handles: vec![], created_at: queued.created_at, }, send_error: queued.error, }, - QueuedRequestKind::MediaUpload { .. } => { + QueuedRequestKind::MediaUpload { .. } + | QueuedRequestKind::GalleryMediaUpload { .. } => { // Don't return uploaded medias as their own things; the accompanying // event represented as a dependent request should be sufficient. return None; @@ -1372,7 +1670,8 @@ impl QueueStorage { }, }), - DependentQueuedRequestKind::UploadFileWithThumbnail { .. } => { + DependentQueuedRequestKind::UploadFileWithThumbnail { .. } + | DependentQueuedRequestKind::UploadGalleryFileOrThumbnail { .. } => { // Don't reflect these: only the associated event is interesting to observers. None } @@ -1391,21 +1690,62 @@ impl QueueStorage { send_handle: SendHandle { room: room.clone(), transaction_id: dep.own_transaction_id.into(), - media_handles: Some(MediaHandles { + media_handles: vec![MediaHandles { upload_thumbnail_txn: thumbnail_info.map(|info| info.txn), upload_file_txn: file_upload, - }), + }], created_at: dep.created_at, }, send_error: None, }, }) } + + DependentQueuedRequestKind::FinishGallery { local_echo, item_infos } => { + // Materialize as an event local echo. + self.create_gallery_local_echo( + dep.own_transaction_id, + room, + dep.created_at, + local_echo, + item_infos, + ) + } }); Ok(local_requests.chain(reactions_and_medias).collect()) } + /// Create a local echo for a gallery message. + fn create_gallery_local_echo( + &self, + transaction_id: ChildTransactionId, + room: &RoomSendQueue, + created_at: MilliSecondsSinceUnixEpoch, + local_echo: RoomMessageEventContent, + item_infos: Vec, + ) -> Option { + Some(LocalEcho { + transaction_id: transaction_id.clone().into(), + content: LocalEchoContent::Event { + serialized_event: SerializableEventContent::new(&local_echo.into()).ok()?, + send_handle: SendHandle { + room: room.clone(), + transaction_id: transaction_id.into(), + media_handles: item_infos + .into_iter() + .map(|i| MediaHandles { + upload_thumbnail_txn: i.thumbnail_info.map(|info| info.txn), + upload_file_txn: i.file_upload, + }) + .collect(), + created_at, + }, + send_error: None, + }, + }) + } + /// Try to apply a single dependent request, whether it's local or remote. /// /// This swallows errors that would retrigger every time if we retried @@ -1625,6 +1965,44 @@ impl QueueStorage { ) .await?; } + + DependentQueuedRequestKind::UploadGalleryFileOrThumbnail { + content_type, + cache_key, + related_to, + depends_on_thumbnail, + } => { + let Some(parent_key) = parent_key else { + // Not finished yet, we should retry later => false. + return Ok(false); + }; + self.handle_dependent_gallery_file_or_thumbnail_upload( + client, + dependent_request.own_transaction_id.into(), + parent_key, + content_type, + cache_key, + related_to, + depends_on_thumbnail, + ) + .await?; + } + + DependentQueuedRequestKind::FinishGallery { local_echo, item_infos } => { + let Some(parent_key) = parent_key else { + // Not finished yet, we should retry later => false. + return Ok(false); + }; + self.handle_dependent_finish_gallery_upload( + client, + dependent_request.own_transaction_id.into(), + parent_key, + local_echo, + item_infos, + new_updates, + ) + .await?; + } } Ok(true) @@ -1719,6 +2097,13 @@ impl QueueStorage { } } +struct GalleryItemQueueInfo { + content_type: Mime, + upload_file_txn: OwnedTransactionId, + file_media_request: MediaRequestParameters, + thumbnail: Option<(FinishUploadThumbnailInfo, MediaRequestParameters, Mime)>, +} + /// The content of a local echo. #[derive(Clone, Debug)] pub enum LocalEchoContent { @@ -1876,6 +2261,10 @@ pub enum RoomSendQueueStorageError { /// Trying to edit a media caption for something that's not a media. #[error("Can't edit a media caption when the underlying event isn't a media")] InvalidMediaCaptionEdit, + + /// Trying to edit a gallery caption for something that's not a gallery. + #[error("Can't edit a gallery caption when the underlying event isn't a gallery")] + InvalidGalleryCaptionEdit, } /// Extra transaction IDs useful during an upload. @@ -1903,7 +2292,7 @@ pub struct SendHandle { transaction_id: OwnedTransactionId, /// Additional handles for a media upload. - media_handles: Option, + media_handles: Vec, /// The time at which the event to be sent has been created. pub created_at: MilliSecondsSinceUnixEpoch, @@ -1911,7 +2300,7 @@ pub struct SendHandle { impl SendHandle { fn nyi_for_uploads(&self) -> Result<(), RoomSendQueueStorageError> { - if self.media_handles.is_some() { + if !self.media_handles.is_empty() { Err(RoomSendQueueStorageError::OperationNotImplementedYet) } else { Ok(()) @@ -1928,7 +2317,7 @@ impl SendHandle { let queue = &self.room.inner.queue; - if let Some(handles) = &self.media_handles { + for handles in &self.media_handles { if queue.abort_upload(&self.transaction_id, handles).await? { // Propagate a cancelled update. let _ = self.room.inner.updates.send(RoomSendQueueUpdate::CancelledLocalEvent { @@ -2061,7 +2450,7 @@ impl SendHandle { // one entry will be updated in the store. The other two are either // done, or dependent requests. - if let Some(handles) = &self.media_handles { + for handles in &self.media_handles { room.queue .mark_as_unwedged(&handles.upload_file_txn) .await @@ -2158,7 +2547,7 @@ impl SendReactionHandle { let handle = SendHandle { room: self.room.clone(), transaction_id: self.transaction_id.clone().into(), - media_handles: None, + media_handles: vec![], created_at: MilliSecondsSinceUnixEpoch::now(), }; @@ -2203,6 +2592,8 @@ fn canonicalize_dependent_requests( DependentQueuedRequestKind::UploadFileWithThumbnail { .. } | DependentQueuedRequestKind::FinishUpload { .. } + | DependentQueuedRequestKind::UploadGalleryFileOrThumbnail { .. } + | DependentQueuedRequestKind::FinishGallery { .. } | DependentQueuedRequestKind::ReactEvent { .. } => { // These requests can't be canonicalized, push them as is. prevs.push(d); diff --git a/crates/matrix-sdk/src/send_queue/upload.rs b/crates/matrix-sdk/src/send_queue/upload.rs index 236d4ba58b0..37e8a7975ac 100644 --- a/crates/matrix-sdk/src/send_queue/upload.rs +++ b/crates/matrix-sdk/src/send_queue/upload.rs @@ -14,19 +14,22 @@ //! Private implementations of the media upload mechanism. +use std::{collections::HashMap, iter::zip}; + use matrix_sdk_base::{ event_cache::store::media::IgnoreMediaRetentionPolicy, - media::{MediaFormat, MediaRequestParameters}, + media::{MediaFormat, MediaRequestParameters, UniqueKey}, store::{ - ChildTransactionId, DependentQueuedRequestKind, FinishUploadThumbnailInfo, - QueuedRequestKind, SentMediaInfo, SentRequestKey, SerializableEventContent, + ChildTransactionId, DependentQueuedRequestKind, FinishGalleryItemInfo, + FinishUploadThumbnailInfo, QueuedRequestKind, SentMediaInfo, SentRequestKey, + SerializableEventContent, }, RoomState, }; use mime::Mime; use ruma::{ events::{ - room::message::{FormattedBody, MessageType, RoomMessageEventContent}, + room::message::{FormattedBody, GalleryItemType, MessageType, RoomMessageEventContent}, AnyMessageLikeEventContent, Mentions, }, MilliSecondsSinceUnixEpoch, OwnedTransactionId, TransactionId, @@ -35,11 +38,11 @@ use tracing::{debug, error, instrument, trace, warn, Span}; use super::{QueueStorage, RoomSendQueue, RoomSendQueueError}; use crate::{ - attachment::AttachmentConfig, + attachment::{AttachmentConfig, GalleryConfig, GalleryItemInfo}, room::edit::update_media_caption, send_queue::{ - LocalEcho, LocalEchoContent, MediaHandles, RoomSendQueueStorageError, RoomSendQueueUpdate, - SendHandle, + GalleryItemQueueInfo, LocalEcho, LocalEchoContent, MediaHandles, RoomSendQueueStorageError, + RoomSendQueueUpdate, SendHandle, }, Client, Media, Room, }; @@ -83,6 +86,84 @@ fn update_media_event_after_upload(echo: &mut RoomMessageEventContent, sent: Sen } } +/// Replace the sources by the final ones in all the media types handled by +/// [`Room::make_gallery_item_type()`]. +fn update_gallery_event_after_upload( + echo: &mut RoomMessageEventContent, + sent: HashMap, +) { + match &mut echo.msgtype { + MessageType::Gallery(gallery) => { + // Some variants look really similar below, but the `event` and `info` are all + // different types… + for itemtype in gallery.itemtypes.iter_mut() { + match itemtype { + GalleryItemType::Audio(event) => match sent.get(&event.source.unique_key()) { + Some(sent) => event.source = sent.file.clone(), + None => error!( + "key for item {:?} does not exist on gallery event", + &event.source + ), + }, + GalleryItemType::File(event) => match sent.get(&event.source.unique_key()) { + Some(sent) => { + event.source = sent.file.clone(); + if let Some(info) = event.info.as_mut() { + info.thumbnail_source = sent.thumbnail.clone(); + } + } + None => error!( + "key for item {:?} does not exist on gallery event", + &event.source + ), + }, + GalleryItemType::Image(event) => match sent.get(&event.source.unique_key()) { + Some(sent) => { + event.source = sent.file.clone(); + if let Some(info) = event.info.as_mut() { + info.thumbnail_source = sent.thumbnail.clone(); + } + } + None => error!( + "key for item {:?} does not exist on gallery event", + &event.source + ), + }, + GalleryItemType::Video(event) => match sent.get(&event.source.unique_key()) { + Some(sent) => { + event.source = sent.file.clone(); + if let Some(info) = event.info.as_mut() { + info.thumbnail_source = sent.thumbnail.clone(); + } + } + None => error!( + "key for item {:?} does not exist on gallery event", + &event.source + ), + }, + + _ => { + // All `GalleryItemType` created by `Room::make_gallery_item_type` should be + // handled here. The only way to end up here is that a item type has + // been tampered with in the database. + error!("Invalid gallery item types in database"); + // Only crash debug builds. + debug_assert!(false, "invalid item type in database {:?}", itemtype); + } + } + } + } + _ => { + // All `GalleryItemType` created by `Room::make_gallery_item_type` should be + // handled here. The only way to end up here is that a item type has + // been tampered with in the database. + error!("Invalid gallery item types in database"); + // Only crash debug builds. + debug_assert!(false, "invalid item type in database {:?}", echo.msgtype()); + } + } +} + impl RoomSendQueue { /// Queues an attachment to be sent to the room, using the send queue. /// @@ -219,7 +300,178 @@ impl RoomSendQueue { let send_handle = SendHandle { room: self.clone(), transaction_id: send_event_txn.clone().into(), - media_handles: Some(MediaHandles { upload_thumbnail_txn, upload_file_txn }), + media_handles: vec![MediaHandles { upload_thumbnail_txn, upload_file_txn }], + created_at, + }; + + let _ = self.inner.updates.send(RoomSendQueueUpdate::NewLocalEvent(LocalEcho { + transaction_id: send_event_txn.clone().into(), + content: LocalEchoContent::Event { + serialized_event: SerializableEventContent::new(&event_content.into()) + .map_err(RoomSendQueueStorageError::JsonSerialization)?, + send_handle: send_handle.clone(), + send_error: None, + }, + })); + + Ok(send_handle) + } + + /// Queues a gallery to be sent to the room, using the send queue. + /// + /// This returns quickly (without sending or uploading anything), and will + /// push the event to be sent into a queue, handled in the background. + /// + /// Callers are expected to consume [`RoomSendQueueUpdate`] via calling + /// the [`Self::subscribe()`] method to get updates about the sending of + /// that event. + /// + /// By default, if sending failed on the first attempt, it will be retried a + /// few times. If sending failed after those retries, the entire + /// client's sending queue will be disabled, and it will need to be + /// manually re-enabled by the caller (e.g. after network is back, or when + /// something has been done about the faulty requests). + /// + /// The attachments and their optional thumbnails are stored in the media + /// cache and can be retrieved at any time, by calling + /// [`Media::get_media_content()`] with the respective `MediaSource` that + /// can be found in the local or remote echo, and using a + /// `MediaFormat::File`. + #[instrument(skip_all, fields(event_txn))] + pub async fn send_gallery( + &self, + item_infos: Vec, + config: GalleryConfig, + ) -> Result { + let Some(room) = self.inner.room.get() else { + return Err(RoomSendQueueError::RoomDisappeared); + }; + + if room.state() != RoomState::Joined { + return Err(RoomSendQueueError::RoomNotJoined); + } + + let send_event_txn = config.txn_id.map_or_else(ChildTransactionId::new, Into::into); + Span::current().record("event_txn", tracing::field::display(&*send_event_txn)); + + let mut item_types = Vec::::new(); + let mut item_queue_infos = Vec::::new(); + let mut media_handles = Vec::::new(); + + for item_info in item_infos { + let filename = item_info.filename; + let content_type = item_info.content_type; + let data = item_info.data; + + let upload_file_txn = TransactionId::new(); + + debug!(filename, %content_type, %upload_file_txn, "uploading a gallery attachment"); + + let file_media_request = Media::make_local_file_media_request(&upload_file_txn); + + let (upload_thumbnail_txn, event_thumbnail_info, queue_thumbnail_info) = { + let client = room.client(); + let cache_store = client + .event_cache_store() + .lock() + .await + .map_err(RoomSendQueueStorageError::LockError)?; + + // Cache the file itself in the cache store. + cache_store + .add_media_content( + &file_media_request, + data.clone(), + // Make sure that the file is stored until it has been uploaded. + IgnoreMediaRetentionPolicy::Yes, + ) + .await + .map_err(RoomSendQueueStorageError::EventCacheStoreError)?; + + // Process the thumbnail, if it's been provided. + if let Some(thumbnail) = item_info.thumbnail { + let txn = TransactionId::new(); + trace!(upload_thumbnail_txn = %txn, "gallery item has a thumbnail"); + + // Create the information required for filling the thumbnail section of the + // gallery event. + let (data, content_type, thumbnail_info) = thumbnail.into_parts(); + + // Cache thumbnail in the cache store. + let thumbnail_media_request = Media::make_local_file_media_request(&txn); + cache_store + .add_media_content( + &thumbnail_media_request, + data, + // Make sure that the thumbnail is stored until it has been uploaded. + IgnoreMediaRetentionPolicy::Yes, + ) + .await + .map_err(RoomSendQueueStorageError::EventCacheStoreError)?; + + ( + Some(txn.clone()), + Some((thumbnail_media_request.source.clone(), thumbnail_info)), + Some(( + FinishUploadThumbnailInfo { txn, width: None, height: None }, + thumbnail_media_request, + content_type, + )), + ) + } else { + Default::default() + } + }; + + item_types.push(room.make_gallery_item_type( + &content_type, + filename, + file_media_request.source.clone(), + item_info.caption, + item_info.formatted_caption, + Some(item_info.attachment_info), + event_thumbnail_info, + )); + + item_queue_infos.push(GalleryItemQueueInfo { + content_type, + upload_file_txn: upload_file_txn.clone(), + file_media_request, + thumbnail: queue_thumbnail_info, + }); + + media_handles.push(MediaHandles { upload_file_txn, upload_thumbnail_txn }); + } + + // Create the content for the gallery event. + let event_content = Room::make_gallery_event( + config.caption, + config.formatted_caption, + item_types, + config.mentions, + ); + + let created_at = MilliSecondsSinceUnixEpoch::now(); + + // Save requests in the queue storage. + self.inner + .queue + .push_gallery( + event_content.clone(), + send_event_txn.clone().into(), + created_at, + item_queue_infos, + ) + .await?; + + trace!("manager sends a gallery to the background task"); + + self.inner.notifier.notify_one(); + + let send_handle = SendHandle { + room: self.clone(), + transaction_id: send_event_txn.clone().into(), + media_handles, created_at, }; @@ -343,6 +595,131 @@ impl QueueStorage { Ok(()) } + /// Consumes a finished gallery upload and queues sending of the final + /// gallery event. + #[allow(clippy::too_many_arguments)] + pub(super) async fn handle_dependent_finish_gallery_upload( + &self, + client: &Client, + event_txn: OwnedTransactionId, + parent_key: SentRequestKey, + mut local_echo: RoomMessageEventContent, + item_infos: Vec, + new_updates: &mut Vec, + ) -> Result<(), RoomSendQueueError> { + // Both uploads are ready: enqueue the event with its final data. + let sent_gallery = parent_key + .into_gallery_media() + .ok_or(RoomSendQueueError::StorageError(RoomSendQueueStorageError::InvalidParentKey))?; + + let mut sent_media_vec = sent_gallery.accumulated; + sent_media_vec + .push(SentMediaInfo { file: sent_gallery.file, thumbnail: sent_gallery.thumbnail }); + + let mut sent_infos = HashMap::::new(); + + for (item_info, sent_media) in zip(item_infos, sent_media_vec) { + let FinishGalleryItemInfo { file_upload: file_upload_txn, thumbnail_info } = item_info; + + // Update cache keys in the cache store. + { + // Do it for the file itself. + let from_req = Media::make_local_file_media_request(&file_upload_txn); + + // Store the sent media under the original cache key for later insertion into + // the local echo + sent_infos.insert(from_req.source.unique_key(), sent_media.clone()); + + trace!(from = ?from_req.source, to = ?sent_media.file, "renaming media file key in cache store"); + let cache_store = client + .event_cache_store() + .lock() + .await + .map_err(RoomSendQueueStorageError::LockError)?; + + // The media can now be removed during cleanups. + cache_store + .set_ignore_media_retention_policy(&from_req, IgnoreMediaRetentionPolicy::No) + .await + .map_err(RoomSendQueueStorageError::EventCacheStoreError)?; + + cache_store + .replace_media_key( + &from_req, + &MediaRequestParameters { + source: sent_media.file.clone(), + format: MediaFormat::File, + }, + ) + .await + .map_err(RoomSendQueueStorageError::EventCacheStoreError)?; + + // Rename the thumbnail too, if needs be. + if let Some((info, new_source)) = + thumbnail_info.as_ref().zip(sent_media.thumbnail.clone()) + { + // Previously the media request used `MediaFormat::Thumbnail`. Handle this case + // for send queue requests that were in the state store before the change. + let from_req = if let Some((height, width)) = info.height.zip(info.width) { + Media::make_local_thumbnail_media_request(&info.txn, height, width) + } else { + Media::make_local_file_media_request(&info.txn) + }; + + trace!(from = ?from_req.source, to = ?new_source, "renaming thumbnail file key in cache store"); + + // The media can now be removed during cleanups. + cache_store + .set_ignore_media_retention_policy( + &from_req, + IgnoreMediaRetentionPolicy::No, + ) + .await + .map_err(RoomSendQueueStorageError::EventCacheStoreError)?; + + cache_store + .replace_media_key( + &from_req, + &MediaRequestParameters { + source: new_source, + format: MediaFormat::File, + }, + ) + .await + .map_err(RoomSendQueueStorageError::EventCacheStoreError)?; + } + } + } + + update_gallery_event_after_upload(&mut local_echo, sent_infos); + + let new_content = SerializableEventContent::new(&local_echo.into()) + .map_err(RoomSendQueueStorageError::JsonSerialization)?; + + // Indicates observers that the upload finished, by editing the local echo for + // the event into its final form before sending. + new_updates.push(RoomSendQueueUpdate::ReplacedLocalEvent { + transaction_id: event_txn.clone(), + new_content: new_content.clone(), + }); + + trace!(%event_txn, "queueing media event after successfully uploading media(s)"); + + client + .store() + .save_send_queue_request( + &self.room_id, + event_txn, + MilliSecondsSinceUnixEpoch::now(), + new_content.into(), + Self::HIGH_PRIORITY, + ) + .await + .map_err(RoomSendQueueStorageError::StateStoreError)?; + + Ok(()) + } + /// Consumes a finished upload of a thumbnail and queues the file upload. pub(super) async fn handle_dependent_file_upload_with_thumbnail( &self, @@ -391,6 +768,74 @@ impl QueueStorage { Ok(()) } + /// Consumes a finished upload of a gallery file (or thumbnail) and queues + /// the next thumbnail (or file) upload. + pub(super) async fn handle_dependent_gallery_file_or_thumbnail_upload( + &self, + client: &Client, + next_upload_txn: OwnedTransactionId, + parent_key: SentRequestKey, + content_type: String, + cache_key: MediaRequestParameters, + event_txn: OwnedTransactionId, + depends_on_thumbnail: bool, + ) -> Result<(), RoomSendQueueError> { + // The previous file (or thumbnail) has been sent, now transform the dependent + // thumbnail (or file) upload request into a ready one. + let sent_media = parent_key + .into_gallery_media() + .ok_or(RoomSendQueueError::StorageError(RoomSendQueueStorageError::InvalidParentKey))?; + + // If the previous upload was a thumbnail, it shouldn't have + // a thumbnail itself. + if depends_on_thumbnail { + debug_assert!(sent_media.thumbnail.is_none()); + if sent_media.thumbnail.is_some() { + warn!("unexpected thumbnail for a thumbnail!"); + } + } + + trace!(related_to = %event_txn, "done uploading thumbnail, now queuing a request to send the media file itself"); + + // If the previous upload was a thumbnail, forward the accumulated sent media + // unchangend. Otherwise, append the sent media to the accumulated + // vector. + let sent_media_infos = if depends_on_thumbnail { + sent_media.accumulated + } else { + let mut new_infos = sent_media.accumulated.clone(); + new_infos.push(SentMediaInfo { + file: sent_media.file.clone(), + thumbnail: sent_media.thumbnail, + }); + new_infos + }; + + let request = QueuedRequestKind::GalleryMediaUpload { + content_type, + cache_key, + // If the previous upload was a thumbnail, it becomse the thumbnail source for the next + // upload. + thumbnail_source: if depends_on_thumbnail { Some(sent_media.file) } else { None }, + related_to: event_txn, + sent_media_infos, + }; + + client + .store() + .save_send_queue_request( + &self.room_id, + next_upload_txn, + MilliSecondsSinceUnixEpoch::now(), + request, + Self::HIGH_PRIORITY, + ) + .await + .map_err(RoomSendQueueStorageError::StateStoreError)?; + + Ok(()) + } + /// Try to abort an upload that would be ongoing. /// /// Return true if any media (media itself or its thumbnail) was being