feat(search): index media, stickers, polls and more message types

crates/matrix-sdk-search/README.md

@@ -34,12 +34,11 @@ The search crate accepts index operations through
 
 ```rust
 use matrix_sdk_search::index::{
-    RoomIndex, RoomIndexOperation,
+    IndexableEvent, RoomIndex, RoomIndexOperation,
     builder::RoomIndexBuilder
 };
-use ruma::events::room::message::OriginalSyncRoomMessageEvent;
 
-async fn add_event(index: &mut RoomIndex, event: OriginalSyncRoomMessageEvent) {
+async fn add_event(index: &mut RoomIndex, event: IndexableEvent) {
     index.execute(RoomIndexOperation::Add(event));
 }
 ```

crates/matrix-sdk-search/changelog.d/6710.changed.md

@@ -0,0 +1 @@
+Introduce an `IndexableEvent` type and decouple message parsing from search indexing.

crates/matrix-sdk-search/src/index/mod.rs

@@ -17,9 +17,7 @@ pub mod builder;
 
 use std::{collections::HashSet, fmt};
 
-use ruma::{
-    EventId, OwnedEventId, OwnedRoomId, RoomId, events::room::message::OriginalSyncRoomMessageEvent,
-};
+use ruma::{EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId};
 use tantivy::{
     Index, IndexReader, TantivyDocument, collector::TopDocs, directory::error::OpenDirectoryError,
     query::QueryParser, schema::Value,
@@ -33,18 +31,37 @@ use crate::{
     writer::SearchIndexWriter,
 };
 
+/// The subset of an event's data required to index it and later retrieve it.
+///
+/// Produced by the matrix-sdk layer, which knows how to extract searchable text
+/// from each event type. This crate stays agnostic to Matrix event content.
+#[derive(Debug, Clone)]
+pub struct IndexableEvent {
+    /// The event's own id (primary key).
+    pub event_id: OwnedEventId,
+    /// The id used as the deletion key: the original event id for edits,
+    /// otherwise the event's own id.
+    pub original_event_id: OwnedEventId,
+    /// The sender of the event.
+    pub sender: OwnedUserId,
+    /// The origin server timestamp of the event.
+    pub timestamp: MilliSecondsSinceUnixEpoch,
+    /// The text to index for this event.
+    pub body: String,
+}
+
 /// A struct to represent the operations on a [`RoomIndex`]
 #[derive(Debug, Clone)]
 pub enum RoomIndexOperation {
     /// Add this event to the index.
-    Add(OriginalSyncRoomMessageEvent),
+    Add(IndexableEvent),
     /// Remove all documents in the index where
     /// `MatrixSearchIndexSchema::deletion_key()` matches this event id.
     Remove(OwnedEventId),
     /// Replace all documents in the index where
     /// `MatrixSearchIndexSchema::deletion_key()` matches this event id with
     /// the new event.
-    Edit(OwnedEventId, OriginalSyncRoomMessageEvent),
+    Edit(OwnedEventId, IndexableEvent),
     /// Do nothing.
     Noop,
 }
@@ -185,7 +202,7 @@ impl RoomIndex {
     fn add(
         &mut self,
         writer: &mut SearchIndexWriter,
-        event: OriginalSyncRoomMessageEvent,
+        event: IndexableEvent,
     ) -> Result<(), IndexError> {
         if !self.contains(&event.event_id) {
             writer.add(self.schema.make_doc(event.clone())?)?;
@@ -342,16 +359,38 @@ mod tests {
         EventId, event_id,
         events::{
             AnySyncMessageLikeEvent,
-            room::message::{OriginalSyncRoomMessageEvent, RoomMessageEventContentWithoutRelation},
+            room::message::{
+                MessageType, OriginalSyncRoomMessageEvent, Relation,
+                RoomMessageEventContentWithoutRelation,
+            },
         },
         room_id, user_id,
     };
 
     use crate::{
         error::IndexError,
-        index::{RoomIndex, RoomIndexOperation, builder::RoomIndexBuilder},
+        index::{IndexableEvent, RoomIndex, RoomIndexOperation, builder::RoomIndexBuilder},
     };
 
+    /// Build an [`IndexableEvent`] from a text room message (tests only handle
+    /// text).
+    fn to_indexable(event: &OriginalSyncRoomMessageEvent) -> IndexableEvent {
+        let MessageType::Text(content) = &event.content.msgtype else {
+            panic!("test helper only supports text messages")
+        };
+        let original_event_id = match &event.content.relates_to {
+            Some(Relation::Replacement(replacement)) => replacement.event_id.clone(),
+            _ => event.event_id.clone(),
+        };
+        IndexableEvent {
+            event_id: event.event_id.clone(),
+            original_event_id,
+            sender: event.sender.clone(),
+            timestamp: event.origin_server_ts,
+            body: content.body.clone(),
+        }
+    }
+
     /// Helper function to add a regular message to the index
     ///
     /// # Panic
@@ -365,7 +404,7 @@ mod tests {
             && let Some(ev) = ev.as_original()
             && ev.content.relates_to.is_none()
         {
-            return index.execute(RoomIndexOperation::Add(ev.clone()));
+            return index.execute(RoomIndexOperation::Add(to_indexable(ev)));
         }
         panic!("Event was not a relationless OriginalSyncRoomMessageEvent.")
     }
@@ -383,7 +422,7 @@ mod tests {
         event_id: &EventId,
         new: OriginalSyncRoomMessageEvent,
     ) -> Result<(), IndexError> {
-        index.execute(RoomIndexOperation::Edit(event_id.to_owned(), new))
+        index.execute(RoomIndexOperation::Edit(event_id.to_owned(), to_indexable(&new)))
     }
 
     #[test]

crates/matrix-sdk-search/src/schema.rs

@@ -12,13 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use ruma::events::room::message::{MessageType, OriginalSyncRoomMessageEvent, Relation};
 use tantivy::{
     DateTime, TantivyDocument, doc,
     schema::{DateOptions, DateTimePrecision, Field, INDEXED, STORED, STRING, Schema, TEXT},
 };
 
-use crate::error::{IndexError, IndexSchemaError};
+use crate::{
+    error::{IndexError, IndexSchemaError},
+    index::IndexableEvent,
+};
 
 pub(crate) trait MatrixSearchIndexSchema {
     fn new() -> Self;
@@ -27,7 +29,7 @@ pub(crate) trait MatrixSearchIndexSchema {
     fn deletion_key(&self) -> Field;
     fn get_field_name(&self, field: Field) -> &str;
     fn as_tantivy_schema(&self) -> Schema;
-    fn make_doc(&self, event: OriginalSyncRoomMessageEvent) -> Result<TantivyDocument, IndexError>;
+    fn make_doc(&self, event: IndexableEvent) -> Result<TantivyDocument, IndexError>;
 }
 
 #[derive(Debug, Clone)]
@@ -92,29 +94,17 @@ impl MatrixSearchIndexSchema for RoomMessageSchema {
         self.inner.clone()
     }
 
-    /// Given an [`OriginalSyncRoomMessageEvent`] return a
-    /// [`TantivyDocument`].
-    fn make_doc(&self, event: OriginalSyncRoomMessageEvent) -> Result<TantivyDocument, IndexError> {
-        let body = match &event.content.msgtype {
-            MessageType::Text(content) => Ok(content.body.clone()),
-            _ => Err(IndexError::MessageTypeNotSupported),
-        }?;
-
-        let mut document = doc!(
+    /// Given an [`IndexableEvent`] return a [`TantivyDocument`].
+    fn make_doc(&self, event: IndexableEvent) -> Result<TantivyDocument, IndexError> {
+        let document = doc!(
             self.event_id_field => event.event_id.to_string(),
-            self.body_field => body,
+            self.body_field => event.body,
             self.date_field =>
-                DateTime::from_timestamp_millis(
-                    event.origin_server_ts.get().into()),
+                DateTime::from_timestamp_millis(event.timestamp.get().into()),
             self.sender_field => event.sender.to_string(),
+            self.original_event_id_field => event.original_event_id.to_string(),
         );
 
-        if let Some(Relation::Replacement(replacement_data)) = &event.content.relates_to {
-            document.add_text(self.original_event_id_field, replacement_data.event_id.clone());
-        } else {
-            document.add_text(self.original_event_id_field, event.event_id);
-        }
-
         Ok(document)
     }
 }

crates/matrix-sdk/changelog.d/6710.changed.md

@@ -0,0 +1 @@
+Index media, stickers, polls and more message types. Move message parsing responsibility outside the search crate and into the main SDK one.