From 78afeb86f51ae6be21868975d468b10a51500f51 Mon Sep 17 00:00:00 2001 From: Gabe Date: Tue, 28 Jul 2026 18:17:25 -0400 Subject: [PATCH 1/4] Add bounded UIDONLY acquisition --- Cargo.lock | 25 +- crates/core/Cargo.toml | 2 +- crates/core/src/account/migration.rs | 5 + crates/core/src/cache/imap/download/flow.rs | 129 +- .../core/src/cache/imap/download/rebuild.rs | 201 +- crates/core/src/envelope/extractor.rs | 277 +- crates/core/src/imap/manager.rs | 67 + crates/core/src/imap/mock_server.rs | 17 +- crates/core/src/imap/mod.rs | 1 + crates/core/src/imap/uidonly_acquisition.rs | 3682 +++++++++++++++++ crates/core/src/mailbox/delete.rs | 17 +- crates/core/src/store/blob.rs | 91 +- crates/core/src/store/tantivy/attachment.rs | 140 +- crates/core/src/store/tantivy/dedup.rs | 85 +- crates/core/src/store/tantivy/envelope.rs | 137 + 15 files changed, 4764 insertions(+), 112 deletions(-) create mode 100644 crates/core/src/imap/uidonly_acquisition.rs diff --git a/Cargo.lock b/Cargo.lock index ad76b59b..8be51953 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -102,7 +102,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -113,7 +113,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -220,7 +220,7 @@ dependencies = [ [[package]] name = "async-imap" version = "0.11.2" -source = "git+https://github.com/rustmailer/async-imap.git?branch=main#7ad47e6b270262ed02c440b3fc0a9985a79e0271" +source = "git+https://github.com/gabeosx/async-imap.git?rev=afdb7a51a46a75b175b8aaa642f7ad1328ce868b#afdb7a51a46a75b175b8aaa642f7ad1328ce868b" dependencies = [ "async-channel 2.5.0", "async-compression", @@ -1369,7 +1369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2221,9 +2221,8 @@ dependencies = [ [[package]] name = "imap-proto" -version = "0.16.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25f6af35c6a517aea5c72314abe90134980d2ae6a763809b50c208b3e429d71f" +version = "0.17.0" +source = "git+https://github.com/gabeosx/tokio-imap?rev=68a4e1dc6beffcb82b9ade4b818d2af8b8594649#68a4e1dc6beffcb82b9ade4b818d2af8b8594649" dependencies = [ "nom 7.1.3", ] @@ -2913,7 +2912,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4025,7 +4024,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4093,7 +4092,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4448,7 +4447,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -4787,7 +4786,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5651,7 +5650,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 9794b0d1..60d346a0 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -43,7 +43,7 @@ sysinfo.workspace = true num_cpus.workspace = true rand.workspace = true encoding_rs.workspace = true -async-imap = { git = "https://github.com/rustmailer/async-imap.git", branch = "main", default-features = false, features = [ +async-imap = { git = "https://github.com/gabeosx/async-imap.git", rev = "afdb7a51a46a75b175b8aaa642f7ad1328ce868b", default-features = false, features = [ "runtime-tokio", "compress", ] } diff --git a/crates/core/src/account/migration.rs b/crates/core/src/account/migration.rs index ed0bcfae..d1121d6d 100644 --- a/crates/core/src/account/migration.rs +++ b/crates/core/src/account/migration.rs @@ -39,6 +39,7 @@ use crate::{ id, oauth2::token::OAuth2AccessToken, raise_error, + settings::dir::DATA_DIR_MANAGER, store::tantivy::{attachment::ATTACHMENT_MANAGER, envelope::ENVELOPE_MANAGER}, users::{payload::UserUpdateRequest, role::DEFAULT_ACCOUNT_MANAGER_ROLE_ID, UserModel}, utc_now, @@ -498,6 +499,10 @@ impl Account { } OAuth2AccessToken::try_delete(account.id)?; UserModel::cleanup_account(account.id)?; + crate::imap::uidonly_acquisition::cleanup_uidonly_account_state( + &DATA_DIR_MANAGER.storage_dir.join("uidonly-acquisition"), + account.id, + )?; MailBox::clean(account.id)?; ENVELOPE_MANAGER .delete_account_envelopes(account.id) diff --git a/crates/core/src/cache/imap/download/flow.rs b/crates/core/src/cache/imap/download/flow.rs index 3fc0b7bb..73c5fd4f 100644 --- a/crates/core/src/cache/imap/download/flow.rs +++ b/crates/core/src/cache/imap/download/flow.rs @@ -35,6 +35,9 @@ use crate::{ imap::executor::{ compress_uid_list, generate_uid_sequence_hashset, ImapExecutor, DEFAULT_BATCH_SIZE, }, + imap::manager::{AcquisitionConnection, ImapConnectionManager}, + imap::uidonly_acquisition::{acquire_bichon_mailbox, AcquisitionLimits}, + settings::dir::DATA_DIR_MANAGER, store::tantivy::envelope::ENVELOPE_MANAGER, }, }; @@ -51,6 +54,23 @@ pub enum FetchDirection { Before, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum UidValiditySyncStrategy { + ReconcileByMessageId, + Incremental, +} + +fn uid_validity_sync_strategy( + local_uid_validity: Option, + remote_uid_validity: u32, +) -> UidValiditySyncStrategy { + if local_uid_validity == Some(remote_uid_validity) { + UidValiditySyncStrategy::Incremental + } else { + UidValiditySyncStrategy::ReconcileByMessageId + } +} + pub async fn fetch_and_save_by_date( account: &AccountModel, date: &str, @@ -262,8 +282,56 @@ pub async fn fetch_and_save_full_mailbox( let mailbox_id = mailbox.id; let account_id = account.id; - let mut session = match ImapExecutor::create_connection(account_id).await { - Ok(session) => session, + let limits = AcquisitionLimits::for_account(account); + let connection = ImapConnectionManager::build_acquisition( + account_id, + limits.response_limits()?, + ) + .await; + let mut session = match connection { + Ok(AcquisitionConnection::Standard(session)) => session, + Ok(AcquisitionConnection::UidOnly { + session, + message_limit, + }) => { + let root = DATA_DIR_MANAGER.storage_dir.join("uidonly-acquisition"); + let report = acquire_bichon_mailbox( + account, + mailbox, + session, + message_limit, + &root, + limits, + token, + ) + .await?; + DownloadState::update_folder_progress( + account_id, + mailbox.name.clone(), + report.planned, + report.processed, + if report.success { + FolderStatus::Success + } else { + FolderStatus::Failed + }, + (!report.success).then(|| { + "UIDONLY snapshot contains unresolved UIDs; checkpoint was not advanced" + .to_string() + }), + )?; + if !report.success { + return Err(raise_error!( + "UIDONLY snapshot incomplete; see durable per-UID ledger".into(), + ErrorCode::ImapUnexpectedResult + )); + } + let mut updated = mailbox.clone(); + updated.uid_validity = Some(report.uid_validity); + updated.highest_uid = report.checkpoint; + MailBox::batch_upsert(&[updated])?; + return Ok(report.checkpoint); + } Err(e) => { let err_msg = format!("Connection failed for this folder: {:#?}", e); DownloadState::update_folder_progress( @@ -784,23 +852,34 @@ pub async fn reconcile_mailboxes( } }; - let new_highest_uid = if local_mailbox.uid_validity != Some(remote_uid_validity) { - info!( - "Account {}: Mailbox '{}' detected with changed uid_validity (local: {:#?}, remote: {:#?}). \ - Comparing by Message-ID to find missing emails.", - account_id, local_mailbox.name, &local_mailbox.uid_validity, &remote_uid_validity - ); + let new_highest_uid = match uid_validity_sync_strategy( + local_mailbox.uid_validity, + remote_uid_validity, + ) { + UidValiditySyncStrategy::ReconcileByMessageId => { + info!( + "Account {}: Mailbox '{}' detected with changed uid_validity (local: {:#?}, remote: {:#?}). \ + Comparing by Message-ID to find missing emails.", + account_id, local_mailbox.name, &local_mailbox.uid_validity, &remote_uid_validity + ); - reconcile_uid_validity_change( - account, - local_mailbox, - remote_mailbox, - token.clone(), - ) - .await? - } else { - perform_incremental_sync(account, local_mailbox, remote_mailbox, token.clone()) + reconcile_uid_validity_change( + account, + local_mailbox, + remote_mailbox, + token.clone(), + ) + .await? + } + UidValiditySyncStrategy::Incremental => { + perform_incremental_sync( + account, + local_mailbox, + remote_mailbox, + token.clone(), + ) .await? + } }; let mut updated = remote_mailbox.clone(); @@ -1043,6 +1122,22 @@ mod tests { assert_ne!(uid, 0, "UIDVALIDITY should not be 0 (reserved)"); } + #[test] + fn uidvalidity_strategy_routes_changes_through_message_id_reconciliation() { + assert_eq!( + uid_validity_sync_strategy(Some(9), 10), + UidValiditySyncStrategy::ReconcileByMessageId + ); + assert_eq!( + uid_validity_sync_strategy(None, 10), + UidValiditySyncStrategy::ReconcileByMessageId + ); + assert_eq!( + uid_validity_sync_strategy(Some(10), 10), + UidValiditySyncStrategy::Incremental + ); + } + // ============================================================ // Integration tests (real IMAP server required) // ============================================================ diff --git a/crates/core/src/cache/imap/download/rebuild.rs b/crates/core/src/cache/imap/download/rebuild.rs index 96d31d9e..88e06a91 100644 --- a/crates/core/src/cache/imap/download/rebuild.rs +++ b/crates/core/src/cache/imap/download/rebuild.rs @@ -29,13 +29,35 @@ use crate::{ SEMAPHORE, }, error::{code::ErrorCode, BichonResult}, + imap::uidonly_acquisition::cleanup_uidonly_mailbox_state, raise_error, + settings::dir::DATA_DIR_MANAGER, store::tantivy::{attachment::ATTACHMENT_MANAGER, envelope::ENVELOPE_MANAGER}, }; +use std::collections::BTreeSet; +use std::future::Future; use tokio_util::sync::CancellationToken; use tracing::{error, info}; +async fn run_rebuild_sequence( + cleanup: Cleanup, + delete_envelope: DeleteEnvelope, + delete_attachment: DeleteAttachment, + reacquire: Reacquire, +) -> BichonResult +where + Cleanup: Future>, + DeleteEnvelope: Future>, + DeleteAttachment: Future>, + Reacquire: Future>, +{ + cleanup.await?; + delete_envelope.await?; + delete_attachment.await?; + reacquire.await +} + pub async fn rebuild_cache( account: &AccountModel, remote_mailboxes: &[MailBox], @@ -205,31 +227,40 @@ pub async fn rebuild_mailbox_cache( remote_mailbox: &MailBox, token: CancellationToken, ) -> BichonResult> { - ENVELOPE_MANAGER - .delete_mailbox_envelopes(account.id, vec![local_mailbox.id]) - .await?; - ATTACHMENT_MANAGER - .delete_mailbox_attachments(account.id, vec![local_mailbox.id]) - .await?; - if remote_mailbox.exists == 0 { - info!( - "Account {}: Mailbox '{}' has no emails on the remote server. The mailbox is empty, no envelopes to fetch.", - account.id, - &local_mailbox.name - ); - DownloadState::update_folder_progress( - account.id, - remote_mailbox.name.clone(), - 0, - 0, - FolderStatus::Success, - None, - )?; - return Ok(None); - } - - let result = fetch_and_save_full_mailbox(account, remote_mailbox, token).await?; - Ok(result) + let names = BTreeSet::from([local_mailbox.name.clone(), remote_mailbox.name.clone()]); + run_rebuild_sequence( + async { + cleanup_uidonly_mailbox_state( + &DATA_DIR_MANAGER.storage_dir.join("uidonly-acquisition"), + account.id, + &names, + )?; + Ok(()) + }, + ENVELOPE_MANAGER.delete_mailbox_envelopes(account.id, vec![local_mailbox.id]), + ATTACHMENT_MANAGER.delete_mailbox_attachments(account.id, vec![local_mailbox.id]), + async { + if remote_mailbox.exists == 0 { + info!( + "Account {}: Mailbox '{}' has no emails on the remote server. The mailbox is empty, no envelopes to fetch.", + account.id, + &local_mailbox.name + ); + DownloadState::update_folder_progress( + account.id, + remote_mailbox.name.clone(), + 0, + 0, + FolderStatus::Success, + None, + )?; + Ok(None) + } else { + fetch_and_save_full_mailbox(account, remote_mailbox, token).await + } + }, + ) + .await } pub async fn rebuild_mailbox_cache_by_date( @@ -240,29 +271,105 @@ pub async fn rebuild_mailbox_cache_by_date( direction: FetchDirection, token: CancellationToken, ) -> BichonResult> { - ENVELOPE_MANAGER - .delete_mailbox_envelopes(account.id, vec![local_mailbox_id]) - .await?; - ATTACHMENT_MANAGER - .delete_mailbox_attachments(account.id, vec![local_mailbox_id]) - .await?; - if remote.exists == 0 { - info!( - "Account {}: Mailbox '{}' has no emails on the remote server. The mailbox is empty, no envelopes to fetch.", - account.id, - &remote.name + let names = BTreeSet::from([remote.name.clone()]); + run_rebuild_sequence( + async { + cleanup_uidonly_mailbox_state( + &DATA_DIR_MANAGER.storage_dir.join("uidonly-acquisition"), + account.id, + &names, + )?; + Ok(()) + }, + ENVELOPE_MANAGER.delete_mailbox_envelopes(account.id, vec![local_mailbox_id]), + ATTACHMENT_MANAGER.delete_mailbox_attachments(account.id, vec![local_mailbox_id]), + async { + if remote.exists == 0 { + info!( + "Account {}: Mailbox '{}' has no emails on the remote server. The mailbox is empty, no envelopes to fetch.", + account.id, + &remote.name + ); + DownloadState::update_folder_progress( + account.id, + remote.name.clone(), + 0, + 0, + FolderStatus::Success, + None, + )?; + Ok(None) + } else { + fetch_and_save_by_date(account, date, remote, direction, token).await + } + }, + ) + .await +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::{Arc, Mutex}; + + #[tokio::test] + async fn rebuild_sequence_cleans_ledger_before_indexes_then_reacquires() { + let events = Arc::new(Mutex::new(Vec::new())); + let record = |event: &'static str| { + let events = events.clone(); + async move { + events.lock().unwrap().push(event); + Ok::<_, crate::error::BichonError>(()) + } + }; + let reacquire_events = events.clone(); + let result = run_rebuild_sequence( + record("cleanup-ledger"), + record("delete-envelope-index"), + record("delete-attachment-index"), + async move { + reacquire_events.lock().unwrap().push("reacquire"); + Ok(77u32) + }, + ) + .await + .unwrap(); + assert_eq!(result, 77); + assert_eq!( + *events.lock().unwrap(), + [ + "cleanup-ledger", + "delete-envelope-index", + "delete-attachment-index", + "reacquire" + ] ); - DownloadState::update_folder_progress( - account.id, - remote.name.clone(), - 0, - 0, - FolderStatus::Success, - None, - )?; - return Ok(None); } - let result = fetch_and_save_by_date(account, date, remote, direction, token).await?; - Ok(result) + #[tokio::test] + async fn rebuild_cleanup_failure_aborts_before_canonical_deletion() { + let events = Arc::new(Mutex::new(Vec::new())); + let later = |event: &'static str| { + let events = events.clone(); + async move { + events.lock().unwrap().push(event); + Ok::<_, crate::error::BichonError>(()) + } + }; + let error = run_rebuild_sequence( + async { + Err(raise_error!( + "synthetic ledger cleanup failure".into(), + ErrorCode::InternalError + )) + }, + later("delete-envelope-index"), + later("delete-attachment-index"), + async { Ok(()) }, + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("synthetic ledger cleanup failure")); + assert!(events.lock().unwrap().is_empty()); + } } diff --git a/crates/core/src/envelope/extractor.rs b/crates/core/src/envelope/extractor.rs index 88195a72..4a24c113 100644 --- a/crates/core/src/envelope/extractor.rs +++ b/crates/core/src/envelope/extractor.rs @@ -27,6 +27,7 @@ use crate::imap::executor::ImapExecutor; use crate::message::content::AttachmentInfo; use crate::store::blob::{DetachedEmail, BLOB_MANAGER}; use crate::store::tantivy::attachment::ATTACHMENT_MANAGER; +use crate::store::tantivy::dedup::UIDONLY_SHARD_ID; use crate::store::tantivy::dedup_cache::DEDUP_CACHE; use crate::store::tantivy::envelope::ENVELOPE_MANAGER; use crate::store::tantivy::model::{AttachmentModel, EnvelopeWithAttachments}; @@ -41,6 +42,24 @@ use tantivy::TantivyDocument; use tantivy::schema::Facet; use tracing::error; use uuid::Uuid; +use tokio_util::sync::CancellationToken; + +#[cfg(test)] +use std::sync::atomic::{AtomicBool, Ordering}; + +#[cfg(test)] +static FAIL_UIDONLY_AFTER_ATTACHMENTS: AtomicBool = AtomicBool::new(false); + +#[cfg(test)] +pub(crate) fn fail_uidonly_after_attachments(enabled: bool) { + FAIL_UIDONLY_AFTER_ATTACHMENTS.store(enabled, Ordering::Release); +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CanonicalProjection { + pub envelope_id: String, + pub content_hash: String, +} pub async fn extract_envelope_and_store_it( fetch: Fetch, @@ -64,7 +83,20 @@ pub async fn extract_envelope_and_store_it( } }; let size = fetch.size.unwrap_or(body.len() as u32); - extract_envelope_core(body, uid, size, internal_date, account_id, mailbox_id).await + extract_envelope_core( + body, + uid, + size, + internal_date, + account_id, + mailbox_id, + false, + false, + None, + None, + ) + .await + .map(|_| ()) } pub async fn extract_envelope_from_eml( @@ -72,7 +104,20 @@ pub async fn extract_envelope_from_eml( account_id: u64, mailbox_id: u64, ) -> BichonResult<()> { - extract_envelope_core(body, 0, body.len() as u32, 0, account_id, mailbox_id).await + extract_envelope_core( + body, + 0, + body.len() as u32, + 0, + account_id, + mailbox_id, + false, + false, + None, + None, + ) + .await + .map(|_| ()) } pub async fn extract_envelope_from_smtp( @@ -87,10 +132,48 @@ pub async fn extract_envelope_from_smtp( utc_now!(), account_id, mailbox_id, + false, + false, + None, + None, ) .await + .map(|_| ()) } +#[allow(clippy::too_many_arguments)] +pub(crate) async fn project_uidonly_message( + body: &[u8], + uid: u32, + size: u32, + internal_date: i64, + account_id: u64, + mailbox_id: u64, + envelope_id: String, + shutdown: CancellationToken, +) -> BichonResult { + extract_envelope_core( + body, + uid, + size, + internal_date, + account_id, + mailbox_id, + true, + true, + Some(envelope_id), + Some(&shutdown), + ) + .await? + .ok_or_else(|| { + raise_error!( + format!("UID {uid} was not projected into the canonical archive"), + ErrorCode::InternalError + ) + }) +} + +#[allow(clippy::too_many_arguments)] async fn extract_envelope_core( body: &[u8], uid: u32, @@ -98,13 +181,17 @@ async fn extract_envelope_core( internal_date: i64, account_id: u64, mailbox_id: u64, -) -> BichonResult<()> { + durable: bool, + preserve_uid_identity: bool, + fixed_envelope_id: Option, + shutdown: Option<&CancellationToken>, +) -> BichonResult> { //The content hash of the original raw EML let email_content_hash = compute_content_hash(body); - if DEDUP_CACHE.contains(account_id, mailbox_id, &email_content_hash) { + if !preserve_uid_identity && DEDUP_CACHE.contains(account_id, mailbox_id, &email_content_hash) { tracing::debug!("Duplicate email detected"); //println!("Duplicate email detected"); - return Ok(()); + return Ok(None); } let message: Message<'_> = MessageParser::new().parse(body).ok_or_else(|| { raise_error!( @@ -136,7 +223,7 @@ async fn extract_envelope_core( subject = subject.as_deref().unwrap_or("?"), "Email filtered out by archive rules" ); - return Ok(()); + return Ok(None); } } } @@ -202,9 +289,11 @@ async fn extract_envelope_core( .and_then(|add| add.address) .unwrap_or_else(|| "unknown".to_string()); let attachment_count = message.attachment_count(); - let attachments = detach_and_store_attachments(body, &message, &email_content_hash, account_id, mailbox_id).await; - - let envelope_id = Uuid::new_v4().to_string(); + let (attachments, detached_email) = + prepare_detached_attachments(body, &message, &email_content_hash, account_id, mailbox_id) + .await; + check_uidonly_projection_cancelled(shutdown)?; + let envelope_id = fixed_envelope_id.unwrap_or_else(|| Uuid::new_v4().to_string()); let now = utc_now!(); @@ -274,7 +363,7 @@ async fn extract_envelope_core( .collect(); let envelope = Envelope { - id: envelope_id, + id: envelope_id.clone(), message_id, account_id, mailbox_id, @@ -303,7 +392,10 @@ async fn extract_envelope_core( envelope, attachments: Some(attachments), }; - let doc = ea.to_document(&body_text, 0)?; + let doc = ea.to_document( + &body_text, + if durable { UIDONLY_SHARD_ID } else { 0 }, + )?; tracing::debug!( "[account {}][mailbox {}] extract: uid={} msg_id={} content_hash={}", account_id, @@ -312,12 +404,134 @@ async fn extract_envelope_core( &ea.envelope.message_id, &ea.envelope.content_hash, ); - ENVELOPE_MANAGER.queue(doc).await; + if durable { + BLOB_MANAGER.store_durable(detached_email).await?; + let commit_result = async { + check_uidonly_projection_cancelled(shutdown)?; + ATTACHMENT_MANAGER.commit_documents(attachment_docs).await?; + check_uidonly_projection_cancelled(shutdown)?; + #[cfg(test)] + if FAIL_UIDONLY_AFTER_ATTACHMENTS.load(Ordering::Acquire) { + return Err(raise_error!( + "synthetic UIDONLY envelope commit failure".into(), + ErrorCode::InternalError + )); + } + ENVELOPE_MANAGER.commit_document(doc).await?; + check_uidonly_projection_cancelled(shutdown) + } + .await; + if let Err(error) = commit_result { + rollback_failed_uidonly_projection( + account_id, + &envelope_id, + &email_content_hash, + ea.attachments.as_deref().unwrap_or_default(), + ) + .await?; + return Err(error); + } + } else { + BLOB_MANAGER.queue(detached_email).await; + ENVELOPE_MANAGER.queue(doc).await; + for doc in attachment_docs { + ATTACHMENT_MANAGER.queue(doc).await; + } + } DEDUP_CACHE.insert(account_id, mailbox_id, &email_content_hash); - for doc in attachment_docs { - ATTACHMENT_MANAGER.queue(doc).await; + Ok(Some(CanonicalProjection { + envelope_id, + content_hash: email_content_hash, + })) +} + +fn check_uidonly_projection_cancelled( + shutdown: Option<&CancellationToken>, +) -> BichonResult<()> { + if shutdown.is_some_and(CancellationToken::is_cancelled) { + Err(raise_error!( + "UIDONLY canonical projection cancelled".into(), + ErrorCode::InternalError + )) + } else { + Ok(()) + } +} + +async fn rollback_failed_uidonly_projection( + account_id: u64, + envelope_id: &str, + email_content_hash: &str, + attachments: &[AttachmentInfo], +) -> BichonResult<()> { + let mut attachment_hashes: std::collections::HashSet = attachments + .iter() + .map(|attachment| attachment.content_hash.clone()) + .collect(); + let mut cleanup_errors = Vec::new(); + match ATTACHMENT_MANAGER + .rollback_documents(account_id, envelope_id) + .await + { + Ok(indexed_hashes) => attachment_hashes.extend(indexed_hashes), + Err(error) => cleanup_errors.push(error.to_string()), + } + if let Err(error) = ENVELOPE_MANAGER + .rollback_uidonly_projection( + account_id, + envelope_id, + email_content_hash.to_string(), + attachment_hashes, + ) + .await + { + cleanup_errors.push(error.to_string()); + } + if cleanup_errors.is_empty() { + Ok(()) + } else { + Err(raise_error!( + format!("UIDONLY canonical rollback failed: {}", cleanup_errors.join("; ")), + ErrorCode::InternalError + )) + } +} + +pub(crate) async fn rollback_uidonly_message( + account_id: u64, + envelope_id: &str, + email_content_hash: &str, +) -> BichonResult<()> { + let mut cleanup_errors = Vec::new(); + let attachment_hashes = match ATTACHMENT_MANAGER + .rollback_documents(account_id, envelope_id) + .await + { + Ok(hashes) => hashes, + Err(error) => { + cleanup_errors.push(error.to_string()); + std::collections::HashSet::new() + } + }; + if let Err(error) = ENVELOPE_MANAGER + .rollback_uidonly_projection( + account_id, + envelope_id, + email_content_hash.to_string(), + attachment_hashes, + ) + .await + { + cleanup_errors.push(error.to_string()); + } + if cleanup_errors.is_empty() { + Ok(()) + } else { + Err(raise_error!( + format!("UIDONLY canonical rollback failed: {}", cleanup_errors.join("; ")), + ErrorCode::InternalError + )) } - Ok(()) } pub fn extract_envelope_from_nested_message( @@ -426,13 +640,13 @@ pub fn extract_references(message: &Message<'_>) -> Option> { } } -pub async fn detach_and_store_attachments( +async fn prepare_detached_attachments( original_body: &[u8], message: &Message<'_>, eml_content_hash: &str, account_id: u64, mailbox_id: u64, -) -> Vec { +) -> (Vec, DetachedEmail) { let rules = if account_id > 0 { AccountModel::get(account_id) .ok() @@ -597,14 +811,31 @@ pub async fn detach_and_store_attachments( } } } - // Step 4: Store the final stripped EML content - BLOB_MANAGER - .queue(DetachedEmail { + ( + attachment_infos, + DetachedEmail { email: (eml_content_hash.to_string(), Bytes::from(stripped_eml)), attachments: Some(attachments), - }) - .await; + }, + ) +} +pub async fn detach_and_store_attachments( + original_body: &[u8], + message: &Message<'_>, + eml_content_hash: &str, + account_id: u64, + mailbox_id: u64, +) -> Vec { + let (attachment_infos, detached_email) = prepare_detached_attachments( + original_body, + message, + eml_content_hash, + account_id, + mailbox_id, + ) + .await; + BLOB_MANAGER.queue(detached_email).await; attachment_infos } @@ -647,7 +878,7 @@ pub fn reattach_eml_content( return Err(raise_error!( format!( "Consistency check failed: envelope.attachment_count ({}) does not match attachments.len ({})", - e.envelope.attachment_count, + e.envelope.attachment_count, actual_count ), ErrorCode::InternalError diff --git a/crates/core/src/imap/manager.rs b/crates/core/src/imap/manager.rs index e048edd0..2a310014 100644 --- a/crates/core/src/imap/manager.rs +++ b/crates/core/src/imap/manager.rs @@ -27,10 +27,35 @@ use crate::imap::session::SessionStream; use crate::oauth2::token::OAuth2AccessToken; use crate::{bichon_version, decrypt, raise_error}; use async_imap::Session; +use async_imap::types::ResponseLimits; use tracing::{error, warn}; pub struct ImapConnectionManager; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum AcquisitionRoute { + Standard, + UidOnly, +} + +fn acquisition_route(uidonly_advertised: bool) -> AcquisitionRoute { + if uidonly_advertised { + AcquisitionRoute::UidOnly + } else { + AcquisitionRoute::Standard + } +} + +/// An authenticated acquisition connection. Existing callers keep using +/// `build`; only the archive acquisition path opts into UIDONLY. +pub(crate) enum AcquisitionConnection { + Standard(Session>), + UidOnly { + session: Session>, + message_limit: Option, + }, +} + impl ImapConnectionManager { async fn create_client(account: &AccountModel) -> BichonResult { assert_eq!(account.account_type, AccountType::IMAP); @@ -168,4 +193,46 @@ impl ImapConnectionManager { Ok(session) } + + /// Builds an acquisition connection and enables RFC 9586 UIDONLY before + /// any mailbox is selected. Calling this method again after a reconnect + /// necessarily re-enables UIDONLY because the mode is connection-scoped. + /// + /// Servers without UIDONLY return the already-authenticated standard + /// session, preserving the pre-existing acquisition behavior. + pub(crate) async fn build_acquisition( + account_id: u64, + response_limits: ResponseLimits, + ) -> BichonResult { + let mut session = Self::build(account_id).await?; + let capabilities = fetch_capabilities(&mut session).await?; + + if acquisition_route(capabilities.has_str("UIDONLY")) == AcquisitionRoute::Standard { + return Ok(AcquisitionConnection::Standard(session)); + } + + session + .set_response_limits(response_limits) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InvalidParameter))?; + session + .enable_uidonly() + .await + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::ImapCommandFailed))?; + + Ok(AcquisitionConnection::UidOnly { + message_limit: capabilities.message_limit(), + session, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn non_uidonly_capabilities_route_to_legacy_standard_acquisition() { + assert_eq!(acquisition_route(false), AcquisitionRoute::Standard); + assert_eq!(acquisition_route(true), AcquisitionRoute::UidOnly); + } } diff --git a/crates/core/src/imap/mock_server.rs b/crates/core/src/imap/mock_server.rs index c37c3455..bf0ba4b8 100644 --- a/crates/core/src/imap/mock_server.rs +++ b/crates/core/src/imap/mock_server.rs @@ -38,7 +38,7 @@ //! ``` use std::net::SocketAddr; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; use tokio::net::{TcpListener, TcpStream}; @@ -47,6 +47,7 @@ type Response = Vec; pub struct MockImapServer { greeting: Vec, script: Vec<(String, Response)>, + transcript: Arc>>, } impl MockImapServer { @@ -54,6 +55,7 @@ impl MockImapServer { Self { greeting: b"* OK Mock IMAP server ready\r\n".to_vec(), script: Vec::new(), + transcript: Arc::new(Mutex::new(Vec::new())), } } @@ -76,6 +78,7 @@ impl MockImapServer { let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); let addr = listener.local_addr().expect("local_addr"); + let transcript = self.transcript.clone(); let server = Arc::new(self); tokio::spawn(async move { @@ -92,7 +95,7 @@ impl MockImapServer { } }); - MockImapServerHandle { addr } + MockImapServerHandle { addr, transcript } } async fn handle_connection(&self, mut stream: TcpStream) { @@ -113,6 +116,11 @@ impl MockImapServer { Err(_) => break, } + self.transcript + .lock() + .expect("transcript lock") + .push(line.trim_end().to_string()); + let tag = extract_tag(&line).unwrap_or("A0"); let matched = self.find_match(&line); if let Some(response) = matched { @@ -151,6 +159,7 @@ impl Default for MockImapServer { /// is dropped. pub struct MockImapServerHandle { addr: SocketAddr, + transcript: Arc>>, } impl MockImapServerHandle { @@ -161,6 +170,10 @@ impl MockImapServerHandle { pub fn port(&self) -> u16 { self.addr.port() } + + pub fn commands(&self) -> Vec { + self.transcript.lock().expect("transcript lock").clone() + } } fn extract_tag(line: &str) -> Option<&str> { diff --git a/crates/core/src/imap/mod.rs b/crates/core/src/imap/mod.rs index 0866f1a1..53ba9218 100644 --- a/crates/core/src/imap/mod.rs +++ b/crates/core/src/imap/mod.rs @@ -24,6 +24,7 @@ pub mod manager; pub mod oauth2; pub mod session; pub mod stats; +pub(crate) mod uidonly_acquisition; #[cfg(test)] mod tests; #[cfg(test)] diff --git a/crates/core/src/imap/uidonly_acquisition.rs b/crates/core/src/imap/uidonly_acquisition.rs new file mode 100644 index 00000000..74870b6d --- /dev/null +++ b/crates/core/src/imap/uidonly_acquisition.rs @@ -0,0 +1,3682 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! UID-safe, restartable mailbox acquisition. +//! +//! This module is intentionally separate from Bichon's legacy sequence-number +//! downloader. A UIDONLY session never reaches code which can issue ordinary +//! FETCH, SEARCH, STORE, COPY, or MOVE commands. + +use crate::account::migration::AccountModel; +use crate::cache::imap::mailbox::MailBox; +use crate::envelope::extractor::{ + project_uidonly_message, reattach_eml_content, rollback_uidonly_message, CanonicalProjection, +}; +use crate::error::code::ErrorCode; +use crate::error::{BichonError, BichonResult}; +use crate::imap::manager::{AcquisitionConnection, ImapConnectionManager}; +use crate::imap::session::SessionStream; +use crate::message::content::AttachmentInfo; +use crate::raise_error; +use crate::store::tantivy::attachment::{CanonicalAttachmentRecord, ATTACHMENT_MANAGER}; +use crate::store::tantivy::dedup::UIDONLY_SHARD_ID; +use crate::store::tantivy::envelope::ENVELOPE_MANAGER; +use crate::utils::compute_content_hash; +use async_imap::types::{PartialRange, ResponseLimits, UidOnlyUnsolicitedResponse}; +use async_imap::Session; +use futures::TryStreamExt; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{self, File, OpenOptions}; +use std::future::Future; +use std::io::{Read, Write}; +use std::ops::RangeInclusive; +use std::path::{Path, PathBuf}; +use std::sync::{ + atomic::{AtomicU64, Ordering}, + LazyLock, +}; +use std::time::{Duration, Instant}; +use tokio_util::sync::CancellationToken; + +const BODY_QUERY: &str = "(UID RFC822.SIZE BODY.PEEK[])"; +const INVENTORY_QUERY: &str = "(UID RFC822.SIZE)"; +const MAX_NETWORK_RETRIES: u32 = 3; +#[cfg(not(test))] +const CANONICAL_CLEANUP_GRACE: Duration = Duration::from_secs(5); +#[cfg(test)] +const CANONICAL_CLEANUP_GRACE: Duration = Duration::from_millis(100); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct AcquisitionLimits { + pub max_messages: usize, + pub max_total_bytes: u64, + pub max_literal_bytes: u64, + pub max_response_bytes: u64, + pub max_runtime: Duration, + pub max_disk_bytes: u64, + pub page_size: u32, +} + +impl AcquisitionLimits { + pub fn for_account(account: &AccountModel) -> Self { + let literal = account.max_email_size_bytes.unwrap_or(100 * 1024 * 1024); + Self { + max_messages: 1_000_000, + max_total_bytes: 100 * 1024 * 1024 * 1024, + max_literal_bytes: literal, + max_response_bytes: literal.saturating_add(1024 * 1024), + max_runtime: Duration::from_secs(6 * 60 * 60), + max_disk_bytes: 120 * 1024 * 1024 * 1024, + page_size: account.download_batch_size.unwrap_or(30).max(1), + } + } + + pub fn response_limits(self) -> BichonResult { + let response = usize::try_from(self.max_response_bytes).map_err(|_| { + raise_error!( + "response ceiling does not fit usize".into(), + ErrorCode::InvalidParameter + ) + })?; + let literal = usize::try_from(self.max_literal_bytes).map_err(|_| { + raise_error!( + "literal ceiling does not fit usize".into(), + ErrorCode::InvalidParameter + ) + })?; + if literal == 0 || response < literal { + return Err(raise_error!( + "UIDONLY response limits must be nonzero and response >= literal".into(), + ErrorCode::InvalidParameter + )); + } + Ok(ResponseLimits::new(response, literal)) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct AcquisitionIdentity { + pub endpoint: String, + pub account_id: u64, + pub canonical_mailbox: String, +} + +impl AcquisitionIdentity { + pub fn from_account(account: &AccountModel, mailbox: &MailBox) -> BichonResult { + let imap = account.imap.as_ref().ok_or_else(|| { + raise_error!( + "IMAP account has no endpoint".into(), + ErrorCode::MissingConfiguration + ) + })?; + Ok(Self { + endpoint: format!("{}:{}", imap.host.to_ascii_lowercase(), imap.port), + account_id: account.id, + canonical_mailbox: mailbox.name.clone(), + }) + } + + fn storage_key(&self) -> String { + let mut hasher = blake3::Hasher::new(); + hasher.update(self.endpoint.as_bytes()); + hasher.update(&[0]); + hasher.update(&self.account_id.to_be_bytes()); + hasher.update(&[0]); + hasher.update(self.canonical_mailbox.as_bytes()); + hasher.finalize().to_hex().to_string() + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "snake_case")] +pub(crate) enum UidState { + Missing, + Pending, + Projecting { + blob_hash: String, + bytes: u64, + canonical_bytes: u64, + }, + Committed { + blob_hash: String, + bytes: u64, + #[serde(default)] + canonical_bytes: u64, + #[serde(default)] + envelope_id: Option, + }, + Vanished, + Failed { + reason: String, + }, + Oversized { + declared: u64, + limit: u64, + }, +} + +impl UidState { + fn reconciled(&self) -> bool { + matches!(self, Self::Committed { .. } | Self::Vanished) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct UidEntry { + pub declared_size: Option, + pub state: UidState, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct AcquisitionLedger { + pub identity: AcquisitionIdentity, + pub uid_validity: u32, + pub snapshot_end: u32, + pub checkpoint: Option, + pub entries: BTreeMap, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct Snapshot { + pub uid_validity: u32, + pub uid_next: u32, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct InventoryItem { + pub uid: u32, + pub size: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct InventoryPage { + pub items: Vec, + pub vanished: Vec>, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum FetchOutcome { + Message { + declared_size: Option, + raw: Vec, + }, + Vanished, + Missing, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct TransportFailure { + pub message: String, + pub network: bool, +} + +impl TransportFailure { + fn command(message: impl Into) -> Self { + Self { + message: message.into(), + network: false, + } + } +} + +#[allow(async_fn_in_trait)] +pub(crate) trait UidOnlyTransport { + async fn snapshot(&mut self, mailbox: &str) -> Result; + async fn inventory_page( + &mut self, + first_uid: u32, + snapshot_end: u32, + page_size: u32, + ) -> Result; + async fn fetch_uid(&mut self, uid: u32) -> Result; + async fn reconnect(&mut self) -> Result<(), TransportFailure>; +} + +#[allow(async_fn_in_trait)] +trait CanonicalArchive { + fn disk_budget(&self, raw: &[u8]) -> BichonResult; + + async fn project( + &mut self, + uid: u32, + raw: &[u8], + declared_size: Option, + shutdown: CancellationToken, + ) -> BichonResult; + + async fn verify(&self, uid: u32, blob_hash: &str, envelope_id: &str) -> BichonResult; + async fn rollback( + &mut self, + uid: u32, + content_hash: &str, + envelope_id: Option<&str>, + ) -> BichonResult<()>; +} + +struct BichonCanonicalArchive { + account_id: u64, + mailbox_id: u64, +} + +static UIDONLY_CANONICAL_WRITE_LOCK: LazyLock> = + LazyLock::new(|| tokio::sync::Mutex::new(())); + +fn canonical_attachment_records( + attachments: Vec, +) -> Vec { + let mut records: Vec<_> = attachments + .into_iter() + .filter(|attachment| !attachment.is_inline()) + .map(|attachment| CanonicalAttachmentRecord { + content_hash: attachment.content_hash, + name: attachment.filename, + size: attachment.size as u64, + content_type: attachment.file_type, + }) + .collect(); + records.sort(); + records +} + +impl BichonCanonicalArchive { + fn new(account_id: u64, mailbox_id: u64) -> Self { + Self { + account_id, + mailbox_id, + } + } + + fn envelope_id(&self, uid: u32, content_hash: &str) -> String { + Self::envelope_id_for(self.account_id, self.mailbox_id, uid, content_hash) + } + + fn envelope_id_for(account_id: u64, mailbox_id: u64, uid: u32, content_hash: &str) -> String { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"bichon-uidonly-envelope-v1"); + hasher.update(&account_id.to_be_bytes()); + hasher.update(&mailbox_id.to_be_bytes()); + hasher.update(&uid.to_be_bytes()); + hasher.update(content_hash.as_bytes()); + format!("uidonly-{}", hasher.finalize().to_hex()) + } + + fn reuse_projection( + uid: u32, + expected_hash: &str, + existing: crate::store::tantivy::envelope::CanonicalProjectionRecord, + ) -> BichonResult { + if existing.shard_id != UIDONLY_SHARD_ID { + return Err(raise_error!( + format!( + "UID {uid} is occupied by a non-UIDONLY canonical record (shard {})", + existing.shard_id + ), + ErrorCode::Incompatible + )); + } + if existing.content_hash != expected_hash { + return Err(raise_error!( + format!( + "canonical UID {uid} has content hash {}, expected {expected_hash}", + existing.content_hash + ), + ErrorCode::Incompatible + )); + } + Ok(CanonicalProjection { + envelope_id: existing.envelope_id, + content_hash: existing.content_hash, + }) + } +} + +impl CanonicalArchive for BichonCanonicalArchive { + fn disk_budget(&self, raw: &[u8]) -> BichonResult { + (raw.len() as u64) + .checked_mul(4) + .and_then(|bytes| bytes.checked_add(64 * 1024)) + .ok_or_else(|| { + raise_error!( + "canonical projection disk budget overflow".into(), + ErrorCode::PayloadTooLarge + ) + }) + } + + async fn project( + &mut self, + uid: u32, + raw: &[u8], + _declared_size: Option, + shutdown: CancellationToken, + ) -> BichonResult { + let account_id = self.account_id; + let mailbox_id = self.mailbox_id; + let body = raw.to_vec(); + let task = tokio::spawn(async move { + // The task owns the body, shutdown token, and serialization guard. + // Dropping its JoinHandle detaches a self-cleaning operation; it + // does not drop the projection future around spawn_blocking I/O. + let _write_guard = UIDONLY_CANONICAL_WRITE_LOCK.lock().await; + if shutdown.is_cancelled() { + return Err(raise_error!( + "UIDONLY canonical projection cancelled".into(), + ErrorCode::InternalError + )); + } + let expected_hash = compute_content_hash(&body); + if let Some(existing) = + ENVELOPE_MANAGER.get_projection_by_uid(account_id, mailbox_id, uid)? + { + return Self::reuse_projection(uid, &expected_hash, existing); + } + let size = u32::try_from(body.len()).map_err(|_| { + raise_error!( + format!("UID {uid} literal length does not fit Bichon's envelope size field"), + ErrorCode::PayloadTooLarge + ) + })?; + let envelope_id = Self::envelope_id_for(account_id, mailbox_id, uid, &expected_hash); + project_uidonly_message( + &body, + uid, + size, + 0, + account_id, + mailbox_id, + envelope_id, + shutdown, + ) + .await + }); + task.await + .map_err(|error| raise_error!(format!("{error:#?}"), ErrorCode::InternalError))? + } + + async fn verify(&self, uid: u32, blob_hash: &str, envelope_id: &str) -> BichonResult { + let Some(record) = + ENVELOPE_MANAGER.get_projection_by_uid(self.account_id, self.mailbox_id, uid)? + else { + return Ok(false); + }; + if record.envelope_id != envelope_id || record.content_hash != blob_hash { + return Ok(false); + } + if record.shard_id != UIDONLY_SHARD_ID { + return Ok(false); + } + let expected_attachments = canonical_attachment_records(record.attachments); + if ATTACHMENT_MANAGER.canonical_records_by_envelope(self.account_id, envelope_id)? + != expected_attachments + { + return Ok(false); + } + let (envelope, raw) = match reattach_eml_content(self.account_id, envelope_id.to_string()) { + Ok(value) => value, + Err(error) if error.code() == ErrorCode::ResourceNotFound => return Ok(false), + Err(error) => return Err(error), + }; + if expected_attachments.len() != envelope.regular_attachment_count { + return Ok(false); + } + Ok(compute_content_hash(&raw) == blob_hash) + } + + async fn rollback( + &mut self, + uid: u32, + content_hash: &str, + envelope_id: Option<&str>, + ) -> BichonResult<()> { + let _write_guard = UIDONLY_CANONICAL_WRITE_LOCK.lock().await; + let envelope_id = envelope_id + .map(ToOwned::to_owned) + .unwrap_or_else(|| self.envelope_id(uid, content_hash)); + rollback_uidonly_message(self.account_id, &envelope_id, content_hash).await + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct AcquisitionReport { + pub uid_validity: u32, + pub planned: u64, + pub processed: u64, + pub checkpoint: Option, + pub success: bool, + pub states: BTreeMap, + #[cfg(test)] + state_bytes_written: u64, +} + +struct DurableArchive { + epoch_dir: PathBuf, + ledger_path: PathBuf, + ledger_entries_dir: PathBuf, + limits: AcquisitionLimits, + disk_bytes: AtomicU64, + #[cfg(test)] + state_bytes_written: AtomicU64, +} + +#[derive(Serialize, Deserialize)] +struct LedgerMetadata { + identity: AcquisitionIdentity, + uid_validity: u32, + snapshot_end: u32, + checkpoint: Option, +} + +#[derive(Serialize, Deserialize)] +struct StagingRecord { + identity: AcquisitionIdentity, + uid_validity: u32, + uid: u32, + blob_hash: String, + bytes: u64, +} + +impl DurableArchive { + fn open( + root: &Path, + identity: &AcquisitionIdentity, + uid_validity: u32, + limits: AcquisitionLimits, + ) -> BichonResult { + let identity_dir = root.join(identity.storage_key()); + fs::create_dir_all(&identity_dir).map_err(io_error)?; + let epoch_marker = identity_dir.join("current-uidvalidity"); + if epoch_marker.exists() { + let existing = fs::read_to_string(&epoch_marker).map_err(io_error)?; + if existing.trim() != uid_validity.to_string() { + // The download flow performs Bichon's Message-ID based + // UIDVALIDITY reconciliation before entering acquisition. A + // new epoch must not reuse the prior UID ledger, but it also + // must not replace that existing reconciliation with a + // campaign-specific terminal error. + atomic_write(&epoch_marker, uid_validity.to_string().as_bytes())?; + } + } else { + atomic_write(&epoch_marker, uid_validity.to_string().as_bytes())?; + } + + let epoch_dir = identity_dir.join(uid_validity.to_string()); + fs::create_dir_all(epoch_dir.join("blobs")).map_err(io_error)?; + fs::create_dir_all(epoch_dir.join("records")).map_err(io_error)?; + let ledger_entries_dir = epoch_dir.join("ledger-entries"); + fs::create_dir_all(&ledger_entries_dir).map_err(io_error)?; + let ledger_path = epoch_dir.join("ledger.json"); + let disk_bytes = directory_size(&identity_dir)?; + if disk_bytes > limits.max_disk_bytes { + return Err(raise_error!( + format!( + "UIDONLY disk ceiling {} bytes already exceeded", + limits.max_disk_bytes + ), + ErrorCode::PayloadTooLarge + )); + } + Ok(Self { + epoch_dir, + ledger_path, + ledger_entries_dir, + limits, + disk_bytes: AtomicU64::new(disk_bytes), + #[cfg(test)] + state_bytes_written: AtomicU64::new(0), + }) + } + + fn reserve_disk(&self, additional: u64) -> BichonResult<()> { + self.disk_bytes + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + let next = current.checked_add(additional)?; + (next <= self.limits.max_disk_bytes).then_some(next) + }) + .map(|_| ()) + .map_err(|_| { + raise_error!( + format!( + "UIDONLY disk ceiling {} bytes exceeded", + self.limits.max_disk_bytes + ), + ErrorCode::PayloadTooLarge + ) + }) + } + + fn release_disk(&self, bytes: u64) { + self.disk_bytes.fetch_sub(bytes, Ordering::AcqRel); + } + + fn load_or_create( + &self, + identity: AcquisitionIdentity, + uid_validity: u32, + snapshot_end: u32, + ) -> BichonResult { + if self.ledger_path.exists() { + let bytes = fs::read(&self.ledger_path).map_err(io_error)?; + let value: serde_json::Value = serde_json::from_slice(&bytes).map_err(|e| { + raise_error!( + format!("invalid UIDONLY ledger: {e}"), + ErrorCode::InternalError + ) + })?; + let mut ledger = if value.get("entries").is_some() { + let legacy: AcquisitionLedger = serde_json::from_value(value).map_err(|e| { + raise_error!( + format!("invalid legacy UIDONLY ledger: {e}"), + ErrorCode::InternalError + ) + })?; + for (&uid, entry) in &legacy.entries { + self.persist_entry(uid, entry)?; + } + self.persist_metadata(&legacy)?; + legacy + } else { + let metadata: LedgerMetadata = serde_json::from_value(value).map_err(|e| { + raise_error!( + format!("invalid UIDONLY ledger metadata: {e}"), + ErrorCode::InternalError + ) + })?; + AcquisitionLedger { + identity: metadata.identity, + uid_validity: metadata.uid_validity, + snapshot_end: metadata.snapshot_end, + checkpoint: metadata.checkpoint, + entries: BTreeMap::new(), + } + }; + for file in fs::read_dir(&self.ledger_entries_dir).map_err(io_error)? { + let file = file.map_err(io_error)?; + if !file.file_type().map_err(io_error)?.is_file() { + continue; + } + let uid = file + .path() + .file_stem() + .and_then(|stem| stem.to_str()) + .and_then(|stem| stem.parse::().ok()) + .ok_or_else(|| { + raise_error!( + "invalid UIDONLY ledger entry filename".into(), + ErrorCode::InternalError + ) + })?; + let entry = + serde_json::from_slice::(&fs::read(file.path()).map_err(io_error)?) + .map_err(|e| { + raise_error!( + format!("invalid UIDONLY ledger entry for UID {uid}: {e}"), + ErrorCode::InternalError + ) + })?; + ledger.entries.insert(uid, entry); + } + if ledger.identity != identity || ledger.uid_validity != uid_validity { + return Err(raise_error!( + "UIDONLY ledger identity mismatch".into(), + ErrorCode::Incompatible + )); + } + return Ok(ledger); + } + let ledger = AcquisitionLedger { + identity, + uid_validity, + snapshot_end, + checkpoint: None, + entries: BTreeMap::new(), + }; + self.persist_metadata(&ledger)?; + Ok(ledger) + } + + fn persist_metadata(&self, ledger: &AcquisitionLedger) -> BichonResult<()> { + let bytes = serde_json::to_vec(&LedgerMetadata { + identity: ledger.identity.clone(), + uid_validity: ledger.uid_validity, + snapshot_end: ledger.snapshot_end, + checkpoint: ledger.checkpoint, + }) + .map_err(|e| { + raise_error!( + format!("cannot serialize UIDONLY ledger metadata: {e}"), + ErrorCode::InternalError + ) + })?; + let previous = fs::metadata(&self.ledger_path) + .map(|metadata| metadata.len()) + .unwrap_or(0); + self.reserve_disk((bytes.len() as u64).saturating_sub(previous))?; + self.record_state_write(bytes.len() as u64); + atomic_write(&self.ledger_path, &bytes) + } + + fn persist_entry(&self, uid: u32, entry: &UidEntry) -> BichonResult<()> { + #[cfg(test)] + if matches!(entry.state, UidState::Committed { .. }) { + let failpoint = self.epoch_dir.join("fail-next-committed-entry-persist"); + if failpoint.exists() { + fs::remove_file(failpoint).map_err(io_error)?; + return Err(raise_error!( + "synthetic committed ledger persist failure".into(), + ErrorCode::InternalError + )); + } + } + let path = self.ledger_entries_dir.join(format!("{uid}.json")); + let bytes = serde_json::to_vec(entry).map_err(|e| { + raise_error!( + format!("cannot serialize UIDONLY ledger entry {uid}: {e}"), + ErrorCode::InternalError + ) + })?; + let previous = fs::metadata(&path) + .map(|metadata| metadata.len()) + .unwrap_or(0); + self.reserve_disk((bytes.len() as u64).saturating_sub(previous))?; + self.record_state_write(bytes.len() as u64); + atomic_write(&path, &bytes) + } + + #[cfg(test)] + fn record_state_write(&self, bytes: u64) { + self.state_bytes_written.fetch_add(bytes, Ordering::Relaxed); + } + + #[cfg(not(test))] + fn record_state_write(&self, _bytes: u64) {} + + fn commit_raw( + &self, + ledger: &AcquisitionLedger, + uid: u32, + raw: &[u8], + ) -> BichonResult<(String, u64)> { + let hash = blake3::hash(raw).to_hex().to_string(); + let blob_path = self.epoch_dir.join("blobs").join(&hash); + let record_path = self.epoch_dir.join("records").join(format!("{uid}.json")); + let blob_was_present = blob_path.exists(); + if !blob_was_present { + self.reserve_disk(raw.len() as u64)?; + } + + if blob_was_present { + let mut existing = Vec::new(); + File::open(&blob_path) + .and_then(|mut f| f.read_to_end(&mut existing)) + .map_err(io_error)?; + if blake3::hash(&existing).to_hex().as_str() != hash || existing != raw { + return Err(raise_error!( + format!("stored blob verification failed for UID {uid}"), + ErrorCode::InternalError + )); + } + } else { + atomic_write(&blob_path, raw)?; + let stored = fs::read(&blob_path).map_err(io_error)?; + if stored.len() != raw.len() || blake3::hash(&stored).to_hex().as_str() != hash { + return Err(raise_error!( + format!("durable blob readback failed for UID {uid}"), + ErrorCode::InternalError + )); + } + } + + let record = serde_json::to_vec_pretty(&StagingRecord { + identity: ledger.identity.clone(), + uid_validity: ledger.uid_validity, + uid, + blob_hash: hash.clone(), + bytes: raw.len() as u64, + }) + .map_err(|e| { + raise_error!( + format!("cannot serialize logical record: {e}"), + ErrorCode::InternalError + ) + })?; + let previous_record = fs::metadata(&record_path) + .map(|metadata| metadata.len()) + .unwrap_or(0); + let record_additional = (record.len() as u64).saturating_sub(previous_record); + self.reserve_disk(record_additional)?; + atomic_write(&record_path, &record)?; + + Ok((hash, raw.len() as u64)) + } + + fn reclaim_committed_staging(&self, ledger: &AcquisitionLedger) -> BichonResult<()> { + for (&uid, entry) in &ledger.entries { + if matches!(entry.state, UidState::Committed { .. }) { + let path = self.epoch_dir.join("records").join(format!("{uid}.json")); + if let Ok(metadata) = fs::metadata(&path) { + fs::remove_file(&path).map_err(io_error)?; + self.disk_bytes.fetch_sub(metadata.len(), Ordering::AcqRel); + } + } + } + + let mut referenced = BTreeSet::new(); + for entry in fs::read_dir(self.epoch_dir.join("records")).map_err(io_error)? { + let entry = entry.map_err(io_error)?; + if !entry.file_type().map_err(io_error)?.is_file() { + continue; + } + let record: StagingRecord = serde_json::from_slice( + &fs::read(entry.path()).map_err(io_error)?, + ) + .map_err(|error| { + raise_error!( + format!("invalid UIDONLY staging record: {error}"), + ErrorCode::InternalError + ) + })?; + referenced.insert(record.blob_hash); + } + + for entry in fs::read_dir(self.epoch_dir.join("blobs")).map_err(io_error)? { + let entry = entry.map_err(io_error)?; + if !entry.file_type().map_err(io_error)?.is_file() { + continue; + } + let hash = entry.file_name().to_string_lossy().to_string(); + if !referenced.contains(&hash) { + let metadata = entry.metadata().map_err(io_error)?; + fs::remove_file(entry.path()).map_err(io_error)?; + self.disk_bytes.fetch_sub(metadata.len(), Ordering::AcqRel); + } + } + File::open(&self.epoch_dir) + .and_then(|directory| directory.sync_all()) + .map_err(io_error) + } +} + +fn io_error(error: std::io::Error) -> crate::error::BichonError { + raise_error!( + format!("UIDONLY durable I/O failed: {error}"), + ErrorCode::InternalError + ) +} + +fn atomic_write(path: &Path, bytes: &[u8]) -> BichonResult<()> { + let parent = path.parent().ok_or_else(|| { + raise_error!( + "atomic write path has no parent".into(), + ErrorCode::InternalError + ) + })?; + fs::create_dir_all(parent).map_err(io_error)?; + let temp = parent.join(format!( + ".{}.{}.tmp", + path.file_name().unwrap_or_default().to_string_lossy(), + uuid::Uuid::new_v4() + )); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&temp) + .map_err(io_error)?; + let result = (|| -> std::io::Result<()> { + file.write_all(bytes)?; + file.sync_all()?; + fs::rename(&temp, path)?; + File::open(parent)?.sync_all()?; + Ok(()) + })(); + if result.is_err() { + let _ = fs::remove_file(&temp); + } + result.map_err(io_error) +} + +fn directory_size(path: &Path) -> BichonResult { + if !path.exists() { + return Ok(0); + } + let mut total = 0u64; + for entry in fs::read_dir(path).map_err(io_error)? { + let entry = entry.map_err(io_error)?; + let metadata = entry.metadata().map_err(io_error)?; + total = total.saturating_add(if metadata.is_dir() { + directory_size(&entry.path())? + } else { + metadata.len() + }); + } + Ok(total) +} + +fn cleanup_uidonly_state( + root: &Path, + matches: impl Fn(&AcquisitionIdentity) -> bool, +) -> BichonResult { + if !root.exists() { + return Ok(0); + } + let mut remove = Vec::new(); + for identity_dir in fs::read_dir(root).map_err(io_error)? { + let identity_dir = identity_dir.map_err(io_error)?; + if !identity_dir.file_type().map_err(io_error)?.is_dir() { + continue; + } + let mut identity = None; + for epoch in fs::read_dir(identity_dir.path()).map_err(io_error)? { + let epoch = epoch.map_err(io_error)?; + if !epoch.file_type().map_err(io_error)?.is_dir() { + continue; + } + let ledger = epoch.path().join("ledger.json"); + if !ledger.exists() { + continue; + } + let value: serde_json::Value = + serde_json::from_slice(&fs::read(&ledger).map_err(io_error)?).map_err(|error| { + raise_error!( + format!( + "invalid UIDONLY cleanup ledger {}: {error}", + ledger.display() + ), + ErrorCode::InternalError + ) + })?; + identity = Some( + serde_json::from_value::( + value.get("identity").cloned().ok_or_else(|| { + raise_error!( + format!( + "UIDONLY cleanup ledger {} has no identity", + ledger.display() + ), + ErrorCode::InternalError + ) + })?, + ) + .map_err(|error| { + raise_error!( + format!( + "invalid UIDONLY cleanup identity {}: {error}", + ledger.display() + ), + ErrorCode::InternalError + ) + })?, + ); + break; + } + if identity.as_ref().map(&matches).unwrap_or(false) { + remove.push(identity_dir.path()); + } + } + for path in &remove { + fs::remove_dir_all(path).map_err(io_error)?; + } + if !remove.is_empty() { + File::open(root) + .and_then(|directory| directory.sync_all()) + .map_err(io_error)?; + } + Ok(remove.len()) +} + +pub(crate) fn cleanup_uidonly_account_state(root: &Path, account_id: u64) -> BichonResult { + cleanup_uidonly_state(root, |identity| identity.account_id == account_id) +} + +pub(crate) fn cleanup_uidonly_mailbox_state( + root: &Path, + account_id: u64, + canonical_mailboxes: &BTreeSet, +) -> BichonResult { + cleanup_uidonly_state(root, |identity| { + identity.account_id == account_id + && canonical_mailboxes.contains(&identity.canonical_mailbox) + }) +} + +fn validate_runtime( + started: Instant, + limits: AcquisitionLimits, + token: &CancellationToken, +) -> BichonResult<()> { + if token.is_cancelled() { + return Err(raise_error!( + "UIDONLY acquisition cancelled".into(), + ErrorCode::InternalError + )); + } + if started.elapsed() > limits.max_runtime { + return Err(raise_error!( + "UIDONLY acquisition runtime ceiling exceeded".into(), + ErrorCode::RequestTimeout + )); + } + Ok(()) +} + +async fn bounded_transport( + operation: F, + started: Instant, + limits: AcquisitionLimits, + token: &CancellationToken, +) -> BichonResult +where + F: Future>, +{ + validate_runtime(started, limits, token)?; + let remaining = limits + .max_runtime + .checked_sub(started.elapsed()) + .ok_or_else(|| { + raise_error!( + "UIDONLY acquisition runtime ceiling exceeded".into(), + ErrorCode::RequestTimeout + ) + })?; + tokio::select! { + _ = token.cancelled() => Err(raise_error!( + "UIDONLY acquisition cancelled".into(), + ErrorCode::InternalError + )), + _ = tokio::time::sleep(remaining) => Err(raise_error!( + "UIDONLY acquisition runtime ceiling exceeded".into(), + ErrorCode::RequestTimeout + )), + result = operation => result.map_err(transport_error), + } +} + +async fn bounded_canonical( + operation: F, + started: Instant, + limits: AcquisitionLimits, + token: &CancellationToken, + operation_shutdown: Option<&CancellationToken>, +) -> Result +where + F: Future>, +{ + validate_runtime(started, limits, token).map_err(BoundedCanonicalFailure::quiesced)?; + let remaining = limits + .max_runtime + .checked_sub(started.elapsed()) + .ok_or_else(|| { + BoundedCanonicalFailure::quiesced(raise_error!( + "UIDONLY acquisition runtime ceiling exceeded".into(), + ErrorCode::RequestTimeout + )) + })?; + tokio::pin!(operation); + enum Boundary { + Cancelled, + Runtime, + } + let boundary = tokio::select! { + _ = token.cancelled() => Some(Boundary::Cancelled), + _ = tokio::time::sleep(remaining) => Some(Boundary::Runtime), + result = &mut operation => return result.map_err(BoundedCanonicalFailure::quiesced), + }; + if let Some(shutdown) = operation_shutdown { + shutdown.cancel(); + } + // The production projection future awaits an owned, self-cleaning task + // that retains the UIDONLY write lock. Give it a bounded opportunity to + // finish. If OS-backed blocking I/O does not return, dropping the + // JoinHandle only detaches that owned task; later projection/rollback is + // serialized behind it until it observes shutdown and rolls itself back. + let cleanup_pending = tokio::time::timeout(CANONICAL_CLEANUP_GRACE, &mut operation) + .await + .is_err(); + let error = match boundary.unwrap() { + Boundary::Cancelled => raise_error!( + "UIDONLY acquisition cancelled during canonical projection".into(), + ErrorCode::InternalError + ), + Boundary::Runtime => raise_error!( + "UIDONLY acquisition runtime ceiling exceeded during canonical projection".into(), + ErrorCode::RequestTimeout + ), + }; + Err(BoundedCanonicalFailure { + error, + cleanup_pending, + }) +} + +struct BoundedCanonicalFailure { + error: BichonError, + cleanup_pending: bool, +} + +impl BoundedCanonicalFailure { + fn quiesced(error: BichonError) -> Self { + Self { + error, + cleanup_pending: false, + } + } +} + +impl From for BichonError { + fn from(failure: BoundedCanonicalFailure) -> Self { + failure.error + } +} + +async fn cleanup_canonical( + canonical: &mut C, + uid: u32, + content_hash: &str, + envelope_id: Option<&str>, +) -> BichonResult<()> { + tokio::time::timeout( + CANONICAL_CLEANUP_GRACE, + canonical.rollback(uid, content_hash, envelope_id), + ) + .await + .map_err(|_| { + raise_error!( + "UIDONLY canonical rollback timed out".into(), + ErrorCode::RequestTimeout + ) + })? +} + +async fn run_acquisition( + transport: &mut T, + canonical: &mut C, + mailbox: &str, + identity: AcquisitionIdentity, + root: &Path, + limits: AcquisitionLimits, + token: CancellationToken, +) -> BichonResult { + let started = Instant::now(); + let snapshot = bounded_transport(transport.snapshot(mailbox), started, limits, &token).await?; + let snapshot_end = snapshot.uid_next.saturating_sub(1); + let archive = DurableArchive::open(root, &identity, snapshot.uid_validity, limits)?; + let mut ledger = archive.load_or_create(identity, snapshot.uid_validity, snapshot_end)?; + let existing_canonical_bytes = ledger + .entries + .values() + .filter_map(|entry| match entry.state { + UidState::Committed { + canonical_bytes, .. + } + | UidState::Projecting { + canonical_bytes, .. + } => Some(canonical_bytes), + _ => None, + }) + .fold(0u64, u64::saturating_add); + archive.reserve_disk(existing_canonical_bytes)?; + + // A Projecting entry proves the reservation was durable before canonical + // writes started, but not that projection reached its success barrier. + // Reconcile it idempotently before retrying the UID in this run. + let interrupted: Vec<_> = ledger + .entries + .iter() + .filter_map(|(&uid, entry)| match &entry.state { + UidState::Projecting { + blob_hash, + canonical_bytes, + .. + } => Some((uid, blob_hash.clone(), *canonical_bytes)), + _ => None, + }) + .collect(); + for (uid, blob_hash, canonical_bytes) in interrupted { + cleanup_canonical(canonical, uid, &blob_hash, None).await?; + archive.release_disk(canonical_bytes); + ledger.entries.get_mut(&uid).unwrap().state = UidState::Missing; + archive.persist_entry(uid, &ledger.entries[&uid])?; + ledger.checkpoint = None; + } + if ledger.checkpoint.is_none() { + archive.persist_metadata(&ledger)?; + } + + let mut invalid_committed = BTreeSet::new(); + let committed: Vec<_> = ledger + .entries + .iter() + .filter_map(|(&uid, entry)| match &entry.state { + UidState::Committed { + blob_hash, + canonical_bytes, + envelope_id, + .. + } => Some(( + uid, + blob_hash.clone(), + *canonical_bytes, + envelope_id.clone(), + )), + _ => None, + }) + .collect(); + for (uid, blob_hash, canonical_bytes, envelope_id) in committed { + let valid = match envelope_id.as_deref() { + Some(envelope_id) => { + bounded_canonical( + canonical.verify(uid, &blob_hash, envelope_id), + started, + limits, + &token, + None, + ) + .await? + } + None => false, + }; + if !valid { + cleanup_canonical(canonical, uid, &blob_hash, envelope_id.as_deref()).await?; + archive.release_disk(canonical_bytes); + ledger.entries.get_mut(&uid).unwrap().state = UidState::Failed { + reason: "committed canonical record or blob failed restart validation".into(), + }; + archive.persist_entry(uid, &ledger.entries[&uid])?; + ledger.checkpoint = None; + invalid_committed.insert(uid); + } + } + if !invalid_committed.is_empty() { + archive.persist_metadata(&ledger)?; + } + // A restart continues the original fixed snapshot even if UIDNEXT grew. + let snapshot_end = ledger.snapshot_end; + + let page_size = limits.page_size.max(1); + let mut first_uid = 1u32; + while first_uid <= snapshot_end { + validate_runtime(started, limits, &token)?; + let page = bounded_transport( + transport.inventory_page(first_uid, snapshot_end, page_size), + started, + limits, + &token, + ) + .await?; + for range in page.vanished { + let mut changed = Vec::new(); + for (&uid, entry) in ledger.entries.range_mut(range) { + if !matches!(entry.state, UidState::Committed { .. } | UidState::Vanished) { + entry.state = UidState::Vanished; + changed.push(uid); + } + } + for uid in changed { + archive.persist_entry(uid, &ledger.entries[&uid])?; + } + } + if page.items.is_empty() { + break; + } + let mut previous = first_uid.saturating_sub(1); + for item in page.items { + if item.uid < first_uid || item.uid > snapshot_end || item.uid <= previous { + return Err(raise_error!( + "UIDONLY inventory was not strictly ascending within the fixed snapshot".into(), + ErrorCode::ImapUnexpectedResult + )); + } + previous = item.uid; + let mut changed = false; + let entry = ledger.entries.entry(item.uid).or_insert_with(|| { + changed = true; + UidEntry { + declared_size: item.size, + state: UidState::Missing, + } + }); + if entry.declared_size.is_none() && item.size.is_some() { + entry.declared_size = item.size; + changed = true; + } + if changed { + archive.persist_entry(item.uid, entry)?; + } + } + first_uid = previous.checked_add(1).ok_or_else(|| { + raise_error!( + "UID cursor overflow".into(), + ErrorCode::ImapUnexpectedResult + ) + })?; + if ledger.entries.len() > limits.max_messages { + return Err(raise_error!( + format!("UIDONLY message ceiling {} exceeded", limits.max_messages), + ErrorCode::PayloadTooLarge + )); + } + } + + let uids: Vec = ledger.entries.keys().copied().collect(); + let mut total_bytes = ledger + .entries + .values() + .filter_map(|entry| match entry.state { + UidState::Committed { bytes, .. } => Some(bytes), + _ => None, + }) + .sum::(); + + for uid in uids { + validate_runtime(started, limits, &token)?; + if invalid_committed.contains(&uid) { + continue; + } + let current = &ledger.entries[&uid]; + if current.state.reconciled() || matches!(current.state, UidState::Oversized { .. }) { + continue; + } + ledger.entries.get_mut(&uid).unwrap().state = UidState::Pending; + archive.persist_entry(uid, &ledger.entries[&uid])?; + + let mut retry = 0; + let outcome = loop { + match bounded_transport(transport.fetch_uid(uid), started, limits, &token).await { + Ok(outcome) => break Ok(outcome), + Err(error) + if error.code() == ErrorCode::NetworkError && retry < MAX_NETWORK_RETRIES => + { + retry += 1; + bounded_transport(transport.reconnect(), started, limits, &token).await?; + let resumed = + bounded_transport(transport.snapshot(mailbox), started, limits, &token) + .await?; + if resumed.uid_validity != ledger.uid_validity { + break Err(raise_error!( + format!( + "UIDVALIDITY changed during reconnect from {} to {}", + ledger.uid_validity, resumed.uid_validity + ), + ErrorCode::Incompatible + )); + } + } + Err(error) => break Err(error), + } + }; + + let outcome = match outcome { + Ok(outcome) => outcome, + Err(error) => { + ledger.entries.get_mut(&uid).unwrap().state = UidState::Failed { + reason: error.to_string(), + }; + archive.persist_entry(uid, &ledger.entries[&uid])?; + continue; + } + }; + + match outcome { + FetchOutcome::Vanished => { + ledger.entries.get_mut(&uid).unwrap().state = UidState::Vanished; + } + FetchOutcome::Missing => { + ledger.entries.get_mut(&uid).unwrap().state = UidState::Missing; + } + FetchOutcome::Message { declared_size, raw } => { + let bytes = raw.len() as u64; + let declared = declared_size.or(ledger.entries[&uid].declared_size); + if bytes > limits.max_literal_bytes { + ledger.entries.get_mut(&uid).unwrap().state = UidState::Oversized { + declared: bytes, + limit: limits.max_literal_bytes, + }; + } else if total_bytes.saturating_add(bytes) > limits.max_total_bytes { + ledger.entries.get_mut(&uid).unwrap().state = UidState::Failed { + reason: format!( + "UIDONLY total byte ceiling {} exceeded", + limits.max_total_bytes + ), + }; + } else { + match archive.commit_raw(&ledger, uid, &raw) { + Ok((blob_hash, bytes)) => { + let budget = canonical.disk_budget(&raw)?; + if let Err(error) = archive.reserve_disk(budget) { + ledger.entries.get_mut(&uid).unwrap().state = UidState::Failed { + reason: error.to_string(), + }; + } else { + ledger.entries.get_mut(&uid).unwrap().state = + UidState::Projecting { + blob_hash: blob_hash.clone(), + bytes, + canonical_bytes: budget, + }; + if let Err(error) = + archive.persist_entry(uid, &ledger.entries[&uid]) + { + archive.release_disk(budget); + return Err(error); + } + let projection_shutdown = token.child_token(); + let projected = bounded_canonical( + canonical.project( + uid, + &raw, + declared, + projection_shutdown.clone(), + ), + started, + limits, + &token, + Some(&projection_shutdown), + ) + .await; + match projected { + Ok(projection) => { + let verified = if projection.content_hash == blob_hash { + bounded_canonical( + canonical.verify( + uid, + &blob_hash, + &projection.envelope_id, + ), + started, + limits, + &token, + None, + ) + .await + } else { + Ok(false) + }; + match verified { + Ok(true) => { + ledger.entries.get_mut(&uid).unwrap().state = + UidState::Committed { + blob_hash: blob_hash.clone(), + bytes, + canonical_bytes: budget, + envelope_id: Some( + projection.envelope_id.clone(), + ), + }; + if let Err(error) = archive + .persist_entry(uid, &ledger.entries[&uid]) + { + cleanup_canonical( + canonical, + uid, + &blob_hash, + Some(&projection.envelope_id), + ) + .await?; + archive.release_disk(budget); + return Err(error); + } + total_bytes = total_bytes.saturating_add(bytes); + continue; + } + Ok(false) => { + cleanup_canonical( + canonical, + uid, + &blob_hash, + Some(&projection.envelope_id), + ) + .await?; + archive.release_disk(budget); + ledger.entries.get_mut(&uid).unwrap().state = + UidState::Failed { + reason: "canonical projection verification failed" + .into(), + }; + } + Err(failure) => { + cleanup_canonical( + canonical, + uid, + &blob_hash, + Some(&projection.envelope_id), + ) + .await?; + archive.release_disk(budget); + return Err(failure.error); + } + } + } + Err(failure) => { + if !failure.cleanup_pending { + cleanup_canonical(canonical, uid, &blob_hash, None) + .await?; + } + archive.release_disk(budget); + let error = failure.error; + if error.code() == ErrorCode::RequestTimeout + || error.to_string().contains("cancelled") + { + return Err(error); + } + ledger.entries.get_mut(&uid).unwrap().state = + UidState::Failed { + reason: error.to_string(), + }; + } + } + } + } + Err(error) => { + ledger.entries.get_mut(&uid).unwrap().state = UidState::Failed { + reason: error.to_string(), + }; + } + } + } + } + } + archive.persist_entry(uid, &ledger.entries[&uid])?; + } + + let final_committed: Vec<_> = ledger + .entries + .iter() + .filter_map(|(&uid, entry)| match &entry.state { + UidState::Committed { + blob_hash, + canonical_bytes, + envelope_id: Some(envelope_id), + .. + } => Some(( + uid, + blob_hash.clone(), + *canonical_bytes, + envelope_id.clone(), + )), + _ => None, + }) + .collect(); + for (uid, blob_hash, canonical_bytes, envelope_id) in final_committed { + if !bounded_canonical( + canonical.verify(uid, &blob_hash, &envelope_id), + started, + limits, + &token, + None, + ) + .await? + { + cleanup_canonical(canonical, uid, &blob_hash, Some(&envelope_id)).await?; + archive.release_disk(canonical_bytes); + ledger.entries.get_mut(&uid).unwrap().state = UidState::Failed { + reason: "canonical record failed final checkpoint revalidation".into(), + }; + archive.persist_entry(uid, &ledger.entries[&uid])?; + ledger.checkpoint = None; + } + } + + let planned = ledger.entries.len() as u64; + let processed = ledger + .entries + .values() + .filter(|entry| entry.state.reconciled()) + .count() as u64; + let success = processed == planned; + if success { + ledger.checkpoint = Some(snapshot_end); + archive.persist_metadata(&ledger)?; + } else if ledger.checkpoint.is_some() { + ledger.checkpoint = None; + archive.persist_metadata(&ledger)?; + } + archive.reclaim_committed_staging(&ledger)?; + #[cfg(test)] + let state_bytes_written = archive.state_bytes_written.load(Ordering::Relaxed); + Ok(AcquisitionReport { + uid_validity: ledger.uid_validity, + planned, + processed, + checkpoint: ledger.checkpoint, + success, + states: ledger + .entries + .into_iter() + .map(|(uid, entry)| (uid, entry.state)) + .collect(), + #[cfg(test)] + state_bytes_written, + }) +} + +fn transport_error(error: TransportFailure) -> crate::error::BichonError { + raise_error!( + error.message, + if error.network { + ErrorCode::NetworkError + } else { + ErrorCode::ImapCommandFailed + } + ) +} + +struct SessionUidOnlyTransport { + account_id: u64, + session: Session>, + message_limit: Option, + response_limits: ResponseLimits, +} + +impl SessionUidOnlyTransport { + fn new( + account_id: u64, + session: Session>, + message_limit: Option, + response_limits: ResponseLimits, + ) -> Self { + Self { + account_id, + session, + message_limit, + response_limits, + } + } + + fn drain_vanished(&self) -> Vec> { + let mut vanished = Vec::new(); + while let Ok(response) = self.session.uidonly_responses.try_recv() { + if let UidOnlyUnsolicitedResponse::Vanished { uids, .. } = response { + for range in uids { + vanished.push(range); + } + } + } + vanished + } +} + +fn classify_transport(error: async_imap::error::Error) -> TransportFailure { + let network = match &error { + async_imap::error::Error::ConnectionLost => true, + async_imap::error::Error::Io(io) => matches!( + io.kind(), + std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::TimedOut + | std::io::ErrorKind::UnexpectedEof + ), + _ => false, + }; + TransportFailure { + message: format!("{error:#?}"), + network, + } +} + +impl UidOnlyTransport for SessionUidOnlyTransport { + async fn snapshot(&mut self, mailbox: &str) -> Result { + let selected = self + .session + .examine(mailbox) + .await + .map_err(classify_transport)?; + let uid_validity = selected + .uid_validity + .ok_or_else(|| TransportFailure::command("selected mailbox omitted UIDVALIDITY"))?; + let uid_next = selected + .uid_next + .ok_or_else(|| TransportFailure::command("selected mailbox omitted UIDNEXT"))?; + Ok(Snapshot { + uid_validity, + uid_next, + }) + } + + async fn inventory_page( + &mut self, + first_uid: u32, + snapshot_end: u32, + page_size: u32, + ) -> Result { + let page_size = self + .message_limit + .map(|limit| limit.min(page_size)) + .unwrap_or(page_size) + .max(1); + let partial = PartialRange::first(page_size).map_err(classify_transport)?; + let uid_set = format!("{first_uid}:{snapshot_end}"); + let mut stream = self + .session + .uid_fetch_uidonly_partial(&uid_set, INVENTORY_QUERY, partial) + .await + .map_err(classify_transport)?; + let mut items = Vec::new(); + while let Some(fetch) = stream.try_next().await.map_err(classify_transport)? { + items.push(InventoryItem { + uid: fetch.uid, + size: fetch.size.map(u64::from), + }); + } + drop(stream); + Ok(InventoryPage { + items, + vanished: self.drain_vanished(), + }) + } + + async fn fetch_uid(&mut self, uid: u32) -> Result { + let mut stream = self + .session + .uid_fetch_uidonly(uid.to_string(), BODY_QUERY) + .await + .map_err(classify_transport)?; + let mut result = None; + while let Some(fetch) = stream.try_next().await.map_err(classify_transport)? { + if result.is_some() || fetch.uid != uid { + return Err(TransportFailure::command(format!( + "unexpected UIDFETCH result while fetching UID {uid}" + ))); + } + result = Some(FetchOutcome::Message { + declared_size: fetch.size.map(u64::from), + raw: fetch + .body() + .ok_or_else(|| TransportFailure::command(format!("UID {uid} omitted BODY[]")))? + .to_vec(), + }); + } + drop(stream); + let vanished = self.drain_vanished(); + Ok(result.unwrap_or_else(|| { + if vanished.iter().any(|range| range.contains(&uid)) { + FetchOutcome::Vanished + } else { + FetchOutcome::Missing + } + })) + } + + async fn reconnect(&mut self) -> Result<(), TransportFailure> { + match ImapConnectionManager::build_acquisition(self.account_id, self.response_limits) + .await + .map_err(|e| TransportFailure { + message: e.to_string(), + network: e.code() == ErrorCode::NetworkError, + })? { + AcquisitionConnection::UidOnly { + session, + message_limit, + } => { + self.session = session; + self.message_limit = message_limit; + Ok(()) + } + AcquisitionConnection::Standard(_) => Err(TransportFailure::command( + "server stopped advertising UIDONLY after reconnect", + )), + } + } +} + +pub(crate) async fn acquire_bichon_mailbox( + account: &AccountModel, + mailbox: &MailBox, + session: Session>, + message_limit: Option, + root: &Path, + limits: AcquisitionLimits, + token: CancellationToken, +) -> BichonResult { + let identity = AcquisitionIdentity::from_account(account, mailbox)?; + let response_limits = limits.response_limits()?; + let mut transport = + SessionUidOnlyTransport::new(account.id, session, message_limit, response_limits); + let mut canonical = BichonCanonicalArchive::new(account.id, mailbox.id); + run_acquisition( + &mut transport, + &mut canonical, + &mailbox.encoded_name(), + identity, + root, + limits, + token, + ) + .await +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::database::{insert_impl, manager::DB_MANAGER}; + use crate::envelope::extractor::fail_uidonly_after_attachments; + use crate::imap::mock_server::{examine_response, MockImapServer}; + use crate::store::blob::BLOB_MANAGER; + use crate::store::tantivy::dedup::dedup_task; + use std::cell::Cell; + use std::collections::VecDeque; + use std::pin::Pin; + use std::task::{Context, Poll}; + use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + use tokio::net::TcpStream; + + #[derive(Debug)] + struct TestStream(TcpStream); + + impl AsyncRead for TestStream { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.0).poll_read(cx, buf) + } + } + + impl AsyncWrite for TestStream { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + Pin::new(&mut self.0).poll_write(cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.0).poll_flush(cx) + } + + fn poll_shutdown( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + Pin::new(&mut self.0).poll_shutdown(cx) + } + } + + impl SessionStream for TestStream {} + + struct FakeTransport { + snapshot: Snapshot, + inventory: Vec, + outcomes: BTreeMap>>, + vanished_on_inventory: BTreeSet, + expunge_after_first_page: Option, + reconnects: u32, + page_requests: Vec<(u32, u32, u32)>, + } + + impl UidOnlyTransport for FakeTransport { + async fn snapshot(&mut self, _mailbox: &str) -> Result { + Ok(self.snapshot) + } + + async fn inventory_page( + &mut self, + first_uid: u32, + end: u32, + page_size: u32, + ) -> Result { + self.page_requests.push((first_uid, end, page_size)); + let items = self + .inventory + .iter() + .filter(|item| item.uid >= first_uid && item.uid <= end) + .take(page_size as usize) + .cloned() + .collect(); + if self.page_requests.len() == 1 { + if let Some(uid) = self.expunge_after_first_page.take() { + self.inventory.retain(|item| item.uid != uid); + } + } + Ok(InventoryPage { + items, + vanished: std::mem::take(&mut self.vanished_on_inventory) + .into_iter() + .map(|uid| uid..=uid) + .collect(), + }) + } + + async fn fetch_uid(&mut self, uid: u32) -> Result { + self.outcomes + .get_mut(&uid) + .and_then(VecDeque::pop_front) + .unwrap_or(Ok(FetchOutcome::Missing)) + } + + async fn reconnect(&mut self) -> Result<(), TransportFailure> { + self.reconnects += 1; + Ok(()) + } + } + + struct HugeVanishedTransport; + + impl UidOnlyTransport for HugeVanishedTransport { + async fn snapshot(&mut self, _mailbox: &str) -> Result { + Ok(Snapshot { + uid_validity: 9, + uid_next: u32::MAX, + }) + } + + async fn inventory_page( + &mut self, + _first_uid: u32, + _snapshot_end: u32, + _page_size: u32, + ) -> Result { + Ok(InventoryPage { + items: Vec::new(), + vanished: vec![1..=u32::MAX], + }) + } + + async fn fetch_uid(&mut self, _uid: u32) -> Result { + panic!("vanished UIDs must not be fetched") + } + + async fn reconnect(&mut self) -> Result<(), TransportFailure> { + Ok(()) + } + } + + struct HangingTransport; + + impl UidOnlyTransport for HangingTransport { + async fn snapshot(&mut self, _mailbox: &str) -> Result { + std::future::pending().await + } + + async fn inventory_page( + &mut self, + _first_uid: u32, + _snapshot_end: u32, + _page_size: u32, + ) -> Result { + unreachable!() + } + + async fn fetch_uid(&mut self, _uid: u32) -> Result { + unreachable!() + } + + async fn reconnect(&mut self) -> Result<(), TransportFailure> { + unreachable!() + } + } + + #[derive(Default)] + struct FakeCanonicalArchive { + records: BTreeMap, + corrupt_blobs: BTreeSet, + fail_projection: BTreeSet, + hang_projection: BTreeSet, + disk_budget_override: Option, + projected_uids: Vec, + fail_verify_on_call: Option, + verify_calls: Cell, + active_projects: usize, + quiesced_projects: Vec, + } + + impl CanonicalArchive for FakeCanonicalArchive { + fn disk_budget(&self, raw: &[u8]) -> BichonResult { + Ok(self.disk_budget_override.unwrap_or(raw.len() as u64 + 128)) + } + + async fn project( + &mut self, + uid: u32, + raw: &[u8], + _declared_size: Option, + shutdown: CancellationToken, + ) -> BichonResult { + if self.hang_projection.contains(&uid) { + self.active_projects += 1; + shutdown.cancelled().await; + self.active_projects -= 1; + self.quiesced_projects.push(uid); + return Err(raise_error!( + "synthetic canonical projection cancelled".into(), + ErrorCode::InternalError + )); + } + if self.fail_projection.contains(&uid) { + return Err(raise_error!( + "synthetic canonical projection failure".into(), + ErrorCode::InternalError + )); + } + self.projected_uids.push(uid); + let projection = CanonicalProjection { + envelope_id: format!("envelope-{uid}"), + content_hash: compute_content_hash(raw), + }; + self.records.insert(uid, projection.clone()); + Ok(projection) + } + + async fn verify(&self, uid: u32, blob_hash: &str, envelope_id: &str) -> BichonResult { + let call = self.verify_calls.get() + 1; + self.verify_calls.set(call); + if self.fail_verify_on_call == Some(call) { + return Ok(false); + } + Ok(!self.corrupt_blobs.contains(&uid) + && self.records.get(&uid).is_some_and(|record| { + record.content_hash == blob_hash && record.envelope_id == envelope_id + })) + } + + async fn rollback( + &mut self, + uid: u32, + _content_hash: &str, + _envelope_id: Option<&str>, + ) -> BichonResult<()> { + self.records.remove(&uid); + Ok(()) + } + } + + fn limits() -> AcquisitionLimits { + AcquisitionLimits { + max_messages: 10_000, + max_total_bytes: 100 * 1024 * 1024, + max_literal_bytes: 10 * 1024 * 1024, + max_response_bytes: 11 * 1024 * 1024, + max_runtime: Duration::from_secs(600), + max_disk_bytes: 1024 * 1024 * 1024, + page_size: 2, + } + } + + #[test] + fn legacy_shard_record_is_never_reused_for_uidonly_projection() { + let error = BichonCanonicalArchive::reuse_projection( + 7, + "expected-hash", + crate::store::tantivy::envelope::CanonicalProjectionRecord { + envelope_id: "legacy-envelope".into(), + content_hash: "expected-hash".into(), + shard_id: 0, + attachments: Vec::new(), + }, + ) + .unwrap_err(); + assert_eq!(error.code(), ErrorCode::Incompatible); + assert!(error.to_string().contains("non-UIDONLY")); + } + + #[test] + fn attachment_verification_rejects_same_count_with_different_metadata() { + let expected = canonical_attachment_records(vec![AttachmentInfo { + file_type: "application/octet-stream".into(), + filename: Some("expected.bin".into()), + size: 4, + content_hash: "attachment-hash".into(), + ..Default::default() + }]); + let actual = vec![CanonicalAttachmentRecord { + content_hash: "attachment-hash".into(), + name: Some("wrong.bin".into()), + size: 4, + content_type: "application/octet-stream".into(), + }]; + assert_eq!(expected.len(), actual.len()); + assert_ne!(expected, actual); + } + + fn identity() -> AcquisitionIdentity { + AcquisitionIdentity { + endpoint: "imap.invalid:993".into(), + account_id: 7, + canonical_mailbox: "INBOX".into(), + } + } + + fn identity_for(account_id: u64, mailbox: &str) -> AcquisitionIdentity { + AcquisitionIdentity { + endpoint: "imap.invalid:993".into(), + account_id, + canonical_mailbox: mailbox.into(), + } + } + + fn temp_root(test: &str) -> PathBuf { + let root = + std::env::temp_dir().join(format!("bichon-uidonly-{test}-{}", uuid::Uuid::new_v4())); + fs::create_dir_all(&root).unwrap(); + root + } + + fn item(uid: u32, size: u64) -> InventoryItem { + InventoryItem { + uid, + size: Some(size), + } + } + fn message(raw: &[u8]) -> Result { + Ok(FetchOutcome::Message { + declared_size: Some(raw.len() as u64), + raw: raw.to_vec(), + }) + } + + fn message_with_size( + raw: &[u8], + declared_size: Option, + ) -> Result { + Ok(FetchOutcome::Message { + declared_size, + raw: raw.to_vec(), + }) + } + + #[tokio::test] + async fn per_uid_ledger_persistence_is_linear_in_total_bytes_written() { + async fn written_for(count: u32) -> u64 { + let root = temp_root(&format!("linear-ledger-{count}")); + let raw = b"mail"; + let inventory: Vec<_> = (1..=count).map(|uid| item(uid, raw.len() as u64)).collect(); + let outcomes = (1..=count) + .map(|uid| (uid, VecDeque::from([message(raw)]))) + .collect(); + let mut transport = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: count + 1, + }, + inventory, + outcomes, + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let mut bounded = limits(); + bounded.page_size = count; + let report = run_acquisition( + &mut transport, + &mut FakeCanonicalArchive::default(), + "INBOX", + identity(), + &root, + bounded, + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(report.success); + fs::remove_dir_all(root).unwrap(); + report.state_bytes_written + } + + let forty = written_for(40).await; + let eighty = written_for(80).await; + assert!(eighty > forty); + assert!( + eighty <= forty.saturating_mul(23) / 10, + "doubling UIDs must keep durable state bytes linear: 40={forty}, 80={eighty}" + ); + } + + #[test] + fn lifecycle_cleanup_removes_only_exact_account_and_mailbox_state() { + let root = temp_root("lifecycle-cleanup"); + let inbox7 = identity_for(7, "INBOX"); + let sent7 = identity_for(7, "Sent"); + let inbox8 = identity_for(8, "INBOX"); + for identity in [&inbox7, &sent7, &inbox8] { + DurableArchive::open(&root, identity, 9, limits()) + .unwrap() + .load_or_create(identity.clone(), 9, 1) + .unwrap(); + } + + assert_eq!( + cleanup_uidonly_mailbox_state(&root, 7, &BTreeSet::from(["INBOX".to_string()])) + .unwrap(), + 1 + ); + assert!(!root.join(inbox7.storage_key()).exists()); + assert!(root.join(sent7.storage_key()).exists()); + assert!(root.join(inbox8.storage_key()).exists()); + + assert_eq!(cleanup_uidonly_account_state(&root, 7).unwrap(), 1); + assert!(!root.join(sent7.storage_key()).exists()); + assert!(root.join(inbox8.storage_key()).exists()); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn canonical_disk_budget_is_enforced_before_projection() { + let root = temp_root("canonical-disk-budget"); + let mut transport = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 2, + }, + inventory: vec![item(1, 4)], + outcomes: [(1, VecDeque::from([message(b"mail")]))].into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let mut canonical = FakeCanonicalArchive { + disk_budget_override: Some(10_000), + ..Default::default() + }; + let mut bounded = limits(); + bounded.max_disk_bytes = 4_096; + let report = run_acquisition( + &mut transport, + &mut canonical, + "INBOX", + identity(), + &root, + bounded, + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(!report.success); + assert_eq!(report.checkpoint, None); + assert!(canonical.projected_uids.is_empty()); + assert!(matches!(report.states[&1], UidState::Failed { .. })); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn checkpoint_final_revalidation_catches_late_canonical_deletion() { + let root = temp_root("final-revalidation"); + let mut transport = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 2, + }, + inventory: vec![item(1, 4)], + outcomes: [(1, VecDeque::from([message(b"mail")]))].into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let mut canonical = FakeCanonicalArchive { + fail_verify_on_call: Some(2), + ..Default::default() + }; + let report = run_acquisition( + &mut transport, + &mut canonical, + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(!report.success); + assert_eq!(report.checkpoint, None); + assert!(matches!(report.states[&1], UidState::Failed { .. })); + assert!(canonical.records.is_empty()); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn cancellation_interrupts_hanging_canonical_projection_and_rolls_back() { + let root = temp_root("cancel-canonical"); + let mut transport = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 2, + }, + inventory: vec![item(1, 4)], + outcomes: [(1, VecDeque::from([message(b"mail")]))].into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let mut canonical = FakeCanonicalArchive { + hang_projection: BTreeSet::from([1]), + ..Default::default() + }; + let token = CancellationToken::new(); + let cancel = token.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(10)).await; + cancel.cancel(); + }); + let error = run_acquisition( + &mut transport, + &mut canonical, + "INBOX", + identity(), + &root, + limits(), + token, + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("cancelled")); + assert!(canonical.records.is_empty()); + assert_eq!(canonical.active_projects, 0); + assert_eq!(canonical.quiesced_projects, vec![1]); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn runtime_ceiling_interrupts_hanging_canonical_projection_and_rolls_back() { + let root = temp_root("runtime-canonical"); + let mut transport = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 2, + }, + inventory: vec![item(1, 4)], + outcomes: [(1, VecDeque::from([message(b"mail")]))].into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let mut canonical = FakeCanonicalArchive { + hang_projection: BTreeSet::from([1]), + ..Default::default() + }; + let mut bounded = limits(); + bounded.max_runtime = Duration::from_millis(200); + let error = run_acquisition( + &mut transport, + &mut canonical, + "INBOX", + identity(), + &root, + bounded, + CancellationToken::new(), + ) + .await + .unwrap_err(); + assert_eq!(error.code(), ErrorCode::RequestTimeout); + assert!(canonical.records.is_empty()); + assert_eq!(canonical.active_projects, 0); + assert!(canonical.quiesced_projects.is_empty() || canonical.quiesced_projects == vec![1]); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn bounded_canonical_runtime_waits_for_projection_quiescence() { + let state = std::sync::Arc::new(AtomicU64::new(0)); + let operation_state = state.clone(); + let operation_shutdown = CancellationToken::new(); + let observed_shutdown = operation_shutdown.clone(); + let acquisition_token = CancellationToken::new(); + let mut bounded = limits(); + bounded.max_runtime = Duration::from_millis(25); + let error = bounded_canonical::<_, ()>( + async move { + operation_state.store(1, Ordering::Release); + observed_shutdown.cancelled().await; + operation_state.store(2, Ordering::Release); + Ok(()) + }, + Instant::now(), + bounded, + &acquisition_token, + Some(&operation_shutdown), + ) + .await + .unwrap_err(); + assert_eq!(error.error.code(), ErrorCode::RequestTimeout); + assert_eq!(state.load(Ordering::Acquire), 2); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn noncooperative_blocking_projection_returns_bounded_and_serializes_late_work() { + let write_lock = std::sync::Arc::new(tokio::sync::Mutex::new(())); + let state = std::sync::Arc::new(AtomicU64::new(0)); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let operation_shutdown = CancellationToken::new(); + let owned_shutdown = operation_shutdown.clone(); + let owned_lock = write_lock.clone(); + let owned_state = state.clone(); + let owned_task = tokio::spawn(async move { + let _guard = owned_lock.lock().await; + owned_state.store(1, Ordering::Release); + // Models a production spawn_blocking Fjall call: shutdown cannot + // cancel the blocking closure, so the owned task must outlive the + // bounded caller and retain serialization until it returns. + tokio::task::spawn_blocking(move || release_rx.recv().unwrap()) + .await + .unwrap(); + assert!(owned_shutdown.is_cancelled()); + // Self-rollback completes before releasing the serialization + // guard. A later projection must observe this state first. + owned_state.store(2, Ordering::Release); + Err::<(), BichonError>(raise_error!( + "synthetic owned projection cancelled".into(), + ErrorCode::InternalError + )) + }); + let operation = async move { + owned_task + .await + .map_err(|error| raise_error!(format!("{error:#?}"), ErrorCode::InternalError))? + }; + let acquisition_token = CancellationToken::new(); + let mut bounded = limits(); + bounded.max_runtime = Duration::from_millis(25); + let started = Instant::now(); + let failure = bounded_canonical( + operation, + started, + bounded, + &acquisition_token, + Some(&operation_shutdown), + ) + .await + .unwrap_err(); + assert_eq!(failure.error.code(), ErrorCode::RequestTimeout); + assert!(failure.cleanup_pending); + assert!(started.elapsed() < Duration::from_millis(500)); + assert_eq!(state.load(Ordering::Acquire), 1); + + let later_lock = write_lock.clone(); + let later_state = state.clone(); + let later = tokio::spawn(async move { + let _guard = later_lock.lock().await; + assert_eq!(later_state.load(Ordering::Acquire), 2); + later_state.store(3, Ordering::Release); + }); + tokio::time::sleep(Duration::from_millis(20)).await; + assert_eq!(state.load(Ordering::Acquire), 1); + release_tx.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(1), later) + .await + .unwrap() + .unwrap(); + assert_eq!(state.load(Ordering::Acquire), 3); + } + + #[tokio::test] + async fn restart_reaccounts_persisted_canonical_disk_reservation() { + let root = temp_root("restart-canonical-disk"); + let mut first = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 2, + }, + inventory: vec![item(1, 4)], + outcomes: [(1, VecDeque::from([message(b"mail")]))].into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let mut canonical = FakeCanonicalArchive::default(); + let first_report = run_acquisition( + &mut first, + &mut canonical, + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(first_report.success); + let canonical_budget = match first_report.states[&1] { + UidState::Committed { + canonical_bytes, .. + } => canonical_bytes, + _ => unreachable!(), + }; + let identity_dir = root.join(identity().storage_key()); + let physical = directory_size(&identity_dir).unwrap(); + let mut restart_limits = limits(); + restart_limits.max_disk_bytes = physical + canonical_budget - 1; + let mut restart = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 2, + }, + inventory: vec![item(1, 4)], + outcomes: BTreeMap::new(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let error = run_acquisition( + &mut restart, + &mut canonical, + "INBOX", + identity(), + &root, + restart_limits, + CancellationToken::new(), + ) + .await + .unwrap_err(); + assert_eq!(error.code(), ErrorCode::PayloadTooLarge); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn sparse_uid_cursor_partial_order_and_identical_bodies_are_safe() { + let root = temp_root("sparse"); + let raw = b"same exact RFC822 bytes"; + let mut transport = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 51, + }, + inventory: vec![ + item(2, raw.len() as u64), + item(30, raw.len() as u64), + item(50, raw.len() as u64), + ], + outcomes: [ + (2, VecDeque::from([message(raw)])), + (30, VecDeque::from([message(raw)])), + (50, VecDeque::from([message(raw)])), + ] + .into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let mut canonical = FakeCanonicalArchive::default(); + let report = run_acquisition( + &mut transport, + &mut canonical, + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(report.success); + assert_eq!( + (report.planned, report.processed, report.checkpoint), + (3, 3, Some(50)) + ); + assert_eq!(transport.page_requests, vec![(1, 50, 2), (31, 50, 2)]); + let epoch = root.join(identity().storage_key()).join("9"); + assert_eq!(canonical.records.len(), 3); + assert_eq!( + canonical.records[&2].content_hash, + canonical.records[&30].content_hash + ); + assert_eq!( + canonical.records[&30].content_hash, + canonical.records[&50].content_hash + ); + assert_ne!( + canonical.records[&2].envelope_id, + canonical.records[&30].envelope_id + ); + assert_eq!( + fs::read_dir(epoch.join("blobs")).unwrap().count(), + 0, + "committed raw bytes are reclaimed from staging" + ); + assert_eq!( + fs::read_dir(epoch.join("records")).unwrap().count(), + 0, + "canonical records, not staging records, own committed identity" + ); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn literal_framing_accepts_equal_different_and_missing_rfc822_size() { + let root = temp_root("literal-size-metadata"); + let raw = b"From: sender@example.invalid\r\n\r\nbody"; + let mut transport = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 4, + }, + inventory: vec![ + InventoryItem { + uid: 1, + size: Some(raw.len() as u64), + }, + InventoryItem { + uid: 2, + size: Some(999), + }, + InventoryItem { uid: 3, size: None }, + ], + outcomes: [ + ( + 1, + VecDeque::from([message_with_size(raw, Some(raw.len() as u64))]), + ), + (2, VecDeque::from([message_with_size(raw, Some(999))])), + (3, VecDeque::from([message_with_size(raw, None)])), + ] + .into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let mut canonical = FakeCanonicalArchive::default(); + let report = run_acquisition( + &mut transport, + &mut canonical, + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(report.success); + assert_eq!((report.planned, report.processed), (3, 3)); + assert_eq!(canonical.records.len(), 3); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn canonical_projection_failure_prevents_success_and_checkpoint() { + let root = temp_root("projection-failure"); + let mut transport = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 8, + }, + inventory: vec![item(7, 4)], + outcomes: [(7, VecDeque::from([message(b"mail")]))].into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let mut canonical = FakeCanonicalArchive { + fail_projection: BTreeSet::from([7]), + ..Default::default() + }; + let report = run_acquisition( + &mut transport, + &mut canonical, + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(!report.success); + assert_eq!(report.checkpoint, None); + assert!(matches!(report.states[&7], UidState::Failed { .. })); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn committed_ledger_persist_failure_rolls_back_canonical_projection() { + let root = temp_root("committed-ledger-persist-failure"); + let identity = identity(); + let archive = DurableArchive::open(&root, &identity, 9, limits()).unwrap(); + archive.load_or_create(identity.clone(), 9, 1).unwrap(); + atomic_write( + &archive.epoch_dir.join("fail-next-committed-entry-persist"), + b"fail", + ) + .unwrap(); + let entry_path = archive.ledger_entries_dir.join("1.json"); + drop(archive); + + let mut transport = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 2, + }, + inventory: vec![item(1, 4)], + outcomes: [(1, VecDeque::from([message(b"mail")]))].into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let mut canonical = FakeCanonicalArchive::default(); + let error = run_acquisition( + &mut transport, + &mut canonical, + "INBOX", + identity.clone(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap_err(); + assert!(error + .to_string() + .contains("synthetic committed ledger persist failure")); + assert!(canonical.records.is_empty()); + let persisted: UidEntry = serde_json::from_slice(&fs::read(entry_path).unwrap()).unwrap(); + assert!(matches!(persisted.state, UidState::Projecting { .. })); + + let mut retry = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 2, + }, + inventory: vec![item(1, 4)], + outcomes: [(1, VecDeque::from([message(b"mail")]))].into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let report = run_acquisition( + &mut retry, + &mut canonical, + "INBOX", + identity, + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(report.success); + assert!(canonical.records.contains_key(&1)); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + #[ignore = "run offline with an isolated BICHON_ROOT_DIR"] + async fn production_canonical_projection_is_queryable_and_restores_exact_raw() { + let account_id = 7_000_000_001; + let mailbox_id = 7_000_000_002; + let first_uid = 77; + let second_uid = 78; + let raw = b"From: sender@example.invalid\r\n\ +To: archive@example.invalid\r\n\ +Subject: canonical roundtrip\r\n\ +Message-ID: \r\n\ +MIME-Version: 1.0\r\n\ +Content-Type: multipart/mixed; boundary=uidonly-boundary\r\n\ +\r\n\ +--uidonly-boundary\r\n\ +Content-Type: text/plain\r\n\ +\r\n\ +exact body bytes\r\n\ +--uidonly-boundary\r\n\ +Content-Type: application/octet-stream\r\n\ +Content-Disposition: attachment; filename=fixture.bin\r\n\ +Content-Transfer-Encoding: base64\r\n\ +\r\n\ +AQIDBA==\r\n\ +--uidonly-boundary--\r\n"; + insert_impl( + DB_MANAGER.db(), + AccountModel { + id: account_id, + email: "archive@example.invalid".into(), + enabled: true, + ..Default::default() + }, + ) + .unwrap(); + MailBox::batch_insert(&[MailBox { + id: mailbox_id, + account_id, + name: "INBOX".into(), + ..Default::default() + }]) + .unwrap(); + let mut canonical = BichonCanonicalArchive::new(account_id, mailbox_id); + let first = canonical + .project(first_uid, raw, None, CancellationToken::new()) + .await + .unwrap(); + let second = canonical + .project(second_uid, raw, None, CancellationToken::new()) + .await + .unwrap(); + assert_ne!(first.envelope_id, second.envelope_id); + + let mut email_writer = ENVELOPE_MANAGER.index_writer().lock().await; + let mut attachment_writer = ATTACHMENT_MANAGER.index_writer().lock().await; + let email_reader = ENVELOPE_MANAGER.create_reader().unwrap(); + dedup_task(&email_reader, &mut email_writer, &mut attachment_writer) + .await + .unwrap(); + drop(attachment_writer); + drop(email_writer); + + for (uid, projection) in [(first_uid, &first), (second_uid, &second)] { + let queried = ENVELOPE_MANAGER + .get_projection_by_uid(account_id, mailbox_id, uid) + .unwrap() + .expect("distinct UIDONLY projection must survive periodic dedup"); + assert_eq!(queried.envelope_id, projection.envelope_id); + assert_eq!(queried.content_hash, compute_content_hash(raw)); + assert_eq!(queried.shard_id, UIDONLY_SHARD_ID); + assert_eq!( + ATTACHMENT_MANAGER + .canonical_records_by_envelope(account_id, &projection.envelope_id) + .unwrap(), + vec![CanonicalAttachmentRecord { + content_hash: compute_content_hash(&[1, 2, 3, 4]), + name: Some("fixture.bin".into()), + size: 4, + content_type: "application/octet-stream".into(), + }] + ); + + let (envelope, restored) = + reattach_eml_content(account_id, projection.envelope_id.clone()).unwrap(); + assert_eq!(envelope.uid, uid); + assert_eq!(restored.as_ref(), raw); + assert!(canonical + .verify(uid, &projection.content_hash, &projection.envelope_id) + .await + .unwrap()); + } + + let failed_uid = 79; + // The failed writer reuses the exact email and attachment blobs owned + // by two committed projections. Rollback must remove only its index + // documents and preserve both shared blob values. + let failed_raw = raw; + let failed_hash = compute_content_hash(failed_raw); + let failed_envelope_id = canonical.envelope_id(failed_uid, &failed_hash); + fail_uidonly_after_attachments(true); + let failure = canonical + .project(failed_uid, failed_raw, None, CancellationToken::new()) + .await; + fail_uidonly_after_attachments(false); + assert!(failure.is_err()); + assert!(ENVELOPE_MANAGER + .get_projection_by_uid(account_id, mailbox_id, failed_uid) + .unwrap() + .is_none()); + assert_eq!( + ATTACHMENT_MANAGER + .count_by_envelope(account_id, &failed_envelope_id) + .unwrap(), + 0 + ); + assert!(BLOB_MANAGER.get_email(&failed_hash).unwrap().is_some()); + for projection in [&first, &second] { + let (_, restored) = + reattach_eml_content(account_id, projection.envelope_id.clone()).unwrap(); + assert_eq!(restored.as_ref(), raw); + } + } + + #[tokio::test] + async fn committed_record_deletion_blocks_restart_success_without_staging_rebuild() { + let root = temp_root("rebuild-canonical-missing"); + let mut first = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 8, + }, + inventory: vec![item(7, 4)], + outcomes: [(7, VecDeque::from([message(b"mail")]))].into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let mut canonical = FakeCanonicalArchive::default(); + assert!( + run_acquisition( + &mut first, + &mut canonical, + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap() + .success + ); + canonical.records.clear(); + let mut restart = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 8, + }, + inventory: vec![item(7, 4)], + outcomes: BTreeMap::new(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let report = run_acquisition( + &mut restart, + &mut canonical, + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(!report.success); + assert_eq!(report.checkpoint, None); + assert!(matches!(report.states[&7], UidState::Failed { .. })); + let epoch = root.join(identity().storage_key()).join("9"); + assert_eq!(fs::read_dir(epoch.join("blobs")).unwrap().count(), 0); + assert_eq!(fs::read_dir(epoch.join("records")).unwrap().count(), 0); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn committed_blob_corruption_blocks_restart_success_without_staging_rebuild() { + let root = temp_root("restart-canonical-blob-corrupt"); + let mut first = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 8, + }, + inventory: vec![item(7, 4)], + outcomes: [(7, VecDeque::from([message(b"mail")]))].into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let mut canonical = FakeCanonicalArchive::default(); + assert!( + run_acquisition( + &mut first, + &mut canonical, + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap() + .success + ); + canonical.corrupt_blobs.insert(7); + let mut restart = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 8, + }, + inventory: vec![item(7, 4)], + outcomes: BTreeMap::new(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let report = run_acquisition( + &mut restart, + &mut canonical, + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(!report.success); + assert_eq!(report.checkpoint, None); + assert!(matches!(report.states[&7], UidState::Failed { .. })); + let epoch = root.join(identity().storage_key()).join("9"); + assert_eq!(fs::read_dir(epoch.join("blobs")).unwrap().count(), 0); + assert_eq!(fs::read_dir(epoch.join("records")).unwrap().count(), 0); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn compact_huge_vanished_range_intersects_only_bounded_planned_uids() { + let root = temp_root("huge-vanished"); + let archive = DurableArchive::open(&root, &identity(), 9, limits()).unwrap(); + let mut ledger = archive.load_or_create(identity(), 9, u32::MAX - 1).unwrap(); + for uid in [2, 30, 50] { + ledger.entries.insert( + uid, + UidEntry { + declared_size: None, + state: UidState::Missing, + }, + ); + } + for uid in [2, 30, 50] { + archive.persist_entry(uid, &ledger.entries[&uid]).unwrap(); + } + let report = run_acquisition( + &mut HugeVanishedTransport, + &mut FakeCanonicalArchive::default(), + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(report.success); + assert_eq!((report.planned, report.processed), (3, 3)); + assert!(report + .states + .values() + .all(|state| matches!(state, UidState::Vanished))); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn cancellation_interrupts_a_pending_transport_operation() { + let root = temp_root("cancel-pending"); + let token = CancellationToken::new(); + let cancel = token.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(10)).await; + cancel.cancel(); + }); + let error = run_acquisition( + &mut HangingTransport, + &mut FakeCanonicalArchive::default(), + "INBOX", + identity(), + &root, + limits(), + token, + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("cancelled")); + fs::remove_dir_all(root).ok(); + } + + #[tokio::test] + async fn runtime_ceiling_interrupts_a_pending_transport_operation() { + let root = temp_root("runtime-pending"); + let mut bounded = limits(); + bounded.max_runtime = Duration::from_millis(10); + let error = run_acquisition( + &mut HangingTransport, + &mut FakeCanonicalArchive::default(), + "INBOX", + identity(), + &root, + bounded, + CancellationToken::new(), + ) + .await + .unwrap_err(); + assert_eq!(error.code(), ErrorCode::RequestTimeout); + fs::remove_dir_all(root).ok(); + } + + #[tokio::test] + async fn message_total_byte_and_disk_ceilings_prevent_checkpoint() { + let cases = ["messages", "total-bytes", "disk"]; + for case in cases { + let root = temp_root(case); + let raw = b"mail"; + let mut bounded = limits(); + match case { + "messages" => bounded.max_messages = 1, + "total-bytes" => bounded.max_total_bytes = raw.len() as u64, + "disk" => bounded.max_disk_bytes = 512, + _ => unreachable!(), + } + let mut transport = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 3, + }, + inventory: vec![item(1, 4), item(2, 4)], + outcomes: [ + (1, VecDeque::from([message(raw)])), + (2, VecDeque::from([message(raw)])), + ] + .into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let result = run_acquisition( + &mut transport, + &mut FakeCanonicalArchive::default(), + "INBOX", + identity(), + &root, + bounded, + CancellationToken::new(), + ) + .await; + match case { + "messages" | "disk" => assert!(result.is_err()), + "total-bytes" => { + let report = result.unwrap(); + assert!(!report.success); + assert_eq!(report.checkpoint, None); + } + _ => unreachable!(), + } + fs::remove_dir_all(root).ok(); + } + } + + #[tokio::test] + async fn oversized_lower_uid_does_not_hide_successful_higher_uid_or_checkpoint() { + let root = temp_root("oversized"); + let mut small_limits = limits(); + small_limits.max_literal_bytes = 4; + let mut transport = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 21, + }, + inventory: vec![item(10, 5), item(20, 2)], + outcomes: [ + (10, VecDeque::from([message(b"large")])), + (20, VecDeque::from([message(b"ok")])), + ] + .into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let mut canonical = FakeCanonicalArchive::default(); + let report = run_acquisition( + &mut transport, + &mut canonical, + "INBOX", + identity(), + &root, + small_limits, + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(!report.success); + assert_eq!( + (report.planned, report.processed, report.checkpoint), + (2, 1, None) + ); + assert!(matches!(report.states[&10], UidState::Oversized { .. })); + assert!(matches!(report.states[&20], UidState::Committed { .. })); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn disconnect_reconnects_and_retries_same_uid_before_checkpoint() { + let root = temp_root("reconnect"); + let mut transport = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 8, + }, + inventory: vec![item(7, 4)], + outcomes: [( + 7, + VecDeque::from([ + Err(TransportFailure { + message: "disconnect during literal".into(), + network: true, + }), + message(b"mail"), + ]), + )] + .into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let report = run_acquisition( + &mut transport, + &mut FakeCanonicalArchive::default(), + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(report.success); + assert_eq!(transport.reconnects, 1); + assert_eq!(report.checkpoint, Some(7)); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn pending_restart_never_advances_and_vanished_reconciles_explicitly() { + let root = temp_root("restart"); + let mut first = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 31, + }, + inventory: vec![item(10, 2), item(30, 2)], + outcomes: [ + (10, VecDeque::from([message(b"ok")])), + (30, VecDeque::from([Ok(FetchOutcome::Missing)])), + ] + .into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let mut canonical = FakeCanonicalArchive::default(); + let report = run_acquisition( + &mut first, + &mut canonical, + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(!report.success); + assert_eq!(report.checkpoint, None); + + let mut restarted = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 99, + }, + inventory: vec![item(10, 2)], + outcomes: BTreeMap::new(), + vanished_on_inventory: BTreeSet::from([30]), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let report = run_acquisition( + &mut restarted, + &mut canonical, + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(report.success); + assert_eq!( + report.checkpoint, + Some(30), + "restart retains the original fixed snapshot" + ); + assert!(matches!(report.states[&30], UidState::Vanished)); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn changed_uidvalidity_starts_a_fresh_staging_epoch() { + let root = temp_root("uidvalidity"); + let mut first = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 2, + }, + inventory: vec![item(1, 1)], + outcomes: [(1, VecDeque::from([message(b"x")]))].into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + run_acquisition( + &mut first, + &mut FakeCanonicalArchive::default(), + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap(); + let mut changed = FakeTransport { + snapshot: Snapshot { + uid_validity: 10, + uid_next: 2, + }, + inventory: vec![], + outcomes: BTreeMap::new(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let report = run_acquisition( + &mut changed, + &mut FakeCanonicalArchive::default(), + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(report.success); + assert!(report.states.is_empty()); + let identity_dir = root.join(identity().storage_key()); + assert_eq!( + fs::read_to_string(identity_dir.join("current-uidvalidity")).unwrap(), + "10" + ); + assert!(identity_dir.join("9").exists()); + assert!(identity_dir.join("10").exists()); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn expunge_between_inventory_pages_cannot_shift_a_uid_behind_cursor() { + let root = temp_root("expunge-pages"); + let inventory = vec![ + item(10, 1), + item(20, 1), + item(30, 1), + item(40, 1), + item(50, 1), + ]; + let outcomes = inventory + .iter() + .map(|item| (item.uid, VecDeque::from([message(b"x")]))) + .collect(); + let mut transport = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 51, + }, + inventory, + outcomes, + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: Some(10), + reconnects: 0, + page_requests: Vec::new(), + }; + let report = run_acquisition( + &mut transport, + &mut FakeCanonicalArchive::default(), + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(report.success); + assert_eq!(report.states.len(), 5); + assert!(report.states.contains_key(&30)); + assert_eq!( + transport.page_requests, + vec![(1, 50, 2), (21, 50, 2), (41, 50, 2)] + ); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn vanished_during_body_fetch_is_an_explicit_reconciliation() { + let root = temp_root("vanished-fetch"); + let mut transport = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 8, + }, + inventory: vec![item(7, 10)], + outcomes: [(7, VecDeque::from([Ok(FetchOutcome::Vanished)]))].into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let report = run_acquisition( + &mut transport, + &mut FakeCanonicalArchive::default(), + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(report.success); + assert!(matches!(report.states[&7], UidState::Vanished)); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn restart_retries_pending_queue_ack_without_checkpointing_it() { + let root = temp_root("pending-queue-ack"); + let archive = DurableArchive::open(&root, &identity(), 9, limits()).unwrap(); + let mut ledger = archive.load_or_create(identity(), 9, 7).unwrap(); + ledger.entries.insert( + 7, + UidEntry { + declared_size: Some(4), + state: UidState::Pending, + }, + ); + archive.persist_entry(7, &ledger.entries[&7]).unwrap(); + + let mut restarted = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 8, + }, + inventory: vec![item(7, 4)], + outcomes: [(7, VecDeque::from([message(b"mail")]))].into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let report = run_acquisition( + &mut restarted, + &mut FakeCanonicalArchive::default(), + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(report.success); + assert_eq!(report.checkpoint, Some(7)); + assert!(matches!(report.states[&7], UidState::Committed { .. })); + fs::remove_dir_all(root).unwrap(); + } + + fn uidfetch_metadata(entries: &[(u32, usize)]) -> Vec { + let mut response = Vec::new(); + for (uid, bytes) in entries { + response.extend_from_slice( + format!("* {uid} UIDFETCH (UID {uid} RFC822.SIZE {bytes})\r\n").as_bytes(), + ); + } + response.extend_from_slice(b"{TAG} OK UID FETCH completed\r\n"); + response + } + + fn uidfetch_body(uid: u32, raw: &[u8]) -> Vec { + let mut response = format!( + "* {uid} UIDFETCH (UID {uid} RFC822.SIZE {} BODY[] {{{}}}\r\n", + raw.len(), + raw.len() + ) + .into_bytes(); + response.extend_from_slice(raw); + response.extend_from_slice(b")\r\n{TAG} OK UID FETCH completed\r\n"); + response + } + + async fn transcript_session( + server: &crate::imap::mock_server::MockImapServerHandle, + limits: ResponseLimits, + ) -> Session> { + let stream = TcpStream::connect((server.host(), server.port())) + .await + .unwrap(); + let mut client = + async_imap::Client::new(Box::new(TestStream(stream)) as Box); + client.read_response().await.unwrap().unwrap(); + let mut session = client + .login("test", "test") + .await + .map_err(|(e, _)| e) + .unwrap(); + session.set_response_limits(limits).unwrap(); + session.enable_uidonly().await.unwrap(); + session + } + + #[tokio::test] + async fn tcp_fake_server_uses_only_uid_commands_after_enable() { + let raw2 = b"From: a@example.invalid\r\n\r\ntwo"; + let raw30 = b"From: b@example.invalid\r\n\r\nthirty"; + let raw50 = b"From: c@example.invalid\r\n\r\nfifty"; + let server = MockImapServer::new() + .respond("LOGIN", b"{TAG} OK LOGIN completed\r\n".to_vec()) + .respond( + "ENABLE UIDONLY", + b"* ENABLED UIDONLY\r\n{TAG} OK ENABLE completed\r\n".to_vec(), + ) + .respond("EXAMINE", examine_response("INBOX", 3, 9, 51)) + .respond( + "UID FETCH 1:50 (UID RFC822.SIZE) (PARTIAL 1:2)", + uidfetch_metadata(&[(2, raw2.len()), (30, raw30.len())]), + ) + .respond( + "UID FETCH 31:50 (UID RFC822.SIZE) (PARTIAL 1:2)", + uidfetch_metadata(&[(50, raw50.len())]), + ) + .respond("UID FETCH 2 ", uidfetch_body(2, raw2)) + .respond("UID FETCH 30 ", uidfetch_body(30, raw30)) + .respond("UID FETCH 50 ", uidfetch_body(50, raw50)) + .start() + .await; + let response_limits = limits().response_limits().unwrap(); + let session = transcript_session(&server, response_limits).await; + let root = temp_root("tcp-fake"); + let mut transport = SessionUidOnlyTransport::new(7, session, Some(2), response_limits); + let report = run_acquisition( + &mut transport, + &mut FakeCanonicalArchive::default(), + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(report.success); + let commands = server.commands(); + let enabled = commands + .iter() + .position(|command| command.contains("ENABLE UIDONLY")) + .unwrap(); + for command in &commands[enabled + 1..] { + assert!( + !command.contains(" SEARCH ") + && !command.contains(" STORE ") + && !command.contains(" COPY ") + && !command.contains(" MOVE ") + && (!command.contains(" FETCH ") || command.contains(" UID FETCH ")), + "UIDONLY session emitted forbidden command: {command}" + ); + } + assert!(commands + .iter() + .any(|command| command.contains("PARTIAL 1:2"))); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn tcp_fake_rejects_declared_literal_before_body_acceptance() { + let server = MockImapServer::new() + .respond("LOGIN", b"{TAG} OK LOGIN completed\r\n".to_vec()) + .respond( + "ENABLE UIDONLY", + b"* ENABLED UIDONLY\r\n{TAG} OK ENABLE completed\r\n".to_vec(), + ) + .respond( + "UID FETCH 7 ", + b"* 7 UIDFETCH (UID 7 RFC822.SIZE 100 BODY[] {100}\r\n".to_vec(), + ) + .start() + .await; + let mut session = transcript_session(&server, ResponseLimits::new(1024, 4)).await; + let mut stream = session.uid_fetch_uidonly("7", BODY_QUERY).await.unwrap(); + let error = stream.try_next().await.unwrap_err(); + assert!(error.to_string().contains("literal") || error.to_string().contains("large")); + } + + #[tokio::test] + #[ignore = "requires an explicitly provisioned disposable localhost Cyrus instance"] + async fn cyrus_uidonly_exact_raw_roundtrip() { + let port: u16 = std::env::var("BICHON_CYRUS_PORT") + .expect("BICHON_CYRUS_PORT") + .parse() + .expect("numeric Cyrus port"); + let root = PathBuf::from( + std::env::var("BICHON_CYRUS_ARCHIVE_ROOT").expect("BICHON_CYRUS_ARCHIVE_ROOT"), + ); + assert!(root.is_absolute()); + fs::create_dir_all(&root).unwrap(); + + let connect = || async move { + let stream = TcpStream::connect(("127.0.0.1", port)).await.unwrap(); + let mut client = + async_imap::Client::new(Box::new(TestStream(stream)) as Box); + client.read_response().await.unwrap().unwrap(); + client + .login("archive-test", "synthetic-only-password") + .await + .map_err(|(error, _)| error) + .unwrap() + }; + + let raw_messages: [&[u8]; 3] = [ + b"From: one@example.invalid\r\nTo: archive@example.invalid\r\nSubject: one\r\n\r\nfirst\r\n", + b"From: two@example.invalid\r\nTo: archive@example.invalid\r\nSubject: two\r\n\r\nsecond\r\n", + b"From: three@example.invalid\r\nTo: archive@example.invalid\r\nSubject: three\r\n\r\nthird\r\n", + ]; + let mut seed = connect().await; + for raw in raw_messages { + seed.append("INBOX", None, None, raw).await.unwrap(); + } + seed.logout().await.unwrap(); + + let mut session = connect().await; + let capabilities = session.capabilities().await.unwrap(); + assert!(capabilities.has_str("UIDONLY")); + assert!(capabilities.has_str("PARTIAL")); + let cyrus_limits = AcquisitionLimits { + max_messages: 100, + max_total_bytes: 100 * 1024 * 1024, + max_literal_bytes: 25 * 1024 * 1024, + max_response_bytes: 26 * 1024 * 1024, + max_runtime: Duration::from_secs(600), + max_disk_bytes: 1024 * 1024 * 1024, + page_size: 2, + }; + let response_limits = cyrus_limits.response_limits().unwrap(); + session.set_response_limits(response_limits).unwrap(); + session.enable_uidonly().await.unwrap(); + let mut transport = SessionUidOnlyTransport::new(7, session, None, response_limits); + let cyrus_identity = AcquisitionIdentity { + endpoint: format!("127.0.0.1:{port}"), + account_id: 7, + canonical_mailbox: "INBOX".into(), + }; + let mut canonical = FakeCanonicalArchive::default(); + let report = run_acquisition( + &mut transport, + &mut canonical, + "INBOX", + cyrus_identity.clone(), + &root, + cyrus_limits, + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(report.success); + assert_eq!((report.planned, report.processed), (3, 3)); + drop(transport); + + let mut restart_session = connect().await; + restart_session + .set_response_limits(response_limits) + .unwrap(); + restart_session.enable_uidonly().await.unwrap(); + let mut restart_transport = + SessionUidOnlyTransport::new(7, restart_session, None, response_limits); + let restart_report = run_acquisition( + &mut restart_transport, + &mut canonical, + "INBOX", + cyrus_identity.clone(), + &root, + cyrus_limits, + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(restart_report.success); + assert_eq!((restart_report.planned, restart_report.processed), (3, 3)); + assert_eq!( + canonical.projected_uids.len(), + 3, + "restart must revalidate committed records without reprojecting bodies" + ); + + let epoch = root + .join(cyrus_identity.storage_key()) + .join(report.uid_validity.to_string()); + assert_eq!(fs::read_dir(epoch.join("records")).unwrap().count(), 0); + assert_eq!(fs::read_dir(epoch.join("blobs")).unwrap().count(), 0); + assert_eq!(canonical.records.len(), 3); + let expected_hashes: BTreeSet<_> = raw_messages + .iter() + .map(|raw| compute_content_hash(raw)) + .collect(); + let projected_hashes: BTreeSet<_> = canonical + .records + .values() + .map(|projection| projection.content_hash.clone()) + .collect(); + let envelope_ids: BTreeSet<_> = canonical + .records + .values() + .map(|projection| projection.envelope_id.clone()) + .collect(); + assert_eq!(projected_hashes, expected_hashes); + assert_eq!(envelope_ids.len(), 3); + } +} diff --git a/crates/core/src/mailbox/delete.rs b/crates/core/src/mailbox/delete.rs index 574323cf..0f5e937c 100644 --- a/crates/core/src/mailbox/delete.rs +++ b/crates/core/src/mailbox/delete.rs @@ -19,8 +19,11 @@ use crate::{ cache::imap::mailbox::MailBox, error::BichonResult, + imap::uidonly_acquisition::cleanup_uidonly_mailbox_state, + settings::dir::DATA_DIR_MANAGER, store::tantivy::{attachment::ATTACHMENT_MANAGER, envelope::ENVELOPE_MANAGER}, }; +use std::collections::BTreeSet; pub async fn delete_mailbox_impl(account_id: u64, mailbox_id: u64) -> BichonResult<()> { let mailbox = MailBox::get(mailbox_id)?; @@ -30,16 +33,26 @@ pub async fn delete_mailbox_impl(account_id: u64, mailbox_id: u64) -> BichonResu let all_mailboxes = MailBox::list_all(account_id)?; let prefix = format!("{}{}", name, delimiter); - let ids_to_delete: Vec = all_mailboxes + let mailboxes_to_delete: Vec<_> = all_mailboxes .into_iter() .filter(|m| m.id == mailbox_id || m.name.starts_with(&prefix)) - .map(|m| m.id) .collect(); + let ids_to_delete: Vec = mailboxes_to_delete.iter().map(|mailbox| mailbox.id).collect(); if ids_to_delete.is_empty() { return Ok(()); } + let names_to_delete: BTreeSet = mailboxes_to_delete + .iter() + .map(|mailbox| mailbox.name.clone()) + .collect(); + cleanup_uidonly_mailbox_state( + &DATA_DIR_MANAGER.storage_dir.join("uidonly-acquisition"), + account_id, + &names_to_delete, + )?; + for id in &ids_to_delete { MailBox::delete(*id)?; } diff --git a/crates/core/src/store/blob.rs b/crates/core/src/store/blob.rs index 145b4959..d2cd97c7 100644 --- a/crates/core/src/store/blob.rs +++ b/crates/core/src/store/blob.rs @@ -26,7 +26,11 @@ use crate::{ use bichon_blob::{Codec, Config, Engine}; use bytes::Bytes; -use std::{io::Cursor, sync::Arc, sync::LazyLock}; +use std::{ + collections::HashSet, + io::Cursor, + sync::{Arc, LazyLock, Mutex as StdMutex}, +}; use tokio::{ sync::{mpsc, Mutex}, task::{self, JoinHandle}, @@ -39,9 +43,17 @@ pub struct DetachedEmail { pub attachments: Option>, } +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub(crate) struct DurableBlobWrite { + pub email_hash: Option, + pub attachment_hashes: Vec, + pub bytes_written: u64, +} + pub struct BlobManager { sender: mpsc::Sender, engine: Arc, + write_lock: Arc>, handle: Mutex>>, } @@ -57,6 +69,59 @@ fn hex_to_key(hex: &str) -> BichonResult<[u8; 32]> { } impl BlobManager { + fn store_detached_email(eml: DetachedEmail, engine: &Engine) -> BichonResult { + let mut written = DurableBlobWrite::default(); + let mut entries = Vec::new(); + let mut new_keys = HashSet::new(); + let (email_hash, email_data) = eml.email; + let email_key = hex_to_key(&email_hash)?; + if !engine + .exists(&email_key) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))? + { + let bytes = email_data.len() as u64; + new_keys.insert(email_key); + entries.push((email_key, email_data.to_vec(), Codec::Lz4)); + written.email_hash = Some(email_hash); + written.bytes_written = written.bytes_written.saturating_add(bytes); + } + + if let Some(attachments) = eml.attachments { + for (attachment_hash, attachment_data) in attachments { + let attachment_key = hex_to_key(&attachment_hash)?; + if !new_keys.contains(&attachment_key) + && !engine + .exists(&attachment_key) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))? + { + let bytes = attachment_data.len() as u64; + new_keys.insert(attachment_key); + entries.push((attachment_key, attachment_data.to_vec(), Codec::Lz4)); + written.attachment_hashes.push(attachment_hash); + written.bytes_written = written.bytes_written.saturating_add(bytes); + } + } + } + + if let Err(error) = engine + .put_batch(&entries) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError)) + { + let rollback_keys: Vec<_> = new_keys.into_iter().collect(); + engine.delete_batch(&rollback_keys).map_err(|rollback_error| { + raise_error!( + format!( + "canonical blob write failed ({error}); rollback failed ({rollback_error:#?})" + ), + ErrorCode::InternalError + ) + })?; + return Err(error); + } + + Ok(written) + } + pub async fn shutdown(&self) { let mut guard = self.handle.lock().await; if let Some(handle) = guard.take() { @@ -127,8 +192,10 @@ impl BlobManager { let engine = Arc::new(engine); let (sender, mut receiver) = mpsc::channel::(100); + let write_lock = Arc::new(StdMutex::new(())); let engine_bg = Arc::clone(&engine); + let handler_write_lock = write_lock.clone(); let handler = task::spawn(async move { let mut shutdown = SIGNAL_MANAGER.subscribe(); loop { @@ -141,7 +208,9 @@ impl BlobManager { batch.push(next_eml); } let engine_bg = Arc::clone(&engine_bg); + let write_lock = handler_write_lock.clone(); if let Err(e) = tokio::task::spawn_blocking(move || { + let _write_guard = write_lock.lock().unwrap(); for eml in batch { Self::process_detached_email(eml, &engine_bg); } @@ -167,7 +236,9 @@ impl BlobManager { ); if !remaining.is_empty() { let engine_bg = Arc::clone(&engine_bg); + let write_lock = handler_write_lock.clone(); if let Err(e) = tokio::task::spawn_blocking(move || { + let _write_guard = write_lock.lock().unwrap(); for eml in remaining { Self::process_detached_email(eml, &engine_bg); } @@ -185,6 +256,7 @@ impl BlobManager { Self { sender, engine, + write_lock, handle: Mutex::new(Some(handler)), } } @@ -195,6 +267,22 @@ impl BlobManager { } } + /// Stores a detached message through the canonical blob engine and waits + /// for its segment and index metadata to be durably synchronized. + pub(crate) async fn store_durable( + &self, + email: DetachedEmail, + ) -> BichonResult { + let engine = Arc::clone(&self.engine); + let write_lock = self.write_lock.clone(); + tokio::task::spawn_blocking(move || { + let _write_guard = write_lock.lock().unwrap(); + Self::store_detached_email(email, &engine) + }) + .await + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))? + } + pub fn get_email(&self, content_hash: &str) -> BichonResult> { self.get(content_hash) } @@ -236,7 +324,6 @@ impl BlobManager { .delete_batch(&keys) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; } - Ok(()) } } diff --git a/crates/core/src/store/tantivy/attachment.rs b/crates/core/src/store/tantivy/attachment.rs index dd727e14..2286448f 100644 --- a/crates/core/src/store/tantivy/attachment.rs +++ b/crates/core/src/store/tantivy/attachment.rs @@ -54,7 +54,7 @@ use tantivy::{ agg_result::{AggregationResult, BucketResult}, AggregationCollector, Key, }, - collector::{Count, FacetCollector, TopDocs}, + collector::{Count, DocSetCollector, FacetCollector, TopDocs}, indexer::{LogMergePolicy, UserOperation}, query::{AllQuery, BooleanQuery, EmptyQuery, Occur, Query, QueryParser, RangeQuery, TermQuery}, schema::{Field, IndexRecordOption, Value}, @@ -77,6 +77,14 @@ pub struct IndexManager { handle: Mutex>>, } +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub(crate) struct CanonicalAttachmentRecord { + pub content_hash: String, + pub name: Option, + pub size: u64, + pub content_type: String, +} + impl IndexManager { pub(crate) fn index_writer(&self) -> &Arc> { &self.index_writer @@ -209,6 +217,136 @@ impl IndexManager { let _ = self.sender.send(doc).await; } + /// Commits the attachment documents before returning so an acquisition + /// checkpoint cannot outrun the canonical attachment index. + pub(crate) async fn commit_documents(&self, docs: Vec) -> BichonResult<()> { + if docs.is_empty() { + return Ok(()); + } + let mut writer = self.index_writer.lock().await; + for doc in docs { + writer + .add_document(doc) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + } + writer + .commit() + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + Ok(()) + } + + pub(crate) async fn rollback_documents( + &self, + account_id: u64, + envelope_id: &str, + ) -> BichonResult> { + let query = self.attachment_query(account_id, envelope_id); + let searcher = self.create_searcher()?; + let fields = SchemaTools::attachment_fields(); + let mut content_hashes = HashSet::new(); + for address in searcher + .search(query.as_ref(), &DocSetCollector) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))? + { + let doc = searcher + .doc::(address) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + if let Some(hash) = doc + .get_first(fields.f_content_hash) + .and_then(|value| value.as_str()) + { + content_hashes.insert(hash.to_string()); + } + } + + let mut writer = self.index_writer.lock().await; + writer + .delete_query(self.attachment_query(account_id, envelope_id)) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + writer + .commit() + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + Ok(content_hashes) + } + + #[cfg(test)] + pub(crate) fn count_by_envelope( + &self, + account_id: u64, + envelope_id: &str, + ) -> BichonResult { + let searcher = self.create_searcher()?; + searcher + .search( + self.attachment_query(account_id, envelope_id).as_ref(), + &Count, + ) + .map(|count| count as u64) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError)) + } + + pub(crate) fn canonical_records_by_envelope( + &self, + account_id: u64, + envelope_id: &str, + ) -> BichonResult> { + let searcher = self.create_searcher()?; + let fields = SchemaTools::attachment_fields(); + let addresses = searcher + .search( + self.attachment_query(account_id, envelope_id).as_ref(), + &DocSetCollector, + ) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + let mut records = Vec::with_capacity(addresses.len()); + for address in addresses { + let doc = searcher + .doc::(address) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + let content_hash = doc + .get_first(fields.f_content_hash) + .and_then(|value| value.as_str()) + .ok_or_else(|| { + raise_error!( + "canonical attachment has no content hash".into(), + ErrorCode::InternalError + ) + })? + .to_string(); + let size = doc + .get_first(fields.f_size) + .and_then(|value| value.as_u64()) + .ok_or_else(|| { + raise_error!( + "canonical attachment has no size".into(), + ErrorCode::InternalError + ) + })?; + let content_type = doc + .get_first(fields.f_content_type) + .and_then(|value| value.as_str()) + .ok_or_else(|| { + raise_error!( + "canonical attachment has no content type".into(), + ErrorCode::InternalError + ) + })? + .to_string(); + let name = doc + .get_first(fields.f_name_exact) + .and_then(|value| value.as_str()) + .map(ToOwned::to_owned); + records.push(CanonicalAttachmentRecord { + content_hash, + name, + size, + content_type, + }); + } + records.sort(); + Ok(records) + } + fn open_or_create_index(index_dir: &PathBuf) -> Index { let need_create = !index_dir.exists() || index_dir diff --git a/crates/core/src/store/tantivy/dedup.rs b/crates/core/src/store/tantivy/dedup.rs index ecd9d698..686bbdd3 100644 --- a/crates/core/src/store/tantivy/dedup.rs +++ b/crates/core/src/store/tantivy/dedup.rs @@ -11,7 +11,7 @@ use crate::raise_error; use crate::store::tantivy::attachment::ATTACHMENT_MANAGER; use crate::store::tantivy::envelope::ENVELOPE_MANAGER; use crate::store::tantivy::fields::{ - F_ACCOUNT_ID, F_CONTENT_HASH, F_ID, F_INGEST_AT, F_MAILBOX_ID, + F_ACCOUNT_ID, F_CONTENT_HASH, F_ID, F_INGEST_AT, F_MAILBOX_ID, F_SHARD_ID, F_UID, }; use crate::store::tantivy::schema::SchemaTools; @@ -30,10 +30,21 @@ struct DedupEntry { email_id: String, } +pub(crate) const UIDONLY_SHARD_ID: u64 = u64::MAX; + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +enum DedupIdentity { + Legacy, + UidOnly(u64), +} + /// Dedup map for one account. -/// Key = (mailbox_id, content_hash) — stable identity across uidvalidity resets +/// Legacy keying remains (mailbox_id, content_hash), preserving the existing +/// UIDVALIDITY-reset behavior. UIDONLY documents use a reserved shard marker +/// and add UID to the identity because RFC 9586 treats distinct UIDs as +/// distinct logical records even when their RFC822 bytes are identical. /// Value = all documents sharing that key, to be reduced to exactly one. -type DedupMap = HashMap<(u64, String), Vec>; +type DedupMap = HashMap<(u64, String, DedupIdentity), Vec>; // ─── Public entry point ─────────────────────────────────────────────────────── @@ -178,6 +189,14 @@ fn dedup_account( .fast_fields() .i64(F_INGEST_AT) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let uid_col = segment_reader + .fast_fields() + .u64(F_UID) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let shard_col = segment_reader + .fast_fields() + .u64(F_SHARD_ID) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; // content_hash and f_id are text fields with FAST; stored as dictionary-encoded strings let hash_col = segment_reader .fast_fields() @@ -202,6 +221,11 @@ fn dedup_account( let mailbox_id = mailbox_col.values.get_val(doc_id); let ingest_at = ingest_col.values.get_val(doc_id); + let identity = if shard_col.values.get_val(doc_id) == UIDONLY_SHARD_ID { + DedupIdentity::UidOnly(uid_col.values.get_val(doc_id)) + } else { + DedupIdentity::Legacy + }; // Read content_hash from the dictionary-encoded string column let hash_ord = hash_col @@ -231,7 +255,7 @@ fn dedup_account( // "DEBUG dedup_account: account={account_id} doc_id={doc_id} mailbox={mailbox_id} hash={content_hash:?} id={email_id:?} ingest_at={ingest_at}" // ); - map.entry((mailbox_id, content_hash)) + map.entry((mailbox_id, content_hash, identity)) .or_default() .push(DedupEntry { ingest_at, @@ -408,6 +432,21 @@ mod tests { mailbox: u64, hash: &str, ingest_at: i64, + ) { + add_email_with_identity(f, w, id, account, mailbox, hash, ingest_at, 0, 0); + } + + #[allow(clippy::too_many_arguments)] + fn add_email_with_identity( + f: &EmailFields, + w: &mut IndexWriter, + id: &str, + account: u64, + mailbox: u64, + hash: &str, + ingest_at: i64, + uid: u64, + shard_id: u64, ) { let mut doc = TantivyDocument::new(); doc.add_text(f.f_id, id); @@ -415,6 +454,8 @@ mod tests { doc.add_u64(f.f_mailbox_id, mailbox); doc.add_text(f.f_content_hash, hash); doc.add_i64(f.f_ingest_at, ingest_at); + doc.add_u64(f.f_uid, uid); + doc.add_u64(f.f_shard_id, shard_id); w.add_document(doc).unwrap(); } @@ -517,6 +558,42 @@ mod tests { .await; } + #[tokio::test] + async fn dedup_preserves_identical_uidonly_bodies_at_distinct_uids() { + Harness::run( + "uidonly-distinct-uids", + |ef, ew, af, aw| { + add_email_with_identity( + ef, + ew, + "uid-10", + 1, + 200, + "hash-same", + 1000, + 10, + UIDONLY_SHARD_ID, + ); + add_email_with_identity( + ef, + ew, + "uid-20", + 1, + 200, + "hash-same", + 2000, + 20, + UIDONLY_SHARD_ID, + ); + add_attachment(af, aw, "att-10", "uid-10", 1, 200); + add_attachment(af, aw, "att-20", "uid-20", 1, 200); + }, + &["uid-10", "uid-20"], + &["att-10", "att-20"], + ) + .await; + } + #[tokio::test] async fn dedup_keeps_latest_among_many_duplicates() { Harness::run( diff --git a/crates/core/src/store/tantivy/envelope.rs b/crates/core/src/store/tantivy/envelope.rs index 9854fbf7..722e9288 100644 --- a/crates/core/src/store/tantivy/envelope.rs +++ b/crates/core/src/store/tantivy/envelope.rs @@ -30,6 +30,7 @@ use crate::{ dashboard::{DashboardStats, Group, LargestEmail, TimeBucket}, error::{code::ErrorCode, BichonResult}, message::{ + content::AttachmentInfo, search::{EmailSearchFilter, SortBy}, tags::{TagAction, TagCount, TagsRequest}, }, @@ -89,6 +90,14 @@ pub struct IndexManager { handle: Mutex>>, } +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CanonicalProjectionRecord { + pub envelope_id: String, + pub content_hash: String, + pub shard_id: u64, + pub attachments: Vec, +} + impl IndexManager { pub(crate) fn index_writer(&self) -> &Arc> { &self.index_writer @@ -229,6 +238,134 @@ impl IndexManager { } } + /// Adds and commits a document before returning. The UIDONLY acquisition + /// path uses this as its canonical projection barrier before advancing a + /// durable checkpoint. + pub(crate) async fn commit_document(&self, doc: TantivyDocument) -> BichonResult<()> { + let mut writer = self.index_writer.lock().await; + writer + .add_document(doc) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + writer + .commit() + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + Ok(()) + } + + pub(crate) fn get_projection_by_uid( + &self, + account_id: u64, + mailbox_id: u64, + uid: u32, + ) -> BichonResult> { + let fields = SchemaTools::email_fields(); + let query = BooleanQuery::new(vec![ + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(fields.f_account_id, account_id), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(fields.f_mailbox_id, mailbox_id), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(fields.f_uid, uid as u64), + IndexRecordOption::Basic, + )), + ), + ]); + let searcher = self.create_searcher()?; + let docs = searcher + .search(&query, &TopDocs::with_limit(2).order_by_score()) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + if docs.len() > 1 { + return Err(raise_error!( + format!( + "multiple canonical records exist for account {account_id}, mailbox {mailbox_id}, UID {uid}" + ), + ErrorCode::Incompatible + )); + } + let Some((_, address)) = docs.first() else { + return Ok(None); + }; + let doc = searcher + .doc::(*address) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + Ok(Some(CanonicalProjectionRecord { + envelope_id: doc + .get_first(fields.f_id) + .and_then(|value| value.as_str()) + .ok_or_else(|| { + raise_error!( + "canonical UID record has no envelope id".into(), + ErrorCode::InternalError + ) + })? + .to_string(), + content_hash: doc + .get_first(fields.f_content_hash) + .and_then(|value| value.as_str()) + .ok_or_else(|| { + raise_error!( + "canonical UID record has no content hash".into(), + ErrorCode::InternalError + ) + })? + .to_string(), + shard_id: doc + .get_first(fields.f_shard_id) + .and_then(|value| value.as_u64()) + .ok_or_else(|| { + raise_error!( + "canonical UID record has no shard id".into(), + ErrorCode::InternalError + ) + })?, + attachments: doc + .get_first(fields.f_attachments) + .and_then(|value| value.as_str()) + .map(serde_json::from_str) + .transpose() + .map_err(|e| { + raise_error!( + format!("canonical UID record has invalid attachment metadata: {e}"), + ErrorCode::InternalError + ) + })? + .unwrap_or_default(), + })) + } + + pub(crate) async fn rollback_uidonly_projection( + &self, + account_id: u64, + envelope_id: &str, + email_content_hash: String, + attachment_content_hashes: HashSet, + ) -> BichonResult<()> { + let mut writer = self.index_writer.lock().await; + writer + .delete_query(self.envelope_query(account_id, envelope_id)) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + writer + .commit() + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + self.cleanup_unused_content( + &mut writer, + HashSet::from([email_content_hash]), + attachment_content_hashes, + ) + } + fn open_or_create_index(index_dir: &PathBuf) -> Index { let need_create = !index_dir.exists() || index_dir From 2aec779ffe424dcaa17b0139f3d8d72664f8be3d Mon Sep 17 00:00:00 2001 From: Gabe Date: Tue, 28 Jul 2026 19:19:25 -0400 Subject: [PATCH 2/4] fix(imap): harden UIDONLY incremental acquisition --- crates/core/src/cache/imap/download/flow.rs | 172 +++++++-- crates/core/src/envelope/extractor.rs | 17 +- crates/core/src/imap/client.rs | 90 ++++- crates/core/src/imap/manager.rs | 26 +- crates/core/src/imap/uidonly_acquisition.rs | 405 +++++++++++++++++++- crates/core/src/store/tantivy/envelope.rs | 94 +++++ 6 files changed, 734 insertions(+), 70 deletions(-) diff --git a/crates/core/src/cache/imap/download/flow.rs b/crates/core/src/cache/imap/download/flow.rs index 73c5fd4f..0db90561 100644 --- a/crates/core/src/cache/imap/download/flow.rs +++ b/crates/core/src/cache/imap/download/flow.rs @@ -36,7 +36,9 @@ use crate::{ compress_uid_list, generate_uid_sequence_hashset, ImapExecutor, DEFAULT_BATCH_SIZE, }, imap::manager::{AcquisitionConnection, ImapConnectionManager}, - imap::uidonly_acquisition::{acquire_bichon_mailbox, AcquisitionLimits}, + imap::uidonly_acquisition::{ + acquire_bichon_mailbox, AcquisitionLimits, AcquisitionReport, + }, settings::dir::DATA_DIR_MANAGER, store::tantivy::envelope::ENVELOPE_MANAGER, }, @@ -47,6 +49,86 @@ use tracing::{debug, error, info, warn}; const MAX_NETWORK_RETRIES: u32 = 3; +async fn build_bounded_acquisition( + account_id: u64, + limits: AcquisitionLimits, + token: &CancellationToken, +) -> BichonResult<(AcquisitionConnection, AcquisitionLimits)> { + let started = Instant::now(); + let response_limits = limits.response_limits()?; + let connection = tokio::select! { + _ = token.cancelled() => return Err(raise_error!( + "UIDONLY acquisition cancelled while connecting".into(), + ErrorCode::InternalError + )), + result = tokio::time::timeout( + limits.max_runtime, + ImapConnectionManager::build_acquisition(account_id, response_limits), + ) => result.map_err(|_| raise_error!( + "UIDONLY acquisition runtime ceiling exceeded while connecting".into(), + ErrorCode::RequestTimeout + ))??, + }; + let remaining = limits + .max_runtime + .checked_sub(started.elapsed()) + .ok_or_else(|| { + raise_error!( + "UIDONLY acquisition runtime ceiling exceeded while connecting".into(), + ErrorCode::RequestTimeout + ) + })?; + Ok(( + connection, + AcquisitionLimits { + max_runtime: remaining, + ..limits + }, + )) +} + +async fn acquire_and_validate_uidonly( + account: &AccountModel, + mailbox: &MailBox, + session: async_imap::Session>, + message_limit: Option, + limits: AcquisitionLimits, + token: CancellationToken, +) -> BichonResult { + let root = DATA_DIR_MANAGER.storage_dir.join("uidonly-acquisition"); + let report = acquire_bichon_mailbox( + account, + mailbox, + session, + message_limit, + &root, + limits, + token, + ) + .await?; + DownloadState::update_folder_progress( + account.id, + mailbox.name.clone(), + report.planned, + report.processed, + if report.success { + FolderStatus::Success + } else { + FolderStatus::Failed + }, + (!report.success).then(|| { + "UIDONLY snapshot contains unresolved UIDs; checkpoint was not advanced".to_string() + }), + )?; + if !report.success { + return Err(raise_error!( + "UIDONLY snapshot incomplete; see durable per-UID ledger".into(), + ErrorCode::ImapUnexpectedResult + )); + } + Ok(report) +} + #[derive(Clone, Debug, Eq, PartialEq)] pub enum FetchDirection { @@ -283,49 +365,25 @@ pub async fn fetch_and_save_full_mailbox( let account_id = account.id; let limits = AcquisitionLimits::for_account(account); - let connection = ImapConnectionManager::build_acquisition( - account_id, - limits.response_limits()?, - ) - .await; + let connection = build_bounded_acquisition(account_id, limits, &token).await; let mut session = match connection { - Ok(AcquisitionConnection::Standard(session)) => session, - Ok(AcquisitionConnection::UidOnly { - session, - message_limit, - }) => { - let root = DATA_DIR_MANAGER.storage_dir.join("uidonly-acquisition"); - let report = acquire_bichon_mailbox( + Ok((AcquisitionConnection::Standard(session), _)) => session, + Ok(( + AcquisitionConnection::UidOnly { + session, + message_limit, + }, + remaining_limits, + )) => { + let report = acquire_and_validate_uidonly( account, mailbox, session, message_limit, - &root, - limits, + remaining_limits, token, ) .await?; - DownloadState::update_folder_progress( - account_id, - mailbox.name.clone(), - report.planned, - report.processed, - if report.success { - FolderStatus::Success - } else { - FolderStatus::Failed - }, - (!report.success).then(|| { - "UIDONLY snapshot contains unresolved UIDs; checkpoint was not advanced" - .to_string() - }), - )?; - if !report.success { - return Err(raise_error!( - "UIDONLY snapshot incomplete; see durable per-UID ledger".into(), - ErrorCode::ImapUnexpectedResult - )); - } let mut updated = mailbox.clone(); updated.uid_validity = Some(report.uid_validity); updated.highest_uid = report.checkpoint; @@ -1066,7 +1124,47 @@ async fn perform_incremental_sync( } }; - let mut session = ImapExecutor::create_connection(account.id).await?; + if account.date_since.is_some() || account.date_before.is_some() { + let mut session = ImapExecutor::create_connection(account.id).await?; + let before_date = account + .date_before + .as_ref() + .map(|r| r.calculate_date()) + .transpose()?; + let new_max_uid = ImapExecutor::fetch_new_mail( + &mut session, + account, + local_mailbox, + start_uid, + before_date.as_deref(), + token, + ) + .await?; + session.logout().await.ok(); + return Ok(new_max_uid.or(local_mailbox.highest_uid)); + } + + let limits = AcquisitionLimits::for_account(account); + let (connection, remaining_limits) = + build_bounded_acquisition(account.id, limits, &token).await?; + let mut session = match connection { + AcquisitionConnection::Standard(session) => session, + AcquisitionConnection::UidOnly { + session, + message_limit, + } => { + let report = acquire_and_validate_uidonly( + account, + remote_mailbox, + session, + message_limit, + remaining_limits, + token, + ) + .await?; + return Ok(report.checkpoint); + } + }; let before_date = account .date_before .as_ref() diff --git a/crates/core/src/envelope/extractor.rs b/crates/core/src/envelope/extractor.rs index 4a24c113..3cb58e45 100644 --- a/crates/core/src/envelope/extractor.rs +++ b/crates/core/src/envelope/extractor.rs @@ -501,18 +501,27 @@ pub(crate) async fn rollback_uidonly_message( account_id: u64, envelope_id: &str, email_content_hash: &str, + raw: Option<&[u8]>, ) -> BichonResult<()> { let mut cleanup_errors = Vec::new(); - let attachment_hashes = match ATTACHMENT_MANAGER + let mut attachment_hashes: std::collections::HashSet = raw + .and_then(|body| MessageParser::new().parse(body)) + .map(|message| { + message + .attachments() + .map(|attachment| compute_content_hash(attachment.contents())) + .collect() + }) + .unwrap_or_default(); + match ATTACHMENT_MANAGER .rollback_documents(account_id, envelope_id) .await { - Ok(hashes) => hashes, + Ok(hashes) => attachment_hashes.extend(hashes), Err(error) => { cleanup_errors.push(error.to_string()); - std::collections::HashSet::new() } - }; + } if let Err(error) = ENVELOPE_MANAGER .rollback_uidonly_projection( account_id, diff --git a/crates/core/src/imap/client.rs b/crates/core/src/imap/client.rs index 85cbceea..9465623f 100644 --- a/crates/core/src/imap/client.rs +++ b/crates/core/src/imap/client.rs @@ -21,10 +21,11 @@ use crate::error::code::ErrorCode; use crate::error::BichonResult; use crate::imap::session::SessionStream; use crate::imap::stats::StatsWrapper; +use crate::raise_error; use crate::utils::net::establish_tcp_connection_with_timeout; use crate::utils::net::establish_tls_connection; use crate::utils::tls::establish_tls_stream; -use crate::raise_error; +use async_imap::types::ResponseLimits; use async_imap::Client as ImapClient; use async_imap::Session as ImapSession; use std::net::SocketAddr; @@ -119,18 +120,44 @@ impl Client { port: u16, use_proxy: Option, dangerous: bool, + ) -> BichonResult { + Self::connection_with_limits(domain, encryption, port, use_proxy, dangerous, None).await + } + + pub(crate) async fn connection_with_limits( + domain: &str, + encryption: &Encryption, + port: u16, + use_proxy: Option, + dangerous: bool, + response_limits: Option, ) -> BichonResult { let resolved_addr = Self::resolve_to_socket_addr(domain, port)?; debug!("Attempting IMAP connection to {domain} ({resolved_addr})."); match encryption { Encryption::Ssl => { - Self::establish_secure_connection(resolved_addr, domain, use_proxy, dangerous).await + Self::establish_secure_connection( + resolved_addr, + domain, + use_proxy, + dangerous, + response_limits, + ) + .await } Encryption::StartTls => { - Self::establish_starttls_connection(resolved_addr, domain, use_proxy, dangerous) - .await + Self::establish_starttls_connection( + resolved_addr, + domain, + use_proxy, + dangerous, + response_limits, + ) + .await + } + Encryption::None => { + Self::establish_insecure_connection(resolved_addr, use_proxy, response_limits).await } - Encryption::None => Self::establish_insecure_connection(resolved_addr, use_proxy).await, } } @@ -139,6 +166,7 @@ impl Client { server_hostname: &str, use_proxy: Option, dangerous: bool, + response_limits: Option, ) -> BichonResult { // Establish the TLS connection with the specified parameters let tls_stream = establish_tls_connection( @@ -156,6 +184,11 @@ impl Client { let session_stream = Box::new(buffered_stream); // Initialize the client with the session stream let mut client = Client::new(session_stream); + if let Some(limits) = response_limits { + client.set_response_limits(limits).map_err(|error| { + raise_error!(format!("{error:#?}"), ErrorCode::InvalidParameter) + })?; + } // Read and validate the greeting response let _greeting = client .read_response() @@ -175,6 +208,7 @@ impl Client { async fn establish_insecure_connection( address: SocketAddr, use_proxy: Option, + response_limits: Option, ) -> BichonResult { // Establish the TCP connection without encryption let tcp_stream = establish_tcp_connection_with_timeout(address, use_proxy).await?; @@ -185,6 +219,11 @@ impl Client { let session_stream: Box = Box::new(buffered_stream); // Initialize the client with the session stream let mut client = Client::new(session_stream); + if let Some(limits) = response_limits { + client.set_response_limits(limits).map_err(|error| { + raise_error!(format!("{error:#?}"), ErrorCode::InvalidParameter) + })?; + } // Read and validate the greeting response let _greeting = client @@ -207,6 +246,7 @@ impl Client { server_hostname: &str, use_proxy: Option, dangerous: bool, + response_limits: Option, ) -> BichonResult { // Establish the initial TCP connection let tcp_stream = establish_tcp_connection_with_timeout(address, use_proxy).await?; @@ -216,6 +256,11 @@ impl Client { // Create a client for communication let mut client = async_imap::Client::new(buffered_tcp_stream); + if let Some(limits) = response_limits { + client.set_response_limits(limits).map_err(|error| { + raise_error!(format!("{error:#?}"), ErrorCode::InvalidParameter) + })?; + } // Read and validate the greeting response let _greeting = client @@ -250,7 +295,12 @@ impl Client { // Create a SessionStream trait object for further communication let session_stream: Box = Box::new(buffered_stream); // Initialize the client with the session stream - let client = Client::new(session_stream); + let mut client = Client::new(session_stream); + if let Some(limits) = response_limits { + client.set_response_limits(limits).map_err(|error| { + raise_error!(format!("{error:#?}"), ErrorCode::InvalidParameter) + })?; + } // Return the established client Ok(client) } @@ -276,3 +326,31 @@ impl Client { }) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::imap::mock_server::MockImapServer; + + #[tokio::test] + async fn acquisition_limits_apply_before_server_greeting() { + let server = MockImapServer::new() + .greeting(format!("* OK {}\r\n", "x".repeat(128))) + .start() + .await; + let error = Client::connection_with_limits( + &server.host(), + &Encryption::None, + server.port(), + None, + false, + Some(ResponseLimits::new(32, 32)), + ) + .await + .unwrap_err(); + assert!( + error.to_string().contains("ResponseTooLarge") + || error.to_string().contains("response") + ); + } +} diff --git a/crates/core/src/imap/manager.rs b/crates/core/src/imap/manager.rs index 2a310014..9fac9276 100644 --- a/crates/core/src/imap/manager.rs +++ b/crates/core/src/imap/manager.rs @@ -26,8 +26,8 @@ use crate::imap::oauth2::OAuth2; use crate::imap::session::SessionStream; use crate::oauth2::token::OAuth2AccessToken; use crate::{bichon_version, decrypt, raise_error}; -use async_imap::Session; use async_imap::types::ResponseLimits; +use async_imap::Session; use tracing::{error, warn}; pub struct ImapConnectionManager; @@ -57,15 +57,19 @@ pub(crate) enum AcquisitionConnection { } impl ImapConnectionManager { - async fn create_client(account: &AccountModel) -> BichonResult { + async fn create_client( + account: &AccountModel, + response_limits: Option, + ) -> BichonResult { assert_eq!(account.account_type, AccountType::IMAP); let imap = account.imap.as_ref().unwrap(); - Client::connection( + Client::connection_with_limits( &imap.host, &imap.encryption, imap.port, imap.use_proxy, account.use_dangerous, + response_limits, ) .await } @@ -119,12 +123,19 @@ impl ImapConnectionManager { } pub async fn build(account_id: u64) -> BichonResult>> { + Self::build_with_limits(account_id, None).await + } + + async fn build_with_limits( + account_id: u64, + response_limits: Option, + ) -> BichonResult>> { let account = AccountModel::get(account_id)?; let account_email = account.email.clone(); let mut client = None; for attempt in 0..3u32 { - match Self::create_client(&account).await { + match Self::create_client(&account, response_limits).await { Ok(c) => { client = Some(c); break; @@ -150,7 +161,10 @@ impl ImapConnectionManager { let client = client.ok_or_else(|| { raise_error!( - format!("Failed to create IMAP {}'s client after 3 attempts", account_email), + format!( + "Failed to create IMAP {}'s client after 3 attempts", + account_email + ), ErrorCode::NetworkError ) })?; @@ -204,7 +218,7 @@ impl ImapConnectionManager { account_id: u64, response_limits: ResponseLimits, ) -> BichonResult { - let mut session = Self::build(account_id).await?; + let mut session = Self::build_with_limits(account_id, Some(response_limits)).await?; let capabilities = fetch_capabilities(&mut session).await?; if acquisition_route(capabilities.has_str("UIDONLY")) == AcquisitionRoute::Standard { diff --git a/crates/core/src/imap/uidonly_acquisition.rs b/crates/core/src/imap/uidonly_acquisition.rs index 74870b6d..ace114a5 100644 --- a/crates/core/src/imap/uidonly_acquisition.rs +++ b/crates/core/src/imap/uidonly_acquisition.rs @@ -187,11 +187,25 @@ pub(crate) struct UidEntry { pub(crate) struct AcquisitionLedger { pub identity: AcquisitionIdentity, pub uid_validity: u32, + #[serde(default = "first_uid")] + pub snapshot_start: u32, pub snapshot_end: u32, pub checkpoint: Option, + #[serde(default)] + pub vanished_ranges: Vec, pub entries: BTreeMap, } +const fn first_uid() -> u32 { + 1 +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct UidRange { + pub start: u32, + pub end: u32, +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) struct Snapshot { pub uid_validity: u32, @@ -250,6 +264,8 @@ pub(crate) trait UidOnlyTransport { #[allow(async_fn_in_trait)] trait CanonicalArchive { + fn begin_epoch(&mut self, uid_validity: u32) -> BichonResult<()>; + fn disk_budget(&self, raw: &[u8]) -> BichonResult; async fn project( @@ -266,12 +282,14 @@ trait CanonicalArchive { uid: u32, content_hash: &str, envelope_id: Option<&str>, + raw: Option<&[u8]>, ) -> BichonResult<()>; } struct BichonCanonicalArchive { account_id: u64, mailbox_id: u64, + uid_validity: Option, } static UIDONLY_CANONICAL_WRITE_LOCK: LazyLock> = @@ -299,18 +317,33 @@ impl BichonCanonicalArchive { Self { account_id, mailbox_id, + uid_validity: None, } } fn envelope_id(&self, uid: u32, content_hash: &str) -> String { - Self::envelope_id_for(self.account_id, self.mailbox_id, uid, content_hash) + Self::envelope_id_for( + self.account_id, + self.mailbox_id, + self.uid_validity + .expect("UIDONLY epoch must be initialized"), + uid, + content_hash, + ) } - fn envelope_id_for(account_id: u64, mailbox_id: u64, uid: u32, content_hash: &str) -> String { + fn envelope_id_for( + account_id: u64, + mailbox_id: u64, + uid_validity: u32, + uid: u32, + content_hash: &str, + ) -> String { let mut hasher = blake3::Hasher::new(); - hasher.update(b"bichon-uidonly-envelope-v1"); + hasher.update(b"bichon-uidonly-envelope-v2"); hasher.update(&account_id.to_be_bytes()); hasher.update(&mailbox_id.to_be_bytes()); + hasher.update(&uid_validity.to_be_bytes()); hasher.update(&uid.to_be_bytes()); hasher.update(content_hash.as_bytes()); format!("uidonly-{}", hasher.finalize().to_hex()) @@ -347,6 +380,11 @@ impl BichonCanonicalArchive { } impl CanonicalArchive for BichonCanonicalArchive { + fn begin_epoch(&mut self, uid_validity: u32) -> BichonResult<()> { + self.uid_validity = Some(uid_validity); + Ok(()) + } + fn disk_budget(&self, raw: &[u8]) -> BichonResult { (raw.len() as u64) .checked_mul(4) @@ -368,6 +406,12 @@ impl CanonicalArchive for BichonCanonicalArchive { ) -> BichonResult { let account_id = self.account_id; let mailbox_id = self.mailbox_id; + let uid_validity = self.uid_validity.ok_or_else(|| { + raise_error!( + "UIDONLY canonical epoch was not initialized".into(), + ErrorCode::InternalError + ) + })?; let body = raw.to_vec(); let task = tokio::spawn(async move { // The task owns the body, shutdown token, and serialization guard. @@ -381,8 +425,10 @@ impl CanonicalArchive for BichonCanonicalArchive { )); } let expected_hash = compute_content_hash(&body); + let envelope_id = + Self::envelope_id_for(account_id, mailbox_id, uid_validity, uid, &expected_hash); if let Some(existing) = - ENVELOPE_MANAGER.get_projection_by_uid(account_id, mailbox_id, uid)? + ENVELOPE_MANAGER.get_projection_by_envelope_id(account_id, &envelope_id)? { return Self::reuse_projection(uid, &expected_hash, existing); } @@ -392,7 +438,6 @@ impl CanonicalArchive for BichonCanonicalArchive { ErrorCode::PayloadTooLarge ) })?; - let envelope_id = Self::envelope_id_for(account_id, mailbox_id, uid, &expected_hash); project_uidonly_message( &body, uid, @@ -411,11 +456,14 @@ impl CanonicalArchive for BichonCanonicalArchive { async fn verify(&self, uid: u32, blob_hash: &str, envelope_id: &str) -> BichonResult { let Some(record) = - ENVELOPE_MANAGER.get_projection_by_uid(self.account_id, self.mailbox_id, uid)? + ENVELOPE_MANAGER.get_projection_by_envelope_id(self.account_id, envelope_id)? else { return Ok(false); }; - if record.envelope_id != envelope_id || record.content_hash != blob_hash { + if record.envelope_id != envelope_id + || record.uid != uid + || record.content_hash != blob_hash + { return Ok(false); } if record.shard_id != UIDONLY_SHARD_ID { @@ -443,12 +491,13 @@ impl CanonicalArchive for BichonCanonicalArchive { uid: u32, content_hash: &str, envelope_id: Option<&str>, + raw: Option<&[u8]>, ) -> BichonResult<()> { let _write_guard = UIDONLY_CANONICAL_WRITE_LOCK.lock().await; let envelope_id = envelope_id .map(ToOwned::to_owned) .unwrap_or_else(|| self.envelope_id(uid, content_hash)); - rollback_uidonly_message(self.account_id, &envelope_id, content_hash).await + rollback_uidonly_message(self.account_id, &envelope_id, content_hash, raw).await } } @@ -459,6 +508,7 @@ pub(crate) struct AcquisitionReport { pub processed: u64, pub checkpoint: Option, pub success: bool, + pub vanished_ranges: Vec, pub states: BTreeMap, #[cfg(test)] state_bytes_written: u64, @@ -478,8 +528,12 @@ struct DurableArchive { struct LedgerMetadata { identity: AcquisitionIdentity, uid_validity: u32, + #[serde(default = "first_uid")] + snapshot_start: u32, snapshot_end: u32, checkpoint: Option, + #[serde(default)] + vanished_ranges: Vec, } #[derive(Serialize, Deserialize)] @@ -600,8 +654,10 @@ impl DurableArchive { AcquisitionLedger { identity: metadata.identity, uid_validity: metadata.uid_validity, + snapshot_start: metadata.snapshot_start, snapshot_end: metadata.snapshot_end, checkpoint: metadata.checkpoint, + vanished_ranges: metadata.vanished_ranges, entries: BTreeMap::new(), } }; @@ -642,8 +698,10 @@ impl DurableArchive { let ledger = AcquisitionLedger { identity, uid_validity, + snapshot_start: 1, snapshot_end, checkpoint: None, + vanished_ranges: Vec::new(), entries: BTreeMap::new(), }; self.persist_metadata(&ledger)?; @@ -654,8 +712,10 @@ impl DurableArchive { let bytes = serde_json::to_vec(&LedgerMetadata { identity: ledger.identity.clone(), uid_validity: ledger.uid_validity, + snapshot_start: ledger.snapshot_start, snapshot_end: ledger.snapshot_end, checkpoint: ledger.checkpoint, + vanished_ranges: ledger.vanished_ranges.clone(), }) .map_err(|e| { raise_error!( @@ -765,6 +825,42 @@ impl DurableArchive { Ok((hash, raw.len() as u64)) } + fn read_staged_raw( + &self, + ledger: &AcquisitionLedger, + uid: u32, + expected_hash: &str, + ) -> BichonResult> { + let record_path = self.epoch_dir.join("records").join(format!("{uid}.json")); + let record: StagingRecord = serde_json::from_slice( + &fs::read(&record_path).map_err(io_error)?, + ) + .map_err(|error| { + raise_error!( + format!("invalid UIDONLY staging record for UID {uid}: {error}"), + ErrorCode::InternalError + ) + })?; + if record.identity != ledger.identity + || record.uid_validity != ledger.uid_validity + || record.uid != uid + || record.blob_hash != expected_hash + { + return Err(raise_error!( + format!("UIDONLY staging record identity mismatch for UID {uid}"), + ErrorCode::Incompatible + )); + } + let raw = fs::read(self.epoch_dir.join("blobs").join(expected_hash)).map_err(io_error)?; + if raw.len() as u64 != record.bytes || compute_content_hash(&raw) != expected_hash { + return Err(raise_error!( + format!("UIDONLY staging blob verification failed for UID {uid}"), + ErrorCode::InternalError + )); + } + Ok(raw) + } + fn reclaim_committed_staging(&self, ledger: &AcquisitionLedger) -> BichonResult<()> { for (&uid, entry) in &ledger.entries { if matches!(entry.state, UidState::Committed { .. }) { @@ -1088,10 +1184,11 @@ async fn cleanup_canonical( uid: u32, content_hash: &str, envelope_id: Option<&str>, + raw: Option<&[u8]>, ) -> BichonResult<()> { tokio::time::timeout( CANONICAL_CLEANUP_GRACE, - canonical.rollback(uid, content_hash, envelope_id), + canonical.rollback(uid, content_hash, envelope_id, raw), ) .await .map_err(|_| { @@ -1102,6 +1199,39 @@ async fn cleanup_canonical( })? } +fn record_vanished_ranges( + ranges: &mut Vec, + additions: impl IntoIterator>, + snapshot_start: u32, + snapshot_end: u32, +) -> bool { + let mut changed = false; + for range in additions { + let start = (*range.start()).max(snapshot_start); + let end = (*range.end()).min(snapshot_end); + if start <= end { + ranges.push(UidRange { start, end }); + changed = true; + } + } + if !changed { + return false; + } + ranges.sort_unstable_by_key(|range| range.start); + let mut merged: Vec = Vec::with_capacity(ranges.len()); + for range in ranges.drain(..) { + if let Some(previous) = merged.last_mut() { + if range.start <= previous.end.saturating_add(1) { + previous.end = previous.end.max(range.end); + continue; + } + } + merged.push(range); + } + *ranges = merged; + changed +} + async fn run_acquisition( transport: &mut T, canonical: &mut C, @@ -1113,9 +1243,16 @@ async fn run_acquisition( ) -> BichonResult { let started = Instant::now(); let snapshot = bounded_transport(transport.snapshot(mailbox), started, limits, &token).await?; + canonical.begin_epoch(snapshot.uid_validity)?; let snapshot_end = snapshot.uid_next.saturating_sub(1); let archive = DurableArchive::open(root, &identity, snapshot.uid_validity, limits)?; let mut ledger = archive.load_or_create(identity, snapshot.uid_validity, snapshot_end)?; + if ledger.checkpoint == Some(ledger.snapshot_end) && snapshot_end > ledger.snapshot_end { + ledger.snapshot_start = ledger.snapshot_end.saturating_add(1); + ledger.snapshot_end = snapshot_end; + ledger.checkpoint = None; + archive.persist_metadata(&ledger)?; + } let existing_canonical_bytes = ledger .entries .values() @@ -1147,7 +1284,8 @@ async fn run_acquisition( }) .collect(); for (uid, blob_hash, canonical_bytes) in interrupted { - cleanup_canonical(canonical, uid, &blob_hash, None).await?; + let raw = archive.read_staged_raw(&ledger, uid, &blob_hash)?; + cleanup_canonical(canonical, uid, &blob_hash, None, Some(&raw)).await?; archive.release_disk(canonical_bytes); ledger.entries.get_mut(&uid).unwrap().state = UidState::Missing; archive.persist_entry(uid, &ledger.entries[&uid])?; @@ -1191,7 +1329,7 @@ async fn run_acquisition( None => false, }; if !valid { - cleanup_canonical(canonical, uid, &blob_hash, envelope_id.as_deref()).await?; + cleanup_canonical(canonical, uid, &blob_hash, envelope_id.as_deref(), None).await?; archive.release_disk(canonical_bytes); ledger.entries.get_mut(&uid).unwrap().state = UidState::Failed { reason: "committed canonical record or blob failed restart validation".into(), @@ -1208,7 +1346,7 @@ async fn run_acquisition( let snapshot_end = ledger.snapshot_end; let page_size = limits.page_size.max(1); - let mut first_uid = 1u32; + let mut first_uid = ledger.snapshot_start; while first_uid <= snapshot_end { validate_runtime(started, limits, &token)?; let page = bounded_transport( @@ -1218,6 +1356,21 @@ async fn run_acquisition( &token, ) .await?; + if page.items.len() > page_size as usize { + return Err(raise_error!( + format!( + "UIDONLY server returned {} inventory items for requested page size {page_size}", + page.items.len() + ), + ErrorCode::ImapUnexpectedResult + )); + } + let metadata_changed = record_vanished_ranges( + &mut ledger.vanished_ranges, + page.vanished.iter().cloned(), + ledger.snapshot_start, + snapshot_end, + ); for range in page.vanished { let mut changed = Vec::new(); for (&uid, entry) in ledger.entries.range_mut(range) { @@ -1230,6 +1383,18 @@ async fn run_acquisition( archive.persist_entry(uid, &ledger.entries[&uid])?; } } + if ledger.vanished_ranges.len() > limits.max_messages { + return Err(raise_error!( + format!( + "UIDONLY VANISHED range ceiling {} exceeded", + limits.max_messages + ), + ErrorCode::PayloadTooLarge + )); + } + if metadata_changed { + archive.persist_metadata(&ledger)?; + } if page.items.is_empty() { break; } @@ -1425,6 +1590,7 @@ async fn run_acquisition( uid, &blob_hash, Some(&projection.envelope_id), + Some(&raw), ) .await?; archive.release_disk(budget); @@ -1439,6 +1605,7 @@ async fn run_acquisition( uid, &blob_hash, Some(&projection.envelope_id), + Some(&raw), ) .await?; archive.release_disk(budget); @@ -1454,6 +1621,7 @@ async fn run_acquisition( uid, &blob_hash, Some(&projection.envelope_id), + Some(&raw), ) .await?; archive.release_disk(budget); @@ -1463,8 +1631,14 @@ async fn run_acquisition( } Err(failure) => { if !failure.cleanup_pending { - cleanup_canonical(canonical, uid, &blob_hash, None) - .await?; + cleanup_canonical( + canonical, + uid, + &blob_hash, + None, + Some(&raw), + ) + .await?; } archive.release_disk(budget); let error = failure.error; @@ -1521,7 +1695,7 @@ async fn run_acquisition( ) .await? { - cleanup_canonical(canonical, uid, &blob_hash, Some(&envelope_id)).await?; + cleanup_canonical(canonical, uid, &blob_hash, Some(&envelope_id), None).await?; archive.release_disk(canonical_bytes); ledger.entries.get_mut(&uid).unwrap().state = UidState::Failed { reason: "canonical record failed final checkpoint revalidation".into(), @@ -1554,6 +1728,7 @@ async fn run_acquisition( processed, checkpoint: ledger.checkpoint, success, + vanished_ranges: ledger.vanished_ranges, states: ledger .entries .into_iter() @@ -1668,6 +1843,11 @@ impl UidOnlyTransport for SessionUidOnlyTransport { .map_err(classify_transport)?; let mut items = Vec::new(); while let Some(fetch) = stream.try_next().await.map_err(classify_transport)? { + if items.len() >= page_size as usize { + return Err(TransportFailure::command(format!( + "server exceeded requested UIDONLY inventory page size {page_size}" + ))); + } items.push(InventoryItem { uid: fetch.uid, size: fetch.size.map(u64::from), @@ -1898,6 +2078,37 @@ mod tests { } } + struct OverfullInventoryTransport; + + impl UidOnlyTransport for OverfullInventoryTransport { + async fn snapshot(&mut self, _mailbox: &str) -> Result { + Ok(Snapshot { + uid_validity: 9, + uid_next: 4, + }) + } + + async fn inventory_page( + &mut self, + _first_uid: u32, + _snapshot_end: u32, + _page_size: u32, + ) -> Result { + Ok(InventoryPage { + items: vec![item(1, 1), item(2, 1), item(3, 1)], + vanished: Vec::new(), + }) + } + + async fn fetch_uid(&mut self, _uid: u32) -> Result { + panic!("overfull inventory must be rejected before body fetch") + } + + async fn reconnect(&mut self) -> Result<(), TransportFailure> { + Ok(()) + } + } + struct HangingTransport; impl UidOnlyTransport for HangingTransport { @@ -1935,9 +2146,14 @@ mod tests { verify_calls: Cell, active_projects: usize, quiesced_projects: Vec, + rollback_raw_hashes: Vec<(u32, String)>, } impl CanonicalArchive for FakeCanonicalArchive { + fn begin_epoch(&mut self, _uid_validity: u32) -> BichonResult<()> { + Ok(()) + } + fn disk_budget(&self, raw: &[u8]) -> BichonResult { Ok(self.disk_budget_override.unwrap_or(raw.len() as u64 + 128)) } @@ -1991,7 +2207,12 @@ mod tests { uid: u32, _content_hash: &str, _envelope_id: Option<&str>, + raw: Option<&[u8]>, ) -> BichonResult<()> { + if let Some(raw) = raw { + self.rollback_raw_hashes + .push((uid, compute_content_hash(raw))); + } self.records.remove(&uid); Ok(()) } @@ -2016,6 +2237,7 @@ mod tests { "expected-hash", crate::store::tantivy::envelope::CanonicalProjectionRecord { envelope_id: "legacy-envelope".into(), + uid: 7, content_hash: "expected-hash".into(), shard_id: 0, attachments: Vec::new(), @@ -2045,6 +2267,13 @@ mod tests { assert_ne!(expected, actual); } + #[test] + fn canonical_identity_is_scoped_to_uidvalidity_epoch() { + let first = BichonCanonicalArchive::envelope_id_for(7, 11, 9, 42, "same-hash"); + let second = BichonCanonicalArchive::envelope_id_for(7, 11, 10, 42, "same-hash"); + assert_ne!(first, second); + } + fn identity() -> AcquisitionIdentity { AcquisitionIdentity { endpoint: "imap.invalid:993".into(), @@ -2712,6 +2941,12 @@ mod tests { .unwrap(); assert!(report.success); assert!(canonical.records.contains_key(&1)); + assert!( + canonical + .rollback_raw_hashes + .contains(&(1, compute_content_hash(b"mail"))), + "restart rollback must recover attachment cleanup evidence from staged raw bytes" + ); fs::remove_dir_all(root).unwrap(); } @@ -2758,6 +2993,7 @@ AQIDBA==\r\n\ }]) .unwrap(); let mut canonical = BichonCanonicalArchive::new(account_id, mailbox_id); + canonical.begin_epoch(9).unwrap(); let first = canonical .project(first_uid, raw, None, CancellationToken::new()) .await @@ -2807,6 +3043,22 @@ AQIDBA==\r\n\ .unwrap()); } + canonical.begin_epoch(10).unwrap(); + let next_epoch = canonical + .project(first_uid, raw, None, CancellationToken::new()) + .await + .unwrap(); + assert_ne!(first.envelope_id, next_epoch.envelope_id); + let next_epoch_record = ENVELOPE_MANAGER + .get_projection_by_envelope_id(account_id, &next_epoch.envelope_id) + .unwrap() + .expect("UID reuse in a new UIDVALIDITY epoch must create a distinct record"); + assert_eq!(next_epoch_record.uid, first_uid); + assert!(canonical + .verify(first_uid, &next_epoch.content_hash, &next_epoch.envelope_id) + .await + .unwrap()); + let failed_uid = 79; // The failed writer reuses the exact email and attachment blobs owned // by two committed projections. Rollback must remove only its index @@ -2998,6 +3250,125 @@ AQIDBA==\r\n\ .states .values() .all(|state| matches!(state, UidState::Vanished))); + assert_eq!( + report.vanished_ranges, + vec![UidRange { + start: 1, + end: u32::MAX - 1, + }] + ); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn unseen_vanished_range_is_durable_compact_evidence() { + let root = temp_root("unseen-vanished-evidence"); + let report = run_acquisition( + &mut HugeVanishedTransport, + &mut FakeCanonicalArchive::default(), + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(report.success); + assert_eq!((report.planned, report.processed), (0, 0)); + assert_eq!( + report.vanished_ranges, + vec![UidRange { + start: 1, + end: u32::MAX - 1, + }] + ); + let archive = DurableArchive::open(&root, &identity(), 9, limits()).unwrap(); + let ledger = archive.load_or_create(identity(), 9, u32::MAX - 1).unwrap(); + assert_eq!(ledger.vanished_ranges, report.vanished_ranges); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn server_cannot_overrun_requested_partial_page() { + let root = temp_root("overfull-partial"); + let error = run_acquisition( + &mut OverfullInventoryTransport, + &mut FakeCanonicalArchive::default(), + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap_err(); + assert_eq!(error.code(), ErrorCode::ImapUnexpectedResult); + let archive = DurableArchive::open(&root, &identity(), 9, limits()).unwrap(); + let ledger = archive.load_or_create(identity(), 9, 3).unwrap(); + assert_eq!(ledger.checkpoint, None); + assert!(ledger.entries.is_empty()); + fs::remove_dir_all(root).unwrap(); + } + + #[tokio::test] + async fn completed_snapshot_extends_from_prior_checkpoint() { + let root = temp_root("periodic-window"); + let mut canonical = FakeCanonicalArchive::default(); + let mut first = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 2, + }, + inventory: vec![item(1, 1)], + outcomes: [(1, VecDeque::from([message(b"a")]))].into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + assert!( + run_acquisition( + &mut first, + &mut canonical, + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap() + .success + ); + + let mut second = FakeTransport { + snapshot: Snapshot { + uid_validity: 9, + uid_next: 3, + }, + inventory: vec![item(1, 1), item(2, 1)], + outcomes: [(2, VecDeque::from([message(b"b")]))].into(), + vanished_on_inventory: BTreeSet::new(), + expunge_after_first_page: None, + reconnects: 0, + page_requests: Vec::new(), + }; + let report = run_acquisition( + &mut second, + &mut canonical, + "INBOX", + identity(), + &root, + limits(), + CancellationToken::new(), + ) + .await + .unwrap(); + assert!(report.success); + assert_eq!(report.checkpoint, Some(2)); + assert_eq!(second.page_requests.first(), Some(&(2, 2, 2))); + assert_eq!(canonical.projected_uids, vec![1, 2]); fs::remove_dir_all(root).unwrap(); } @@ -3085,8 +3456,8 @@ AQIDBA==\r\n\ ) .await; match case { - "messages" | "disk" => assert!(result.is_err()), - "total-bytes" => { + "messages" => assert!(result.is_err()), + "total-bytes" | "disk" => { let report = result.unwrap(); assert!(!report.success); assert_eq!(report.checkpoint, None); diff --git a/crates/core/src/store/tantivy/envelope.rs b/crates/core/src/store/tantivy/envelope.rs index 722e9288..f17ca606 100644 --- a/crates/core/src/store/tantivy/envelope.rs +++ b/crates/core/src/store/tantivy/envelope.rs @@ -93,6 +93,7 @@ pub struct IndexManager { #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct CanonicalProjectionRecord { pub envelope_id: String, + pub uid: u32, pub content_hash: String, pub shard_id: u64, pub attachments: Vec, @@ -252,6 +253,7 @@ impl IndexManager { Ok(()) } + #[cfg(test)] pub(crate) fn get_projection_by_uid( &self, account_id: u64, @@ -311,6 +313,98 @@ impl IndexManager { ) })? .to_string(), + uid: doc + .get_first(fields.f_uid) + .and_then(|value| value.as_u64()) + .and_then(|value| u32::try_from(value).ok()) + .ok_or_else(|| { + raise_error!( + "canonical UID record has no valid UID".into(), + ErrorCode::InternalError + ) + })?, + content_hash: doc + .get_first(fields.f_content_hash) + .and_then(|value| value.as_str()) + .ok_or_else(|| { + raise_error!( + "canonical UID record has no content hash".into(), + ErrorCode::InternalError + ) + })? + .to_string(), + shard_id: doc + .get_first(fields.f_shard_id) + .and_then(|value| value.as_u64()) + .ok_or_else(|| { + raise_error!( + "canonical UID record has no shard id".into(), + ErrorCode::InternalError + ) + })?, + attachments: doc + .get_first(fields.f_attachments) + .and_then(|value| value.as_str()) + .map(serde_json::from_str) + .transpose() + .map_err(|e| { + raise_error!( + format!("canonical UID record has invalid attachment metadata: {e}"), + ErrorCode::InternalError + ) + })? + .unwrap_or_default(), + })) + } + + pub(crate) fn get_projection_by_envelope_id( + &self, + account_id: u64, + envelope_id: &str, + ) -> BichonResult> { + let fields = SchemaTools::email_fields(); + let searcher = self.create_searcher()?; + let docs = searcher + .search( + self.envelope_query(account_id, envelope_id).as_ref(), + &TopDocs::with_limit(2).order_by_score(), + ) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + if docs.len() > 1 { + return Err(raise_error!( + format!( + "multiple canonical records exist for account {account_id}, envelope {envelope_id}" + ), + ErrorCode::Incompatible + )); + } + let Some((_, address)) = docs.first() else { + return Ok(None); + }; + let doc = searcher + .doc::(*address) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + Ok(Some(CanonicalProjectionRecord { + envelope_id: doc + .get_first(fields.f_id) + .and_then(|value| value.as_str()) + .ok_or_else(|| { + raise_error!( + "canonical UID record has no envelope id".into(), + ErrorCode::InternalError + ) + })? + .to_string(), + uid: doc + .get_first(fields.f_uid) + .and_then(|value| value.as_u64()) + .and_then(|value| u32::try_from(value).ok()) + .ok_or_else(|| { + raise_error!( + "canonical UID record has no valid UID".into(), + ErrorCode::InternalError + ) + })?, content_hash: doc .get_first(fields.f_content_hash) .and_then(|value| value.as_str()) From 6bf3e331d318d3c11cfcc15f375ff4a68b4de2bf Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 29 Jul 2026 10:07:57 -0400 Subject: [PATCH 3/4] test(imap): add disposable Cyrus UIDONLY harness --- crates/core/src/imap/uidonly_acquisition.rs | 16 ++- crates/core/tests/cyrus/Dockerfile | 51 +++++++ crates/core/tests/cyrus/README.md | 30 ++++ .../core/tests/cyrus/container-entrypoint.sh | 20 +++ crates/core/tests/cyrus/cyrus.conf | 9 ++ crates/core/tests/cyrus/imapd.conf | 10 ++ crates/core/tests/cyrus/run.sh | 135 ++++++++++++++++++ 7 files changed, 265 insertions(+), 6 deletions(-) create mode 100644 crates/core/tests/cyrus/Dockerfile create mode 100644 crates/core/tests/cyrus/README.md create mode 100755 crates/core/tests/cyrus/container-entrypoint.sh create mode 100644 crates/core/tests/cyrus/cyrus.conf create mode 100644 crates/core/tests/cyrus/imapd.conf create mode 100755 crates/core/tests/cyrus/run.sh diff --git a/crates/core/src/imap/uidonly_acquisition.rs b/crates/core/src/imap/uidonly_acquisition.rs index ace114a5..b3dbeadd 100644 --- a/crates/core/src/imap/uidonly_acquisition.rs +++ b/crates/core/src/imap/uidonly_acquisition.rs @@ -3928,7 +3928,7 @@ AQIDBA==\r\n\ } #[tokio::test] - #[ignore = "requires an explicitly provisioned disposable localhost Cyrus instance"] + #[ignore = "run crates/core/tests/cyrus/run.sh"] async fn cyrus_uidonly_exact_raw_roundtrip() { let port: u16 = std::env::var("BICHON_CYRUS_PORT") .expect("BICHON_CYRUS_PORT") @@ -3940,30 +3940,34 @@ AQIDBA==\r\n\ assert!(root.is_absolute()); fs::create_dir_all(&root).unwrap(); - let connect = || async move { + let connect = |username: &'static str| async move { let stream = TcpStream::connect(("127.0.0.1", port)).await.unwrap(); let mut client = async_imap::Client::new(Box::new(TestStream(stream)) as Box); client.read_response().await.unwrap().unwrap(); client - .login("archive-test", "synthetic-only-password") + .login(username, "synthetic-only-password") .await .map_err(|(error, _)| error) .unwrap() }; + let mut admin = connect("cyrus").await; + admin.create("user/archive-test").await.unwrap(); + admin.logout().await.unwrap(); + let raw_messages: [&[u8]; 3] = [ b"From: one@example.invalid\r\nTo: archive@example.invalid\r\nSubject: one\r\n\r\nfirst\r\n", b"From: two@example.invalid\r\nTo: archive@example.invalid\r\nSubject: two\r\n\r\nsecond\r\n", b"From: three@example.invalid\r\nTo: archive@example.invalid\r\nSubject: three\r\n\r\nthird\r\n", ]; - let mut seed = connect().await; + let mut seed = connect("archive-test").await; for raw in raw_messages { seed.append("INBOX", None, None, raw).await.unwrap(); } seed.logout().await.unwrap(); - let mut session = connect().await; + let mut session = connect("archive-test").await; let capabilities = session.capabilities().await.unwrap(); assert!(capabilities.has_str("UIDONLY")); assert!(capabilities.has_str("PARTIAL")); @@ -4001,7 +4005,7 @@ AQIDBA==\r\n\ assert_eq!((report.planned, report.processed), (3, 3)); drop(transport); - let mut restart_session = connect().await; + let mut restart_session = connect("archive-test").await; restart_session .set_response_limits(response_limits) .unwrap(); diff --git a/crates/core/tests/cyrus/Dockerfile b/crates/core/tests/cyrus/Dockerfile new file mode 100644 index 00000000..30591fb0 --- /dev/null +++ b/crates/core/tests/cyrus/Dockerfile @@ -0,0 +1,51 @@ +FROM debian:bookworm-slim + +ARG CYRUS_SOURCE=cyrus-imapd-3.12.2.tar.gz +ARG CYRUS_SHA256=681ca57483b3dd9ee91f171e11e5ee21684d1da87262e27ea6ff9bd076e9514d + +LABEL org.opencontainers.image.version="3.12.2" \ + org.opencontainers.image.source="https://github.com/cyrusimap/cyrus-imapd" \ + bichon.cyrus.source-sha256="681ca57483b3dd9ee91f171e11e5ee21684d1da87262e27ea6ff9bd076e9514d" + +COPY ${CYRUS_SOURCE} /tmp/${CYRUS_SOURCE} + +RUN set -eu; \ + echo "${CYRUS_SHA256} /tmp/${CYRUS_SOURCE}" | sha256sum -c -; \ + apt-get update; \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + autoconf automake bison build-essential flex gosu libical-dev \ + libjansson-dev libkrb5-dev libpcre2-dev libsasl2-dev libsasl2-modules \ + libsqlite3-dev libssl-dev libtool libxml2-dev pkg-config sasl2-bin \ + uuid-dev zlib1g-dev; \ + groupadd --force mail; \ + groupadd --system cyrus; \ + useradd --system --gid cyrus --groups mail --home-dir /var/lib/cyrus \ + --shell /usr/sbin/nologin cyrus; \ + mkdir -p /usr/src/cyrus; \ + tar -xzf /tmp/${CYRUS_SOURCE} -C /usr/src/cyrus --strip-components=1; \ + cd /usr/src/cyrus; \ + ./configure \ + --prefix=/usr/local/cyrus \ + --with-cyrus-user=cyrus \ + --with-sqlite=yes \ + --without-clamav \ + --without-zeroskip \ + --without-chardet \ + --without-cld2 \ + --without-guesstz \ + --without-nghttp2 \ + --without-wslay \ + --without-brotli \ + --without-zstd; \ + make -j2; \ + make install; \ + /usr/local/cyrus/sbin/cyr_buildinfo \ + | grep -Eq '"CYRUS_VERSION"[[:space:]]*:[[:space:]]*"3\.12\.2"'; \ + rm -rf /usr/src/cyrus /tmp/${CYRUS_SOURCE} /var/lib/apt/lists/* + +COPY container-entrypoint.sh /usr/local/sbin/bichon-cyrus-entrypoint +RUN chmod 0755 /usr/local/sbin/bichon-cyrus-entrypoint + +EXPOSE 143 + +ENTRYPOINT ["/usr/local/sbin/bichon-cyrus-entrypoint"] diff --git a/crates/core/tests/cyrus/README.md b/crates/core/tests/cyrus/README.md new file mode 100644 index 00000000..b4030a19 --- /dev/null +++ b/crates/core/tests/cyrus/README.md @@ -0,0 +1,30 @@ +# Cyrus UIDONLY interoperability test + +This opt-in test runs Bichon's UIDONLY acquisition path against a disposable +Cyrus IMAP 3.12.2 server. It uses three synthetic messages and verifies exact +raw-message acquisition, restart behavior, canonical records, and staging +cleanup. + +Run it from the repository root: + +```sh +crates/core/tests/cyrus/run.sh +``` + +The first run downloads the Cyrus 3.12.2 source release, verifies its SHA-256, +and builds a local test image. Docker binds the server to a random localhost +port and uses two uniquely named volumes. The script removes its container, +volumes, temporary archive, and newly built image when it exits. + +Set `BICHON_CYRUS_KEEP_IMAGE=1` to retain a newly built image for later runs, or +set `BICHON_CYRUS_IMAGE` to use an existing compatible Cyrus 3.12.2 image. + +Requirements: + +- Docker +- curl +- Python 3 +- the Rust toolchain used to build Bichon + +Cyrus implements UIDONLY and PARTIAL but not MESSAGELIMIT. MESSAGELIMIT and +malformed-response behavior remain covered by deterministic Rust tests. diff --git a/crates/core/tests/cyrus/container-entrypoint.sh b/crates/core/tests/cyrus/container-entrypoint.sh new file mode 100755 index 00000000..37abe0b6 --- /dev/null +++ b/crates/core/tests/cyrus/container-entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -eu + +state_uid="$(id -u cyrus)" +state_gid="$(id -g cyrus)" +for state_dir in /var/lib/cyrus /var/spool/cyrus/mail; do + mkdir -p "$state_dir" + if [ -z "$(find "$state_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]; then + chown "$state_uid:$state_gid" "$state_dir" + fi + actual="$(stat -c '%u:%g' "$state_dir")" + if [ "$actual" != "$state_uid:$state_gid" ]; then + echo "unexpected Cyrus state owner for $state_dir" >&2 + exit 78 + fi +done + +exec /usr/sbin/gosu cyrus:cyrus \ + /usr/local/cyrus/libexec/master -D \ + -C /etc/imapd.conf -M /etc/cyrus.conf diff --git a/crates/core/tests/cyrus/cyrus.conf b/crates/core/tests/cyrus/cyrus.conf new file mode 100644 index 00000000..f7ffb10f --- /dev/null +++ b/crates/core/tests/cyrus/cyrus.conf @@ -0,0 +1,9 @@ +START { + recover cmd="/usr/local/cyrus/sbin/ctl_cyrusdb -r" +} +SERVICES { + imap cmd="/usr/local/cyrus/libexec/imapd -N" listen="143" prefork=1 +} +EVENTS { + checkpoint cmd="/usr/local/cyrus/sbin/ctl_cyrusdb -c" period=30 +} diff --git a/crates/core/tests/cyrus/imapd.conf b/crates/core/tests/cyrus/imapd.conf new file mode 100644 index 00000000..d4a0c2f0 --- /dev/null +++ b/crates/core/tests/cyrus/imapd.conf @@ -0,0 +1,10 @@ +configdirectory: /var/lib/cyrus +partition-default: /var/spool/cyrus/mail +admins: cyrus +servername: synthetic.invalid +sasl_pwcheck_method: auxprop +sasl_auxprop_plugin: sasldb +sasl_sasldb_path: /var/lib/cyrus/sasldb2 +allowplaintext: yes +autocreate_quota: -1 +unixhierarchysep: yes diff --git a/crates/core/tests/cyrus/run.sh b/crates/core/tests/cyrus/run.sh new file mode 100755 index 00000000..f849fe94 --- /dev/null +++ b/crates/core/tests/cyrus/run.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +set -euo pipefail + +harness_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "$harness_dir/../../../.." && pwd)" +source_url="https://github.com/cyrusimap/cyrus-imapd/releases/download/cyrus-imapd-3.12.2/cyrus-imapd-3.12.2.tar.gz" +source_sha256="681ca57483b3dd9ee91f171e11e5ee21684d1da87262e27ea6ff9bd076e9514d" +image="${BICHON_CYRUS_IMAGE:-bichon-cyrus-test:3.12.2}" +cargo_bin="${CARGO:-cargo}" +run_id="bichon-cyrus-uidonly-$$" +container="$run_id" +state_volume="${run_id}-state" +spool_volume="${run_id}-spool" +work_root="$(mktemp -d "${TMPDIR:-/tmp}/bichon-cyrus-uidonly.XXXXXX")" +built_image=0 + +cleanup() { + status=$? + trap - EXIT + + if docker container inspect "$container" >/dev/null 2>&1; then + if [ "$status" -ne 0 ]; then + docker logs --tail 80 "$container" >&2 || true + fi + if [ "$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)" = "true" ]; then + docker stop -t 20 "$container" >/dev/null || true + fi + docker rm "$container" >/dev/null 2>&1 || true + fi + docker volume rm "$state_volume" "$spool_volume" >/dev/null 2>&1 || true + if [ "$built_image" -eq 1 ] && [ "${BICHON_CYRUS_KEEP_IMAGE:-0}" != "1" ]; then + docker image rm "$image" >/dev/null 2>&1 || true + fi + rm -rf "$work_root" + exit "$status" +} +trap cleanup EXIT +trap 'exit 130' INT TERM + +for command in docker curl python3 "$cargo_bin"; do + if ! command -v "$command" >/dev/null 2>&1; then + echo "required command not found: $command" >&2 + exit 2 + fi +done + +if ! docker image inspect "$image" >/dev/null 2>&1; then + echo "Building pinned Cyrus 3.12.2 test image..." + curl --fail --location --silent --show-error \ + "$source_url" \ + --output "$work_root/cyrus-imapd-3.12.2.tar.gz" + actual_sha256="$( + python3 - "$work_root/cyrus-imapd-3.12.2.tar.gz" <<'PY' +import hashlib +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +digest = hashlib.sha256() +with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + digest.update(block) +print(digest.hexdigest()) +PY + )" + if [ "$actual_sha256" != "$source_sha256" ]; then + echo "Cyrus source checksum mismatch" >&2 + exit 2 + fi + cp "$harness_dir/Dockerfile" "$work_root/Dockerfile" + cp "$harness_dir/container-entrypoint.sh" "$work_root/container-entrypoint.sh" + docker build --tag "$image" "$work_root" + built_image=1 +fi + +docker volume create --label "bichon.test=$run_id" "$state_volume" >/dev/null +docker volume create --label "bichon.test=$run_id" "$spool_volume" >/dev/null +docker run --detach \ + --name "$container" \ + --label "bichon.test=$run_id" \ + --read-only \ + --tmpfs /run:rw,nosuid,nodev,noexec,size=16m \ + --tmpfs /tmp:rw,nosuid,nodev,noexec,size=16m \ + --publish 127.0.0.1::143 \ + --mount "type=volume,source=$state_volume,target=/var/lib/cyrus" \ + --mount "type=volume,source=$spool_volume,target=/var/spool/cyrus" \ + --mount "type=bind,source=$harness_dir/imapd.conf,target=/etc/imapd.conf,readonly" \ + --mount "type=bind,source=$harness_dir/cyrus.conf,target=/etc/cyrus.conf,readonly" \ + "$image" >/dev/null + +port_line="$(docker port "$container" 143/tcp)" +case "$port_line" in + 127.0.0.1:*) port="${port_line##*:}" ;; + *) + echo "Cyrus was not bound exclusively to localhost: $port_line" >&2 + exit 2 + ;; +esac + +printf '%s\n' 'synthetic-only-password' \ + | docker exec -i "$container" saslpasswd2 -p -c \ + -f /var/lib/cyrus/sasldb2 -u synthetic.invalid cyrus +printf '%s\n' 'synthetic-only-password' \ + | docker exec -i "$container" saslpasswd2 -p -c \ + -f /var/lib/cyrus/sasldb2 -u synthetic.invalid archive-test +docker exec "$container" chown cyrus:mail /var/lib/cyrus/sasldb2 +docker exec "$container" chmod 640 /var/lib/cyrus/sasldb2 + +python3 - "$port" <<'PY' +import socket +import sys +import time + +port = int(sys.argv[1]) +deadline = time.monotonic() + 20 +while time.monotonic() < deadline: + try: + with socket.create_connection(("127.0.0.1", port), timeout=1) as stream: + if stream.recv(4096).startswith(b"* OK"): + break + except OSError: + time.sleep(0.1) +else: + raise SystemExit("Cyrus did not become ready") +PY + +echo "Running Bichon UIDONLY Cyrus interoperability test..." +( + cd "$repo_root" + BICHON_CYRUS_PORT="$port" \ + BICHON_CYRUS_ARCHIVE_ROOT="$work_root/archive" \ + "$cargo_bin" test -p bichon-core --lib --no-default-features --locked \ + imap::uidonly_acquisition::tests::cyrus_uidonly_exact_raw_roundtrip \ + -- --ignored --exact --test-threads=1 +) From 09e7c75c5568b02f0bae33a6c53db797d11decdd Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 29 Jul 2026 10:37:14 -0400 Subject: [PATCH 4/4] chore(imap): rebase UIDONLY dependency stack --- Cargo.lock | 6 +++--- crates/core/Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8be51953..afc1e572 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -219,8 +219,8 @@ dependencies = [ [[package]] name = "async-imap" -version = "0.11.2" -source = "git+https://github.com/gabeosx/async-imap.git?rev=afdb7a51a46a75b175b8aaa642f7ad1328ce868b#afdb7a51a46a75b175b8aaa642f7ad1328ce868b" +version = "0.11.3" +source = "git+https://github.com/gabeosx/async-imap.git?rev=905a005234f89d046c2f439cee27daf947917b41#905a005234f89d046c2f439cee27daf947917b41" dependencies = [ "async-channel 2.5.0", "async-compression", @@ -2222,7 +2222,7 @@ dependencies = [ [[package]] name = "imap-proto" version = "0.17.0" -source = "git+https://github.com/gabeosx/tokio-imap?rev=68a4e1dc6beffcb82b9ade4b818d2af8b8594649#68a4e1dc6beffcb82b9ade4b818d2af8b8594649" +source = "git+https://github.com/gabeosx/tokio-imap?rev=05439a90033d67297892c3fe206b8c4285df3821#05439a90033d67297892c3fe206b8c4285df3821" dependencies = [ "nom 7.1.3", ] diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 60d346a0..4484f1fe 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -43,7 +43,7 @@ sysinfo.workspace = true num_cpus.workspace = true rand.workspace = true encoding_rs.workspace = true -async-imap = { git = "https://github.com/gabeosx/async-imap.git", rev = "afdb7a51a46a75b175b8aaa642f7ad1328ce868b", default-features = false, features = [ +async-imap = { git = "https://github.com/gabeosx/async-imap.git", rev = "905a005234f89d046c2f439cee27daf947917b41", default-features = false, features = [ "runtime-tokio", "compress", ] }