perf: Round 2 — Remove poll loop, fire-and-forget with retry, mtime skip-early, combined Redis FCALL

src/lua/rag_helpers.lua

@@ -15,47 +15,69 @@ redis.register_function('get_formatted_context', function(keys, args)
     return '> **' .. label .. ':**\n> ' .. ctx .. '\n\n'
 end)
 
--- Function 2: check_file_exists
--- Returns 1 if key exists, 0 if not.
-redis.register_function('check_file_exists', function(keys, args)
-    return redis.call('EXISTS', keys[1])
-end)
-
--- Function 3: verify_file_hash
--- Compares the stored content_hash with the provided local hash.
--- Returns 1 on match, 0 on mismatch or missing.
-redis.register_function('verify_file_hash', function(keys, args)
+-- Function 2: check_and_compare (COMBINED — replaces check_file_exists + verify_file_hash)
+-- Single round-trip: checks existence, compares mtime+size, then hash if needed.
+-- Args: local_mtime, local_size, local_content_hash
+-- Returns:
+--   0 = file does not exist in Redis (new file)
+--   1 = file exists AND hash matches AND not dirty (skip)
+--   2 = file exists BUT hash/metadata differs OR dirty (needs update)
+redis.register_function('check_and_compare', function(keys, args)
     local key = keys[1]
-    local local_hash = args[1]
-    local stored = redis.call('HGET', key, 'content_hash')
-    if not stored or stored ~= local_hash then
-        return 0
+    local local_mtime = args[1]
+    local local_size = args[2]
+    local local_hash = args[3]
+
+    local exists = redis.call('EXISTS', key)
+    if exists == 0 then
+        return 0 -- New file
     end
-    -- Even if hash matches, force re-upload if context was updated
+
+    -- Check context_dirty first (always forces re-upload)
     local dirty = redis.call('HGET', key, 'context_dirty')
     if dirty == 'true' then
-        return 0
+        return 2 -- Needs update (context changed)
     end
-    return 1
+
+    -- Fast path: check mtime + size before expensive hash comparison
+    local stored_mtime = redis.call('HGET', key, 'file_mtime')
+    local stored_size = redis.call('HGET', key, 'file_size')
+    if stored_mtime and stored_size and stored_mtime == local_mtime and stored_size == local_size then
+        return 1 -- Metadata matches — skip (no need to even check hash)
+    end
+
+    -- Slow path: metadata changed, check hash
+    local stored_hash = redis.call('HGET', key, 'content_hash')
+    if stored_hash and stored_hash == local_hash then
+        -- Hash still matches despite metadata change — update stored metadata, skip upload
+        redis.call('HSET', key, 'file_mtime', local_mtime, 'file_size', local_size)
+        return 1
+    end
+
+    return 2 -- Hash mismatch — needs update
 end)
 
--- Function 4: upsert_sync_state
--- Updates absolute_path, content_hash, and openwebui_file_id without altering context_text.
+-- Function 3: upsert_sync_state (UPDATED — now stores mtime + size)
+-- Updates absolute_path, content_hash, mtime, size, and openwebui_file_id without altering context_text.
 redis.register_function('upsert_sync_state', function(keys, args)
     local key = keys[1]
     local absolute_path = args[1]
     local content_hash = args[2]
     local openwebui_file_id = args[3]
+    local file_mtime = args[4] or ''
+    local file_size = args[5] or ''
     redis.call('HSET', key,
         'absolute_path', absolute_path,
         'content_hash', content_hash,
         'openwebui_file_id', openwebui_file_id,
-        'context_dirty', 'false'
+        'context_dirty', 'false',
+        'file_mtime', file_mtime,
+        'file_size', file_size
     )
     return 1
 end)
 
--- Function 5: get_cleanup_batch
+-- Function 4: get_cleanup_batch
 -- Performs a SCAN and retrieves absolute_path and openwebui_file_id for each key.
 -- Returns: [new_cursor, key1, path1, id1, key2, path2, id2, ...]
 redis.register_function('get_cleanup_batch', function(keys, args)
@@ -80,3 +102,22 @@ redis.register_function('get_cleanup_batch', function(keys, args)
 
     return output
 end)
+
+-- LEGACY compatibility: keep old functions so existing callers don't crash during rolling updates
+redis.register_function('check_file_exists', function(keys, args)
+    return redis.call('EXISTS', keys[1])
+end)
+
+redis.register_function('verify_file_hash', function(keys, args)
+    local key = keys[1]
+    local local_hash = args[1]
+    local stored = redis.call('HGET', key, 'content_hash')
+    if not stored or stored ~= local_hash then
+        return 0
+    end
+    local dirty = redis.call('HGET', key, 'context_dirty')
+    if dirty == 'true' then
+        return 0
+    end
+    return 1
+end)

src/redis_client.rs

@@ -26,6 +26,42 @@ pub async fn load_functions(
     }
 }
 
+/// Result of the combined check_and_compare FCALL.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum FileCheckResult {
+    /// File does not exist in Redis — new file, needs upload
+    New,
+    /// File exists, hash matches, not dirty — skip
+    Unchanged,
+    /// File exists but hash/metadata changed or dirty — needs re-upload
+    Changed,
+}
+
+/// FCALL check_and_compare 1 <key> <mtime> <size> <content_hash>
+/// Single round-trip: checks existence, compares mtime+size, then hash if needed.
+pub async fn fcall_check_and_compare(
+    con: &mut redis::aio::MultiplexedConnection,
+    key: &str,
+    mtime: &str,
+    size: &str,
+    content_hash: &str,
+) -> Result<FileCheckResult, Box<dyn std::error::Error + Send + Sync>> {
+    let result: i64 = redis::cmd("FCALL")
+        .arg("check_and_compare")
+        .arg(1)
+        .arg(key)
+        .arg(mtime)
+        .arg(size)
+        .arg(content_hash)
+        .query_async(con)
+        .await?;
+    match result {
+        0 => Ok(FileCheckResult::New),
+        1 => Ok(FileCheckResult::Unchanged),
+        _ => Ok(FileCheckResult::Changed),
+    }
+}
+
 /// FCALL get_formatted_context 1 <key>
 pub async fn fcall_get_formatted_context(
     con: &mut redis::aio::MultiplexedConnection,
@@ -40,7 +76,7 @@ pub async fn fcall_get_formatted_context(
     Ok(result)
 }
 
-/// FCALL check_file_exists 1 <key>
+/// FCALL check_file_exists 1 <key> (LEGACY — kept for tests)
 pub async fn fcall_check_file_exists(
     con: &mut redis::aio::MultiplexedConnection,
     key: &str,
@@ -54,7 +90,7 @@ pub async fn fcall_check_file_exists(
     Ok(result == 1)
 }
 
-/// FCALL verify_file_hash 1 <key> <local_content_hash>
+/// FCALL verify_file_hash 1 <key> <local_content_hash> (LEGACY — kept for tests)
 pub async fn fcall_verify_file_hash(
     con: &mut redis::aio::MultiplexedConnection,
     key: &str,
@@ -70,13 +106,15 @@ pub async fn fcall_verify_file_hash(
     Ok(result == 1)
 }
 
-/// FCALL upsert_sync_state 1 <key> <absolute_path> <content_hash> <openwebui_file_id>
+/// FCALL upsert_sync_state 1 <key> <absolute_path> <content_hash> <openwebui_file_id> <mtime> <size>
 pub async fn fcall_upsert_sync_state(
     con: &mut redis::aio::MultiplexedConnection,
     key: &str,
     absolute_path: &str,
     content_hash: &str,
     openwebui_file_id: &str,
+    mtime: &str,
+    size: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     redis::cmd("FCALL")
         .arg("upsert_sync_state")
@@ -85,6 +123,8 @@ pub async fn fcall_upsert_sync_state(
         .arg(absolute_path)
         .arg(content_hash)
         .arg(openwebui_file_id)
+        .arg(mtime)
+        .arg(size)
         .query_async::<i64>(con)
         .await?;
     Ok(())
@@ -122,6 +162,7 @@ pub async fn fcall_get_cleanup_batch(
 }
 
 /// Scan all tracked file keys and return (key, absolute_path, context_text) for each.
+/// Uses HMGET for batch field retrieval (single call per key instead of 2).
 pub async fn scan_all_tracked_files(
     con: &mut redis::aio::MultiplexedConnection,
 ) -> Result<Vec<(String, String, String)>, Box<dyn std::error::Error + Send + Sync>> {
@@ -140,18 +181,17 @@ pub async fn scan_all_tracked_files(
             if key == "config:global" {
                 continue;
             }
-            let path: Option<String> = redis::cmd("HGET")
+            // HMGET: single round-trip for both fields (was 2 separate HGET calls)
+            let fields: Vec<Option<String>> = redis::cmd("HMGET")
                 .arg(key)
                 .arg("absolute_path")
-                .query_async(con)
-                .await?;
-            let ctx: Option<String> = redis::cmd("HGET")
-                .arg(key)
                 .arg("context_text")
                 .query_async(con)
                 .await?;
-            if let Some(path) = path {
-                files.push((key.clone(), path, ctx.unwrap_or_default()));
+
+            if let Some(path) = fields.get(0).and_then(|f| f.clone()) {
+                let ctx = fields.get(1).and_then(|f| f.clone()).unwrap_or_default();
+                files.push((key.clone(), path, ctx));
             }
         }