From 44df0a02819661b61eaae945e6666bb8617336f3 Mon Sep 17 00:00:00 2001 From: Daniel JB Clark Date: Fri, 31 Jul 2026 09:16:57 -0400 Subject: [PATCH] fix: paginate mobile delta sync --- backend/api.py | 8 +++- backend/core/database.py | 6 +-- backend/tests/test_sync_pagination.py | 52 ++++++++++++++++++++++ superbrain-app/src/services/api.ts | 20 ++++++--- superbrain-app/src/services/syncService.ts | 15 ++++++- 5 files changed, 89 insertions(+), 12 deletions(-) create mode 100644 backend/tests/test_sync_pagination.py diff --git a/backend/api.py b/backend/api.py index 57fac65..d7d063f 100644 --- a/backend/api.py +++ b/backend/api.py @@ -760,6 +760,7 @@ async def get_recent_analyses( async def sync_posts( since: str = Query(..., description="ISO timestamp — return posts updated after this time"), limit: int = Query(default=500, ge=1, le=1000), + offset: int = Query(default=0, ge=0, description="Number of matching rows to skip"), token: str = Depends(verify_token), ): """ @@ -769,12 +770,17 @@ async def sync_posts( """ try: db = get_db() - results = db.get_posts_since(since, limit=limit) + page = db.get_posts_since(since, limit=limit + 1, offset=offset) + has_more = len(page) > limit + results = page[:limit] return { "success": True, "count": len(results), "since": since, + "offset": offset, + "next_offset": offset + len(results) if has_more else None, + "has_more": has_more, "data": results } diff --git a/backend/core/database.py b/backend/core/database.py index 14da1c2..84f0a80 100644 --- a/backend/core/database.py +++ b/backend/core/database.py @@ -294,7 +294,7 @@ def get_recent_light(self, limit=50, offset=0): print(f"[WARNING] Error retrieving recent (light): {e}") return [] - def get_posts_since(self, updated_after: str, limit=1000): + def get_posts_since(self, updated_after: str, limit=1000, offset=0): """Return posts updated after the given ISO timestamp (delta sync). Includes soft-deleted posts so the app knows to hide them.""" if not self.is_connected(): @@ -304,8 +304,8 @@ def get_posts_since(self, updated_after: str, limit=1000): cur.execute( f"SELECT {self.LIGHT_COLUMNS} FROM analyses " "WHERE updated_at > ? " - "ORDER BY updated_at ASC LIMIT ?", - (updated_after, limit) + "ORDER BY updated_at ASC, shortcode ASC LIMIT ? OFFSET ?", + (updated_after, limit, offset) ) return [self._row_to_dict(r) for r in cur.fetchall()] except Exception as e: diff --git a/backend/tests/test_sync_pagination.py b/backend/tests/test_sync_pagination.py new file mode 100644 index 0000000..18abd69 --- /dev/null +++ b/backend/tests/test_sync_pagination.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Regression tests for deterministic delta-sync pagination.""" + +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import core.database as database_module # noqa: E402 + + +class SyncPaginationTests(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.original_db_path = database_module.DB_PATH + database_module.DB_PATH = Path(self.temp_dir.name) / "superbrain.db" + self.db = database_module.Database() + + rows = [ + ("charlie", "2026-01-02T12:00:00"), + ("alpha", "2026-01-02T12:00:00"), + ("echo", "2026-01-03T12:00:00"), + ("bravo", "2026-01-02T12:00:00"), + ("delta", "2026-01-03T12:00:00"), + ] + self.db._conn.executemany( + "INSERT INTO analyses (shortcode, updated_at, tags) VALUES (?, ?, '[]')", + rows, + ) + self.db._conn.commit() + + def tearDown(self): + self.db._conn.close() + database_module.DB_PATH = self.original_db_path + self.temp_dir.cleanup() + + def test_pages_use_stable_updated_at_and_shortcode_order(self): + since = "2026-01-01T00:00:00" + + first = self.db.get_posts_since(since, limit=2, offset=0) + second = self.db.get_posts_since(since, limit=2, offset=2) + third = self.db.get_posts_since(since, limit=2, offset=4) + + self.assertEqual([row["shortcode"] for row in first], ["alpha", "bravo"]) + self.assertEqual([row["shortcode"] for row in second], ["charlie", "delta"]) + self.assertEqual([row["shortcode"] for row in third], ["echo"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/superbrain-app/src/services/api.ts b/superbrain-app/src/services/api.ts index d178677..4225e17 100644 --- a/superbrain-app/src/services/api.ts +++ b/superbrain-app/src/services/api.ts @@ -305,18 +305,27 @@ class ApiService { /** * Delta sync — returns posts modified after the given ISO timestamp. */ - async syncPosts(since: string): Promise { + async syncPosts( + since: string, + limit: number = 200, + offset: number = 0, + ): Promise<{ data: Post[]; hasMore: boolean }> { try { const headers = await this.getHeaders(); const baseUrl = await this.getBaseUrl(); - const response = await axios.get<{ success: boolean; data: Post[] }>( - `${baseUrl}/sync?since=${encodeURIComponent(since)}&limit=1000`, + const response = await axios.get<{ + success: boolean; data: Post[]; has_more: boolean + }>( + `${baseUrl}/sync?since=${encodeURIComponent(since)}&limit=${limit}&offset=${offset}`, { headers, timeout: 30000 } ); - return (response.data.data || []).map(normalizePost); + return { + data: (response.data.data || []).map(normalizePost), + hasMore: response.data.has_more, + }; } catch (error: any) { console.error('Error syncing posts:', error.response?.data?.detail || error.message); - return []; + throw error; } } @@ -756,4 +765,3 @@ class ApiService { } export default new ApiService(); - diff --git a/superbrain-app/src/services/syncService.ts b/superbrain-app/src/services/syncService.ts index 45109ab..f6ce20e 100644 --- a/superbrain-app/src/services/syncService.ts +++ b/superbrain-app/src/services/syncService.ts @@ -56,8 +56,19 @@ async function deltaSync(): Promise { return fullSync(); } - // Fetch changed posts - const changedPosts = await apiService.syncPosts(since); + // Fetch every changed row before advancing the cursor. A delta can exceed + // the server's page size after many changes occur between app launches. + const changedPosts: Post[] = []; + let offset = 0; + while (true) { + const page = await apiService.syncPosts(since, BATCH_SIZE, offset); + changedPosts.push(...page.data); + offset += page.data.length; + if (!page.hasMore) break; + if (page.data.length === 0) { + throw new Error('Delta sync returned an empty page before completion'); + } + } // Filter out hidden (soft-deleted) posts for upsert; delete them locally instead const toUpsert: Post[] = [];