From 5975a02a41930d26c80ec077b3fdfbf27e09708a Mon Sep 17 00:00:00 2001 From: Daniel JB Clark Date: Sat, 1 Aug 2026 09:10:25 -0400 Subject: [PATCH 1/4] feat: Android taxonomy sync gated behind GET /taxonomy Defensive client support for config-driven categories. Without /taxonomy (upstream today), behavior matches mainline chips and sync. With /taxonomy, honor use_default_categories, taxonomy_version full-resync, and custom chips. Co-authored-by: Cursor --- superbrain-app/src/constants/categories.ts | 24 +++- superbrain-app/src/screens/HomeScreen.tsx | 119 +++++++++++++----- .../src/screens/PostDetailScreen.tsx | 49 +++++++- superbrain-app/src/services/api.ts | 54 +++++++- superbrain-app/src/services/localDb.ts | 19 +++ superbrain-app/src/services/syncService.ts | 54 +++++++- .../src/services/taxonomySupport.ts | 33 +++++ superbrain-app/src/theme/index.ts | 6 + 8 files changed, 311 insertions(+), 47 deletions(-) create mode 100644 superbrain-app/src/services/taxonomySupport.ts diff --git a/superbrain-app/src/constants/categories.ts b/superbrain-app/src/constants/categories.ts index 53ea5d7..9f18dae 100644 --- a/superbrain-app/src/constants/categories.ts +++ b/superbrain-app/src/constants/categories.ts @@ -1,8 +1,14 @@ -export const DEFAULT_CATEGORIES = [ - { id: 'all', name: 'All', icon: 'star', count: 0 }, +/** Chip shown for the unfiltered feed. */ +export const ALL_CATEGORY = { id: 'all', name: 'All', icon: 'star', count: 0 }; + +/** + * Built-in mainline categories (product/places/food/…). + * Used when GET /taxonomy is unavailable, or when the server sets + * use_default_categories=true. Matching upstream default chip set. + */ +export const BUILTIN_DEFAULT_CATEGORIES = [ { id: 'product', name: 'Product', icon: 'cube', count: 0 }, { id: 'places', name: 'Places', icon: 'location', count: 0 }, - { id: 'food', name: 'Food', icon: 'restaurant', count: 0 }, { id: 'software', name: 'Software', icon: 'code-slash', count: 0 }, { id: 'book', name: 'Book', icon: 'book', count: 0 }, @@ -13,8 +19,20 @@ export const DEFAULT_CATEGORIES = [ { id: 'other', name: 'Other', icon: 'pricetag', count: 0 }, ]; +/** + * Default chip set for servers without config-driven taxonomy (upstream path). + * Custom taxonomies replace this via GET /taxonomy when present. + */ +export const DEFAULT_CATEGORIES = [ALL_CATEGORY, ...BUILTIN_DEFAULT_CATEGORIES]; + export const CATEGORY_ICONS: Record = { 'all': 'star', + 'sysadmin': 'terminal-outline', + 'science': 'flask-outline', + 'technology': 'hardware-chip-outline', + 'history': 'hourglass-outline', + 'humanities': 'library-outline', + 'politics': 'newspaper-outline', 'product': 'cube-outline', 'places': 'location-outline', 'food': 'restaurant-outline', diff --git a/superbrain-app/src/screens/HomeScreen.tsx b/superbrain-app/src/screens/HomeScreen.tsx index 4cd51d1..6b5557e 100644 --- a/superbrain-app/src/screens/HomeScreen.tsx +++ b/superbrain-app/src/screens/HomeScreen.tsx @@ -40,7 +40,12 @@ import { RootStackParamList } from '../../App'; import CustomToast from '../components/CustomToast'; import BottomNav from '../components/BottomNav'; import { getCollectionIconName, getCollectionIconColor } from '../constants/icons'; -import { DEFAULT_CATEGORIES, CATEGORY_ICONS } from '../constants/categories'; +import { ALL_CATEGORY, DEFAULT_CATEGORIES, CATEGORY_ICONS } from '../constants/categories'; +import { + TaxonomyPayload, + isTaxonomyApiActive, + usesStrictCustomTaxonomy, +} from '../services/taxonomySupport'; type NavigationProp = NativeStackNavigationProp; @@ -71,6 +76,8 @@ const HomeScreen = () => { const [onboardingStep, setOnboardingStep] = useState(0); const [categories, setCategories] = useState(DEFAULT_CATEGORIES); const lastFocusRefreshRef = useRef(0); + /** Caches the GET /taxonomy result within a single loadPosts call. */ + const taxonomyRef = useRef(undefined); useEffect(() => { const bootstrap = async () => { @@ -201,32 +208,60 @@ const HomeScreen = () => { }, ]; - const loadCategories = async () => { + const loadCategories = async ( + prefetchedTaxonomy?: TaxonomyPayload | null, + ) => { try { - const cats = await apiService.getCategories(); - if (cats && cats.length > 0) { - // Always keep full default pill set visible, then overlay live counts from backend. - const mergedById = new Map( - DEFAULT_CATEGORIES - .filter(c => c.id !== 'all') - .map(c => [c.id, { ...c, count: 0 }]) - ); + // Reuse a pre-fetched taxonomy when available to avoid redundant HTTP calls. + // On init (no prefetch), fetch taxonomy alongside categories. + const [cats, taxonomy] = await Promise.all([ + apiService.getCategories(), + prefetchedTaxonomy !== undefined + ? Promise.resolve(prefetchedTaxonomy) + : apiService.getTaxonomy().catch(() => null), + ]); - for (const c of cats) { - const id = c.id.toLowerCase(); - const existing = mergedById.get(id); - mergedById.set(id, { - id, - name: existing?.name || c.name, - icon: existing?.icon || CATEGORY_ICONS[c.name.trim().toLowerCase()] || 'pricetag-outline', - count: c.count, - }); - } + // No GET /taxonomy (upstream today) → keep full built-in pill set + counts. + // Taxonomy present + use_default_categories=false → configured chips only. + // Taxonomy present + defaults on → seed from server taxonomy list. + const taxonomyActive = isTaxonomyApiActive(taxonomy); + const strictCustom = usesStrictCustomTaxonomy(taxonomy); + + let seed: Array<{ id: string; name: string; icon: string; count: number }>; + if (taxonomyActive) { + seed = taxonomy!.categories.map(c => ({ + id: c.id, + name: c.name, + icon: CATEGORY_ICONS[c.id] || CATEGORY_ICONS[c.name.trim().toLowerCase()] || 'pricetag-outline', + count: 0, + })); + } else { + seed = DEFAULT_CATEGORIES.filter(c => c.id !== 'all').map(c => ({ ...c, count: 0 })); + } + + const mergedById = new Map(seed.map(c => [c.id.toLowerCase(), { ...c }])); + const configuredIds = new Set(mergedById.keys()); - const merged = Array.from(mergedById.values()); - const totalCount = merged.reduce((sum, c) => sum + c.count, 0); - setCategories([{ id: 'all', name: 'All', icon: 'star', count: totalCount }, ...merged]); + for (const c of cats || []) { + const id = c.id.toLowerCase(); + if (strictCustom && !configuredIds.has(id)) { + continue; + } + const existing = mergedById.get(id); + mergedById.set(id, { + id, + name: existing?.name || c.name, + icon: existing?.icon || CATEGORY_ICONS[c.name.trim().toLowerCase()] || 'pricetag-outline', + count: c.count, + }); } + + const merged = Array.from(mergedById.values()); + // Upstream totals from visible chips; strict taxonomy still counts all posts in All. + const totalCount = strictCustom + ? (cats || []).reduce((sum, c) => sum + (c.count || 0), 0) + : merged.reduce((sum, c) => sum + c.count, 0); + setCategories([{ ...ALL_CATEGORY, count: totalCount }, ...merged]); } catch (e) { console.warn('Failed to load categories, using defaults:', e); } @@ -263,6 +298,8 @@ const HomeScreen = () => { }; const loadPosts = async (forceRefresh: boolean = false) => { + // Reset taxonomy cache for this load cycle; re-fetched lazily below. + taxonomyRef.current = undefined; try { // Reconcile: if a post was in the failed list AND still stuck in analyzing, clean it up. const failedList = await postsCache.getFailedPosts(); @@ -296,9 +333,22 @@ const HomeScreen = () => { setPosts(merged); setLoading(false); - // If not forcing refresh and no analyzing posts, we're done + // Upstream path: return early when local data is enough. + // Taxonomy-aware servers: always continue so migrations reach the device. + // Defer the taxonomy check until after we confirm connectivity (below) + // to avoid a 404 round-trip on upstream servers that lack GET /taxonomy. if (!forceRefresh && analyzingShortcodes.length === 0) { - return; + const isOnlineForGate = await apiService.testConnection().catch(() => false); + if (!isOnlineForGate) { + return; + } + const taxonomy = await apiService.getTaxonomy().catch(() => null); + if (!isTaxonomyApiActive(taxonomy)) { + return; + } + // taxonomy is active — fall through to background sync below. + // Stash result so syncIfNeeded and post-sync reload can reuse it. + taxonomyRef.current = taxonomy; } } else { // No local data at all — show loading spinner @@ -306,7 +356,7 @@ const HomeScreen = () => { } // ── Step 2: Background sync with backend ── - // This runs non-blocking: UI is already showing local data (if any) + // UI may already be showing local data; refresh if sync changes anything. const isOnline = await apiService.testConnection().catch(() => false); if (isOnline) { @@ -314,11 +364,20 @@ const HomeScreen = () => { await postsCache.flushPendingPostMutations(); await postsCache.flushPendingAnalyses(); - // Delta or full sync depending on DB state - const dataChanged = await syncService.syncIfNeeded(); + // forceFull / taxonomy_version resync only activate when GET /taxonomy exists + // (see syncService); without it this matches upstream delta-only sync. + // Reuse any taxonomy payload already fetched above to avoid a redundant call. + const taxonomy = taxonomyRef.current !== undefined + ? taxonomyRef.current + : await apiService.getTaxonomy().catch(() => null); + const dataChanged = await syncService.syncIfNeeded(forceRefresh, taxonomy); // If sync brought new data, re-read from local DB and update UI if (dataChanged || forceRefresh) { + // Category chip reload is taxonomy-aware; skip extra work on mainline. + if (isTaxonomyApiActive(taxonomy)) { + await loadCategories(taxonomy); + } const freshPosts = await localDb.getAllPosts(); if (freshPosts.length > 0) { // Clear analyzing state for posts that now exist in the synced data @@ -421,7 +480,9 @@ const HomeScreen = () => { (post.summary && post.summary.toLowerCase().includes(searchQuery.toLowerCase())) || (post.tags && post.tags.some(tag => tag.toLowerCase().includes(searchQuery.toLowerCase()))); - const matchesCategory = selectedCategory === 'all' || post.category === selectedCategory; + const matchesCategory = + selectedCategory === 'all' || + (post.category || '').trim().toLowerCase() === selectedCategory.trim().toLowerCase(); return matchesSearch && matchesCategory; }); diff --git a/superbrain-app/src/screens/PostDetailScreen.tsx b/superbrain-app/src/screens/PostDetailScreen.tsx index 31eb392..611a2f5 100644 --- a/superbrain-app/src/screens/PostDetailScreen.tsx +++ b/superbrain-app/src/screens/PostDetailScreen.tsx @@ -13,12 +13,19 @@ import { collectionsService } from '../services/collections'; import { Collection } from '../types'; import { schedulePostWatchLaterNotification, sendImmediateWatchLaterNotification, sendImmediateSavedNotification } from '../services/notificationService'; import { getCollectionIconName, getCollectionIconColor } from '../constants/icons'; -import { DEFAULT_CATEGORIES, CATEGORY_ICONS } from '../constants/categories'; +import { BUILTIN_DEFAULT_CATEGORIES, CATEGORY_ICONS } from '../constants/categories'; +import { isTaxonomyApiActive } from '../services/taxonomySupport'; type Props = NativeStackScreenProps; -// Filter out 'all' from categories to use for the edit dropdown -const CATEGORIES = DEFAULT_CATEGORIES.filter(c => c.id !== 'all'); +type CategoryOption = { id: string; name: string; icon: string }; + +/** Upstream / no-taxonomy edit list (built-in mainline categories). */ +const FALLBACK_CATEGORIES: CategoryOption[] = BUILTIN_DEFAULT_CATEGORIES.map(c => ({ + id: c.id, + name: c.name, + icon: c.icon, +})); const PostDetailScreen = ({ route, navigation }: Props) => { const { post } = route.params; @@ -28,6 +35,7 @@ const PostDetailScreen = ({ route, navigation }: Props) => { const [editedTitle, setEditedTitle] = useState(post.title); const [editedSummary, setEditedSummary] = useState(post.summary); const [saving, setSaving] = useState(false); + const [categoryOptions, setCategoryOptions] = useState(FALLBACK_CATEGORIES); const [deleting, setDeleting] = useState(false); const [reanalyzing, setReanalyzing] = useState(false); const [toast, setToast] = useState({ visible: false, message: '', type: 'info' as 'success' | 'error' | 'warning' | 'info' }); @@ -46,6 +54,35 @@ const PostDetailScreen = ({ route, navigation }: Props) => { } }, [showEditModal]); + useEffect(() => { + let cancelled = false; + (async () => { + try { + // Gate behind connectivity to avoid a wasted 404 round-trip on + // upstream servers that lack GET /taxonomy. + const isOnline = await apiService.testConnection().catch(() => false); + if (!isOnline || cancelled) return; + const taxonomy = await apiService.getTaxonomy(); + if (cancelled) return; + // Only replace the built-in picker when the server exposes taxonomy. + if (isTaxonomyApiActive(taxonomy)) { + setCategoryOptions( + taxonomy!.categories.map(c => ({ + id: c.id, + name: c.name, + icon: CATEGORY_ICONS[c.id] || CATEGORY_ICONS[c.name.trim().toLowerCase()] || 'pricetag', + })) + ); + } + } catch { + // Keep FALLBACK_CATEGORIES (upstream path) + } + })(); + return () => { + cancelled = true; + }; + }, []); + const getPostImageUrl = (post: Post) => { if (post.thumbnail_url) return post.thumbnail_url; if (post.thumbnail) { @@ -390,8 +427,10 @@ const PostDetailScreen = ({ route, navigation }: Props) => { contentContainerStyle={styles.categoriesContent} keyboardShouldPersistTaps="always" > - {CATEGORIES.map((cat) => { - const isActive = editedCategory === cat.id; + {categoryOptions.map((cat) => { + const isActive = + (editedCategory || '').trim().toLowerCase() === cat.id.toLowerCase() || + (editedCategory || '').trim().toLowerCase() === cat.name.trim().toLowerCase(); const catColor = getCategoryColor(cat.id); return ( { + async syncPosts( + since: string, + limit: number = 200, + offset: number = 0, + ): Promise<{ data: Post[]; hasMore: boolean }> { try { const headers = await this.getHeaders(); const baseUrl = await this.getBaseUrl(); - const response = await axios.get<{ success: boolean; data: Post[] }>( - `${baseUrl}/sync?since=${encodeURIComponent(since)}&limit=1000`, + const response = await axios.get<{ + success: boolean; data: Post[]; has_more?: boolean + }>( + `${baseUrl}/sync?since=${encodeURIComponent(since)}&limit=${limit}&offset=${offset}`, { headers, timeout: 30000 } ); - return (response.data.data || []).map(normalizePost); + const data = (response.data.data || []).map(normalizePost); + return { + data, + hasMore: response.data.has_more ?? data.length === limit, + }; } catch (error: any) { console.error('Error syncing posts:', error.response?.data?.detail || error.message); - return []; + return { data: [], hasMore: false }; } } @@ -404,6 +415,38 @@ class ApiService { } } + async getTaxonomy(): Promise { + try { + const headers = await this.getHeaders(); + const baseUrl = await this.getBaseUrl(); + const response = await axios.get<{ + success: boolean; + categories: Array<{ id: string; name: string; precedence: number; guidance: string }>; + allow_multiple_categories: boolean; + fallback_category: string; + use_default_categories: boolean; + taxonomy_version?: string; + }>( + `${baseUrl}/taxonomy`, + { headers, timeout: DEFAULT_TIMEOUT } + ); + if (!response.data?.success) return null; + return { + categories: response.data.categories || [], + allow_multiple_categories: !!response.data.allow_multiple_categories, + fallback_category: response.data.fallback_category || 'Other', + use_default_categories: !!response.data.use_default_categories, + taxonomy_version: response.data.taxonomy_version || '', + }; + } catch (error: any) { + // 404 = upstream / pre-taxonomy servers — silent by design + if (error?.response?.status !== 404) { + console.error('Error fetching taxonomy:', error); + } + return null; + } + } + async checkCache(shortcode: string): Promise { try { const headers = await this.getHeaders(); @@ -756,4 +799,3 @@ class ApiService { } export default new ApiService(); - diff --git a/superbrain-app/src/services/localDb.ts b/superbrain-app/src/services/localDb.ts index a9c39b5..c20d4bf 100644 --- a/superbrain-app/src/services/localDb.ts +++ b/superbrain-app/src/services/localDb.ts @@ -230,6 +230,23 @@ async function setLastSyncTime(isoTimestamp: string): Promise { ); } +async function getSyncMeta(key: string): Promise { + const db = await getDb(); + const row = await db.getFirstAsync<{ value: string }>( + 'SELECT value FROM sync_meta WHERE key = ?', + [key] + ); + return row?.value ?? null; +} + +async function setSyncMeta(key: string, value: string): Promise { + const db = await getDb(); + await db.runAsync( + 'INSERT OR REPLACE INTO sync_meta (key, value) VALUES (?, ?)', + [key, value] + ); +} + // ─── Utilities ───────────────────────────────────────────────────── /** Drop all data — used for hard reset. */ @@ -258,6 +275,8 @@ const localDb = { updatePost, getLastSyncTime, setLastSyncTime, + getSyncMeta, + setSyncMeta, clearAll, isEmpty, }; diff --git a/superbrain-app/src/services/syncService.ts b/superbrain-app/src/services/syncService.ts index 45109ab..8195d02 100644 --- a/superbrain-app/src/services/syncService.ts +++ b/superbrain-app/src/services/syncService.ts @@ -10,6 +10,7 @@ import localDb from './localDb'; import apiService from './api'; import { Post } from '../types'; +import { TaxonomyPayload, isTaxonomyApiActive } from './taxonomySupport'; const BATCH_SIZE = 200; // posts per batch during full sync @@ -56,8 +57,16 @@ async function deltaSync(): Promise { return fullSync(); } - // Fetch changed posts - const changedPosts = await apiService.syncPosts(since); + // Fetch every changed row before advancing the cursor. A single delta can + // exceed the server's page size after a large playlist import. + const changedPosts: Post[] = []; + let offset = 0; + while (true) { + const page = await apiService.syncPosts(since, BATCH_SIZE, offset); + changedPosts.push(...page.data); + offset += page.data.length; + if (!page.hasMore || page.data.length === 0) break; + } // Filter out hidden (soft-deleted) posts for upsert; delete them locally instead const toUpsert: Post[] = []; @@ -99,14 +108,51 @@ async function deltaSync(): Promise { /** * Decides whether to do a full or delta sync. * - Empty local DB → full sync + * - forceFull → full sync only when GET /taxonomy is active (taxonomy-aware + * servers / pull-to-refresh after migrations). Without taxonomy, forceFull + * is ignored so behavior matches upstream mainline (delta only). + * - taxonomy_version change → full sync (no-op when endpoint absent) * - Has data → delta sync + * + * Accepts an optional pre-fetched taxonomy payload so callers that already + * fetched GET /taxonomy do not trigger a redundant round-trip. * Returns true if any data changed. */ -async function syncIfNeeded(): Promise { +async function syncIfNeeded( + forceFull: boolean = false, + prefetchedTaxonomy?: TaxonomyPayload | null, +): Promise { try { const empty = await localDb.isEmpty(); - if (empty) { + let taxonomyChanged = false; + let taxonomyVersion = ''; + let taxonomyActive = false; + try { + const taxonomy = prefetchedTaxonomy !== undefined + ? prefetchedTaxonomy + : await apiService.getTaxonomy(); + taxonomyActive = isTaxonomyApiActive(taxonomy); + taxonomyVersion = taxonomyActive ? (taxonomy?.taxonomy_version || '') : ''; + if (taxonomyVersion) { + const localVersion = await localDb.getSyncMeta('taxonomy_version'); + taxonomyChanged = localVersion !== taxonomyVersion; + if (taxonomyChanged) { + console.log( + `[Sync] Taxonomy version changed (${localVersion || 'none'} → ${taxonomyVersion}); forcing full sync` + ); + } + } + } catch { + /* offline / taxonomy endpoint unavailable — upstream path */ + } + + const effectiveForceFull = forceFull && taxonomyActive; + + if (empty || effectiveForceFull || taxonomyChanged) { const count = await fullSync(); + if (taxonomyVersion) { + await localDb.setSyncMeta('taxonomy_version', taxonomyVersion); + } return count > 0; } else { const changes = await deltaSync(); diff --git a/superbrain-app/src/services/taxonomySupport.ts b/superbrain-app/src/services/taxonomySupport.ts new file mode 100644 index 0000000..6b41adf --- /dev/null +++ b/superbrain-app/src/services/taxonomySupport.ts @@ -0,0 +1,33 @@ +/** + * Helpers for optional server-driven category taxonomy (`GET /taxonomy`). + * + * When the endpoint is absent (upstream mainline today), clients must keep + * historical APK behavior. New chip/sync rules activate only when the server + * returns a successful taxonomy payload with at least one category. + */ + +export type TaxonomyPayload = { + categories: Array<{ id: string; name: string; precedence?: number; guidance?: string }>; + allow_multiple_categories: boolean; + fallback_category: string; + use_default_categories: boolean; + taxonomy_version: string; +}; + +/** True when the backend exposes a usable config-driven taxonomy. */ +export function isTaxonomyApiActive( + taxonomy: TaxonomyPayload | null | undefined +): boolean { + return !!(taxonomy && Array.isArray(taxonomy.categories) && taxonomy.categories.length > 0); +} + +/** + * Strict custom taxonomy: hide built-in product/places/food chips and any + * labels outside the configured list. Only when the server explicitly sets + * use_default_categories=false. + */ +export function usesStrictCustomTaxonomy( + taxonomy: TaxonomyPayload | null | undefined +): boolean { + return isTaxonomyApiActive(taxonomy) && taxonomy!.use_default_categories === false; +} diff --git a/superbrain-app/src/theme/index.ts b/superbrain-app/src/theme/index.ts index 1e2e6a1..2a1103b 100644 --- a/superbrain-app/src/theme/index.ts +++ b/superbrain-app/src/theme/index.ts @@ -54,6 +54,12 @@ export const colors = { // Category Colors (for tagging) categories: { + sysadmin: '#0f766e', // Teal-700 + science: '#2563eb', // Blue-600 + technology: '#7c3aed', // Violet-600 + history: '#a16207', // Yellow-700 + humanities: '#be185d', // Pink-700 + politics: '#b91c1c', // Red-700 product: '#f59e0b', // Amber places: '#3b82f6', // Blue food: '#ef4444', // Red (alias) From 878a1dc6b7748f9d99560a1b71d166534aefb499 Mon Sep 17 00:00:00 2001 From: Daniel JB Clark Date: Sat, 1 Aug 2026 09:58:43 -0400 Subject: [PATCH 2/4] fix: delta-sync pagination safety, dedupe /taxonomy calls, tighten types - syncService.ts: the delta-sync pagination loop could spin forever if the server doesn't implement `offset` (this PR adds client-side pagination with no matching backend change). Add a hard page cap plus non-advancing- cursor detection so it terminates safely either way, warning rather than silently duplicating posts. - HomeScreen.tsx: loadCategories() and loadPosts() fired independent, redundant /taxonomy fetches when run in parallel at bootstrap (the existing taxonomyRef caching only deduped within loadPosts's own steps, not across the two functions) -- a real HTTP call doubling plus a narrow race if one fetch succeeded while the other timed out under flaky network. Fetch /taxonomy once in initializeAndLoad and share the single in-flight promise with both, each still resolving it lazily at the point they actually need it (keeps the fast local-data paint path unblocked). - api.ts: getTaxonomy()'s raw response type declared precedence/guidance as required while TaxonomyPayload declares them optional. Align them. TypeScript passes. --- superbrain-app/src/screens/HomeScreen.tsx | 48 ++++++++++++++++------ superbrain-app/src/services/api.ts | 2 +- superbrain-app/src/services/syncService.ts | 38 ++++++++++++++++- 3 files changed, 72 insertions(+), 16 deletions(-) diff --git a/superbrain-app/src/screens/HomeScreen.tsx b/superbrain-app/src/screens/HomeScreen.tsx index 6b5557e..6add0ea 100644 --- a/superbrain-app/src/screens/HomeScreen.tsx +++ b/superbrain-app/src/screens/HomeScreen.tsx @@ -209,7 +209,7 @@ const HomeScreen = () => { ]; const loadCategories = async ( - prefetchedTaxonomy?: TaxonomyPayload | null, + prefetchedTaxonomy?: TaxonomyPayload | null | Promise, ) => { try { // Reuse a pre-fetched taxonomy when available to avoid redundant HTTP calls. @@ -279,11 +279,18 @@ const HomeScreen = () => { } setIsConfigured(true); // Fire categories, collections sync, and posts loading in parallel - // for faster first paint + // for faster first paint. Fetch /taxonomy exactly once here (not + // awaited yet — same in-flight promise shared with both loadCategories + // and loadPosts, each awaiting it only at the point they actually need + // it) instead of each independently re-fetching, which both wasted a + // redundant HTTP call and could race under flaky network (one fetch + // succeeding while the other timed out, leaving rendered chips + // inconsistent with what sync just persisted). + const taxonomyPromise = apiService.getTaxonomy().catch(() => null); const [, ,] = await Promise.all([ - loadCategories(), + loadCategories(taxonomyPromise), collectionsService.syncFromBackend().catch(() => { /* offline */ }), - loadPosts(false), + loadPosts(false, taxonomyPromise), ]); // Reschedule Watch Later notifications with (possibly restored) collection data if (!deferNotificationPrompt) { @@ -297,9 +304,29 @@ const HomeScreen = () => { } }; - const loadPosts = async (forceRefresh: boolean = false) => { - // Reset taxonomy cache for this load cycle; re-fetched lazily below. + const loadPosts = async ( + forceRefresh: boolean = false, + prefetchedTaxonomy?: TaxonomyPayload | null | Promise, + ) => { + // Reset taxonomy cache for this load cycle; resolved lazily (below) at + // whichever point in this function first actually needs it, not eagerly + // here — keeps the fast local-data paint path (right below) unblocked by + // any network call, including a caller-supplied prefetch promise. taxonomyRef.current = undefined; + // Resolves the taxonomy exactly once per loadPosts call, reusing + // taxonomyRef.current if a previous call site within this same cycle + // already resolved it, otherwise awaiting prefetchedTaxonomy (a caller's + // in-flight fetch, e.g. bootstrap sharing one /taxonomy call with + // loadCategories instead of each independently re-fetching), otherwise + // fetching fresh. + const resolveTaxonomy = async (): Promise => { + if (taxonomyRef.current !== undefined) return taxonomyRef.current; + const resolved = prefetchedTaxonomy !== undefined + ? await prefetchedTaxonomy + : await apiService.getTaxonomy().catch(() => null); + taxonomyRef.current = resolved; + return resolved; + }; try { // Reconcile: if a post was in the failed list AND still stuck in analyzing, clean it up. const failedList = await postsCache.getFailedPosts(); @@ -342,13 +369,11 @@ const HomeScreen = () => { if (!isOnlineForGate) { return; } - const taxonomy = await apiService.getTaxonomy().catch(() => null); + const taxonomy = await resolveTaxonomy(); if (!isTaxonomyApiActive(taxonomy)) { return; } // taxonomy is active — fall through to background sync below. - // Stash result so syncIfNeeded and post-sync reload can reuse it. - taxonomyRef.current = taxonomy; } } else { // No local data at all — show loading spinner @@ -366,10 +391,7 @@ const HomeScreen = () => { // forceFull / taxonomy_version resync only activate when GET /taxonomy exists // (see syncService); without it this matches upstream delta-only sync. - // Reuse any taxonomy payload already fetched above to avoid a redundant call. - const taxonomy = taxonomyRef.current !== undefined - ? taxonomyRef.current - : await apiService.getTaxonomy().catch(() => null); + const taxonomy = await resolveTaxonomy(); const dataChanged = await syncService.syncIfNeeded(forceRefresh, taxonomy); // If sync brought new data, re-read from local DB and update UI diff --git a/superbrain-app/src/services/api.ts b/superbrain-app/src/services/api.ts index 26124bf..0a10481 100644 --- a/superbrain-app/src/services/api.ts +++ b/superbrain-app/src/services/api.ts @@ -421,7 +421,7 @@ class ApiService { const baseUrl = await this.getBaseUrl(); const response = await axios.get<{ success: boolean; - categories: Array<{ id: string; name: string; precedence: number; guidance: string }>; + categories: Array<{ id: string; name: string; precedence?: number; guidance?: string }>; allow_multiple_categories: boolean; fallback_category: string; use_default_categories: boolean; diff --git a/superbrain-app/src/services/syncService.ts b/superbrain-app/src/services/syncService.ts index 8195d02..3f9057a 100644 --- a/superbrain-app/src/services/syncService.ts +++ b/superbrain-app/src/services/syncService.ts @@ -59,13 +59,47 @@ async function deltaSync(): Promise { // Fetch every changed row before advancing the cursor. A single delta can // exceed the server's page size after a large playlist import. + // + // Defensive against a server that doesn't implement `offset` (this PR adds + // client-side pagination without a matching backend change — if `/sync` + // silently ignores an unknown `offset` param and keeps returning the same + // page, `hasMore`'s length-based fallback would never go false and this + // loop would never terminate). Two independent safety nets: + // 1. Hard cap on page count — always terminates regardless of server behavior. + // 2. Non-advancing-cursor detection — if consecutive pages start with the + // same post, the server isn't honoring `offset`; stop and warn rather + // than loop forever accumulating duplicates. + const MAX_SYNC_PAGES = 50; // 50 * BATCH_SIZE(200) = 10,000 posts per delta sync const changedPosts: Post[] = []; let offset = 0; - while (true) { + let previousFirstShortcode: string | undefined; + let hitPageCap = true; + for (let pageNum = 0; pageNum < MAX_SYNC_PAGES; pageNum++) { const page = await apiService.syncPosts(since, BATCH_SIZE, offset); + if (page.data.length === 0) { + hitPageCap = false; + break; + } + const firstShortcode = page.data[0].shortcode; + if (firstShortcode === previousFirstShortcode) { + console.warn( + '[Sync] Pagination cursor did not advance (server may not support offset) — stopping delta sync early' + ); + hitPageCap = false; + break; + } + previousFirstShortcode = firstShortcode; changedPosts.push(...page.data); offset += page.data.length; - if (!page.hasMore || page.data.length === 0) break; + if (!page.hasMore) { + hitPageCap = false; + break; + } + } + if (hitPageCap) { + console.warn( + `[Sync] Delta sync hit the ${MAX_SYNC_PAGES}-page safety cap — some changes may not be synced this cycle; next sync will continue from the last successful point.` + ); } // Filter out hidden (soft-deleted) posts for upsert; delete them locally instead From d629ccfcc9612790b6b3ac89173053e092f86317 Mon Sep 17 00:00:00 2001 From: Daniel JB Clark Date: Sat, 1 Aug 2026 10:36:07 -0400 Subject: [PATCH 3/4] fix: don't advance delta-sync cursor on incomplete pagination CodeRabbit review on PR #10: stopping early at the page cap or on a repeated cursor still advanced lastSyncTime to now, silently skipping whatever changes existed past that point on every future sync. Only advance the cursor when pagination genuinely completed (empty page or hasMore === false); otherwise the next sync retries the same window. --- superbrain-app/src/services/syncService.ts | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/superbrain-app/src/services/syncService.ts b/superbrain-app/src/services/syncService.ts index 3f9057a..e3307ed 100644 --- a/superbrain-app/src/services/syncService.ts +++ b/superbrain-app/src/services/syncService.ts @@ -73,11 +73,11 @@ async function deltaSync(): Promise { const changedPosts: Post[] = []; let offset = 0; let previousFirstShortcode: string | undefined; - let hitPageCap = true; + let paginationComplete = false; for (let pageNum = 0; pageNum < MAX_SYNC_PAGES; pageNum++) { const page = await apiService.syncPosts(since, BATCH_SIZE, offset); if (page.data.length === 0) { - hitPageCap = false; + paginationComplete = true; break; } const firstShortcode = page.data[0].shortcode; @@ -85,20 +85,20 @@ async function deltaSync(): Promise { console.warn( '[Sync] Pagination cursor did not advance (server may not support offset) — stopping delta sync early' ); - hitPageCap = false; break; } previousFirstShortcode = firstShortcode; changedPosts.push(...page.data); offset += page.data.length; if (!page.hasMore) { - hitPageCap = false; + paginationComplete = true; break; } } - if (hitPageCap) { + if (!paginationComplete) { console.warn( - `[Sync] Delta sync hit the ${MAX_SYNC_PAGES}-page safety cap — some changes may not be synced this cycle; next sync will continue from the last successful point.` + `[Sync] Delta sync stopped before fetching all changes (page cap or non-advancing cursor) — ` + + `lastSyncTime will NOT advance, so the next sync retries this same window from '${since}'.` ); } @@ -127,8 +127,12 @@ async function deltaSync(): Promise { await localDb.deletePosts(deletedShortcodes); } - // Update sync cursor - await localDb.setLastSyncTime(new Date().toISOString()); + // Update sync cursor — only when pagination genuinely completed. Advancing + // this after an incomplete cycle (page cap / non-advancing cursor) would + // permanently skip whatever changes existed past the point sync stopped. + if (paginationComplete) { + await localDb.setLastSyncTime(new Date().toISOString()); + } const totalChanges = toUpsert.length + deletedShortcodes.length; if (totalChanges > 0) { From 3acb694bad2ff916b61315519d0731a137490ada Mon Sep 17 00:00:00 2001 From: Daniel JB Clark Date: Sat, 1 Aug 2026 10:49:15 -0400 Subject: [PATCH 4/4] fix: real /sync pagination, distinguish fetch failure from end-of-data, remove taxonomy-cache race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Independent second-opinion review (Fable 5) on the prior CodeRabbit-driven fix caught a deeper bug: /sync never actually supported the `offset` param the client was already sending — get_posts_since() always queried from the same `since` with no OFFSET, and the endpoint never returned `has_more`. The client's non-advancing-cursor safety net correctly stopped the loop instead of looping forever, but since lastSyncTime never advanced either, every retry hit the identical wall — a guaranteed permanent sync stall for any backlog over 200 changed posts, not just a hypothetical. - backend: get_posts_since()/`/sync` now honor offset and return real has_more (fetch limit+1, trim, compare). Added shortcode as a secondary ORDER BY key since updated_at alone isn't a stable pagination cursor when timestamps tie. - client: syncPosts() now marks a transient fetch failure distinctly (`failed: true`) instead of returning the same shape as a legitimate empty/final page — a network blip could otherwise be mistaken for sync completion and silently advance the cursor past unfetched changes. - HomeScreen: taxonomyRef was a component-level ref reset at the top of every loadPosts() call but only ever read/written within that same call — turned into a plain local variable so overlapping loadPosts calls (focus listener vs. poll-interval refresh) can no longer stomp each other's cache and reintroduce the redundant /taxonomy fetches this PR was written to eliminate. Verified offset/has_more pagination against an in-memory sqlite db (paged fetch of 5 rows at limit=2 returns all rows once, in order, with has_more flipping false on the last page). Typechecked clean. --- backend/api.py | 4 +++- backend/core/database.py | 21 ++++++++++++-------- superbrain-app/src/screens/HomeScreen.tsx | 23 +++++++++++----------- superbrain-app/src/services/api.ts | 6 ++++-- superbrain-app/src/services/syncService.ts | 19 ++++++++++++------ 5 files changed, 45 insertions(+), 28 deletions(-) diff --git a/backend/api.py b/backend/api.py index 57fac65..7eca338 100644 --- a/backend/api.py +++ b/backend/api.py @@ -760,6 +760,7 @@ async def get_recent_analyses( async def sync_posts( since: str = Query(..., description="ISO timestamp — return posts updated after this time"), limit: int = Query(default=500, ge=1, le=1000), + offset: int = Query(default=0, ge=0), token: str = Depends(verify_token), ): """ @@ -769,12 +770,13 @@ async def sync_posts( """ try: db = get_db() - results = db.get_posts_since(since, limit=limit) + results, has_more = db.get_posts_since(since, limit=limit, offset=offset) return { "success": True, "count": len(results), "since": since, + "has_more": has_more, "data": results } diff --git a/backend/core/database.py b/backend/core/database.py index 14da1c2..991b76c 100644 --- a/backend/core/database.py +++ b/backend/core/database.py @@ -294,23 +294,28 @@ def get_recent_light(self, limit=50, offset=0): print(f"[WARNING] Error retrieving recent (light): {e}") return [] - def get_posts_since(self, updated_after: str, limit=1000): - """Return posts updated after the given ISO timestamp (delta sync). - Includes soft-deleted posts so the app knows to hide them.""" + def get_posts_since(self, updated_after: str, limit=1000, offset=0): + """Return posts updated after the given ISO timestamp (delta sync), paginated. + Includes soft-deleted posts so the app knows to hide them. + Returns (results, has_more) — has_more indicates additional pages remain.""" if not self.is_connected(): - return [] + return [], False try: cur = self._conn.cursor() + # updated_at alone isn't a stable ORDER BY/OFFSET key when rows share a + # timestamp; shortcode (primary key) breaks ties so paging is deterministic. cur.execute( f"SELECT {self.LIGHT_COLUMNS} FROM analyses " "WHERE updated_at > ? " - "ORDER BY updated_at ASC LIMIT ?", - (updated_after, limit) + "ORDER BY updated_at ASC, shortcode ASC LIMIT ? OFFSET ?", + (updated_after, limit + 1, offset) ) - return [self._row_to_dict(r) for r in cur.fetchall()] + rows = cur.fetchall() + has_more = len(rows) > limit + return [self._row_to_dict(r) for r in rows[:limit]], has_more except Exception as e: print(f"[WARNING] Error getting posts since {updated_after}: {e}") - return [] + return [], False def get_deleted_since(self, since: str): """Return shortcodes of posts deleted after the given ISO timestamp.""" diff --git a/superbrain-app/src/screens/HomeScreen.tsx b/superbrain-app/src/screens/HomeScreen.tsx index 6add0ea..ef29fd5 100644 --- a/superbrain-app/src/screens/HomeScreen.tsx +++ b/superbrain-app/src/screens/HomeScreen.tsx @@ -76,8 +76,6 @@ const HomeScreen = () => { const [onboardingStep, setOnboardingStep] = useState(0); const [categories, setCategories] = useState(DEFAULT_CATEGORIES); const lastFocusRefreshRef = useRef(0); - /** Caches the GET /taxonomy result within a single loadPosts call. */ - const taxonomyRef = useRef(undefined); useEffect(() => { const bootstrap = async () => { @@ -308,23 +306,26 @@ const HomeScreen = () => { forceRefresh: boolean = false, prefetchedTaxonomy?: TaxonomyPayload | null | Promise, ) => { - // Reset taxonomy cache for this load cycle; resolved lazily (below) at - // whichever point in this function first actually needs it, not eagerly - // here — keeps the fast local-data paint path (right below) unblocked by - // any network call, including a caller-supplied prefetch promise. - taxonomyRef.current = undefined; + // Resolved lazily (below) at whichever point in this function first + // actually needs it, not eagerly here — keeps the fast local-data paint + // path (right below) unblocked by any network call, including a + // caller-supplied prefetch promise. `taxonomyCache` is a local (not a + // ref) so it's scoped to this single loadPosts call — overlapping calls + // (focus listener vs. poll-interval refresh) each get their own cache + // instead of racing to reset/read a shared one. + let taxonomyCache: TaxonomyPayload | null | undefined = undefined; // Resolves the taxonomy exactly once per loadPosts call, reusing - // taxonomyRef.current if a previous call site within this same cycle - // already resolved it, otherwise awaiting prefetchedTaxonomy (a caller's + // taxonomyCache if a previous call site within this same cycle already + // resolved it, otherwise awaiting prefetchedTaxonomy (a caller's // in-flight fetch, e.g. bootstrap sharing one /taxonomy call with // loadCategories instead of each independently re-fetching), otherwise // fetching fresh. const resolveTaxonomy = async (): Promise => { - if (taxonomyRef.current !== undefined) return taxonomyRef.current; + if (taxonomyCache !== undefined) return taxonomyCache; const resolved = prefetchedTaxonomy !== undefined ? await prefetchedTaxonomy : await apiService.getTaxonomy().catch(() => null); - taxonomyRef.current = resolved; + taxonomyCache = resolved; return resolved; }; try { diff --git a/superbrain-app/src/services/api.ts b/superbrain-app/src/services/api.ts index 0a10481..02a8e2a 100644 --- a/superbrain-app/src/services/api.ts +++ b/superbrain-app/src/services/api.ts @@ -310,7 +310,7 @@ class ApiService { since: string, limit: number = 200, offset: number = 0, - ): Promise<{ data: Post[]; hasMore: boolean }> { + ): Promise<{ data: Post[]; hasMore: boolean; failed?: boolean }> { try { const headers = await this.getHeaders(); const baseUrl = await this.getBaseUrl(); @@ -327,7 +327,9 @@ class ApiService { }; } catch (error: any) { console.error('Error syncing posts:', error.response?.data?.detail || error.message); - return { data: [], hasMore: false }; + // `failed: true` distinguishes a transient fetch error from a legitimate + // empty/final page — callers must not treat this as "sync complete". + return { data: [], hasMore: false, failed: true }; } } diff --git a/superbrain-app/src/services/syncService.ts b/superbrain-app/src/services/syncService.ts index e3307ed..a1ba16d 100644 --- a/superbrain-app/src/services/syncService.ts +++ b/superbrain-app/src/services/syncService.ts @@ -60,15 +60,18 @@ async function deltaSync(): Promise { // Fetch every changed row before advancing the cursor. A single delta can // exceed the server's page size after a large playlist import. // - // Defensive against a server that doesn't implement `offset` (this PR adds - // client-side pagination without a matching backend change — if `/sync` - // silently ignores an unknown `offset` param and keeps returning the same - // page, `hasMore`'s length-based fallback would never go false and this - // loop would never terminate). Two independent safety nets: + // `/sync` now honors `offset` and returns real `has_more` (backend/api.py), + // but the client stays defensive against any server that doesn't — an + // `offset`-blind `/sync` would return the same page forever, and + // `hasMore`'s length-based fallback would never go false. Two independent + // safety nets, both against a server that ignores/lacks `offset` support: // 1. Hard cap on page count — always terminates regardless of server behavior. // 2. Non-advancing-cursor detection — if consecutive pages start with the // same post, the server isn't honoring `offset`; stop and warn rather // than loop forever accumulating duplicates. + // Either safety net stopping early leaves `paginationComplete` false, so the + // cursor doesn't advance and the next sync retries the same window — no + // permanent stall, just a delayed catch-up once offset support is correct. const MAX_SYNC_PAGES = 50; // 50 * BATCH_SIZE(200) = 10,000 posts per delta sync const changedPosts: Post[] = []; let offset = 0; @@ -76,6 +79,10 @@ async function deltaSync(): Promise { let paginationComplete = false; for (let pageNum = 0; pageNum < MAX_SYNC_PAGES; pageNum++) { const page = await apiService.syncPosts(since, BATCH_SIZE, offset); + if (page.failed) { + console.warn('[Sync] Delta sync page fetch failed — stopping this cycle without advancing the cursor'); + break; + } if (page.data.length === 0) { paginationComplete = true; break; @@ -97,7 +104,7 @@ async function deltaSync(): Promise { } if (!paginationComplete) { console.warn( - `[Sync] Delta sync stopped before fetching all changes (page cap or non-advancing cursor) — ` + + `[Sync] Delta sync stopped before fetching all changes (page cap, non-advancing cursor, or fetch failure) — ` + `lastSyncTime will NOT advance, so the next sync retries this same window from '${since}'.` ); }