Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 35 additions & 13 deletions superbrain-app/src/screens/HomeScreen.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ const HomeScreen = () => {
];

const loadCategories = async (
prefetchedTaxonomy?: TaxonomyPayload | null,
prefetchedTaxonomy?: TaxonomyPayload | null | Promise<TaxonomyPayload | null>,
) => {
try {
// Reuse a pre-fetched taxonomy when available to avoid redundant HTTP calls.
Expand Down Expand Up @@ -279,11 +279,18 @@ const HomeScreen = () => {
}
setIsConfigured(true);
// Fire categories, collections sync, and posts loading in parallel
// for faster first paint
// for faster first paint. Fetch /taxonomy exactly once here (not
// awaited yet — same in-flight promise shared with both loadCategories
// and loadPosts, each awaiting it only at the point they actually need
// it) instead of each independently re-fetching, which both wasted a
// redundant HTTP call and could race under flaky network (one fetch
// succeeding while the other timed out, leaving rendered chips
// inconsistent with what sync just persisted).
const taxonomyPromise = apiService.getTaxonomy().catch(() => null);
const [, ,] = await Promise.all([
loadCategories(),
loadCategories(taxonomyPromise),
collectionsService.syncFromBackend().catch(() => { /* offline */ }),
loadPosts(false),
loadPosts(false, taxonomyPromise),
]);
// Reschedule Watch Later notifications with (possibly restored) collection data
if (!deferNotificationPrompt) {
Expand All @@ -297,9 +304,29 @@ const HomeScreen = () => {
}
};

const loadPosts = async (forceRefresh: boolean = false) => {
// Reset taxonomy cache for this load cycle; re-fetched lazily below.
const loadPosts = async (
forceRefresh: boolean = false,
prefetchedTaxonomy?: TaxonomyPayload | null | Promise<TaxonomyPayload | null>,
) => {
// Reset taxonomy cache for this load cycle; resolved lazily (below) at
// whichever point in this function first actually needs it, not eagerly
// here — keeps the fast local-data paint path (right below) unblocked by
// any network call, including a caller-supplied prefetch promise.
taxonomyRef.current = undefined;
// Resolves the taxonomy exactly once per loadPosts call, reusing
// taxonomyRef.current if a previous call site within this same cycle
// already resolved it, otherwise awaiting prefetchedTaxonomy (a caller's
// in-flight fetch, e.g. bootstrap sharing one /taxonomy call with
// loadCategories instead of each independently re-fetching), otherwise
// fetching fresh.
const resolveTaxonomy = async (): Promise<TaxonomyPayload | null> => {
if (taxonomyRef.current !== undefined) return taxonomyRef.current;
const resolved = prefetchedTaxonomy !== undefined
? await prefetchedTaxonomy
: await apiService.getTaxonomy().catch(() => null);
taxonomyRef.current = resolved;
return resolved;
};
try {
// Reconcile: if a post was in the failed list AND still stuck in analyzing, clean it up.
const failedList = await postsCache.getFailedPosts();
Expand Down Expand Up @@ -342,13 +369,11 @@ const HomeScreen = () => {
if (!isOnlineForGate) {
return;
}
const taxonomy = await apiService.getTaxonomy().catch(() => null);
const taxonomy = await resolveTaxonomy();
if (!isTaxonomyApiActive(taxonomy)) {
return;
}
// taxonomy is active — fall through to background sync below.
// Stash result so syncIfNeeded and post-sync reload can reuse it.
taxonomyRef.current = taxonomy;
}
} else {
// No local data at all — show loading spinner
Expand All @@ -366,10 +391,7 @@ const HomeScreen = () => {

// forceFull / taxonomy_version resync only activate when GET /taxonomy exists
// (see syncService); without it this matches upstream delta-only sync.
// Reuse any taxonomy payload already fetched above to avoid a redundant call.
const taxonomy = taxonomyRef.current !== undefined
? taxonomyRef.current
: await apiService.getTaxonomy().catch(() => null);
const taxonomy = await resolveTaxonomy();
const dataChanged = await syncService.syncIfNeeded(forceRefresh, taxonomy);

// If sync brought new data, re-read from local DB and update UI
Expand Down
2 changes: 1 addition & 1 deletion superbrain-app/src/services/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -421,7 +421,7 @@ class ApiService {
const baseUrl = await this.getBaseUrl();
const response = await axios.get<{
success: boolean;
categories: Array<{ id: string; name: string; precedence: number; guidance: string }>;
categories: Array<{ id: string; name: string; precedence?: number; guidance?: string }>;
allow_multiple_categories: boolean;
fallback_category: string;
use_default_categories: boolean;
Expand Down
38 changes: 36 additions & 2 deletions superbrain-app/src/services/syncService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,13 +59,47 @@ async function deltaSync(): Promise<number> {

// Fetch every changed row before advancing the cursor. A single delta can
// exceed the server's page size after a large playlist import.
//
// Defensive against a server that doesn't implement `offset` (this PR adds
// client-side pagination without a matching backend change — if `/sync`
// silently ignores an unknown `offset` param and keeps returning the same
// page, `hasMore`'s length-based fallback would never go false and this
// loop would never terminate). Two independent safety nets:
// 1. Hard cap on page count — always terminates regardless of server behavior.
// 2. Non-advancing-cursor detection — if consecutive pages start with the
// same post, the server isn't honoring `offset`; stop and warn rather
// than loop forever accumulating duplicates.
const MAX_SYNC_PAGES = 50; // 50 * BATCH_SIZE(200) = 10,000 posts per delta sync
const changedPosts: Post[] = [];
let offset = 0;
while (true) {
let previousFirstShortcode: string | undefined;
let hitPageCap = true;
for (let pageNum = 0; pageNum < MAX_SYNC_PAGES; pageNum++) {
const page = await apiService.syncPosts(since, BATCH_SIZE, offset);
if (page.data.length === 0) {
hitPageCap = false;
break;
}
const firstShortcode = page.data[0].shortcode;
if (firstShortcode === previousFirstShortcode) {
console.warn(
'[Sync] Pagination cursor did not advance (server may not support offset) — stopping delta sync early'
);
hitPageCap = false;
break;
}
previousFirstShortcode = firstShortcode;
changedPosts.push(...page.data);
offset += page.data.length;
if (!page.hasMore || page.data.length === 0) break;
if (!page.hasMore) {
hitPageCap = false;
break;
}
}
if (hitPageCap) {
console.warn(
`[Sync] Delta sync hit the ${MAX_SYNC_PAGES}-page safety cap — some changes may not be synced this cycle; next sync will continue from the last successful point.`
);
Comment on lines +99 to +102

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🗄️ Data Integrity & Integration | 🟠 Major | 🏗️ Heavy lift

Do not advance the delta timestamp after incomplete pagination.

When the loop stops at the page cap or on a repeated cursor, deltaSync later stores the current time as lastSyncTime. The next /sync request then excludes changes that were not fetched in this cycle. The warning at Line 101 says the next sync continues, but no offset or cursor is persisted.

Persist a continuation checkpoint with the original since value and offset or cursor. At minimum, update lastSyncTime only after an empty page or hasMore === false confirms completion.

Proposed minimum data-loss safeguard
+  let paginationComplete = false;
   for (let pageNum = 0; pageNum < MAX_SYNC_PAGES; pageNum++) {
     const page = await apiService.syncPosts(since, BATCH_SIZE, offset);
     if (page.data.length === 0) {
+      paginationComplete = true;
       hitPageCap = false;
       break;
     }
     // ...
     if (!page.hasMore) {
+      paginationComplete = true;
       hitPageCap = false;
       break;
     }
   }

-  await localDb.setLastSyncTime(new Date().toISOString());
+  if (paginationComplete) {
+    await localDb.setLastSyncTime(new Date().toISOString());
+  }
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@superbrain-app/src/services/syncService.ts` around lines 99 - 102, Update
deltaSync so pagination stopping at MAX_SYNC_PAGES or a repeated cursor does not
advance lastSyncTime; retain the original since value and persist the current
offset or cursor as the continuation checkpoint. Only update lastSyncTime after
an empty page or hasMore === false confirms synchronization completed.

}

// Filter out hidden (soft-deleted) posts for upsert; delete them locally instead
Expand Down