From 1909ee68d6fe323ec6e739e8f43fee47954a38be Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 21:21:44 +0000 Subject: [PATCH 1/2] Don't drop pub.leaflet documents without publishedAt from post lists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every post-list surface (publication home, archive, posts-list blocks, dashboard, RSS, tag pages, reader feed) funnels documents through normalizeDocument, which returned null for pub.leaflet.document records missing publishedAt — an optional field in that lexicon. Documents that only exist under pub.leaflet.document (third-party publishers) could legitimately lack it and silently vanished from every list, while site.standard.document records could never hit the drop since their lexicon requires publishedAt. Normalize such records with publishedAt left undefined instead: every consumer already guards missing dates (sorting falls back to epoch, date display is conditional, prev/next links filter undated docs), so only the type needed loosening. NormalizedDocument uses OmitKey rather than Omit because the generated Record's index signature makes plain Omit collapse every known key to unknown. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011UUDcdHkoLxf1bVFhriAFD --- lexicons/src/normalize.ts | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/lexicons/src/normalize.ts b/lexicons/src/normalize.ts index 6d12be92..e771c899 100644 --- a/lexicons/src/normalize.ts +++ b/lexicons/src/normalize.ts @@ -21,12 +21,21 @@ import type * as SiteStandardPublication from "../api/types/site/standard/public import type * as SiteStandardThemeBasic from "../api/types/site/standard/theme/basic"; import type * as SiteStandardThemeColor from "../api/types/site/standard/theme/color"; import type * as PubLeafletThemeColor from "../api/types/pub/leaflet/theme/color"; -import type { $Typed } from "../api/util"; +import type { $Typed, OmitKey } from "../api/util"; import { AtUri } from "@atproto/syntax"; // Normalized document type - uses the generated site.standard.document type -// with an additional optional theme field for backwards compatibility -export type NormalizedDocument = SiteStandardDocument.Record & { +// with an additional optional theme field for backwards compatibility. +// publishedAt is optional here even though site.standard.document requires it: +// pub.leaflet.document records may legitimately lack it, and consumers all +// handle a missing date, so normalization must not drop those documents. +// (OmitKey rather than Omit: the generated Record's index signature makes +// plain Omit collapse every known key to `unknown`.) +export type NormalizedDocument = OmitKey< + SiteStandardDocument.Record, + "publishedAt" +> & { + publishedAt?: string; // Keep the original theme for components that need leaflet-specific styling theme?: PubLeafletPublication.Theme; preferences?: SiteStandardPublication.Preferences; @@ -234,12 +243,6 @@ export function normalizeDocument( if (isLeafletDocument(record)) { // Convert from pub.leaflet to site.standard - const publishedAt = record.publishedAt; - - if (!publishedAt) { - return null; - } - // For standalone documents (no publication), construct a site URL from the author // This matches the pattern used in publishToPublication.ts for new standalone docs const site = record.publication || `https://leaflet.pub/p/${record.author}`; @@ -264,7 +267,7 @@ export function normalizeDocument( title: record.title, site, path, - publishedAt, + publishedAt: record.publishedAt, description: record.description, tags: record.tags, coverImage: record.coverImage, From ad6d520e8c01b939418a0ff59aea3501b53ff8b5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 00:30:45 +0000 Subject: [PATCH 2/2] List a publication's documents across both namespace URI variants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A publication can be indexed under both its site.standard.publication and legacy pub.leaflet.publication URIs (migration intentionally keeps the old rows), and each document links in documents_in_publications to whichever variant its own record names. Every listing surface resolved one publications row (.order uri desc puts site.standard first) and read only that row's embedded links — so a document that only has a pub.leaflet.document record, linked to the legacy publication URI, vanished from the publication home page, archive, posts-list blocks, RSS, sitemap, dashboard, and search, while its post page (which starts from the documents table) kept rendering fine. Fetch documents_in_publications for both URI variants of the resolved publication instead, deduping documents that exist under both namespaces (preferring the site.standard copy) so migrated posts don't double-list: - fetchPublicationForPage (home page, /archive, published-page posts-list blocks) - generateFeed (rss/atom/json) - per-domain sitemap.xml - get_publication_data (dashboard posts list; merged links reattached to the returned publication for the theme-preview client) - search_publication_documents The bsky feed generator (lish/feeds), reader getSubscriptions, and the post page's prev/next embed still read a single row's links and could get the same treatment. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011UUDcdHkoLxf1bVFhriAFD --- .../lish/[did]/[publication]/generateFeed.ts | 27 ++++++++++---- .../[publication]/getPublicationForPage.ts | 33 +++++++++++++---- .../[did]/[publication]/sitemap.xml/route.ts | 26 ++++++++----- app/api/rpc/[command]/get_publication_data.ts | 37 ++++++++++++++----- .../[command]/search_publication_documents.ts | 12 ++++-- src/utils/deduplicateRecords.ts | 28 ++++++++++++++ src/utils/uriHelpers.ts | 24 ++++++++++++ 7 files changed, 151 insertions(+), 36 deletions(-) diff --git a/app/(app)/lish/[did]/[publication]/generateFeed.ts b/app/(app)/lish/[did]/[publication]/generateFeed.ts index d8c71205..f5f5660a 100644 --- a/app/(app)/lish/[did]/[publication]/generateFeed.ts +++ b/app/(app)/lish/[did]/[publication]/generateFeed.ts @@ -11,7 +11,11 @@ import { hasLeafletContent, isLeafletPublication, } from "src/utils/normalizeRecords"; -import { publicationNameOrUriFilter } from "src/utils/uriHelpers"; +import { + publicationNameOrUriFilter, + publicationUriVariants, +} from "src/utils/uriHelpers"; +import { dedupeDocumentsInPublications } from "src/utils/deduplicateRecords"; import { getDocumentURL, getPublicationURL, @@ -61,11 +65,7 @@ export async function generateFeed( ); let { data: publications, error } = await supabaseServerClient .from("publications") - .select( - `*, - documents_in_publications(documents(*)) - `, - ) + .select("*") .eq("identity_did", did) .or(publicationNameOrUriFilter(did, publication_name)) .order("uri", { ascending: false }) @@ -79,6 +79,19 @@ export async function generateFeed( let publication = publications?.[0]; if (!publication) return new NextResponse(null, { status: 404 }); + // Documents link to whichever namespace URI their record names, and the + // publication may be indexed under both — fetch links for both variants so + // legacy pub.leaflet-only posts still show up alongside migrated ones. + let { data: docLinks, error: docsError } = await supabaseServerClient + .from("documents_in_publications") + .select("documents(*)") + .in("publication", publicationUriVariants(publication.uri)); + if (docsError) { + console.error(docsError); + return new NextResponse(null, { status: 500 }); + } + const documentsInPublication = dedupeDocumentsInPublications(docLinks ?? []); + const pubRecord = normalizePublicationRecord(publication.record); // Legacy pub.leaflet publications without a configured domain don't // normalize (no URL), but they're still browsable at leaflet.pub/lish/…, @@ -97,7 +110,7 @@ export async function generateFeed( ); const description = pubRecord?.description ?? rawRecord?.description; - let docs = publication.documents_in_publications + let docs = documentsInPublication .sort((a, b) => { const dateA = a.documents?.sort_date ? new Date(a.documents.sort_date).getTime() diff --git a/app/(app)/lish/[did]/[publication]/getPublicationForPage.ts b/app/(app)/lish/[did]/[publication]/getPublicationForPage.ts index 5b7b83a1..a00feb84 100644 --- a/app/(app)/lish/[did]/[publication]/getPublicationForPage.ts +++ b/app/(app)/lish/[did]/[publication]/getPublicationForPage.ts @@ -1,5 +1,9 @@ import { supabaseServerClient } from "supabase/serverClient"; -import { publicationNameOrUriFilter } from "src/utils/uriHelpers"; +import { + publicationNameOrUriFilter, + publicationUriVariants, +} from "src/utils/uriHelpers"; +import { dedupeDocumentsInPublications } from "src/utils/deduplicateRecords"; export async function fetchPublicationForPage( did: string, @@ -10,17 +14,32 @@ export async function fetchPublicationForPage( .select( `uri, name, identity_did, record, publication_newsletter_settings(enabled), - publication_pages(id, path, title, record, record_uri, sort_order), - documents_in_publications(documents(uri, data, - comments_on_documents(count), - document_mentions_in_bsky(count), - recommends_on_documents(count)))`, + publication_pages(id, path, title, record, record_uri, sort_order)`, ) .eq("identity_did", did) .or(publicationNameOrUriFilter(did, publicationName)) .order("uri", { ascending: false }) .limit(1); - return data?.[0] ?? null; + const publication = data?.[0]; + if (!publication) return null; + + // Documents link to whichever namespace URI their record names, and the + // publication may be indexed under both — fetch links for both variants so + // legacy pub.leaflet-only posts still show up alongside migrated ones. + const { data: docLinks } = await supabaseServerClient + .from("documents_in_publications") + .select( + `documents(uri, data, + comments_on_documents(count), + document_mentions_in_bsky(count), + recommends_on_documents(count))`, + ) + .in("publication", publicationUriVariants(publication.uri)); + + return { + ...publication, + documents_in_publications: dedupeDocumentsInPublications(docLinks ?? []), + }; } export type PublicationForPage = NonNullable< diff --git a/app/(app)/lish/[did]/[publication]/sitemap.xml/route.ts b/app/(app)/lish/[did]/[publication]/sitemap.xml/route.ts index 68e147b0..f0fbffe1 100644 --- a/app/(app)/lish/[did]/[publication]/sitemap.xml/route.ts +++ b/app/(app)/lish/[did]/[publication]/sitemap.xml/route.ts @@ -1,6 +1,10 @@ import { AtUri } from "@atproto/syntax"; import { supabaseServerClient } from "supabase/serverClient"; -import { publicationNameOrUriFilter } from "src/utils/uriHelpers"; +import { + publicationNameOrUriFilter, + publicationUriVariants, +} from "src/utils/uriHelpers"; +import { dedupeDocumentsInPublications } from "src/utils/deduplicateRecords"; import { normalizeDocumentRecord } from "src/utils/normalizeRecords"; import { isMainSiteHost } from "src/utils/customDomain"; import { isExternalLink } from "src/utils/externalPublicationLink"; @@ -36,11 +40,7 @@ export async function GET( let { data: publications } = await supabaseServerClient .from("publications") - .select( - `uri, - documents_in_publications(documents(uri, data, sort_date)), - publication_pages(path, record)`, - ) + .select(`uri, publication_pages(path, record)`) .eq("identity_did", did) .or(publicationNameOrUriFilter(did, publication_name)) .order("uri", { ascending: false }) @@ -49,6 +49,14 @@ export async function GET( let publication = publications?.[0]; if (!did || !publication) return new Response(null, { status: 404 }); + // Documents link to whichever namespace URI their record names, and the + // publication may be indexed under both — fetch links for both variants so + // legacy pub.leaflet-only posts still show up alongside migrated ones. + let { data: docLinks } = await supabaseServerClient + .from("documents_in_publications") + .select("documents(uri, data, sort_date)") + .in("publication", publicationUriVariants(publication.uri)); + let base = `https://${host}`; // Collect one entry per public URL, keyed by path so a post and a published @@ -59,9 +67,9 @@ export async function GET( if (!entries.has(path)) entries.set(path, { loc: base + path, lastmod }); }; - let posts = (publication.documents_in_publications ?? []) - .map((dip) => dip.documents) - .filter((d): d is NonNullable => !!d); + let posts = dedupeDocumentsInPublications(docLinks ?? []).map( + (dip) => dip.documents, + ); // Home page lastmod tracks the most recently published post. let latest = posts.reduce((acc, d) => { diff --git a/app/api/rpc/[command]/get_publication_data.ts b/app/api/rpc/[command]/get_publication_data.ts index 8f85be91..a64fb93c 100644 --- a/app/api/rpc/[command]/get_publication_data.ts +++ b/app/api/rpc/[command]/get_publication_data.ts @@ -5,6 +5,8 @@ import { AtUri } from "@atproto/syntax"; import { normalizeDocumentRecord } from "src/utils/normalizeRecords"; import { getIdentityData } from "actions/getIdentityData"; import { ids } from "lexicons/api/lexicons"; +import { publicationUriVariants } from "src/utils/uriHelpers"; +import { dedupeDocumentsInPublications } from "src/utils/deduplicateRecords"; export type GetPublicationDataReturnType = Awaited< ReturnType<(typeof get_publication_data)["handler"]> @@ -39,13 +41,6 @@ export const get_publication_data = makeRoute({ .from("publications") .select( `*, - documents_in_publications(members_only, documents( - *, - comments_on_documents(count), - document_mentions_in_bsky(count), - recommends_on_documents(count), - publication_post_sends(status, subscriber_count) - )), publication_subscriptions(*, identities(atp_did)), publication_email_subscribers(*, identities(atp_did)), publication_domains(*), @@ -88,8 +83,27 @@ export const get_publication_data = makeRoute({ return { result: { publication: null, documents: [], drafts: [] } }; } + // Documents link to whichever namespace URI their record names, and the + // publication may be indexed under both — fetch links for both variants so + // legacy pub.leaflet-only posts still show up alongside migrated ones. + const { data: docLinks } = await supabase + .from("documents_in_publications") + .select( + `members_only, documents( + *, + comments_on_documents(count), + document_mentions_in_bsky(count), + recommends_on_documents(count), + publication_post_sends(status, subscriber_count) + )`, + ) + .in("publication", publicationUriVariants(publication.uri)); + const documentsInPublication = dedupeDocumentsInPublications( + docLinks ?? [], + ); + // Pre-normalize documents from documents_in_publications - const documents = (publication?.documents_in_publications || []) + const documents = documentsInPublication .map((dip) => { if (!dip.documents) return null; const normalized = normalizeDocumentRecord( @@ -128,7 +142,12 @@ export const get_publication_data = makeRoute({ return { result: { - publication, + // Reattach the merged document links; clients (e.g. the theme preview) + // read publication.documents_in_publications off this object. + publication: { + ...publication, + documents_in_publications: documentsInPublication, + }, documents, drafts, }, diff --git a/app/api/rpc/[command]/search_publication_documents.ts b/app/api/rpc/[command]/search_publication_documents.ts index 7be59c7f..1a45611c 100644 --- a/app/api/rpc/[command]/search_publication_documents.ts +++ b/app/api/rpc/[command]/search_publication_documents.ts @@ -4,6 +4,8 @@ import { makeRoute } from "../lib"; import type { Env } from "./route"; import { getDocumentURL } from "app/(app)/lish/createPub/getPublicationURL"; import { normalizeDocumentRecord } from "src/utils/normalizeRecords"; +import { publicationUriVariants } from "src/utils/uriHelpers"; +import { dedupeDocumentsInPublications } from "src/utils/deduplicateRecords"; export type SearchPublicationDocumentsReturnType = Awaited< ReturnType<(typeof search_publication_documents)["handler"]> @@ -20,14 +22,16 @@ export const search_publication_documents = makeRoute({ { publication_uri, query, limit }, { supabase }: Pick, ) => { - // Get documents in the publication, filtering by title using JSON operator - // Also join with publications to get the record for URL construction + // Get documents in the publication, filtering by title using JSON operator. + // Also join with publications to get the record for URL construction. + // Documents link to whichever namespace URI their record names, so match + // both variants of the publication URI. const { data: documents, error } = await supabase .from("documents_in_publications") .select( "document, documents!inner(uri, data), publications!inner(uri, record)", ) - .eq("publication", publication_uri) + .in("publication", publicationUriVariants(publication_uri)) .ilike("documents.data->>title", `%${query}%`) .limit(limit); @@ -37,7 +41,7 @@ export const search_publication_documents = makeRoute({ ); } - const result = documents.map((d) => { + const result = dedupeDocumentsInPublications(documents).map((d) => { const normalizedDoc = normalizeDocumentRecord(d.documents.data, d.documents.uri); return { diff --git a/src/utils/deduplicateRecords.ts b/src/utils/deduplicateRecords.ts index 2dadaf5c..b350e3cf 100644 --- a/src/utils/deduplicateRecords.ts +++ b/src/utils/deduplicateRecords.ts @@ -82,6 +82,34 @@ export function deduplicateByUri(records: T[]): T[] { return Array.from(recordsByKey.values()); } +/** + * Deduplicates documents_in_publications rows fetched across a publication's + * namespace URI variants (see publicationUriVariants). A migrated document is + * linked under both the site.standard and pub.leaflet publication URIs, so the + * merged rows can carry the same document identity twice — keep one row per + * identity, preferring the site.standard document. Rows without a joined + * document are dropped. + */ +export function dedupeDocumentsInPublications< + T extends { documents: { uri: string } | null }, +>(rows: T[]): (T & { documents: NonNullable })[] { + const withDocs = rows.filter( + (r): r is T & { documents: NonNullable } => !!r.documents, + ); + const kept = new Set( + deduplicateByUriOrdered(withDocs.map((r) => r.documents)).map( + (d) => d.uri, + ), + ); + const seen = new Set(); + return withDocs.filter((r) => { + const uri = r.documents.uri; + if (!kept.has(uri) || seen.has(uri)) return false; + seen.add(uri); + return true; + }); +} + /** * Deduplicates records while preserving the original order based on the first * occurrence of each unique record. diff --git a/src/utils/uriHelpers.ts b/src/utils/uriHelpers.ts index 23336e5c..b8abe685 100644 --- a/src/utils/uriHelpers.ts +++ b/src/utils/uriHelpers.ts @@ -27,6 +27,30 @@ export function publicationUriFilter(did: string, rkey: string): string { return `uri.eq.${standard},uri.eq.${legacy}`; } +/** + * Both namespace URIs a publication may be indexed under (same DID and rkey, + * in site.standard.publication and pub.leaflet.publication). A publication can + * have a row for each, and documents link to whichever variant their record + * names — so queries over a publication's documents must match both. + */ +export function publicationUriVariants(uri: string): string[] { + try { + const aturi = new AtUri(uri); + if ( + aturi.collection !== ids.SiteStandardPublication && + aturi.collection !== ids.PubLeafletPublication + ) { + return [uri]; + } + return [ + AtUri.make(aturi.host, ids.SiteStandardPublication, aturi.rkey).toString(), + AtUri.make(aturi.host, ids.PubLeafletPublication, aturi.rkey).toString(), + ]; + } catch { + return [uri]; + } +} + /** * Returns an OR filter string for Supabase queries to match a publication by name * or by either namespace URI. Used when the rkey might be the publication name.