diff --git a/app/(app)/lish/[did]/[publication]/generateFeed.ts b/app/(app)/lish/[did]/[publication]/generateFeed.ts index d8c71205..f5f5660a 100644 --- a/app/(app)/lish/[did]/[publication]/generateFeed.ts +++ b/app/(app)/lish/[did]/[publication]/generateFeed.ts @@ -11,7 +11,11 @@ import { hasLeafletContent, isLeafletPublication, } from "src/utils/normalizeRecords"; -import { publicationNameOrUriFilter } from "src/utils/uriHelpers"; +import { + publicationNameOrUriFilter, + publicationUriVariants, +} from "src/utils/uriHelpers"; +import { dedupeDocumentsInPublications } from "src/utils/deduplicateRecords"; import { getDocumentURL, getPublicationURL, @@ -61,11 +65,7 @@ export async function generateFeed( ); let { data: publications, error } = await supabaseServerClient .from("publications") - .select( - `*, - documents_in_publications(documents(*)) - `, - ) + .select("*") .eq("identity_did", did) .or(publicationNameOrUriFilter(did, publication_name)) .order("uri", { ascending: false }) @@ -79,6 +79,19 @@ export async function generateFeed( let publication = publications?.[0]; if (!publication) return new NextResponse(null, { status: 404 }); + // Documents link to whichever namespace URI their record names, and the + // publication may be indexed under both — fetch links for both variants so + // legacy pub.leaflet-only posts still show up alongside migrated ones. + let { data: docLinks, error: docsError } = await supabaseServerClient + .from("documents_in_publications") + .select("documents(*)") + .in("publication", publicationUriVariants(publication.uri)); + if (docsError) { + console.error(docsError); + return new NextResponse(null, { status: 500 }); + } + const documentsInPublication = dedupeDocumentsInPublications(docLinks ?? []); + const pubRecord = normalizePublicationRecord(publication.record); // Legacy pub.leaflet publications without a configured domain don't // normalize (no URL), but they're still browsable at leaflet.pub/lish/…, @@ -97,7 +110,7 @@ export async function generateFeed( ); const description = pubRecord?.description ?? rawRecord?.description; - let docs = publication.documents_in_publications + let docs = documentsInPublication .sort((a, b) => { const dateA = a.documents?.sort_date ? new Date(a.documents.sort_date).getTime() diff --git a/app/(app)/lish/[did]/[publication]/getPublicationForPage.ts b/app/(app)/lish/[did]/[publication]/getPublicationForPage.ts index 5b7b83a1..a00feb84 100644 --- a/app/(app)/lish/[did]/[publication]/getPublicationForPage.ts +++ b/app/(app)/lish/[did]/[publication]/getPublicationForPage.ts @@ -1,5 +1,9 @@ import { supabaseServerClient } from "supabase/serverClient"; -import { publicationNameOrUriFilter } from "src/utils/uriHelpers"; +import { + publicationNameOrUriFilter, + publicationUriVariants, +} from "src/utils/uriHelpers"; +import { dedupeDocumentsInPublications } from "src/utils/deduplicateRecords"; export async function fetchPublicationForPage( did: string, @@ -10,17 +14,32 @@ export async function fetchPublicationForPage( .select( `uri, name, identity_did, record, publication_newsletter_settings(enabled), - publication_pages(id, path, title, record, record_uri, sort_order), - documents_in_publications(documents(uri, data, - comments_on_documents(count), - document_mentions_in_bsky(count), - recommends_on_documents(count)))`, + publication_pages(id, path, title, record, record_uri, sort_order)`, ) .eq("identity_did", did) .or(publicationNameOrUriFilter(did, publicationName)) .order("uri", { ascending: false }) .limit(1); - return data?.[0] ?? null; + const publication = data?.[0]; + if (!publication) return null; + + // Documents link to whichever namespace URI their record names, and the + // publication may be indexed under both — fetch links for both variants so + // legacy pub.leaflet-only posts still show up alongside migrated ones. + const { data: docLinks } = await supabaseServerClient + .from("documents_in_publications") + .select( + `documents(uri, data, + comments_on_documents(count), + document_mentions_in_bsky(count), + recommends_on_documents(count))`, + ) + .in("publication", publicationUriVariants(publication.uri)); + + return { + ...publication, + documents_in_publications: dedupeDocumentsInPublications(docLinks ?? []), + }; } export type PublicationForPage = NonNullable< diff --git a/app/(app)/lish/[did]/[publication]/sitemap.xml/route.ts b/app/(app)/lish/[did]/[publication]/sitemap.xml/route.ts index 68e147b0..f0fbffe1 100644 --- a/app/(app)/lish/[did]/[publication]/sitemap.xml/route.ts +++ b/app/(app)/lish/[did]/[publication]/sitemap.xml/route.ts @@ -1,6 +1,10 @@ import { AtUri } from "@atproto/syntax"; import { supabaseServerClient } from "supabase/serverClient"; -import { publicationNameOrUriFilter } from "src/utils/uriHelpers"; +import { + publicationNameOrUriFilter, + publicationUriVariants, +} from "src/utils/uriHelpers"; +import { dedupeDocumentsInPublications } from "src/utils/deduplicateRecords"; import { normalizeDocumentRecord } from "src/utils/normalizeRecords"; import { isMainSiteHost } from "src/utils/customDomain"; import { isExternalLink } from "src/utils/externalPublicationLink"; @@ -36,11 +40,7 @@ export async function GET( let { data: publications } = await supabaseServerClient .from("publications") - .select( - `uri, - documents_in_publications(documents(uri, data, sort_date)), - publication_pages(path, record)`, - ) + .select(`uri, publication_pages(path, record)`) .eq("identity_did", did) .or(publicationNameOrUriFilter(did, publication_name)) .order("uri", { ascending: false }) @@ -49,6 +49,14 @@ export async function GET( let publication = publications?.[0]; if (!did || !publication) return new Response(null, { status: 404 }); + // Documents link to whichever namespace URI their record names, and the + // publication may be indexed under both — fetch links for both variants so + // legacy pub.leaflet-only posts still show up alongside migrated ones. + let { data: docLinks } = await supabaseServerClient + .from("documents_in_publications") + .select("documents(uri, data, sort_date)") + .in("publication", publicationUriVariants(publication.uri)); + let base = `https://${host}`; // Collect one entry per public URL, keyed by path so a post and a published @@ -59,9 +67,9 @@ export async function GET( if (!entries.has(path)) entries.set(path, { loc: base + path, lastmod }); }; - let posts = (publication.documents_in_publications ?? []) - .map((dip) => dip.documents) - .filter((d): d is NonNullable => !!d); + let posts = dedupeDocumentsInPublications(docLinks ?? []).map( + (dip) => dip.documents, + ); // Home page lastmod tracks the most recently published post. let latest = posts.reduce((acc, d) => { diff --git a/app/api/rpc/[command]/get_publication_data.ts b/app/api/rpc/[command]/get_publication_data.ts index 8f85be91..a64fb93c 100644 --- a/app/api/rpc/[command]/get_publication_data.ts +++ b/app/api/rpc/[command]/get_publication_data.ts @@ -5,6 +5,8 @@ import { AtUri } from "@atproto/syntax"; import { normalizeDocumentRecord } from "src/utils/normalizeRecords"; import { getIdentityData } from "actions/getIdentityData"; import { ids } from "lexicons/api/lexicons"; +import { publicationUriVariants } from "src/utils/uriHelpers"; +import { dedupeDocumentsInPublications } from "src/utils/deduplicateRecords"; export type GetPublicationDataReturnType = Awaited< ReturnType<(typeof get_publication_data)["handler"]> @@ -39,13 +41,6 @@ export const get_publication_data = makeRoute({ .from("publications") .select( `*, - documents_in_publications(members_only, documents( - *, - comments_on_documents(count), - document_mentions_in_bsky(count), - recommends_on_documents(count), - publication_post_sends(status, subscriber_count) - )), publication_subscriptions(*, identities(atp_did)), publication_email_subscribers(*, identities(atp_did)), publication_domains(*), @@ -88,8 +83,27 @@ export const get_publication_data = makeRoute({ return { result: { publication: null, documents: [], drafts: [] } }; } + // Documents link to whichever namespace URI their record names, and the + // publication may be indexed under both — fetch links for both variants so + // legacy pub.leaflet-only posts still show up alongside migrated ones. + const { data: docLinks } = await supabase + .from("documents_in_publications") + .select( + `members_only, documents( + *, + comments_on_documents(count), + document_mentions_in_bsky(count), + recommends_on_documents(count), + publication_post_sends(status, subscriber_count) + )`, + ) + .in("publication", publicationUriVariants(publication.uri)); + const documentsInPublication = dedupeDocumentsInPublications( + docLinks ?? [], + ); + // Pre-normalize documents from documents_in_publications - const documents = (publication?.documents_in_publications || []) + const documents = documentsInPublication .map((dip) => { if (!dip.documents) return null; const normalized = normalizeDocumentRecord( @@ -128,7 +142,12 @@ export const get_publication_data = makeRoute({ return { result: { - publication, + // Reattach the merged document links; clients (e.g. the theme preview) + // read publication.documents_in_publications off this object. + publication: { + ...publication, + documents_in_publications: documentsInPublication, + }, documents, drafts, }, diff --git a/app/api/rpc/[command]/search_publication_documents.ts b/app/api/rpc/[command]/search_publication_documents.ts index 7be59c7f..1a45611c 100644 --- a/app/api/rpc/[command]/search_publication_documents.ts +++ b/app/api/rpc/[command]/search_publication_documents.ts @@ -4,6 +4,8 @@ import { makeRoute } from "../lib"; import type { Env } from "./route"; import { getDocumentURL } from "app/(app)/lish/createPub/getPublicationURL"; import { normalizeDocumentRecord } from "src/utils/normalizeRecords"; +import { publicationUriVariants } from "src/utils/uriHelpers"; +import { dedupeDocumentsInPublications } from "src/utils/deduplicateRecords"; export type SearchPublicationDocumentsReturnType = Awaited< ReturnType<(typeof search_publication_documents)["handler"]> @@ -20,14 +22,16 @@ export const search_publication_documents = makeRoute({ { publication_uri, query, limit }, { supabase }: Pick, ) => { - // Get documents in the publication, filtering by title using JSON operator - // Also join with publications to get the record for URL construction + // Get documents in the publication, filtering by title using JSON operator. + // Also join with publications to get the record for URL construction. + // Documents link to whichever namespace URI their record names, so match + // both variants of the publication URI. const { data: documents, error } = await supabase .from("documents_in_publications") .select( "document, documents!inner(uri, data), publications!inner(uri, record)", ) - .eq("publication", publication_uri) + .in("publication", publicationUriVariants(publication_uri)) .ilike("documents.data->>title", `%${query}%`) .limit(limit); @@ -37,7 +41,7 @@ export const search_publication_documents = makeRoute({ ); } - const result = documents.map((d) => { + const result = dedupeDocumentsInPublications(documents).map((d) => { const normalizedDoc = normalizeDocumentRecord(d.documents.data, d.documents.uri); return { diff --git a/lexicons/src/normalize.ts b/lexicons/src/normalize.ts index 6d12be92..e771c899 100644 --- a/lexicons/src/normalize.ts +++ b/lexicons/src/normalize.ts @@ -21,12 +21,21 @@ import type * as SiteStandardPublication from "../api/types/site/standard/public import type * as SiteStandardThemeBasic from "../api/types/site/standard/theme/basic"; import type * as SiteStandardThemeColor from "../api/types/site/standard/theme/color"; import type * as PubLeafletThemeColor from "../api/types/pub/leaflet/theme/color"; -import type { $Typed } from "../api/util"; +import type { $Typed, OmitKey } from "../api/util"; import { AtUri } from "@atproto/syntax"; // Normalized document type - uses the generated site.standard.document type -// with an additional optional theme field for backwards compatibility -export type NormalizedDocument = SiteStandardDocument.Record & { +// with an additional optional theme field for backwards compatibility. +// publishedAt is optional here even though site.standard.document requires it: +// pub.leaflet.document records may legitimately lack it, and consumers all +// handle a missing date, so normalization must not drop those documents. +// (OmitKey rather than Omit: the generated Record's index signature makes +// plain Omit collapse every known key to `unknown`.) +export type NormalizedDocument = OmitKey< + SiteStandardDocument.Record, + "publishedAt" +> & { + publishedAt?: string; // Keep the original theme for components that need leaflet-specific styling theme?: PubLeafletPublication.Theme; preferences?: SiteStandardPublication.Preferences; @@ -234,12 +243,6 @@ export function normalizeDocument( if (isLeafletDocument(record)) { // Convert from pub.leaflet to site.standard - const publishedAt = record.publishedAt; - - if (!publishedAt) { - return null; - } - // For standalone documents (no publication), construct a site URL from the author // This matches the pattern used in publishToPublication.ts for new standalone docs const site = record.publication || `https://leaflet.pub/p/${record.author}`; @@ -264,7 +267,7 @@ export function normalizeDocument( title: record.title, site, path, - publishedAt, + publishedAt: record.publishedAt, description: record.description, tags: record.tags, coverImage: record.coverImage, diff --git a/src/utils/deduplicateRecords.ts b/src/utils/deduplicateRecords.ts index 2dadaf5c..b350e3cf 100644 --- a/src/utils/deduplicateRecords.ts +++ b/src/utils/deduplicateRecords.ts @@ -82,6 +82,34 @@ export function deduplicateByUri(records: T[]): T[] { return Array.from(recordsByKey.values()); } +/** + * Deduplicates documents_in_publications rows fetched across a publication's + * namespace URI variants (see publicationUriVariants). A migrated document is + * linked under both the site.standard and pub.leaflet publication URIs, so the + * merged rows can carry the same document identity twice — keep one row per + * identity, preferring the site.standard document. Rows without a joined + * document are dropped. + */ +export function dedupeDocumentsInPublications< + T extends { documents: { uri: string } | null }, +>(rows: T[]): (T & { documents: NonNullable })[] { + const withDocs = rows.filter( + (r): r is T & { documents: NonNullable } => !!r.documents, + ); + const kept = new Set( + deduplicateByUriOrdered(withDocs.map((r) => r.documents)).map( + (d) => d.uri, + ), + ); + const seen = new Set(); + return withDocs.filter((r) => { + const uri = r.documents.uri; + if (!kept.has(uri) || seen.has(uri)) return false; + seen.add(uri); + return true; + }); +} + /** * Deduplicates records while preserving the original order based on the first * occurrence of each unique record. diff --git a/src/utils/uriHelpers.ts b/src/utils/uriHelpers.ts index 23336e5c..b8abe685 100644 --- a/src/utils/uriHelpers.ts +++ b/src/utils/uriHelpers.ts @@ -27,6 +27,30 @@ export function publicationUriFilter(did: string, rkey: string): string { return `uri.eq.${standard},uri.eq.${legacy}`; } +/** + * Both namespace URIs a publication may be indexed under (same DID and rkey, + * in site.standard.publication and pub.leaflet.publication). A publication can + * have a row for each, and documents link to whichever variant their record + * names — so queries over a publication's documents must match both. + */ +export function publicationUriVariants(uri: string): string[] { + try { + const aturi = new AtUri(uri); + if ( + aturi.collection !== ids.SiteStandardPublication && + aturi.collection !== ids.PubLeafletPublication + ) { + return [uri]; + } + return [ + AtUri.make(aturi.host, ids.SiteStandardPublication, aturi.rkey).toString(), + AtUri.make(aturi.host, ids.PubLeafletPublication, aturi.rkey).toString(), + ]; + } catch { + return [uri]; + } +} + /** * Returns an OR filter string for Supabase queries to match a publication by name * or by either namespace URI. Used when the rkey might be the publication name.