From 3cde39e5dfd8f7a764a48a1d876d50cc690879c8 Mon Sep 17 00:00:00 2001 From: Fadhlan Ridhwanallah Date: Mon, 13 Jul 2026 17:23:49 +0700 Subject: [PATCH 1/6] Search: mixed cards + files entry search with scope + client parity Extend the realm `entry` search so one query can span card instances and files, and address the review feedback on how kind selection and client-side parity work. Engine - Replace the shared `jsonb_array_elements_text(types)` cross-join alias with a self-contained `TypesContains` membership predicate (pg `@>`, sqlite `json_each` EXISTS, COALESCE for NULL/absent types). Type conditions now compose correctly: `not:{type}` genuinely excludes, `every:[{type},{type}]` intersects, and rows with empty types survive negation. - Support a mixed 'all' entry scope alongside 'instance'/'file'. Wire scope - Add `scope: 'cards' | 'files' | 'all'` (default 'all') to the entry wire query; it pins boxel_index.type directly, independent of filter shape. The federated-search cache key includes it. Consumers (store.search, host-mode head prefetch, create-listing, recents, choosers) pin scope instead of a synthetic-key incantation; `anchorQueryToCardInstances` is removed. Client parity - Teach the instance-filter matcher the synthetic search-doc keys (`_title`, `_cardType`, `_isCardInstanceFile`) via a shim at the top of resolvePath, and add an isClientEvaluable guard so an unshimmed underscore key is server-only rather than blanking reconciled results. `friendlyCardType` moves to runtime-common so the index stamp and matcher share one implementation. Keys and types - Name the dual-`.json` marker `_isCardInstanceFile` with canonical `eq: false` (existence-test compile, adapter-agnostic); one home in search-doc-keys.ts plus an `excludeCardInstanceFileRows()` helper. - Append BaseDef to instance and file type chains so `{ type: baseRef }` spans both kinds and composes. Cleanup + docs - cardsOnly choosers hide file types in the type picker; trim unread markdown/isolated_html from the mixed file projection; fold searchEntriesFileMeta into searchEntries(..., 'file'). - Document scope + the mixed-search shared-key contract in docs/search.md. Requires a full reindex on deploy (stamp rename + BaseDef in types); there is no index-version knob, so trigger it explicitly. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/search.md | 29 ++ packages/base/cards-grid.gts | 7 + packages/boxel-cli/src/commands/search.ts | 6 + .../components/card-chooser/mini/index.gts | 2 + .../app/components/card-chooser/modal.gts | 6 +- .../host/app/components/card-prerender.gts | 13 + .../app/components/card-search/constants.ts | 7 +- .../components/card-search/panel-content.gts | 16 +- .../host/app/components/card-search/panel.gts | 4 + .../operator-mode/create-listing-modal.gts | 12 +- packages/host/app/resources/type-summaries.ts | 28 +- packages/host/app/routes/render/meta.ts | 11 +- .../host/app/services/host-mode-service.ts | 11 +- packages/host/app/services/store.ts | 22 +- packages/host/app/tools/search-cards.ts | 2 + .../app/utils/card-search/query-builder.ts | 93 ++++- .../utils/file-def-attributes-extractor.ts | 8 +- packages/host/app/utils/render-error.ts | 10 +- .../tests/acceptance/prerender-meta-test.gts | 1 + .../components/operator-mode-ui-test.gts | 26 ++ .../tests/integration/realm-indexing-test.gts | 6 +- .../tests/integration/realm-querying-test.gts | 3 + .../host/tests/integration/realm-test.gts | 28 +- .../unit/card-search-query-builder-test.ts | 105 +++++- .../tests/unit/index-query-engine-test.ts | 58 +++ packages/host/tests/unit/index-writer-test.ts | 10 +- .../unit/instance-filter-matcher-test.ts | 74 ++++ .../realm-server/handlers/handle-search.ts | 8 + .../tests/eq-containment-integration-test.ts | 26 +- packages/realm-server/tests/indexing-test.ts | 8 +- .../realm-server/tests/prerendering-test.ts | 1 + .../tests/search-entries-engine-test.ts | 213 ++++++++++- .../tests/searchable-parity-diff-test.ts | 6 +- packages/runtime-common/expression.ts | 43 +++ .../helpers/card-type-display-name.ts | 15 + packages/runtime-common/index-query-engine.ts | 332 ++++++++++++------ .../index-runner/file-indexer.ts | 16 +- packages/runtime-common/index-writer.ts | 2 +- .../runtime-common/instance-filter-matcher.ts | 88 +++++ packages/runtime-common/query-field-utils.ts | 13 + .../realm-index-query-engine.ts | 263 +++++++------- packages/runtime-common/search-doc-keys.ts | 33 ++ packages/runtime-common/search-entry.ts | 31 +- packages/runtime-common/searchable-parity.ts | 11 +- 44 files changed, 1383 insertions(+), 324 deletions(-) create mode 100644 packages/runtime-common/search-doc-keys.ts diff --git a/docs/search.md b/docs/search.md index d02e0e4947..80626df908 100644 --- a/docs/search.md +++ b/docs/search.md @@ -229,3 +229,32 @@ let response = await request ``` The response is an `entry` collection document: each entry resolves to prerendered HTML (the fast path) or a live serialization, and `fields: ['item']` asks for the full card/file serialization in `included`. + +### Result kinds and `scope` + +The index holds two row kinds: card instances and files (a card's `.json` is dual-indexed — it appears as both an `instance` row and a `file` row that share the same URL). A search spans both kinds by default. + +Select which kinds a query returns with the `scope` member (default `'all'`): + +- `scope: 'cards'` — card instances only. +- `scope: 'files'` — file rows only (including a card `.json`'s file row). +- `scope: 'all'` — both kinds. This returns **both** rows of a dual-indexed card `.json`. To keep each card once (drop the duplicate `.json` file row) add the dedup filter `excludeCardInstanceFileRows()` — i.e. `eq: { _isCardInstanceFile: false }`. + +`scope` pins the row kind directly, independent of the filter, so it works with any filter shape. Prefer it over discriminating kind through a `{ type: … }` anchor. + +```ts +// Card instances only, across realms: +searchEntryWireQueryFromQuery(query, { fields: ['item'], scope: 'cards' }); +``` + +### The mixed-search key contract + +Cards and files carry non-overlapping fields, so a filter that names a card field narrows to cards by construction, and a FileDef-typed filter narrows to files. A mixed (`scope: 'all'`) query can filter and sort across both kinds only through the keys both row kinds carry: + +- `matches` — full-text search over both kinds' content. +- `_title` — the row's display title (a card's `cardTitle`, a file's name); usable in `contains` and `sort`. +- `_cardType` — the card type's display name. +- `_isCardInstanceFile` — stamped `true` only on a card `.json`'s file row; the dedup key (`eq: false` keeps everything else). +- `{ type: }` — BaseDef terminates both kinds' type chains, so a BaseDef-anchored type filter matches every row and composes with other conditions. + +Any other field key narrows the result to a single kind. diff --git a/packages/base/cards-grid.gts b/packages/base/cards-grid.gts index a05ce9ad15..a4f9c1491f 100644 --- a/packages/base/cards-grid.gts +++ b/packages/base/cards-grid.gts @@ -175,6 +175,13 @@ class Isolated extends Component { displayName: 'All Cards', icon: AllCardsIcon, query: { + // Cards-only by construction: file rows (both plain files and a + // card's dual-indexed `.json`) never carry `_cardType`, so the + // `not eq` excludes them via NULL semantics even though search is + // mixed by default — while errored instances (which have `_cardType` + // but may lack a `types` array) are kept, since this is a field + // filter with no type-anchor cross-join. The "All Files" group + // (`type: baseFileRef`) owns the file side. filter: { not: { eq: { diff --git a/packages/boxel-cli/src/commands/search.ts b/packages/boxel-cli/src/commands/search.ts index 77ed8b2912..32f2c71fcb 100644 --- a/packages/boxel-cli/src/commands/search.ts +++ b/packages/boxel-cli/src/commands/search.ts @@ -108,6 +108,9 @@ interface SearchEntryRequestBody { sort?: Record[]; page?: unknown; cardUrls?: unknown; + // Which row kinds to span: 'cards' | 'files' | 'all' (default 'all' server- + // side). Pass 'cards' to restrict to card instances. + scope?: unknown; } /** @@ -150,6 +153,9 @@ export function searchEntryRequestBody( if (query.cardUrls !== undefined) { body.cardUrls = query.cardUrls; } + if (query.scope !== undefined) { + body.scope = query.scope; + } return body; } diff --git a/packages/host/app/components/card-chooser/mini/index.gts b/packages/host/app/components/card-chooser/mini/index.gts index 534c9b76ab..113dcd39bb 100644 --- a/packages/host/app/components/card-chooser/mini/index.gts +++ b/packages/host/app/components/card-chooser/mini/index.gts @@ -54,6 +54,7 @@ export default class MiniCardChooser extends Component {
@@ -66,6 +67,7 @@ export default class MiniCardChooser extends Component {
{ @initialSelectedRealms={{this.initialSelectedRealmsForPanel}} @initialSelectedTypes={{this.initialSelectedTypesForPanel}} @lockSelectedRealms={{state.lockConsumingRealm}} + @cardsOnly={{true}} as |Bar Content| > { <:content> { }); let preselectedCardUrl: string | undefined; if (opts?.preselectedCardTypeQuery) { + // The result is used as a card instance; store.search pins + // `scope: 'cards'`, so the raw query resolves to card instances only. let instances: CardDef[] = await this.store.search( - opts.preselectedCardTypeQuery!, + opts.preselectedCardTypeQuery, this.realmServer.availableRealmIdentifiers, ); if (instances?.[0]?.id) { diff --git a/packages/host/app/components/card-prerender.gts b/packages/host/app/components/card-prerender.gts index 70c5e02068..fa4f93c556 100644 --- a/packages/host/app/components/card-prerender.gts +++ b/packages/host/app/components/card-prerender.gts @@ -14,7 +14,9 @@ import Component from '@glimmer/component'; import { didCancel, enqueueTask } from 'ember-concurrency'; import { + baseRef, CardError, + internalKeysFor, SupportedMimeType, type CardErrorsJSONAPI, type LooseSingleCardDocument, @@ -713,8 +715,19 @@ export default class CardPrerender extends Component { types: string[], renderOptions?: RenderRouteOptions, ) => { + // BaseDef is part of the searchable `types` chain (so `{ type: baseRef }` + // spans cards and files) but it is abstract — a card coerced to BaseDef + // has no meaningful fitted/embedded template, so rendering that level is + // pure index bloat. Skip it; the ancestor renderings cover the concrete + // card types through CardDef. + let baseDefKeys = new Set( + internalKeysFor(baseRef, undefined, this.network.virtualNetwork), + ); let ancestors: Record = {}; for (let i = 0; i < types.length; i++) { + if (baseDefKeys.has(types[i])) { + continue; + } let res = await this.renderHTML.perform(url, format, i, renderOptions); ancestors[types[i]] = res as string; } diff --git a/packages/host/app/components/card-search/constants.ts b/packages/host/app/components/card-search/constants.ts index 6b50dafcdd..49aa24f601 100644 --- a/packages/host/app/components/card-search/constants.ts +++ b/packages/host/app/components/card-search/constants.ts @@ -55,7 +55,12 @@ export const SORT_OPTIONS: SortOption[] = [ module: baseRRI('card-api'), name: 'CardDef', }, - by: 'cardTitle', + // The sheet is mixed cards + files, so sort on the synthetic `_title` + // key that both row types carry (a card's title — mirrored from + // `cardTitle` at index time — and a file's name). Sorting on + // `cardTitle` alone would leave file rows NULL and sink them all below + // the cards under NULLS LAST. + by: '_title', direction: 'asc', }, ], diff --git a/packages/host/app/components/card-search/panel-content.gts b/packages/host/app/components/card-search/panel-content.gts index c1e344f0a9..4e8f5c2104 100644 --- a/packages/host/app/components/card-search/panel-content.gts +++ b/packages/host/app/components/card-search/panel-content.gts @@ -30,6 +30,7 @@ import type RecentCards from '@cardstack/host/services/recent-cards-service'; import { buildRecentsQuery, buildSearchQuery, + searchScopeForOptions, shouldSkipSearchQuery, } from '@cardstack/host/utils/card-search/query-builder'; import { SectionPagination } from '@cardstack/host/utils/card-search/section-pagination'; @@ -112,6 +113,9 @@ interface Signature { realmFilter: RealmFilter; typeFilter: TypeFilter; baseFilter?: Filter; + // Pin the result set to card instances. Search is mixed (cards + files) + // by default; the card choosers set this so file rows never surface. + cardsOnly?: boolean; isCompact: boolean; handleSelect: (selection: string | NewCardArgs) => void; selectedCards?: (string | NewCardArgs)[]; @@ -206,7 +210,9 @@ export default class PanelContent extends Component { this.args.activeSort, this.args.baseFilter, selectedTypeIds, + { cardsOnly: this.args.cardsOnly }, ), + { scope: searchScopeForOptions({ cardsOnly: this.args.cardsOnly }) }, ), realms: this.args.realmFilter.selectedURLs, // Cap each realm's results at the focused-section display limit — the @@ -286,7 +292,11 @@ export default class PanelContent extends Component { } if (this.args.isCompact) { return this.withMiniHtmlQuery({ - ...searchEntryWireQueryFromQuery({}), + // A recent's `.json` URL also names the card's dual-indexed file row, so + // an unfiltered `cardUrls` query would surface each recent twice. + // Recents are always cards, so `scope: 'cards'` keeps only the instance + // rows (regardless of the sheet's cardsOnly mode). + ...searchEntryWireQueryFromQuery({}, { scope: 'cards' }), realms: this.realms, cardUrls: this.recentCardUrls, }); @@ -309,7 +319,11 @@ export default class PanelContent extends Component { this.args.activeSort, this.args.baseFilter, selectedTypeIds, + // Recents are always cards, so pin the card scope regardless of the + // sheet's cardsOnly mode (this also skips the redundant dedup filter). + { cardsOnly: true }, ), + { scope: 'cards' }, ), realms: this.recentsSearchRealms, cardUrls: this.recentCardUrls, diff --git a/packages/host/app/components/card-search/panel.gts b/packages/host/app/components/card-search/panel.gts index 8bad0e27a4..5acd63506a 100644 --- a/packages/host/app/components/card-search/panel.gts +++ b/packages/host/app/components/card-search/panel.gts @@ -28,6 +28,9 @@ interface Signature { * and the realm picker is disabled. */ lockSelectedRealms?: boolean; + // A cards-only chooser: the type picker offers card types only (file types + // are hidden so one can't be selected against the card scope). + cardsOnly?: boolean; onRealmChange?: (selectedRealms: URL[]) => void; onTypeChange?: (selectedTypes: ResolvedCodeRef[]) => void; }; @@ -59,6 +62,7 @@ export default class SearchPanel extends Component { realmURLs: this.selectedRealmURLs, baseFilter: this.args.baseFilter, initialSelectedTypes: this.args.initialSelectedTypes, + cardsOnly: this.args.cardsOnly, })); private get initialFocusedSectionId(): string | null { diff --git a/packages/host/app/components/operator-mode/create-listing-modal.gts b/packages/host/app/components/operator-mode/create-listing-modal.gts index 5cdb0eb203..4454df05cd 100644 --- a/packages/host/app/components/operator-mode/create-listing-modal.gts +++ b/packages/host/app/components/operator-mode/create-listing-modal.gts @@ -113,13 +113,19 @@ export default class CreateListingModal extends Component { // chosen card URLs (matched by index file URL, hence the `.json`-suffixed // `selectedExampleCardUrls`) and their realms, with the `atom` rendering // bound through the filter's `htmlQuery` (the way to select a prerendered - // format). The eq carries only that binding, which the engine lifts out, - // leaving the result set scoped purely by `cardUrls`. + // format). The engine lifts the htmlQuery binding out of the eq (which then + // dissolves). `scope: 'cards'` pins the instance row, dropping each example's + // dual-indexed `.json` file row that shares its `cardUrls` URL. private get exampleSearchQuery(): SearchEntryWireQuery { return { cardUrls: this.selectedExampleCardUrls, realms: this.selectedExampleRealms, - filter: { eq: { htmlQuery: { eq: { format: 'atom' } } } }, + scope: 'cards', + filter: { + eq: { + htmlQuery: { eq: { format: 'atom' } }, + }, + }, }; } diff --git a/packages/host/app/resources/type-summaries.ts b/packages/host/app/resources/type-summaries.ts index fb5c51f239..4e6c2746f5 100644 --- a/packages/host/app/resources/type-summaries.ts +++ b/packages/host/app/resources/type-summaries.ts @@ -35,7 +35,15 @@ export interface TypeOption { type TypeSummaryItem = { id: string; type: string; - attributes: { displayName: string; total: number; iconHTML: string }; + attributes: { + displayName: string; + total: number; + iconHTML: string; + // The `_federated-types` response tags each entry as a card-instance or a + // file type. A cards-only chooser hides the file entries. Optional for + // back-compat with the legacy response (no kind — treated as 'instance'). + kind?: 'instance' | 'file'; + }; meta?: { realmURL: string }; }; @@ -44,6 +52,9 @@ export interface TypeSummariesArgs { realmURLs: string[]; baseFilter: Filter | undefined; initialSelectedTypes: ResolvedCodeRef[] | undefined; + // When true (a cards-only chooser), file-type entries are excluded from the + // picker options so a file type can't be selected against a card scope. + cardsOnly?: boolean; owner: Owner; }; } @@ -81,12 +92,16 @@ export class TypeSummariesResource extends Resource { }); } + #cardsOnly = false; + modify(_positional: never[], named: TypeSummariesArgs['named']) { - let { realmURLs, baseFilter, initialSelectedTypes, owner } = named; + let { realmURLs, baseFilter, initialSelectedTypes, cardsOnly, owner } = + named; setOwner(this, owner); this.#baseFilter = baseFilter; this.#initialSelectedTypes = initialSelectedTypes; + this.#cardsOnly = cardsOnly ?? false; // Only re-fetch when realmURLs change (search key changes are handled by onSearchChange) let realmURLsKey = realmURLs.join(','); @@ -299,6 +314,13 @@ export class TypeSummariesResource extends Resource { continue; } + // A cards-only chooser offers card types only — hide file-type entries so + // one can't be picked against the card scope (which would just return no + // results). + if (this.#cardsOnly && item.attributes.kind === 'file') { + continue; + } + // Never show root types — they are internal meta types if (rootTypeKeys.has(codeRef)) { continue; @@ -384,6 +406,7 @@ export function getTypeSummaries( realmURLs: string[]; baseFilter: Filter | undefined; initialSelectedTypes: ResolvedCodeRef[] | undefined; + cardsOnly?: boolean; }, ) { let resource = TypeSummariesResource.from(parent, () => ({ @@ -391,6 +414,7 @@ export function getTypeSummaries( realmURLs: getArgs().realmURLs, baseFilter: getArgs().baseFilter, initialSelectedTypes: getArgs().initialSelectedTypes, + cardsOnly: getArgs().cardsOnly, owner, }, })); diff --git a/packages/host/app/routes/render/meta.ts b/packages/host/app/routes/render/meta.ts index ff1dd7b186..a1bf5309c4 100644 --- a/packages/host/app/routes/render/meta.ts +++ b/packages/host/app/routes/render/meta.ts @@ -452,10 +452,19 @@ export function getTypes(klass: typeof BaseDef): CodeRef[] { while (current) { let ref = identifyCard(current); - if (!ref || isEqual(ref, baseRef)) { + if (!ref) { break; } types.push(ref); + // Include BaseDef (the common ancestor of CardDef and FileDef) as the final + // element, then stop — so a `{ type: baseRef }` filter matches both card + // and file rows (e.g. a `scope: 'all'` search using one type ref). The file + // chains (file-indexer.ts, file-def-attributes-extractor.ts) terminate at + // BaseDef the same way. `getDisplayNames` deliberately does NOT — no + // consumer reads past element 0, and a BaseDef display name would be noise. + if (isEqual(ref, baseRef)) { + break; + } current = Reflect.getPrototypeOf(current) as typeof BaseDef | undefined; } return types; diff --git a/packages/host/app/services/host-mode-service.ts b/packages/host/app/services/host-mode-service.ts index 021869b647..0620ceab5b 100644 --- a/packages/host/app/services/host-mode-service.ts +++ b/packages/host/app/services/host-mode-service.ts @@ -320,7 +320,16 @@ export default class HostModeService extends Service { body: JSON.stringify({ realms: [realmRoot], cardUrls: [cardJsonURL], - filter: { eq: { htmlQuery: { eq: { format: 'head' } } } }, + // `scope: 'cards'` pins the instance row, dropping the card `.json`'s + // dual-indexed file row that shares the `cardUrls` URL — a by-URL card + // lookup, so it's exactly the scope's purpose (and immune to the + // un-restamped-rows window a stamp-based dedup depends on). + scope: 'cards', + filter: { + eq: { + htmlQuery: { eq: { format: 'head' } }, + }, + }, fields: { entry: ['html'] }, }), }); diff --git a/packages/host/app/services/store.ts b/packages/host/app/services/store.ts index 684565478b..69c2e7eae4 100644 --- a/packages/host/app/services/store.ts +++ b/packages/host/app/services/store.ts @@ -33,6 +33,7 @@ import { isSparseItemResource, resolveFileDefCodeRef, searchEntryWireQueryFromQuery, + getTypeRefsFromFilter, X_BOXEL_JOB_PRIORITY_HEADER, userInitiatedPriority, Deferred, @@ -76,6 +77,7 @@ import { type StoreReadType, type CardResource, type SearchEntryResults, + type SearchEntryScope, type SearchEntryWireQuery, type EntrySingleDocument, isEntrySingleDocument, @@ -1263,8 +1265,26 @@ export default class StoreService extends Service implements StoreInterface { query: Query, realms: string[], ): Promise { + // Search spans card instances and files. A query with a positive type ref + // already selects a kind (a card type -> instances, a FileDef type -> + // files), so it passes through with the default 'all' scope and its filter + // discriminates. An otherwise-unscoped query is pinned to 'cards' so the + // common "search for cards" case doesn't surface a card's dual-indexed + // `.json` file row (or plain files) — the choke point that replaces the + // former per-call-site card anchor, while leaving file/typed searches + // (e.g. SearchResource's file-meta queries) untouched. + let typeRefs = query.filter + ? getTypeRefsFromFilter(query.filter) + : undefined; + let hasPositiveType = typeRefs?.some((r) => !r.negated) ?? false; + let scope: SearchEntryScope | undefined = hasPositiveType + ? undefined + : 'cards'; return await this.fetchSearchEntryDoc( - searchEntryWireQueryFromQuery(query, { fields: ['item'] }), + searchEntryWireQueryFromQuery(query, { + fields: ['item'], + ...(scope ? { scope } : {}), + }), realms, ); } diff --git a/packages/host/app/tools/search-cards.ts b/packages/host/app/tools/search-cards.ts index 5ee7d4e1f0..5e3b78130f 100644 --- a/packages/host/app/tools/search-cards.ts +++ b/packages/host/app/tools/search-cards.ts @@ -77,6 +77,8 @@ export class SearchCardsByQueryTool extends HostBaseTool< let realmUrls = this.realmServer.availableRealmIdentifiers; let instances: CardDef[] = []; try { + // store.search pins `scope: 'cards'`, so the raw query already resolves + // to card instances only. instances = await this.store.search(input.query, realmUrls); } catch (e) { console.error(`Error searching in realms:`, e, input.query); diff --git a/packages/host/app/utils/card-search/query-builder.ts b/packages/host/app/utils/card-search/query-builder.ts index 57baab96f7..e2053dc885 100644 --- a/packages/host/app/utils/card-search/query-builder.ts +++ b/packages/host/app/utils/card-search/query-builder.ts @@ -3,9 +3,12 @@ import type { Filter, Query, ResolvedCodeRef, + SearchEntryScope, } from '@cardstack/runtime-common'; import { codeRefFromInternalKey, + excludeCardInstanceFileRows, + getTypeRefsFromFilter, isAnyFilter, isCardTypeFilter, isEveryFilter, @@ -29,9 +32,11 @@ export function shouldSkipSearchQuery( // Removes CardTypeFilter nodes from a filter tree. // Returns undefined if the entire filter was type-only. -// Used to avoid combining two type conditions in an `every` clause, -// which produces impossible SQL (the query engine shares one cross-join alias -// for all tableValuedEach('types') references). +// When a subtype is picked from the type filter, the base filter's (parent) +// type condition is redundant — the picked subtype's `types` already contains +// the parent — so dropping it keeps the query minimal. (Type conditions now +// compose as an intersection via self-contained membership predicates, so +// keeping both would also be correct; this is a simplification, not a fix.) function stripTypeFromFilter(filter: Filter): Filter | undefined { // Preserve the `on` property (TypedFilter) so field-scoped filters // like {on: specRef, every: [{eq: {isCard: true}}]} keep their type context. @@ -84,28 +89,78 @@ function buildTypeFilter( return { any: codeRefs.map((ref) => ({ type: ref })) }; } -// OR-combine full-text markdown search with a cardTitle substring match so -// short prefixes (e.g. "ma" → "mango", "mark") still find cards by title +// OR-combine full-text markdown search with a `_title` substring match so +// short prefixes (e.g. "ma" → "mango", "mark") still find results by title // while longer/natural-language queries tap markdown content via `matches`. +// `_title` is the synthetic key stamped on both card and file rows (a card's +// title, a file's name), so the term matches files by name too. function buildSearchTermFilter(searchTerm: string): Filter { return { - any: [{ matches: searchTerm }, { contains: { cardTitle: searchTerm } }], + any: [{ matches: searchTerm }, { contains: { _title: searchTerm } }], }; } +// Search spans card and file rows in one query. Kind selection is a wire-level +// concern now: a `opts.cardsOnly` caller sends `scope: 'cards'` (see +// `searchScopeForOptions`), which pins `boxel_index.type` to instance rows +// server-side — no filter anchor needed, and immune to a stray file-type filter +// (which just yields no results rather than leaking files). The mixed sheet +// (no `cardsOnly`, default `scope: 'all'`) still discriminates the one case +// scope can't: dropping a card's dual-indexed `.json` file row so the card +// shows once. See `scopeFilters`. +export interface BuildQueryOptions { + cardsOnly?: boolean; +} + +// The wire scope for a set of build options. cardsOnly pins card-instance rows; +// the mixed default leaves scope unset ('all'). +export function searchScopeForOptions( + opts: BuildQueryOptions | undefined, +): SearchEntryScope | undefined { + return opts?.cardsOnly ? 'cards' : undefined; +} + +function hasPositiveTypeRef(filter: Filter | undefined): boolean { + if (!filter) { + return false; + } + return getTypeRefsFromFilter(filter)?.some((r) => !r.negated) ?? false; +} + +function scopeFilters( + filters: Filter[], + opts: BuildQueryOptions | undefined, + hasPositiveType: boolean, +): Filter[] { + // cardsOnly is enforced by `scope: 'cards'` on the wire, so nothing to add. + if (opts?.cardsOnly) { + return filters; + } + // Mixed (`scope: 'all'`) sheet: drop a card's dual-indexed `.json` file row so + // the card shows once (via its instance row). Skipped when the filter already + // carries a positive type ref — a picked card type matches only instance rows + // (no dupe to drop), and a picked file type must stay free to surface a + // `.json` file row that legitimately matches it. + if (hasPositiveType) { + return filters; + } + return [...filters, excludeCardInstanceFileRows()]; +} + export function buildSearchQuery( searchKey: string, activeSort: SortOption, baseFilter?: Filter, selectedTypeIds?: string[], + opts?: BuildQueryOptions, ): Query { const typeFilter = buildTypeFilter(selectedTypeIds); const searchTerm = searchKey?.trim() || undefined; let filters: Filter[]; if (baseFilter) { - // When typeFilter is present, strip the baseFilter's type constraint - // to avoid SQL conflict where two type conditions share one cross-join alias. - // This is safe because the type picker only offers subtypes of the baseFilter type. + // When typeFilter is present, strip the baseFilter's (parent) type + // constraint as redundant — the picked subtype already implies it. Safe + // because the type picker only offers subtypes of the baseFilter type. const effectiveBaseFilter = typeFilter ? stripTypeFromFilter(baseFilter) : baseFilter; @@ -121,6 +176,11 @@ export function buildSearchQuery( ...(searchTerm ? [buildSearchTermFilter(searchTerm)] : []), ]; } + filters = scopeFilters( + filters, + opts, + Boolean(typeFilter) || hasPositiveTypeRef(baseFilter), + ); return { filter: filters.length === 1 ? filters[0] : { every: filters }, sort: activeSort.sort, @@ -129,21 +189,27 @@ export function buildSearchQuery( // Narrower query for the Recents section. Matches the pre-prerendered // client-side behavior where a search term filtered recents only by -// cardTitle substring — unlike the realm search which also runs +// title substring — unlike the realm search which also runs // full-text `matches` on markdown. Using `matches` here picks up // linked-card content (e.g. Fadhlan's card markdown includes its // linked Mango pet, so "man" matches via "mango"), producing false // positives the previous UX never showed. +// +// Recents are always cards (their URLs are card `.json`s, applied by the +// caller through `cardUrls`), but the same URL also names the card's +// dual-indexed file row — so the mixed/cards-only scoping here is what keeps +// each recent from surfacing twice. export function buildRecentsQuery( searchTerm: string | undefined, activeSort: SortOption, baseFilter?: Filter, selectedTypeIds?: string[], + opts?: BuildQueryOptions, ): Query { const typeFilter = buildTypeFilter(selectedTypeIds); const term = searchTerm?.trim() || undefined; const termFilter: Filter | undefined = term - ? { contains: { cardTitle: term } } + ? { contains: { _title: term } } : undefined; let filters: Filter[]; if (baseFilter) { @@ -162,6 +228,11 @@ export function buildRecentsQuery( ...(termFilter ? [termFilter] : []), ]; } + filters = scopeFilters( + filters, + opts, + Boolean(typeFilter) || hasPositiveTypeRef(baseFilter), + ); return { filter: filters.length === 1 ? filters[0] : { every: filters }, sort: activeSort.sort, diff --git a/packages/host/app/utils/file-def-attributes-extractor.ts b/packages/host/app/utils/file-def-attributes-extractor.ts index 80caf68eed..dec68d5461 100644 --- a/packages/host/app/utils/file-def-attributes-extractor.ts +++ b/packages/host/app/utils/file-def-attributes-extractor.ts @@ -390,10 +390,16 @@ export function getTypes(klass: FileDefConstructor): CodeRef[] { while (current) { let ref = identifyCard(current as unknown as typeof BaseDef); - if (!ref || isEqual(ref, baseRef)) { + if (!ref) { break; } types.push(ref); + // Include BaseDef as the final element then stop, matching the card-side + // getTypes (routes/render/meta.ts) so a `{ type: baseRef }` filter spans + // file and card rows. getDisplayNames below intentionally does not. + if (isEqual(ref, baseRef)) { + break; + } current = Reflect.getPrototypeOf(current) as FileDefConstructor | undefined; } return types; diff --git a/packages/host/app/utils/render-error.ts b/packages/host/app/utils/render-error.ts index 224903b623..466046f3a5 100644 --- a/packages/host/app/utils/render-error.ts +++ b/packages/host/app/utils/render-error.ts @@ -12,9 +12,15 @@ import { isCardErrorJSONAPI, serializableError, } from '@cardstack/runtime-common/error'; +// Re-exported so existing host imports of `friendlyCardType` from this module +// keep working; the implementation lives in runtime-common so the index stamp +// and the client-side matcher share one definition. +import { friendlyCardType } from '@cardstack/runtime-common/helpers/card-type-display-name'; import type { CardDef } from '@cardstack/base/card-api'; +export { friendlyCardType }; + export interface RenderErrorContext { cardId?: string; normalizeCardId?(id: string): string; @@ -213,10 +219,6 @@ export class RenderCardTypeTracker { } } -export function friendlyCardType(card: typeof CardDef): string { - return card.displayName === 'Card' ? card.name : card.displayName; -} - export function mergeDeps( ...depLists: (string[] | undefined)[] ): string[] | undefined { diff --git a/packages/host/tests/acceptance/prerender-meta-test.gts b/packages/host/tests/acceptance/prerender-meta-test.gts index 4b1815b5d8..795327c18e 100644 --- a/packages/host/tests/acceptance/prerender-meta-test.gts +++ b/packages/host/tests/acceptance/prerender-meta-test.gts @@ -355,6 +355,7 @@ module('Acceptance | prerender | meta', function (hooks) { `${testRealmURL}cat/Cat`, `${testRealmURL}pet/Pet`, `${baseRealmRRI}card-api/CardDef`, + `${baseRealmRRI}card-api/BaseDef`, ], 'types are correct', ); diff --git a/packages/host/tests/integration/components/operator-mode-ui-test.gts b/packages/host/tests/integration/components/operator-mode-ui-test.gts index 21bcf9f8e4..6d5c0f275f 100644 --- a/packages/host/tests/integration/components/operator-mode-ui-test.gts +++ b/packages/host/tests/integration/components/operator-mode-ui-test.gts @@ -450,6 +450,32 @@ module('Integration | operator-mode | ui', function (hooks) { .hasValue(`${testRealmURL}BlogPost/1.json${someRandomText}`); }); + test('cards-grid "All Cards" group stays cards-only', async function (assert) { + // Search spans card and file rows in one query, but the "All Cards" + // group's `not: { eq: { _cardType } }` filter keeps it cards-only: + // file rows (module files and the cards' own dual-indexed `.json`) + // never carry `_cardType`, so they drop out via NULL semantics. + ctx.setCardInOperatorModeState(`${testRealmURL}grid`); + await renderComponent( + class TestDriver extends GlimmerComponent { + + }, + ); + await waitFor(`[data-test-stack-card="${testRealmURL}grid"]`); + await click(`[data-test-boxel-filter-list-button="All Cards"]`); + await waitFor(`[data-test-cards-grid-item]`); + + assert + .dom(`[data-test-cards-grid-item="${testRealmURL}BlogPost/1"]`) + .exists( + { count: 1 }, + 'a card appears once — its `.json` file row is not a second grid item', + ); + assert + .dom(`[data-test-cards-grid-item="${testRealmURL}blog-post"]`) + .doesNotExist('a module file row is not an "All Cards" grid item'); + }); + test(`can open and close search sheet`, async function (assert) { ctx.setCardInOperatorModeState(`${testRealmURL}grid`); await renderComponent( diff --git a/packages/host/tests/integration/realm-indexing-test.gts b/packages/host/tests/integration/realm-indexing-test.gts index 56b5d51c33..2b725e0e47 100644 --- a/packages/host/tests/integration/realm-indexing-test.gts +++ b/packages/host/tests/integration/realm-indexing-test.gts @@ -148,7 +148,11 @@ module(`Integration | realm indexing`, function (hooks) { }, }); let queryEngine = realm.realmIndexQueryEngine; - let { data: cards } = await searchCardsForTest(queryEngine, {}); + // anchored to cards — an unanchored entry query also returns the card + // `.json` file rows + let { data: cards } = await searchCardsForTest(queryEngine, { + filter: { type: baseCardRef }, + }); assert.deepEqual(cards, [ { id: testRRI('empty'), diff --git a/packages/host/tests/integration/realm-querying-test.gts b/packages/host/tests/integration/realm-querying-test.gts index afc0be513c..f31599ba61 100644 --- a/packages/host/tests/integration/realm-querying-test.gts +++ b/packages/host/tests/integration/realm-querying-test.gts @@ -1202,7 +1202,10 @@ module(`Integration | realm querying`, function (hooks) { }); test('can sort by card display name (card type shown in the interface)', async function (assert) { + // Anchor to card instances: search is mixed cards + files by default, and + // this test asserts the card ordering, so exclude file rows via the type. let { data: matching } = await searchCardsForTest(queryEngine, { + filter: { type: baseCardRef }, sort: [ { on: baseCardRef, diff --git a/packages/host/tests/integration/realm-test.gts b/packages/host/tests/integration/realm-test.gts index cbc9e1016e..46901edf9e 100644 --- a/packages/host/tests/integration/realm-test.gts +++ b/packages/host/tests/integration/realm-test.gts @@ -7,6 +7,7 @@ import { validate as uuidValidate } from 'uuid'; import type { Realm } from '@cardstack/runtime-common'; import { + baseCardRef, baseRealmRRI, rri, searchEntryWireQueryFromQuery, @@ -462,7 +463,12 @@ module('Integration | realm', function (hooks) { 'Content-Type': 'application/json', }, body: JSON.stringify( - searchEntryWireQueryFromQuery({}, { fields: ['item'] }), + // anchored to cards — an unanchored entry query also returns the + // card `.json` file rows + searchEntryWireQueryFromQuery( + { filter: { type: baseCardRef } }, + { fields: ['item'] }, + ), ), }), ); @@ -2720,7 +2726,9 @@ module('Integration | realm', function (hooks) { let queryEngine = realm.realmIndexQueryEngine; - let { data: cards } = await searchCardsForTest(queryEngine, {}); + let { data: cards } = await searchCardsForTest(queryEngine, { + filter: { type: baseCardRef }, + }); assert.strictEqual(cards.length, 2, 'two cards found'); let result = await queryEngine.cardDocument( @@ -2771,7 +2779,11 @@ module('Integration | realm', function (hooks) { 'card 1 is still there', ); - cards = (await searchCardsForTest(queryEngine, {})).data; + cards = ( + await searchCardsForTest(queryEngine, { + filter: { type: baseCardRef }, + }) + ).data; assert.strictEqual(cards.length, 1, 'only one card remains'); }); @@ -3026,7 +3038,12 @@ module('Integration | realm', function (hooks) { 'Content-Type': 'application/json', }, body: JSON.stringify( - searchEntryWireQueryFromQuery({}, { fields: ['item'] }), + // anchored to cards — an unanchored entry query also returns the + // card `.json` file rows + searchEntryWireQueryFromQuery( + { filter: { type: baseCardRef } }, + { fields: ['item'] }, + ), ), }), ); @@ -3120,6 +3137,9 @@ module('Integration | realm', function (hooks) { body: JSON.stringify( searchEntryWireQueryFromQuery( { + // anchored to cards — an unanchored entry query also returns + // the card `.json` file rows + filter: { type: baseCardRef }, sort: [ { by: 'id', diff --git a/packages/host/tests/unit/card-search-query-builder-test.ts b/packages/host/tests/unit/card-search-query-builder-test.ts index 4f90141da0..d6e9d125ee 100644 --- a/packages/host/tests/unit/card-search-query-builder-test.ts +++ b/packages/host/tests/unit/card-search-query-builder-test.ts @@ -2,6 +2,7 @@ import { module, test } from 'qunit'; import { baseCardRef, + excludeCardInstanceFileRows, specRef, type Filter, type RealmResourceIdentifier, @@ -11,7 +12,9 @@ import { import type { SortOption } from '@cardstack/host/components/card-search/constants'; import { + buildRecentsQuery, buildSearchQuery, + searchScopeForOptions, shouldSkipSearchQuery, } from '@cardstack/host/utils/card-search/query-builder'; @@ -20,17 +23,22 @@ const SORT_AZ: SortOption = { sort: [{ by: 'cardTitle', direction: 'asc' }] as Sort, }; +// The mixed-search (`scope: 'all'`) dedup: drops a card `.json`'s dual-indexed +// file row while keeping cards and plain files. Restricting to a single kind is +// a wire-`scope` concern (see searchScopeForOptions), not a filter. +const DEDUP_FILTER: Filter = excludeCardInstanceFileRows(); + module('Unit | card-search/query-builder', function () { module('buildSearchQuery', function () { - test('empty search key with no baseFilter collapses to the single not-spec filter', function (assert) { + test('empty search key with no baseFilter combines not-spec with the card-json dedup', function (assert) { let query = buildSearchQuery('', SORT_AZ); assert.deepEqual(query, { - filter: { not: { type: specRef } }, + filter: { every: [{ not: { type: specRef } }, DEDUP_FILTER] }, sort: SORT_AZ.sort, }); }); - test('non-empty search key with no baseFilter OR-combines matches with cardTitle substring', function (assert) { + test('non-empty search key with no baseFilter OR-combines matches with a _title substring', function (assert) { let query = buildSearchQuery('xylophone', SORT_AZ); assert.deepEqual(query, { filter: { @@ -39,9 +47,10 @@ module('Unit | card-search/query-builder', function () { { any: [ { matches: 'xylophone' }, - { contains: { cardTitle: 'xylophone' } }, + { contains: { _title: 'xylophone' } }, ], }, + DEDUP_FILTER, ], }, sort: SORT_AZ.sort, @@ -51,12 +60,12 @@ module('Unit | card-search/query-builder', function () { test('whitespace-only search key is treated as empty', function (assert) { let query = buildSearchQuery(' ', SORT_AZ); assert.deepEqual(query, { - filter: { not: { type: specRef } }, + filter: { every: [{ not: { type: specRef } }, DEDUP_FILTER] }, sort: SORT_AZ.sort, }); }); - test('baseFilter alone returns that filter without matches', function (assert) { + test('a positively-typed baseFilter passes through alone — the picked type already selects one kind', function (assert) { let baseFilter: Filter = { type: baseCardRef }; let query = buildSearchQuery('', SORT_AZ, baseFilter); assert.deepEqual(query, { @@ -65,7 +74,7 @@ module('Unit | card-search/query-builder', function () { }); }); - test('baseFilter with non-empty search wraps in every and OR-combines matches + cardTitle', function (assert) { + test('baseFilter with non-empty search wraps in every and OR-combines matches + _title', function (assert) { let baseFilter: Filter = { type: baseCardRef }; let query = buildSearchQuery('puppy', SORT_AZ, baseFilter); assert.deepEqual(query, { @@ -73,7 +82,7 @@ module('Unit | card-search/query-builder', function () { every: [ baseFilter, { - any: [{ matches: 'puppy' }, { contains: { cardTitle: 'puppy' } }], + any: [{ matches: 'puppy' }, { contains: { _title: 'puppy' } }], }, ], }, @@ -94,13 +103,91 @@ module('Unit | card-search/query-builder', function () { { not: { type: specRef } }, { type: authorRef }, { - any: [{ matches: 'droid' }, { contains: { cardTitle: 'droid' } }], + any: [{ matches: 'droid' }, { contains: { _title: 'droid' } }], }, ], }, sort: SORT_AZ.sort, }); }); + + test('cardsOnly adds no filter anchor — the card scope is pinned on the wire', function (assert) { + let query = buildSearchQuery('', SORT_AZ, undefined, undefined, { + cardsOnly: true, + }); + assert.deepEqual( + query, + { + filter: { not: { type: specRef } }, + sort: SORT_AZ.sort, + }, + 'no baseCardRef anchor and no dedup filter — scope: cards handles it', + ); + }); + + test('cardsOnly with a positively-typed baseFilter passes the filter through unchanged', function (assert) { + let authorRef = { + module: 'http://test-realm/test/author' as RealmResourceIdentifier, + name: 'Author', + }; + let baseFilter: Filter = { type: authorRef }; + let query = buildSearchQuery('', SORT_AZ, baseFilter, undefined, { + cardsOnly: true, + }); + assert.deepEqual(query, { + filter: baseFilter, + sort: SORT_AZ.sort, + }); + }); + }); + + module('buildRecentsQuery', function () { + test('no term with no baseFilter combines not-spec with the card-json dedup', function (assert) { + let query = buildRecentsQuery(undefined, SORT_AZ); + assert.deepEqual(query, { + filter: { every: [{ not: { type: specRef } }, DEDUP_FILTER] }, + sort: SORT_AZ.sort, + }); + }); + + test('a term filters recents by _title substring only (no full-text matches)', function (assert) { + let query = buildRecentsQuery('mango', SORT_AZ); + assert.deepEqual(query, { + filter: { + every: [ + { not: { type: specRef } }, + { contains: { _title: 'mango' } }, + DEDUP_FILTER, + ], + }, + sort: SORT_AZ.sort, + }); + }); + + test('cardsOnly adds no filter anchor to recents — the card scope is pinned on the wire', function (assert) { + let query = buildRecentsQuery(undefined, SORT_AZ, undefined, undefined, { + cardsOnly: true, + }); + assert.deepEqual(query, { + filter: { not: { type: specRef } }, + sort: SORT_AZ.sort, + }); + }); + }); + + module('searchScopeForOptions', function () { + test('cardsOnly maps to the "cards" wire scope', function (assert) { + assert.strictEqual(searchScopeForOptions({ cardsOnly: true }), 'cards'); + }); + + test('the mixed default leaves the scope unset (all)', function (assert) { + assert.strictEqual(searchScopeForOptions({}), undefined); + assert.strictEqual(searchScopeForOptions(undefined), undefined); + assert.strictEqual( + searchScopeForOptions({ cardsOnly: false }), + undefined, + ); + }); }); module('shouldSkipSearchQuery', function () { diff --git a/packages/host/tests/unit/index-query-engine-test.ts b/packages/host/tests/unit/index-query-engine-test.ts index 64734d9b4b..823d262493 100644 --- a/packages/host/tests/unit/index-query-engine-test.ts +++ b/packages/host/tests/unit/index-query-engine-test.ts @@ -398,6 +398,64 @@ module('Unit | query', function (hooks) { ); }); + test('not: { type } genuinely excludes rows whose types chain contains the ref', async function (assert) { + // mango is a FancyPerson, so its `types` chain contains Person. With the + // per-row membership predicate `not: { type: Person }` must exclude it (and + // vangogh/ringo); before the fix the shared cross-join alias let a + // FancyPerson row survive via its own non-Person element. + let { paper } = testCards; + await setupIndex(dbAdapter, [testCards.mango, testCards.vangogh, paper]); + + let personType = await personCardType(testCards); + let { cards: results, meta } = await indexQueryEngine.searchCards( + new URL(testRealmURL), + { filter: { not: { type: personType } } }, + ); + + assert.strictEqual(meta.page.total, 1, 'only the non-Person card remains'); + assert.deepEqual( + getIds(results), + [paper.id], + 'the Cat is kept; both Person-chain cards are excluded', + ); + }); + + test('every: [{ type: A }, { type: B }] is an intersection, satisfiable within one chain', async function (assert) { + // mango's chain contains both FancyPerson and Person, so the intersection + // matches it; ringo (plain Person) lacks FancyPerson. Before the fix this + // conjunction was unsatisfiable (no single array element equals both). + let { mango, ringo } = testCards; + await setupIndex(dbAdapter, [mango, ringo]); + + let personType = await personCardType(testCards); + let fancyType = internalKeyToCodeRef([...(await getTypes(mango))].shift()!); + let { cards: results, meta } = await indexQueryEngine.searchCards( + new URL(testRealmURL), + { filter: { every: [{ type: fancyType }, { type: personType }] } }, + ); + + assert.strictEqual(meta.page.total, 1, 'the intersection matches one card'); + assert.deepEqual(getIds(results), [mango.id], 'only the FancyPerson'); + }); + + test('{ type: baseRef } matches every card instance (BaseDef terminates the chain)', async function (assert) { + // getTypes now ends each chain at BaseDef, so a BaseDef-anchored filter is a + // universe selector across kinds. Here (instance-only fixtures) it should + // match all cards regardless of their concrete type. + let { mango, paper } = testCards; + await setupIndex(dbAdapter, [mango, paper]); + + // BaseDef is the last element of any card's types chain. + let baseType = internalKeyToCodeRef([...(await getTypes(mango))].pop()!); + let { cards: results, meta } = await indexQueryEngine.searchCards( + new URL(testRealmURL), + { filter: { type: baseType } }, + ); + + assert.strictEqual(meta.page.total, 2, 'both cards match BaseDef'); + assert.deepEqual(getIds(results), [mango.id, paper.id]); + }); + test(`can filter using 'eq'`, async function (assert) { let { mango, vangogh, paper } = testCards; await setupIndex(dbAdapter, [ diff --git a/packages/host/tests/unit/index-writer-test.ts b/packages/host/tests/unit/index-writer-test.ts index 939cedae1a..1a29443b4f 100644 --- a/packages/host/tests/unit/index-writer-test.ts +++ b/packages/host/tests/unit/index-writer-test.ts @@ -1642,7 +1642,7 @@ module('Unit | index-writer', function (hooks) { }); test('error entry keeps freshly-stamped synthetic search keys over the last-known-good doc', async function (assert) { - // Rollout case: a `file` row indexed before `_isCardInstance` / `_title` + // Rollout case: a `file` row indexed before `_isCardInstanceFile` / `_title` // existed, then hit by a dependency-error reindex. The fresh searchData // carries the synthetic keys and must survive rather than being clobbered // by the production doc, while last-known-good fields (here `extra`) still @@ -1671,7 +1671,11 @@ module('Unit | index-writer', function (hooks) { await batch.updateEntry(new URL(`${testRealmURL}1.json`), { type: 'file-error', error: { message: 'dep in error', status: 500, additionalErrors: [] }, - searchData: { name: '1.json', _isCardInstance: true, _title: '1.json' }, + searchData: { + name: '1.json', + _isCardInstanceFile: true, + _title: '1.json', + }, }); await batch.done(); @@ -1684,7 +1688,7 @@ module('Unit | index-writer', function (hooks) { { name: '1.json', extra: 'keep-me', - _isCardInstance: true, + _isCardInstanceFile: true, _title: '1.json', }, 'error row merges freshly-stamped synthetic keys onto the last-known-good doc', diff --git a/packages/host/tests/unit/instance-filter-matcher-test.ts b/packages/host/tests/unit/instance-filter-matcher-test.ts index c506d40c8d..b387a90139 100644 --- a/packages/host/tests/unit/instance-filter-matcher-test.ts +++ b/packages/host/tests/unit/instance-filter-matcher-test.ts @@ -650,4 +650,78 @@ module('Unit | instance-filter-matcher', function (hooks) { 'mango sorts before ringo by URL', ); }); + + // -- synthetic search-doc keys (server parity) ------------------------------ + // The engine lets filters/sorts address `_title`, `_cardType`, and + // `_isCardInstanceFile` even though they are stamped into the search doc, not + // real fields. Without a client shim the matcher resolves them to nothing and + // the reconciler blanks correct server results — these guard that parity. + + test('_title contains resolves to the card title', function (assert) { + let { mangoBirthday, vangoghBirthday } = cards; + assert.strictEqual( + match(mangoBirthday, { contains: { _title: 'Mango' } }), + 'match', + "mangoBirthday's title contains Mango", + ); + assert.strictEqual( + match(vangoghBirthday, { contains: { _title: 'Mango' } }), + 'no-match', + "vangoghBirthday's title does not contain Mango", + ); + }); + + test('_title comparator orders by the card title, not URL', function (assert) { + let { mangoBirthday, vangoghBirthday } = cards; + let sort: Sort = [{ on: eventRef, by: '_title', direction: 'asc' }]; + let sorted = [vangoghBirthday, mangoBirthday].sort( + makeInstanceComparator(sort, api), + ); + assert.deepEqual( + sorted.map((c) => (c as any).cardTitle), + ["Mango's Birthday", "Van Gogh's Birthday"], + 'A-Z by title (would be URL order without the _title shim)', + ); + }); + + test('_cardType is non-null for a card instance', function (assert) { + let { mango } = cards; + // cards-grid relies on `not: { eq: { _cardType: null } }` matching cards; + // the shim must make `_cardType` resolve to a non-null value. + assert.strictEqual( + match(mango, { eq: { _cardType: null } }), + 'no-match', + '_cardType is not null on a hydrated card', + ); + assert.strictEqual( + match(mango, { not: { eq: { _cardType: null } } }), + 'match', + ); + }); + + test('_isCardInstanceFile: false matches a hydrated card (the dedup filter keeps cards)', function (assert) { + let { mango } = cards; + assert.strictEqual( + match(mango, { eq: { _isCardInstanceFile: false } }), + 'match', + 'a hydrated card is not a card-instance file, so the dedup keeps it', + ); + assert.strictEqual( + match(mango, { eq: { _isCardInstanceFile: true } }), + 'no-match', + ); + }); + + test('isClientEvaluable keeps shimmed synthetic keys but rejects unshimmed underscore keys', function (assert) { + assert.true( + isClientEvaluable({ eq: { _isCardInstanceFile: false } }), + 'a shimmed synthetic key stays client-evaluable', + ); + assert.true(isClientEvaluable({ contains: { _title: 'x' } })); + assert.true(isClientEvaluable({ eq: { name: 'Mango' } }), 'a real field'); + assert.false( + isClientEvaluable({ eq: { _mysteryServerKey: 1 } }), + 'an unshimmed underscore key forces server-only evaluation', + ); + }); }); diff --git a/packages/realm-server/handlers/handle-search.ts b/packages/realm-server/handlers/handle-search.ts index b958772c72..096783e8f0 100644 --- a/packages/realm-server/handlers/handle-search.ts +++ b/packages/realm-server/handlers/handle-search.ts @@ -127,6 +127,14 @@ export default function handleSearch(opts: { if (parsed.cardUrls?.length) { cacheKeyOpts.cardUrls = parsed.cardUrls; } + // `scope` changes which row kinds the response contains, so it must key the + // cache — otherwise a `scope: 'cards'` and a `scope: 'all'` request for the + // same query would collide on one ETag/body. Fold only when set (absent == + // the default 'all', so keying `undefined` would not fragment but omitting + // it keeps the key identical to pre-scope requests). + if (parsed.scope) { + cacheKeyOpts.scope = parsed.scope; + } // `loggingCorrelationId` / `timings` deliberately stay OUT of the // cache-key opts (per-request values would make every key unique) and diff --git a/packages/realm-server/tests/eq-containment-integration-test.ts b/packages/realm-server/tests/eq-containment-integration-test.ts index ce0a1f2018..7a17500920 100644 --- a/packages/realm-server/tests/eq-containment-integration-test.ts +++ b/packages/realm-server/tests/eq-containment-integration-test.ts @@ -331,6 +331,14 @@ module(basename(import.meta.filename), function () { return executedSql.filter((sql) => !/^\s*INSERT\b/i.test(sql)).join('\n'); } + // The `on:` type anchor compiles to its own `COALESCE(i.types, …) @> …` + // containment predicate, so a bare `@>` search no longer isolates the field + // eq. "Did the field eq use containment" is specifically containment on the + // `search_doc` column. + function usesFieldContainment(): boolean { + return /search_doc\s*@>/.test(lastFilterSql()); + } + test('singular string eq is served by `@>` containment', async function (assert) { let { cards, meta } = await engine.searchCards(new URL(testRealmURL), { filter: { on: policyRef, eq: { policyId: 'P1' } }, @@ -338,7 +346,7 @@ module(basename(import.meta.filename), function () { assert.strictEqual(meta.page.total, 1, 'one row matches'); assert.deepEqual(ids(cards), [`${testRealmURL}Policy/p1`]); assert.ok( - lastFilterSql().includes('@>'), + usesFieldContainment(), 'a `@>` containment predicate was emitted', ); }); @@ -353,7 +361,7 @@ module(basename(import.meta.filename), function () { `${testRealmURL}Policy/p3`, ]); assert.ok( - lastFilterSql().includes('@>'), + usesFieldContainment(), 'the linksTo `.id` path uses `@>` from the root', ); }); @@ -367,7 +375,7 @@ module(basename(import.meta.filename), function () { `${testRealmURL}Policy/p1`, `${testRealmURL}Policy/p3`, ]); - assert.ok(lastFilterSql().includes('@>'), 'nested object path uses `@>`'); + assert.ok(usesFieldContainment(), 'nested object path uses `@>`'); }); test('numeric eq keeps the `->>` extraction form', async function (assert) { @@ -381,7 +389,7 @@ module(basename(import.meta.filename), function () { `${testRealmURL}Policy/p4`, ]); assert.notOk( - lastFilterSql().includes('@>'), + usesFieldContainment(), 'numeric leaf is excluded from containment', ); }); @@ -392,7 +400,7 @@ module(basename(import.meta.filename), function () { }); assert.strictEqual(meta.page.total, 3); assert.notOk( - lastFilterSql().includes('@>'), + usesFieldContainment(), 'boolean leaf is excluded from containment', ); assert.deepEqual(ids(cards), [ @@ -412,7 +420,7 @@ module(basename(import.meta.filename), function () { `${testRealmURL}Policy/p3`, ]); assert.notOk( - lastFilterSql().includes('@>'), + usesFieldContainment(), 'plural path stays on json_tree, never `@>`', ); }); @@ -434,7 +442,7 @@ module(basename(import.meta.filename), function () { `${testRealmURL}Policy/p3`, ]); assert.notOk( - lastFilterSql().includes('@>'), + usesFieldContainment(), 'an interior plural segment forces json_tree, never `@>`', ); assert.ok( @@ -456,7 +464,7 @@ module(basename(import.meta.filename), function () { ]); assert.strictEqual(meta.page.total, 2); assert.notOk( - lastFilterSql().includes('@>'), + usesFieldContainment(), 'negated eq must not use `@>` (FALSE vs NULL diverges under NOT)', ); }); @@ -468,7 +476,7 @@ module(basename(import.meta.filename), function () { assert.strictEqual(meta.page.total, 1); assert.deepEqual(ids(cards), [`${testRealmURL}Policy/p1`]); assert.ok( - lastFilterSql().includes('@>'), + usesFieldContainment(), 'two NOTs cancel: positive polarity uses `@>`', ); }); diff --git a/packages/realm-server/tests/indexing-test.ts b/packages/realm-server/tests/indexing-test.ts index b415487929..c70eb3fdc5 100644 --- a/packages/realm-server/tests/indexing-test.ts +++ b/packages/realm-server/tests/indexing-test.ts @@ -1114,8 +1114,8 @@ module(basename(import.meta.filename), function () { 'file entry includes contentSize', ); assert.true( - entry?.searchDoc?._isCardInstance, - 'file entry for a card instance json is marked _isCardInstance', + entry?.searchDoc?._isCardInstanceFile, + 'file entry for a card instance json is marked _isCardInstanceFile', ); assert.strictEqual( entry?.searchDoc?._title, @@ -1175,8 +1175,8 @@ module(basename(import.meta.filename), function () { 'search_doc _title is the file name', ); assert.false( - '_isCardInstance' in searchDoc, - 'non-card file search_doc does not carry _isCardInstance', + '_isCardInstanceFile' in searchDoc, + 'non-card file search_doc does not carry _isCardInstanceFile', ); }); diff --git a/packages/realm-server/tests/prerendering-test.ts b/packages/realm-server/tests/prerendering-test.ts index 1cefff8073..721ea06b7b 100644 --- a/packages/realm-server/tests/prerendering-test.ts +++ b/packages/realm-server/tests/prerendering-test.ts @@ -4384,6 +4384,7 @@ module(basename(import.meta.filename), function () { assert.deepEqual(result.types, [ `${realmURL2}cat/Cat`, '@cardstack/base/card-api/CardDef', + '@cardstack/base/card-api/BaseDef', ]); }); diff --git a/packages/realm-server/tests/search-entries-engine-test.ts b/packages/realm-server/tests/search-entries-engine-test.ts index 18571842d2..15344e9d53 100644 --- a/packages/realm-server/tests/search-entries-engine-test.ts +++ b/packages/realm-server/tests/search-entries-engine-test.ts @@ -369,11 +369,18 @@ module(basename(import.meta.filename), function () { test('html.format: head scoped to a single card URL returns only that card head markup', async function (assert) { // Mirrors the host-mode published-view head prefetch: an html-only query - // at html.format: head, scoped to the single card by cardUrls. + // at html.format: head, scoped to the single card by cardUrls. `scope: + // 'cards'` pins the instance row, dropping the card `.json`'s dual-indexed + // file row that shares the `cardUrls` URL. let doc = await testRealm.realmIndexQueryEngine.searchEntries( parseSearchEntryQueryFromPayload({ cardUrls: [`${johnId}.json`], - filter: { eq: { htmlQuery: { eq: { format: 'head' } } } }, + scope: 'cards', + filter: { + eq: { + htmlQuery: { eq: { format: 'head' } }, + }, + }, fields: { entry: ['html'] }, }), ); @@ -647,6 +654,208 @@ module(basename(import.meta.filename), function () { ); } }); + + test('mixed default: an anchorless entry query returns both card instances and files', async function (assert) { + let doc = await testRealm.realmIndexQueryEngine.searchEntries( + parseSearchEntryQueryFromPayload({ fields: { entry: ['item'] } }), + ); + + assert.ok( + entryFor(doc, johnId), + 'the card instance surfaces as an entry', + ); + assert.strictEqual( + itemIn(doc, johnId)?.type, + 'card', + 'a card instance carries a card item', + ); + + let helloUrl = `${realmHref}hello.md`; + assert.ok(entryFor(doc, helloUrl), 'a plain file surfaces as an entry'); + assert.strictEqual( + itemIn(doc, helloUrl)?.type, + 'file-meta', + 'a plain file carries a file-meta item', + ); + + // By default nothing is deduped: a card `.json` surfaces both as its + // instance entry (`.../john`) and its dual-indexed file entry + // (`.../john.json`), kept distinct by the `(url, type)` grouping. + assert.ok( + entryFor(doc, `${johnId}.json`), + 'the card-instance `.json` file row also surfaces by default', + ); + assert.strictEqual( + itemIn(doc, `${johnId}.json`)?.type, + 'file-meta', + 'the card-instance `.json` row carries a file-meta item', + ); + }); + + test('eq item._isCardInstanceFile false dedups a dual-indexed card .json file row', async function (assert) { + let doc = await testRealm.realmIndexQueryEngine.searchEntries( + parseSearchEntryQueryFromPayload({ + // `_isCardInstanceFile` is stamped only on a card `.json`'s file row, + // so `eq: false` (absent-as-false) keeps cards + plain files and drops + // that row — the explicit mixed-scope dedup. + filter: { eq: { 'item._isCardInstanceFile': false } }, + fields: { entry: ['item'] }, + }), + ); + + assert.ok(entryFor(doc, johnId), 'the card instance is kept'); + assert.ok( + entryFor(doc, `${realmHref}hello.md`), + 'a plain file is kept (no `_isCardInstanceFile` key)', + ); + assert.notOk( + entryFor(doc, `${johnId}.json`), + 'the card-instance `.json` file row is dropped', + ); + }); + + test("scope: 'all' returns both rows of a dual-indexed card .json", async function (assert) { + let doc = await testRealm.realmIndexQueryEngine.searchEntries( + parseSearchEntryQueryFromPayload({ + scope: 'all', + fields: { entry: ['item'] }, + }), + ); + assert.ok(entryFor(doc, johnId), 'the card instance row is present'); + assert.ok( + entryFor(doc, `${johnId}.json`), + 'the dual-indexed .json file row is ALSO present (dedup is explicit)', + ); + assert.ok(entryFor(doc, `${realmHref}hello.md`), 'a plain file too'); + }); + + test("scope: 'cards' pins card-instance rows (no file rows at all)", async function (assert) { + let doc = await testRealm.realmIndexQueryEngine.searchEntries( + parseSearchEntryQueryFromPayload({ + scope: 'cards', + fields: { entry: ['item'] }, + }), + ); + assert.ok(entryFor(doc, johnId), 'the card instance is kept'); + assert.notOk( + entryFor(doc, `${johnId}.json`), + 'the card .json file row is excluded by the card scope', + ); + assert.notOk( + entryFor(doc, `${realmHref}hello.md`), + 'a plain file is excluded by the card scope', + ); + }); + + test("scope: 'files' pins file rows (no card-instance rows)", async function (assert) { + let doc = await testRealm.realmIndexQueryEngine.searchEntries( + parseSearchEntryQueryFromPayload({ + scope: 'files', + fields: { entry: ['item'] }, + }), + ); + assert.ok( + entryFor(doc, `${realmHref}hello.md`), + 'a plain file row is present', + ); + assert.ok( + entryFor(doc, `${johnId}.json`), + 'a card .json file row is present', + ); + assert.notOk( + entryFor(doc, johnId), + 'the card-instance row is excluded by the file scope', + ); + }); + + test('eq item._isCardInstanceFile true selects only the dual-indexed card .json file rows', async function (assert) { + let doc = await testRealm.realmIndexQueryEngine.searchEntries( + parseSearchEntryQueryFromPayload({ + filter: { eq: { 'item._isCardInstanceFile': true } }, + fields: { entry: ['item'] }, + }), + ); + assert.ok( + entryFor(doc, `${johnId}.json`), + 'the card .json file row is selected', + ); + assert.notOk(entryFor(doc, johnId), 'the card-instance row is not'); + assert.notOk( + entryFor(doc, `${realmHref}hello.md`), + 'a plain file (no key) is not', + ); + }); + + test('a positive card-type anchor keeps a mixed-default query cards-only', async function (assert) { + // The entry wire grammar expresses a card-type anchor as `item.on`. + let doc = await testRealm.realmIndexQueryEngine.searchEntries( + parseSearchEntryQueryFromPayload({ + filter: { + 'item.on': { module: `${realmHref}person`, name: 'Person' }, + }, + fields: { entry: ['item'] }, + }), + ); + + assert.strictEqual( + doc.meta.page.total, + 2, + 'only the two Person instances match', + ); + assert.ok(entryFor(doc, johnId)); + assert.ok(entryFor(doc, janeId)); + let fileItems = (doc.included ?? []).filter( + (resource) => resource.type === 'file-meta', + ); + assert.strictEqual(fileItems.length, 0, 'no file rows leak in'); + }); + + test('A-Z `_title` sort gives file rows real sort values (not a NULL that sinks them)', async function (assert) { + // Regression guard for the mixed-search A-Z sort. A file row carries the + // synthetic `_title` but not `cardTitle`, so sorting on `cardTitle` + // leaves every file's sort value NULL — under NULLS LAST they collapse to + // the `url` tiebreaker (always ascending), so their order is invariant to + // `direction`. Sorting on `_title` gives files distinct, real sort values, + // so their relative order reverses between asc and desc. The sort anchors + // on a card type only to resolve the synthetic key; it does not filter, + // so the mixed set (here undeduped, guaranteeing several file rows) keeps + // its files. + let titleSort = (direction: 'asc' | 'desc') => + parseSearchEntryQueryFromPayload({ + sort: [ + { + by: 'item._title', + 'item.on': { module: `${realmHref}person`, name: 'Person' }, + direction, + }, + ], + page: { size: 100 }, + fields: { entry: ['item'] }, + }); + let fileOrder = (doc: EntryCollectionDocument) => + doc.data + .map((entry) => entry.id) + .filter((id) => itemIn(doc, id)?.type === 'file-meta'); + + let ascDoc = await testRealm.realmIndexQueryEngine.searchEntries( + titleSort('asc'), + ); + let descDoc = await testRealm.realmIndexQueryEngine.searchEntries( + titleSort('desc'), + ); + let asc = fileOrder(ascDoc); + let desc = fileOrder(descDoc); + + assert.true( + asc.length >= 2, + 'the mixed set carries at least two file rows to order', + ); + assert.deepEqual( + desc, + [...asc].reverse(), + 'file rows reverse with sort direction — they carry a real `_title` sort value, not a NULL pinned to the url tiebreaker', + ); + }); }); module('searchEntries css + render-type', function (hooks) { diff --git a/packages/realm-server/tests/searchable-parity-diff-test.ts b/packages/realm-server/tests/searchable-parity-diff-test.ts index 064f603374..f59517a04b 100644 --- a/packages/realm-server/tests/searchable-parity-diff-test.ts +++ b/packages/realm-server/tests/searchable-parity-diff-test.ts @@ -5,7 +5,7 @@ import { diffDoc, isShallowLink, shallowIds } from '@cardstack/runtime-common'; // Unit coverage for the parity comparison logic shared by the realm-scale // validator (`scripts/searchable-parity-diff.ts`) and the generation tests. -// The differ ignores synthetic keys (`_cardType`, `_title`, `_isCardInstance`), +// The differ ignores synthetic keys (`_cardType`, `_title`, `_isCardInstanceFile`), // normalizes object key order, and (under // --ignore-shallow-links) treats the store-driven omit-vs-keep-`{id}` difference // as equivalent — at any nesting depth — while still catching a CHANGED @@ -53,11 +53,11 @@ module('Unit | searchable-parity-diff', function () { ); }); - test('synthetic _title / _isCardInstance keys are ignored', function (assert) { + test('synthetic _title / _isCardInstanceFile keys are ignored', function (assert) { assert.deepEqual( diffDoc( { title: 'A' }, - { title: 'A', _title: 'A', _isCardInstance: true }, + { title: 'A', _title: 'A', _isCardInstanceFile: true }, false, ), [], diff --git a/packages/runtime-common/expression.ts b/packages/runtime-common/expression.ts index c2f0a56d56..901e8fd97b 100644 --- a/packages/runtime-common/expression.ts +++ b/packages/runtime-common/expression.ts @@ -11,6 +11,7 @@ export type Expression = ( | TableValuedEach | TableValuedTree | JsonContains + | TypesContains | DBSpecificExpression )[]; @@ -94,6 +95,19 @@ export interface JsonContains { value: Param; } +// Self-contained membership test: does the JSON array in `column` contain +// `key`? Rendered per adapter (Postgres `@>`, SQLite `json_each` EXISTS). +// Unlike a `jsonb_array_elements_text` cross join — which fans a row out into +// one row per array element and so gives a type condition exists-one-element +// semantics that miscompose under AND/NOT — this is a single per-row scalar +// predicate, so type conditions compose correctly (a real `NOT` exclusion, an +// AND intersection) and no GROUP BY is needed to recollapse the fan-out. +export interface TypesContains { + kind: 'types-contains'; + column: string; + key: string; +} + export interface FieldArity { type: CodeRef; path: string; @@ -111,6 +125,7 @@ export type CardExpression = ( | TableValuedEach | TableValuedTree | JsonContains + | TypesContains | JsonContainsQuery | FieldQuery | FieldValue @@ -221,6 +236,14 @@ export function jsonContainsQuery( }; } +export function typesContains(key: string, column = 'i.types'): TypesContains { + return { + kind: 'types-contains', + column, + key, + }; +} + export function fieldQuery( path: string, type: CodeRef, @@ -541,6 +564,26 @@ export function expressionToSql( return [column, '@>', param(nested as JSONTypes.Object), '::jsonb'] .map(renderElement) .join(' '); + } else if (element.kind === 'types-contains') { + // Per-row array membership. COALESCE keeps a NULL/absent `types` array a + // definite FALSE (not SQL NULL) at positive polarity, so an enclosing + // `NOT (...)` keeps rows whose types never indexed rather than dropping + // them — the fan-out approach eliminated those rows before WHERE ran. + let { column, key } = element; + if (dbAdapterKind === 'sqlite') { + return [ + 'EXISTS (SELECT 1 FROM json_each(COALESCE(', + column, + `, '[]')) WHERE value =`, + param(key), + ')', + ] + .map(renderElement) + .join(' '); + } + return ['COALESCE(', column, `, '[]'::jsonb) @>`, param([key]), '::jsonb'] + .map(renderElement) + .join(' '); } else { throw assertNever(element); } diff --git a/packages/runtime-common/helpers/card-type-display-name.ts b/packages/runtime-common/helpers/card-type-display-name.ts index 11c118a372..1e52bf010f 100644 --- a/packages/runtime-common/helpers/card-type-display-name.ts +++ b/packages/runtime-common/helpers/card-type-display-name.ts @@ -6,6 +6,21 @@ import type { import { getField } from '../code-ref.ts'; +// The card type's friendly display name as stamped into a row's `search_doc` +// `_cardType` key by the prerender meta route (routes/render/meta.ts): a class +// whose displayName is exactly 'Card' (i.e. `CardDef` itself) falls back to its +// class name. Kept here as the single source of truth so the index stamp and +// the client-side matcher shim (instance-filter-matcher.ts) agree byte-for-byte. +// NOTE: this is deliberately NOT `cardTypeDisplayName` below — that one calls +// `getDisplayName` and lacks the 'Card' → class-name fallback, so it would +// diverge from the stamped value. +export function friendlyCardType(klass: { + displayName: string; + name: string; +}): string { + return klass.displayName === 'Card' ? klass.name : klass.displayName; +} + export function cardTypeDisplayName(cardOrField: BaseDef): string { // A not-yet-loaded or broken relationship link can surface an undefined // model to a card's own template (the linksTo component only renders the diff --git a/packages/runtime-common/index-query-engine.ts b/packages/runtime-common/index-query-engine.ts index 46b604ffca..875ccc3dd8 100644 --- a/packages/runtime-common/index-query-engine.ts +++ b/packages/runtime-common/index-query-engine.ts @@ -20,8 +20,8 @@ import { type JsonContainsQuery, param, isParam, - tableValuedEach, tableValuedTree, + typesContains, separatedByCommas, addExplicitParens, any, @@ -64,6 +64,10 @@ import { type FieldDefinition, } from './definitions.ts'; import { matchSearchableRoutes, routesForField } from './searchable-routes.ts'; +import { + CARD_INSTANCE_FILE_KEY, + isSyntheticSearchDocKey, +} from './search-doc-keys.ts'; import { isFilterRefersToNonexistentTypeError, type DefinitionLookup, @@ -610,18 +614,12 @@ export class IndexQueryEngine { let typeKeys = internalKeysFor(ref, undefined, this.#virtualNetwork); let rows = (await this.#query([ 'SELECT 1', - `FROM ${tableFromOpts(opts)} AS i ${tableValuedFunctionsPlaceholder}`, + `FROM ${tableFromOpts(opts)} AS i`, 'WHERE', ...every([ ['i.realm_url =', param(realmURL.href)], ['i.type =', param('file')], - any( - typeKeys.map((typeKey) => [ - tableValuedEach('types'), - '=', - param(typeKey), - ]), - ), + any(typeKeys.map((typeKey) => [typesContains(typeKey)])), ]), 'LIMIT 1', ] as Expression)) as unknown as { 1: number }[]; @@ -639,18 +637,12 @@ export class IndexQueryEngine { let typeKeys = internalKeysFor(ref, undefined, this.#virtualNetwork); let rows = (await this.#query([ 'SELECT 1', - `FROM ${tableFromOpts(opts)} AS i ${tableValuedFunctionsPlaceholder}`, + `FROM ${tableFromOpts(opts)} AS i`, 'WHERE', ...every([ ['i.realm_url =', param(realmURL.href)], ['i.type =', param('instance')], - any( - typeKeys.map((typeKey) => [ - tableValuedEach('types'), - '=', - param(typeKey), - ]), - ), + any(typeKeys.map((typeKey) => [typesContains(typeKey)])), ]), 'LIMIT 1', ] as Expression)) as unknown as { 1: number }[]; @@ -675,7 +667,7 @@ export class IndexQueryEngine { { filter, sort, page }: Query, opts: QueryOptions, selectClauseExpression: CardExpression, - entryType: 'instance' | 'file' = 'instance', + entryType: 'instance' | 'file' | 'all' = 'instance', // When set, the grouped projection is wrapped in an outer select so a // conditional live `pristine_doc` can reference the computed `html` column // once (see `search()`'s render branch). The inner projection must alias @@ -692,17 +684,34 @@ export class IndexQueryEngine { ['i.is_deleted = FALSE OR i.is_deleted IS NULL'], ]; - if (opts.includeErrors) { + // "not errored" spans both error channels (index + current render error), + // matching the single-kind branches below — so a file row with a current + // render error is excluded from the mixed scope too. + let notErrored: CardExpression = [`NOT ${effectiveHasError()}`]; + if (entryType === 'all') { + // Mixed scope. Instance rows follow `includeErrors` (their error + // markup is rendered through the html branch); file rows are only + // ever surfaced in a healthy state, so they always require + // `has_error` false regardless of `includeErrors`. + let instanceBranch: CardExpression = opts.includeErrors + ? ['i.type =', param('instance')] + : every([['i.type =', param('instance')], notErrored]); + let fileBranch = every([['i.type =', param('file')], notErrored]); + conditions.push(any([instanceBranch, fileBranch])); + } else if (opts.includeErrors) { conditions.push(['i.type =', param(entryType)]); } else { - conditions.push( - every([ - ['i.type =', param(entryType)], - [`NOT ${effectiveHasError()}`], - ]), - ); + conditions.push(every([['i.type =', param(entryType)], notErrored])); } + // A card-instance `.json` is dual-indexed: an `instance` row and a + // `file` row sharing the same `i.url`, distinguished only by `i.type`. + // `GROUP BY i.url, i.type` (below) keeps them as two distinct rows so a + // mixed query surfaces both; a consumer that wants to dedup the `file` + // row filters on the `_isCardInstanceFile` search-doc key (stamped only on + // that row) — e.g. `eq: { _isCardInstanceFile: false }` keeps cards + plain + // files and drops the card `.json`. + if (opts.cardUrls && opts.cardUrls.length > 0) { // Restrict to this URL subset. Instance rows key on their `.json` file // URL; file rows on their canonical file URL — the same `i.url` column, @@ -744,7 +753,7 @@ export class IndexQueryEngine { )} ${tableValuedFunctionsPlaceholder}`, 'WHERE', ...everyCondition, - 'GROUP BY i.url', + 'GROUP BY i.url, i.type', ') AS sub', ...outerOrderBy, ...limitClause, @@ -757,7 +766,7 @@ export class IndexQueryEngine { )} ${tableValuedFunctionsPlaceholder}`, 'WHERE', ...everyCondition, - 'GROUP BY i.url', + 'GROUP BY i.url, i.type', ...this.orderExpression(sort), ...limitClause, ]; @@ -765,14 +774,20 @@ export class IndexQueryEngine { // Count over the same tables as the data query (via `opts`) so `total` // stays consistent with the result set — including in WIP mode and for a // `matches` predicate, which is shared with the data query through - // `everyCondition` and references `ph.markdown`. + // `everyCondition` and references `ph.markdown`. Count distinct + // `(url, type)` pairs to match the `GROUP BY i.url, i.type` grouping — a + // dual-indexed card `.json` contributes both its `instance` and `file` + // row. Expressed as a `COUNT(*)` over a `DISTINCT` subquery because + // SQLite's `COUNT(DISTINCT …)` takes only a single expression. let queryCount = [ - 'SELECT COUNT(DISTINCT i.url) AS total', + 'SELECT COUNT(*) AS total FROM (', + 'SELECT DISTINCT i.url, i.type', `FROM ${tableFromOpts(opts)} AS i ${prerenderedJoin( opts, )} ${tableValuedFunctionsPlaceholder}`, 'WHERE', ...everyCondition, + ') AS distinct_entries', ]; let [results, totalResults] = await Promise.all([ @@ -808,6 +823,13 @@ export class IndexQueryEngine { { filter, sort, page }: Query, opts: QueryOptions, projection: SearchProjection, + // 'all' searches both `instance` and `file` rows in one query (kind is + // discriminated by the caller's filter, with the card-instance `.json` + // exclusion applied); 'file' pins file rows only; the file columns needed + // to synthesize a file row's resource + renderings ride the projection when + // file rows are in scope. Defaults to instance-only so `searchCards` and + // other single-kind callers are unchanged. + entryType: 'instance' | 'file' | 'all' = 'instance', ): Promise<{ meta: QueryResultsMeta; results: (Partial & { @@ -819,31 +841,65 @@ export class IndexQueryEngine { html_generation?: number | null; })[]; }> { + // File-only columns the assembly loop reads to build a `file` row's + // resource + native renderings. Each is gated on `i.type = 'file'` so a + // mixed query never drags a card row's (potentially multi-megabyte) + // `search_doc` / `isolated_html` / `markdown`: within a `(url, type)` + // group the type is constant, so the CASE yields the value for a file + // group and NULL for an instance group (which the assembly loop ignores + // — it only calls `fileEntryFromResult` on `file` rows). Appended to + // either projection only when file rows are in scope. + let fileOnly = (col: string) => + `ANY_VALUE(CASE WHEN i.type = 'file' THEN ${col} END)`; + // Only the columns the file-row assembly actually reads + // (`fileResourceFromIndex` + `fileEntryFromResult`): the row's search_doc, + // timestamps, realm, and index generation. `markdown` and `isolated_html` + // are deliberately NOT here — neither the file-meta resource nor its + // renderings read them, and `markdown` can be multi-megabyte. The standalone + // `searchFiles` projection (below) still carries them for its own consumers. + let fileColumns = `, ${fileOnly('i.search_doc')} as search_doc, ${fileOnly( + 'i.last_modified', + )} as last_modified, ${fileOnly( + 'i.resource_created_at', + )} as resource_created_at, ${fileOnly('i.realm_url')} as realm_url, ${fileOnly( + 'i.indexed_at', + )} as indexed_at`; let selectClauseExpression: CardExpression; if (projection.kind === 'dataOnly') { - selectClauseExpression = [ - `SELECT i.url AS url, ANY_VALUE(i.type) as type, ANY_VALUE(${effectiveHasError()}) as has_error, ANY_VALUE(i.pristine_doc) as pristine_doc, ANY_VALUE(${effectiveErrorDoc()}) as error_doc, ANY_VALUE(i.generation) as generation`, - ]; + // The live serialization only — no rendered HTML entries (dataOnly). + // When file rows are in scope, also carry the columns the file-row + // assembly reads to synthesize each file's `file-meta` resource + type + // descriptor. The effective error columns fold in current render errors + // from the prerendered_html channel. + let dataOnly = `SELECT i.url AS url, ANY_VALUE(i.type) as type, ANY_VALUE(${effectiveHasError()}) as has_error, ANY_VALUE(i.pristine_doc) as pristine_doc, ANY_VALUE(${effectiveErrorDoc()}) as error_doc, ANY_VALUE(i.generation) as generation`; + selectClauseExpression = + entryType !== 'instance' + ? [ + `${dataOnly}, ANY_VALUE(i.types) as types, ANY_VALUE(i.display_names) as display_names, ANY_VALUE(${dualReadColumn( + 'deps', + )}) as deps, ANY_VALUE(i.icon_html) as icon_html${fileColumns}`, + ] + : [dataOnly]; } else { // The full rendering set: every per-format HTML column whole (the // fitted/embedded JSONB maps keyed by render type, the scalar // atom/head columns), plus the live serialization on every row. The // caller enumerates candidate renderings and selects from the set. - selectClauseExpression = [ - `SELECT i.url AS url, ANY_VALUE(i.type) as type, ANY_VALUE(${effectiveHasError()}) as has_error, ANY_VALUE(i.file_alias) as file_alias, ANY_VALUE(${dualReadColumn( - 'fitted_html', - )}) as fitted_html, ANY_VALUE(${dualReadColumn( - 'embedded_html', - )}) as embedded_html, ANY_VALUE(${dualReadColumn( - 'atom_html', - )}) as atom_html, ANY_VALUE(${dualReadColumn( - 'head_html', - )}) as head_html, ANY_VALUE(i.types) as types, ANY_VALUE(${dualReadColumn( - 'deps', - )}) as deps, ANY_VALUE(i.display_names) as display_names, ANY_VALUE(i.icon_html) as icon_html, ANY_VALUE(${effectiveErrorDoc()}) as error_doc, ANY_VALUE(i.pristine_doc) as pristine_doc, ANY_VALUE(i.generation) as generation, ANY_VALUE(${dualReadColumn( - 'generation', - )}) as html_generation`, - ]; + let renderSet = `SELECT i.url AS url, ANY_VALUE(i.type) as type, ANY_VALUE(${effectiveHasError()}) as has_error, ANY_VALUE(i.file_alias) as file_alias, ANY_VALUE(${dualReadColumn( + 'fitted_html', + )}) as fitted_html, ANY_VALUE(${dualReadColumn( + 'embedded_html', + )}) as embedded_html, ANY_VALUE(${dualReadColumn( + 'atom_html', + )}) as atom_html, ANY_VALUE(${dualReadColumn( + 'head_html', + )}) as head_html, ANY_VALUE(i.types) as types, ANY_VALUE(${dualReadColumn( + 'deps', + )}) as deps, ANY_VALUE(i.display_names) as display_names, ANY_VALUE(i.icon_html) as icon_html, ANY_VALUE(${effectiveErrorDoc()}) as error_doc, ANY_VALUE(i.pristine_doc) as pristine_doc, ANY_VALUE(i.generation) as generation, ANY_VALUE(${dualReadColumn( + 'generation', + )}) as html_generation`; + selectClauseExpression = + entryType !== 'instance' ? [`${renderSet}${fileColumns}`] : [renderSet]; } return (await this._search( @@ -851,7 +907,7 @@ export class IndexQueryEngine { { filter, sort, page }, opts, selectClauseExpression, - 'instance', + entryType, )) as { meta: QueryResultsMeta; results: (Partial & { @@ -913,56 +969,10 @@ export class IndexQueryEngine { 'file', ); - let files = results.map((result) => this.fileEntryFromResult(result)); + let files = results.map((result) => fileEntryFromResult(result)); return { files, meta }; } - private fileEntryFromResult(result: Partial): IndexedFile { - let canonicalURL = result.url; - if (!canonicalURL) { - throw new Error('expected file search result to include url'); - } - let lastModified = - typeof result.last_modified === 'string' - ? parseInt(result.last_modified) - : (result.last_modified ?? null); - let resourceCreatedAt = - typeof result.resource_created_at === 'string' - ? parseInt(result.resource_created_at) - : (result.resource_created_at ?? null); - let indexedAt = - typeof result.indexed_at === 'string' - ? parseInt(result.indexed_at) - : (result.indexed_at ?? null); - return { - type: 'file', - canonicalURL, - searchDoc: (result.search_doc as Record | null) ?? null, - resource: (result.pristine_doc as FileMetaResource | null) ?? null, - types: (result.types as string[] | null) ?? null, - displayNames: (result.display_names as string[] | null) ?? null, - deps: (result.deps as string[] | null) ?? null, - isolatedHtml: result.isolated_html ?? null, - headHtml: result.head_html ?? null, - embeddedHtml: - (result.embedded_html as { [refURL: string]: string } | null) ?? null, - fittedHtml: - (result.fitted_html as { [refURL: string]: string } | null) ?? null, - atomHtml: result.atom_html ?? null, - iconHtml: result.icon_html ?? null, - markdown: result.markdown ?? null, - lastModified, - resourceCreatedAt, - generation: result.generation ?? 0, - htmlGeneration: - (result as { html_generation?: number | null }).html_generation ?? - result.generation ?? - 0, - realmURL: result.realm_url ?? '', - indexedAt, - }; - } - private generalFieldSortColumn(field: string) { let mappedField = generalSortFields[field]; if (mappedField) { @@ -974,7 +984,9 @@ export class IndexQueryEngine { private orderExpression(sort: Sort | undefined): CardExpression { if (!sort) { - return ['ORDER BY i.url COLLATE "POSIX"']; + // `i.type` follows `url` so a dual-indexed card `.json`'s two rows + // (same url, `instance` vs `file`) order deterministically. + return ['ORDER BY i.url COLLATE "POSIX", i.type']; } return [ 'ORDER BY', @@ -990,8 +1002,10 @@ export class IndexQueryEngine { s.direction ?? 'asc', 'NULLS LAST', ]), - // we include 'url' as the final sort key for deterministic results + // `url` then `type` are the final sort keys for deterministic results + // (the two rows of a dual-indexed card `.json` share a url). ['i.url COLLATE "POSIX"'], + ['i.type'], ]), ]; } @@ -1006,9 +1020,12 @@ export class IndexQueryEngine { outerOrderBy: CardExpression; } { if (!sort) { + // `type` follows `url` to match `orderExpression`'s tiebreaker — the two + // rows of a dual-indexed card `.json` share a url. The inner projection + // aliases `ANY_VALUE(i.type) as type`, so the outer references it bare. return { innerSortColumns: [], - outerOrderBy: ['ORDER BY url COLLATE "POSIX"'], + outerOrderBy: ['ORDER BY url COLLATE "POSIX", type'], }; } let innerSortColumns: CardExpression = []; @@ -1024,8 +1041,11 @@ export class IndexQueryEngine { ); outerKeys.push([alias, s.direction ?? 'asc', 'NULLS LAST']); }); - // the `url` tiebreaker matches `orderExpression` for deterministic results + // `url` then `type` are the final tiebreakers, matching `orderExpression` + // for deterministic results (a dual-indexed card `.json`'s two rows share + // a url; the inner projection aliases `ANY_VALUE(i.type) as type`). outerKeys.push(['url COLLATE "POSIX"']); + outerKeys.push(['type']); return { innerSortColumns, outerOrderBy: ['ORDER BY', ...separatedByCommas(outerKeys)], @@ -1107,11 +1127,17 @@ export class IndexQueryEngine { // Match any equivalent spelling of the type key (RRI / real-URL / // virtual-alias), so rows indexed before references were canonicalized to // RRI still satisfy the filter without a reindex or DB migration. + // + // Each key is a self-contained `types-contains` membership predicate rather + // than a comparison against a shared `jsonb_array_elements_text(types)` + // cross-join alias. The cross join gave every type condition in a query + // exists-one-element semantics (all type conditions dedupe onto one alias), + // which miscompose: `not: { type: X }` failed to exclude X, and + // `every: [{ type: A }, { type: B }]` was unsatisfiable. Per-row membership + // makes negation a true exclusion and conjunction a true intersection. return any( internalKeysFor(ref, undefined, this.#virtualNetwork).map((typeKey) => [ - tableValuedEach('types'), - '=', - param(typeKey), + typesContains(typeKey), ]), ); } @@ -1308,6 +1334,31 @@ export class IndexQueryEngine { onRef: CodeRef, polarity: FilterPolarity = 'positive', ): CardExpression { + if ( + currentField(key) === CARD_INSTANCE_FILE_KEY && + typeof value === 'boolean' + ) { + // Synthetic boolean search-doc key, stamped `true` only on the file row of + // a dual-indexed card `.json` and omitted (never stamped `false`) + // everywhere else. Because the key is present exactly when it is true, + // membership is an existence test: `eq: false` ("not a card-instance + // file", which must also match rows lacking the key) is `->> IS NULL`, and + // `eq: true` is `->> IS NOT NULL`. This deliberately avoids comparing the + // extracted value against a boolean literal: `->>` of a JSON boolean + // renders as 'true'/'false' on Postgres but 1/0 on SQLite, so a + // `= 'true'` comparison would silently diverge between adapters. `IS + // [NOT] NULL` is adapter-agnostic and inverts cleanly when `notCondition` + // wraps this in `NOT (...)`, so polarity needs no special handling here. + let extracted = fieldQuery(key, onRef, false, 'filter'); + return [ + fieldArity({ + type: onRef, + path: key, + value: [extracted, value ? 'IS NOT NULL' : 'IS NULL'], + errorHint: 'filter', + }), + ]; + } if (value === null) { let query = fieldQuery(key, onRef, true, 'filter'); return [ @@ -1476,7 +1527,8 @@ export class IndexQueryEngine { typeof element === 'string' || element.kind === 'table-valued-each' || element.kind === 'table-valued-tree' || - element.kind === 'json-contains' + element.kind === 'json-contains' || + element.kind === 'types-contains' ) { return Promise.resolve([element]); } else if (element.kind === 'field-query') { @@ -1918,16 +1970,19 @@ async function getField( return await definitionLookup.lookupDefinition(codeRef); }); if (!field) { - if ( - currentField(pathTraveled) === '_cardType' || - currentField(pathTraveled) === '_title' - ) { - // this is a little awkward--we have the need to treat '_cardType' and - // '_title' as string fields that we can query against from the index - // (e.g. the cards grid sorts by the card's display name; a mixed - // cards+files search sorts/filters both row types on '_title'). the - // index-runner / prerender meta route inject these into the searchDoc - // during index time. + if (isSyntheticSearchDocKey(currentField(pathTraveled))) { + // this is a little awkward--we have the need to treat the synthetic + // search-doc keys (see search-doc-keys.ts) as fields that we can query + // against from the index (e.g. the cards grid sorts by the card's display + // name via `_cardType`; a mixed cards+files search sorts/filters both row + // types on `_title`, and dedups a card's dual-indexed `.json` file row + // with `eq: { _isCardInstanceFile: false }`). the index-runner / prerender + // meta route inject these into the searchDoc during index time. The string + // field shape here is enough for the non-null string keys and the `IS + // NULL` legacy spelling; `_isCardInstanceFile`'s boolean `eq: true/false` + // is special-cased in `fieldEqFilter` so it never depends on the + // adapter-specific `->>` rendering of a JSON boolean. Kept in sync with + // the matcher shim (instance-filter-matcher.ts) and searchable-parity.ts. return { type: 'contains', isPrimitive: true, @@ -2046,6 +2101,57 @@ function dualReadColumn(col: string): string { return `CASE WHEN ph.url IS NULL THEN i.${col} ELSE ph.${col} END`; } +// Maps a raw `file`-row search result to the `IndexedFile` view-model. Shared +// by `searchFiles` and the mixed `searchEntries` assembly loop (which builds +// a file's resource + native renderings from the same shape). +export function fileEntryFromResult( + result: Partial, +): IndexedFile { + let canonicalURL = result.url; + if (!canonicalURL) { + throw new Error('expected file search result to include url'); + } + let lastModified = + typeof result.last_modified === 'string' + ? parseInt(result.last_modified) + : (result.last_modified ?? null); + let resourceCreatedAt = + typeof result.resource_created_at === 'string' + ? parseInt(result.resource_created_at) + : (result.resource_created_at ?? null); + let indexedAt = + typeof result.indexed_at === 'string' + ? parseInt(result.indexed_at) + : (result.indexed_at ?? null); + return { + type: 'file', + canonicalURL, + searchDoc: (result.search_doc as Record | null) ?? null, + resource: (result.pristine_doc as FileMetaResource | null) ?? null, + types: (result.types as string[] | null) ?? null, + displayNames: (result.display_names as string[] | null) ?? null, + deps: (result.deps as string[] | null) ?? null, + isolatedHtml: result.isolated_html ?? null, + headHtml: result.head_html ?? null, + embeddedHtml: + (result.embedded_html as { [refURL: string]: string } | null) ?? null, + fittedHtml: + (result.fitted_html as { [refURL: string]: string } | null) ?? null, + atomHtml: result.atom_html ?? null, + iconHtml: result.icon_html ?? null, + markdown: result.markdown ?? null, + lastModified, + resourceCreatedAt, + generation: result.generation ?? 0, + htmlGeneration: + (result as { html_generation?: number | null }).html_generation ?? + result.generation ?? + 0, + realmURL: result.realm_url ?? '', + indexedAt, + }; +} + // Dual-read HTML/markdown reads, appended after a `SELECT i.*` so these win over // the same-named boxel_index columns (duplicate result columns resolve to the // last one in both the pg and sqlite adapters). `icon_html` is excluded — the diff --git a/packages/runtime-common/index-runner/file-indexer.ts b/packages/runtime-common/index-runner/file-indexer.ts index 0a12980ebf..2d183c4a8c 100644 --- a/packages/runtime-common/index-runner/file-indexer.ts +++ b/packages/runtime-common/index-runner/file-indexer.ts @@ -24,6 +24,8 @@ import { BASE_FILE_DEF_CODE_REF, resolveFileDefCodeRef, } from '../file-def-code-ref.ts'; +import { baseRef } from '../constants.ts'; +import { CARD_INSTANCE_FILE_KEY } from '../search-doc-keys.ts'; import type { VirtualNetwork } from '../virtual-network.ts'; export interface FileIndexerOptions { @@ -86,6 +88,12 @@ export async function performFileIndexing({ ) { fileTypeRefs.push(BASE_FILE_DEF_CODE_REF); } + // Terminate the chain at BaseDef (the common ancestor of FileDef and CardDef) + // so a `{ type: baseRef }` filter matches both file and card-instance rows — + // e.g. `scope: 'all'` searches wanting one type ref for "everything". The + // instance chain (render/meta.ts `getTypes`) and the extractor chain + // (file-def-attributes-extractor.ts `getTypes`) append it too. + fileTypeRefs.push(baseRef); let extractResult: FileExtractResponse | undefined = precomputedExtractResult; let uncaughtError: Error | undefined; @@ -171,11 +179,13 @@ export async function performFileIndexing({ // Shared by the success entry and the dependency-error entry below, so the // two rows carry the same search keys. Two of them are synthetic (stamped // after the extractor's searchDoc so they win deterministically): - // - `_isCardInstance` marks the `file` row of a dual-indexed card-instance + // - `_isCardInstanceFile` marks the `file` row of a dual-indexed card-instance // .json so mixed cards+files search can exclude it (the card already // appears via its `instance` row). It rides on the dependency-error entry // too, so a card .json whose card is in an error state stays out of file - // search. Stamped only when true; plain file docs don't carry the key. + // search. Stamped only when true; plain file docs don't carry the key — so + // `eq: { _isCardInstanceFile: false }` (absent-as-false) keeps cards + plain + // files and drops this row. // - `_title` is the row's display title (a file's is its name) under a // neutral key that card docs also carry (stamped alongside `_cardType` // during card render), so one mixed query can substring-match @@ -187,7 +197,7 @@ export async function performFileIndexing({ name, contentType, ...(extractResult.searchDoc ?? {}), - ...(isCardInstance ? { _isCardInstance: true } : {}), + ...(isCardInstance ? { [CARD_INSTANCE_FILE_KEY]: true } : {}), _title: name, }; diff --git a/packages/runtime-common/index-writer.ts b/packages/runtime-common/index-writer.ts index 87c9b6b502..364e486d22 100644 --- a/packages/runtime-common/index-writer.ts +++ b/packages/runtime-common/index-writer.ts @@ -847,7 +847,7 @@ export class Batch { // favor the last known good types over the types derived from the error state ...production, // Assign search_doc AFTER the production spread so the freshly-stamped - // synthetic keys (`_title`, `_isCardInstance`, `_cardType`) survive + // synthetic keys (`_title`, `_isCardInstanceFile`, `_cardType`) survive // rather than being clobbered by the last-known-good doc. Overlaying // the current searchData onto that doc keeps an instance's rich fields // when it degrades to a sparse error searchData, while a file / diff --git a/packages/runtime-common/instance-filter-matcher.ts b/packages/runtime-common/instance-filter-matcher.ts index 4b5bb9875a..ab07a66a50 100644 --- a/packages/runtime-common/instance-filter-matcher.ts +++ b/packages/runtime-common/instance-filter-matcher.ts @@ -16,6 +16,14 @@ import { type Sort, } from './query.ts'; +import { + CARD_INSTANCE_FILE_KEY, + CARD_TITLE_KEY, + CARD_TYPE_KEY, + isSyntheticSearchDocKey, +} from './search-doc-keys.ts'; +import { friendlyCardType } from './helpers/card-type-display-name.ts'; + import type { CodeRef } from './code-ref.ts'; import type { VirtualNetwork } from './virtual-network.ts'; import type { BaseDef, CardDef, Field } from '@cardstack/base/card-api'; @@ -86,12 +94,40 @@ export function isClientEvaluable(filter: Filter): boolean { isRangeFilter(filter) || isCardTypeFilter(filter) ) { + // A synthetic underscore-prefixed key that the matcher doesn't shim (see + // resolveSyntheticKey) can't be evaluated locally without silently + // diverging from the server, so force server-only evaluation for it. This + // makes a future synthetic key added on the server without a matcher shim + // degrade to "no instant results" rather than "reconciler blanks correct + // server results". + for (let op of ['eq', 'contains', 'in', 'range'] as const) { + let operator = (filter as unknown as Record)[op]; + if ( + operator && + typeof operator === 'object' && + !operatorPathsClientEvaluable(operator as Record) + ) { + return false; + } + } return true; } // Unknown / non-replicable operator. return false; } +// A filter operator object is keyed by field paths. An underscore-prefixed path +// head is a synthetic search-doc key; only the ones the matcher shims are +// client-evaluable. +function operatorPathsClientEvaluable( + operator: Record, +): boolean { + return Object.keys(operator).every((path) => { + let head = path.split('.')[0]; + return !head.startsWith('_') || isSyntheticSearchDocKey(head); + }); +} + export function matchInstanceAgainstFilter( instance: CardDef, filter: Filter, @@ -182,11 +218,63 @@ interface PathResolution { sawUnresolvable: boolean; } +// Client-side counterpart to the engine's `getField` synthetic-key shim +// (index-query-engine.ts) plus the `_isCardInstanceFile` special case in +// `fieldEqFilter`. These keys are stamped into a row's search_doc at index time +// (see search-doc-keys.ts), so they are not real fields on a hydrated instance; +// resolve them to the value the server agrees they denote. Kept in sync with the +// engine and the parity differ (searchable-parity.ts) — a synthetic key added +// on the server without a shim here re-introduces the reconciler-blanks-results +// bug this guards against (see `isClientEvaluable`). +function resolveSyntheticKey( + instance: BaseDef, + path: string, + api: CardAPIForMatching, +): PathResolution | undefined { + if (!isSyntheticSearchDocKey(path)) { + return undefined; + } + // A CardDef instance can reproduce all three keys; a file-meta instance + // cannot derive them from its attributes (the stamp lives in the search doc, + // not the file's fields), so it is 'unresolvable' — which the reconciler + // treats as "trust the server" rather than dropping a correct result. + let isCard = Boolean(getField(instance, 'cardTitle')); + if (!isCard) { + return { values: [], leafField: undefined, sawUnresolvable: true }; + } + if (path === CARD_TITLE_KEY) { + // Stamped as `searchDoc.cardTitle`; `cardTitle` is a real CardDef field, so + // resolve it the normal way (identical value, correct leafField for + // formatting). + return resolvePath(instance, 'cardTitle', api); + } + if (path === CARD_TYPE_KEY) { + let ctor = instance.constructor as { displayName: string; name: string }; + return { + values: [friendlyCardType(ctor)], + leafField: undefined, + sawUnresolvable: false, + }; + } + if (path === CARD_INSTANCE_FILE_KEY) { + // The stamp marks a card `.json`'s *file* row, which hydrates as a FileDef + // (handled by the non-card branch above), never as this CardDef — so a + // hydrated card is definitively not a card-instance file. The value is + // `false`, so the canonical dedup filter `eq: { …: false }` matches. + return { values: [false], leafField: undefined, sawUnresolvable: false }; + } + return { values: [], leafField: undefined, sawUnresolvable: true }; +} + function resolvePath( instance: BaseDef, path: string, api: CardAPIForMatching, ): PathResolution { + let synthetic = resolveSyntheticKey(instance, path, api); + if (synthetic) { + return synthetic; + } let segments = path.split('.'); let nodes: BaseDef[] = [instance]; diff --git a/packages/runtime-common/query-field-utils.ts b/packages/runtime-common/query-field-utils.ts index ec6a6d6011..7df98c124d 100644 --- a/packages/runtime-common/query-field-utils.ts +++ b/packages/runtime-common/query-field-utils.ts @@ -1,5 +1,6 @@ import { codeRefWithAbsoluteIdentifier, type CodeRef } from './code-ref.ts'; import { rri, type RealmResourceIdentifier } from './realm-identifiers.ts'; +import { CARD_INSTANCE_FILE_KEY } from './search-doc-keys.ts'; import type { FieldDefinition } from './definitions.ts'; import type { FileMetaResource, @@ -403,6 +404,18 @@ export function getTypeRefsFromFilter( return undefined; } +// The dedup filter for a mixed (`scope: 'all'`) entry search: keep card +// instances and plain files, drop a card's dual-indexed `.json` file row (the +// card already appears via its `instance` row). The `_isCardInstanceFile` key +// is stamped only on that file row, so `eq: false` (absent-as-false) keeps +// every other row and drops it. Restricting to a single kind is better done +// with the wire `scope` member (`'cards'`/`'files'`), which pins +// `boxel_index.type` directly; this filter is only for the both-kinds case that +// still wants each card once. +export function excludeCardInstanceFileRows(): Filter { + return { eq: { [CARD_INSTANCE_FILE_KEY]: false } }; +} + export function cloneRelationship( relationship?: Relationship, ): Relationship | undefined { diff --git a/packages/runtime-common/realm-index-query-engine.ts b/packages/runtime-common/realm-index-query-engine.ts index f7766afc35..79a4d21ce6 100644 --- a/packages/runtime-common/realm-index-query-engine.ts +++ b/packages/runtime-common/realm-index-query-engine.ts @@ -9,6 +9,7 @@ import { maxLinkDepth, maybeURL, IndexQueryEngine, + fileEntryFromResult, codeRefWithAbsoluteIdentifier, logger, CardResourceType, @@ -272,15 +273,33 @@ export class RealmIndexQueryEngine { async searchEntries( searchEntryQuery: SearchEntryQuery, opts?: Options, + // Internal override for the single-URL GET path, which pins 'instance' so a + // bare file URL never resolves as a card entry. When omitted, the scope + // comes from the wire `searchEntryQuery.scope` (mapped below): 'cards' -> + // 'instance', 'files' -> 'file', 'all'/absent -> 'all'. + entryTypeScopeOverride?: 'instance' | 'file' | 'all', ): Promise { - let { itemQuery: query, htmlQuery, fieldset, cardUrls } = searchEntryQuery; + let { + itemQuery: query, + htmlQuery, + fieldset, + cardUrls, + scope, + } = searchEntryQuery; let engineOpts: Options = { ...opts, ...(cardUrls && cardUrls.length > 0 ? { cardUrls } : {}), }; - if (await this.queryTargetsFileMeta(query.filter, engineOpts)) { - return await this.searchEntriesFileMeta(searchEntryQuery, engineOpts); - } + + // `scope` pins `boxel_index.type` directly. 'all' searches both kinds in one + // query — a card row and a file row carry non-overlapping `types`, so the + // caller's filter also discriminates by kind. A mixed 'all' query returns + // both a card's `instance` row and its dual-indexed `.json` `file` row; a + // consumer that wants the file row dropped does so through its own filter + // (`eq: { item._isCardInstanceFile: false }`). + let entryTypeScope: 'instance' | 'file' | 'all' = + entryTypeScopeOverride ?? + (scope === 'cards' ? 'instance' : scope === 'files' ? 'file' : 'all'); let itemOnEveryRow = fieldset.item.kind !== 'none'; let projection: SearchProjection = fieldset.html @@ -288,16 +307,26 @@ export class RealmIndexQueryEngine { : { kind: 'dataOnly' }; // Error rows surface only through the `html` branch (their renderings // carry `isError`); the item-only projection matches the live search - // path, which excludes them. - let sqlOpts: QueryOptions = fieldset.html - ? { ...engineOpts, includeErrors: true } - : engineOpts; + // path, which excludes them. The 'files' scope never includes errors — + // files are only ever surfaced healthy (the mixed 'all' scope forces the + // same for its file branch in `_search`) — so it strips includeErrors even + // when an html fieldset would otherwise set it. + let sqlOpts: QueryOptions; + if (entryTypeScope === 'file') { + let { includeErrors: _drop, ...rest } = engineOpts; + sqlOpts = rest; + } else if (fieldset.html) { + sqlOpts = { ...engineOpts, includeErrors: true }; + } else { + sqlOpts = engineOpts; + } let runSql = () => this.#indexQueryEngine.search( new URL(this.#realm.url), query, sqlOpts, projection, + entryTypeScope, ); let { results, meta } = opts?.timings ? await opts.timings.time('sql', runSql) @@ -318,6 +347,82 @@ export class RealmIndexQueryEngine { let fullItemRoots: (CardResource | FileMetaResource)[] = []; for (let row of results) { + // A `file` row (mixed 'all' scope) renders natively and carries no + // ancestor coercion — its renderings hang off its own type's entry with + // no renderTypeKey, and its resource is the synthesized `file-meta`. This + // is the file counterpart of the instance branch below; kind is decided + // per row on `row.type`. + if (row.type === 'file') { + let file = fileEntryFromResult(row); + let url = file.canonicalURL; + if (!url) { + continue; + } + let fileHtmlIds: string[] | undefined; + let fileIconId = collectIconId( + fieldset.html ? file.types?.[0] : undefined, + fieldset.html ? (file.iconHtml ?? undefined) : undefined, + file.displayNames?.[0] ?? '', + iconById, + ); + if (fieldset.html) { + let matched = enumerateFileRenderings(file).filter((candidate) => + htmlQueryMatches(resolvedHtmlQuery, candidate), + ); + let cssIds: string[] = []; + if (matched.length > 0) { + for (let href of scopedCssHrefsFromDeps(file.deps)) { + let css = buildCssResource(href); + if (!cssById.has(css.id)) { + cssById.set(css.id, css); + } + cssIds.push(css.id); + } + } + let ids: string[] = []; + for (let candidate of matched) { + let htmlResource = buildHtmlResource({ + url, + format: candidate.format, + html: candidate.html, + cardType: file.displayNames?.[0] ?? '', + cssIds, + generation: file.htmlGeneration ?? file.generation, + }); + htmlResources.push(htmlResource); + ids.push(htmlResource.id); + } + fileHtmlIds = + fieldset.itemAsFallback && ids.length === 0 ? undefined : ids; + } + let emitFileItem = + itemOnEveryRow || + (fieldset.itemAsFallback && fileHtmlIds === undefined); + let fileItemEmitted = false; + if (emitFileItem) { + let item: FileMetaResource = fileResourceFromIndex( + new URL(url), + file, + ); + if (fieldset.item.kind === 'sparse') { + item = buildSparseItemResource(item, fieldset.item.fields); + } else { + fullItemRoots.push(item); + } + itemResources.push(item); + fileItemEmitted = true; + } + data.push( + buildEntryResource({ + url, + htmlIds: fileHtmlIds, + itemType: fileItemEmitted ? FileMetaResourceType : undefined, + iconId: fileIconId, + generation: file.generation, + }), + ); + continue; + } let fileUrl = row.url; if (!fileUrl) { continue; @@ -486,18 +591,17 @@ export class RealmIndexQueryEngine { fieldset, cardUrls: [url.href], }; - let collection = - kind === 'file' - ? // The file path is reached only through this explicit `kind` — an - // empty membership query never routes to file-meta on its own (see - // `queryTargetsFileMeta`), so a file's entry must be requested by - // accept header. `cardUrls` rides in `opts` for the file path (it's - // read there, not off the SearchEntryQuery). - await this.searchEntriesFileMeta(searchEntryQuery, { - ...opts, - cardUrls: [url.href], - }) - : await this.searchEntries(searchEntryQuery, opts); + // Pin the scope by kind: a membership query resolves a card `.json`'s URL + // to its `instance` entry, so a file's entry must be requested explicitly by + // accept header ('file'), and a by-URL card lookup pins 'instance' so a bare + // file URL 404s rather than resolving as a file-meta entry. Both scopes run + // through the one `searchEntries` assembly loop (the file rows via its + // `row.type === 'file'` branch). + let collection = await this.searchEntries( + searchEntryQuery, + opts, + kind === 'file' ? 'file' : 'instance', + ); let [entry] = collection.data; if (!entry) { return undefined; @@ -509,125 +613,6 @@ export class RealmIndexQueryEngine { return doc; } - // The file-meta counterpart of `searchEntries`. Files are indexed as - // `type: 'file'` rows, which the instance-only projections skip, so they - // resolve through `searchFiles` (per-format HTML + the full `file-meta` - // resource), with the same fieldset semantics. A file renders natively — - // there is no ancestor coercion — so a file rendering carries no - // renderType, its composite id is just `#`, and a - // renderType predicate in the htmlQuery never matches a file rendering. - private async searchEntriesFileMeta( - searchEntryQuery: SearchEntryQuery, - opts?: Options, - ): Promise { - let { itemQuery: query, htmlQuery, fieldset } = searchEntryQuery; - let { includeErrors: _includeErrors, ...fileOpts } = opts ?? {}; - let runSql = () => - this.#indexQueryEngine.searchFiles( - new URL(this.#realm.url), - query, - fileOpts, - ); - let { files, meta } = opts?.timings - ? await opts.timings.time('sql', runSql) - : await runSql(); - - let resolvedHtmlQuery = resolveHtmlQuery(htmlQuery, (ref) => - internalKeyFor(ref, undefined, this.#realm.virtualNetwork), - ); - - let itemOnEveryRow = fieldset.item.kind !== 'none'; - let data: EntryResource[] = []; - let htmlResources: EntryIncludedResource[] = []; - let itemResources: (CardResource | FileMetaResource)[] = []; - let cssById = new Map(); - let iconById = new Map(); - let fullItemRoots: (CardResource | FileMetaResource)[] = []; - - for (let file of files) { - let url = file.canonicalURL; - if (!url) { - continue; - } - - let htmlIds: string[] | undefined; - // A file's type icon, deduped by its native-type internal key — carried - // on the `entry` so a no-HTML file row (e.g. a `.gts`/`.ts` - // FileDef with no fitted rendering) still resolves its icon. - let iconId = collectIconId( - fieldset.html ? file.types?.[0] : undefined, - fieldset.html ? (file.iconHtml ?? undefined) : undefined, - file.displayNames?.[0] ?? '', - iconById, - ); - if (fieldset.html) { - let matched = enumerateFileRenderings(file).filter((candidate) => - htmlQueryMatches(resolvedHtmlQuery, candidate), - ); - let cssIds: string[] = []; - if (matched.length > 0) { - for (let href of scopedCssHrefsFromDeps(file.deps)) { - let css = buildCssResource(href); - if (!cssById.has(css.id)) { - cssById.set(css.id, css); - } - cssIds.push(css.id); - } - } - let ids: string[] = []; - for (let candidate of matched) { - let htmlResource = buildHtmlResource({ - url, - format: candidate.format, - html: candidate.html, - cardType: file.displayNames?.[0] ?? '', - cssIds, - generation: file.htmlGeneration ?? file.generation, - }); - htmlResources.push(htmlResource); - ids.push(htmlResource.id); - } - htmlIds = fieldset.itemAsFallback && ids.length === 0 ? undefined : ids; - } - - let emitItem = - itemOnEveryRow || (fieldset.itemAsFallback && htmlIds === undefined); - let itemEmitted = false; - if (emitItem) { - // The file's live serialization — the same synthesized resource the - // live search path serves (raw pristine rows can lag the synthesized - // attributes, e.g. inferred contentType / searchDoc extras). - let item: FileMetaResource = fileResourceFromIndex(new URL(url), file); - if (fieldset.item.kind === 'sparse') { - item = buildSparseItemResource(item, fieldset.item.fields); - } else { - fullItemRoots.push(item); - } - itemResources.push(item); - itemEmitted = true; - } - - data.push( - buildEntryResource({ - url, - htmlIds, - itemType: itemEmitted ? FileMetaResourceType : undefined, - iconId, - generation: file.generation, - }), - ); - } - - let metaWithEcho: EntryCollectionDocument['meta'] = fieldset.html - ? { ...meta, htmlQuery } - : meta; - return await this.assembleSearchEntryDoc( - { data, meta: metaWithEcho }, - { htmlResources, cssById, iconById, itemResources, fullItemRoots }, - opts, - ); - } - // Shared tail of the two searchEntries paths: stitch `included` together // (html renderings first, then the deduped css, then the items), expand // links for the full items only (sparse items are field-limited data reads diff --git a/packages/runtime-common/search-doc-keys.ts b/packages/runtime-common/search-doc-keys.ts new file mode 100644 index 0000000000..4d7f1376b9 --- /dev/null +++ b/packages/runtime-common/search-doc-keys.ts @@ -0,0 +1,33 @@ +// Synthetic keys stamped into a row's `search_doc` after render (see +// index-runner/{card,file}-indexer.ts). They are not real card/file fields, but +// the query engine (index-query-engine.ts `getField` shim + `fieldEqFilter`), +// the client-side matcher (instance-filter-matcher.ts), and the search-doc +// parity differ (searchable-parity.ts) all treat them as addressable so cards +// and files can be filtered/sorted through one query. Keeping the spellings in +// one module means a rename is a single-file change rather than a cross-package +// grep. + +// The row's display title under a kind-neutral key (a card's is its +// `cardTitle`, a file's is its name), so one mixed query can substring-match +// and A-Z sort cards and files uniformly. +export const CARD_TITLE_KEY = '_title'; + +// The card type's friendly display name (see `friendlyCardType`). +export const CARD_TYPE_KEY = '_cardType'; + +// Stamped `true` only on the `file` row of a dual-indexed card `.json` (the +// same URL also has an `instance` row); absent on card-instance rows and on +// plain file rows. So the canonical spelling that keeps cards + plain files and +// drops the duplicate `.json` file row is `eq: false` (which matches an absent +// key too), not `eq: true`. +export const CARD_INSTANCE_FILE_KEY = '_isCardInstanceFile'; + +export const SYNTHETIC_SEARCH_DOC_KEYS = [ + CARD_TITLE_KEY, + CARD_TYPE_KEY, + CARD_INSTANCE_FILE_KEY, +] as const; + +export function isSyntheticSearchDocKey(key: string): boolean { + return (SYNTHETIC_SEARCH_DOC_KEYS as readonly string[]).includes(key); +} diff --git a/packages/runtime-common/search-entry.ts b/packages/runtime-common/search-entry.ts index 101702eb06..c1ee946f34 100644 --- a/packages/runtime-common/search-entry.ts +++ b/packages/runtime-common/search-entry.ts @@ -104,8 +104,19 @@ const SEARCH_ENTRY_QUERY_MEMBERS = [ 'realms', 'fields', 'cardUrls', + 'scope', ]; +// Which row kinds a search spans. Pins `boxel_index.type` directly, so it works +// for any filter shape and is independent of index-stamp state — unlike +// discriminating kind through a `{ type: … }` anchor or the +// `_isCardInstanceFile` dedup key. `'all'` (the default) returns both kinds, +// including both rows of a dual-indexed card `.json`; a caller that wants the +// `.json` file row dropped adds the dedup filter explicitly. +export type SearchEntryScope = 'cards' | 'files' | 'all'; + +const SEARCH_ENTRY_SCOPES: SearchEntryScope[] = ['cards', 'files', 'all']; + // The operator members whose value is an object keyed by field paths. const FIELD_KEYED_OPERATORS = ['eq', 'contains', 'in', 'range']; @@ -135,6 +146,7 @@ export interface SearchEntryQuery { fieldset: SearchEntryFieldset; realms?: string[]; cardUrls?: string[]; + scope?: SearchEntryScope; } function invalidQuery(message: string): SearchRequestError { @@ -604,6 +616,18 @@ export function parseSearchEntryQueryFromPayload( } } + let scope: SearchEntryScope | undefined; + if (record.scope !== undefined) { + if (!SEARCH_ENTRY_SCOPES.includes(record.scope as SearchEntryScope)) { + throw invalidQuery( + `scope must be one of ${SEARCH_ENTRY_SCOPES.map((s) => `"${s}"`).join( + ', ', + )}`, + ); + } + scope = record.scope as SearchEntryScope; + } + let capture: HtmlQueryCapture = {}; let filter: Record | undefined; if (record.filter !== undefined) { @@ -655,6 +679,7 @@ export function parseSearchEntryQueryFromPayload( fieldset, realms, cardUrls, + scope, }; } @@ -851,6 +876,7 @@ export interface SearchEntryWireQuery { fields?: { entry: string[] }; cardUrls?: string[]; realms?: string[]; + scope?: SearchEntryScope; } function wireFilterFromFilter(filter: Filter): SearchEntryWireFilter { @@ -886,7 +912,7 @@ function wireFilterFromFilter(filter: Filter): SearchEntryWireFilter { export function searchEntryWireQueryFromQuery( query: Query, - opts?: { fields?: string[] }, + opts?: { fields?: string[]; scope?: SearchEntryScope }, ): SearchEntryWireQuery { // the legacy `realm`/`realms` members are deliberately not carried — the // caller addresses realms at the request level; `asData`/`fields` are the @@ -909,6 +935,9 @@ export function searchEntryWireQueryFromQuery( if (opts?.fields) { wire.fields = { entry: [...opts.fields] }; } + if (opts?.scope) { + wire.scope = opts.scope; + } return wire; } diff --git a/packages/runtime-common/searchable-parity.ts b/packages/runtime-common/searchable-parity.ts index d61cbd52b5..4bbeb8fa03 100644 --- a/packages/runtime-common/searchable-parity.ts +++ b/packages/runtime-common/searchable-parity.ts @@ -1,5 +1,7 @@ import stringify from 'safe-stable-stringify'; +import { SYNTHETIC_SEARCH_DOC_KEYS } from './search-doc-keys.ts'; + // Parity comparison between a store-driven search doc and a searchable-driven // one. Shared by the realm-scale validator (`scripts/searchable-parity-diff.ts`) // and the generation tests so both judge "parity" by exactly the same rule. @@ -81,9 +83,9 @@ function diffValue( if (isObject(lv) && isObject(gv)) { let keys = new Set([...Object.keys(lv), ...Object.keys(gv)]); - keys.delete('_cardType'); - keys.delete('_isCardInstance'); - keys.delete('_title'); + for (let syntheticKey of SYNTHETIC_SEARCH_DOC_KEYS) { + keys.delete(syntheticKey); + } for (let key of keys) { diffValue( path ? `${path}.${key}` : key, @@ -118,8 +120,7 @@ function diffValue( // Compare two search docs and return a list of human-readable divergences // (empty when equivalent). Synthetic keys stamped after generation -// (`_cardType`, `_title`, `_isCardInstance`) are ignored; object key order is -// normalized. +// (SYNTHETIC_SEARCH_DOC_KEYS) are ignored; object key order is normalized. export function diffDoc( live: SearchDoc, generated: SearchDoc, From 0aedc2cb13a963c988361d9b69bd2b66818d3442 Mon Sep 17 00:00:00 2001 From: Fadhlan Ridhwanallah Date: Mon, 13 Jul 2026 20:53:01 +0700 Subject: [PATCH 2/6] Search sheet: include files in term search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sheet's base filter was `any: [CardDef, FieldDef]`, which no file row satisfies (file type chains run MarkdownDef → FileDef → BaseDef), so a typed term could never surface a file even though the search runs at the mixed scope — while picking a file type worked, because a picked type strips the type-only base filter entirely. - Base the sheet on `{ type: baseRef }`: BaseDef is the common ancestor stamped on both instance and file type chains, so one ref spans cards (including specs), field instances, and files. - Send the wire scope explicitly: `searchScopeForOptions` now returns 'all' for the mixed case instead of leaving scope unset. - Root refs (BaseDef/CardDef/FieldDef/FileDef) no longer suppress the card-json dedup filter: they span kinds rather than narrowing to one, so a base filter built from them still needs `excludeCardInstanceFileRows()` — otherwise every card would surface twice via its dual-indexed `.json` file row. A picked (narrowing) type still skips the dedup. - Add `baseFileRef` to the type picker's root set so the widened base filter reads as unconstrained and base FileDef stays out of the picker options. - Placeholder copy now mentions files. Tests: new acceptance module covering files-by-term, card dedup, and spec inclusion in the sheet; query-builder unit coverage for the root-ref dedup rules; the chooser search test's term becomes "Mark Jackson" since word-AND full-text over the now-included skills-realm docs also matched "Mark J". Co-Authored-By: Claude Fable 5 --- .../app/components/search-sheet/index.gts | 14 +- .../app/utils/card-search/query-builder.ts | 66 +++++-- .../host/app/utils/card-search/type-filter.ts | 4 +- .../search-sheet-files-test.gts | 164 ++++++++++++++++++ .../operator-mode-card-chooser-test.gts | 9 +- .../unit/card-search-query-builder-test.ts | 79 +++++++-- 6 files changed, 299 insertions(+), 37 deletions(-) create mode 100644 packages/host/tests/acceptance/interact-submode/search-sheet-files-test.gts diff --git a/packages/host/app/components/search-sheet/index.gts b/packages/host/app/components/search-sheet/index.gts index 768a3b73c6..79b34fe5fb 100644 --- a/packages/host/app/components/search-sheet/index.gts +++ b/packages/host/app/components/search-sheet/index.gts @@ -23,8 +23,7 @@ import { IconSearch } from '@cardstack/boxel-ui/icons'; import { type Filter, type ResolvedCodeRef, - baseCardRef, - baseFieldRef, + baseRef, } from '@cardstack/runtime-common'; import type RealmServerService from '@cardstack/host/services/realm-server'; @@ -80,9 +79,12 @@ const repositionDropdownsOnTransitionEnd = modifier((element: Element) => { element.removeEventListener('transitionend', handler as EventListener); }); -const BASE_FILTER: Filter = { - any: [{ type: baseCardRef }, { type: baseFieldRef }], -}; +// The sheet searches everything the index knows — cards (including specs, +// which the default `not: specRef` query path excludes), field instances, and +// files. BaseDef is the common ancestor stamped on both instance and file type +// chains, so this one ref spans all kinds; `scope: 'all'` rides the wire +// alongside (see `searchScopeForOptions`). +const BASE_FILTER: Filter = { type: baseRef }; export default class SearchSheet extends Component { @tracked private searchKey = ''; @@ -122,7 +124,7 @@ export default class SearchSheet extends Component { mode == SearchSheetModes.SearchPrompt || mode == SearchSheetModes.ChoosePrompt ) { - return 'Search for cards or enter card URL'; + return 'Search for cards and files or enter card URL'; } return 'Search for…'; } diff --git a/packages/host/app/utils/card-search/query-builder.ts b/packages/host/app/utils/card-search/query-builder.ts index e2053dc885..7760921f2f 100644 --- a/packages/host/app/utils/card-search/query-builder.ts +++ b/packages/host/app/utils/card-search/query-builder.ts @@ -6,6 +6,10 @@ import type { SearchEntryScope, } from '@cardstack/runtime-common'; import { + baseCardRef, + baseFieldRef, + baseFileRef, + baseRef, codeRefFromInternalKey, excludeCardInstanceFileRows, getTypeRefsFromFilter, @@ -13,6 +17,7 @@ import { isCardTypeFilter, isEveryFilter, isNotFilter, + isResolvedCodeRef, specRef, } from '@cardstack/runtime-common'; @@ -105,32 +110,59 @@ function buildSearchTermFilter(searchTerm: string): Filter { // `searchScopeForOptions`), which pins `boxel_index.type` to instance rows // server-side — no filter anchor needed, and immune to a stray file-type filter // (which just yields no results rather than leaking files). The mixed sheet -// (no `cardsOnly`, default `scope: 'all'`) still discriminates the one case -// scope can't: dropping a card's dual-indexed `.json` file row so the card -// shows once. See `scopeFilters`. +// (no `cardsOnly`, `scope: 'all'`) still discriminates the one case scope +// can't: dropping a card's dual-indexed `.json` file row so the card shows +// once. See `scopeFilters`. export interface BuildQueryOptions { cardsOnly?: boolean; } -// The wire scope for a set of build options. cardsOnly pins card-instance rows; -// the mixed default leaves scope unset ('all'). +// The wire scope for a set of build options. cardsOnly pins card-instance +// rows; anything else is the mixed cards + files search. export function searchScopeForOptions( opts: BuildQueryOptions | undefined, -): SearchEntryScope | undefined { - return opts?.cardsOnly ? 'cards' : undefined; +): SearchEntryScope { + return opts?.cardsOnly ? 'cards' : 'all'; } -function hasPositiveTypeRef(filter: Filter | undefined): boolean { +// The root refs span kinds rather than narrowing to one — BaseDef is the +// common ancestor of every row, and CardDef/FieldDef/FileDef are each kind's +// own root (see `getRootTypeKeys` in type-filter.ts for the picker-side +// analogue). A base filter built from them (e.g. the search sheet's +// `{ type: baseRef }`) matches a card's dual-indexed `.json` file row too, so +// it must not suppress the mixed-scope dedup the way a genuinely narrowing +// type ref does. +const ROOT_TYPE_REFS: ResolvedCodeRef[] = [ + baseRef, + baseCardRef, + baseFieldRef, + baseFileRef, +]; + +function isRootTypeRef(ref: CodeRef): boolean { + return ( + isResolvedCodeRef(ref) && + ROOT_TYPE_REFS.some( + (root) => root.module === ref.module && root.name === ref.name, + ) + ); +} + +function hasNarrowingPositiveTypeRef(filter: Filter | undefined): boolean { if (!filter) { return false; } - return getTypeRefsFromFilter(filter)?.some((r) => !r.negated) ?? false; + return ( + getTypeRefsFromFilter(filter)?.some( + (r) => !r.negated && !isRootTypeRef(r.ref), + ) ?? false + ); } function scopeFilters( filters: Filter[], opts: BuildQueryOptions | undefined, - hasPositiveType: boolean, + hasNarrowingType: boolean, ): Filter[] { // cardsOnly is enforced by `scope: 'cards'` on the wire, so nothing to add. if (opts?.cardsOnly) { @@ -138,10 +170,12 @@ function scopeFilters( } // Mixed (`scope: 'all'`) sheet: drop a card's dual-indexed `.json` file row so // the card shows once (via its instance row). Skipped when the filter already - // carries a positive type ref — a picked card type matches only instance rows - // (no dupe to drop), and a picked file type must stay free to surface a - // `.json` file row that legitimately matches it. - if (hasPositiveType) { + // carries a kind-narrowing positive type ref — a picked card type matches only + // instance rows (no dupe to drop), and a picked file type must stay free to + // surface a `.json` file row that legitimately matches it. Root refs + // (BaseDef/CardDef/FieldDef/FileDef) don't count: they span kinds, so the + // dedup is still needed (see `hasNarrowingPositiveTypeRef`). + if (hasNarrowingType) { return filters; } return [...filters, excludeCardInstanceFileRows()]; @@ -179,7 +213,7 @@ export function buildSearchQuery( filters = scopeFilters( filters, opts, - Boolean(typeFilter) || hasPositiveTypeRef(baseFilter), + Boolean(typeFilter) || hasNarrowingPositiveTypeRef(baseFilter), ); return { filter: filters.length === 1 ? filters[0] : { every: filters }, @@ -231,7 +265,7 @@ export function buildRecentsQuery( filters = scopeFilters( filters, opts, - Boolean(typeFilter) || hasPositiveTypeRef(baseFilter), + Boolean(typeFilter) || hasNarrowingPositiveTypeRef(baseFilter), ); return { filter: filters.length === 1 ? filters[0] : { every: filters }, diff --git a/packages/host/app/utils/card-search/type-filter.ts b/packages/host/app/utils/card-search/type-filter.ts index ce59e197a8..a62603e15c 100644 --- a/packages/host/app/utils/card-search/type-filter.ts +++ b/packages/host/app/utils/card-search/type-filter.ts @@ -7,6 +7,7 @@ import type { import { baseCardRef, baseFieldRef, + baseFileRef, baseRef, getTypeRefsFromFilter, identifyCard, @@ -18,7 +19,7 @@ import { import type { CardDef } from '@cardstack/base/card-api'; /** - * Internal key strings for root types (CardDef, FieldDef, BaseDef). + * Internal key strings for root types (CardDef, FieldDef, FileDef, BaseDef). * These represent the base of the type hierarchy and are excluded * from type picker options and type-constraint checks. Pass the * caller's VirtualNetwork so the keys produced here match those @@ -28,6 +29,7 @@ export function getRootTypeKeys(virtualNetwork: VirtualNetwork): Set { return new Set([ internalKeyFor(baseCardRef, undefined, virtualNetwork), internalKeyFor(baseFieldRef, undefined, virtualNetwork), + internalKeyFor(baseFileRef, undefined, virtualNetwork), internalKeyFor(baseRef, undefined, virtualNetwork), ]); } diff --git a/packages/host/tests/acceptance/interact-submode/search-sheet-files-test.gts b/packages/host/tests/acceptance/interact-submode/search-sheet-files-test.gts new file mode 100644 index 0000000000..bf29427f20 --- /dev/null +++ b/packages/host/tests/acceptance/interact-submode/search-sheet-files-test.gts @@ -0,0 +1,164 @@ +import { click, fillIn, waitFor } from '@ember/test-helpers'; + +import { module, test } from 'qunit'; + +import { baseRealm, specRef } from '@cardstack/runtime-common'; + +import { + setupLocalIndexing, + setupRealmCacheTeardown, + testRealmURL, + setupOnSave, + setupAcceptanceTestRealm, + SYSTEM_CARD_FIXTURE_CONTENTS, + visitOperatorMode, + setupAuthEndpoints, + setupUserSubscription, + withCachedRealmSetup, + realmConfigCardJSON, +} from '../../helpers'; +import { setupMockMatrix } from '../../helpers/mock-matrix'; +import { setupApplicationTest } from '../../helpers/setup'; + +const testRealmFiles: Record = { + 'realm.json': realmConfigCardJSON({ + name: 'Test Workspace', + iconURL: 'https://boxel-images.boxel.ai/icons/Letter-t.png', + }), + 'pet.gts': ` + import { CardDef, Component, StringField, field, contains } from "@cardstack/base/card-api"; + export default class Pet extends CardDef { + static displayName = 'Pet'; + @field name = contains(StringField); + @field cardTitle = contains(StringField, { + computeVia: function (this: Pet) { + return this.name; + }, + }); + static embedded = class Embedded extends Component { + + } + } + `, + 'spec/pet.json': { + data: { + type: 'card', + attributes: { + cardTitle: 'Pet Spec', + cardDescription: 'Spec for Pet', + specType: 'card', + ref: { module: `../pet`, name: 'default' }, + }, + meta: { adoptsFrom: specRef }, + }, + }, + 'Pet/mango.json': { + data: { + attributes: { name: 'Mango' }, + meta: { + adoptsFrom: { + module: `../pet`, + name: 'default', + }, + }, + }, + }, + 'mango-care-notes.md': `# Mango Care Notes + +Feed twice a day. No couch privileges. +`, + 'garden-tips.md': `# Garden Tips + +Water the blueberries weekly. +`, +}; + +module( + 'Acceptance | interact submode | search sheet files tests', + function (hooks) { + setupApplicationTest(hooks); + setupLocalIndexing(hooks); + setupOnSave(hooks); + setupRealmCacheTeardown(hooks); + + let mockMatrixUtils = setupMockMatrix(hooks, { + loggedInAs: '@testuser:localhost', + activeRealms: [baseRealm.url, testRealmURL], + }); + let { setRealmPermissions, createAndJoinRoom } = mockMatrixUtils; + + hooks.beforeEach(async function () { + await withCachedRealmSetup(async () => { + await setupAcceptanceTestRealm({ + mockMatrixUtils, + realmURL: testRealmURL, + contents: { + ...SYSTEM_CARD_FIXTURE_CONTENTS, + ...testRealmFiles, + }, + }); + }); + + createAndJoinRoom({ + sender: '@testuser:localhost', + name: 'room-test', + }); + setupUserSubscription(); + setupAuthEndpoints(); + + setRealmPermissions({ + [baseRealm.url]: ['read'], + [testRealmURL]: ['read', 'write'], + }); + }); + + test('searching by term returns matching files alongside cards', async function (assert) { + await visitOperatorMode({}); + + await click('[data-test-open-search-field]'); + await fillIn('[data-test-search-field]', 'mango'); + + await waitFor( + `[data-test-search-result="${testRealmURL}mango-care-notes"]`, + ); + assert + .dom(`[data-test-search-result="${testRealmURL}mango-care-notes"]`) + .exists('the .md file matching the term by name is a search result'); + assert + .dom(`[data-test-search-result="${testRealmURL}Pet/mango"]`) + .exists( + { count: 1 }, + 'the card appears exactly once (its dual-indexed .json file row is deduped)', + ); + }); + + test('a term matching only a file name returns the file', async function (assert) { + await visitOperatorMode({}); + + await click('[data-test-open-search-field]'); + await fillIn('[data-test-search-field]', 'garden-tips'); + + await waitFor(`[data-test-search-result="${testRealmURL}garden-tips"]`); + assert + .dom(`[data-test-search-result="${testRealmURL}garden-tips"]`) + .exists('the file is found by its name'); + assert + .dom(`[data-test-search-result="${testRealmURL}Pet/mango"]`) + .doesNotExist('unrelated cards do not match'); + }); + + test('specs still match by title', async function (assert) { + await visitOperatorMode({}); + + await click('[data-test-open-search-field]'); + await fillIn('[data-test-search-field]', 'Pet Spec'); + + await waitFor(`[data-test-search-result="${testRealmURL}spec/pet"]`); + assert + .dom(`[data-test-search-result="${testRealmURL}spec/pet"]`) + .exists('spec cards remain searchable in the sheet'); + }); + }, +); diff --git a/packages/host/tests/integration/components/operator-mode-card-chooser-test.gts b/packages/host/tests/integration/components/operator-mode-card-chooser-test.gts index 076e35272a..a322b5cb21 100644 --- a/packages/host/tests/integration/components/operator-mode-card-chooser-test.gts +++ b/packages/host/tests/integration/components/operator-mode-card-chooser-test.gts @@ -440,9 +440,12 @@ module('Integration | operator-mode | card chooser', function (hooks) { await click(`[data-test-search-sheet-cancel-button]`); await click(`[data-test-open-search-field]`); - await fillIn(`[data-test-search-field]`, 'Mark J'); - // Wait for the specific result (Author/mark matches "Mark J" via its - // cardTitle "Mark Jackson") rather than polling the label's innerText; + // "Mark Jackson" (Author/mark's cardTitle) rather than a shorter prefix: + // the sheet spans files too, and full-text word-AND matching would let a + // vaguer term (e.g. "Mark J") also hit skills-realm docs that contain + // "mark" and "j" — this assertion wants exactly the one card. + await fillIn(`[data-test-search-field]`, 'Mark Jackson'); + // Wait for the specific result rather than polling the label's innerText; // DOM-element waits aren't racy against the transient "Searching…" state. await waitFor(`[data-test-search-result="${testRealmURL}Author/mark"]`, { timeout: 15000, diff --git a/packages/host/tests/unit/card-search-query-builder-test.ts b/packages/host/tests/unit/card-search-query-builder-test.ts index d6e9d125ee..fc6c820353 100644 --- a/packages/host/tests/unit/card-search-query-builder-test.ts +++ b/packages/host/tests/unit/card-search-query-builder-test.ts @@ -2,6 +2,7 @@ import { module, test } from 'qunit'; import { baseCardRef, + baseRef, excludeCardInstanceFileRows, specRef, type Filter, @@ -65,8 +66,13 @@ module('Unit | card-search/query-builder', function () { }); }); - test('a positively-typed baseFilter passes through alone — the picked type already selects one kind', function (assert) { - let baseFilter: Filter = { type: baseCardRef }; + test('a narrowing-typed baseFilter passes through alone — the picked type already selects one kind', function (assert) { + let baseFilter: Filter = { + type: { + module: 'http://test-realm/test/author' as RealmResourceIdentifier, + name: 'Author', + }, + }; let query = buildSearchQuery('', SORT_AZ, baseFilter); assert.deepEqual(query, { filter: baseFilter, @@ -74,8 +80,13 @@ module('Unit | card-search/query-builder', function () { }); }); - test('baseFilter with non-empty search wraps in every and OR-combines matches + _title', function (assert) { - let baseFilter: Filter = { type: baseCardRef }; + test('narrowing baseFilter with non-empty search wraps in every and OR-combines matches + _title', function (assert) { + let baseFilter: Filter = { + type: { + module: 'http://test-realm/test/author' as RealmResourceIdentifier, + name: 'Author', + }, + }; let query = buildSearchQuery('puppy', SORT_AZ, baseFilter); assert.deepEqual(query, { filter: { @@ -90,6 +101,55 @@ module('Unit | card-search/query-builder', function () { }); }); + test('a kind-spanning root baseFilter keeps the card-json dedup', function (assert) { + // The search sheet's base filter: BaseDef spans cards and files, so a + // card's dual-indexed `.json` file row matches it too and must still be + // deduped — unlike a narrowing type, a root ref doesn't select one kind. + let baseFilter: Filter = { type: baseRef }; + let query = buildSearchQuery('mango', SORT_AZ, baseFilter); + assert.deepEqual(query, { + filter: { + every: [ + baseFilter, + { + any: [{ matches: 'mango' }, { contains: { _title: 'mango' } }], + }, + DEDUP_FILTER, + ], + }, + sort: SORT_AZ.sort, + }); + }); + + test('root refs inside a compound baseFilter also keep the dedup', function (assert) { + let baseFilter: Filter = { + any: [{ type: baseCardRef }, { type: baseRef }], + }; + let query = buildSearchQuery('', SORT_AZ, baseFilter); + assert.deepEqual(query, { + filter: { every: [baseFilter, DEDUP_FILTER] }, + sort: SORT_AZ.sort, + }); + }); + + test('a picked type strips the root baseFilter and skips the dedup', function (assert) { + let markdownRef = { + module: + 'https://cardstack.com/base/markdown-file-def' as RealmResourceIdentifier, + name: 'MarkdownDef', + }; + let typeKey = `${markdownRef.module}/${markdownRef.name}`; + let query = buildSearchQuery('', SORT_AZ, { type: baseRef }, [typeKey]); + assert.deepEqual( + query, + { + filter: { type: markdownRef }, + sort: SORT_AZ.sort, + }, + 'the picked file type must stay free to surface card .json file rows', + ); + }); + test('search with a selected type produces a type filter alongside the search-term filter', function (assert) { let authorRef = { module: 'http://test-realm/test/author' as RealmResourceIdentifier, @@ -180,13 +240,10 @@ module('Unit | card-search/query-builder', function () { assert.strictEqual(searchScopeForOptions({ cardsOnly: true }), 'cards'); }); - test('the mixed default leaves the scope unset (all)', function (assert) { - assert.strictEqual(searchScopeForOptions({}), undefined); - assert.strictEqual(searchScopeForOptions(undefined), undefined); - assert.strictEqual( - searchScopeForOptions({ cardsOnly: false }), - undefined, - ); + test('anything else is the mixed "all" wire scope', function (assert) { + assert.strictEqual(searchScopeForOptions({}), 'all'); + assert.strictEqual(searchScopeForOptions(undefined), 'all'); + assert.strictEqual(searchScopeForOptions({ cardsOnly: false }), 'all'); }); }); From 32496e457e514adf0c28176038a826adce47ebae Mon Sep 17 00:00:00 2001 From: Fadhlan Ridhwanallah Date: Tue, 14 Jul 2026 12:17:32 +0700 Subject: [PATCH 3/6] Search sheet: keep file extensions in result ids; kind-neutral copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `removeFileExtension` stripped ANY trailing extension, a leftover from when every search result was a card `.json`. With files in the results, that mangled file identity in three ways: the result/overlay data-test attributes collapsed same-named files (`notes.md` ≡ `notes.txt`) and misreported file ids, deleting a card `` would also trim an open file stack item `.md` from the stack, and multi-select ids for file rows lost their extension. A file's extension is part of its canonical id — only the card `.json` convention is decorative. - Rename the helper to `removeCardJsonExtension` and strip only `.json`. `create-listing-modal` now uses this host helper (it previously reached the same-named generic helper in runtime-common, which stays for URL contexts). - Kind-neutral copy for the mixed results: section show-more buttons say "Show N more results", and the empty state says "No results found". Cards-only chooser copy elsewhere is untouched. The file click-through needed no change: the tile's select payload was already the unstripped entry id, and `detectStackItemTypeForTarget` resolves it as a file via `knownFileMetaUrls`. New acceptance coverage locks that in — clicking a `.md` search result opens the file stack item under its full URL — and the file-result assertions now expect extension-bearing ids. Co-Authored-By: Claude Fable 5 --- .../components/card-chooser/mini/index.gts | 4 +-- .../components/card-search/result-section.gts | 12 ++++--- .../components/card-search/result-tile.gts | 12 ++++--- .../components/card-search/sheet-results.gts | 2 +- .../operator-mode/create-listing-modal.gts | 6 ++-- .../operator-mode/interact-submode.gts | 4 +-- .../operator-mode/operator-mode-overlays.gts | 8 ++--- .../services/operator-mode-state-service.ts | 4 +-- packages/host/app/utils/card-search/types.ts | 8 +++-- .../search-sheet-files-test.gts | 34 ++++++++++++++++--- .../operator-mode-card-chooser-test.gts | 4 +-- .../components/overlay-menu-items-test.gts | 7 ++-- 12 files changed, 70 insertions(+), 35 deletions(-) diff --git a/packages/host/app/components/card-chooser/mini/index.gts b/packages/host/app/components/card-chooser/mini/index.gts index 113dcd39bb..363bb82179 100644 --- a/packages/host/app/components/card-chooser/mini/index.gts +++ b/packages/host/app/components/card-chooser/mini/index.gts @@ -6,7 +6,7 @@ import type { Filter } from '@cardstack/runtime-common'; import SearchPanel from '@cardstack/host/components/card-search/panel'; import { - removeFileExtension, + removeCardJsonExtension, type NewCardArgs, } from '@cardstack/host/utils/card-search/types'; @@ -43,7 +43,7 @@ export default class MiniCardChooser extends Component { if (typeof selection !== 'string') { return; } - let normalized = removeFileExtension(selection); + let normalized = removeCardJsonExtension(selection); if (normalized) { this.args.onSelect(normalized); } diff --git a/packages/host/app/components/card-search/result-section.gts b/packages/host/app/components/card-search/result-section.gts index 7489eb1bbc..d08e65b037 100644 --- a/packages/host/app/components/card-search/result-section.gts +++ b/packages/host/app/components/card-search/result-section.gts @@ -26,7 +26,7 @@ import type { } from '@cardstack/host/utils/card-search/sections'; import { - removeFileExtension, + removeCardJsonExtension, type NewCardArgs, } from '@cardstack/host/utils/card-search/types'; @@ -400,7 +400,7 @@ export default class ResultSection extends Component { Show {{this.nextShowMoreCount}} more - {{pluralize 'card' this.nextShowMoreCount}} + {{pluralize 'result' this.nextShowMoreCount}} ({{this.remainingCount}} not shown) @@ -470,7 +470,9 @@ export default class ResultSection extends Component { @adorn={{@adorn}} @adornStrokeClass={{@adornStrokeClass}} @adornPositionLabel={{@adornPositionLabel}} - data-test-recent-card-result={{removeFileExtension card.id}} + data-test-recent-card-result={{removeCardJsonExtension + card.id + }} /> <:after> @@ -552,7 +554,9 @@ export default class ResultSection extends Component { > Show {{this.nextShowMoreCount}} - more cards ({{this.remainingCount}} + more + {{pluralize 'result' this.nextShowMoreCount}} + ({{this.remainingCount}} not shown) {{/if}} diff --git a/packages/host/app/components/card-search/result-tile.gts b/packages/host/app/components/card-search/result-tile.gts index 4ac83856ac..9cbed37f5e 100644 --- a/packages/host/app/components/card-search/result-tile.gts +++ b/packages/host/app/components/card-search/result-tile.gts @@ -22,7 +22,7 @@ import AdornSelectChip from '@cardstack/host/components/adorn/adorn-select-chip' import { htmlComponent } from '@cardstack/host/lib/html-component'; import { - removeFileExtension, + removeCardJsonExtension, type NewCardArgs, } from '@cardstack/host/utils/card-search/types'; @@ -207,7 +207,7 @@ export default class SearchResultTile extends Component { {{on 'keydown' this.handleKeydown}} {{this.registerCardEl}} data-test-item-button-create-new={{@newCard.realmURL}} - data-test-item-button={{removeFileExtension this.resolvedItemId}} + data-test-item-button={{removeCardJsonExtension this.resolvedItemId}} data-test-item-button-selected={{if @isSelected 'true'}} ...attributes > @@ -272,7 +272,9 @@ export default class SearchResultTile extends Component { {{else if @entry}} <@entry.component class='hide-boundaries' - data-test-search-result={{removeFileExtension this.resolvedItemId}} + data-test-search-result={{removeCardJsonExtension + this.resolvedItemId + }} /> {{else if @card}} { @format='fitted' @codeRef={{defaultResultsCardRef}} @displayContainer={{false}} - data-test-search-result={{removeFileExtension this.resolvedItemId}} + data-test-search-result={{removeCardJsonExtension + this.resolvedItemId + }} /> {{/if}} diff --git a/packages/host/app/components/card-search/sheet-results.gts b/packages/host/app/components/card-search/sheet-results.gts index 3ab9894246..fe0c2f01d5 100644 --- a/packages/host/app/components/card-search/sheet-results.gts +++ b/packages/host/app/components/card-search/sheet-results.gts @@ -312,7 +312,7 @@ export default class SheetResults extends Component { {{#if this.hasNoResults}}
- No cards available + No results found
{{/if}} {{/if}} diff --git a/packages/host/app/components/operator-mode/create-listing-modal.gts b/packages/host/app/components/operator-mode/create-listing-modal.gts index 4454df05cd..5430847ec3 100644 --- a/packages/host/app/components/operator-mode/create-listing-modal.gts +++ b/packages/host/app/components/operator-mode/create-listing-modal.gts @@ -21,7 +21,6 @@ import { IconX, IconPlus } from '@cardstack/boxel-ui/icons'; import { chooseCard, isResolvedCodeRef, - removeFileExtension, rri, type ToolContext, type ResolvedCodeRef, @@ -37,6 +36,7 @@ import type LoaderService from '@cardstack/host/services/loader-service'; import type OperatorModeStateService from '@cardstack/host/services/operator-mode-state-service'; import type RealmService from '@cardstack/host/services/realm'; import type ToolService from '@cardstack/host/services/tool-service'; +import { removeCardJsonExtension } from '@cardstack/host/utils/card-search/types'; interface Signature { Args: {}; @@ -152,9 +152,9 @@ export default class CreateListingModal extends Component { }); @action private removeSelectedExample(urlToRemove: string) { - let normalizedUrlToRemove = removeFileExtension(urlToRemove); + let normalizedUrlToRemove = removeCardJsonExtension(urlToRemove); this._selectedExampleURLs = this.selectedExampleURLs.filter( - (url) => removeFileExtension(url) !== normalizedUrlToRemove, + (url) => removeCardJsonExtension(url) !== normalizedUrlToRemove, ); } diff --git a/packages/host/app/components/operator-mode/interact-submode.gts b/packages/host/app/components/operator-mode/interact-submode.gts index f0a885e943..e813017d88 100644 --- a/packages/host/app/components/operator-mode/interact-submode.gts +++ b/packages/host/app/components/operator-mode/interact-submode.gts @@ -61,7 +61,7 @@ import { idFromCardOrURL } from '@cardstack/host/utils/id-from-card-or-url'; import consumeContext from '../../helpers/consume-context'; -import { removeFileExtension } from '../../utils/card-search/types'; +import { removeCardJsonExtension } from '../../utils/card-search/types'; import CopyButton from './copy-button'; import DeleteModal from './delete-modal'; @@ -580,7 +580,7 @@ export default class InteractSubmode extends Component { .map((cardDefOrId) => { let raw = typeof cardDefOrId === 'string' ? cardDefOrId : cardDefOrId.id; - return raw ? removeFileExtension(raw) : undefined; + return raw ? removeCardJsonExtension(raw) : undefined; }) .filter(Boolean) as string[]; diff --git a/packages/host/app/components/operator-mode/operator-mode-overlays.gts b/packages/host/app/components/operator-mode/operator-mode-overlays.gts index 10da49ec57..2159a38f5c 100644 --- a/packages/host/app/components/operator-mode/operator-mode-overlays.gts +++ b/packages/host/app/components/operator-mode/operator-mode-overlays.gts @@ -49,7 +49,7 @@ import { AdornCheckmarkSelected, } from '@cardstack/host/components/adorn/adorn-select-checkmark'; -import { removeFileExtension } from '@cardstack/host/utils/card-search/types'; +import { removeCardJsonExtension } from '@cardstack/host/utils/card-search/types'; import { htmlComponent } from '../../lib/html-component'; import { knownFileMetaUrls } from '../../lib/known-file-meta-urls'; @@ -117,9 +117,9 @@ export default class OperatorModeOverlays extends Overlays { {{this.trackCompact renderedCard.element}} data-test-overlay-selected={{if isSelected - (removeFileExtension cardId) + (removeCardJsonExtension cardId) }} - data-test-overlay-card={{removeFileExtension cardId}} + data-test-overlay-card={{removeCardJsonExtension cardId}} style={{renderedCard.overlayZIndexStyle}} ...attributes > @@ -206,7 +206,7 @@ export default class OperatorModeOverlays extends Overlays { class='overlay-select-button' {{! @glint-ignore (glint thinks toggleSelect is not in this scope but it actually is - we check for it in the condition above) }} {{on 'click' (fn @toggleSelect cardDefOrId)}} - data-test-overlay-select={{removeFileExtension cardId}} + data-test-overlay-select={{removeCardJsonExtension cardId}} /> {{/if}}
diff --git a/packages/host/app/services/operator-mode-state-service.ts b/packages/host/app/services/operator-mode-state-service.ts index 814c84fa96..1f7ec668c2 100644 --- a/packages/host/app/services/operator-mode-state-service.ts +++ b/packages/host/app/services/operator-mode-state-service.ts @@ -50,7 +50,7 @@ import type RealmServer from '@cardstack/host/services/realm-server'; import type RecentCardsService from '@cardstack/host/services/recent-cards-service'; import type RecentFilesService from '@cardstack/host/services/recent-files-service'; -import { removeFileExtension } from '../utils/card-search/types'; +import { removeCardJsonExtension } from '../utils/card-search/types'; import { AiAssistantOpen, @@ -396,7 +396,7 @@ export default class OperatorModeStateService extends Service { for (let stack of this._state.stacks || []) { items.push( ...(stack.filter( - (i: StackItem) => i.id && removeFileExtension(i.id) === cardId, + (i: StackItem) => i.id && removeCardJsonExtension(i.id) === cardId, ) as StackItem[]), ); } diff --git a/packages/host/app/utils/card-search/types.ts b/packages/host/app/utils/card-search/types.ts index 0f419fd7bf..f7f330d207 100644 --- a/packages/host/app/utils/card-search/types.ts +++ b/packages/host/app/utils/card-search/types.ts @@ -6,6 +6,10 @@ export interface NewCardArgs { realmURL: string; } -export function removeFileExtension(cardId: string | undefined) { - return cardId?.replace(/\.[^/.]+$/, ''); +// Normalizes an id that may carry the card `.json` file convention down to the +// canonical extensionless card id. Only `.json` is stripped: a file entry's id +// (`….md`, `….gts`, …) IS its canonical id, and stripping its extension would +// collide same-named files and mismatch it against open file stack items. +export function removeCardJsonExtension(cardId: string | undefined) { + return cardId?.replace(/\.json$/, ''); } diff --git a/packages/host/tests/acceptance/interact-submode/search-sheet-files-test.gts b/packages/host/tests/acceptance/interact-submode/search-sheet-files-test.gts index bf29427f20..2d7d7b7f27 100644 --- a/packages/host/tests/acceptance/interact-submode/search-sheet-files-test.gts +++ b/packages/host/tests/acceptance/interact-submode/search-sheet-files-test.gts @@ -121,11 +121,13 @@ module( await fillIn('[data-test-search-field]', 'mango'); await waitFor( - `[data-test-search-result="${testRealmURL}mango-care-notes"]`, + `[data-test-search-result="${testRealmURL}mango-care-notes.md"]`, ); assert - .dom(`[data-test-search-result="${testRealmURL}mango-care-notes"]`) - .exists('the .md file matching the term by name is a search result'); + .dom(`[data-test-search-result="${testRealmURL}mango-care-notes.md"]`) + .exists( + 'the .md file matching the term by name is a search result, under its extension-bearing id', + ); assert .dom(`[data-test-search-result="${testRealmURL}Pet/mango"]`) .exists( @@ -140,15 +142,37 @@ module( await click('[data-test-open-search-field]'); await fillIn('[data-test-search-field]', 'garden-tips'); - await waitFor(`[data-test-search-result="${testRealmURL}garden-tips"]`); + await waitFor( + `[data-test-search-result="${testRealmURL}garden-tips.md"]`, + ); assert - .dom(`[data-test-search-result="${testRealmURL}garden-tips"]`) + .dom(`[data-test-search-result="${testRealmURL}garden-tips.md"]`) .exists('the file is found by its name'); assert .dom(`[data-test-search-result="${testRealmURL}Pet/mango"]`) .doesNotExist('unrelated cards do not match'); }); + test('clicking a file result opens the file in the stack', async function (assert) { + await visitOperatorMode({}); + + await click('[data-test-open-search-field]'); + await fillIn('[data-test-search-field]', 'garden-tips'); + + await waitFor( + `[data-test-search-result="${testRealmURL}garden-tips.md"]`, + ); + await click(`[data-test-search-result="${testRealmURL}garden-tips.md"]`); + + await waitFor(`[data-test-stack-card="${testRealmURL}garden-tips.md"]`); + assert + .dom(`[data-test-stack-card="${testRealmURL}garden-tips.md"]`) + .exists('the file opens as a file stack item under its full URL'); + assert + .dom(`[data-test-stack-card="${testRealmURL}garden-tips.md"]`) + .containsText('Garden Tips', 'the markdown content renders'); + }); + test('specs still match by title', async function (assert) { await visitOperatorMode({}); diff --git a/packages/host/tests/integration/components/operator-mode-card-chooser-test.gts b/packages/host/tests/integration/components/operator-mode-card-chooser-test.gts index a322b5cb21..9d7019f217 100644 --- a/packages/host/tests/integration/components/operator-mode-card-chooser-test.gts +++ b/packages/host/tests/integration/components/operator-mode-card-chooser-test.gts @@ -651,9 +651,7 @@ module('Integration | operator-mode | card chooser', function (hooks) { await fillIn(`[data-test-search-field]`, `friend`); await waitFor('[data-test-item-button]', { count: 0 }); - assert - .dom(`[data-test-search-content-empty]`) - .hasText('No cards available'); + assert.dom(`[data-test-search-content-empty]`).hasText('No results found'); }); test(`can filter by realm after searching in card chooser`, async function (assert) { diff --git a/packages/host/tests/integration/components/overlay-menu-items-test.gts b/packages/host/tests/integration/components/overlay-menu-items-test.gts index baaf42cb0e..577e0e5dbd 100644 --- a/packages/host/tests/integration/components/overlay-menu-items-test.gts +++ b/packages/host/tests/integration/components/overlay-menu-items-test.gts @@ -295,9 +295,10 @@ module('Integration | overlay-menu-items', function (hooks) { }, ); let fileURL = `${testRealmURL}ParentWithFile/notes.md`; - // The overlay strips the file extension from cardId for its data-test - // attributes (removeFileExtension), so data-test-overlay-card lacks `.md`. - let fileCardSelector = `[data-test-overlay-card="${testRealmURL}ParentWithFile/notes"]`; + // Only the card `.json` convention is stripped from overlay data-test + // attributes (removeCardJsonExtension); a file's extension is part of its + // canonical id, so data-test-overlay-card keeps the `.md`. + let fileCardSelector = `[data-test-overlay-card="${fileURL}"]`; await waitFor(`[data-test-card="${fileURL}"]`); await triggerEvent(`[data-test-card="${fileURL}"]`, 'mouseenter'); await waitFor(`${fileCardSelector} [data-test-overlay-more-options]`); From a64287345780be1958ad34581c92d2fcfb6bf12c Mon Sep 17 00:00:00 2001 From: Fadhlan Ridhwanallah Date: Tue, 14 Jul 2026 17:55:44 +0700 Subject: [PATCH 4/6] Update card-chooser pagination assertion for kind-neutral copy The show-more button says "result(s)" now that search results span cards and files; this assertion still expected the old "card(s)" wording. Co-Authored-By: Claude Fable 5 --- .../host/tests/integration/components/card-chooser-test.gts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/host/tests/integration/components/card-chooser-test.gts b/packages/host/tests/integration/components/card-chooser-test.gts index 5a552cd67b..2dab5eea29 100644 --- a/packages/host/tests/integration/components/card-chooser-test.gts +++ b/packages/host/tests/integration/components/card-chooser-test.gts @@ -276,7 +276,7 @@ module('Integration | card-chooser', function (hooks) { .exists('show pagination button for test realm'); assert .dom(`[data-test-realm="${realmName}"] [data-test-show-more-cards]`) - .containsText('Show 1 more card (1 not shown)'); + .containsText('Show 1 more result (1 not shown)'); await click( `[data-test-realm="${realmName}"] [data-test-show-more-cards]`, From 5bf3f5d5ef0aa39d01d40ac310fa2f0a7b02404c Mon Sep 17 00:00:00 2001 From: Fadhlan Ridhwanallah Date: Tue, 14 Jul 2026 22:34:08 +0700 Subject: [PATCH 5/6] Search: fall back to cardTitle when title term-matching The mixed search sheet and recents match a title term against the synthetic `_title` search-doc key, which only exists on rows indexed after it was introduced. On not-yet-reindexed cards `_title` is absent, so a `_title`-only match silently misses them until a realm reindexes. OR in the long-standing `cardTitle` key as a fallback so title search keeps working immediately, independent of index-stamp state. It resolves against the default CardDef type context and is NULL-safe on file rows, so it never throws in the mixed scope and never leaks files; on reindexed rows `_title` equals `cardTitle`, so the branch is redundant, not wrong. Remove once every realm has been reindexed. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../app/utils/card-search/query-builder.ts | 33 +++++++++++++++---- .../unit/card-search-query-builder-test.ts | 21 +++++++----- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/packages/host/app/utils/card-search/query-builder.ts b/packages/host/app/utils/card-search/query-builder.ts index 7760921f2f..632fc46544 100644 --- a/packages/host/app/utils/card-search/query-builder.ts +++ b/packages/host/app/utils/card-search/query-builder.ts @@ -94,14 +94,32 @@ function buildTypeFilter( return { any: codeRefs.map((ref) => ({ type: ref })) }; } -// OR-combine full-text markdown search with a `_title` substring match so -// short prefixes (e.g. "ma" → "mango", "mark") still find results by title -// while longer/natural-language queries tap markdown content via `matches`. -// `_title` is the synthetic key stamped on both card and file rows (a card's -// title, a file's name), so the term matches files by name too. +// A title substring match against `_title` plus, for backward compatibility, +// the legacy `cardTitle` key. +// +// BACKWARD-COMPAT FALLBACK (remove once every realm has been reindexed): +// `_title` is a synthetic key that only exists on rows indexed after it was +// introduced. On not-yet-reindexed cards it is absent, so a `_title`-only match +// would silently miss them. A card's title has long been stamped under the +// `cardTitle` key, so OR that branch in — it resolves against the default +// `CardDef` type context and is NULL-safe on file rows (which have no +// `cardTitle`), so it never throws and never leaks files. On reindexed rows +// `_title` equals `cardTitle`, so the extra branch is redundant, not wrong. +function buildTitleTermFilters(searchTerm: string): Filter[] { + return [ + { contains: { _title: searchTerm } }, + { contains: { cardTitle: searchTerm } }, + ]; +} + +// OR-combine full-text markdown search with a title substring match so short +// prefixes (e.g. "ma" → "mango", "mark") still find results by title while +// longer/natural-language queries tap markdown content via `matches`. `_title` +// is the synthetic key stamped on both card and file rows (a card's title, a +// file's name), so the term matches files by name too. function buildSearchTermFilter(searchTerm: string): Filter { return { - any: [{ matches: searchTerm }, { contains: { _title: searchTerm } }], + any: [{ matches: searchTerm }, ...buildTitleTermFilters(searchTerm)], }; } @@ -242,8 +260,9 @@ export function buildRecentsQuery( ): Query { const typeFilter = buildTypeFilter(selectedTypeIds); const term = searchTerm?.trim() || undefined; + // `_title` plus the legacy `cardTitle` fallback — see `buildTitleTermFilters`. const termFilter: Filter | undefined = term - ? { contains: { _title: term } } + ? { any: buildTitleTermFilters(term) } : undefined; let filters: Filter[]; if (baseFilter) { diff --git a/packages/host/tests/unit/card-search-query-builder-test.ts b/packages/host/tests/unit/card-search-query-builder-test.ts index fc6c820353..b4077f25d0 100644 --- a/packages/host/tests/unit/card-search-query-builder-test.ts +++ b/packages/host/tests/unit/card-search-query-builder-test.ts @@ -29,6 +29,12 @@ const SORT_AZ: SortOption = { // a wire-`scope` concern (see searchScopeForOptions), not a filter. const DEDUP_FILTER: Filter = excludeCardInstanceFileRows(); +// A title term matches `_title` plus the legacy `cardTitle` backward-compat +// fallback (present until every realm is reindexed — see buildTitleTermFilters). +function titleBranches(term: string): Filter[] { + return [{ contains: { _title: term } }, { contains: { cardTitle: term } }]; +} + module('Unit | card-search/query-builder', function () { module('buildSearchQuery', function () { test('empty search key with no baseFilter combines not-spec with the card-json dedup', function (assert) { @@ -46,10 +52,7 @@ module('Unit | card-search/query-builder', function () { every: [ { not: { type: specRef } }, { - any: [ - { matches: 'xylophone' }, - { contains: { _title: 'xylophone' } }, - ], + any: [{ matches: 'xylophone' }, ...titleBranches('xylophone')], }, DEDUP_FILTER, ], @@ -93,7 +96,7 @@ module('Unit | card-search/query-builder', function () { every: [ baseFilter, { - any: [{ matches: 'puppy' }, { contains: { _title: 'puppy' } }], + any: [{ matches: 'puppy' }, ...titleBranches('puppy')], }, ], }, @@ -112,7 +115,7 @@ module('Unit | card-search/query-builder', function () { every: [ baseFilter, { - any: [{ matches: 'mango' }, { contains: { _title: 'mango' } }], + any: [{ matches: 'mango' }, ...titleBranches('mango')], }, DEDUP_FILTER, ], @@ -163,7 +166,7 @@ module('Unit | card-search/query-builder', function () { { not: { type: specRef } }, { type: authorRef }, { - any: [{ matches: 'droid' }, { contains: { _title: 'droid' } }], + any: [{ matches: 'droid' }, ...titleBranches('droid')], }, ], }, @@ -210,13 +213,13 @@ module('Unit | card-search/query-builder', function () { }); }); - test('a term filters recents by _title substring only (no full-text matches)', function (assert) { + test('a term filters recents by title substring only (no full-text matches)', function (assert) { let query = buildRecentsQuery('mango', SORT_AZ); assert.deepEqual(query, { filter: { every: [ { not: { type: specRef } }, - { contains: { _title: 'mango' } }, + { any: titleBranches('mango') }, DEDUP_FILTER, ], }, From 27b95988ae9c4f8d92c5cb707b7334f315c001b6 Mon Sep 17 00:00:00 2001 From: Fadhlan Ridhwanallah Date: Tue, 14 Jul 2026 23:39:55 +0700 Subject: [PATCH 6/6] Search review hardening: BaseDef pins cards scope; drop TableValuedEach - store.search: a BaseDef type ref no longer counts as kind-selecting (it matches both row kinds), so such queries pin scope 'cards' like untyped ones; integration test added - expression.ts: remove the consumer-less TableValuedEach node and its render branch; drop the sqlite adjustSQL `case jsonb_typeof` strip that only its SQL used - handle-search: fold an explicit scope 'all' into the same cache key as an absent scope - docs + search-doc-keys: _cardType documented as a card-only synthetic, scope vs BaseDef-anchor guidance, and `_`-prefixed top-level search-doc keys reserved for synthetics Co-Authored-By: Claude Fable 5 --- docs/search.md | 7 +++-- packages/host/app/lib/sqlite-adapter.ts | 1 - packages/host/app/services/store.ts | 27 ++++++++++++------- .../tests/integration/store-search-test.gts | 20 ++++++++++++++ .../realm-server/handlers/handle-search.ts | 9 ++++--- packages/runtime-common/expression.ts | 27 ------------------- packages/runtime-common/index-query-engine.ts | 1 - packages/runtime-common/search-doc-keys.ts | 8 ++++++ 8 files changed, 56 insertions(+), 44 deletions(-) diff --git a/docs/search.md b/docs/search.md index 80626df908..b94fe78f37 100644 --- a/docs/search.md +++ b/docs/search.md @@ -253,8 +253,11 @@ Cards and files carry non-overlapping fields, so a filter that names a card fiel - `matches` — full-text search over both kinds' content. - `_title` — the row's display title (a card's `cardTitle`, a file's name); usable in `contains` and `sort`. -- `_cardType` — the card type's display name. - `_isCardInstanceFile` — stamped `true` only on a card `.json`'s file row; the dedup key (`eq: false` keeps everything else). - `{ type: }` — BaseDef terminates both kinds' type chains, so a BaseDef-anchored type filter matches every row and composes with other conditions. -Any other field key narrows the result to a single kind. +Any other field key narrows the result to a single kind. `_cardType` (the card type's display name) is a card-only synthetic — file rows never carry it — which makes it a kind discriminator in mixed queries (`not: { eq: { _cardType: null } }` is a cards-only filter) but a poor mixed sort key: every file sinks to the NULLS-LAST tail. + +When does a BaseDef anchor beat `scope: 'all'`? `scope` is query-global — it pins the row kind for the whole request — while `{ type: }` is a filter node, composable inside the tree: one branch of an `any:` can span both kinds while a sibling branch is card-typed, or it can intersect with another type condition under `every:`. If the whole query wants both kinds, `scope: 'all'` is the spelling; the BaseDef anchor is for kind-spanning _within_ a filter composition. + +`_`-prefixed top-level keys are reserved for these synthetics — the query engine resolves them by name before any field-definition lookup, so a user-defined field with a colliding name would silently get the synthetic's semantics (see `runtime-common/search-doc-keys.ts`). diff --git a/packages/host/app/lib/sqlite-adapter.ts b/packages/host/app/lib/sqlite-adapter.ts index be19d72c02..a335780527 100644 --- a/packages/host/app/lib/sqlite-adapter.ts +++ b/packages/host/app/lib/sqlite-adapter.ts @@ -422,7 +422,6 @@ export default class SQLiteAdapter implements DBAdapter { } return `ON CONFLICT (${pkColumns})`; }) - .replace(/\(case jsonb_typeof(\([^)]*\)) when 'array' .+ end\)/, '$1') .split('ANY_VALUE(') .reduce((acc, segment, index) => { // SQLite has no ANY_VALUE; every non-grouped select column is diff --git a/packages/host/app/services/store.ts b/packages/host/app/services/store.ts index 1b6517e8e9..ea1de30fd5 100644 --- a/packages/host/app/services/store.ts +++ b/packages/host/app/services/store.ts @@ -21,6 +21,7 @@ import { TrackedObject, TrackedMap } from 'tracked-built-ins'; import { baseFileRef, + baseRef, CardError, hasExecutableExtension, isCardError, @@ -1270,18 +1271,26 @@ export default class StoreService extends Service implements StoreInterface { query: Query, realms: string[], ): Promise { - // Search spans card instances and files. A query with a positive type ref - // already selects a kind (a card type -> instances, a FileDef type -> - // files), so it passes through with the default 'all' scope and its filter - // discriminates. An otherwise-unscoped query is pinned to 'cards' so the - // common "search for cards" case doesn't surface a card's dual-indexed - // `.json` file row (or plain files) — the choke point that replaces the - // former per-call-site card anchor, while leaving file/typed searches - // (e.g. SearchResource's file-meta queries) untouched. + // Search spans card instances and files. A query with a positive + // *concrete* type ref already selects a kind (a card type -> instances, a + // FileDef type -> files), so it passes through with the default 'all' + // scope and its filter discriminates. An otherwise-unscoped query is + // pinned to 'cards' so the common "search for cards" case doesn't surface + // a card's dual-indexed `.json` file row (or plain files) — the choke + // point that replaces the former per-call-site card anchor, while leaving + // file/typed searches (e.g. SearchResource's file-meta queries) untouched. + // + // A BaseDef ref is *not* kind-selecting — it terminates both kinds' type + // chains, so it matches every row — and is pinned to 'cards' like an + // untyped query. Known gap: a mixed `any:` whose one branch is card-typed + // and another untyped counts as positively typed, so its untyped branch + // can still match file rows in 'all' scope; no caller composes that shape + // today. let typeRefs = query.filter ? getTypeRefsFromFilter(query.filter) : undefined; - let hasPositiveType = typeRefs?.some((r) => !r.negated) ?? false; + let hasPositiveType = + typeRefs?.some((r) => !r.negated && !isEqual(r.ref, baseRef)) ?? false; let scope: SearchEntryScope | undefined = hasPositiveType ? undefined : 'cards'; diff --git a/packages/host/tests/integration/store-search-test.gts b/packages/host/tests/integration/store-search-test.gts index 46528ff53d..5bbedb09d8 100644 --- a/packages/host/tests/integration/store-search-test.gts +++ b/packages/host/tests/integration/store-search-test.gts @@ -7,6 +7,7 @@ import { module, test } from 'qunit'; import { isCardInstance, baseFileRef, + baseRef, isCardResource, isFileMetaResource, type Loader, @@ -101,6 +102,25 @@ module('Integration | store search public API', function (hooks) { ); }); + test('search with a BaseDef type filter stays pinned to card instances', async function (assert) { + // BaseDef terminates both kinds' type chains, so as a filter it matches + // file rows too — but it selects no kind, so `search` pins the 'cards' + // scope for it just like an untyped query. + let instances = await storeService.search( + { filter: { type: baseRef } }, + [testRealmURL], + ); + + assert.true( + instances.every((instance) => isCardInstance(instance)), + 'every result is a card instance — no plain files, no dual-indexed `.json` file rows', + ); + assert.deepEqual( + instances.map((instance) => (instance as any).title).sort(), + ['Mango', 'Van Gogh'], + ); + }); + test('search with includeMeta returns the single { instances, meta } shape', async function (assert) { let result = await storeService.search( { filter: { type: bookRef } }, diff --git a/packages/realm-server/handlers/handle-search.ts b/packages/realm-server/handlers/handle-search.ts index 096783e8f0..4fed2a314e 100644 --- a/packages/realm-server/handlers/handle-search.ts +++ b/packages/realm-server/handlers/handle-search.ts @@ -129,10 +129,11 @@ export default function handleSearch(opts: { } // `scope` changes which row kinds the response contains, so it must key the // cache — otherwise a `scope: 'cards'` and a `scope: 'all'` request for the - // same query would collide on one ETag/body. Fold only when set (absent == - // the default 'all', so keying `undefined` would not fragment but omitting - // it keeps the key identical to pre-scope requests). - if (parsed.scope) { + // same query would collide on one ETag/body. An explicit `'all'` folds the + // same as an absent scope (both mean the default), so the two spellings + // share one cache entry/ETag — which also keeps the key identical to + // pre-scope requests. + if (parsed.scope && parsed.scope !== 'all') { cacheKeyOpts.scope = parsed.scope; } diff --git a/packages/runtime-common/expression.ts b/packages/runtime-common/expression.ts index 901e8fd97b..9a2db4bb7c 100644 --- a/packages/runtime-common/expression.ts +++ b/packages/runtime-common/expression.ts @@ -8,7 +8,6 @@ import type { CodeRef, DBAdapter, TypeCoercion } from './index.ts'; export type Expression = ( | string | Param - | TableValuedEach | TableValuedTree | JsonContains | TypesContains @@ -59,11 +58,6 @@ export interface FieldValue { kind: 'field-value'; } -export interface TableValuedEach { - kind: 'table-valued-each'; - column: string; -} - export interface TableValuedTree { kind: 'table-valued-tree'; column: string; @@ -122,7 +116,6 @@ export type CardExpression = ( | string | Param | DBSpecificExpression - | TableValuedEach | TableValuedTree | JsonContains | TypesContains @@ -201,13 +194,6 @@ export function isDbExpression( ); } -export function tableValuedEach(column: string): TableValuedEach { - return { - kind: 'table-valued-each', - column, - }; -} - export function tableValuedTree( column: string, rootPath: string, @@ -528,19 +514,6 @@ export function expressionToSql( }); } return `${name}.${treeColumn}`; - } else if (element.kind === 'table-valued-each') { - let { column } = element; - let key = `each_${column}`; - let { name } = tableValuedFunctions.get(key) ?? {}; - if (!name) { - name = `${column}${nonce++}_array_element`; - - tableValuedFunctions.set(key, { - name, - fn: `jsonb_array_elements_text(case jsonb_typeof(${column}) when 'array' then ${column} else '[]' end) as ${name}`, - }); - } - return name; } else if (element.kind === 'json-contains') { // Render the containment of `column` by {segments: value}. Both branches // re-use renderElement so binds are pushed in left-to-right order. diff --git a/packages/runtime-common/index-query-engine.ts b/packages/runtime-common/index-query-engine.ts index 2ad322ba53..9795b2a478 100644 --- a/packages/runtime-common/index-query-engine.ts +++ b/packages/runtime-common/index-query-engine.ts @@ -1507,7 +1507,6 @@ export class IndexQueryEngine { isParam(element) || isDbExpression(element) || typeof element === 'string' || - element.kind === 'table-valued-each' || element.kind === 'table-valued-tree' || element.kind === 'json-contains' || element.kind === 'types-contains' diff --git a/packages/runtime-common/search-doc-keys.ts b/packages/runtime-common/search-doc-keys.ts index 4d7f1376b9..35cd2146e7 100644 --- a/packages/runtime-common/search-doc-keys.ts +++ b/packages/runtime-common/search-doc-keys.ts @@ -6,6 +6,14 @@ // and files can be filtered/sorted through one query. Keeping the spellings in // one module means a rename is a single-file change rather than a cross-package // grep. +// +// `_`-prefixed top-level search-doc keys are reserved for these synthetics: +// the query engine special-cases them by leaf name before any field-definition +// lookup (e.g. `_isCardInstanceFile` compiles to an existence test, not a +// value comparison), and the client-side matcher treats any unshimmed +// `_`-prefixed key as server-only. A user-defined field whose name collides +// with one of these at the search-doc top level would silently get the +// synthetic's semantics instead of its own value's. // The row's display title under a kind-neutral key (a card's is its // `cardTitle`, a file's is its name), so one mixed query can substring-match