diff --git a/.changeset/moody-socks-look.md b/.changeset/moody-socks-look.md new file mode 100644 index 00000000..20e083b4 --- /dev/null +++ b/.changeset/moody-socks-look.md @@ -0,0 +1,5 @@ +--- +'@fuzdev/fuz_code': patch +--- + +feat: add `CodeTextarea` diff --git a/package-lock.json b/package-lock.json index afd34422..5da74b2d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,7 @@ "devDependencies": { "@changesets/changelog-git": "^0.2.1", "@fuzdev/blake3_wasm": "^0.1.1", - "@fuzdev/fuz_css": "^0.63.0", + "@fuzdev/fuz_css": "^0.63.2", "@fuzdev/fuz_ui": "^0.205.0", "@fuzdev/fuz_util": "^0.65.1", "@fuzdev/gro": "^0.204.0", @@ -759,9 +759,9 @@ } }, "node_modules/@fuzdev/fuz_css": { - "version": "0.63.0", - "resolved": "https://registry.npmjs.org/@fuzdev/fuz_css/-/fuz_css-0.63.0.tgz", - "integrity": "sha512-DuSlmlY9e0taTW4yUU+BHU79uKvIJSV2Uf+ZwmGU4AHOr67jp0iJbDlW14bGmOL2K/NPlJ/fFhDrrn2dshObAw==", + "version": "0.63.2", + "resolved": "https://registry.npmjs.org/@fuzdev/fuz_css/-/fuz_css-0.63.2.tgz", + "integrity": "sha512-df2j970oRPHPJVGoMIhkHltRI6ZMTrhVoBEPfek8hY8RiFMnJMW8cFq+X6oTSGpdfo1COQpjHbeLZemd74z+mg==", "dev": true, "license": "MIT", "engines": { diff --git a/package.json b/package.json index 58c0b447..0f05a3e3 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "devDependencies": { "@changesets/changelog-git": "^0.2.1", "@fuzdev/blake3_wasm": "^0.1.1", - "@fuzdev/fuz_css": "^0.63.0", + "@fuzdev/fuz_css": "^0.63.2", "@fuzdev/fuz_ui": "^0.205.0", "@fuzdev/fuz_util": "^0.65.1", "@fuzdev/gro": "^0.204.0", diff --git a/src/lib/CodeHighlight.svelte b/src/lib/CodeHighlight.svelte index 3672be7d..fdfe591b 100644 --- a/src/lib/CodeHighlight.svelte +++ b/src/lib/CodeHighlight.svelte @@ -6,18 +6,13 @@ * Requires importing `theme_highlight.css` instead of `theme.css`. */ - import {onDestroy, type Snippet} from 'svelte'; - import {DEV} from 'esm-env'; + import type {Snippet} from 'svelte'; import type {SvelteHTMLElements} from 'svelte/elements'; import {syntax_styler_global} from './syntax_styler_global.js'; import type {SyntaxStyler, SyntaxGrammar} from './syntax_styler.js'; - import {tokenize_syntax} from './tokenize_syntax.js'; - import { - HighlightManager, - supports_css_highlight_api, - type HighlightMode, - } from './highlight_manager.js'; + import {supports_css_highlight_api, type HighlightMode} from './highlight_manager.js'; + import {create_range_highlighting} from './range_highlighting.svelte.js'; const { content, @@ -121,58 +116,27 @@ const supports_ranges = supports_css_highlight_api(); - const highlight_manager = supports_ranges ? new HighlightManager() : null; - const use_ranges = $derived(supports_ranges && (mode === 'ranges' || mode === 'auto')); - const language_supported = $derived(lang !== null && !!syntax_styler.langs[lang]); - - const highlighting_disabled = $derived(lang === null || (!language_supported && !grammar)); - - // DEV-only validation warnings - if (DEV) { - $effect(() => { - if (lang && !language_supported && !grammar) { - const langs = Object.keys(syntax_styler.langs).join(', '); - // eslint-disable-next-line no-console - console.error( - `[CodeHighlight] Language "${lang}" is not supported and no custom grammar provided. ` + - `Highlighting disabled. Supported: ${langs}`, - ); - } - }); - } + const rh = create_range_highlighting({ + element: () => code_element, + text: () => content, + enabled: () => use_ranges, + lang: () => lang, + grammar: () => grammar, + syntax_styler: () => syntax_styler, + dev_label: 'CodeHighlight', + }); // Generate HTML markup for syntax highlighting in non-range mode const html_content = $derived.by(() => { - if (use_ranges || !content || highlighting_disabled) { + if (use_ranges || !content || rh.highlighting_disabled) { return ''; } return syntax_styler.stylize(content, lang!, grammar); // ! is safe bc of the `highlighting_disabled` calculation }); - // Apply highlights for range mode - if (highlight_manager) { - $effect(() => { - if (!code_element || !content || !use_ranges || highlighting_disabled) { - highlight_manager.clear_element_ranges(); - return; - } - - // Get tokens from syntax styler - const tokens = tokenize_syntax(content, grammar || syntax_styler.get_lang(lang!)); // ! is safe bc of the `highlighting_disabled` calculation - - // Apply highlights - highlight_manager.highlight_from_syntax_tokens(code_element, tokens); - }); - } - - onDestroy(() => { - highlight_manager?.destroy(); - }); - - // TODO use intersect attachment from fuz_ui to optimize ranges // TODO do syntax styling at compile-time in the normal case, and don't import these at runtime // TODO @html making me nervous @@ -182,7 +146,7 @@ {#if use_ranges && children}{@render children( content, - )}{:else if use_ranges || highlighting_disabled}{content}{:else if children}{@render children( + )}{:else if use_ranges || rh.highlighting_disabled}{content}{:else if children}{@render children( html_content, )}{:else}{@html html_content}{/if} diff --git a/src/lib/CodeTextarea.svelte b/src/lib/CodeTextarea.svelte new file mode 100644 index 00000000..f44bef75 --- /dev/null +++ b/src/lib/CodeTextarea.svelte @@ -0,0 +1,149 @@ + + +
+ + +
+ + diff --git a/src/lib/highlight_manager.ts b/src/lib/highlight_manager.ts index 477963c2..c3fb110b 100644 --- a/src/lib/highlight_manager.ts +++ b/src/lib/highlight_manager.ts @@ -1,3 +1,5 @@ +import {DEV} from 'esm-env'; + import type {SyntaxTokenStream} from './syntax_token.js'; import {highlight_priorities} from './highlight_priorities.js'; @@ -10,10 +12,55 @@ export const supports_css_highlight_api = (): boolean => !!(globalThis.CSS?.highlights && globalThis.Highlight); /** - * Manages CSS Custom Highlight API ranges for a single element. - * Tracks ranges per element and only removes its own ranges when clearing. + * How a manager builds ranges for the Highlight API. * - * **Experimental** — limited browser support. Use `Code.svelte` for production. + * `StaticRange` is preferred: it's an immutable snapshot the browser does *not* + * track across DOM mutations. Since highlights are rebuilt wholesale whenever + * content changes, liveness buys nothing and only costs per-mutation boundary + * bookkeeping and paint work — the main efficiency (and Safari-stability) lever. + * Falls back to a live `Range` where `StaticRange` is unavailable. + */ +type RangeKind = 'static' | 'live'; + +const detect_range_kind = (): RangeKind => + typeof globalThis.StaticRange === 'function' ? 'static' : 'live'; + +/** + * Finds the first text node child of `element`, or `null` if there is none. + * + * The text node might not be `firstChild` because frameworks (e.g. Svelte) can + * insert comment/anchor nodes around it. + */ +const find_text_node = (element: Element): Node | null => { + for (const node of element.childNodes) { + if (node.nodeType === Node.TEXT_NODE) return node; + } + return null; +}; + +const has_tokens = (tokens: SyntaxTokenStream): boolean => + tokens.some((t) => typeof t !== 'string'); + +const push_range = ( + ranges_by_name: Map>, + name: string, + range: AbstractRange, +): void => { + const existing = ranges_by_name.get(name); + if (existing) { + existing.push(range); + } else { + ranges_by_name.set(name, [range]); + } +}; + +/** + * Manages CSS Custom Highlight API ranges for a single element's text node. + * Tracks ranges per element and only removes its own ranges when clearing, + * cooperating with other managers that share the global `CSS.highlights` registry. + * + * **Experimental** — limited browser support. Use `Code.svelte` for production + * block code; this powers the experimental `CodeHighlight` and `CodeTextarea`. * * @example * ```ts @@ -22,62 +69,94 @@ export const supports_css_highlight_api = (): boolean => * ``` */ export class HighlightManager { - element_ranges: Map>; + /** + * This manager's ranges, keyed by prefixed highlight name (e.g. `token_keyword`). + * A single range object may be shared across several names (a token type plus + * its aliases), since one range can belong to multiple `Highlight` sets. + */ + element_ranges: Map>; + + #range_kind: RangeKind; constructor() { if (!supports_css_highlight_api()) { throw Error('CSS Highlights API not supported'); } this.element_ranges = new Map(); + this.#range_kind = detect_range_kind(); } /** - * Highlights from a `SyntaxTokenStream` produced by `tokenize_syntax`. + * Highlights `element`'s text node from a `SyntaxTokenStream` produced by + * `tokenize_syntax`. Clears this manager's previous ranges first. + * + * In production this never throws on a tokenizer/DOM mismatch: out-of-bounds + * tokens are clamped and a missing text node is a no-op. In DEV the same + * conditions throw loudly to surface grammar bugs. */ highlight_from_syntax_tokens(element: Element, tokens: SyntaxTokenStream): void { - // Find the text node (it might not be firstChild due to Svelte comment nodes) - let text_node: Node | null = null; - for (const node of element.childNodes) { - if (node.nodeType === Node.TEXT_NODE) { - text_node = node; - break; - } - } + this.clear_element_ranges(); + const text_node = find_text_node(element); if (!text_node) { - throw new Error('no text node to highlight'); + if (has_tokens(tokens)) { + if (DEV) { + throw new Error('no text node to highlight'); + } else { + // eslint-disable-next-line no-console + console.error('[HighlightManager] tokens present but no text node to highlight'); + } + } + return; } - this.clear_element_ranges(); + try { + this.#apply(text_node, tokens); + } catch (err) { + // some engines may reject `StaticRange` in `Highlight.add` -- fall back to + // live `Range` once rather than letting the throw escape into the effect + if (this.#range_kind === 'static') { + this.#range_kind = 'live'; + this.clear_element_ranges(); // undo any partial application + this.#apply(text_node, tokens); + } else { + throw err; + } + } + } - const ranges_by_type: Map> = new Map(); - const final_pos = this.#create_all_ranges(tokens, text_node, ranges_by_type, 0); + #apply(text_node: Node, tokens: SyntaxTokenStream): void { + const ranges_by_name: Map> = new Map(); + const final_pos = this.#collect_ranges(tokens, text_node, ranges_by_name, 0); - // Validate that token positions matched text node length - const text_length = text_node.textContent?.length ?? 0; - if (final_pos !== text_length) { - throw new Error( - `Token stream length mismatch: tokens covered ${final_pos} chars but text node has ${text_length} chars`, - ); + if (DEV) { + const text_length = text_node.textContent?.length ?? 0; + if (final_pos !== text_length) { + throw new Error( + `Token stream length mismatch: tokens covered ${final_pos} chars but text node has ${text_length} chars`, + ); + } } - // Apply highlights - for (const [type, ranges] of ranges_by_type) { - const prefixed_type = `token_${type}`; - // Track ranges for this element - this.element_ranges.set(prefixed_type, ranges); + // TODO: cross-instance coupling -- all managers share one global `Highlight` + // per token type, so re-highlighting one element (e.g. a textarea on every + // keystroke) mutates highlights that also hold ranges from every other code + // block on the page, forcing the browser to re-evaluate the shared set. + // Isolating per-instance needs unique highlight names + runtime-injected + // `::highlight()` CSS (≈50 token types × N instances), trading the static + // theme file for generated CSS. Only worth it with many concurrently-updating + // instances; revisit if profiling shows it. + for (const [name, ranges] of ranges_by_name) { + this.element_ranges.set(name, ranges); - // Get or create the shared highlight - let highlight = CSS.highlights.get(prefixed_type); + let highlight = CSS.highlights.get(name); if (!highlight) { highlight = new Highlight(); - // Set priority based on CSS cascade order (higher = later in CSS = wins) - highlight.priority = - highlight_priorities[prefixed_type as keyof typeof highlight_priorities] ?? 0; - CSS.highlights.set(prefixed_type, highlight); + // priority follows CSS cascade order (higher = later in CSS = wins) + highlight.priority = highlight_priorities[name as keyof typeof highlight_priorities] ?? 0; + CSS.highlights.set(name, highlight); } - // Add all ranges to the highlight for (const range of ranges) { highlight.add(range); } @@ -85,14 +164,14 @@ export class HighlightManager { } /** - * Clears only this element's ranges from highlights. + * Clears only this manager's ranges from the shared highlights. Defensive: + * a highlight may already be gone (e.g. another manager removed the last + * range, or HMR reset the registry), which is a valid state, not an error. */ clear_element_ranges(): void { for (const [name, ranges] of this.element_ranges) { const highlight = CSS.highlights.get(name); - if (!highlight) { - throw new Error('Expected to find CSS highlight: ' + name); - } + if (!highlight) continue; for (const range of ranges) { highlight.delete(range); @@ -110,13 +189,29 @@ export class HighlightManager { this.clear_element_ranges(); } + #make_range(text_node: Node, start: number, end: number): AbstractRange { + if (this.#range_kind === 'static') { + return new StaticRange({ + startContainer: text_node, + startOffset: start, + endContainer: text_node, + endOffset: end, + }); + } + const range = new Range(); + range.setStart(text_node, start); + range.setEnd(text_node, end); + return range; + } + /** - * Creates ranges for all tokens in the tree. + * Walks the token tree, collecting one range per non-empty token (shared + * across its type and aliases). Returns the end position covered. */ - #create_all_ranges( + #collect_ranges( tokens: SyntaxTokenStream, text_node: Node, - ranges_by_type: Map>, + ranges_by_name: Map>, offset: number, ): number { const text_length = text_node.textContent?.length ?? 0; @@ -128,61 +223,36 @@ export class HighlightManager { continue; } - const length = token.length; - const end_pos = pos + length; + const end_pos = pos + token.length; - // Validate positions are within text node bounds before creating ranges - if (end_pos > text_length) { + if (DEV && end_pos > text_length) { throw new Error( `Token ${token.type} extends beyond text node: position ${end_pos} > length ${text_length}`, ); } - try { - const range = new Range(); - range.setStart(text_node, pos); - range.setEnd(text_node, end_pos); - - // Add range for the token type - const type = token.type; - if (!ranges_by_type.has(type)) { - ranges_by_type.set(type, []); - } - ranges_by_type.get(type)!.push(range); - - // Also add range for any aliases (alias is always an array) + // production-safe: clamp rather than throw on a tokenizer edge case + const safe_end = end_pos > text_length ? text_length : end_pos; + if (safe_end > pos) { + // one range shared across the token type and all its aliases -- + // the same range object can belong to multiple `Highlight` sets + const range = this.#make_range(text_node, pos, safe_end); + push_range(ranges_by_name, `token_${token.type}`, range); for (const alias of token.alias) { - if (!ranges_by_type.has(alias)) { - ranges_by_type.set(alias, []); - } - // Create a new range for each alias (ranges can't be reused) - const alias_range = new Range(); - alias_range.setStart(text_node, pos); - alias_range.setEnd(text_node, end_pos); - ranges_by_type.get(alias)!.push(alias_range); + push_range(ranges_by_name, `token_${alias}`, range); } - } catch (e) { - throw new Error(`Failed to create range for ${token.type}: ${e}`); } - // Process nested tokens if (Array.isArray(token.content)) { - const actual_end_pos = this.#create_all_ranges( - token.content, - text_node, - ranges_by_type, - pos, - ); - // Validate that nested tokens match the parent token's claimed length - if (actual_end_pos !== end_pos) { + const nested_end = this.#collect_ranges(token.content, text_node, ranges_by_name, pos); + if (DEV && nested_end !== end_pos) { throw new Error( - `Token ${token.type} length mismatch: claimed ${length} chars (${pos}-${end_pos}) but nested content covered ${actual_end_pos - pos} chars (${pos}-${actual_end_pos})`, + `Token ${token.type} length mismatch: claimed ${token.length} chars (${pos}-${end_pos}) but nested content covered ${nested_end - pos} chars (${pos}-${nested_end})`, ); } - pos = actual_end_pos; - } else { - pos = end_pos; } + + pos = end_pos; } return pos; diff --git a/src/lib/range_highlighting.svelte.ts b/src/lib/range_highlighting.svelte.ts new file mode 100644 index 00000000..4b6f2a00 --- /dev/null +++ b/src/lib/range_highlighting.svelte.ts @@ -0,0 +1,100 @@ +import {onDestroy} from 'svelte'; +import {DEV} from 'esm-env'; + +import type {SyntaxStyler, SyntaxGrammar} from './syntax_styler.js'; +import {HighlightManager, supports_css_highlight_api} from './highlight_manager.js'; + +/** + * Reactive inputs for `create_range_highlighting`. All values are getters so the + * helper can track the consuming component's reactive state across the call + * boundary (the Svelte 5 getter-injection pattern). + */ +export interface RangeHighlightingOptions { + /** The element whose first text node receives the highlight ranges. */ + element: () => Element | undefined; + /** + * The text to tokenize. Must match the element's text node exactly (e.g. a + * textarea backdrop includes its trailing newline here too). + */ + text: () => string; + /** Language id; `null` disables highlighting. */ + lang: () => string | null; + /** Optional custom grammar; takes precedence over `lang` for tokenization. */ + grammar: () => SyntaxGrammar | undefined; + /** The syntax styler whose registered grammars back `lang` lookups. */ + syntax_styler: () => SyntaxStyler; + /** Extra gate — ranges are only applied when this returns true. Defaults to always-on. */ + enabled?: () => boolean; + /** Component name used in DEV warnings. */ + dev_label: string; +} + +/** Reactive outputs from `create_range_highlighting`. */ +export interface RangeHighlighting { + readonly highlighting_disabled: boolean; +} + +/** + * Wires up CSS Custom Highlight API range highlighting for a single element's + * text node, shared by `CodeHighlight` and `CodeTextarea`. Creates a + * `HighlightManager`, memoizes tokenization, applies/clears ranges in an effect, + * emits DEV warnings for unsupported languages, and tears down on destroy. + * + * Must be called during component initialization (it uses `$effect`/`onDestroy`). + */ +export const create_range_highlighting = (options: RangeHighlightingOptions): RangeHighlighting => { + const manager = supports_css_highlight_api() ? new HighlightManager() : null; + const is_enabled = options.enabled ?? (() => true); + + const language_supported = $derived( + options.lang() !== null && !!options.syntax_styler().langs[options.lang()!], + ); + const highlighting_disabled = $derived( + options.lang() === null || (!language_supported && !options.grammar()), + ); + + // tokenize once per (text, grammar, lang) change -- memoized so unrelated + // reactivity doesn't trigger a full re-tokenization + const range_tokens = $derived.by(() => { + if (!manager || !is_enabled() || highlighting_disabled) return null; + const text = options.text(); + if (!text) return null; + // route through the styler so `before_tokenize`/`after_tokenize` hooks run, + // matching `stylize`'s HTML path; `grammar` undefined falls back to the + // registered lang grammar (`! safe bc of `highlighting_disabled`) + return options.syntax_styler().tokenize(text, options.lang()!, options.grammar()); + }); + + if (manager) { + $effect(() => { + const element = options.element(); + if (!element || !range_tokens) { + manager.clear_element_ranges(); + return; + } + manager.highlight_from_syntax_tokens(element, range_tokens); + }); + } + + if (DEV) { + $effect(() => { + // a lang was requested but we can't highlight it (unknown id, no grammar) + if (options.lang() && highlighting_disabled) { + const langs = Object.keys(options.syntax_styler().langs).join(', '); + // eslint-disable-next-line no-console + console.error( + `[${options.dev_label}] Language "${options.lang()}" is not supported and no custom grammar provided. ` + + `Highlighting disabled. Supported: ${langs}`, + ); + } + }); + } + + onDestroy(() => manager?.destroy()); + + return { + get highlighting_disabled() { + return highlighting_disabled; + }, + }; +}; diff --git a/src/lib/syntax_styler.ts b/src/lib/syntax_styler.ts index 729c38f0..a48211c4 100644 --- a/src/lib/syntax_styler.ts +++ b/src/lib/syntax_styler.ts @@ -3,6 +3,13 @@ import {tokenize_syntax} from './tokenize_syntax.js'; export type AddSyntaxGrammar = (syntax_styler: SyntaxStyler) => void; +/** + * Maps a matched `&`, `<`, or non-breaking space in text content to its + * HTML-safe form. Used as the replacer in `stringify_token` for leaf strings + * (non-breaking spaces are normalized to a regular space). + */ +const escape_text_char = (ch: string): string => (ch === '&' ? '&' : ch === '<' ? '<' : ' '); + /** * Based on Prism (https://github.com/PrismJS/prism) * by Lea Verou (https://lea.verou.me/) @@ -127,7 +134,42 @@ export class SyntaxStyler { lang: string, grammar: SyntaxGrammar | undefined = this.get_lang(lang), ): string { - var ctx: HookBeforeTokenizeCallbackContext = { + // stringify with the post-hook `lang`, which a `before_tokenize` hook may + // have rewritten (it flows into each token's `wrap` hook context) + const c = this.#tokenize_hooked(text, lang, grammar); + return this.stringify_token(c.tokens, c.lang); + } + + /** + * Tokenizes `text` into a `SyntaxTokenStream`, running the `before_tokenize` + * and `after_tokenize` hooks. This is the tokenization half of `stylize` — use + * it when you need the token stream itself (e.g. CSS Custom Highlight API range + * highlighting) rather than HTML. + * + * @param text - source to tokenize + * @param lang - language identifier; passed to the tokenize hooks + * @param grammar - grammar to tokenize with; defaults to `this.get_lang(lang)` + * @returns the resulting token stream + */ + tokenize( + text: string, + lang: string, + grammar: SyntaxGrammar | undefined = this.get_lang(lang), + ): SyntaxTokenStream { + return this.#tokenize_hooked(text, lang, grammar).tokens; + } + + /** + * Runs `before_tokenize` → `tokenize_syntax` → `after_tokenize`, returning the + * resolved context. Shared by `stylize` (which also needs the post-hook `lang`) + * and `tokenize` (which only needs `tokens`). + */ + #tokenize_hooked( + text: string, + lang: string, + grammar: SyntaxGrammar, + ): HookAfterTokenizeCallbackContext { + const ctx: HookBeforeTokenizeCallbackContext = { code: text, grammar, lang, @@ -137,7 +179,7 @@ export class SyntaxStyler { const c = ctx as any as HookAfterTokenizeCallbackContext; c.tokens = tokenize_syntax(c.code, c.grammar); this.run_hook_after_tokenize(c); - return this.stringify_token(c.tokens, c.lang); + return c; } /** @@ -263,10 +305,9 @@ export class SyntaxStyler { */ stringify_token(o: string | SyntaxToken | SyntaxTokenStream, lang: string): string { if (typeof o === 'string') { - return o - .replace(/&/g, '&') - .replace(/ with no + // attributes, so skip the per-token context object and hook dispatch + if (this.hooks_wrap.length === 0) { + return '' + content + ''; + } + var ctx: HookWrapCallbackContext = { type: o.type, - content: this.stringify_token(o.content, lang), + content, tag: 'span', - classes: [`token_${o.type}`], + classes: classes.split(' '), attributes: {}, lang, }; - var aliases = o.alias; - // alias is always an array after normalization - for (const a of aliases) { - ctx.classes.push(`token_${a}`); - } - this.run_hook_wrap(ctx); var attributes = ''; diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index 40fd757c..c03fd8a9 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -63,6 +63,7 @@ import Code from '@fuzdev/fuz_code/Code.svelte';`}
samples + textarea benchmark about
diff --git a/src/routes/textarea/+page.svelte b/src/routes/textarea/+page.svelte new file mode 100644 index 00000000..a126b92a --- /dev/null +++ b/src/routes/textarea/+page.svelte @@ -0,0 +1,62 @@ + + +
+
+ +
+ +
+

Editable highlighted textarea (experimental)

+ + +
+ {#each sample_langs as l (l)} + + {/each} +
+ + +
+ +
+
diff --git a/src/test/highlight_manager.test.ts b/src/test/highlight_manager.test.ts index a27e0ed1..5a1bf919 100644 --- a/src/test/highlight_manager.test.ts +++ b/src/test/highlight_manager.test.ts @@ -80,13 +80,24 @@ describe('initialization', () => { }); describe('DOM element handling', () => { - test('throws if element has no text node', () => { + test('no-op when no text node and no tokens', () => { const manager = new HighlightManager(); const element = { childNodes: [], // No children } as unknown as Element; const tokens: SyntaxTokenStream = []; + assert.doesNotThrow(() => manager.highlight_from_syntax_tokens(element, tokens)); + assert.equal(manager.element_ranges.size, 0); + }); + + test('throws in DEV when tokens present but no text node', () => { + const manager = new HighlightManager(); + const element = { + childNodes: [], // No children + } as unknown as Element; + const tokens: SyntaxTokenStream = [new SyntaxToken('keyword', 'const', undefined, 'const')]; + assert.throws( () => manager.highlight_from_syntax_tokens(element, tokens), /no text node to highlight/, @@ -280,7 +291,7 @@ describe('range creation', () => { assert.equal(part3_ranges[0]!.endOffset, 5); }); - test('creates separate ranges for aliases', () => { + test('shares one range across a token type and its aliases', () => { const manager = new HighlightManager(); const element = create_code_element('function'); @@ -295,10 +306,65 @@ describe('range creation', () => { assert.ok(manager.element_ranges.has('token_reserved')); assert.ok(manager.element_ranges.has('token_special_keyword')); - // Each should have one range + // Each name has one range assert.equal(manager.element_ranges.get('token_keyword')!.length, 1); assert.equal(manager.element_ranges.get('token_reserved')!.length, 1); assert.equal(manager.element_ranges.get('token_special_keyword')!.length, 1); + + // ...and it's the SAME range object reused across all of them (one range + // can belong to multiple `Highlight` sets, so we don't allocate per alias) + const range = manager.element_ranges.get('token_keyword')![0]; + assert.equal(manager.element_ranges.get('token_reserved')![0], range); + assert.equal(manager.element_ranges.get('token_special_keyword')![0], range); + }); + + test('falls back to live Range when StaticRange is unavailable', () => { + const saved_static = (globalThis as any).StaticRange; + delete (globalThis as any).StaticRange; + try { + const manager = new HighlightManager(); + const element = create_code_element('const'); + const tokens: SyntaxTokenStream = [new SyntaxToken('keyword', 'const', undefined, 'const')]; + + manager.highlight_from_syntax_tokens(element, tokens); + + const ranges = manager.element_ranges.get('token_keyword')!; + assert.equal(ranges.length, 1); + assert.equal(ranges[0]!.startOffset, 0); + assert.equal(ranges[0]!.endOffset, 5); + } finally { + (globalThis as any).StaticRange = saved_static; + } + }); + + test('downgrades to live Range if Highlight.add rejects StaticRange', () => { + // simulate an engine that accepts live Range but rejects StaticRange + class RejectingHighlight extends Set { + priority = 0; + override add(range: AbstractRange): this { + if (range instanceof (globalThis as any).StaticRange) { + throw new Error('StaticRange not supported'); + } + return super.add(range); + } + } + const saved_highlight = (globalThis as any).Highlight; + (globalThis as any).Highlight = RejectingHighlight; + try { + const manager = new HighlightManager(); + const element = create_code_element('const'); + const tokens: SyntaxTokenStream = [new SyntaxToken('keyword', 'const', undefined, 'const')]; + + assert.doesNotThrow(() => manager.highlight_from_syntax_tokens(element, tokens)); + + // fell back to live Range and still registered the highlight + assert.ok(CSS.highlights.has('token_keyword')); + assert.equal(CSS.highlights.get('token_keyword')!.size, 1); + const range = manager.element_ranges.get('token_keyword')![0]!; + assert.equal(range instanceof (globalThis as any).StaticRange, false); + } finally { + (globalThis as any).Highlight = saved_highlight; + } }); test('handles token stream with only strings', () => { @@ -585,7 +651,7 @@ describe('lifecycle management', () => { assert.ok(CSS.highlights.has('token_keyword')); // Highlight still exists }); - test('clear_element_ranges throws if highlight unexpectedly missing', () => { + test('clear_element_ranges is a safe no-op if highlight already removed', () => { const manager = new HighlightManager(); const element = create_code_element('const'); @@ -593,13 +659,12 @@ describe('lifecycle management', () => { manager.highlight_from_syntax_tokens(element, tokens); - // Manually delete the highlight to simulate unexpected state + // Another manager (or HMR) removed the shared highlight -- a valid state, + // not an error, so clearing must not throw CSS.highlights.delete('token_keyword'); - assert.throws( - () => manager.clear_element_ranges(), - /Expected to find CSS highlight: token_keyword/, - ); + assert.doesNotThrow(() => manager.clear_element_ranges()); + assert.equal(manager.element_ranges.size, 0); }); test('clear_element_ranges deletes highlight when last range removed', () => { diff --git a/src/test/highlight_test_helpers.ts b/src/test/highlight_test_helpers.ts index 32a37b1b..92d3fa02 100644 --- a/src/test/highlight_test_helpers.ts +++ b/src/test/highlight_test_helpers.ts @@ -6,10 +6,30 @@ */ // Mock implementation of CSS Custom Highlight API -export class MockHighlight extends Set { +export class MockHighlight extends Set { priority = 0; } +// Mock StaticRange (the manager's preferred, non-live range kind) +export class MockStaticRange { + startContainer: Node; + startOffset: number; + endContainer: Node; + endOffset: number; + + constructor(init: { + startContainer: Node; + startOffset: number; + endContainer: Node; + endOffset: number; + }) { + this.startContainer = init.startContainer; + this.startOffset = init.startOffset; + this.endContainer = init.endContainer; + this.endOffset = init.endOffset; + } +} + // Mock Range class with bounds validation export class MockRange { start_container: Node | null = null; @@ -55,6 +75,7 @@ export interface SavedGlobals { css: any; highlight: any; range: any; + static_range: any; node: any; } @@ -66,6 +87,7 @@ export function setup_mock_highlight_api(): SavedGlobals { css: (globalThis as any).CSS, highlight: (globalThis as any).Highlight, range: (globalThis as any).Range, + static_range: (globalThis as any).StaticRange, node: (globalThis as any).Node, }; @@ -75,6 +97,7 @@ export function setup_mock_highlight_api(): SavedGlobals { }; (globalThis as any).Highlight = MockHighlight; (globalThis as any).Range = MockRange; + (globalThis as any).StaticRange = MockStaticRange; // Mock Node with TEXT_NODE constant (globalThis as any).Node = { @@ -93,6 +116,7 @@ export function restore_globals(saved: SavedGlobals): void { (globalThis as any).CSS = saved.css; (globalThis as any).Highlight = saved.highlight; (globalThis as any).Range = saved.range; + (globalThis as any).StaticRange = saved.static_range; (globalThis as any).Node = saved.node; } diff --git a/src/test/syntax_styler.stringify.test.ts b/src/test/syntax_styler.stringify.test.ts new file mode 100644 index 00000000..43504d1e --- /dev/null +++ b/src/test/syntax_styler.stringify.test.ts @@ -0,0 +1,148 @@ +import {test, assert, describe} from 'vitest'; + +import {SyntaxStyler} from '$lib/syntax_styler.js'; +import {syntax_styler_global} from '$lib/syntax_styler_global.js'; +import {SyntaxToken} from '$lib/syntax_token.js'; + +/** + * Tests for `SyntaxStyler.stringify_token` — HTML escaping of leaf text, span + * wrapping (the no-hook fast path), and the `wrap` hook (the slow path). + * + * `stringify_token` needs no grammar, so token streams are built by hand. The + * non-breaking space is built with `String.fromCharCode` so the intent can't be + * silently mangled into a regular space by an editor or formatter. + */ + +const nbsp = String.fromCharCode(0xa0); + +describe('stringify_token text escaping', () => { + test('escapes & and < in leaf text', () => { + assert.equal(syntax_styler_global.stringify_token('a & b < c', 'js'), 'a & b < c'); + }); + + test('does not escape > or " in text content', () => { + // only `&` and `<` are required to be escaped in HTML text nodes + assert.equal(syntax_styler_global.stringify_token('a > b " c', 'js'), 'a > b " c'); + }); + + test('normalizes a non-breaking space to a regular space', () => { + assert.equal(syntax_styler_global.stringify_token('a' + nbsp + 'b', 'js'), 'a b'); + }); + + test('does not double-escape the & introduced by escaping <', () => { + // `<` -> `<`, and the produced `&` must NOT become `&lt;` + assert.equal(syntax_styler_global.stringify_token('<', 'js'), '<'); + }); + + test('escapes a literal ampersand once', () => { + assert.equal(syntax_styler_global.stringify_token('<', 'js'), '&lt;'); + }); + + test('handles all three escapes together in one pass', () => { + // & -> &, < -> <, nbsp -> space, > untouched + assert.equal(syntax_styler_global.stringify_token('&<' + nbsp + '>', 'js'), '&< >'); + }); +}); + +describe('stringify_token span wrapping (no-hook fast path)', () => { + test('wraps a token in a span with its token_ class', () => { + const token = new SyntaxToken('keyword', 'const', undefined, 'const'); + assert.equal( + syntax_styler_global.stringify_token(token, 'js'), + 'const', + ); + }); + + test('appends a token_ class for each alias', () => { + const token = new SyntaxToken('keyword', 'fn', ['special_keyword', 'reserved'], 'fn'); + assert.equal( + syntax_styler_global.stringify_token(token, 'js'), + 'fn', + ); + }); + + test('escapes string content inside the span', () => { + const token = new SyntaxToken('string', 'a & b', undefined, 'a & b'); + assert.equal( + syntax_styler_global.stringify_token(token, 'js'), + 'a & b', + ); + }); + + test('recurses into nested token content, escaping leaves', () => { + const token = new SyntaxToken( + 'tag', + [new SyntaxToken('punctuation', '<', undefined, '<'), 'a & b'], + undefined, + '<a & b', + ); + }); + + test('concatenates a token stream array', () => { + const stream = [new SyntaxToken('keyword', 'const', undefined, 'const'), ' x']; + assert.equal( + syntax_styler_global.stringify_token(stream, 'js'), + 'const x', + ); + }); +}); + +describe('stringify_token wrap hook (slow path)', () => { + test('a registered wrap hook can add attributes', () => { + const styler = new SyntaxStyler(); + styler.add_hook_wrap((ctx) => { + ctx.attributes['data-type'] = ctx.type; + }); + const token = new SyntaxToken('keyword', 'const', undefined, 'const'); + assert.equal( + styler.stringify_token(token, 'js'), + 'const', + ); + }); + + test('a wrap hook can change the tag and push classes', () => { + const styler = new SyntaxStyler(); + styler.add_hook_wrap((ctx) => { + ctx.tag = 'mark'; + ctx.classes.push('extra'); + }); + const token = new SyntaxToken('keyword', 'const', undefined, 'const'); + assert.equal( + styler.stringify_token(token, 'js'), + 'const', + ); + }); + + test('attribute values have their double quotes escaped', () => { + const styler = new SyntaxStyler(); + styler.add_hook_wrap((ctx) => { + ctx.attributes['data-x'] = 'a"b'; + }); + const token = new SyntaxToken('keyword', 'const', undefined, 'const'); + assert.equal( + styler.stringify_token(token, 'js'), + 'const', + ); + }); + + test('the hook receives the resolved token type, content, and classes', () => { + const styler = new SyntaxStyler(); + let seen_type: string | undefined; + let seen_content: string | undefined; + let seen_classes: Array | undefined; + styler.add_hook_wrap((ctx) => { + seen_type = ctx.type; + seen_content = ctx.content; + seen_classes = [...ctx.classes]; + }); + const token = new SyntaxToken('string', 'a & b', ['quoted'], 'a & b'); + styler.stringify_token(token, 'ts'); + assert.equal(seen_type, 'string'); + assert.equal(seen_content, 'a & b'); // content is already escaped when the hook runs + assert.deepEqual(seen_classes, ['token_string', 'token_quoted']); + }); +}); diff --git a/src/test/syntax_styler.test.ts b/src/test/syntax_styler.test.ts index ed7c241d..ea2d321b 100644 --- a/src/test/syntax_styler.test.ts +++ b/src/test/syntax_styler.test.ts @@ -593,3 +593,66 @@ describe('tokenization consistency', () => { assert.ok(result.includes(long_content), 'Should preserve long string content'); }); }); + +describe('tokenize (hooked tokenization)', () => { + test('matches the bare tokenize_syntax stream when no hooks are registered', () => { + const syntax_styler = create_styler_with_grammars(); + const code = 'const x = 1;'; + assert.deepEqual( + syntax_styler.tokenize(code, 'js'), + tokenize_syntax(code, syntax_styler.get_lang('js')), + ); + }); + + test('runs before_tokenize and after_tokenize hooks (which bare tokenize_syntax skips)', () => { + const syntax_styler = create_styler_with_grammars(); + let before_lang: string | undefined; + let after_token_count: number | undefined; + syntax_styler.add_hook_before_tokenize((ctx) => { + before_lang = ctx.lang; + }); + syntax_styler.add_hook_after_tokenize((ctx) => { + after_token_count = ctx.tokens.length; + }); + + const tokens = syntax_styler.tokenize('const x = 1;', 'js'); + + assert.equal(before_lang, 'js'); + assert.equal(after_token_count, tokens.length); + }); + + test('a before_tokenize hook can rewrite the code before tokenization', () => { + const syntax_styler = create_styler_with_grammars(); + syntax_styler.add_hook_before_tokenize((ctx) => { + ctx.code = ctx.code.replace('var', 'const'); + }); + + const tokens = syntax_styler.tokenize('var x = 1;', 'js'); + const keyword = tokens.find((t) => typeof t !== 'string' && t.type === 'keyword'); + assert.ok(keyword && typeof keyword !== 'string'); + assert.equal(keyword.content, 'const'); + }); + + test('an after_tokenize hook can replace the token stream', () => { + const syntax_styler = create_styler_with_grammars(); + syntax_styler.add_hook_after_tokenize((ctx) => { + ctx.tokens = ['replaced']; + }); + assert.deepEqual(syntax_styler.tokenize('const x = 1;', 'js'), ['replaced']); + }); + + test('stylize stringifies with the lang a before_tokenize hook rewrote', () => { + const syntax_styler = create_styler_with_grammars(); + let wrap_lang: string | undefined; + // the rewritten lang must flow through to each token's wrap hook context + syntax_styler.add_hook_before_tokenize((ctx) => { + ctx.lang = 'rewritten'; + }); + syntax_styler.add_hook_wrap((ctx) => { + wrap_lang = ctx.lang; + }); + + syntax_styler.stylize('const x = 1;', 'js'); + assert.equal(wrap_lang, 'rewritten'); + }); +});