From 7e127fc55b8cfa7b461e9e2d650ade04a1611613 Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Sat, 13 Jun 2026 19:08:42 -0400 Subject: [PATCH 01/12] wip --- TODO_TEXTAREA_HIGHLIGHTING.md | 63 ++++++++ src/lib/CodeHighlight.svelte | 16 +- src/lib/CodeTextarea.svelte | 155 ++++++++++++++++++++ src/lib/highlight_manager.ts | 226 ++++++++++++++++++----------- src/routes/textarea/+page.svelte | 62 ++++++++ src/test/highlight_manager.test.ts | 83 +++++++++-- src/test/highlight_test_helpers.ts | 26 +++- 7 files changed, 532 insertions(+), 99 deletions(-) create mode 100644 TODO_TEXTAREA_HIGHLIGHTING.md create mode 100644 src/lib/CodeTextarea.svelte create mode 100644 src/routes/textarea/+page.svelte diff --git a/TODO_TEXTAREA_HIGHLIGHTING.md b/TODO_TEXTAREA_HIGHLIGHTING.md new file mode 100644 index 00000000..d03a6069 --- /dev/null +++ b/TODO_TEXTAREA_HIGHLIGHTING.md @@ -0,0 +1,63 @@ +# TODO: textarea range highlighting + +Adding live syntax highlighting to a ` + + + diff --git a/src/lib/highlight_manager.ts b/src/lib/highlight_manager.ts index 477963c2..0b3b4702 100644 --- a/src/lib/highlight_manager.ts +++ b/src/lib/highlight_manager.ts @@ -1,3 +1,5 @@ +import {DEV} from 'esm-env'; + import type {SyntaxTokenStream} from './syntax_token.js'; import {highlight_priorities} from './highlight_priorities.js'; @@ -10,10 +12,55 @@ export const supports_css_highlight_api = (): boolean => !!(globalThis.CSS?.highlights && globalThis.Highlight); /** - * Manages CSS Custom Highlight API ranges for a single element. - * Tracks ranges per element and only removes its own ranges when clearing. + * How a manager builds ranges for the Highlight API. + * + * `StaticRange` is preferred: it's an immutable snapshot the browser does *not* + * track across DOM mutations. Since highlights are rebuilt wholesale whenever + * content changes, liveness buys nothing and only costs per-mutation boundary + * bookkeeping and paint work — the main efficiency (and Safari-stability) lever. + * Falls back to a live `Range` where `StaticRange` is unavailable. + */ +type RangeKind = 'static' | 'live'; + +const detect_range_kind = (): RangeKind => + typeof globalThis.StaticRange === 'function' ? 'static' : 'live'; + +/** + * Finds the first text node child of `element`, or `null` if there is none. + * + * The text node might not be `firstChild` because frameworks (e.g. Svelte) can + * insert comment/anchor nodes around it. + */ +const find_text_node = (element: Element): Node | null => { + for (const node of element.childNodes) { + if (node.nodeType === Node.TEXT_NODE) return node; + } + return null; +}; + +const has_tokens = (tokens: SyntaxTokenStream): boolean => + tokens.some((t) => typeof t !== 'string'); + +const push_range = ( + ranges_by_name: Map>, + name: string, + range: AbstractRange, +): void => { + const existing = ranges_by_name.get(name); + if (existing) { + existing.push(range); + } else { + ranges_by_name.set(name, [range]); + } +}; + +/** + * Manages CSS Custom Highlight API ranges for a single element's text node. + * Tracks ranges per element and only removes its own ranges when clearing, + * cooperating with other managers that share the global `CSS.highlights` registry. * - * **Experimental** — limited browser support. Use `Code.svelte` for production. + * **Experimental** — limited browser support. Use `Code.svelte` for production + * block code; this powers the experimental `CodeHighlight` and `CodeTextarea`. * * @example * ```ts @@ -22,62 +69,86 @@ export const supports_css_highlight_api = (): boolean => * ``` */ export class HighlightManager { - element_ranges: Map>; + /** + * This manager's ranges, keyed by prefixed highlight name (e.g. `token_keyword`). + * A single range object may be shared across several names (a token type plus + * its aliases), since one range can belong to multiple `Highlight` sets. + */ + element_ranges: Map>; + + #range_kind: RangeKind; constructor() { if (!supports_css_highlight_api()) { throw Error('CSS Highlights API not supported'); } this.element_ranges = new Map(); + this.#range_kind = detect_range_kind(); } /** - * Highlights from a `SyntaxTokenStream` produced by `tokenize_syntax`. + * Highlights `element`'s text node from a `SyntaxTokenStream` produced by + * `tokenize_syntax`. Clears this manager's previous ranges first. + * + * In production this never throws on a tokenizer/DOM mismatch: out-of-bounds + * tokens are clamped and a missing text node is a no-op. In DEV the same + * conditions throw loudly to surface grammar bugs. */ highlight_from_syntax_tokens(element: Element, tokens: SyntaxTokenStream): void { - // Find the text node (it might not be firstChild due to Svelte comment nodes) - let text_node: Node | null = null; - for (const node of element.childNodes) { - if (node.nodeType === Node.TEXT_NODE) { - text_node = node; - break; - } - } + this.clear_element_ranges(); + const text_node = find_text_node(element); if (!text_node) { - throw new Error('no text node to highlight'); + if (has_tokens(tokens)) { + if (DEV) { + throw new Error('no text node to highlight'); + } else { + // eslint-disable-next-line no-console + console.error('[HighlightManager] tokens present but no text node to highlight'); + } + } + return; } - this.clear_element_ranges(); + try { + this.#apply(text_node, tokens); + } catch (err) { + // some engines may reject `StaticRange` in `Highlight.add` -- fall back to + // live `Range` once rather than letting the throw escape into the effect + if (this.#range_kind === 'static') { + this.#range_kind = 'live'; + this.clear_element_ranges(); // undo any partial application + this.#apply(text_node, tokens); + } else { + throw err; + } + } + } - const ranges_by_type: Map> = new Map(); - const final_pos = this.#create_all_ranges(tokens, text_node, ranges_by_type, 0); + #apply(text_node: Node, tokens: SyntaxTokenStream): void { + const ranges_by_name: Map> = new Map(); + const final_pos = this.#collect_ranges(tokens, text_node, ranges_by_name, 0); - // Validate that token positions matched text node length - const text_length = text_node.textContent?.length ?? 0; - if (final_pos !== text_length) { - throw new Error( - `Token stream length mismatch: tokens covered ${final_pos} chars but text node has ${text_length} chars`, - ); + if (DEV) { + const text_length = text_node.textContent?.length ?? 0; + if (final_pos !== text_length) { + throw new Error( + `Token stream length mismatch: tokens covered ${final_pos} chars but text node has ${text_length} chars`, + ); + } } - // Apply highlights - for (const [type, ranges] of ranges_by_type) { - const prefixed_type = `token_${type}`; - // Track ranges for this element - this.element_ranges.set(prefixed_type, ranges); + for (const [name, ranges] of ranges_by_name) { + this.element_ranges.set(name, ranges); - // Get or create the shared highlight - let highlight = CSS.highlights.get(prefixed_type); + let highlight = CSS.highlights.get(name); if (!highlight) { highlight = new Highlight(); - // Set priority based on CSS cascade order (higher = later in CSS = wins) - highlight.priority = - highlight_priorities[prefixed_type as keyof typeof highlight_priorities] ?? 0; - CSS.highlights.set(prefixed_type, highlight); + // priority follows CSS cascade order (higher = later in CSS = wins) + highlight.priority = highlight_priorities[name as keyof typeof highlight_priorities] ?? 0; + CSS.highlights.set(name, highlight); } - // Add all ranges to the highlight for (const range of ranges) { highlight.add(range); } @@ -85,14 +156,14 @@ export class HighlightManager { } /** - * Clears only this element's ranges from highlights. + * Clears only this manager's ranges from the shared highlights. Defensive: + * a highlight may already be gone (e.g. another manager removed the last + * range, or HMR reset the registry), which is a valid state, not an error. */ clear_element_ranges(): void { for (const [name, ranges] of this.element_ranges) { const highlight = CSS.highlights.get(name); - if (!highlight) { - throw new Error('Expected to find CSS highlight: ' + name); - } + if (!highlight) continue; for (const range of ranges) { highlight.delete(range); @@ -110,13 +181,29 @@ export class HighlightManager { this.clear_element_ranges(); } + #make_range(text_node: Node, start: number, end: number): AbstractRange { + if (this.#range_kind === 'static') { + return new StaticRange({ + startContainer: text_node, + startOffset: start, + endContainer: text_node, + endOffset: end, + }); + } + const range = new Range(); + range.setStart(text_node, start); + range.setEnd(text_node, end); + return range; + } + /** - * Creates ranges for all tokens in the tree. + * Walks the token tree, collecting one range per non-empty token (shared + * across its type and aliases). Returns the end position covered. */ - #create_all_ranges( + #collect_ranges( tokens: SyntaxTokenStream, text_node: Node, - ranges_by_type: Map>, + ranges_by_name: Map>, offset: number, ): number { const text_length = text_node.textContent?.length ?? 0; @@ -128,61 +215,36 @@ export class HighlightManager { continue; } - const length = token.length; - const end_pos = pos + length; + const end_pos = pos + token.length; - // Validate positions are within text node bounds before creating ranges - if (end_pos > text_length) { + if (DEV && end_pos > text_length) { throw new Error( `Token ${token.type} extends beyond text node: position ${end_pos} > length ${text_length}`, ); } - try { - const range = new Range(); - range.setStart(text_node, pos); - range.setEnd(text_node, end_pos); - - // Add range for the token type - const type = token.type; - if (!ranges_by_type.has(type)) { - ranges_by_type.set(type, []); - } - ranges_by_type.get(type)!.push(range); - - // Also add range for any aliases (alias is always an array) + // production-safe: clamp rather than throw on a tokenizer edge case + const safe_end = end_pos > text_length ? text_length : end_pos; + if (safe_end > pos) { + // one range shared across the token type and all its aliases -- + // the same range object can belong to multiple `Highlight` sets + const range = this.#make_range(text_node, pos, safe_end); + push_range(ranges_by_name, `token_${token.type}`, range); for (const alias of token.alias) { - if (!ranges_by_type.has(alias)) { - ranges_by_type.set(alias, []); - } - // Create a new range for each alias (ranges can't be reused) - const alias_range = new Range(); - alias_range.setStart(text_node, pos); - alias_range.setEnd(text_node, end_pos); - ranges_by_type.get(alias)!.push(alias_range); + push_range(ranges_by_name, `token_${alias}`, range); } - } catch (e) { - throw new Error(`Failed to create range for ${token.type}: ${e}`); } - // Process nested tokens if (Array.isArray(token.content)) { - const actual_end_pos = this.#create_all_ranges( - token.content, - text_node, - ranges_by_type, - pos, - ); - // Validate that nested tokens match the parent token's claimed length - if (actual_end_pos !== end_pos) { + const nested_end = this.#collect_ranges(token.content, text_node, ranges_by_name, pos); + if (DEV && nested_end !== end_pos) { throw new Error( - `Token ${token.type} length mismatch: claimed ${length} chars (${pos}-${end_pos}) but nested content covered ${actual_end_pos - pos} chars (${pos}-${actual_end_pos})`, + `Token ${token.type} length mismatch: claimed ${token.length} chars (${pos}-${end_pos}) but nested content covered ${nested_end - pos} chars (${pos}-${nested_end})`, ); } - pos = actual_end_pos; - } else { - pos = end_pos; } + + pos = end_pos; } return pos; diff --git a/src/routes/textarea/+page.svelte b/src/routes/textarea/+page.svelte new file mode 100644 index 00000000..a126b92a --- /dev/null +++ b/src/routes/textarea/+page.svelte @@ -0,0 +1,62 @@ + + +
+
+ +
+ +
+

Editable highlighted textarea (experimental)

+ + +
+ {#each sample_langs as l (l)} + + {/each} +
+ + +
+ +
+
diff --git a/src/test/highlight_manager.test.ts b/src/test/highlight_manager.test.ts index a27e0ed1..5a1bf919 100644 --- a/src/test/highlight_manager.test.ts +++ b/src/test/highlight_manager.test.ts @@ -80,13 +80,24 @@ describe('initialization', () => { }); describe('DOM element handling', () => { - test('throws if element has no text node', () => { + test('no-op when no text node and no tokens', () => { const manager = new HighlightManager(); const element = { childNodes: [], // No children } as unknown as Element; const tokens: SyntaxTokenStream = []; + assert.doesNotThrow(() => manager.highlight_from_syntax_tokens(element, tokens)); + assert.equal(manager.element_ranges.size, 0); + }); + + test('throws in DEV when tokens present but no text node', () => { + const manager = new HighlightManager(); + const element = { + childNodes: [], // No children + } as unknown as Element; + const tokens: SyntaxTokenStream = [new SyntaxToken('keyword', 'const', undefined, 'const')]; + assert.throws( () => manager.highlight_from_syntax_tokens(element, tokens), /no text node to highlight/, @@ -280,7 +291,7 @@ describe('range creation', () => { assert.equal(part3_ranges[0]!.endOffset, 5); }); - test('creates separate ranges for aliases', () => { + test('shares one range across a token type and its aliases', () => { const manager = new HighlightManager(); const element = create_code_element('function'); @@ -295,10 +306,65 @@ describe('range creation', () => { assert.ok(manager.element_ranges.has('token_reserved')); assert.ok(manager.element_ranges.has('token_special_keyword')); - // Each should have one range + // Each name has one range assert.equal(manager.element_ranges.get('token_keyword')!.length, 1); assert.equal(manager.element_ranges.get('token_reserved')!.length, 1); assert.equal(manager.element_ranges.get('token_special_keyword')!.length, 1); + + // ...and it's the SAME range object reused across all of them (one range + // can belong to multiple `Highlight` sets, so we don't allocate per alias) + const range = manager.element_ranges.get('token_keyword')![0]; + assert.equal(manager.element_ranges.get('token_reserved')![0], range); + assert.equal(manager.element_ranges.get('token_special_keyword')![0], range); + }); + + test('falls back to live Range when StaticRange is unavailable', () => { + const saved_static = (globalThis as any).StaticRange; + delete (globalThis as any).StaticRange; + try { + const manager = new HighlightManager(); + const element = create_code_element('const'); + const tokens: SyntaxTokenStream = [new SyntaxToken('keyword', 'const', undefined, 'const')]; + + manager.highlight_from_syntax_tokens(element, tokens); + + const ranges = manager.element_ranges.get('token_keyword')!; + assert.equal(ranges.length, 1); + assert.equal(ranges[0]!.startOffset, 0); + assert.equal(ranges[0]!.endOffset, 5); + } finally { + (globalThis as any).StaticRange = saved_static; + } + }); + + test('downgrades to live Range if Highlight.add rejects StaticRange', () => { + // simulate an engine that accepts live Range but rejects StaticRange + class RejectingHighlight extends Set { + priority = 0; + override add(range: AbstractRange): this { + if (range instanceof (globalThis as any).StaticRange) { + throw new Error('StaticRange not supported'); + } + return super.add(range); + } + } + const saved_highlight = (globalThis as any).Highlight; + (globalThis as any).Highlight = RejectingHighlight; + try { + const manager = new HighlightManager(); + const element = create_code_element('const'); + const tokens: SyntaxTokenStream = [new SyntaxToken('keyword', 'const', undefined, 'const')]; + + assert.doesNotThrow(() => manager.highlight_from_syntax_tokens(element, tokens)); + + // fell back to live Range and still registered the highlight + assert.ok(CSS.highlights.has('token_keyword')); + assert.equal(CSS.highlights.get('token_keyword')!.size, 1); + const range = manager.element_ranges.get('token_keyword')![0]!; + assert.equal(range instanceof (globalThis as any).StaticRange, false); + } finally { + (globalThis as any).Highlight = saved_highlight; + } }); test('handles token stream with only strings', () => { @@ -585,7 +651,7 @@ describe('lifecycle management', () => { assert.ok(CSS.highlights.has('token_keyword')); // Highlight still exists }); - test('clear_element_ranges throws if highlight unexpectedly missing', () => { + test('clear_element_ranges is a safe no-op if highlight already removed', () => { const manager = new HighlightManager(); const element = create_code_element('const'); @@ -593,13 +659,12 @@ describe('lifecycle management', () => { manager.highlight_from_syntax_tokens(element, tokens); - // Manually delete the highlight to simulate unexpected state + // Another manager (or HMR) removed the shared highlight -- a valid state, + // not an error, so clearing must not throw CSS.highlights.delete('token_keyword'); - assert.throws( - () => manager.clear_element_ranges(), - /Expected to find CSS highlight: token_keyword/, - ); + assert.doesNotThrow(() => manager.clear_element_ranges()); + assert.equal(manager.element_ranges.size, 0); }); test('clear_element_ranges deletes highlight when last range removed', () => { diff --git a/src/test/highlight_test_helpers.ts b/src/test/highlight_test_helpers.ts index 32a37b1b..92d3fa02 100644 --- a/src/test/highlight_test_helpers.ts +++ b/src/test/highlight_test_helpers.ts @@ -6,10 +6,30 @@ */ // Mock implementation of CSS Custom Highlight API -export class MockHighlight extends Set { +export class MockHighlight extends Set { priority = 0; } +// Mock StaticRange (the manager's preferred, non-live range kind) +export class MockStaticRange { + startContainer: Node; + startOffset: number; + endContainer: Node; + endOffset: number; + + constructor(init: { + startContainer: Node; + startOffset: number; + endContainer: Node; + endOffset: number; + }) { + this.startContainer = init.startContainer; + this.startOffset = init.startOffset; + this.endContainer = init.endContainer; + this.endOffset = init.endOffset; + } +} + // Mock Range class with bounds validation export class MockRange { start_container: Node | null = null; @@ -55,6 +75,7 @@ export interface SavedGlobals { css: any; highlight: any; range: any; + static_range: any; node: any; } @@ -66,6 +87,7 @@ export function setup_mock_highlight_api(): SavedGlobals { css: (globalThis as any).CSS, highlight: (globalThis as any).Highlight, range: (globalThis as any).Range, + static_range: (globalThis as any).StaticRange, node: (globalThis as any).Node, }; @@ -75,6 +97,7 @@ export function setup_mock_highlight_api(): SavedGlobals { }; (globalThis as any).Highlight = MockHighlight; (globalThis as any).Range = MockRange; + (globalThis as any).StaticRange = MockStaticRange; // Mock Node with TEXT_NODE constant (globalThis as any).Node = { @@ -93,6 +116,7 @@ export function restore_globals(saved: SavedGlobals): void { (globalThis as any).CSS = saved.css; (globalThis as any).Highlight = saved.highlight; (globalThis as any).Range = saved.range; + (globalThis as any).StaticRange = saved.static_range; (globalThis as any).Node = saved.node; } From 04f8ca7a77657044dc9843ca0f59a2ca10f96fe6 Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Sat, 13 Jun 2026 19:10:12 -0400 Subject: [PATCH 02/12] wip --- TODO_TEXTAREA_HIGHLIGHTING.md | 63 ----------------------------------- 1 file changed, 63 deletions(-) delete mode 100644 TODO_TEXTAREA_HIGHLIGHTING.md diff --git a/TODO_TEXTAREA_HIGHLIGHTING.md b/TODO_TEXTAREA_HIGHLIGHTING.md deleted file mode 100644 index d03a6069..00000000 --- a/TODO_TEXTAREA_HIGHLIGHTING.md +++ /dev/null @@ -1,63 +0,0 @@ -# TODO: textarea range highlighting - -Adding live syntax highlighting to a ` @@ -104,6 +112,9 @@ white-space: pre-wrap; overflow-wrap: break-word; overflow: auto; + /* reserve gutter on both layers so the textarea's scrollbar doesn't shrink + its wrap width relative to the backdrop, which would drift highlights */ + scrollbar-gutter: stable; } .code_textarea_backdrop { diff --git a/src/lib/range_highlighting.svelte.ts b/src/lib/range_highlighting.svelte.ts index e2ce8682..5dbfeb51 100644 --- a/src/lib/range_highlighting.svelte.ts +++ b/src/lib/range_highlighting.svelte.ts @@ -1,7 +1,6 @@ import {onDestroy} from 'svelte'; import {DEV} from 'esm-env'; -import {tokenize_syntax} from './tokenize_syntax.js'; import type {SyntaxStyler, SyntaxGrammar} from './syntax_styler.js'; import {HighlightManager, supports_css_highlight_api} from './highlight_manager.js'; @@ -61,10 +60,10 @@ export const create_range_highlighting = (options: RangeHighlightingOptions): Ra if (!manager || !is_enabled() || highlighting_disabled) return null; const text = options.text(); if (!text) return null; - return tokenize_syntax( - text, - options.grammar() || options.syntax_styler().get_lang(options.lang()!), - ); // ! safe bc of `highlighting_disabled` + // route through the styler so `before_tokenize`/`after_tokenize` hooks run, + // matching `stylize`'s HTML path; `grammar` undefined falls back to the + // registered lang grammar (`! safe bc of `highlighting_disabled`) + return options.syntax_styler().tokenize(text, options.lang()!, options.grammar()); }); if (manager) { diff --git a/src/lib/syntax_styler.ts b/src/lib/syntax_styler.ts index a5c03c31..a48211c4 100644 --- a/src/lib/syntax_styler.ts +++ b/src/lib/syntax_styler.ts @@ -134,7 +134,42 @@ export class SyntaxStyler { lang: string, grammar: SyntaxGrammar | undefined = this.get_lang(lang), ): string { - var ctx: HookBeforeTokenizeCallbackContext = { + // stringify with the post-hook `lang`, which a `before_tokenize` hook may + // have rewritten (it flows into each token's `wrap` hook context) + const c = this.#tokenize_hooked(text, lang, grammar); + return this.stringify_token(c.tokens, c.lang); + } + + /** + * Tokenizes `text` into a `SyntaxTokenStream`, running the `before_tokenize` + * and `after_tokenize` hooks. This is the tokenization half of `stylize` — use + * it when you need the token stream itself (e.g. CSS Custom Highlight API range + * highlighting) rather than HTML. + * + * @param text - source to tokenize + * @param lang - language identifier; passed to the tokenize hooks + * @param grammar - grammar to tokenize with; defaults to `this.get_lang(lang)` + * @returns the resulting token stream + */ + tokenize( + text: string, + lang: string, + grammar: SyntaxGrammar | undefined = this.get_lang(lang), + ): SyntaxTokenStream { + return this.#tokenize_hooked(text, lang, grammar).tokens; + } + + /** + * Runs `before_tokenize` → `tokenize_syntax` → `after_tokenize`, returning the + * resolved context. Shared by `stylize` (which also needs the post-hook `lang`) + * and `tokenize` (which only needs `tokens`). + */ + #tokenize_hooked( + text: string, + lang: string, + grammar: SyntaxGrammar, + ): HookAfterTokenizeCallbackContext { + const ctx: HookBeforeTokenizeCallbackContext = { code: text, grammar, lang, @@ -144,7 +179,7 @@ export class SyntaxStyler { const c = ctx as any as HookAfterTokenizeCallbackContext; c.tokens = tokenize_syntax(c.code, c.grammar); this.run_hook_after_tokenize(c); - return this.stringify_token(c.tokens, c.lang); + return c; } /** diff --git a/src/test/syntax_styler.test.ts b/src/test/syntax_styler.test.ts index ed7c241d..ea2d321b 100644 --- a/src/test/syntax_styler.test.ts +++ b/src/test/syntax_styler.test.ts @@ -593,3 +593,66 @@ describe('tokenization consistency', () => { assert.ok(result.includes(long_content), 'Should preserve long string content'); }); }); + +describe('tokenize (hooked tokenization)', () => { + test('matches the bare tokenize_syntax stream when no hooks are registered', () => { + const syntax_styler = create_styler_with_grammars(); + const code = 'const x = 1;'; + assert.deepEqual( + syntax_styler.tokenize(code, 'js'), + tokenize_syntax(code, syntax_styler.get_lang('js')), + ); + }); + + test('runs before_tokenize and after_tokenize hooks (which bare tokenize_syntax skips)', () => { + const syntax_styler = create_styler_with_grammars(); + let before_lang: string | undefined; + let after_token_count: number | undefined; + syntax_styler.add_hook_before_tokenize((ctx) => { + before_lang = ctx.lang; + }); + syntax_styler.add_hook_after_tokenize((ctx) => { + after_token_count = ctx.tokens.length; + }); + + const tokens = syntax_styler.tokenize('const x = 1;', 'js'); + + assert.equal(before_lang, 'js'); + assert.equal(after_token_count, tokens.length); + }); + + test('a before_tokenize hook can rewrite the code before tokenization', () => { + const syntax_styler = create_styler_with_grammars(); + syntax_styler.add_hook_before_tokenize((ctx) => { + ctx.code = ctx.code.replace('var', 'const'); + }); + + const tokens = syntax_styler.tokenize('var x = 1;', 'js'); + const keyword = tokens.find((t) => typeof t !== 'string' && t.type === 'keyword'); + assert.ok(keyword && typeof keyword !== 'string'); + assert.equal(keyword.content, 'const'); + }); + + test('an after_tokenize hook can replace the token stream', () => { + const syntax_styler = create_styler_with_grammars(); + syntax_styler.add_hook_after_tokenize((ctx) => { + ctx.tokens = ['replaced']; + }); + assert.deepEqual(syntax_styler.tokenize('const x = 1;', 'js'), ['replaced']); + }); + + test('stylize stringifies with the lang a before_tokenize hook rewrote', () => { + const syntax_styler = create_styler_with_grammars(); + let wrap_lang: string | undefined; + // the rewritten lang must flow through to each token's wrap hook context + syntax_styler.add_hook_before_tokenize((ctx) => { + ctx.lang = 'rewritten'; + }); + syntax_styler.add_hook_wrap((ctx) => { + wrap_lang = ctx.lang; + }); + + syntax_styler.stylize('const x = 1;', 'js'); + assert.equal(wrap_lang, 'rewritten'); + }); +}); From dd052be504a662bf61d1f66eb5f8a683a5738f6a Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Sat, 13 Jun 2026 22:16:54 -0400 Subject: [PATCH 07/12] wip --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index afd34422..071d12e6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,7 @@ "devDependencies": { "@changesets/changelog-git": "^0.2.1", "@fuzdev/blake3_wasm": "^0.1.1", - "@fuzdev/fuz_css": "^0.63.0", + "@fuzdev/fuz_css": "^0.63.1", "@fuzdev/fuz_ui": "^0.205.0", "@fuzdev/fuz_util": "^0.65.1", "@fuzdev/gro": "^0.204.0", @@ -759,9 +759,9 @@ } }, "node_modules/@fuzdev/fuz_css": { - "version": "0.63.0", - "resolved": "https://registry.npmjs.org/@fuzdev/fuz_css/-/fuz_css-0.63.0.tgz", - "integrity": "sha512-DuSlmlY9e0taTW4yUU+BHU79uKvIJSV2Uf+ZwmGU4AHOr67jp0iJbDlW14bGmOL2K/NPlJ/fFhDrrn2dshObAw==", + "version": "0.63.1", + "resolved": "https://registry.npmjs.org/@fuzdev/fuz_css/-/fuz_css-0.63.1.tgz", + "integrity": "sha512-Ik2uBPjO63cmf2nLfAWfVmf3cbLgHMR+jAF1eowdEVrvH1H4HDdcjcRsKw2/UZx2sDpCLAZguSDN/EEFV2UQFQ==", "dev": true, "license": "MIT", "engines": { diff --git a/package.json b/package.json index 58c0b447..42ade574 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "devDependencies": { "@changesets/changelog-git": "^0.2.1", "@fuzdev/blake3_wasm": "^0.1.1", - "@fuzdev/fuz_css": "^0.63.0", + "@fuzdev/fuz_css": "^0.63.1", "@fuzdev/fuz_ui": "^0.205.0", "@fuzdev/fuz_util": "^0.65.1", "@fuzdev/gro": "^0.204.0", From 15458020102623c7a5851e214c9a22c50b48c3e1 Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Sat, 13 Jun 2026 22:49:00 -0400 Subject: [PATCH 08/12] wip --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 071d12e6..5da74b2d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,7 @@ "devDependencies": { "@changesets/changelog-git": "^0.2.1", "@fuzdev/blake3_wasm": "^0.1.1", - "@fuzdev/fuz_css": "^0.63.1", + "@fuzdev/fuz_css": "^0.63.2", "@fuzdev/fuz_ui": "^0.205.0", "@fuzdev/fuz_util": "^0.65.1", "@fuzdev/gro": "^0.204.0", @@ -759,9 +759,9 @@ } }, "node_modules/@fuzdev/fuz_css": { - "version": "0.63.1", - "resolved": "https://registry.npmjs.org/@fuzdev/fuz_css/-/fuz_css-0.63.1.tgz", - "integrity": "sha512-Ik2uBPjO63cmf2nLfAWfVmf3cbLgHMR+jAF1eowdEVrvH1H4HDdcjcRsKw2/UZx2sDpCLAZguSDN/EEFV2UQFQ==", + "version": "0.63.2", + "resolved": "https://registry.npmjs.org/@fuzdev/fuz_css/-/fuz_css-0.63.2.tgz", + "integrity": "sha512-df2j970oRPHPJVGoMIhkHltRI6ZMTrhVoBEPfek8hY8RiFMnJMW8cFq+X6oTSGpdfo1COQpjHbeLZemd74z+mg==", "dev": true, "license": "MIT", "engines": { diff --git a/package.json b/package.json index 42ade574..0f05a3e3 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "devDependencies": { "@changesets/changelog-git": "^0.2.1", "@fuzdev/blake3_wasm": "^0.1.1", - "@fuzdev/fuz_css": "^0.63.1", + "@fuzdev/fuz_css": "^0.63.2", "@fuzdev/fuz_ui": "^0.205.0", "@fuzdev/fuz_util": "^0.65.1", "@fuzdev/gro": "^0.204.0", From 2e4c03a1f4941e6add265b1b6922a7b654b25774 Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Sat, 13 Jun 2026 23:00:38 -0400 Subject: [PATCH 09/12] wip --- src/lib/CodeTextarea.svelte | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/lib/CodeTextarea.svelte b/src/lib/CodeTextarea.svelte index c9e569c9..d557872c 100644 --- a/src/lib/CodeTextarea.svelte +++ b/src/lib/CodeTextarea.svelte @@ -85,16 +85,10 @@