Fix markdown serializer boundary-whitespace defects on the…

.changeset/serialize-boundary-whitespace-fidelity.md

@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge-core": patch
+---
+
+Fix markdown serializer boundary-whitespace defects on the WYSIWYG-to-source path. An insignificant trailing space or tab at a block edge now serializes to a literal character instead of a visible ` ` / `	` character reference, so source mode no longer shows the escape where a space was typed (a leading tab, or a leading run of four or more spaces, still encodes, since it would otherwise trigger an indented code block on re-parse and turn the paragraph into a code block). Strikethrough (`~~`) and highlight (`==`) marks authored with boundary whitespace now char-ref-encode that whitespace so the mark survives re-parse instead of silently dropping. Emphasis and strong already behaved correctly and are unchanged.

packages/app/tests/integration/bug3-source-mode-writeback.test.ts

@@ -0,0 +1,113 @@
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import { setTimeout as wait } from 'node:timers/promises';
+import { updateYFragment } from '@tiptap/y-tiptap';
+import { HARNESS_BOOT_TIMEOUT_MS } from './harness-boot-timeout';
+import {
+  agentWriteMd,
+  awaitDocQuiescence,
+  createTestClient,
+  createTestServer,
+  getServerState,
+  mdManager,
+  schema,
+  type TestClient,
+  type TestServer,
+} from './test-harness';
+
+const STEPS = [
+  '<Steps>',
+  '',
+  '<Step>',
+  '',
+  'Content one.',
+  '',
+  '</Step>',
+  '',
+  '<Step>',
+  '',
+  'Content two.',
+  '',
+  '</Step>',
+  '',
+  '</Steps>',
+  '',
+].join('\n');
+
+let server: TestServer;
+
+beforeAll(async () => {
+  server = await createTestServer();
+}, HARNESS_BOOT_TIMEOUT_MS);
+
+afterAll(async () => {
+  await server.cleanup();
+});
+
+/** A genuine WYSIWYG-side fragment commit (null origin => server Observer A sees
+ *  a real WYSIWYG mutation, xmlDirty=true) — the same channel the hidden-but-
+ *  mounted TipTap binding republishes through. Non-vacuous: it really changes
+ *  the fragment, so Observer A runs and its serialize-vs-ytext diff is exercised. */
+function applyWysiwygEdit(client: TestClient, markdownAfterEdit: string): void {
+  const pmNode = schema.nodeFromJSON(mdManager.parse(markdownAfterEdit));
+  client.doc.transact(() => {
+    updateYFragment(client.doc, client.fragment, pmNode, {
+      mapping: new Map(),
+      isOMark: new Map(),
+    });
+  });
+}
+
+const INDENTED_STEP = /\n[ \t]+<\/?Step\b/; // a <Step>/</Step> tag gaining leading indentation
+const INDENTED_STEPS = /\n[ \t]+<\/?Steps\b/;
+
+describe('bug #3 — source-mode write-back guard (re-indent facet closed by #1991)', () => {
+  test('the faithful <Steps> parses to a jsxComponent and is a serialize fixed point', () => {
+    const tree = mdManager.parse(STEPS) as { content?: Array<{ type?: string }> };
+    const topTypes = (tree.content ?? []).map((n) => n.type);
+    expect(topTypes).toContain('jsxComponent');
+    expect(mdManager.serialize(mdManager.parse(STEPS))).toBe(
+      '<Steps>\n\n<Step>\n\nContent one.\n\n</Step>\n\n<Step>\n\nContent two.\n\n</Step>\n\n</Steps>\n',
+    );
+  });
+
+  test('V1 baseline: an isolated source keystroke stays byte-verbatim (no Observer-A write-back)', async () => {
+    const docName = `bug3-v1-${crypto.randomUUID()}`;
+    await agentWriteMd(server.port, STEPS, { docName, position: 'replace' });
+    await wait(300);
+    const client = await createTestClient(server.port, docName);
+    try {
+      const ytext = client.doc.getText('source');
+      await awaitDocQuiescence(client.doc);
+      expect(ytext.toString()).toBe(STEPS); // seed landed verbatim
+      const at = ytext.toString().indexOf('Content one.') + 'Content one'.length;
+      client.doc.transact(() => ytext.insert(at, 'X'));
+      const expected = ytext.toString();
+      await awaitDocQuiescence(client.doc);
+      expect(getServerState(server, docName)?.ytext.toString()).toBe(expected);
+    } finally {
+      await client.cleanup();
+    }
+  });
+
+  test('a concurrent WYSIWYG fragment commit does NOT re-indent the <Steps> in Y.Text', async () => {
+    const docName = `bug3-writeback-${crypto.randomUUID()}`;
+    await agentWriteMd(server.port, STEPS, { docName, position: 'replace' });
+    await wait(300);
+    const client = await createTestClient(server.port, docName);
+    try {
+      const ytext = client.doc.getText('source');
+      await awaitDocQuiescence(client.doc);
+      expect(ytext.toString()).toBe(STEPS);
+
+      applyWysiwygEdit(client, STEPS.replace('Content two.', 'Content two, edited.'));
+      await awaitDocQuiescence(client.doc);
+
+      const after = getServerState(server, docName)?.ytext.toString() ?? '';
+      expect(after).toContain('Content two, edited.'); // the edit landed (non-vacuous)
+      expect(after).not.toMatch(INDENTED_STEP);
+      expect(after).not.toMatch(INDENTED_STEPS);
+    } finally {
+      await client.cleanup();
+    }
+  });
+});

packages/core/src/markdown/serialize-fidelity.test-helper.ts

@@ -0,0 +1,39 @@
+import { expect } from 'bun:test';
+import type { JSONContent } from '@tiptap/core';
+import type { Nodes } from 'mdast';
+import { sharedExtensions } from '../extensions/shared.ts';
+import { MarkdownManager } from './index.ts';
+import { isInlineWhitespaceNumericCharRef } from './whitespace-char-ref.ts';
+
+const md = new MarkdownManager({ extensions: sharedExtensions });
+
+const NUMERIC_CHAR_REF_SCAN = /&#(?:x[0-9A-Fa-f]+|X[0-9A-Fa-f]+|[0-9]+);/g;
+
+export function assertFragmentSerializesTo(fragment: JSONContent, expectedBytes: string): void {
+  expect(md.serialize(fragment)).toBe(expectedBytes);
+}
+
+export function assertNoBoundaryLeak(fragment: JSONContent): void {
+  const out = md.serialize(fragment);
+  const leaked = (out.match(NUMERIC_CHAR_REF_SCAN) ?? []).filter(isInlineWhitespaceNumericCharRef);
+  expect(leaked).toEqual([]);
+}
+
+export function assertMarkSurvives(fragment: JSONContent, markType: string): void {
+  const out = md.serialize(fragment);
+  expect(mdastNodeTypes(out)).toContain(markType);
+}
+
+function mdastNodeTypes(markdown: string): string[] {
+  const out: string[] = [];
+  const walk = (node: Nodes): void => {
+    out.push(node.type);
+    if ('children' in node) {
+      for (const child of node.children) {
+        walk(child);
+      }
+    }
+  };
+  walk(md.parseToMdast(markdown));
+  return out;
+}

packages/core/src/markdown/to-markdown-handlers.ts

@@ -567,11 +567,15 @@ export const toMarkdownHandlers = {
     const tracker = state.createTracker(info);
     const exit = state.enter('mark');
     let value = tracker.move('==');
-    value += state.containerPhrasing(node as Parents, {
-      before: value,
-      after: '==',
-      ...tracker.current(),
-    });
+    value += encodeAttentionBoundaries(
+      state.containerPhrasing(node as Parents, {
+        before: value,
+        after: '==',
+        ...tracker.current(),
+      }),
+      info,
+      '==',
+    );
     value += tracker.move('==');
     exit();
     return value;
@@ -786,6 +790,12 @@ function safeText(state: State, value: string, info: Info): string {
       return false;
     }
     if (u.character === '=' && u.atBreak === true) return false;
+    if (u.character === ' ' && u.inConstruct === 'phrasing' && u.after === '[\\r\\n]') {
+      return false;
+    }
+    if (u.character === '\t' && u.inConstruct === 'phrasing' && u.after === '[\\r\\n]') {
+      return false;
+    }
     return true;
   });
   let result: string;
@@ -873,14 +883,16 @@ function escapeActiveDelimiterRuns(value: string, info: Info): string {
 }
 
 function encodeAttentionBoundaries(between: string, info: Info, delim: string): string {
-  const marker = delim.startsWith('_') ? '_' : '*';
+  const marker: '*' | '_' = delim.startsWith('_') ? '_' : '*';
+  const before = info.before ?? '';
+  const after = info.after ?? '';
   let result = between;
   const head = result.charCodeAt(0);
-  if (shouldEncodeAttentionBoundary(info.before.charCodeAt(info.before.length - 1), head, marker)) {
+  if (shouldEncodeAttentionBoundary(before.charCodeAt(before.length - 1), head, marker)) {
     result = encodeCharacterReference(head) + result.slice(1);
   }
   const tail = result.charCodeAt(result.length - 1);
-  if (shouldEncodeAttentionBoundary(info.after.charCodeAt(0), tail, marker)) {
+  if (shouldEncodeAttentionBoundary(after.charCodeAt(0), tail, marker)) {
     result = result.slice(0, -1) + encodeCharacterReference(tail);
   }
   return result;
@@ -950,11 +962,15 @@ function serializeDelete(
   const tracker = state.createTracker(info);
   const exit = state.enter('strikethrough');
   let value = tracker.move(delim);
-  value += state.containerPhrasing(node, {
-    ...tracker.current(),
-    before: value,
-    after: '~',
-  });
+  value += encodeAttentionBoundaries(
+    state.containerPhrasing(node, {
+      ...tracker.current(),
+      before: value,
+      after: '~',
+    }),
+    info,
+    delim,
+  );
   value += tracker.move(delim);
   exit();
   return value;

turbo.json

@@ -30,7 +30,7 @@
     "test": {
       "dependsOn": ["^build"],
       "cache": true,
-      "inputs": ["src/**/*.ts", "src/**/*.tsx", "tests/contract/**/*.ts"]
+      "inputs": ["src/**/*.ts", "src/**/*.tsx", "src/**/*.json", "tests/contract/**/*.ts"]
     },
     "test:dom": {
       "dependsOn": ["^build"],