From e096d6f72ad5a26337d314054ad46dbbf0d525d9 Mon Sep 17 00:00:00 2001 From: Fu-Jie <33599649+Fu-Jie@users.noreply.github.com> Date: Mon, 29 Jun 2026 12:09:40 +0000 Subject: [PATCH 1/2] feat(markdown-normalizer): add intra-word emphasis fix for issue #97 - New `enable_intra_word_emphasis_fix` Valve moves the opening emphasis marker to the start of a word when the wrapped content is pure punctuation, e.g. `series**,**` -> `**series,**` so added punctuation renders with the word (common in grammar/spelling diff outputs) - Supports **, __, ~~, ***, ___; code blocks and inline code are skipped - Bump version to 1.2.9; sync bilingual READMEs, docs mirror, and FEATURES_CN; add v1.2.9 release notes - Add tests under tests/ and plugins/.../tests/ covering the fix, code protection, and disabled-default behavior Closes #97 --- docs/plugins/filters/index.md | 2 +- docs/plugins/filters/index.zh.md | 2 +- docs/plugins/filters/markdown_normalizer.md | 8 +- .../plugins/filters/markdown_normalizer.zh.md | 8 +- .../markdown_normalizer/FEATURES_CN.md | 9 + plugins/filters/markdown_normalizer/README.md | 8 +- .../filters/markdown_normalizer/README_CN.md | 8 +- .../markdown_normalizer.py | 87 +++++++++- .../markdown_normalizer/tests/conftest.py | 24 +++ .../tests/test_intra_word_emphasis.py | 164 ++++++++++++++++++ plugins/filters/markdown_normalizer/v1.2.9.md | 15 ++ .../filters/markdown_normalizer/v1.2.9_CN.md | 15 ++ .../test_issue_97_regression.py | 103 +++++++++++ 13 files changed, 446 insertions(+), 7 deletions(-) create mode 100644 plugins/filters/markdown_normalizer/tests/test_intra_word_emphasis.py create mode 100644 plugins/filters/markdown_normalizer/v1.2.9.md create mode 100644 plugins/filters/markdown_normalizer/v1.2.9_CN.md create mode 100644 tests/plugins/filters/markdown_normalizer/test_issue_97_regression.py diff --git a/docs/plugins/filters/index.md b/docs/plugins/filters/index.md index 0a21dfe9..ffa4ccd3 100644 --- a/docs/plugins/filters/index.md +++ b/docs/plugins/filters/index.md @@ -52,7 +52,7 @@ Filters act as middleware in the message pipeline: Fixes common Markdown formatting issues in LLM outputs, including Mermaid syntax, code blocks, and LaTeX formulas. - **Version:** 1.2.8 + **Version:** 1.2.9 [:octicons-arrow-right-24: Documentation](markdown_normalizer.md) diff --git a/docs/plugins/filters/index.zh.md b/docs/plugins/filters/index.zh.md index 9fc33e79..d1107bdd 100644 --- a/docs/plugins/filters/index.zh.md +++ b/docs/plugins/filters/index.zh.md @@ -52,7 +52,7 @@ Filter 充当消息管线中的中间件： 修复 LLM 输出中常见的 Markdown 格式问题，包括 Mermaid 语法、代码块和 LaTeX 公式。 - **版本：** 1.2.8 + **版本：** 1.2.9 [:octicons-arrow-right-24: 查看文档](markdown_normalizer.zh.md) diff --git a/docs/plugins/filters/markdown_normalizer.md b/docs/plugins/filters/markdown_normalizer.md index 796e6d88..d956498c 100644 --- a/docs/plugins/filters/markdown_normalizer.md +++ b/docs/plugins/filters/markdown_normalizer.md @@ -1,6 +1,6 @@ # Markdown Normalizer Filter -| By [Fu-Jie](https://github.com/Fu-Jie) · v1.2.8 | [⭐ Star this repo](https://github.com/Fu-Jie/openwebui-extensions) | +| By [Fu-Jie](https://github.com/Fu-Jie) · v1.2.9 | [⭐ Star this repo](https://github.com/Fu-Jie/openwebui-extensions) | | :--- | ---: | | ![followers](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_followers.json&label=%F0%9F%91%A5&style=flat) | ![points](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_points.json&label=%E2%AD%90&style=flat) | ![top](https://img.shields.io/badge/%F0%9F%8F%86-Top%20%3C1%25-10b981?style=flat) | ![contributions](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_contributions.json&label=%F0%9F%93%A6&style=flat) | ![downloads](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_downloads.json&label=%E2%AC%87%EF%B8%8F&style=flat) | ![saves](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_saves.json&label=%F0%9F%92%BE&style=flat) | ![views](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_views.json&label=%F0%9F%91%81%EF%B8%8F&style=flat) | @@ -27,6 +27,10 @@ When the selection dialog opens, search for this plugin, check it, and continue. > [!IMPORTANT] > If the official OpenWebUI Community version is already installed, remove it first. After that, Batch Install Plugins can keep this plugin updated in future runs. +## 🔥 What's New in v1.2.9 + +* **Intra-word Emphasis Repair (issue [#97](https://github.com/Fu-Jie/openwebui-extensions/issues/97))**: New `enable_intra_word_emphasis_fix` Valve. When LLMs mark up punctuation-only additions inside a word boundary — e.g. `series**,**` (added comma in bold) or `word~~,~~` (added strikethrough) — most Markdown parsers refuse to render the inline emphasis because the opening delimiter is intra-word. The plugin now moves the opening marker to the start of the word (`series**,**` → `**series,**`) so the added punctuation renders together with the word. Perfect for grammar/spelling diff outputs. Only transforms cases where the wrapped content is pure punctuation, so legitimate `**bold**` / `word**bold**` markup is left untouched. Off by default. + ## 🔥 What's New in v1.2.8 * **Safe-by-Default Strategy**: The `enable_escape_fix` feature is now **disabled by default**. This prevents unwanted modifications to valid technical text like Windows file paths (`C:\new\test`) or complex LaTeX formulas. @@ -62,6 +66,7 @@ Before making any changes, the plugin builds a semantic map of the text to prote - **Details Tag Normalization**: `
` blocks (often used for Chain of Thought) require strict spacing to render correctly. The plugin automatically injects blank lines after `
` and self-closing `
` tags. * **Mermaid Syntax Fixer**: One of the most common LLM errors is omitting quotes in Mermaid diagrams (e.g., `A --> B(Some text)`). This plugin parses the Mermaid syntax and auto-quotes labels and citations to guarantee the graph renders. * **Emphasis Spacing Fix**: Fixes formatting-breaking extra spaces inside bold/italic markers (e.g., `** text **` becomes `**text**`) while cleverly ignoring math expressions like `2 * 3 * 4`. +* **Intra-word Emphasis Repair**: When LLMs mark up punctuation-only additions inside a word (e.g., `series**,**` for an added comma), most parsers refuse to render the inline emphasis. The plugin moves the opening marker to the start of the word (`series**,**` → `**series,**`) so the added punctuation renders together with the word. Useful for grammar/spelling diff outputs. * **Intelligent Escape Character Cleanup**: Removes excessive literal `\n` and `\t` generated by some models and converts them to actual structural newlines (only in safe text areas). * **LaTeX Standardization**: Automatically upgrades old-school LaTeX delimiters (`\[...\]` and `\(...\)`) to modern Markdown standards (`$$...$$` and `$ ... $`). * **Thought Tag Unification**: Standardizes various model thought outputs (``, ``) into a unified `` tag. @@ -102,6 +107,7 @@ The plugin UI and status notifications automatically switch based on your langua | `enable_table_fix` | `True` | Add missing closing pipe in tables. | | `enable_xml_tag_cleanup` | `True` | Remove leftover XML artifacts. | | `enable_emphasis_spacing_fix` | `False` | Fix extra spaces in emphasis formatting. | +| `enable_intra_word_emphasis_fix` | `False` | Move opening emphasis marker to the start of a word (e.g., `series**,**` → `**series,**`) so punctuation-only additions render. Useful for grammar/spelling diff outputs. | | `show_status` | `True` | Show UI status notification when a fix is actively applied. | | `show_debug_log` | `False` | Print detailed before/after diffs to browser console (F12). | diff --git a/docs/plugins/filters/markdown_normalizer.zh.md b/docs/plugins/filters/markdown_normalizer.zh.md index 63cfc0f7..669e2ebe 100644 --- a/docs/plugins/filters/markdown_normalizer.zh.md +++ b/docs/plugins/filters/markdown_normalizer.zh.md @@ -1,6 +1,6 @@ # Markdown 格式化过滤器 (Markdown Normalizer) -| 作者：[Fu-Jie](https://github.com/Fu-Jie) · v1.2.8 | [⭐ 点个 Star 支持项目](https://github.com/Fu-Jie/openwebui-extensions) | +| 作者：[Fu-Jie](https://github.com/Fu-Jie) · v1.2.9 | [⭐ 点个 Star 支持项目](https://github.com/Fu-Jie/openwebui-extensions) | | :--- | ---: | | ![followers](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_followers.json&label=%F0%9F%91%A5&style=flat) | ![points](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_points.json&label=%E2%AD%90&style=flat) | ![top](https://img.shields.io/badge/%F0%9F%8F%86-Top%20%3C1%25-10b981?style=flat) | ![contributions](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_contributions.json&label=%F0%9F%93%A6&style=flat) | ![downloads](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_downloads.json&label=%E2%AC%87%EF%B8%8F&style=flat) | ![saves](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_saves.json&label=%F0%9F%92%BE&style=flat) | ![views](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_views.json&label=%F0%9F%91%81%EF%B8%8F&style=flat) | @@ -27,6 +27,10 @@ > [!IMPORTANT] > 如果你已经安装了 OpenWebUI 官方社区里的同名版本，请先删除旧版本，否则重新安装时可能报错。删除后，Batch Install Plugins 后续就可以继续负责更新这个插件。 +## 🔥 最新更新 v1.2.9 + +* **词内强调修复 (issue [#97](https://github.com/Fu-Jie/openwebui-extensions/issues/97))**：新增 `enable_intra_word_emphasis_fix` 配置项。当 LLM 在词边界内标记仅含标点的添加时——例如 `series**,**`（添加的逗号用粗体）或 `word~~,~~`（添加的删除线）——大多数 Markdown 解析器会因为开头定界符位于词内而拒绝渲染该强调。插件现在会将开头标记移动到词的开头（`series**,**` → `**series,**`），让添加的标点能与单词一起渲染。非常适合语法/拼写差异输出。仅对包含纯标点内容的情况进行转换，因此合法的 `**bold**` / `word**bold**` 标记不会被改动。默认关闭。 + ## 🔥 最新更新 v1.2.8 * **“默认安全”策略 (Safe-by-Default)**：`enable_escape_fix` 功能现在**默认禁用**。这能有效防止插件在未经授权的情况下误改 Windows 路径 (`C:\new\test`) 或复杂的 LaTeX 公式。 @@ -60,6 +64,7 @@ - **Details 标签排版修复**：`
` 块要求极为严格的空行才能正确渲染内部内容。插件会自动在 `
` 以及自闭合 `
` 标签后注入安全的换行符。 * **Mermaid 语法急救**：自动修复最常见的 Mermaid 错误——为未加引号的节点标签（如 `A --> B(Some text)`）自动补充双引号，甚至支持多行标签和引用，确保拓扑图 100% 渲染。 * **强调语法间距修复**：修复加粗/斜体语法内部多余的空格（如 `** 文本 **` 变为 `**文本**`，否则 OpenWebUI 无法加粗），同时智能忽略数学算式（如 `2 * 3 * 4`）。 +* **词内强调修复**：当 LLM 在词边界内标记仅含标点的添加时（如 `series**,**` 表示添加了逗号），大多数解析器会拒绝渲染该强调。插件会将开头标记移动到词的开头（`series**,**` → `**series,**`），让添加的标点能与单词一起渲染。适合语法/拼写差异输出。 * **智能转义字符清理**：将模型过度转义生成的字面量 `\n` 和 `\t` 转化为真正的换行和缩进（仅在安全的纯文本区域执行）。 * **LaTeX 现代化转换**：自动将旧式的 LaTeX 定界符（`\[...\]` 和 `\(...\)`）升级为现代 Markdown 标准（`$$...$$` 和 `$ ... $`）。 * **思维标签大一统**：无论模型输出的是 `` 还是 ``，统一标准化为 `` 标签。 @@ -100,6 +105,7 @@ | `enable_table_fix` | `True` | 修复表格中缺失的闭合管道符。 | | `enable_xml_tag_cleanup` | `True` | 清理残留的 XML 分析标签。 | | `enable_emphasis_spacing_fix` | `False` | 修复强调语法（加粗/斜体）内部的多余空格。 | +| `enable_intra_word_emphasis_fix` | `False` | 将开头的强调标记移动到词的开头（如 `series**,**` → `**series,**`），让仅含标点的添加能正常渲染。适合语法/拼写差异输出。 | | `show_status` | `True` | 当触发任何修复规则时，在页面底部显示提示气泡。 | | `show_debug_log` | `False` | 在浏览器控制台 (F12) 打印修改前后的详细对比日志。 | diff --git a/plugins/filters/markdown_normalizer/FEATURES_CN.md b/plugins/filters/markdown_normalizer/FEATURES_CN.md index 6670a6fa..304a2209 100644 --- a/plugins/filters/markdown_normalizer/FEATURES_CN.md +++ b/plugins/filters/markdown_normalizer/FEATURES_CN.md @@ -186,3 +186,12 @@ Here is the result hidden metadata. **示例**: * **Before**: `| Col 1 | Col 2` * **After**: `| Col 1 | Col 2 |` + +## 11. 词内强调修复 (Intra-word Emphasis Fix) +**功能**: 当 LLM 在词边界内标记仅含标点的添加时（如语法/拼写差异输出中的 `series**,**` 表示添加了逗号），大多数 Markdown 解析器会拒绝渲染该强调，因为开头的定界符位于词内。本插件会将开头标记移动到词的开头，让添加的标点能与单词一起渲染。仅对包含纯标点内容的情况生效，合法的 `**bold**` / `word**bold**` 等标记不会被改动；代码块与行内代码也受到保护。 +**默认**: 关闭 (`enable_intra_word_emphasis_fix = False`) +**示例**: +* **Before**: `series**,**` (加粗的逗号无法渲染) +* **After**: `**series,**` (整个 "series," 加粗渲染) + +支持 `**` (加粗)、`~~` (删除线)、`__` (加粗)、`***` / `___` (加粗+斜体) 等所有多字符强调标记。 diff --git a/plugins/filters/markdown_normalizer/README.md b/plugins/filters/markdown_normalizer/README.md index 796e6d88..d956498c 100644 --- a/plugins/filters/markdown_normalizer/README.md +++ b/plugins/filters/markdown_normalizer/README.md @@ -1,6 +1,6 @@ # Markdown Normalizer Filter -| By [Fu-Jie](https://github.com/Fu-Jie) · v1.2.8 | [⭐ Star this repo](https://github.com/Fu-Jie/openwebui-extensions) | +| By [Fu-Jie](https://github.com/Fu-Jie) · v1.2.9 | [⭐ Star this repo](https://github.com/Fu-Jie/openwebui-extensions) | | :--- | ---: | | ![followers](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_followers.json&label=%F0%9F%91%A5&style=flat) | ![points](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_points.json&label=%E2%AD%90&style=flat) | ![top](https://img.shields.io/badge/%F0%9F%8F%86-Top%20%3C1%25-10b981?style=flat) | ![contributions](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_contributions.json&label=%F0%9F%93%A6&style=flat) | ![downloads](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_downloads.json&label=%E2%AC%87%EF%B8%8F&style=flat) | ![saves](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_saves.json&label=%F0%9F%92%BE&style=flat) | ![views](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_views.json&label=%F0%9F%91%81%EF%B8%8F&style=flat) | @@ -27,6 +27,10 @@ When the selection dialog opens, search for this plugin, check it, and continue. > [!IMPORTANT] > If the official OpenWebUI Community version is already installed, remove it first. After that, Batch Install Plugins can keep this plugin updated in future runs. +## 🔥 What's New in v1.2.9 + +* **Intra-word Emphasis Repair (issue [#97](https://github.com/Fu-Jie/openwebui-extensions/issues/97))**: New `enable_intra_word_emphasis_fix` Valve. When LLMs mark up punctuation-only additions inside a word boundary — e.g. `series**,**` (added comma in bold) or `word~~,~~` (added strikethrough) — most Markdown parsers refuse to render the inline emphasis because the opening delimiter is intra-word. The plugin now moves the opening marker to the start of the word (`series**,**` → `**series,**`) so the added punctuation renders together with the word. Perfect for grammar/spelling diff outputs. Only transforms cases where the wrapped content is pure punctuation, so legitimate `**bold**` / `word**bold**` markup is left untouched. Off by default. + ## 🔥 What's New in v1.2.8 * **Safe-by-Default Strategy**: The `enable_escape_fix` feature is now **disabled by default**. This prevents unwanted modifications to valid technical text like Windows file paths (`C:\new\test`) or complex LaTeX formulas. @@ -62,6 +66,7 @@ Before making any changes, the plugin builds a semantic map of the text to prote - **Details Tag Normalization**: `
` blocks (often used for Chain of Thought) require strict spacing to render correctly. The plugin automatically injects blank lines after `
` and self-closing `
` tags. * **Mermaid Syntax Fixer**: One of the most common LLM errors is omitting quotes in Mermaid diagrams (e.g., `A --> B(Some text)`). This plugin parses the Mermaid syntax and auto-quotes labels and citations to guarantee the graph renders. * **Emphasis Spacing Fix**: Fixes formatting-breaking extra spaces inside bold/italic markers (e.g., `** text **` becomes `**text**`) while cleverly ignoring math expressions like `2 * 3 * 4`. +* **Intra-word Emphasis Repair**: When LLMs mark up punctuation-only additions inside a word (e.g., `series**,**` for an added comma), most parsers refuse to render the inline emphasis. The plugin moves the opening marker to the start of the word (`series**,**` → `**series,**`) so the added punctuation renders together with the word. Useful for grammar/spelling diff outputs. * **Intelligent Escape Character Cleanup**: Removes excessive literal `\n` and `\t` generated by some models and converts them to actual structural newlines (only in safe text areas). * **LaTeX Standardization**: Automatically upgrades old-school LaTeX delimiters (`\[...\]` and `\(...\)`) to modern Markdown standards (`$$...$$` and `$ ... $`). * **Thought Tag Unification**: Standardizes various model thought outputs (``, ``) into a unified `` tag. @@ -102,6 +107,7 @@ The plugin UI and status notifications automatically switch based on your langua | `enable_table_fix` | `True` | Add missing closing pipe in tables. | | `enable_xml_tag_cleanup` | `True` | Remove leftover XML artifacts. | | `enable_emphasis_spacing_fix` | `False` | Fix extra spaces in emphasis formatting. | +| `enable_intra_word_emphasis_fix` | `False` | Move opening emphasis marker to the start of a word (e.g., `series**,**` → `**series,**`) so punctuation-only additions render. Useful for grammar/spelling diff outputs. | | `show_status` | `True` | Show UI status notification when a fix is actively applied. | | `show_debug_log` | `False` | Print detailed before/after diffs to browser console (F12). | diff --git a/plugins/filters/markdown_normalizer/README_CN.md b/plugins/filters/markdown_normalizer/README_CN.md index 63cfc0f7..669e2ebe 100644 --- a/plugins/filters/markdown_normalizer/README_CN.md +++ b/plugins/filters/markdown_normalizer/README_CN.md @@ -1,6 +1,6 @@ # Markdown 格式化过滤器 (Markdown Normalizer) -| 作者：[Fu-Jie](https://github.com/Fu-Jie) · v1.2.8 | [⭐ 点个 Star 支持项目](https://github.com/Fu-Jie/openwebui-extensions) | +| 作者：[Fu-Jie](https://github.com/Fu-Jie) · v1.2.9 | [⭐ 点个 Star 支持项目](https://github.com/Fu-Jie/openwebui-extensions) | | :--- | ---: | | ![followers](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_followers.json&label=%F0%9F%91%A5&style=flat) | ![points](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_points.json&label=%E2%AD%90&style=flat) | ![top](https://img.shields.io/badge/%F0%9F%8F%86-Top%20%3C1%25-10b981?style=flat) | ![contributions](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_contributions.json&label=%F0%9F%93%A6&style=flat) | ![downloads](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_downloads.json&label=%E2%AC%87%EF%B8%8F&style=flat) | ![saves](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_saves.json&label=%F0%9F%92%BE&style=flat) | ![views](https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2FFu-Jie%2Fdb3d95687075a880af6f1fba76d679c6%2Fraw%2Fbadge_views.json&label=%F0%9F%91%81%EF%B8%8F&style=flat) | @@ -27,6 +27,10 @@ > [!IMPORTANT] > 如果你已经安装了 OpenWebUI 官方社区里的同名版本，请先删除旧版本，否则重新安装时可能报错。删除后，Batch Install Plugins 后续就可以继续负责更新这个插件。 +## 🔥 最新更新 v1.2.9 + +* **词内强调修复 (issue [#97](https://github.com/Fu-Jie/openwebui-extensions/issues/97))**：新增 `enable_intra_word_emphasis_fix` 配置项。当 LLM 在词边界内标记仅含标点的添加时——例如 `series**,**`（添加的逗号用粗体）或 `word~~,~~`（添加的删除线）——大多数 Markdown 解析器会因为开头定界符位于词内而拒绝渲染该强调。插件现在会将开头标记移动到词的开头（`series**,**` → `**series,**`），让添加的标点能与单词一起渲染。非常适合语法/拼写差异输出。仅对包含纯标点内容的情况进行转换，因此合法的 `**bold**` / `word**bold**` 标记不会被改动。默认关闭。 + ## 🔥 最新更新 v1.2.8 * **“默认安全”策略 (Safe-by-Default)**：`enable_escape_fix` 功能现在**默认禁用**。这能有效防止插件在未经授权的情况下误改 Windows 路径 (`C:\new\test`) 或复杂的 LaTeX 公式。 @@ -60,6 +64,7 @@ - **Details 标签排版修复**：`
` 块要求极为严格的空行才能正确渲染内部内容。插件会自动在 `
` 以及自闭合 `
` 标签后注入安全的换行符。 * **Mermaid 语法急救**：自动修复最常见的 Mermaid 错误——为未加引号的节点标签（如 `A --> B(Some text)`）自动补充双引号，甚至支持多行标签和引用，确保拓扑图 100% 渲染。 * **强调语法间距修复**：修复加粗/斜体语法内部多余的空格（如 `** 文本 **` 变为 `**文本**`，否则 OpenWebUI 无法加粗），同时智能忽略数学算式（如 `2 * 3 * 4`）。 +* **词内强调修复**：当 LLM 在词边界内标记仅含标点的添加时（如 `series**,**` 表示添加了逗号），大多数解析器会拒绝渲染该强调。插件会将开头标记移动到词的开头（`series**,**` → `**series,**`），让添加的标点能与单词一起渲染。适合语法/拼写差异输出。 * **智能转义字符清理**：将模型过度转义生成的字面量 `\n` 和 `\t` 转化为真正的换行和缩进（仅在安全的纯文本区域执行）。 * **LaTeX 现代化转换**：自动将旧式的 LaTeX 定界符（`\[...\]` 和 `\(...\)`）升级为现代 Markdown 标准（`$$...$$` 和 `$ ... $`）。 * **思维标签大一统**：无论模型输出的是 `` 还是 ``，统一标准化为 `` 标签。 @@ -100,6 +105,7 @@ | `enable_table_fix` | `True` | 修复表格中缺失的闭合管道符。 | | `enable_xml_tag_cleanup` | `True` | 清理残留的 XML 分析标签。 | | `enable_emphasis_spacing_fix` | `False` | 修复强调语法（加粗/斜体）内部的多余空格。 | +| `enable_intra_word_emphasis_fix` | `False` | 将开头的强调标记移动到词的开头（如 `series**,**` → `**series,**`），让仅含标点的添加能正常渲染。适合语法/拼写差异输出。 | | `show_status` | `True` | 当触发任何修复规则时，在页面底部显示提示气泡。 | | `show_debug_log` | `False` | 在浏览器控制台 (F12) 打印修改前后的详细对比日志。 | diff --git a/plugins/filters/markdown_normalizer/markdown_normalizer.py b/plugins/filters/markdown_normalizer/markdown_normalizer.py index 5028b05f..4bfd04ec 100644 --- a/plugins/filters/markdown_normalizer/markdown_normalizer.py +++ b/plugins/filters/markdown_normalizer/markdown_normalizer.py @@ -3,7 +3,7 @@ author: Fu-Jie author_url: https://github.com/Fu-Jie/openwebui-extensions funding_url: https://github.com/open-webui -version: 1.2.8 +version: 1.2.9 openwebui_id: baaa8732-9348-40b7-8359-7e009660e23c description: A content normalizer filter that fixes common Markdown formatting issues in LLM outputs, such as broken code blocks, LaTeX formulas, and list formatting. Including LaTeX command protection. """ @@ -38,6 +38,7 @@ "fix_table": "Table Format", "fix_xml": "XML Cleanup", "fix_emphasis": "Emphasis Spacing", + "fix_intra_word": "Intra-word Emphasis", "fix_custom": "Custom Cleaner", }, "zh-CN": { @@ -55,6 +56,7 @@ "fix_table": "表格格式", "fix_xml": "XML清理", "fix_emphasis": "强调空格", + "fix_intra_word": "词内强调", "fix_custom": "自定义清理", }, "zh-HK": { @@ -72,6 +74,7 @@ "fix_table": "表格格式", "fix_xml": "XML清理", "fix_emphasis": "強調空格", + "fix_intra_word": "詞內強調", "fix_custom": "自訂清理", }, "zh-TW": { @@ -89,6 +92,7 @@ "fix_table": "表格格式", "fix_xml": "XML清理", "fix_emphasis": "強調空格", + "fix_intra_word": "詞內強調", "fix_custom": "自訂清理", }, "ko-KR": { @@ -106,6 +110,7 @@ "fix_table": "표 형식", "fix_xml": "XML 정리", "fix_emphasis": "강조 공백", + "fix_intra_word": "단어 내 강조", "fix_custom": "사용자 정의 정리", }, "ja-JP": { @@ -123,6 +128,7 @@ "fix_table": "表形式", "fix_xml": "XMLクリーンアップ", "fix_emphasis": "強調の空白", + "fix_intra_word": "単語内強調", "fix_custom": "カスタムクリーナー", }, "fr-FR": { @@ -140,6 +146,7 @@ "fix_table": "Format de tableau", "fix_xml": "Nettoyage XML", "fix_emphasis": "Espacement d'emphase", + "fix_intra_word": "Emphase intra-mot", "fix_custom": "Nettoyeur personnalisé", }, "de-DE": { @@ -157,6 +164,7 @@ "fix_table": "Tabellenformat", "fix_xml": "XML-Bereinigung", "fix_emphasis": "Hervorhebungsabstände", + "fix_intra_word": "Wortinterne Hervorhebung", "fix_custom": "Benutzerdefinierter Reiniger", }, "es-ES": { @@ -174,6 +182,7 @@ "fix_table": "Formato de tabla", "fix_xml": "Limpieza XML", "fix_emphasis": "Espaciado de énfasis", + "fix_intra_word": "Énfasis intra-palabra", "fix_custom": "Limpiador personalizado", }, "it-IT": { @@ -191,6 +200,7 @@ "fix_table": "Formato tabella", "fix_xml": "Pulizia XML", "fix_emphasis": "Spaziatura enfasi", + "fix_intra_word": "Enfasi intra-parola", "fix_custom": "Pulitore personalizzato", }, "vi-VN": { @@ -208,6 +218,7 @@ "fix_table": "Định dạng bảng", "fix_xml": "Dọn dẹp XML", "fix_emphasis": "Khoảng cách nhấn mạnh", + "fix_intra_word": "Nhấn mạnh trong từ", "fix_custom": "Trình dọn dẹp tùy chỉnh", }, "id-ID": { @@ -225,6 +236,7 @@ "fix_table": "Format tabel", "fix_xml": "Pembersihan XML", "fix_emphasis": "Spasi penekanan", + "fix_intra_word": "Penekanan intra-kata", "fix_custom": "Pembersih kustom", }, } @@ -256,6 +268,9 @@ class NormalizerConfig: enable_table_fix: bool = True # Fix missing closing pipe in tables enable_xml_tag_cleanup: bool = True # Cleanup leftover XML tags enable_emphasis_spacing_fix: bool = False # Fix spaces inside **emphasis** + enable_intra_word_emphasis_fix: bool = ( + False # Move opening emphasis marker to word start (e.g. series**,** -> **series,**) + ) # Custom cleaner functions (for advanced extension) custom_cleaners: List[Callable[[str], str]] = field(default_factory=list) @@ -329,6 +344,18 @@ class ContentNormalizer: "emphasis_spacing": re.compile( r"(?(?:(?!\1)[^\n])*?)(\1)(?!\*|_)" ), + # Intra-word emphasis: series**,** -> **series,** + # Matches an alphanumeric word directly followed by an opening emphasis marker + # (~~, **, __, ***, ___) that wraps ONLY punctuation (no whitespace/word chars), + # then the same closing marker. The opening marker is moved before the word so + # the punctuation (e.g. an added comma) renders together with the word. + # NOTE: Alternation lists longer markers first so *** / ___ win over ** / __. + "intra_word_emphasis": re.compile( + r"(?P[^\W_]+)" + r"(?P\*\*\*|___|\*\*|__|~~)" + r"(?P[^\s\w]+?)" + r"(?P=marker)" + ), } def __init__(self, config: Optional[NormalizerConfig] = None): @@ -435,6 +462,14 @@ def normalize(self, content: str) -> str: if content != original: self.applied_fixes.append("Fix Emphasis Spacing") + # 13. Intra-word emphasis fix (run after emphasis spacing so any + # inner spaces are already collapsed, e.g. `** , **` -> `**, **` first) + if self.config.enable_intra_word_emphasis_fix: + original = content + content = self._fix_intra_word_emphasis(content) + if content != original: + self.applied_fixes.append("Fix Intra-word Emphasis") + # 9. Custom cleaners for cleaner in self.config.custom_cleaners: original = content @@ -718,6 +753,46 @@ def replacer(match): parts[i] = new_part return "```".join(parts) + def _fix_intra_word_emphasis(self, content: str) -> str: + """Move opening emphasis marker to the start of an intra-word group. + + When LLMs mark up punctuation-only additions inside a word boundary, + e.g. `series**,**` (added comma) or `word~~,~~` (added strikethrough), + many Markdown parsers refuse to render the inline emphasis because the + opening delimiter is intra-word. Moving the opening marker to the start + of the word produces a renderable `**series,**` / `~~word,~~`. + + Only transforms cases where the wrapped content is *pure punctuation* + (no whitespace, no word characters) so legitimate `**bold**` / + `__bold__` / `~~strike~~` markup is left untouched. Code blocks and + inline code are skipped via the standard `split("```")` partitioning. + """ + + def replacer(match): + word = match.group("word") + marker = match.group("marker") + inner = match.group("content") + # Skip if content somehow slipped through with whitespace or word + # chars (defensive; the regex already restricts this). + if any(c.isspace() or c.isalnum() for c in inner): + return match.group(0) + return f"{marker}{word}{inner}{marker}" + + parts = content.split("```") + for i in range(0, len(parts), 2): # Even indices are markdown text + # Protect inline code (backtick spans) from modification. + inline_parts = parts[i].split("`") + for k in range(0, len(inline_parts), 2): # Even indices = non-code + while True: + new_segment = self._PATTERNS["intra_word_emphasis"].sub( + replacer, inline_parts[k] + ) + if new_segment == inline_parts[k]: + break + inline_parts[k] = new_segment + parts[i] = "`".join(inline_parts) + return "```".join(parts) + class Filter: class Valves(BaseModel): @@ -781,6 +856,14 @@ class Valves(BaseModel): default=False, description="Fix spaces inside **emphasis** (e.g. ** text ** -> **text**).", ) + enable_intra_word_emphasis_fix: bool = Field( + default=False, + description=( + "Move opening emphasis marker to the start of an intra-word " + "group (e.g. series**,** -> **series,**) so added punctuation " + "renders. Useful for grammar/spelling diff outputs." + ), + ) show_status: bool = Field( default=True, description="Show status notification when fixes are applied.", @@ -964,6 +1047,7 @@ async def _emit_status( "Fix Tables": "fix_table", "Cleanup XML Tags": "fix_xml", "Fix Emphasis Spacing": "fix_emphasis", + "Fix Intra-word Emphasis": "fix_intra_word", "Custom Cleaner": "fix_custom", } @@ -1061,6 +1145,7 @@ async def outlet( enable_table_fix=self.valves.enable_table_fix, enable_xml_tag_cleanup=self.valves.enable_xml_tag_cleanup, enable_emphasis_spacing_fix=self.valves.enable_emphasis_spacing_fix, + enable_intra_word_emphasis_fix=self.valves.enable_intra_word_emphasis_fix, ) normalizer = ContentNormalizer(config) diff --git a/plugins/filters/markdown_normalizer/tests/conftest.py b/plugins/filters/markdown_normalizer/tests/conftest.py index 3ace6e28..5a3371ec 100644 --- a/plugins/filters/markdown_normalizer/tests/conftest.py +++ b/plugins/filters/markdown_normalizer/tests/conftest.py @@ -29,6 +29,7 @@ def normalizer(): enable_table_fix=True, enable_xml_tag_cleanup=True, enable_emphasis_spacing_fix=True, + enable_intra_word_emphasis_fix=True, ) return ContentNormalizer(config) @@ -50,6 +51,29 @@ def emphasis_only_normalizer(): enable_table_fix=False, enable_xml_tag_cleanup=False, enable_emphasis_spacing_fix=True, + enable_intra_word_emphasis_fix=False, + ) + return ContentNormalizer(config) + + +@pytest.fixture +def intra_word_only_normalizer(): + """Normalizer with only intra-word emphasis fix enabled.""" + config = NormalizerConfig( + enable_escape_fix=False, + enable_thought_tag_fix=False, + enable_details_tag_fix=False, + enable_code_block_fix=False, + enable_latex_fix=False, + enable_list_fix=False, + enable_unclosed_block_fix=False, + enable_fullwidth_symbol_fix=False, + enable_mermaid_fix=False, + enable_heading_fix=False, + enable_table_fix=False, + enable_xml_tag_cleanup=False, + enable_emphasis_spacing_fix=False, + enable_intra_word_emphasis_fix=True, ) return ContentNormalizer(config) diff --git a/plugins/filters/markdown_normalizer/tests/test_intra_word_emphasis.py b/plugins/filters/markdown_normalizer/tests/test_intra_word_emphasis.py new file mode 100644 index 00000000..0662549a --- /dev/null +++ b/plugins/filters/markdown_normalizer/tests/test_intra_word_emphasis.py @@ -0,0 +1,164 @@ +""" +Tests for intra-word emphasis fix (issue #97). + +Covers the case where LLMs mark up punctuation-only additions inside a word +boundary, e.g. `series**,**` (added comma rendered in bold) which most Markdown +parsers refuse to render because the opening delimiter is intra-word. The fix +moves the opening marker to the start of the word so the wrapped punctuation +renders together with the word. +""" + +import pytest + + +class TestIntraWordEmphasisFix: + """Test that intra-word emphasis markers are moved to the word start.""" + + @pytest.mark.parametrize( + "input_str,expected", + [ + # Bold (**) wrapping a single added punctuation char + ("series**,**", "**series,**"), + ("series**.**", "**series.**"), + ("series**!**", "**series!**"), + ("series**;**", "**series;**"), + ("series**:**", "**series:**"), + # Strikethrough (~~) — the issue's primary use case + ("series~~,~~", "~~series,~~"), + ("word~~.~~", "~~word.~~"), + # Underscore bold (__) wrapping punctuation + ("word__!__", "__word!__"), + ("word__,__", "__word,__"), + # Triple markers (*** / ___) wrapping punctuation + ("word***,***", "***word,***"), + ("word___,___", "___word,___"), + # Multiple punctuation chars wrapped + ("series**?!**", "**series?!**"), + ("series**...**", "**series...**"), + # Numeric "word" prefix is also handled + ("100**,**", "**100,**"), + # Multiple occurrences in one line + ( + "series**,** and word2**.**", + "**series,** and **word2.**", + ), + # Inside a sentence + ( + "The text series**,** is now fixed.", + "The text **series,** is now fixed.", + ), + # CJK word prefix + ("中文**,**", "**中文,**"), + ], + ) + def test_intra_word_emphasis_fixed( + self, intra_word_only_normalizer, input_str, expected + ): + """Intra-word emphasis wrapping pure punctuation should be moved.""" + assert intra_word_only_normalizer.normalize(input_str) == expected + + @pytest.mark.parametrize( + "input_str", + [ + # Already-correct emphasis at line start (no leading word) + "**bold**", + "~~strike~~", + "__bold__", + "***bold italic***", + "___bold italic___", + # Legitimate emphasis preceded by a word but wrapping text + "word**bold**", + "word~~strike~~", + "word__bold__", + # Math-like intra-word emphasis wrapping digits/letters (legitimate) + "a**2**", + "x**2**b", + # Emphasis wrapping content with whitespace (already renderable) + "series** , **", + "series**a b**", + # Underscore inside identifiers (snake_case) — must not match + "my_var__bold__", + "word_with_underscore", + "a_b_c_d", + # Horizontal rules + "---", + "***", + "___", + ], + ) + def test_safe_cases_unchanged(self, intra_word_only_normalizer, input_str): + """Legitimate emphasis and identifiers must not be modified.""" + assert intra_word_only_normalizer.normalize(input_str) == input_str + + +class TestIntraWordEmphasisCodeProtection: + """Test that code blocks and inline code are never modified.""" + + def test_intra_word_emphasis_in_code_block_unchanged( + self, intra_word_only_normalizer + ): + """Code blocks should be completely skipped.""" + input_str = "```python\nseries**,** should stay\n```" + assert intra_word_only_normalizer.normalize(input_str) == input_str + + def test_intra_word_emphasis_in_inline_code_unchanged( + self, intra_word_only_normalizer + ): + """Inline code spans should be completely skipped.""" + input_str = "Use `series**,**` as a variable name." + assert intra_word_only_normalizer.normalize(input_str) == input_str + + def test_mixed_text_and_code_block( + self, intra_word_only_normalizer + ): + """Only markdown text outside code blocks should be fixed.""" + input_str = ( + "Outside series**,** text\n" + "```python\nseries**,** unchanged\n```\n" + "More word**.** outside." + ) + expected = ( + "Outside **series,** text\n" + "```python\nseries**,** unchanged\n```\n" + "More **word.** outside." + ) + assert intra_word_only_normalizer.normalize(input_str) == expected + + def test_mixed_text_and_inline_code( + self, intra_word_only_normalizer + ): + """Only text outside inline code should be fixed.""" + input_str = "Fix series**,** but skip `series**,**` here." + expected = "Fix **series,** but skip `series**,**` here." + assert intra_word_only_normalizer.normalize(input_str) == expected + + +class TestIntraWordEmphasisCombinedWithSpacing: + """Test interaction with the emphasis spacing fix when both are enabled.""" + + def test_spacing_then_intra_word(self, normalizer): + """`series** , **` -> spacing fixes inner spaces -> intra-word moves marker. + + With both fixes enabled the result should be `**series,**`. + """ + input_str = "series** , **" + expected = "**series,**" + assert normalizer.normalize(input_str) == expected + + def test_already_correct_unchanged_with_both(self, normalizer): + """Already-correct `**series,**` should remain unchanged.""" + input_str = "**series,**" + assert normalizer.normalize(input_str) == input_str + + +class TestIntraWordEmphasisDisabled: + """Test that the fix is a no-op when disabled.""" + + def test_disabled_does_not_modify(self, emphasis_only_normalizer): + """When only emphasis_spacing is enabled (intra-word off), the + punctuation-only intra-word pattern must NOT be moved. + """ + input_str = "series**,**" + # emphasis_spacing only strips inner spaces, content `,` is untouched, + # so the intra-word structure remains. + assert emphasis_only_normalizer.normalize(input_str) == input_str diff --git a/plugins/filters/markdown_normalizer/v1.2.9.md b/plugins/filters/markdown_normalizer/v1.2.9.md new file mode 100644 index 00000000..c4d07653 --- /dev/null +++ b/plugins/filters/markdown_normalizer/v1.2.9.md @@ -0,0 +1,15 @@ +# v1.2.9 Release Notes + +This release adds a new opt-in Valve that fixes a long-standing rendering issue when LLMs mark up punctuation-only additions inside a word boundary (e.g. for grammar/spelling diff outputs). + +## New Features + +- **Intra-word Emphasis Repair** (issue [#97](https://github.com/Fu-Jie/openwebui-extensions/issues/97)): New `enable_intra_word_emphasis_fix` Valve. When LLMs mark up punctuation-only additions inside a word boundary — e.g. `series**,**` (added comma rendered in bold) or `word~~,~~` (added strikethrough) — most Markdown parsers refuse to render the inline emphasis because the opening delimiter is intra-word. The plugin now moves the opening marker to the start of the word so the added punctuation renders together with the word: + + | Before | After | + | --- | --- | + | `series**,**` | `**series,**` | + | `word~~,~~` | `~~word,~~` | + | `word__,__` | `__word,__` | + + Only transforms cases where the wrapped content is pure punctuation (no whitespace, no word characters), so legitimate `**bold**` / `word**bold**` markup, math-like `a**2**`, and snake_case identifiers (`my_var__bold__`) are left untouched. Code blocks and inline code spans are also protected. Off by default for safety. diff --git a/plugins/filters/markdown_normalizer/v1.2.9_CN.md b/plugins/filters/markdown_normalizer/v1.2.9_CN.md new file mode 100644 index 00000000..d9b5aa74 --- /dev/null +++ b/plugins/filters/markdown_normalizer/v1.2.9_CN.md @@ -0,0 +1,15 @@ +# v1.2.9 版本发布说明 + +本次更新新增了一个可选启用的配置项，用于修复 LLM 在词边界内标记仅含标点的添加时（例如语法/拼写差异输出场景）长期存在的渲染问题。 + +## 新功能 + +- **词内强调修复 (Intra-word Emphasis Repair)**（issue [#97](https://github.com/Fu-Jie/openwebui-extensions/issues/97)）：新增 `enable_intra_word_emphasis_fix` 配置项。当 LLM 在词边界内标记仅含标点的添加时——例如 `series**,**`（添加的逗号用粗体）或 `word~~,~~`（添加的删除线）——大多数 Markdown 解析器会因为开头定界符位于词内而拒绝渲染该强调。插件现在会将开头标记移动到词的开头，让添加的标点能与单词一起渲染： + + | 修改前 | 修改后 | + | --- | --- | + | `series**,**` | `**series,**` | + | `word~~,~~` | `~~word,~~` | + | `word__,__` | `__word,__` | + + 仅对包含纯标点内容（无空格、无单词字符）的情况进行转换，因此合法的 `**bold**` / `word**bold**` 标记、数学算式 `a**2**` 以及 snake_case 标识符（`my_var__bold__`）均不会被改动。代码块和行内代码也受到保护。默认关闭以保证安全。 diff --git a/tests/plugins/filters/markdown_normalizer/test_issue_97_regression.py b/tests/plugins/filters/markdown_normalizer/test_issue_97_regression.py new file mode 100644 index 00000000..b0c31d58 --- /dev/null +++ b/tests/plugins/filters/markdown_normalizer/test_issue_97_regression.py @@ -0,0 +1,103 @@ +"""Regression test for issue #97: intra-word emphasis rendering. + +Issue: When LLMs mark up punctuation-only additions inside a word boundary, +e.g. `series**,**` (added comma rendered in bold) or `word~~,~~` (added +strikethrough), the Open WebUI Markdown parser refuses to render the inline +emphasis because the opening delimiter is intra-word. + +The fix moves the opening emphasis marker to the start of the word so the +wrapped punctuation renders together with the word: + `series**,**` -> `**series,**` + `word~~,~~` -> `~~word,~~` +""" + +from plugins.filters.markdown_normalizer.markdown_normalizer import ( + ContentNormalizer, + NormalizerConfig, +) + + +def _norm(**kwargs): + """Build a normalizer with all fixes off, then enable the requested ones.""" + config_kwargs = dict( + enable_escape_fix=False, + enable_escape_fix_in_code_blocks=False, + enable_thought_tag_fix=False, + enable_details_tag_fix=False, + enable_code_block_fix=False, + enable_latex_fix=False, + enable_list_fix=False, + enable_unclosed_block_fix=False, + enable_fullwidth_symbol_fix=False, + enable_mermaid_fix=False, + enable_heading_fix=False, + enable_table_fix=False, + enable_xml_tag_cleanup=False, + enable_emphasis_spacing_fix=False, + enable_intra_word_emphasis_fix=False, + ) + config_kwargs.update(kwargs) + return ContentNormalizer(NormalizerConfig(**config_kwargs)) + + +def test_intra_word_bold_punctuation_moved(): + """`series**,**` should become `**series,**`.""" + norm = _norm(enable_intra_word_emphasis_fix=True) + assert norm.normalize("series**,**") == "**series,**" + + +def test_intra_word_strikethrough_punctuation_moved(): + """`word~~,~~` should become `~~word,~~`.""" + norm = _norm(enable_intra_word_emphasis_fix=True) + assert norm.normalize("word~~,~~") == "~~word,~~" + + +def test_intra_word_underscore_bold_punctuation_moved(): + """`word__,__` should become `__word,__`.""" + norm = _norm(enable_intra_word_emphasis_fix=True) + assert norm.normalize("word__,__") == "__word,__" + + +def test_legitimate_emphasis_preserved(): + """Already-correct emphasis and emphasis wrapping text must not change.""" + norm = _norm(enable_intra_word_emphasis_fix=True) + for unchanged in ( + "**bold**", + "word**bold**", # content has letters -> legitimate + "a**2**", # content is digit -> math-like, leave alone + "my_var__bold__", # snake_case identifier, no match + ): + assert norm.normalize(unchanged) == unchanged, repr(unchanged) + + +def test_code_block_and_inline_code_protected(): + """Intra-word emphasis inside code blocks and inline code is preserved.""" + norm = _norm(enable_intra_word_emphasis_fix=True) + code_block = "```python\nseries**,** unchanged\n```" + assert norm.normalize(code_block) == code_block + inline_code = "Use `series**,**` here." + assert norm.normalize(inline_code) == inline_code + + +def test_intra_word_disabled_by_default(): + """When disabled (the default), the intra-word pattern is left as-is.""" + # Use the default config: enable_intra_word_emphasis_fix defaults to False. + norm = ContentNormalizer(NormalizerConfig()) + assert norm.normalize("series**,**") == "series**,**" + + +if __name__ == "__main__": + for fn in ( + test_intra_word_bold_punctuation_moved, + test_intra_word_strikethrough_punctuation_moved, + test_intra_word_underscore_bold_punctuation_moved, + test_legitimate_emphasis_preserved, + test_code_block_and_inline_code_protected, + test_intra_word_disabled_by_default, + ): + try: + fn() + print(f"✅ {fn.__name__} passed.") + except AssertionError as e: + print(f"❌ {fn.__name__} FAILED: {e}") + raise From 5a287405ae94573722bca8476d709661b4cc5276 Mon Sep 17 00:00:00 2001 From: Fu-Jie Date: Tue, 30 Jun 2026 12:36:02 +0000 Subject: [PATCH 2/2] fix(markdown-normalizer): address Gemini review on intra-word emphasis 1. Refine word pattern: `[^\W_]+` -> `[^\W_]+(?:_[^\W_]+)*` - Snake_case identifiers (`my_var**,**`) now stay intact instead of being split into `my_` + `var` (Gemini review, high priority). - Trailing underscores are left for the marker alternation to consume, so triple-underscore markers (`word___,___`) keep working correctly. - Legitimate `my_var__bold__` is still protected (content excludes word chars, so the pattern never matches). 2. Remove redundant `while True` loop in `_fix_intra_word_emphasis`. - The pattern cannot nest (content is restricted to pure punctuation), so a single `re.sub` pass is sufficient (Gemini review, medium). 3. Add snake_case test cases covering `my_var**,**`, `my_var__,__`, `my_var~~,~~`, and `snake_case_var**.**`. Test results: 110 passed, 3 failed (all 3 failures are pre-existing on `main` and unrelated to this change). --- .../markdown_normalizer.py | 23 ++++++++++++------- .../tests/test_intra_word_emphasis.py | 7 ++++++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/plugins/filters/markdown_normalizer/markdown_normalizer.py b/plugins/filters/markdown_normalizer/markdown_normalizer.py index 4bfd04ec..73c654d7 100644 --- a/plugins/filters/markdown_normalizer/markdown_normalizer.py +++ b/plugins/filters/markdown_normalizer/markdown_normalizer.py @@ -350,8 +350,16 @@ class ContentNormalizer: # then the same closing marker. The opening marker is moved before the word so # the punctuation (e.g. an added comma) renders together with the word. # NOTE: Alternation lists longer markers first so *** / ___ win over ** / __. + # NOTE: `word` is `[^\W_]+(?:_[^\W_]+)*` so snake_case identifiers like + # `my_var` stay intact instead of being split into `my_` + `var`. The + # trailing `(?:_[^\W_]+)*` only allows underscores that sit *between* + # word-char runs, so a run of trailing underscores (e.g. `word___`) + # is left for the marker alternation to consume as `___` rather than + # being swallowed by `word`. This keeps triple-underscore markers + # (`word___,___`) working while still protecting `my_var__bold__` + # (which never matches because `content` excludes word chars). "intra_word_emphasis": re.compile( - r"(?P[^\W_]+)" + r"(?P[^\W_]+(?:_[^\W_]+)*)" r"(?P\*\*\*|___|\*\*|__|~~)" r"(?P[^\s\w]+?)" r"(?P=marker)" @@ -783,13 +791,12 @@ def replacer(match): # Protect inline code (backtick spans) from modification. inline_parts = parts[i].split("`") for k in range(0, len(inline_parts), 2): # Even indices = non-code - while True: - new_segment = self._PATTERNS["intra_word_emphasis"].sub( - replacer, inline_parts[k] - ) - if new_segment == inline_parts[k]: - break - inline_parts[k] = new_segment + # Single-pass is sufficient: the pattern cannot nest because + # `content` is restricted to pure punctuation, so a match cannot + # contain another word+marker+content+marker sequence. + inline_parts[k] = self._PATTERNS["intra_word_emphasis"].sub( + replacer, inline_parts[k] + ) parts[i] = "`".join(inline_parts) return "```".join(parts) diff --git a/plugins/filters/markdown_normalizer/tests/test_intra_word_emphasis.py b/plugins/filters/markdown_normalizer/tests/test_intra_word_emphasis.py index 0662549a..3a832152 100644 --- a/plugins/filters/markdown_normalizer/tests/test_intra_word_emphasis.py +++ b/plugins/filters/markdown_normalizer/tests/test_intra_word_emphasis.py @@ -49,6 +49,13 @@ class TestIntraWordEmphasisFix: ), # CJK word prefix ("中文**,**", "**中文,**"), + # snake_case variable name with punctuation addition — the + # underscore inside the identifier must NOT split the word, so + # `my_var` stays intact and the marker moves to its start. + ("my_var__,__", "__my_var,__"), + ("my_var**,**", "**my_var,**"), + ("my_var~~,~~", "~~my_var,~~"), + ("snake_case_var**.**", "**snake_case_var.**"), ], ) def test_intra_word_emphasis_fixed(