mathjax渲染器方案优化

llm_web_kit/extractor/config.py

@@ -6,4 +6,5 @@
     {'url': 'stackexchange.com', 'tag': '//*[contains(@class, "d-none")]'},  # 任意标签，class包含d-none，限制在stackexchange.com网站
     {'url': 'mathoverflow.net', 'tag': '//*[contains(@class, "d-none")]'},  # 任意标签，class包含d-none，限制在mathoverflow.net网站
     {'url': 'blog.csdn.net', 'tag': '//span[contains(@class, "katex-html")]'},  # 仅针对 blog.csdn.net 域名，删除所有 class 包含 katex-html 的 <span> 标签及其内容（用于移除数学公式渲染的 HTML 部分）
+    {'url': 'math.libretexts.org', 'tag': '//div[contains(@class, "Headertext")]'},  # 仅针对 bmath.libretexts.org 域名，删除所有 class 包含 Headertext 的 <div> 标签及其内容
 ]

llm_web_kit/extractor/html/recognizer/cc_math/common.py

@@ -88,6 +88,10 @@ class ZHIHU:
     DOMAIN = 'zhihu.com'
 
 
+class MATHINSIGHT:
+    DOMAIN = 'mathinsight.org'
+
+
 # 行内行间公式，MathJax中一般也可以通过配置来区分行内行间公式
 EQUATION_INLINE = DocElementType.EQUATION_INLINE
 EQUATION_INTERLINE = DocElementType.EQUATION_INTERLINE
@@ -145,27 +149,182 @@ def text_strip(text):
 transform = etree.XSLT(xslt)
 
 
+def MATHINSIGHT_convert_to_standard_latex(text):
+    """将MathInsight网站使用的自定义LaTeX宏转换为标准LaTeX格式。
+
+    基于 https://mathinsight.org/static/mathjaxconfig/midefault.js?rev=2.6.1 配置文件中定义的宏。
+
+    当前支持的转换：
+    - 向量表示、雅可比矩阵
+    - 微分和多阶微分、偏微分和多阶偏微分、范数
+    - 各种积分形式（线积分、面积分、闭合积分等）
+    - 参数化积分（线积分、面积分等）
+    - 特殊函数符号（div、curl、tr）、默认字母和符号、集合符号
+    - 换行和格式控制、颜色标记
+
+    Args:
+        text: 包含MathInsight宏的LaTeX文本
+
+    Returns:
+        转换后的标准LaTeX文本
+    """
+    replacements = {
+        # 向量表示
+        r'\\vc{([^}]*)}': r'\\mathbf{\1}',
+
+        # 微分和偏微分
+        r'\\diff{([^}]*)}{([^}]*)}': r'\\frac{\\mathrm{d} \1}{\\mathrm{d} \2}',
+        r'\\diffn{([^}]*)}{([^}]*)}{([^}]*)}': r'\\frac{\\mathrm{d}^{\3} \1}{\\mathrm{d} \2^{\3}}',
+        r'\\pdiff{([^}]*)}{([^}]*)}': r'\\frac{\\partial \1}{\\partial \2}',
+        r'\\pdiffn{([^}]*)}{([^}]*)}{([^}]*)}': r'\\frac{\\partial^{\3} \1}{\\partial \2^{\3}}',
+
+        # 矩阵和行列式
+        r'\\jacm{([^}]*)}': r'D\1',
+        r'\\JacobianMatrix{([^}]*)}': r'D\1',
+
+        # 范数
+        r'\\norm{([^}]*)}': r'\\|\1\\|',
+
+        # 线积分
+        r'\\lint{([^}]*)}{([^}]*)}': r'\\int_{\1} \2 \\cdot d\\mathbf{s}',
+        r'\\clint{([^}]*)}{([^}]*)}': r'\\oint_{\1} \2 \\cdot d\\mathbf{s}',
+        r'\\slint{([^}]*)}{([^}]*)}': r'\\int_{\1} \2 \\,ds',
+        r'\\cslint{([^}]*)}{([^}]*)}': r'\\oint_{\1} \2 \\,ds',
+
+        # 面积分
+        r'\\sint{([^}]*)}{([^}]*)}': r'\\iint_{\1} \2 \\cdot d\\mathbf{S}',
+        r'\\ssint{([^}]*)}{([^}]*)}': r'\\iint_{\1} \2 \\,dS',
+
+        # 参数化积分（线积分）
+        r'\\plint{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}': r'\\int_{\1}^{\2} \3(\4(t)) \\cdot \4\'(t) dt',
+        r'\\pslint{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}': r'\\int_{\1}^{\2} \3(\4(t)) \\|\\4\'(t)\\| dt',
+
+        # 参数化积分（面积分）
+        r'\\psint{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}':
+            r'\\int_{\1}^{\2}\\int_{\3}^{\4} \5(\6(\7,\8)) \\cdot \\left( \\frac{\\partial \6}{\\partial \7}(\7,\8) \\times \\frac{\\partial \6}{\\partial \8}(\7,\8) \\right) d\7\\,d\8',
+
+        r'\\psintrn{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}':
+            r'\\int_{\1}^{\2}\\int_{\3}^{\4} \5(\6(\7,\8)) \\cdot \\left( \\frac{\\partial \6}{\\partial \8}(\7,\8) \\times \\frac{\\partial \6}{\\partial \7}(\7,\8) \\right) d\7\\,d\8',
+
+        r'\\psintrnro{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}':
+            r'\\int_{\1}^{\2}\\int_{\3}^{\4} \5(\6(\7,\8)) \\cdot \\left( \\frac{\\partial \6}{\\partial \8}(\7,\8) \\times \\frac{\\partial \6}{\\partial \7}(\7,\8) \\right) d\8\\,d\7',
+
+        r'\\psintro{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}':
+            r'\\int_{\1}^{\2}\\int_{\3}^{\4} \5(\6(\7,\8)) \\cdot \\left( \\frac{\\partial \6}{\\partial \7}(\7,\8) \\times \\frac{\\partial \6}{\\partial \8}(\7,\8) \\right) d\8\\,d\7',
+
+        r'\\psintor{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}':
+            r'\\iint_{\1} \2(\3(\4,\5)) \\cdot \\left( \\frac{\\partial \3}{\\partial \4}(\4,\5) \\times \\frac{\\partial \3}{\\partial \5}(\4,\5) \\right) d\4\\,d\5',
+
+        # 参数化标量面积分
+        r'\\pssint{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}':
+            r'\\int_{\1}^{\2}\\int_{\3}^{\4} \5(\6(\7,\8)) \\left\\| \\frac{\\partial \6}{\\partial \7}(\7,\8) \\times \\frac{\\partial \6}{\\partial \8}(\7,\8) \\right\\| d\7\\,d\8',
+
+        r'\\pssintro{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}':
+            r'\\int_{\1}^{\2}\\int_{\3}^{\4} \5(\6(\7,\8)) \\left\\| \\frac{\\partial \6}{\\partial \7}(\7,\8) \\times \\frac{\\partial \6}{\\partial \8}(\7,\8) \\right\\| d\8\\,d\7',
+
+        r'\\pssintor{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}{([^}]*)}':
+            r'\\iint_{\1} \2(\3(\4,\5)) \\left\\| \\frac{\\partial \3}{\\partial \4}(\4,\5) \\times \\frac{\\partial \3}{\\partial \5}(\4,\5) \\right\\| d\4\\,d\5',
+
+        # 特殊函数
+        r'\\div': r'\\mathop{\\text{div}}',
+        r'\\curl': r'\\mathop{\\text{curl}}',
+        r'\\tr': r'\\mathop{\\rm tr}',
+
+        # 默认字母替换
+        r'\\dlvf': r'\\mathbf{F}',
+        r'\\dlvfc': r'F',
+        r'\\adlvf': r'\\mathbf{G}',
+        r'\\adlvfc': r'G',
+        r'\\dlc': r'C',
+        r'\\adlc': r'B',
+        r'\\sadlc': r'E',
+        r'\\dlsi': r'f',
+        r'\\dlr': r'D',
+        r'\\dlv': r'W',
+        r'\\dls': r'S',
+        r'\\dlpf': r'f',  # 势函数
+
+        # 参数化
+        r'\\dllp': r'\\mathbf{c}',
+        r'\\dllpc': r'c',
+        r'\\adllp': r'\\mathbf{p}',
+        r'\\adllpc': r'p',
+        r'\\sadllp': r'\\mathbf{q}',
+        r'\\sadllpc': r'q',
+        r'\\tadllp': r'\\mathbf{d}',
+        r'\\tadllpc': r'd',
+        r'\\dlsp': r'\\mathbf{\\Phi}',
+        r'\\dlspc': r'\\Phi',
+
+        # 参数变量
+        r'\\spfv': r'u',
+        r'\\spsv': r'v',
+        r'\\cvarf': r'\\mathbf{T}',
+        r'\\cvarfc': r'T',
+        r'\\cvarfv': r'u',
+        r'\\cvarsv': r'v',
+        r'\\cvartv': r'w',
+
+        # 集合符号
+        r'\\R': r'\\mathbb{R}',
+
+        # 弧长和曲面符号
+        r'\\als': r's',
+        r'\\lis': r'\\mathbf{s}',
+        r'\\sas': r'S',
+        r'\\sid': r'\\mathbf{S}',
+
+        # 不可见运算符
+        r'\\invisibletimes': r'\\unicode{x2062}',
+        r'\\cdotbadbreak': r'\\mmlToken{mo}[linebreak="badbreak"]{\\u22C5}',
+        r'\\timesbadbreak': r'\\mmlToken{mo}[linebreak="badbreak"]{\\u00D7}',
+
+        # 颜色标记
+        r'\\blue': r'\\color{blue}{\\textbf{blue}}',
+        r'\\red': r'\\color{red}{\\textbf{red}}',
+        r'\\green': r'\\color{green}{\\textbf{green}}',
+        r'\\cyan': r'\\color{cyan}{\\textbf{cyan}}',
+        r'\\magenta': r'\\color{magenta}{\\textbf{magenta}}',
+
+        # 换行控制
+        r'\\goodbreak': r'\\mmlToken{mo}[linebreak="goodbreak"]{}',
+        r'\\badbreak{([^}]*)}': r'\\mmlToken{mo}[linebreak="badbreak"]{\1}',
+        r'\\nobreak{([^}]*)}': r'\\mmlToken{mo}[linebreak="nobreak"]{\1}',
+    }
+
+    # 应用所有替换规则
+    for pattern, replacement in replacements.items():
+        text = re.sub(pattern, replacement, text)
+
+    # 处理默认组合的积分形式
+    default_combos = {
+        # 默认线积分组合
+        r'\\dlint': r'\\int_{C} \\mathbf{F} \\cdot d\\mathbf{s}',
+        r'\\dclint': r'\\oint_{C} \\mathbf{F} \\cdot d\\mathbf{s}',
+        r'\\dslint': r'\\int_{C} f \\,ds',
+        r'\\dcslint': r'\\oint_{C} f \\,ds',
+
+        # 默认面积分组合
+        r'\\dsint': r'\\iint_{S} \\mathbf{F} \\cdot d\\mathbf{S}',
+        r'\\dssint': r'\\iint_{S} f \\,dS',
+
+        # 默认参数化积分组合
+        r'\\dplint': r'\\int_{a}^{b} \\mathbf{F}(\\mathbf{c}(t)) \\cdot \\mathbf{c}\'(t) dt',
+        r'\\dpslint': r'\\int_{a}^{b} f(\\mathbf{c}(t)) \\|\\mathbf{c}\'(t)\\| dt',
+        r'\\dpsint': r'\\iint_{D} \\mathbf{F}(\\mathbf{\\Phi}(u,v)) \\cdot \\left( \\frac{\\partial \\mathbf{\\Phi}}{\\partial u}(u,v) \\times \\frac{\\partial \\mathbf{\\Phi}}{\\partial v}(u,v) \\right) du\\,dv',
+        r'\\dpssint': r'\\iint_{D} f(\\mathbf{\\Phi}(u,v)) \\left\\| \\frac{\\partial \\mathbf{\\Phi}}{\\partial u}(u,v) \\times \\frac{\\partial \\mathbf{\\Phi}}{\\partial v}(u,v) \\right\\| du\\,dv',
+    }
+
+    for pattern, replacement in default_combos.items():
+        text = re.sub(pattern, replacement, text)
+
+    return text
+
+
 class CCMATH():
     def __init__(self):
         self.url = ''
 
-    # end def
-    def wrap_math(self, s, display=False):
-        """根据行间行内公式加上$$或$"""
-        s = re.sub(r'\s+', ' ', s)
-        s = color_regex.sub('', s)
-        s = s.replace('$', '')
-        s = s.replace('\n', ' ').replace('\\n', '')
-        s = s.strip()
-        if len(s) == 0:
-            return s
-        # Don't wrap if it's already in \align
-        if '\\begin' in s:
-            return s
-        if display:
-            return '$$' + s + '$$'
-        return '$' + s + '$'
-
     def wrap_math_md(self, s):
         """去掉latex公式头尾的$$或$或\\(\\)或\\[\\]"""
         if not s:
@@ -183,6 +342,9 @@ def wrap_math_md(self, s):
         if s.startswith('`') and s.endswith('`'):
             return s.replace('`', '').strip()
         s = self.wrap_math_md_custom(s)
+        # 处理MathInsight网站的特殊宏
+        if MATHINSIGHT.DOMAIN in self.url:
+            s = MATHINSIGHT_convert_to_standard_latex(s)
         return s.strip()
 
     # 循环MATH_MD_CUSTOM_CONFIG，如果url匹配，则去掉特殊网站的公式奇怪的起始结尾
@@ -442,8 +604,6 @@ def build_cc_exception_tag(self, text, math_type, math_render) -> str:
     # print(cm.get_equation_type(r'<p>abc [itex]x^2 abc</p>'))
     print(cm.get_equation_type(r'<p>\begin{align} a^2+b=c\end{align}</p>'))
     print(cm.get_equation_type(r'<p>\begin{abc} a^2+b=c\end{abc}</p>'))
-    print(cm.wrap_math(r'{\displaystyle \operatorname {Var} (X)=\operatorname {E} \left[(X-\mu)^{2}\right].}'))
-    print(cm.wrap_math(r'$$a^2 + b^2 = c^2$$'))
     print(cm.wrap_math_md(r'{\displaystyle \operatorname {Var} (X)=\operatorname {E} \left[(X-\mu)^{2}\right].}'))
     print(cm.wrap_math_md(r'$$a^2 + b^2 = c^2$$'))
     print(cm.wrap_math_md(r'\(a^2 + b^2 = c^2\)'))

llm_web_kit/extractor/html/recognizer/cc_math/render/mathjax.py

@@ -1,6 +1,7 @@
 import re
 from typing import Any, Dict, List
 
+from llm_web_kit.extractor.html.recognizer.cc_math.common import CCMATH
 from llm_web_kit.extractor.html.recognizer.cc_math.render.render import (
     BaseMathRender, MathRenderType)
 from llm_web_kit.libs.html_utils import HtmlElement, html_to_element
@@ -16,7 +17,7 @@
     'skipTags': ['script', 'noscript', 'style', 'textarea', 'pre', 'code'],
     'ignoreClass': 'tex2jax_ignore',
     'processClass': 'tex2jax_process',
-    'elements': ['body']
+    'elements': ['body'],
 }
 
 
@@ -34,6 +35,7 @@ def __init__(self):
             'config': '',
             'version': ''
         }
+        self.ccmath = CCMATH()  # 初始化CCMATH实例
 
     def get_render_type(self) -> str:
         """获取渲染器类型."""
@@ -228,6 +230,12 @@ def find_math(self, root: HtmlElement) -> None:
             pattern = f'{start_escaped}(.*?){end_escaped}'
             display_patterns.append(pattern)
 
+        # 添加对环境的支持
+        if MATHJAX_OPTIONS.get('processEnvironments', True):
+            # 通用匹配任何 \begin{...}\end{...} 环境的模式，保证环境名称相同时才匹配
+            env_pattern = r'(\\begin\{(?P<env>[^}]+)\}.*?\\end\{(?P=env)\})'
+            display_patterns.append(env_pattern)
+
         # 编译正则表达式
         inline_pattern = re.compile('|'.join(inline_patterns), re.DOTALL)
         display_pattern = re.compile('|'.join(display_patterns), re.DOTALL)
@@ -255,7 +263,7 @@ def _find_math_in_element(self, element: HtmlElement, inline_pattern: re.Pattern
                 element.tail = self._process_math_in_text(element, element.tail, inline_pattern, False, True)
 
         # 跳过特定标签
-        skip_tags = MATHJAX_OPTIONS.get('skipTags', ['script', 'noscript', 'style', 'textarea', 'pre', 'code'])
+        skip_tags = MATHJAX_OPTIONS['skipTags']
         if element.tag in skip_tags:
             return
         # 跳过ccmath标签
@@ -332,8 +340,10 @@ def _process_math_in_text(
         if not text:
             return text
 
-        # 查找所有匹配
+        # 首先查找所有分隔符形式的匹配
         matches = list(pattern.finditer(text))
+
+        # 如果没有匹配到分隔符形式的公式，直接返回原文本
         if not matches:
             return text
 
@@ -359,6 +369,8 @@ def _process_math_in_text(
             if self._is_escaped_delimiter(text, match.start()):
                 continue
 
+            formula = self.ccmath.wrap_math_md(formula)
+
             start_pos = match.start()
             end_pos = match.end()
 

llm_web_kit/extractor/html/recognizer/cc_math/render/render.py

@@ -25,6 +25,7 @@ def __init__(self):
         """初始化渲染器基类."""
         self.options = {}
         self.render_type = None
+        self.url = ''  # 添加url属性的正确方式
 
     @abstractmethod
     def get_render_type(self) -> str:

llm_web_kit/extractor/html/recognizer/cc_math/tag_math.py

@@ -23,7 +23,6 @@ def modify_tree(cm: CCMATH, math_render: str, o_html: str, node: HtmlElement, pa
         if len(annotation_tags) > 0:
             annotation_tag = annotation_tags[0]
             text = annotation_tag.text
-            # wrapped_text = cm.wrap_math(r'{}'.format(text), display=display)
             style_value = parent.get('style')
             if style_value:
                 normalized_style_value = style_value.lower().strip().replace(' ', '').replace(';', '')

llm_web_kit/extractor/html/recognizer/cc_math/tag_mjx.py

@@ -4,6 +4,8 @@
                                                                   CCTag,
                                                                   MathType,
                                                                   text_strip)
+from llm_web_kit.extractor.html.recognizer.recognizer import \
+    BaseHTMLElementRecognizer
 from llm_web_kit.libs.html_utils import build_cc_element, replace_element
 
 
@@ -21,6 +23,12 @@ def modify_tree(cm: CCMATH, math_render: str, o_html: str, node: HtmlElement):
     """
     # 处理mjx-container标签
     if node.tag == 'mjx-container':
+        # 查找formula标签
+        formula = node.find('.//formula')
+        if formula is not None and formula.text:
+            # 如果已经包含指定的ccmath标签，不进行替换
+            if BaseHTMLElementRecognizer.is_cc_html(formula, [CCTag.CC_MATH_INTERLINE, CCTag.CC_MATH_INLINE]):
+                return
         display = node.get('display', 'false').lower() == 'true'
         if display:
             new_tag = CCTag.CC_MATH_INTERLINE