新增知乎公式提取

.gitignore

@@ -48,3 +48,4 @@ coverage.xml
 llm_web_kit.egg-info/*
 .llm-web-kit.jsonc
 .llm-web-kit-pageclassify.jsonc
+tests/llm_web_kit/extractor/ygq_testmd

llm_web_kit/extractor/html/recognizer/cc_math/common.py

@@ -74,6 +74,20 @@ class MATH_TYPE_PATTERN:
     DISPLAYMATH = 'displayMath'
 
 
+# CSDN博客的KaTeX标签
+class CSDN:
+    INLINE = 'katex--inline'
+    DISPLAY = 'katex--display'
+    MATH = 'katex-mathml'
+    DOMAIN = 'blog.csdn.net'
+
+
+# 知乎的数学公式标签
+class ZHIHU:
+    MATH = 'ztext-math'
+    DOMAIN = 'zhihu.com'
+
+
 # 行内行间公式，MathJax中一般也可以通过配置来区分行内行间公式
 EQUATION_INLINE = DocElementType.EQUATION_INLINE
 EQUATION_INTERLINE = DocElementType.EQUATION_INTERLINE
@@ -272,6 +286,14 @@ def check_delimiters(delims_list, s):
             if (sub_elements and any(text_strip(elem.text) for elem in sub_elements)) or \
                 (sup_elements and any(text_strip(elem.text) for elem in sup_elements)):
                 result.append((EQUATION_INLINE, MathType.HTMLMATH))
+
+            # 检查当前节点是否是katex元素（CSDN）
+            if CSDN.DOMAIN in self.url and node.tag == 'span' and node.get('class'):
+                node_class = node.get('class')
+                if CSDN.INLINE in node_class:
+                    result.append((EQUATION_INLINE, MathType.LATEX))
+                elif CSDN.DISPLAY in node_class:
+                    result.append((EQUATION_INTERLINE, MathType.LATEX))
         return self.equation_type_to_tag(result)
 
     def equation_type_to_tag(self, type_math_type: List[Tuple[str, str]]) -> List[Tuple[str, str]]:

llm_web_kit/extractor/html/recognizer/cc_math/tag_script.py

@@ -5,7 +5,7 @@
 
 from llm_web_kit.exception.exception import HtmlMathRecognizerException
 from llm_web_kit.extractor.html.recognizer.cc_math.common import (
-    CCMATH, CCMATH_INTERLINE, MathType, text_strip)
+    CCMATH, CCMATH_INLINE, CCMATH_INTERLINE, CSDN, MathType, text_strip)
 from llm_web_kit.libs.html_utils import (build_cc_element, element_to_html,
                                          replace_element)
 
@@ -59,32 +59,52 @@
 
 
 def process_katex_mathml(cm, math_render, node):
-    # 只处理katex-mathml节点
-    if node.tag == 'span' and node.get('class') == 'katex-mathml':
-        # 向上查找父节点，判断行内/行间
-        parent = node.getparent()
-        is_inline = False
-        while parent is not None:
-            parent_class = parent.get('class') or ''
-            if 'katex--inline' in parent_class:
-                is_inline = True
-                break
-            if 'katex--display' in parent_class:
-                is_inline = False
-                break
-            parent = parent.getparent()
-        # 提取latex公式（取最后一行非空内容）
-        lines = [line.strip() for line in node.text_content().splitlines() if line.strip()]
-        if not lines:
+    try:
+        # 根据节点class确定公式类型
+        equation_type = CCMATH_INLINE if CSDN.INLINE in node.get('class') else CCMATH_INTERLINE
+        # 查找内部的katex-mathml节点提取公式
+        mathml_nodes = node.xpath(f'.//span[@class="{CSDN.MATH}"]')
+        if mathml_nodes:
+            mathml_node = mathml_nodes[0]
+            # 提取latex公式（取最后一行非空内容）
+            lines = [line.strip() for line in mathml_node.text_content().splitlines() if line.strip()]
+            if lines:
+                latex = lines[-1]
+                if latex:
+                    html_with_formula = element_to_html(node)
+                    # 创建新元素替换原节点
+                    new_span = build_cc_element(
+                        html_tag_name=equation_type,
+                        text=cm.wrap_math_md(latex),
+                        tail=text_strip(node.tail),
+                        type='latex',
+                        by=math_render,
+                        html=html_with_formula
+                    )
+                    replace_element(node, new_span)
+    except Exception as e:
+        raise HtmlMathRecognizerException(f'处理CSDN博客数学公式失败: {e}')
+
+
+def process_zhihu_custom_tag(cm, math_render, node):
+    try:
+        # 从data-tex属性获取LaTeX公式
+        latex = node.get('data-tex')
+        if not latex:
             return
-        latex = lines[-1]
-        tag = 'ccmath-inline' if is_inline else 'ccmath-interline'
+        html_with_formula = f'<span>{latex}</span>'
+        tag_math_type_list = cm.get_equation_type(html_with_formula)
+        # 如果无法确定类型，默认为行内公式
+        if not tag_math_type_list:
+            tag_math_type_list = [(CCMATH_INLINE, MathType.LATEX)]
         new_span = build_cc_element(
-            html_tag_name=tag,
+            html_tag_name=tag_math_type_list[0][0],
             text=cm.wrap_math_md(latex),
-            tail=None,
-            type='latex',
+            tail=text_strip(node.tail),
+            type=tag_math_type_list[0][1],
             by=math_render,
-            html=None
+            html=element_to_html(node)
         )
         replace_element(node, new_span)
+    except Exception as e:
+        raise HtmlMathRecognizerException(f'处理知乎数学公式失败: {e}')

llm_web_kit/extractor/html/recognizer/ccmath.py

@@ -9,7 +9,8 @@
                                                            tag_common_modify,
                                                            tag_img, tag_math,
                                                            tag_mjx, tag_script)
-from llm_web_kit.extractor.html.recognizer.cc_math.common import CCMATH
+from llm_web_kit.extractor.html.recognizer.cc_math.common import (CCMATH, CSDN,
+                                                                  ZHIHU)
 from llm_web_kit.extractor.html.recognizer.cc_math.render.render import (
     BaseMathRender, MathRenderType)
 from llm_web_kit.extractor.html.recognizer.recognizer import (
@@ -129,10 +130,17 @@ def process_ccmath_html(self, cc_html: str, o_html: str, math_render: BaseMathRe
                 original_html = self._element_to_html(node)
                 parent = node.getparent()
 
-                # 针对csdn博客中的katex-mathml标签，提取latex公式
-                if node.tag == 'span' and node.get('class') == 'katex-mathml' and 'blog.csdn.net' in self.cm.url:
+                # 针对csdn博客中的katex标签，提取latex公式
+                if (CSDN.DOMAIN in self.cm.url and
+                        node.tag == 'span' and
+                        node.get('class') in [CSDN.INLINE, CSDN.DISPLAY]):
                     tag_script.process_katex_mathml(self.cm, math_render_type, node)
 
+                if ZHIHU.DOMAIN in self.cm.url and node.tag == 'span' and node.get('class') == ZHIHU.MATH:
+                    tag_script.process_zhihu_custom_tag(self.cm, math_render_type, node)
+
+                # if 'mathinsight.org' in self.cm.url and node.tag == 'span' and node.get('class') == '':
+
                 # tag = span， class 为 math-containerm， 或者 mathjax 或者 wp-katex-eq
                 if node.tag == 'span' and node.get('class') and (
                         'math-container' in node.get('class') or
@@ -145,7 +153,8 @@ def process_ccmath_html(self, cc_html: str, o_html: str, math_render: BaseMathRe
 
                 # script[type="math/tex"]
                 # if node.tag == 'script' and node.get('type') and 'math/tex' in node.get('type'):
-                    # tag_common_modify.modify_tree(cm, math_render_type, original_html, node, parent)
+                #     print('匹配到script标签: ', node.get('type'))
+                #     tag_common_modify.modify_tree(cm, math_render_type, original_html, node, parent)
 
                 # math tags
                 if node.tag == 'math' or node.tag.endswith(':math'):