增加元素识别和抽取magic-html的接口

README.md

@@ -75,6 +75,8 @@ llm-web-kit is a python library that ..
 
 ## Quick Start
 
+### extract magic_html+recognize
+
 ```python
 from llm_web_kit.simple import extract_html_to_md
 import traceback
@@ -95,6 +97,49 @@ if __name__=="__main__":
     markdown = extract(url, html)
 ```
 
+### only extract by recognize
+
+```python
+from llm_web_kit.simple import extract_pure_html_to_md
+import traceback
+from loguru import logger
+
+def extract(url:str, html:str) -> str:
+    try:
+        nlp_md = extract_html_to_md(url, html, clip_html=False)
+        return nlp_md
+    except Exception as e:
+        logger.exception(e)
+    return None
+
+if __name__=="__main__":
+    url = ""
+    html = ""
+    markdown = extract(url, html)
+```
+
+### only extract by magic-html
+
+```python
+from llm_web_kit.simple import extract_magic_html
+import traceback
+from loguru import logger
+
+def extract(url:str, html:str) -> str:
+    try:
+        nlp_md = extract_main_html_by_maigic_html(url, html)
+        # or mm_nlp_md = extract_pure_html_to_mm_md(url, html)
+        return nlp_md
+    except Exception as e:
+        logger.exception(e)
+    return None
+
+if __name__=="__main__":
+    url = ""
+    html = ""
+    markdown = extract(url, html)
+```
+
 ## Pipeline
 
 1. [HTML pre-dedup](jupyter/html-pre-dedup/main.ipynb)

llm_web_kit/config/pipe_tpl/html-test.jsonc

@@ -23,7 +23,7 @@
         "extractor": [
             {
                 "enable": true,
-                "python_class": "llm_web_kit.extractor.html.extractor.HTMLFileFormatExtractor",
+                "python_class": "llm_web_kit.extractor.html.extractor.MagicHTMLFIleFormatorExtractor",
                 "class_init_kwargs": {}
             }
         ],

llm_web_kit/config/pipe_tpl/html.jsonc

@@ -16,7 +16,7 @@
             "extractor": [
                 {
                     "enable": true,
-                    "python_class": "llm_web_kit.extractor.html.extractor.HTMLFileFormatExtractor",
+                    "python_class": "llm_web_kit.extractor.html.extractor.MagicHTMLFIleFormatorExtractor",
                     "class_init_kwargs": {}
                 }
             ],

llm_web_kit/config/pipe_tpl/noclip_html.jsonc

@@ -0,0 +1,30 @@
+{
+    "extractor_pipe": {
+            "enable": true,
+            "validate_input_format": false,
+            "pre_extractor": [
+                {
+                    "enable": true,
+                    "python_class": "llm_web_kit.extractor.html.pre_extractor.HTMLFileFormatFilterTablePreExtractor"
+                },
+                {
+                    "enable": true,
+                    "python_class": "llm_web_kit.extractor.html.pre_extractor.HTMLFileFormatCleanTagsPreExtractor",
+                    "class_init_kwargs": {}
+                }
+            ],
+            "extractor": [
+                {
+                    "enable": true,
+                    "python_class": "llm_web_kit.extractor.html.extractor.NoClipHTMLFIleFormatorExtractor",
+                    "class_init_kwargs": {}
+                }
+            ],
+            "post_extractor": [
+            {
+                "enable": true,
+                "python_class": "llm_web_kit.extractor.html.post_extractor.HTMLStripSpacePostExtractor"
+            }
+        ]
+    }
+}

llm_web_kit/extractor/html/extractor.py

@@ -33,7 +33,7 @@ class HTMLPageLayoutType:
     LAYOUT_LIST = 'list'
 
 
-class HTMLFileFormatExtractor(BaseFileFormatExtractor):
+class NoClipHTMLFIleFormatorExtractor(BaseFileFormatExtractor):
     """一个从html文件中提取数据的提取器."""
 
     def __init__(self, config: dict):
@@ -65,8 +65,6 @@ def __init__(self, config: dict):
             CCTag.CC_TEXT: self.__paragraph_recognizer
         }
 
-        self.__magic_html_extractor = self.__build_extractor()
-
     @override
     def _filter_by_rule(self, data_json: DataJson) -> bool:
         """根据规则过滤content_list.
@@ -91,9 +89,10 @@ def _do_extract(self, data_json: DataJson) -> DataJson:
         # 第三步将解析结果存入content_list中
         raw_html:str = data_json['html']
         base_url:str = data_json['url']
-        page_layout_type:str = data_json.get('page_layout_type', HTMLPageLayoutType.LAYOUT_ARTICLE)  # 默认是文章类型
+        main_html:str = data_json['main_html']
+        # page_layout_type:str = data_json.get('page_layout_type', HTMLPageLayoutType.LAYOUT_ARTICLE)  # 默认是文章类型
 
-        main_html, method, title = self._extract_main_html(raw_html, base_url, page_layout_type)
+        # main_html, method, title = self._extract_main_html(raw_html, base_url, page_layout_type)
         main_html_element = html_to_element(main_html)
         parsed_html = [(main_html_element, raw_html)]
         for extract_func in [self._extract_code, self._extract_table, self._extract_math, self._extract_list,
@@ -102,24 +101,9 @@ def _do_extract(self, data_json: DataJson) -> DataJson:
             parsed_html = extract_func(base_url, parsed_html, raw_html)
         content_list:ContentList = self._export_to_content_list(base_url, parsed_html, raw_html)
         data_json['content_list'] = content_list
-        data_json['title'] = title
+        # data_json['title'] = title
         return data_json
 
-    def _extract_main_html(self, raw_html:str, base_url:str, page_layout_type:str) -> Tuple[str, str, str]:
-        """从html文本中提取主要的内容.
-
-        Args:
-            raw_html (str): html文本
-            base_url (str): html文本的网页地址
-            page_layout_type (str): 网页的布局类型
-
-        Returns:
-            str1: 主要的内容
-            str2: 获得内容的方式，可对质量进行评估
-        """
-        dict_result = self.__magic_html_extractor.extract(raw_html, base_url=base_url, precision=False, html_type=page_layout_type)
-        return dict_result['html'], dict_result['xp_num'], dict_result.get('title', '')
-
     def _extract_code(self, base_url:str, html_lst:List[Tuple[HtmlElement, HtmlElement]], raw_html:str) -> List[Tuple[HtmlElement,HtmlElement]]:
         """从html文本中提取代码.
 
@@ -374,6 +358,54 @@ def __get_cc_node(self, html:HtmlElement) -> (HtmlElement, str):
             # return element_to_html(nodes[0]), nodes[0].tag
             return nodes[0], nodes[0].tag
 
+
+class MagicHTMLFIleFormatorExtractor(NoClipHTMLFIleFormatorExtractor):
+    """一个从html文件中提取数据的提取器."""
+
+    def __init__(self, config: dict):
+        """从参数指定的配置中初始化这个流水线链.
+
+        Args:
+            config (dict): 配置字典
+        """
+        super().__init__(config)
+        self.__magic_html_extractor = self.__build_extractor()
+
+    @override
+    def _do_extract(self, data_json: DataJson) -> DataJson:
+        """实现真正的数据提取.
+
+        Args:
+            data_json (DataJson): 需要处理的数据集
+        """
+        raw_html:str = data_json['html']
+        base_url:str = data_json['url']
+        page_layout_type:str = data_json.get('page_layout_type', HTMLPageLayoutType.LAYOUT_ARTICLE)  # 默认是文章类型
+
+        # 使用magic-html提取主要内容
+        main_html, method, title = self._extract_main_html(raw_html, base_url, page_layout_type)
+        data_json['main_html'] = main_html
+        # 调用父类的提取方法
+        data_json = super()._do_extract(data_json)
+        # 添加标题
+        data_json['title'] = title
+        return data_json
+
+    def _extract_main_html(self, raw_html:str, base_url:str, page_layout_type:str) -> Tuple[str, str, str]:
+        """从html文本中提取主要的内容.
+
+        Args:
+            raw_html (str): html文本
+            base_url (str): html文本的网页地址
+            page_layout_type (str): 网页的布局类型
+
+        Returns:
+            str1: 主要的内容
+            str2: 获得内容的方式，可对质量进行评估
+        """
+        dict_result = self.__magic_html_extractor.extract(raw_html, base_url=base_url, precision=False, html_type=page_layout_type)
+        return dict_result['html'], dict_result['xp_num'], dict_result.get('title', '')
+
     def __build_extractor(self):
         """
         结合自定义域名规则，构建一个抽取器。

llm_web_kit/extractor/html/recognizer/ccmath.py

@@ -49,7 +49,6 @@ def recognize(self, base_url: str, main_html_lst: List[Tuple[HtmlElement, HtmlEl
                 result.extend(self.process_ccmath_html(cc_html, o_html, math_render, base_url))
             else:
                 result.append((cc_html, o_html))
-
         return result
 
     @override
@@ -85,7 +84,6 @@ def to_content_list_node(self, base_url: str, parsed_content: HtmlElement, raw_h
             # 获取math_content
             math_content = inter_ele[0].text
             math_content = self.cm.wrap_math_md(math_content)
-
             return {
                 'type': DocElementType.EQUATION_INTERLINE,
                 'raw_content': raw_html_segment,
@@ -97,7 +95,6 @@ def to_content_list_node(self, base_url: str, parsed_content: HtmlElement, raw_h
             }
         elif len(in_els) > 0:
             math_content = in_els[0].text
-
             return {
                 'type': DocElementType.EQUATION_INLINE,
                 'raw_content': raw_html_segment,

llm_web_kit/libs/html_utils.py

@@ -284,9 +284,10 @@ def extract_magic_html(html, base_url, page_layout_type):
         base_url: str: 基础url
         page_layout_type: str: 页面布局类型
     """
-    from llm_web_kit.extractor.html.extractor import HTMLFileFormatExtractor
+    from llm_web_kit.extractor.html.extractor import \
+        MagicHTMLFIleFormatorExtractor
 
-    extractor = HTMLFileFormatExtractor({})
+    extractor = MagicHTMLFIleFormatorExtractor({})
     try:
         main_html, _, _ = extractor._extract_main_html(html, base_url, page_layout_type)
         return main_html

llm_web_kit/simple.py

@@ -5,6 +5,9 @@
 
 from llm_web_kit.config.cfg_reader import load_pipe_tpl
 from llm_web_kit.extractor.extractor_chain import ExtractSimpleFactory
+from llm_web_kit.extractor.html.extractor import (
+    HTMLPageLayoutType, MagicHTMLFIleFormatorExtractor,
+    NoClipHTMLFIleFormatorExtractor)
 from llm_web_kit.input.datajson import DataJson
 
 
@@ -32,6 +35,29 @@
             raise ValueError(f'Invalid extractor type: {extractor_type}')
 
 
+def __extract_main_html_by_no_clip_html(url:str, html_content: str) -> DataJson:
+    extractor = NoClipHTMLFIleFormatorExtractor(load_pipe_tpl('noclip_html'))
+    input_data_dict = {
+        'track_id': str(uuid.uuid4()),
+        'url': url,
+        'html': html_content,
+        'main_html': html_content,
+        'dataset_name': 'llm-web-kit-pure-quickstart',
+        'data_source_category': 'HTML',
+        'file_bytes': len(html_content),
+        'meta_info': {'input_datetime': datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
+    }
+    d = DataJson(input_data_dict)
+    result = extractor.extract(d)
+    return result
+
+
+def __extract_main_html_by_maigic_html(url:str, html_str: str, page_layout_type:str) -> DataJson:
+    magic_html_extractor = MagicHTMLFIleFormatorExtractor(load_pipe_tpl('html'))
+    main_html, method, title = magic_html_extractor._extract_main_html(html_str, url, page_layout_type)
+    return main_html, title
+
+
 def __extract_html(url:str, html_content: str) -> DataJson:
     extractor = ExtractorFactory.get_extractor(ExtractorType.HTML)
     input_data_dict = {
@@ -48,14 +74,25 @@
     return result
 
 
-def extract_html_to_md(url:str, html_content: str) -> str:
+def extract_html_to_md(url:str, html_content: str, clip_html=True) -> str:
     """extract html to markdown without images."""
-    result = __extract_html(url, html_content)
+    if clip_html:
+        result = __extract_html(url, html_content)
+    else:
+        result = __extract_main_html_by_no_clip_html(url, html_content)
     return result.get_content_list().to_nlp_md()
 
 
-def extract_html_to_mm_md(url:str, html_content: str) -> str:
+def extract_html_to_mm_md(url:str, html_content: str, clip_html=True) -> str:
     """extract html to markdown with images."""
-
-    result = __extract_html(url, html_content)
+    if clip_html:
+        result = __extract_html(url, html_content)
+    else:
+        result = __extract_main_html_by_no_clip_html(url, html_content)
     return result.get_content_list().to_mm_md()
+
+
+def extract_main_html_by_maigic_html(url:str, html_str: str, page_layout_type:str = HTMLPageLayoutType.LAYOUT_ARTICLE) -> str:
+    """extract main html."""
+    result = __extract_main_html_by_maigic_html(url, html_str, page_layout_type)
+    return result[0], result[1]

llm_web_kit/tools/cli.py

@@ -5,7 +5,7 @@
 import click
 from loguru import logger
 
-from llm_web_kit.extractor.html.extractor import HTMLFileFormatExtractor
+from llm_web_kit.extractor.html.extractor import MagicHTMLFIleFormatorExtractor
 from llm_web_kit.input.datajson import DataJson
 
 
@@ -55,7 +55,7 @@ def cli(input_path, output_path, debug_mode):
             else:
                 raise ValueError('Input JSON must contain either html or path field')
 
-        extractor = HTMLFileFormatExtractor({})
+        extractor = MagicHTMLFIleFormatorExtractor({})
         data_e = extractor.extract(DataJson(input_data))
         output_json = data_e.to_json()
         if output_path: