From 85dc9c06d6567cd77970ed68e0218629608ea100 Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Mon, 9 Jun 2025 16:03:36 +0800 Subject: [PATCH 01/11] =?UTF-8?q?feat:=E6=B8=85=E7=90=86=E5=85=83=E7=B4=A0?= =?UTF-8?q?=E5=B1=9E=E6=80=A7=EF=BC=8C=E4=BF=9D=E7=95=99=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E7=9A=84=E6=9C=89=E6=95=88src=EF=BC=88=E6=8E=92=E9=99=A4base64?= =?UTF-8?q?=EF=BC=89=E3=80=81alt=EF=BC=8C=E4=BB=A5=E5=8F=8A=E6=89=80?= =?UTF-8?q?=E6=9C=89=E5=85=83=E7=B4=A0=E7=9A=84class=E5=92=8Cid"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../simplify_html/simplify_html.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py index b47de995..94e5d629 100644 --- a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py +++ b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py @@ -417,23 +417,34 @@ def is_meaningful_content(element) -> bool: def clean_attributes(element): - """清理元素属性，只保留图片的有效src以及所有元素的class和id.""" + """清理元素属性，保留图片的有效src（排除base64）、alt，以及所有元素的class和id.""" if element.tag == 'img': + # 获取图片相关属性 src = element.get('src', '').strip() + alt = element.get('alt', '').strip() class_attr = element.get('class', '').strip() id_attr = element.get('id', '').strip() - element.attrib.clear() # 先清除所有属性 - if src: + + element.attrib.clear() # 清除所有属性 + + # 保留非base64的src + if src and not src.startswith('data:image/'): element.set('src', src) + # 保留alt（如果非空） + if alt: + element.set('alt', alt) + # 保留class和id（如果非空） if class_attr: element.set('class', class_attr) if id_attr: element.set('id', id_attr) else: - # 对于其他元素，只保留class和id + # 非图片元素：只保留class和id class_attr = element.get('class', '').strip() id_attr = element.get('id', '').strip() - element.attrib.clear() # 先清除所有属性 + + element.attrib.clear() # 清除所有属性 + if class_attr: element.set('class', class_attr) if id_attr: From 6cffbb6f93c4f44b24fde8964fdf44e03b3cbdb8 Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Mon, 9 Jun 2025 19:15:51 +0800 Subject: [PATCH 02/11] =?UTF-8?q?feat:=20=E7=B2=BE=E7=AE=80=E6=8E=A7?= =?UTF-8?q?=E5=88=B6=E6=98=AF=E5=90=A6=E8=8E=B7=E5=8F=96XPATH?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- llm_web_kit/input/pre_data_json.py | 1 + llm_web_kit/main_html_parser/parser/tag_simplifier.py | 3 ++- .../llm_web_kit/main_html_parser/parser/test_tag_simplifier.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/llm_web_kit/input/pre_data_json.py b/llm_web_kit/input/pre_data_json.py index c8d602c0..2153073f 100644 --- a/llm_web_kit/input/pre_data_json.py +++ b/llm_web_kit/input/pre_data_json.py @@ -15,6 +15,7 @@ class PreDataJsonKey: TYPICAL_RAW_HTML = 'typical_raw_html' TYPICAL_RAW_TAG_HTML = 'typical_raw_tag_html' + IS_XPATH = True XPATH_MAPPING = 'xpath_mapping' TYPICAL_SIMPLIFIED_HTML = 'typical_simplified_html' # 模型打标字典 diff --git a/llm_web_kit/main_html_parser/parser/tag_simplifier.py b/llm_web_kit/main_html_parser/parser/tag_simplifier.py index 1d705c1b..eede6cfb 100644 --- a/llm_web_kit/main_html_parser/parser/tag_simplifier.py +++ b/llm_web_kit/main_html_parser/parser/tag_simplifier.py @@ -19,11 +19,12 @@ def parse(self, pre_data: PreDataJson) -> PreDataJson: """ # 获取输入数据 typical_raw_html = pre_data.get(PreDataJsonKey.TYPICAL_RAW_HTML, '') + is_xpath = pre_data.get(PreDataJsonKey.IS_XPATH, True) # layout_file_list = pre_data.get(PreDataJsonKey.LAYOUT_FILE_LIST, []) # 执行HTML标签简化逻辑 try: - simplified_html, original_html, _ = simplify_html(typical_raw_html) + simplified_html, original_html, _ = simplify_html(typical_raw_html, is_xpath=is_xpath) except TagSimplifiedParserException as e1: raise e1 except Exception as e2: diff --git a/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py b/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py index 3d656811..ea5d57e9 100644 --- a/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py +++ b/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py @@ -57,7 +57,7 @@ def test_tag_simplifier4(self): file_path = base_dir / 'assets/test_html_data/1.html' with open(file_path, 'r', encoding='utf-8') as file: raw_html = file.read() - data_dict = {PreDataJsonKey.TYPICAL_RAW_HTML: raw_html} + data_dict = {PreDataJsonKey.TYPICAL_RAW_HTML: raw_html, PreDataJsonKey.IS_XPATH: False} pre_data = PreDataJson(data_dict) pre_data_result = HtmlTagSimplifierParser({}).parse(pre_data) simplifier_raw_html = pre_data_result.get(PreDataJsonKey.TYPICAL_SIMPLIFIED_HTML, '') From c96bbf9a242919d996ce0ccfb895fac5e30d2023 Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Mon, 9 Jun 2025 19:44:34 +0800 Subject: [PATCH 03/11] =?UTF-8?q?feat:=20=E7=B2=BE=E7=AE=80=E6=8E=A7?= =?UTF-8?q?=E5=88=B6=E6=98=AF=E5=90=A6=E8=8E=B7=E5=8F=96XPATH?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- llm_web_kit/input/pre_data_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_web_kit/input/pre_data_json.py b/llm_web_kit/input/pre_data_json.py index 2153073f..7902d093 100644 --- a/llm_web_kit/input/pre_data_json.py +++ b/llm_web_kit/input/pre_data_json.py @@ -15,7 +15,7 @@ class PreDataJsonKey: TYPICAL_RAW_HTML = 'typical_raw_html' TYPICAL_RAW_TAG_HTML = 'typical_raw_tag_html' - IS_XPATH = True + IS_XPATH = 'is_xpath' XPATH_MAPPING = 'xpath_mapping' TYPICAL_SIMPLIFIED_HTML = 'typical_simplified_html' # 模型打标字典 From 61fd634b15f55917c4e447df6ec2335da4eb260d Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Tue, 17 Jun 2025 16:40:22 +0800 Subject: [PATCH 04/11] =?UTF-8?q?feat:=20=E8=87=AA=E5=AE=9A=E4=B9=89?= =?UTF-8?q?=E6=A0=87=E7=AD=BE'marked-tail',=20'marked-text'=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E4=B8=BA=E8=A1=8C=E5=86=85=E6=A0=87=E7=AD=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- llm_web_kit/main_html_parser/simplify_html/simplify_html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py index 94e5d629..679b0742 100644 --- a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py +++ b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py @@ -12,7 +12,7 @@ 'samp', 'blink', 'b', 'code', 'nobr', 'strike', 'bdo', 'basefont', 'abbr', 'var', 'i', 'cccode-inline', 'select', 's', 'pic', 'label', 'mark', 'object', 'dd', 'dt', 'ccmath-inline', 'svg', 'li', 'button', 'a', 'font', 'dfn', 'sup', 'kbd', 'q', 'script', 'acronym', 'option', 'img', 'big', 'cite', - 'em', + 'em', 'marked-tail', 'marked-text' # 'td', 'th' } From a73c2b40da775e822ad43f2e1b1654a9d91d8737 Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Thu, 19 Jun 2025 19:07:03 +0800 Subject: [PATCH 05/11] =?UTF-8?q?fix:=20=E9=87=8D=E5=91=BD=E5=90=8D?= =?UTF-8?q?=E8=87=AA=E5=AE=9A=E4=B9=89=E6=A0=87=E7=AD=BE=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main_html_parser/simplify_html/simplify_html.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py index 679b0742..312fb3cd 100644 --- a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py +++ b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py @@ -47,6 +47,9 @@ # '-header', '_header', # 有特例，可能自定义的header中有标题，先注释 } +# 自定义标签 +tail_block_tag = "cc-alg-uc-text" + def add_data_uids(dom: html.HtmlElement) -> None: """为DOM所有节点添加data-uid属性（递归所有子节点）""" @@ -762,7 +765,7 @@ def process_paragraphs(paragraphs: List[Dict[str, str]], uid_map: Dict[str, html # trailing_text = last_child.tail # 创建wrapper元素 - wrapper = etree.Element('cc-alg-uc-tex') + wrapper = etree.Element(tail_block_tag) wrapper.set('_item_id', current_id) # 设置前面的文本 @@ -798,7 +801,7 @@ def process_paragraphs(paragraphs: List[Dict[str, str]], uid_map: Dict[str, html # 检查父节点的text if original_parent.text and original_parent.text.strip() == root_for_xpath.text.strip(): # 创建wrapper - wrapper = etree.Element('cc-alg-uc-tex') + wrapper = etree.Element(tail_block_tag) wrapper.set('_item_id', current_id) wrapper.text = original_parent.text @@ -818,7 +821,7 @@ def process_paragraphs(paragraphs: List[Dict[str, str]], uid_map: Dict[str, html for child in original_parent.iterchildren(): if child.tail and child.tail.strip() == root_for_xpath.text.strip(): # 创建wrapper - wrapper = etree.Element('cc-alg-uc-tex') + wrapper = etree.Element(tail_block_tag) wrapper.set('_item_id', current_id) wrapper.text = child.tail @@ -835,7 +838,7 @@ def process_paragraphs(paragraphs: List[Dict[str, str]], uid_map: Dict[str, html # 如果没有找到匹配的文本节点，使用父节点作为包裹对象 if not found: - wrapper = etree.Element('cc-alg-uc-tex') + wrapper = etree.Element(tail_block_tag) wrapper.set('_item_id', current_id) wrapper.text = root_for_xpath.text # 将父节点的内容移动到wrapper中 From 87d9f822620c6263ed52ba9836c6354a357f8cfa Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Thu, 19 Jun 2025 19:14:04 +0800 Subject: [PATCH 06/11] =?UTF-8?q?fix:=20=E9=87=8D=E5=91=BD=E5=90=8D?= =?UTF-8?q?=E8=87=AA=E5=AE=9A=E4=B9=89=E6=A0=87=E7=AD=BE=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- llm_web_kit/main_html_parser/simplify_html/simplify_html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py index 312fb3cd..f472c3a8 100644 --- a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py +++ b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py @@ -48,7 +48,7 @@ } # 自定义标签 -tail_block_tag = "cc-alg-uc-text" +tail_block_tag = 'cc-alg-uc-text' def add_data_uids(dom: html.HtmlElement) -> None: From ff910e9d750d2d9eacb75da8e7a89528855c0ad8 Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Thu, 19 Jun 2025 19:44:35 +0800 Subject: [PATCH 07/11] =?UTF-8?q?fix:=20=E5=8E=BB=E6=8E=89=E5=86=97?= =?UTF-8?q?=E4=BD=99=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main_html_parser/simplify_html/simplify_html.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py index f472c3a8..bfff8f29 100644 --- a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py +++ b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py @@ -833,21 +833,8 @@ def process_paragraphs(paragraphs: List[Dict[str, str]], uid_map: Dict[str, html index = parent.index(child) parent.insert(index + 1, wrapper) - found = True break - # 如果没有找到匹配的文本节点，使用父节点作为包裹对象 - if not found: - wrapper = etree.Element(tail_block_tag) - wrapper.set('_item_id', current_id) - wrapper.text = root_for_xpath.text - # 将父节点的内容移动到wrapper中 - for child in list(original_parent.iterchildren()): - wrapper.append(child) - original_parent.remove(child) - - # 添加wrapper到父节点 - original_parent.append(wrapper) else: # 块级元素直接设置属性 original_parent.set('_item_id', current_id) From d7e09d1ca72f5061aa19694a1e92a0d7db544404 Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Thu, 10 Jul 2025 19:57:25 +0800 Subject: [PATCH 08/11] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E5=A4=9A?= =?UTF-8?q?=E8=AF=AD=E7=A7=8D=E6=8B=BC=E6=8E=A5=E8=A7=84=E5=88=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- llm_web_kit/extractor/html/extractor.py | 8 +- llm_web_kit/extractor/html/recognizer/text.py | 24 +- .../good_data/html/br.html | 780 ++++++++++++++++++ .../good_data/html/zh.html | 663 +++++++++++++++ .../extractor/html/recognizer/test_text.py | 44 + .../extractor/test_extractor_chain.py | 2 +- 6 files changed, 1507 insertions(+), 14 deletions(-) create mode 100644 tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/br.html create mode 100644 tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/zh.html diff --git a/llm_web_kit/extractor/html/extractor.py b/llm_web_kit/extractor/html/extractor.py index 6f71c271..7ec6e1b1 100644 --- a/llm_web_kit/extractor/html/extractor.py +++ b/llm_web_kit/extractor/html/extractor.py @@ -90,6 +90,7 @@ def _do_extract(self, data_json: DataJson) -> DataJson: raw_html:str = data_json['html'] base_url:str = data_json['url'] main_html:str = data_json['main_html'] + language:str = data_json.get('language', 'en') # page_layout_type:str = data_json.get('page_layout_type', HTMLPageLayoutType.LAYOUT_ARTICLE) # 默认是文章类型 # main_html, method, title = self._extract_main_html(raw_html, base_url, page_layout_type) @@ -97,8 +98,9 @@ def _do_extract(self, data_json: DataJson) -> DataJson: parsed_html = [(main_html_element, raw_html)] for extract_func in [self._extract_code, self._extract_table, self._extract_math, self._extract_list, self._extract_image, - self._extract_title, self._extract_paragraph]: + self._extract_title]: parsed_html = extract_func(base_url, parsed_html, raw_html) + parsed_html = self._extract_paragraph(base_url, parsed_html, raw_html, language) # 过滤掉包含script和style标签的元素 filtered_parsed_html = [] @@ -222,7 +224,7 @@ def _extract_title(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:s lst = self.__title_recognizer.recognize(base_url, html_lst, raw_html) return lst - def _extract_paragraph(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str) -> List[Tuple[str,str]]: + def _extract_paragraph(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str, language:str) -> List[Tuple[str,str]]: """从html文本中提取段落. Args: @@ -233,7 +235,7 @@ def _extract_paragraph(self, base_url:str, html_lst:List[Tuple[str,str]], raw_ht Returns: """ - lst = self.__paragraph_recognizer.recognize(base_url, html_lst, raw_html) + lst = self.__paragraph_recognizer.recognize(base_url, html_lst, raw_html, language) return lst def __is_valid_node(self, node: dict) -> bool: diff --git a/llm_web_kit/extractor/html/recognizer/text.py b/llm_web_kit/extractor/html/recognizer/text.py index 22cc07f9..65b60d45 100644 --- a/llm_web_kit/extractor/html/recognizer/text.py +++ b/llm_web_kit/extractor/html/recognizer/text.py @@ -67,6 +67,9 @@ 'mjx-container', 'mjx-assistive-mml', 'strike', 'wbr', 'ins' } +# 词间无分隔符的语言 +no_separation_language = ['zh', 'ja', 'ko', 'wuu', 'th', 'km', 'lo', 'bo', 'ii', 'jv'] + class TextParagraphRecognizer(BaseHTMLElementRecognizer): """解析文本段落元素.""" @@ -93,7 +96,7 @@ def to_content_list_node(self, base_url: str, parsed_content: HtmlElement, raw_h return node @override - def recognize(self, base_url:str, main_html_lst: List[Tuple[HtmlElement | str, HtmlElement | str]], raw_html:str) -> List[Tuple[HtmlElement, HtmlElement]]: + def recognize(self, base_url:str, main_html_lst: List[Tuple[HtmlElement | str, HtmlElement | str]], raw_html:str, language:str = 'en') -> List[Tuple[HtmlElement, HtmlElement]]: """父类，解析文本段落元素. Args: @@ -111,11 +114,11 @@ def recognize(self, base_url:str, main_html_lst: List[Tuple[HtmlElement | str, H new_html_lst.append((html_element, raw_html_element)) else: lst = list(self.__extract_paragraphs(html_element)) - new_lst = self.__to_cctext_lst(lst) + new_lst = self.__to_cctext_lst(lst, language) new_html_lst.extend(new_lst) return new_html_lst - def __to_cctext_lst(self, lst: List[Tuple[HtmlElement | str, HtmlElement | str]]) -> List[Tuple[HtmlElement, HtmlElement]]: + def __to_cctext_lst(self, lst: List[Tuple[HtmlElement | str, HtmlElement | str]], language:str) -> List[Tuple[HtmlElement, HtmlElement]]: """将lst[Element, raw_html] 进行处理. 提出Element里的文字，做成<>标签. Args: @@ -129,7 +132,7 @@ def __to_cctext_lst(self, lst: List[Tuple[HtmlElement | str, HtmlElement | str]] el_element = html_to_element(el) if isinstance(el, str) else el raw_html_element = html_to_element(raw_html) if isinstance(raw_html, str) else raw_html - para_text = self.__get_paragraph_text(el_element) + para_text = self.__get_paragraph_text(el_element, language) if para_text: cctext_el = self._build_cc_element(CCTag.CC_TEXT, json.dumps(para_text, ensure_ascii=False, indent=4), '', html=element_to_html_unescaped(raw_html_element)) new_lst.append((cctext_el, raw_html_element)) @@ -185,20 +188,20 @@ def __combine_text(self, text1:str, text2:str, lang='en') -> str: lang: str: 语言 TODO 实现根据语言连接文本的不同方式, 还有就是一些特殊符号开头的连接不加空格。 """ text1 = text1.strip(' ') if text1 else '' - text2 = text2.strip(' ') if text2 else '' - if lang == 'zh': + text2 = text2.rstrip(' ') if text2 else '' + if lang in no_separation_language: txt = text1 + text2 return self.replace_entities(txt.strip(), entities_map) else: # 根据text1的最后一个字符和text2的第一个字符判断两个text之间的连接 if (text2[0] in string.punctuation) or (text2[0] in special_symbols) or (text2[0] in other_symbols) or (text1 and text1[-1] in other_symbols): words_sep = '' - else : + else: words_sep = ' ' txt = text1 + words_sep + text2 return self.replace_entities(txt.strip(), entities_map) - def __get_paragraph_text(self, root: HtmlElement) -> List[dict]: + def __get_paragraph_text(self, root: HtmlElement, language:str = 'en') -> List[dict]: """ 获取段落全部的文本. 对于段落里的行内公式需要特定处理，转换为段落格式： @@ -235,7 +238,7 @@ def __get_paragraph_text_recusive(el: HtmlElement, text: str) -> str: pass else: if el.text and el.text.strip(): - text = self.__combine_text(text, el.text.strip()) + text = self.__combine_text(text, el.text.strip(), language) for child in el: text = __get_paragraph_text_recusive(child, text) @@ -244,7 +247,8 @@ def __get_paragraph_text_recusive(el: HtmlElement, text: str) -> str: if is_sub_sup: text += el.tail else: - text = self.__combine_text(text, el.tail.strip()) + new_tail = f' {el.tail.strip()}' if el.tail.startswith(' ') and el.tail.strip()[0] in string.punctuation else el.tail.strip() + text = self.__combine_text(text, new_tail, language) return text diff --git a/tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/br.html b/tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/br.html new file mode 100644 index 00000000..d9b7ab22 --- /dev/null +++ b/tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/br.html @@ -0,0 +1,780 @@ + + + + +Faustina an Henañ - Wikipedia + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Mont d’an endalc’had +
+
+
+ + + + +
+
+ + + + + +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+ +
+
+
+
+
+ +

Faustina an Henañ

+ + +
+
+
+
+ +
+
+ + + +
+
+
+
+
+ + +
+
+
+
+
+
+ +
Eus Wikipedia
+
+
+ + +
Ur pennad Faustina zo ivez.
+
Faostina an Henañ
+
Aureus da Faustina an Henañ.
+

Annia Galeria Faustina, lesanvet Faustina Maior pe Faustina an Henañ (c. 100 - c. 140), a oa un impalaerez roman, pried da Antoninus Pius, impalaer eus 138 da 161. +

+ +

He buhez

[kemmañ | kemmañ ar vammenn]
+

Bugaleaj

[kemmañ | kemmañ ar vammenn]
+

Ganet e oa e Roma, en un tiegezh roman eus Hispania. Merc'h e oa d'ar c'honsul Marcus Annius Verus. He mamm, Rupilia Faustina, a oa gournizez d'an impalaer Trajan. Nizez e oa da Vibia Sabina (83 – 136 pe 137), pried an impalaer Hadrian. +

C'hoar e oa da Varcus Annius Libo, konsul e 128, ha da Varcus Annius Verus, praetor, marvet e 124, a oa pried da Zomitia Lucilla Minor ha tad da Varcus Aurelius ; ur c'hoar he doa ivez, Annia Cornificia Faustina[1]. +

+

Yaouankiz

[kemmañ | kemmañ ar vammenn]
+

Dimeziñ a reas da Antoninus Pius etre 110 ha 115. Pevar bugel o doe : +

+ +

Notennoù

[kemmañ | kemmañ ar vammenn]
+
    +
  1. ↑ Historia Augusta, Buhez de Marcus-Aurelius, 1. +
    2. +
+ + + + +
+
Adtapet diwar « https://br.wikipedia.org/w/index.php?title=Faustina_an_Henañ&oldid=1958641 »
+ +
+
+ +
+
+ + + +
+
+
+
+
+
+ + + +
+ + +
+
+ +
+
+
+
    + +
+
+ + + +
\ No newline at end of file diff --git a/tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/zh.html b/tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/zh.html new file mode 100644 index 00000000..2d550c07 --- /dev/null +++ b/tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/zh.html @@ -0,0 +1,663 @@ + + + + + + + 浙江省淡水水产研究所 + + + + + + + + + + + + + + + +
+ + + +
+
+
+ 内网入口 +
+
+ +
+
+
+ + + + +
+
+ +
+
+ + +
+ +
+ + + +
+
+
+ + + + + +
+ + +
+
+
+ +
+ +
+ 内网入口 +
+ + + +
+
+ + + + +
+ + + + +
+ + + + + +
+ + + +
+ +
+
+ + + + +
+ + + + + + + +
+ +
+ +
+
+
浙江省淡水水产研究所召开干部职工大会暨党风廉政建设大会
+ +
+ +
+

+ IMG_8429.JPG +

+

为贯彻落实省农业农村厅干部职工大会精神，2月19日下午，浙江省淡水水产研究所召开干部职工大会暨党风廉政建设大会。会议传达了王通林厅长重要讲话精神，全体职工观看了廉政警示教育片，层层签订了党风廉政建设责任书。会议要求，全所干部职工要围绕农业科技创新工作，比学赶超、奋勇争先，以“开局就要奔跑、起步就要跃进”的姿态迅速投入工作；对照干部队伍建设8个“还有之”和党风廉政建设5个“还有之”开展查摆自纠。一要把好方向盘厘清工作思路，明确主次，确保重要工作做踏实、细小工作不遗漏；二要紧盯绩效表抓好工作落实，树牢问题导向和目标导向，坚持“主动想、重点干、善协调、出经验”工作理念，鼓励全所干部职工创先争优，全力打造第一等的标志性成果；三要拧紧安全阀守牢工作底线，按照无规定疫病苗种场的建设要求，扎实做好水生动物疫病防控工作，确保种源生物安全，从实从细抓好安全生产工作和党风廉政建设，营造清正廉洁、干净干事的良好氛围。

+ +

 

+ +
+ +
+ 返回列表 +
+
+
+
+ + + +
+
相关推荐
+
+
+ +
+
+ + +
+
+
+
陈浩调研粮食安全和现代农业发展：守牢粮食安全底线 大力发展现代农业
+
+ +
+
+ +
+
+ + +
+
+
+
浙江省淡水水产研究所在衢州开展罗氏沼虾良种良法推广
+
+ +
+
+ + +
+
+ + +
+
+
+ +
+ +
+ + + + +
+ +
+
+
+
+
+ 友情链接 +
+ +
+
+
+ 联系我们 +
+
+ 浙江省湖州市杭长桥南路999号0572-2045189 +
+
+ +
+
+ +
+
+ +
+
微信公众号
+
+
+
+ +
+
+
+ + +
+
+ +
+ 网站建设： +
+ +
+ + + ©2025 浙江省淡水水产研究所  浙ICP备14041274-1号  + + + +
+
+ +
+ + + + + + +
+ + + + + + + + \ No newline at end of file diff --git a/tests/llm_web_kit/extractor/html/recognizer/test_text.py b/tests/llm_web_kit/extractor/html/recognizer/test_text.py index db392b29..5173e55d 100644 --- a/tests/llm_web_kit/extractor/html/recognizer/test_text.py +++ b/tests/llm_web_kit/extractor/html/recognizer/test_text.py @@ -266,3 +266,47 @@ def test_text_line_exception(self): result = chain.extract(input_data) content_md = result.get_content_list().to_mm_md() assert 'sensi dell’art.33' in content_md + + def test_no_separation_language(self): + """ + Returns: + + """ + chain = ExtractSimpleFactory.create(self.config) + self.assertIsNotNone(chain) + test_data = { + 'track_id': 'text_md', + 'dataset_name': 'text_md', + 'url': 'https://www.zjfish.org/Talent/Detail/876289162604750/2174994673059649', + 'data_source_category': 'HTML', + 'path': 'zh.html', + 'file_bytes': 1000, + 'meta_info': {'input_datetime': '2020-01-01 00:00:00'}, + 'language': 'zh' + } + input_data = DataJson(test_data) + result = chain.extract(input_data) + content_md = result.get_content_list().to_mm_md() + assert '成果； 三' not in content_md + + def test_tail_space(self): + """ + Returns: + + """ + chain = ExtractSimpleFactory.create(self.config) + self.assertIsNotNone(chain) + test_data = { + 'track_id': 'text_md', + 'dataset_name': 'text_md', + 'url': 'https://br.wikipedia.org/wiki/Faustina_an_Hena%C3%B1', + 'data_source_category': 'HTML', + 'path': 'br.html', + 'file_bytes': 1000, + 'meta_info': {'input_datetime': '2020-01-01 00:00:00'}, + 'language': 'br' + } + input_data = DataJson(test_data) + result = chain.extract(input_data) + content_md = result.get_content_list().to_mm_md() + assert 'Henañ (c' in content_md diff --git a/tests/llm_web_kit/extractor/test_extractor_chain.py b/tests/llm_web_kit/extractor/test_extractor_chain.py index 14eb2ec1..8c209e15 100644 --- a/tests/llm_web_kit/extractor/test_extractor_chain.py +++ b/tests/llm_web_kit/extractor/test_extractor_chain.py @@ -451,7 +451,7 @@ def test_para_is_short(self): input_data = DataJson(test_data) result = chain.extract(input_data) content_txt = result.get_content_list().to_nlp_md() - assert len(content_txt) == 1982 + assert len(content_txt) == 1983 def test_xml_tag(self): """测试xml标签.""" From 9113a0e968cda6c9b2d014bf5aba3427bce96c5f Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Wed, 16 Jul 2025 15:20:24 +0800 Subject: [PATCH 09/11] =?UTF-8?q?fix:=20=E6=96=B0=E5=A2=9Elanguage?= =?UTF-8?q?=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- llm_web_kit/extractor/html/extractor.py | 37 +++++++++---------- .../extractor/html/recognizer/audio.py | 2 +- .../extractor/html/recognizer/cccode.py | 3 +- .../extractor/html/recognizer/ccmath.py | 2 +- .../extractor/html/recognizer/image.py | 2 +- llm_web_kit/extractor/html/recognizer/list.py | 2 +- .../extractor/html/recognizer/recognizer.py | 2 +- .../extractor/html/recognizer/table.py | 3 +- .../extractor/html/recognizer/title.py | 2 +- .../extractor/html/recognizer/video.py | 2 +- 10 files changed, 29 insertions(+), 28 deletions(-) diff --git a/llm_web_kit/extractor/html/extractor.py b/llm_web_kit/extractor/html/extractor.py index 5dfeb208..f8509bf8 100644 --- a/llm_web_kit/extractor/html/extractor.py +++ b/llm_web_kit/extractor/html/extractor.py @@ -98,9 +98,8 @@ def _do_extract(self, data_json: DataJson) -> DataJson: parsed_html = [(main_html_element, raw_html)] for extract_func in [self._extract_code, self._extract_table, self._extract_math, self._extract_list, self._extract_image, - self._extract_title]: - parsed_html = extract_func(base_url, parsed_html, raw_html) - parsed_html = self._extract_paragraph(base_url, parsed_html, raw_html, language) + self._extract_title, self._extract_paragraph]: + parsed_html = extract_func(base_url, parsed_html, raw_html, language) # 过滤掉包含script和style标签的元素,在这里改，是因为math提取需要保留script标签 filtered_parsed_html = [] @@ -113,7 +112,7 @@ def _do_extract(self, data_json: DataJson) -> DataJson: # data_json['title'] = title return data_json - def _extract_code(self, base_url:str, html_lst:List[Tuple[HtmlElement, HtmlElement]], raw_html:str) -> List[Tuple[HtmlElement,HtmlElement]]: + def _extract_code(self, base_url:str, html_lst:List[Tuple[HtmlElement, HtmlElement]], raw_html:str, language:str) -> List[Tuple[HtmlElement,HtmlElement]]: """从html文本中提取代码. Args: @@ -123,10 +122,10 @@ def _extract_code(self, base_url:str, html_lst:List[Tuple[HtmlElement, HtmlEleme Returns: """ - lst = self.__code_recognizer.recognize(base_url, html_lst, raw_html) + lst = self.__code_recognizer.recognize(base_url, html_lst, raw_html, language) return lst - def _extract_math(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str) -> List[Tuple[str,str]]: + def _extract_math(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str, language:str) -> List[Tuple[str,str]]: """从html文本中提取数学公式. Args: @@ -137,10 +136,10 @@ def _extract_math(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:st Returns: """ - lst = self.__math_recognizer.recognize(base_url, html_lst, raw_html) + lst = self.__math_recognizer.recognize(base_url, html_lst, raw_html, language) return lst - def _extract_image(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str) -> List[Tuple[str,str]]: + def _extract_image(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str, language:str) -> List[Tuple[str,str]]: """从html文本中提取图片. Args: @@ -151,10 +150,10 @@ def _extract_image(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:s Returns: """ - lst = self.__image_recognizer.recognize(base_url, html_lst, raw_html) + lst = self.__image_recognizer.recognize(base_url, html_lst, raw_html, language) return lst - def _extract_audio(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str) -> List[Tuple[str,str]]: + def _extract_audio(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str, language:str) -> List[Tuple[str,str]]: """从html文本中提取音频. Args: @@ -165,10 +164,10 @@ def _extract_audio(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:s Returns: """ - lst = self.__audio_recognizer.recognize(base_url, html_lst, raw_html) + lst = self.__audio_recognizer.recognize(base_url, html_lst, raw_html, language) return lst - def _extract_video(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str) -> List[Tuple[str,str]]: + def _extract_video(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str, language:str) -> List[Tuple[str,str]]: """从html文本中提取视频. Args: @@ -179,10 +178,10 @@ def _extract_video(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:s Returns: """ - lst = self.__video_recognizer.recognize(base_url, html_lst, raw_html) + lst = self.__video_recognizer.recognize(base_url, html_lst, raw_html, language) return lst - def _extract_table(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str) -> List[Tuple[str,str]]: + def _extract_table(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str, language:str) -> List[Tuple[str,str]]: """从html文本中提取表格. Args: @@ -193,10 +192,10 @@ def _extract_table(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:s Returns: """ - lst = self.__table_recognizer.recognize(base_url, html_lst, raw_html) + lst = self.__table_recognizer.recognize(base_url, html_lst, raw_html, language) return lst - def _extract_list(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str) -> List[Tuple[str,str]]: + def _extract_list(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str, language:str) -> List[Tuple[str,str]]: """从html文本中提取列表. Args: @@ -207,10 +206,10 @@ def _extract_list(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:st Returns: """ - lst = self.__list_recognizer.recognize(base_url, html_lst, raw_html) + lst = self.__list_recognizer.recognize(base_url, html_lst, raw_html, language) return lst - def _extract_title(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str) -> List[Tuple[str,str]]: + def _extract_title(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str, language:str) -> List[Tuple[str,str]]: """从html文本中提取标题. Args: @@ -221,7 +220,7 @@ def _extract_title(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:s Returns: """ - lst = self.__title_recognizer.recognize(base_url, html_lst, raw_html) + lst = self.__title_recognizer.recognize(base_url, html_lst, raw_html, language) return lst def _extract_paragraph(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str, language:str) -> List[Tuple[str,str]]: diff --git a/llm_web_kit/extractor/html/recognizer/audio.py b/llm_web_kit/extractor/html/recognizer/audio.py index f9e74a7b..b43ead6c 100644 --- a/llm_web_kit/extractor/html/recognizer/audio.py +++ b/llm_web_kit/extractor/html/recognizer/audio.py @@ -10,7 +10,7 @@ class AudioRecognizer(BaseHTMLElementRecognizer): """解析音频元素.""" @override - def recognize(self, base_url:str, main_html_lst: List[Tuple[HtmlElement,HtmlElement]], raw_html:str) -> List[Tuple[HtmlElement,HtmlElement]]: + def recognize(self, base_url:str, main_html_lst: List[Tuple[HtmlElement,HtmlElement]], raw_html:str, language:str = 'en') -> List[Tuple[HtmlElement,HtmlElement]]: """父类，解析音频元素. Args: diff --git a/llm_web_kit/extractor/html/recognizer/cccode.py b/llm_web_kit/extractor/html/recognizer/cccode.py index b84ce207..0baf436c 100644 --- a/llm_web_kit/extractor/html/recognizer/cccode.py +++ b/llm_web_kit/extractor/html/recognizer/cccode.py @@ -27,7 +27,8 @@ def recognize( self, base_url: str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], - raw_html: str + raw_html: str, + language:str = 'en' ) -> List[Tuple[HtmlElement, HtmlElement]]: """父类，解析代码元素. diff --git a/llm_web_kit/extractor/html/recognizer/ccmath.py b/llm_web_kit/extractor/html/recognizer/ccmath.py index 47896359..585c4f06 100644 --- a/llm_web_kit/extractor/html/recognizer/ccmath.py +++ b/llm_web_kit/extractor/html/recognizer/ccmath.py @@ -27,7 +27,7 @@ def __init__(self): self.cm = CCMATH() @override - def recognize(self, base_url: str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], raw_html: str) -> List[Tuple[HtmlElement, HtmlElement]]: + def recognize(self, base_url: str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], raw_html: str, language:str = 'en') -> List[Tuple[HtmlElement, HtmlElement]]: """父类，解析数学公式元素. Args: diff --git a/llm_web_kit/extractor/html/recognizer/image.py b/llm_web_kit/extractor/html/recognizer/image.py index 495bdc0f..30f8241d 100644 --- a/llm_web_kit/extractor/html/recognizer/image.py +++ b/llm_web_kit/extractor/html/recognizer/image.py @@ -67,7 +67,7 @@ def __ccimg_to_content_list(self, raw_html_segment: str, html_obj: HtmlElement) return result @override - def recognize(self, base_url: str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], raw_html: str) -> List[ + def recognize(self, base_url: str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], raw_html: str, language:str = 'en') -> List[ Tuple[HtmlElement, HtmlElement]]: """父类，解析图片元素. diff --git a/llm_web_kit/extractor/html/recognizer/list.py b/llm_web_kit/extractor/html/recognizer/list.py index 38d57f38..6be54bef 100644 --- a/llm_web_kit/extractor/html/recognizer/list.py +++ b/llm_web_kit/extractor/html/recognizer/list.py @@ -48,7 +48,7 @@ def to_content_list_node(self, base_url: str, parsed_content: HtmlElement, raw_h return ele_node @override - def recognize(self, base_url: str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], raw_html: str) -> List[Tuple[HtmlElement, HtmlElement]]: + def recognize(self, base_url: str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], raw_html: str, language:str = 'en') -> List[Tuple[HtmlElement, HtmlElement]]: """父类，解析列表元素. Args: diff --git a/llm_web_kit/extractor/html/recognizer/recognizer.py b/llm_web_kit/extractor/html/recognizer/recognizer.py index 570bf129..63497c9e 100644 --- a/llm_web_kit/extractor/html/recognizer/recognizer.py +++ b/llm_web_kit/extractor/html/recognizer/recognizer.py @@ -29,7 +29,7 @@ class BaseHTMLElementRecognizer(ABC): """基本的元素解析类.""" @abstractmethod - def recognize(self, base_url:str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], raw_html:str) -> List[Tuple[HtmlElement, HtmlElement]]: + def recognize(self, base_url:str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], raw_html:str, language:str) -> List[Tuple[HtmlElement, HtmlElement]]: """父类，解析html中的元素. Args: diff --git a/llm_web_kit/extractor/html/recognizer/table.py b/llm_web_kit/extractor/html/recognizer/table.py index fbcc8eec..d0faa969 100644 --- a/llm_web_kit/extractor/html/recognizer/table.py +++ b/llm_web_kit/extractor/html/recognizer/table.py @@ -24,7 +24,8 @@ def __init__(self): def recognize(self, base_url: str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], - raw_html: str) -> List[Tuple[HtmlElement, HtmlElement]]: + raw_html: str, + language:str = 'en') -> List[Tuple[HtmlElement, HtmlElement]]: """父类，解析表格元素. Args: diff --git a/llm_web_kit/extractor/html/recognizer/title.py b/llm_web_kit/extractor/html/recognizer/title.py index 4d7ea4fd..2f711d4b 100644 --- a/llm_web_kit/extractor/html/recognizer/title.py +++ b/llm_web_kit/extractor/html/recognizer/title.py @@ -39,7 +39,7 @@ def to_content_list_node(self, base_url: str, parsed_content: HtmlElement, raw_h return cctitle_content_node @override - def recognize(self, base_url: str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], raw_html: str) -> List[Tuple[HtmlElement, HtmlElement]]: + def recognize(self, base_url: str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], raw_html: str, language:str = 'en') -> List[Tuple[HtmlElement, HtmlElement]]: """父类，解析标题元素. Args: diff --git a/llm_web_kit/extractor/html/recognizer/video.py b/llm_web_kit/extractor/html/recognizer/video.py index bed7df5a..ac61650b 100644 --- a/llm_web_kit/extractor/html/recognizer/video.py +++ b/llm_web_kit/extractor/html/recognizer/video.py @@ -10,7 +10,7 @@ class VideoRecognizer(BaseHTMLElementRecognizer): """解析视元素.""" @override - def recognize(self, base_url:str, main_html_lst: List[Tuple[HtmlElement,HtmlElement]], raw_html:str) -> List[Tuple[HtmlElement,HtmlElement]]: + def recognize(self, base_url:str, main_html_lst: List[Tuple[HtmlElement,HtmlElement]], raw_html:str, language:str = 'en') -> List[Tuple[HtmlElement,HtmlElement]]: """父类，解析视频元素. Args: From 0e6deeece7df568ec59ff0da9331d2be0dd6f7b8 Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Wed, 16 Jul 2025 15:39:52 +0800 Subject: [PATCH 10/11] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E6=97=A0?= =?UTF-8?q?=E6=B3=95=E5=A4=84=E7=90=86xml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- llm_web_kit/main_html_parser/typical_html/typical_html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_web_kit/main_html_parser/typical_html/typical_html.py b/llm_web_kit/main_html_parser/typical_html/typical_html.py index 72e9e24b..22ef66a9 100644 --- a/llm_web_kit/main_html_parser/typical_html/typical_html.py +++ b/llm_web_kit/main_html_parser/typical_html/typical_html.py @@ -39,7 +39,7 @@ def select_representative_html(html_strings): continue # 将字符串转换为文件对象 - file_obj = StringIO(html_dict['html']) + file_obj = StringIO(html_str) tree = html.parse(file_obj) # 找到body元素 From f39020697c1f09762e73583a8586c97d711fdafa Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Tue, 22 Jul 2025 16:52:13 +0800 Subject: [PATCH 11/11] =?UTF-8?q?feat:=20noclip=E7=AE=A1=E7=BA=BF=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E9=A2=84=E5=A4=84=E7=90=86=EF=BC=9A=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E8=A1=A8=E5=8D=95=E4=BA=A4=E4=BA=92=E5=BC=8F=E5=85=83=E7=B4=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../config/pipe_tpl/noclip_html_test.jsonc | 42 + llm_web_kit/extractor/html/pre_extractor.py | 73 ++ .../html/delete_interactive_element1.html | 814 ++++++++++++++++++ .../delete_interactive_element1_main.html | 73 ++ .../extractor/html/recognizer/test_text.py | 23 + 5 files changed, 1025 insertions(+) create mode 100644 llm_web_kit/config/pipe_tpl/noclip_html_test.jsonc create mode 100644 tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/delete_interactive_element1.html create mode 100644 tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/delete_interactive_element1_main.html diff --git a/llm_web_kit/config/pipe_tpl/noclip_html_test.jsonc b/llm_web_kit/config/pipe_tpl/noclip_html_test.jsonc new file mode 100644 index 00000000..0e471ce6 --- /dev/null +++ b/llm_web_kit/config/pipe_tpl/noclip_html_test.jsonc @@ -0,0 +1,42 @@ +{ + "extractor_pipe": { + "enable": true, + "validate_input_format": false, + "pre_extractor": [ + { + "enable": true, + "python_class": "llm_web_kit.extractor.html.pre_extractor.TestHTMLFileToDataJsonPreExtractor", + "class_init_kwargs": { + "html_parent_dir": "tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/" + } + }, + { + "enable": true, + "python_class": "llm_web_kit.extractor.html.pre_extractor.HTMLFileFormatNoClipPreExtractor", + "class_init_kwargs": {} + }, + { + "enable": true, + "python_class": "llm_web_kit.extractor.html.pre_extractor.HTMLFileFormatFilterTablePreExtractor" + }, + { + "enable": true, + "python_class": "llm_web_kit.extractor.html.pre_extractor.HTMLFileFormatCleanTagsPreExtractor", + "class_init_kwargs": {} + } + ], + "extractor": [ + { + "enable": true, + "python_class": "llm_web_kit.extractor.html.extractor.NoClipHTMLFIleFormatorExtractor", + "class_init_kwargs": {} + } + ], + "post_extractor": [ + { + "enable": true, + "python_class": "llm_web_kit.extractor.html.post_extractor.HTMLStripSpacePostExtractor" + } + ] + } +} diff --git a/llm_web_kit/extractor/html/pre_extractor.py b/llm_web_kit/extractor/html/pre_extractor.py index a9323a9d..d95f06e5 100644 --- a/llm_web_kit/extractor/html/pre_extractor.py +++ b/llm_web_kit/extractor/html/pre_extractor.py @@ -111,3 +111,76 @@ def __clean_invisible_elements(self, data_json: DataJson) -> str: for element in elements: remove_element(element) return element_to_html(tree) + + +class TestHTMLFileToDataJsonPreExtractor(HTMLFileFormatFilterPreExtractor): + """为了方便noclip管线对测试数据进行测试，根据路径读取html文件和main_html文件，然后转换为DataJson格式。""" + + def __init__(self, config: dict, html_parent_dir: str): + """ + 初始化函数 + Args: + config: + html_parent_dir: + """ + super().__init__(config) + self.__html_parent_path = html_parent_dir + + @override + def _do_pre_extract(self, data_json: DataJson) -> DataJson: + """对输入的html和main_html拼装到DataJson中，形成标准输入格式.""" + proj_root_dir = get_proj_root_dir() + html_file_path = os.path.join(proj_root_dir, self.__html_parent_path, data_json.get('path')) + main_html_file_path = os.path.join(proj_root_dir, self.__html_parent_path, data_json.get('main_path')) + + with open(html_file_path, 'r', encoding='utf-8') as f: + html = f.read() + data_json['html'] = html + del data_json['path'] + + with open(main_html_file_path, 'r', encoding='utf-8') as f: + main_html = f.read() + data_json['main_html'] = main_html + del data_json['main_path'] + return data_json + + +class HTMLFileFormatNoClipPreExtractor(HTMLFileFormatFilterPreExtractor): + """noclip管线对main_html预处理.""" + def __init__(self, config: dict): + super().__init__(config) + + @override + def _do_pre_extract(self, data_json: DataJson) -> DataJson: + data_json['main_html'] = self.__clean_interactive_elements(data_json) + return data_json + + def __clean_interactive_elements(self, data_json: DataJson) -> str: + """清除main_html中交互式元素.""" + html_content = data_json['main_html'] + tree = html_to_element(html_content) + interactive_tags = ['input', 'select', 'textarea', 'button'] + # 删除内的交互标签及关联label + for tag in interactive_tags: + for element in tree.xpath(f'//body//{tag}'): + # 删除标签本身 + parent = element.getparent() + if parent is not None: + parent.remove(element) + + # 删除关联的label（通过for属性匹配） + if 'id' in element.attrib: + for label in tree.xpath(f'//body//label[@for="{element.attrib["id"]}"]'): + label.getparent().remove(label) + + # 处理
内的交互标签及关联label + for form in tree.xpath('//form'): + # 删除表单内所有交互标签 + form_elements = form.xpath('.//input | .//select | .//textarea | .//button | .//label | .//img') + for element in form_elements: + element.getparent().remove(element) + + # 检查表单是否为空（无子元素或仅剩空白文本） + if len(form.getchildren()) == 0 or not form.text_content().strip(): + form.getparent().remove(form) + return element_to_html(tree) diff --git a/tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/delete_interactive_element1.html b/tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/delete_interactive_element1.html new file mode 100644 index 00000000..4d93c41e --- /dev/null +++ b/tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/delete_interactive_element1.html @@ -0,0 +1,814 @@ + + + + + + + + + Serve the Gay Muscle Stud | Straight Master + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+
+
+
+
+
Masters +
+

Serve the Gay Muscle Stud

+
+
+

Gay Muscle Stud

+

You know you’ve been thinking about serving a gay muscle stud like me.

+

I am the Alpha Male that makes you so weak you can’t do anything but submit to my + needs. Your an inferior little bottom boy who wants to please a real man like me.

+

Whether it’s my ripped body, hot  meat or my tight virgin ass. Most likely you can’t + say the same for your hole, can you?

+

Kneel Before the Gay Muscle Stud

+

You’d cough your holes up to me in a heart beat so don’t fool yourself. It + boils down to this; I am the Master and you are a slave. So take your place and kneel in + the presence of a real gay muscle stud. Always up for anything so you bring it on and + see if you can shock me; it’s impossible.

+

If you are not lucky enough to find me when I am online, you can always get used by one + of my fellow gay + muscle hunks.

+
+
+ +

You might also like

+
+
+
Black dick worship cam master posing.
+

Black Dick + Worship Cam for Gay Cock Slaves

+
+
+
+
Hung Master for Slaves +
+

+ Hung Master for Maggot Slaves

+
+
+
+
+
Alpha Master for Fag Slaves +
+

Alpha Master for Fag Slaves to + Serve

+
+
+
+
+
+

Leave + a Reply

+

Your email address will not be published.

+

+

+

+ +

+

+

+

+
+
+
+
+
+ +
+
+ +
+ + + + \ No newline at end of file diff --git a/tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/delete_interactive_element1_main.html b/tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/delete_interactive_element1_main.html new file mode 100644 index 00000000..24b5a35c --- /dev/null +++ b/tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/delete_interactive_element1_main.html @@ -0,0 +1,73 @@ + + +
+
+
+
+
+
+

Gay Muscle Stud

+

You know you’ve been thinking about serving a gay muscle stud like me.

+

I am the Alpha Male that makes you so weak you can’t do anything but submit to my needs. + Your an inferior little bottom boy who wants to please a real man like me.

+

Whether it’s my ripped body, hot  meat or my tight virgin ass. Most likely you can’t say + the same for your hole, can you?

+

Kneel Before the Gay Muscle Stud

+

You’d cough your holes up to me in a heart beat so don’t fool yourself. It boils down to + this; I am the Master and you are a slave. So take your place and kneel in the presence + of a real gay muscle stud. Always up for anything so you bring it on and see if you can + shock me; it’s impossible.

+

If you are not lucky enough to find me when I am online, you can always get used by one + of my fellow gay + muscle hunks.

+
+
+
+

Leave + a Reply

+

Your email address will not be published. + +

+

+

+ +

+

+
+
+
+
+
+
+
+
+ + \ No newline at end of file diff --git a/tests/llm_web_kit/extractor/html/recognizer/test_text.py b/tests/llm_web_kit/extractor/html/recognizer/test_text.py index 5173e55d..a011299d 100644 --- a/tests/llm_web_kit/extractor/html/recognizer/test_text.py +++ b/tests/llm_web_kit/extractor/html/recognizer/test_text.py @@ -310,3 +310,26 @@ def test_tail_space(self): result = chain.extract(input_data) content_md = result.get_content_list().to_mm_md() assert 'Henañ (c' in content_md + + def test_interactive_element(self): + """ + Returns: + + """ + chain = ExtractSimpleFactory.create(load_pipe_tpl('noclip_html_test')) + self.assertIsNotNone(chain) + test_data = { + 'track_id': 'text_md', + 'dataset_name': 'text_md', + 'url': 'https://br.wikipedia.org/wiki/Faustina_an_Hena%C3%B1', + 'data_source_category': 'HTML', + 'path': 'delete_interactive_element1.html', + 'main_path': 'delete_interactive_element1_main.html', + 'file_bytes': 1000, + 'meta_info': {'input_datetime': '2020-01-01 00:00:00'}, + 'language': 'en' + } + input_data = DataJson(test_data) + result = chain.extract(input_data) + main_html = result.get_content_list().to_main_html() + assert '