From 85dc9c06d6567cd77970ed68e0218629608ea100 Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Mon, 9 Jun 2025 16:03:36 +0800 Subject: [PATCH 1/7] =?UTF-8?q?feat:=E6=B8=85=E7=90=86=E5=85=83=E7=B4=A0?= =?UTF-8?q?=E5=B1=9E=E6=80=A7=EF=BC=8C=E4=BF=9D=E7=95=99=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E7=9A=84=E6=9C=89=E6=95=88src=EF=BC=88=E6=8E=92=E9=99=A4base64?= =?UTF-8?q?=EF=BC=89=E3=80=81alt=EF=BC=8C=E4=BB=A5=E5=8F=8A=E6=89=80?= =?UTF-8?q?=E6=9C=89=E5=85=83=E7=B4=A0=E7=9A=84class=E5=92=8Cid"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../simplify_html/simplify_html.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py index b47de995..94e5d629 100644 --- a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py +++ b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py @@ -417,23 +417,34 @@ def is_meaningful_content(element) -> bool: def clean_attributes(element): - """清理元素属性，只保留图片的有效src以及所有元素的class和id.""" + """清理元素属性，保留图片的有效src（排除base64）、alt，以及所有元素的class和id.""" if element.tag == 'img': + # 获取图片相关属性 src = element.get('src', '').strip() + alt = element.get('alt', '').strip() class_attr = element.get('class', '').strip() id_attr = element.get('id', '').strip() - element.attrib.clear() # 先清除所有属性 - if src: + + element.attrib.clear() # 清除所有属性 + + # 保留非base64的src + if src and not src.startswith('data:image/'): element.set('src', src) + # 保留alt（如果非空） + if alt: + element.set('alt', alt) + # 保留class和id（如果非空） if class_attr: element.set('class', class_attr) if id_attr: element.set('id', id_attr) else: - # 对于其他元素，只保留class和id + # 非图片元素：只保留class和id class_attr = element.get('class', '').strip() id_attr = element.get('id', '').strip() - element.attrib.clear() # 先清除所有属性 + + element.attrib.clear() # 清除所有属性 + if class_attr: element.set('class', class_attr) if id_attr: From 6cffbb6f93c4f44b24fde8964fdf44e03b3cbdb8 Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Mon, 9 Jun 2025 19:15:51 +0800 Subject: [PATCH 2/7] =?UTF-8?q?feat:=20=E7=B2=BE=E7=AE=80=E6=8E=A7?= =?UTF-8?q?=E5=88=B6=E6=98=AF=E5=90=A6=E8=8E=B7=E5=8F=96XPATH?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- llm_web_kit/input/pre_data_json.py | 1 + llm_web_kit/main_html_parser/parser/tag_simplifier.py | 3 ++- .../llm_web_kit/main_html_parser/parser/test_tag_simplifier.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/llm_web_kit/input/pre_data_json.py b/llm_web_kit/input/pre_data_json.py index c8d602c0..2153073f 100644 --- a/llm_web_kit/input/pre_data_json.py +++ b/llm_web_kit/input/pre_data_json.py @@ -15,6 +15,7 @@ class PreDataJsonKey: TYPICAL_RAW_HTML = 'typical_raw_html' TYPICAL_RAW_TAG_HTML = 'typical_raw_tag_html' + IS_XPATH = True XPATH_MAPPING = 'xpath_mapping' TYPICAL_SIMPLIFIED_HTML = 'typical_simplified_html' # 模型打标字典 diff --git a/llm_web_kit/main_html_parser/parser/tag_simplifier.py b/llm_web_kit/main_html_parser/parser/tag_simplifier.py index 1d705c1b..eede6cfb 100644 --- a/llm_web_kit/main_html_parser/parser/tag_simplifier.py +++ b/llm_web_kit/main_html_parser/parser/tag_simplifier.py @@ -19,11 +19,12 @@ def parse(self, pre_data: PreDataJson) -> PreDataJson: """ # 获取输入数据 typical_raw_html = pre_data.get(PreDataJsonKey.TYPICAL_RAW_HTML, '') + is_xpath = pre_data.get(PreDataJsonKey.IS_XPATH, True) # layout_file_list = pre_data.get(PreDataJsonKey.LAYOUT_FILE_LIST, []) # 执行HTML标签简化逻辑 try: - simplified_html, original_html, _ = simplify_html(typical_raw_html) + simplified_html, original_html, _ = simplify_html(typical_raw_html, is_xpath=is_xpath) except TagSimplifiedParserException as e1: raise e1 except Exception as e2: diff --git a/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py b/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py index 3d656811..ea5d57e9 100644 --- a/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py +++ b/tests/llm_web_kit/main_html_parser/parser/test_tag_simplifier.py @@ -57,7 +57,7 @@ def test_tag_simplifier4(self): file_path = base_dir / 'assets/test_html_data/1.html' with open(file_path, 'r', encoding='utf-8') as file: raw_html = file.read() - data_dict = {PreDataJsonKey.TYPICAL_RAW_HTML: raw_html} + data_dict = {PreDataJsonKey.TYPICAL_RAW_HTML: raw_html, PreDataJsonKey.IS_XPATH: False} pre_data = PreDataJson(data_dict) pre_data_result = HtmlTagSimplifierParser({}).parse(pre_data) simplifier_raw_html = pre_data_result.get(PreDataJsonKey.TYPICAL_SIMPLIFIED_HTML, '') From c96bbf9a242919d996ce0ccfb895fac5e30d2023 Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Mon, 9 Jun 2025 19:44:34 +0800 Subject: [PATCH 3/7] =?UTF-8?q?feat:=20=E7=B2=BE=E7=AE=80=E6=8E=A7?= =?UTF-8?q?=E5=88=B6=E6=98=AF=E5=90=A6=E8=8E=B7=E5=8F=96XPATH?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- llm_web_kit/input/pre_data_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_web_kit/input/pre_data_json.py b/llm_web_kit/input/pre_data_json.py index 2153073f..7902d093 100644 --- a/llm_web_kit/input/pre_data_json.py +++ b/llm_web_kit/input/pre_data_json.py @@ -15,7 +15,7 @@ class PreDataJsonKey: TYPICAL_RAW_HTML = 'typical_raw_html' TYPICAL_RAW_TAG_HTML = 'typical_raw_tag_html' - IS_XPATH = True + IS_XPATH = 'is_xpath' XPATH_MAPPING = 'xpath_mapping' TYPICAL_SIMPLIFIED_HTML = 'typical_simplified_html' # 模型打标字典 From 61fd634b15f55917c4e447df6ec2335da4eb260d Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Tue, 17 Jun 2025 16:40:22 +0800 Subject: [PATCH 4/7] =?UTF-8?q?feat:=20=E8=87=AA=E5=AE=9A=E4=B9=89?= =?UTF-8?q?=E6=A0=87=E7=AD=BE'marked-tail',=20'marked-text'=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E4=B8=BA=E8=A1=8C=E5=86=85=E6=A0=87=E7=AD=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- llm_web_kit/main_html_parser/simplify_html/simplify_html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py index 94e5d629..679b0742 100644 --- a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py +++ b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py @@ -12,7 +12,7 @@ 'samp', 'blink', 'b', 'code', 'nobr', 'strike', 'bdo', 'basefont', 'abbr', 'var', 'i', 'cccode-inline', 'select', 's', 'pic', 'label', 'mark', 'object', 'dd', 'dt', 'ccmath-inline', 'svg', 'li', 'button', 'a', 'font', 'dfn', 'sup', 'kbd', 'q', 'script', 'acronym', 'option', 'img', 'big', 'cite', - 'em', + 'em', 'marked-tail', 'marked-text' # 'td', 'th' } From a73c2b40da775e822ad43f2e1b1654a9d91d8737 Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Thu, 19 Jun 2025 19:07:03 +0800 Subject: [PATCH 5/7] =?UTF-8?q?fix:=20=E9=87=8D=E5=91=BD=E5=90=8D=E8=87=AA?= =?UTF-8?q?=E5=AE=9A=E4=B9=89=E6=A0=87=E7=AD=BE=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main_html_parser/simplify_html/simplify_html.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py index 679b0742..312fb3cd 100644 --- a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py +++ b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py @@ -47,6 +47,9 @@ # '-header', '_header', # 有特例，可能自定义的header中有标题，先注释 } +# 自定义标签 +tail_block_tag = "cc-alg-uc-text" + def add_data_uids(dom: html.HtmlElement) -> None: """为DOM所有节点添加data-uid属性（递归所有子节点）""" @@ -762,7 +765,7 @@ def process_paragraphs(paragraphs: List[Dict[str, str]], uid_map: Dict[str, html # trailing_text = last_child.tail # 创建wrapper元素 - wrapper = etree.Element('cc-alg-uc-tex') + wrapper = etree.Element(tail_block_tag) wrapper.set('_item_id', current_id) # 设置前面的文本 @@ -798,7 +801,7 @@ def process_paragraphs(paragraphs: List[Dict[str, str]], uid_map: Dict[str, html # 检查父节点的text if original_parent.text and original_parent.text.strip() == root_for_xpath.text.strip(): # 创建wrapper - wrapper = etree.Element('cc-alg-uc-tex') + wrapper = etree.Element(tail_block_tag) wrapper.set('_item_id', current_id) wrapper.text = original_parent.text @@ -818,7 +821,7 @@ def process_paragraphs(paragraphs: List[Dict[str, str]], uid_map: Dict[str, html for child in original_parent.iterchildren(): if child.tail and child.tail.strip() == root_for_xpath.text.strip(): # 创建wrapper - wrapper = etree.Element('cc-alg-uc-tex') + wrapper = etree.Element(tail_block_tag) wrapper.set('_item_id', current_id) wrapper.text = child.tail @@ -835,7 +838,7 @@ def process_paragraphs(paragraphs: List[Dict[str, str]], uid_map: Dict[str, html # 如果没有找到匹配的文本节点，使用父节点作为包裹对象 if not found: - wrapper = etree.Element('cc-alg-uc-tex') + wrapper = etree.Element(tail_block_tag) wrapper.set('_item_id', current_id) wrapper.text = root_for_xpath.text # 将父节点的内容移动到wrapper中 From 87d9f822620c6263ed52ba9836c6354a357f8cfa Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Thu, 19 Jun 2025 19:14:04 +0800 Subject: [PATCH 6/7] =?UTF-8?q?fix:=20=E9=87=8D=E5=91=BD=E5=90=8D=E8=87=AA?= =?UTF-8?q?=E5=AE=9A=E4=B9=89=E6=A0=87=E7=AD=BE=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- llm_web_kit/main_html_parser/simplify_html/simplify_html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py index 312fb3cd..f472c3a8 100644 --- a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py +++ b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py @@ -48,7 +48,7 @@ } # 自定义标签 -tail_block_tag = "cc-alg-uc-text" +tail_block_tag = 'cc-alg-uc-text' def add_data_uids(dom: html.HtmlElement) -> None: From ff910e9d750d2d9eacb75da8e7a89528855c0ad8 Mon Sep 17 00:00:00 2001 From: houlinfeng Date: Thu, 19 Jun 2025 19:44:35 +0800 Subject: [PATCH 7/7] =?UTF-8?q?fix:=20=E5=8E=BB=E6=8E=89=E5=86=97=E4=BD=99?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main_html_parser/simplify_html/simplify_html.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py index f472c3a8..bfff8f29 100644 --- a/llm_web_kit/main_html_parser/simplify_html/simplify_html.py +++ b/llm_web_kit/main_html_parser/simplify_html/simplify_html.py @@ -833,21 +833,8 @@ def process_paragraphs(paragraphs: List[Dict[str, str]], uid_map: Dict[str, html index = parent.index(child) parent.insert(index + 1, wrapper) - found = True break - # 如果没有找到匹配的文本节点，使用父节点作为包裹对象 - if not found: - wrapper = etree.Element(tail_block_tag) - wrapper.set('_item_id', current_id) - wrapper.text = root_for_xpath.text - # 将父节点的内容移动到wrapper中 - for child in list(original_parent.iterchildren()): - wrapper.append(child) - original_parent.remove(child) - - # 添加wrapper到父节点 - original_parent.append(wrapper) else: # 块级元素直接设置属性 original_parent.set('_item_id', current_id)