From fae8300494367e7940fe9649e8ffa975966ba33a Mon Sep 17 00:00:00 2001 From: liukaiwen Date: Tue, 15 Jul 2025 19:03:40 +0800 Subject: [PATCH 1/5] : add extract main html by model response to README.md --- README.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/README.md b/README.md index bb51386c..9902b852 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,38 @@ if __name__=="__main__": main_html = extract(url, html) ``` +### extract main_html by model response + +```python +import traceback +from loguru import logger +from llm_web_kit.main_html_parser.simplify_html.simplify_html import simplify_html +from llm_web_kit.input.pre_data_json import PreDataJson, PreDataJsonKey +from llm_web_kit.main_html_parser.parser.tag_mapping import MapItemToHtmlTagsParser + +def extract(response_json: dict, html:str) -> str: + try: + _, typical_raw_tag_html, _ = simplify_html(html) + pre_data = PreDataJson({}) + pre_data[PreDataJsonKey.TYPICAL_RAW_TAG_HTML] = typical_raw_tag_html + pre_data[PreDataJsonKey.TYPICAL_RAW_HTML] = html + pre_data['success_label_enable'] = True + pre_data[PreDataJsonKey.LLM_RESPONSE] = response_json + parser = MapItemToHtmlTagsParser({}) + pre_data = parser.parse_single(pre_data) + main_html = pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML] + is_success = pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML_SUCCESS] + return main_html, is_success + except Exception as e: + logger.exception(e) + return None + +if __name__=="__main__": + response_json = {'item_id 1': 0, 'item_id 2': 1, 'item_id 3': 1} + html = "" + main_html, is_success = extract(response_json, html) +``` + ## Pipeline 1. [HTML pre-dedup](jupyter/html-pre-dedup/main.ipynb) From 385597475fcb49d4eac35e14f7895fc365408a0f Mon Sep 17 00:00:00 2001 From: liukaiwen Date: Tue, 15 Jul 2025 19:07:17 +0800 Subject: [PATCH 2/5] : add extract main html by model response to README.md --- tests/llm_web_kit/main_html_parser/processor/test_tag_mapping.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/llm_web_kit/main_html_parser/processor/test_tag_mapping.py b/tests/llm_web_kit/main_html_parser/processor/test_tag_mapping.py index 44b8b9c3..2ae8341c 100644 --- a/tests/llm_web_kit/main_html_parser/processor/test_tag_mapping.py +++ b/tests/llm_web_kit/main_html_parser/processor/test_tag_mapping.py @@ -72,6 +72,7 @@ def test_parse_single(self): mock_dict = data[0] pre_data = PreDataJson(mock_dict['pre_data']) pre_data[PreDataJsonKey.TYPICAL_RAW_HTML] = pre_data[PreDataJsonKey.TYPICAL_RAW_TAG_HTML] + pre_data['success_label_enable'] = True parser = MapItemToHtmlTagsParser({}) pre_data = parser.parse_single(pre_data) content_list = pre_data[PreDataJsonKey.HTML_TARGET_LIST] From 2d9ac782bc49c8b044b933801f6a23d0159dc5d5 Mon Sep 17 00:00:00 2001 From: liukaiwen Date: Tue, 15 Jul 2025 19:10:05 +0800 Subject: [PATCH 3/5] : add extract main html by model response to README.md --- .../main_html_parser/parser/tag_mapping.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/llm_web_kit/main_html_parser/parser/tag_mapping.py b/llm_web_kit/main_html_parser/parser/tag_mapping.py index d1f99090..c7c2b8e2 100644 --- a/llm_web_kit/main_html_parser/parser/tag_mapping.py +++ b/llm_web_kit/main_html_parser/parser/tag_mapping.py @@ -84,10 +84,25 @@ def parse_single(self, pre_data: PreDataJson) -> PreDataJson: try: template_tag_html = pre_data[PreDataJsonKey.TYPICAL_RAW_TAG_HTML] response_json = pre_data[PreDataJsonKey.LLM_RESPONSE] + source_html = pre_data[PreDataJsonKey.TYPICAL_RAW_HTML] root = html.fromstring(template_tag_html) # 直接抽取正文 content_list = self.tag_main_html(response_json, root) template_extract_html = self.__extract_main_directly(root) + if pre_data.get('success_label_enable', False): + feature1 = get_feature(source_html) + feature2 = get_feature(template_extract_html) + layer = 6 + template_sim = None + if feature1 is not None and feature2 is not None: + template_sim = similarity(feature1, feature2, layer_n=layer) + + # 比较模版正文html与原html相似度 + if template_sim is None or template_sim > SIMILAR_THRESHOLD: + pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML_SUCCESS] = False + else: + pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML_SUCCESS] = True + pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML] = template_extract_html pre_data[PreDataJsonKey.HTML_TARGET_LIST] = content_list except Exception as e: From 288efe657415e1b9edab6b5033a2d1f87b70c588 Mon Sep 17 00:00:00 2001 From: liukaiwen Date: Tue, 15 Jul 2025 19:14:41 +0800 Subject: [PATCH 4/5] : add extract main html by model response to README.md --- .../main_html_parser/parser/tag_mapping.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/llm_web_kit/main_html_parser/parser/tag_mapping.py b/llm_web_kit/main_html_parser/parser/tag_mapping.py index c7c2b8e2..d1f99090 100644 --- a/llm_web_kit/main_html_parser/parser/tag_mapping.py +++ b/llm_web_kit/main_html_parser/parser/tag_mapping.py @@ -84,25 +84,10 @@ def parse_single(self, pre_data: PreDataJson) -> PreDataJson: try: template_tag_html = pre_data[PreDataJsonKey.TYPICAL_RAW_TAG_HTML] response_json = pre_data[PreDataJsonKey.LLM_RESPONSE] - source_html = pre_data[PreDataJsonKey.TYPICAL_RAW_HTML] root = html.fromstring(template_tag_html) # 直接抽取正文 content_list = self.tag_main_html(response_json, root) template_extract_html = self.__extract_main_directly(root) - if pre_data.get('success_label_enable', False): - feature1 = get_feature(source_html) - feature2 = get_feature(template_extract_html) - layer = 6 - template_sim = None - if feature1 is not None and feature2 is not None: - template_sim = similarity(feature1, feature2, layer_n=layer) - - # 比较模版正文html与原html相似度 - if template_sim is None or template_sim > SIMILAR_THRESHOLD: - pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML_SUCCESS] = False - else: - pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML_SUCCESS] = True - pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML] = template_extract_html pre_data[PreDataJsonKey.HTML_TARGET_LIST] = content_list except Exception as e: From dd364fc0178f49f74c1ec70af4f51f8c23432766 Mon Sep 17 00:00:00 2001 From: liukaiwen Date: Tue, 15 Jul 2025 20:30:58 +0800 Subject: [PATCH 5/5] : add extract main html by model response to README.md --- .../main_html_parser/parser/tag_mapping.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/llm_web_kit/main_html_parser/parser/tag_mapping.py b/llm_web_kit/main_html_parser/parser/tag_mapping.py index d1f99090..c7c2b8e2 100644 --- a/llm_web_kit/main_html_parser/parser/tag_mapping.py +++ b/llm_web_kit/main_html_parser/parser/tag_mapping.py @@ -84,10 +84,25 @@ def parse_single(self, pre_data: PreDataJson) -> PreDataJson: try: template_tag_html = pre_data[PreDataJsonKey.TYPICAL_RAW_TAG_HTML] response_json = pre_data[PreDataJsonKey.LLM_RESPONSE] + source_html = pre_data[PreDataJsonKey.TYPICAL_RAW_HTML] root = html.fromstring(template_tag_html) # 直接抽取正文 content_list = self.tag_main_html(response_json, root) template_extract_html = self.__extract_main_directly(root) + if pre_data.get('success_label_enable', False): + feature1 = get_feature(source_html) + feature2 = get_feature(template_extract_html) + layer = 6 + template_sim = None + if feature1 is not None and feature2 is not None: + template_sim = similarity(feature1, feature2, layer_n=layer) + + # 比较模版正文html与原html相似度 + if template_sim is None or template_sim > SIMILAR_THRESHOLD: + pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML_SUCCESS] = False + else: + pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML_SUCCESS] = True + pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML] = template_extract_html pre_data[PreDataJsonKey.HTML_TARGET_LIST] = content_list except Exception as e: