diff --git a/README.md b/README.md index bb51386c..9902b852 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,38 @@ if __name__=="__main__": main_html = extract(url, html) ``` +### extract main_html by model response + +```python +import traceback +from loguru import logger +from llm_web_kit.main_html_parser.simplify_html.simplify_html import simplify_html +from llm_web_kit.input.pre_data_json import PreDataJson, PreDataJsonKey +from llm_web_kit.main_html_parser.parser.tag_mapping import MapItemToHtmlTagsParser + +def extract(response_json: dict, html:str) -> str: + try: + _, typical_raw_tag_html, _ = simplify_html(html) + pre_data = PreDataJson({}) + pre_data[PreDataJsonKey.TYPICAL_RAW_TAG_HTML] = typical_raw_tag_html + pre_data[PreDataJsonKey.TYPICAL_RAW_HTML] = html + pre_data['success_label_enable'] = True + pre_data[PreDataJsonKey.LLM_RESPONSE] = response_json + parser = MapItemToHtmlTagsParser({}) + pre_data = parser.parse_single(pre_data) + main_html = pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML] + is_success = pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML_SUCCESS] + return main_html, is_success + except Exception as e: + logger.exception(e) + return None + +if __name__=="__main__": + response_json = {'item_id 1': 0, 'item_id 2': 1, 'item_id 3': 1} + html = "" + main_html, is_success = extract(response_json, html) +``` + ## Pipeline 1. [HTML pre-dedup](jupyter/html-pre-dedup/main.ipynb) diff --git a/llm_web_kit/main_html_parser/parser/tag_mapping.py b/llm_web_kit/main_html_parser/parser/tag_mapping.py index d1f99090..c7c2b8e2 100644 --- a/llm_web_kit/main_html_parser/parser/tag_mapping.py +++ b/llm_web_kit/main_html_parser/parser/tag_mapping.py @@ -84,10 +84,25 @@ def parse_single(self, pre_data: PreDataJson) -> PreDataJson: try: template_tag_html = pre_data[PreDataJsonKey.TYPICAL_RAW_TAG_HTML] response_json = pre_data[PreDataJsonKey.LLM_RESPONSE] + source_html = pre_data[PreDataJsonKey.TYPICAL_RAW_HTML] root = html.fromstring(template_tag_html) # 直接抽取正文 content_list = self.tag_main_html(response_json, root) template_extract_html = self.__extract_main_directly(root) + if pre_data.get('success_label_enable', False): + feature1 = get_feature(source_html) + feature2 = get_feature(template_extract_html) + layer = 6 + template_sim = None + if feature1 is not None and feature2 is not None: + template_sim = similarity(feature1, feature2, layer_n=layer) + + # 比较模版正文html与原html相似度 + if template_sim is None or template_sim > SIMILAR_THRESHOLD: + pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML_SUCCESS] = False + else: + pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML_SUCCESS] = True + pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML] = template_extract_html pre_data[PreDataJsonKey.HTML_TARGET_LIST] = content_list except Exception as e: diff --git a/tests/llm_web_kit/main_html_parser/processor/test_tag_mapping.py b/tests/llm_web_kit/main_html_parser/processor/test_tag_mapping.py index 44b8b9c3..2ae8341c 100644 --- a/tests/llm_web_kit/main_html_parser/processor/test_tag_mapping.py +++ b/tests/llm_web_kit/main_html_parser/processor/test_tag_mapping.py @@ -72,6 +72,7 @@ def test_parse_single(self): mock_dict = data[0] pre_data = PreDataJson(mock_dict['pre_data']) pre_data[PreDataJsonKey.TYPICAL_RAW_HTML] = pre_data[PreDataJsonKey.TYPICAL_RAW_TAG_HTML] + pre_data['success_label_enable'] = True parser = MapItemToHtmlTagsParser({}) pre_data = parser.parse_single(pre_data) content_list = pre_data[PreDataJsonKey.HTML_TARGET_LIST]