<doc>: add extract main html by model response to README.md

README.md

@@ -141,6 +141,38 @@ if __name__=="__main__":
     main_html = extract(url, html)
 ```
 
+### extract main_html by model response
+
+```python
+import traceback
+from loguru import logger
+from llm_web_kit.main_html_parser.simplify_html.simplify_html import simplify_html
+from llm_web_kit.input.pre_data_json import PreDataJson, PreDataJsonKey
+from llm_web_kit.main_html_parser.parser.tag_mapping import MapItemToHtmlTagsParser
+
+def extract(response_json: dict, html:str) -> str:
+    try:
+        _, typical_raw_tag_html, _ = simplify_html(html)
+        pre_data = PreDataJson({})
+        pre_data[PreDataJsonKey.TYPICAL_RAW_TAG_HTML] = typical_raw_tag_html
+        pre_data[PreDataJsonKey.TYPICAL_RAW_HTML] = html
+        pre_data['success_label_enable'] = True
+        pre_data[PreDataJsonKey.LLM_RESPONSE] = response_json
+        parser = MapItemToHtmlTagsParser({})
+        pre_data = parser.parse_single(pre_data)
+        main_html = pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML]
+        is_success = pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML_SUCCESS]
+        return main_html, is_success
+    except Exception as e:
+        logger.exception(e)
+    return None
+
+if __name__=="__main__":
+    response_json =  {'item_id 1': 0, 'item_id 2': 1, 'item_id 3': 1}
+    html = ""
+    main_html, is_success = extract(response_json, html)
+```
+
 ## Pipeline
 
 1. [HTML pre-dedup](jupyter/html-pre-dedup/main.ipynb)

llm_web_kit/main_html_parser/parser/tag_mapping.py

@@ -84,10 +84,25 @@
         try:
             template_tag_html = pre_data[PreDataJsonKey.TYPICAL_RAW_TAG_HTML]
             response_json = pre_data[PreDataJsonKey.LLM_RESPONSE]
+            source_html = pre_data[PreDataJsonKey.TYPICAL_RAW_HTML]
             root = html.fromstring(template_tag_html)
             # 直接抽取正文
             content_list = self.tag_main_html(response_json, root)
             template_extract_html = self.__extract_main_directly(root)
+            if pre_data.get('success_label_enable', False):
+                feature1 = get_feature(source_html)
+                feature2 = get_feature(template_extract_html)
+                layer = 6
+                template_sim = None
+                if feature1 is not None and feature2 is not None:
+                    template_sim = similarity(feature1, feature2, layer_n=layer)
+
+                # 比较模版正文html与原html相似度
+                if template_sim is None or template_sim > SIMILAR_THRESHOLD:
+                    pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML_SUCCESS] = False
+                else:
+                    pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML_SUCCESS] = True
+
             pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML] = template_extract_html
             pre_data[PreDataJsonKey.HTML_TARGET_LIST] = content_list
         except Exception as e:

tests/llm_web_kit/main_html_parser/processor/test_tag_mapping.py

@@ -72,6 +72,7 @@ def test_parse_single(self):
         mock_dict = data[0]
         pre_data = PreDataJson(mock_dict['pre_data'])
         pre_data[PreDataJsonKey.TYPICAL_RAW_HTML] = pre_data[PreDataJsonKey.TYPICAL_RAW_TAG_HTML]
+        pre_data['success_label_enable'] = True
         parser = MapItemToHtmlTagsParser({})
         pre_data = parser.parse_single(pre_data)
         content_list = pre_data[PreDataJsonKey.HTML_TARGET_LIST]