<fix>: fix image loss problem in new tag and modified the dynamic_classid_similarity_threshold

llm_web_kit/input/pre_data_json.py

@@ -36,6 +36,8 @@ class PreDataJsonKey:
     DYNAMIC_ID_ENABLE = 'dynamic_id_enable'
     # 动态classid开关
     DYNAMIC_CLASSID_ENABLE = 'dynamic_classid_enable'
+    # 动态classid相似度阈值
+    DYNAMIC_CLASSID_SIM_THRESH = 'dynamic_classid_similarity_threshold'
     # 正文噪音开关
     MORE_NOISE_ENABLE = 'more_noise_enable'
     # 推广原网页

llm_web_kit/main_html_parser/parser/layout_batch_parser.py

@@ -28,6 +28,7 @@ def __init__(self, template_data: str | dict):
         self.dynamic_id_enable = False
         self.dynamic_classid_enable = False
         self.more_noise_enable = False
+        self.dynamic_classid_similarity_threshold = 0.85
 
     def parse_tuple_key(self, key_str):
         if key_str.startswith('(') and key_str.endswith(')'):
@@ -44,6 +45,7 @@ def parse(self, pre_data: PreDataJson) -> PreDataJson:
         self.dynamic_id_enable = pre_data.get(PreDataJsonKey.DYNAMIC_ID_ENABLE, False)
         self.dynamic_classid_enable = pre_data.get(PreDataJsonKey.DYNAMIC_CLASSID_ENABLE, False)
         self.more_noise_enable = pre_data.get(PreDataJsonKey.MORE_NOISE_ENABLE, False)
+        self.dynamic_classid_similarity_threshold = pre_data.get(PreDataJsonKey.DYNAMIC_CLASSID_SIM_THRESH, 0.85)
         template_data_str = pre_data[PreDataJsonKey.HTML_ELEMENT_DICT]
         template_data = dict()
         if isinstance(template_data_str, str):
@@ -149,7 +151,6 @@ def find_blocks_drop(self, element, depth, element_dict, parent_keyy, parent_lab
                 child_key = self.normalize_key(child_ori_key)
                 child_str = html.tostring(child, encoding='utf-8').decode()
                 current_layer_keys[child_key] = (child_ori_key, child_str)
-
         # 匹配正文节点
         has_red = False
         layer_nodes_dict = dict()
@@ -237,11 +238,13 @@ def find_blocks_drop(self, element, depth, element_dict, parent_keyy, parent_lab
                 else:
                     parent = element.getparent()
                     if parent is not None:
+                        if element.tail:
+                            element.tail = None
                         parent.remove(element)
                     return
             else:
                 label = 'red'
-        elif length > 0 or length_tail > 0:
+        elif length > 0 or length_tail > 0 or tag in ['figure', 'img']:
             return
 
         for child in element:
@@ -340,6 +343,8 @@ def __match_tag_class(self, layer_nodes, current_layer_key, parent_key, node_htm
     def __match_tag(self, layer_nodes, current_layer_key, parent_key, node_html, template_doc, class_must=False,
                     id_exist=False):
         current_norm_key = (self.normalize_key((current_layer_key[0], None, None)), parent_key)
+        current_norm_key_with_first_class = (
+            self.normalize_key((current_layer_key[0], current_layer_key[1].strip().split(' ')[0], None)), parent_key)
         for ele_keyy, ele_value in layer_nodes.items():
             # class id要存在
             if class_must and not ele_keyy[1]:
@@ -354,6 +359,7 @@ def __match_tag(self, layer_nodes, current_layer_key, parent_key, node_html, tem
             ele_label = ele_value[0]
             norm_ele_keyy = self.normalize_key((ele_keyy[0], None, None))
             norm_ele_keyy_parent = (norm_ele_keyy, ele_parent_keyy)
+            # 相似度方案
             if current_norm_key == norm_ele_keyy_parent:
                 # 计算3层相似度
                 ele_root = template_doc.xpath(xpath)[0]
@@ -363,8 +369,14 @@ def __match_tag(self, layer_nodes, current_layer_key, parent_key, node_html, tem
                 if feature1 is None or feature2 is None:
                     continue
                 template_sim = similarity(feature1, feature2, layer_n=3)
-                if template_sim > 0.85:
+                if template_sim >= self.dynamic_classid_similarity_threshold:
                     return ele_label, self.normalize_key(ele_keyy[0:3])
+            # first class方案
+            norm_ele_keyy_with_first_class = self.normalize_key((ele_keyy[0], ele_keyy[1].strip().split(' ')[0], None))
+            norm_ele_keyy_parent_with_first_class = (norm_ele_keyy_with_first_class, ele_parent_keyy)
+            if current_norm_key_with_first_class == norm_ele_keyy_parent_with_first_class:
+                return ele_label, self.normalize_key(ele_keyy[0:3])
+
         return None, None
 
     def __is_natural_language(self, text, min_words=3):

tests/llm_web_kit/input/test_pre_data_json.py

@@ -392,6 +392,7 @@ def test_pre_data_json_key_constants(self):
         assert hasattr(PreDataJsonKey, 'TYPICAL_DICT_HTML')
         assert hasattr(PreDataJsonKey, 'DYNAMIC_ID_ENABLE')
         assert hasattr(PreDataJsonKey, 'DYNAMIC_CLASSID_ENABLE')
+        assert hasattr(PreDataJsonKey, 'DYNAMIC_CLASSID_SIM_THRESH')
         assert hasattr(PreDataJsonKey, 'MORE_NOISE_ENABLE')
 
         # Check actual values

tests/llm_web_kit/main_html_parser/parser/test_layout_parser.py

@@ -280,3 +280,41 @@ def test_more_noise_enable(self):
         parts = parser.parse(pre_data)
         main_html_body = parts[PreDataJsonKey.MAIN_HTML_BODY]
         assert 'test more noise' in main_html_body
+
+    def test_classid_with_first_class(self):
+        """测试动态classid失败的情况下，采取first classid方案判断 e.g. "class-main" vs "class-
+        main middle-content"只取第一个classid来判断."""
+        # 构造测试html
+        html_source = base_dir.joinpath('assets/input_layout_batch_parser/www.wdi.it.html').read_text(
+            encoding='utf-8')
+        template_source = re.sub(
+            'clearfix post post-37041 type-post status-publish format-standard hentry category-economia ' +
+            'category-societa tag-camera tag-commercio tag-cosenza tag-diritto tag-economia-2 tag-imprese tag-libro ' +
+            'tag-usi item-wrap',
+            'post-classid', html_source)
+        template_source = re.sub('post-37041', '', template_source)
+        expand_source = re.sub('post-classid', 'post-classid template-classid', template_source)
+        # 简化网页
+        simplified_html, typical_raw_tag_html, _ = simplify_html(template_source)
+        # 模型结果格式改写
+        llm_path = base_dir.joinpath(TEST_CASES[0]['input'][2][0])
+        llm_response = json.loads(llm_path.read_text(encoding='utf-8'))
+        for key in llm_response.keys():
+            llm_response[key] = 1 if llm_response[key] == 'Yes' else 0
+        pre_data = {'typical_raw_tag_html': typical_raw_tag_html, 'typical_raw_html': template_source,
+                    'llm_response': llm_response}
+        pre_data = PreDataJson(pre_data)
+        # 映射
+        parser = MapItemToHtmlTagsParser({})
+        pre_data = parser.parse(pre_data)
+        element_dict = pre_data.get(PreDataJsonKey.HTML_ELEMENT_DICT, {})
+        # 推广
+        pre_data[PreDataJsonKey.HTML_SOURCE] = expand_source
+        pre_data[PreDataJsonKey.DYNAMIC_ID_ENABLE] = True
+        pre_data[PreDataJsonKey.DYNAMIC_CLASSID_ENABLE] = True
+        pre_data[PreDataJsonKey.MORE_NOISE_ENABLE] = True
+        pre_data[PreDataJsonKey.DYNAMIC_CLASSID_SIM_THRESH] = 1
+        parser = LayoutBatchParser(element_dict)
+        parts = parser.parse(pre_data)
+        main_html_body = parts[PreDataJsonKey.MAIN_HTML_BODY]
+        assert 'Permalink link a questo articolo' not in main_html_body and 'Con la stesura di un' in main_html_body