diff --git a/llm_web_kit/input/pre_data_json.py b/llm_web_kit/input/pre_data_json.py index 03dd802b..206325c2 100644 --- a/llm_web_kit/input/pre_data_json.py +++ b/llm_web_kit/input/pre_data_json.py @@ -36,6 +36,8 @@ class PreDataJsonKey: DYNAMIC_ID_ENABLE = 'dynamic_id_enable' # 动态classid开关 DYNAMIC_CLASSID_ENABLE = 'dynamic_classid_enable' + # 动态classid相似度阈值 + DYNAMIC_CLASSID_SIM_THRESH = 'dynamic_classid_similarity_threshold' # 正文噪音开关 MORE_NOISE_ENABLE = 'more_noise_enable' # 推广原网页 diff --git a/llm_web_kit/main_html_parser/parser/layout_batch_parser.py b/llm_web_kit/main_html_parser/parser/layout_batch_parser.py index 9a8cbd27..1de5af3c 100644 --- a/llm_web_kit/main_html_parser/parser/layout_batch_parser.py +++ b/llm_web_kit/main_html_parser/parser/layout_batch_parser.py @@ -28,6 +28,7 @@ def __init__(self, template_data: str | dict): self.dynamic_id_enable = False self.dynamic_classid_enable = False self.more_noise_enable = False + self.dynamic_classid_similarity_threshold = 0.85 def parse_tuple_key(self, key_str): if key_str.startswith('(') and key_str.endswith(')'): @@ -44,6 +45,7 @@ def parse(self, pre_data: PreDataJson) -> PreDataJson: self.dynamic_id_enable = pre_data.get(PreDataJsonKey.DYNAMIC_ID_ENABLE, False) self.dynamic_classid_enable = pre_data.get(PreDataJsonKey.DYNAMIC_CLASSID_ENABLE, False) self.more_noise_enable = pre_data.get(PreDataJsonKey.MORE_NOISE_ENABLE, False) + self.dynamic_classid_similarity_threshold = pre_data.get(PreDataJsonKey.DYNAMIC_CLASSID_SIM_THRESH, 0.85) template_data_str = pre_data[PreDataJsonKey.HTML_ELEMENT_DICT] template_data = dict() if isinstance(template_data_str, str): @@ -149,7 +151,6 @@ def find_blocks_drop(self, element, depth, element_dict, parent_keyy, parent_lab child_key = self.normalize_key(child_ori_key) child_str = html.tostring(child, encoding='utf-8').decode() current_layer_keys[child_key] = (child_ori_key, child_str) - # 匹配正文节点 has_red = False layer_nodes_dict = dict() @@ -237,11 +238,13 @@ def find_blocks_drop(self, element, depth, element_dict, parent_keyy, parent_lab else: parent = element.getparent() if parent is not None: + if element.tail: + element.tail = None parent.remove(element) return else: label = 'red' - elif length > 0 or length_tail > 0: + elif length > 0 or length_tail > 0 or tag in ['figure', 'img']: return for child in element: @@ -340,6 +343,8 @@ def __match_tag_class(self, layer_nodes, current_layer_key, parent_key, node_htm def __match_tag(self, layer_nodes, current_layer_key, parent_key, node_html, template_doc, class_must=False, id_exist=False): current_norm_key = (self.normalize_key((current_layer_key[0], None, None)), parent_key) + current_norm_key_with_first_class = ( + self.normalize_key((current_layer_key[0], current_layer_key[1].strip().split(' ')[0], None)), parent_key) for ele_keyy, ele_value in layer_nodes.items(): # class id要存在 if class_must and not ele_keyy[1]: @@ -354,6 +359,7 @@ def __match_tag(self, layer_nodes, current_layer_key, parent_key, node_html, tem ele_label = ele_value[0] norm_ele_keyy = self.normalize_key((ele_keyy[0], None, None)) norm_ele_keyy_parent = (norm_ele_keyy, ele_parent_keyy) + # 相似度方案 if current_norm_key == norm_ele_keyy_parent: # 计算3层相似度 ele_root = template_doc.xpath(xpath)[0] @@ -363,8 +369,14 @@ def __match_tag(self, layer_nodes, current_layer_key, parent_key, node_html, tem if feature1 is None or feature2 is None: continue template_sim = similarity(feature1, feature2, layer_n=3) - if template_sim > 0.85: + if template_sim >= self.dynamic_classid_similarity_threshold: return ele_label, self.normalize_key(ele_keyy[0:3]) + # first class方案 + norm_ele_keyy_with_first_class = self.normalize_key((ele_keyy[0], ele_keyy[1].strip().split(' ')[0], None)) + norm_ele_keyy_parent_with_first_class = (norm_ele_keyy_with_first_class, ele_parent_keyy) + if current_norm_key_with_first_class == norm_ele_keyy_parent_with_first_class: + return ele_label, self.normalize_key(ele_keyy[0:3]) + return None, None def __is_natural_language(self, text, min_words=3): diff --git a/tests/llm_web_kit/input/test_pre_data_json.py b/tests/llm_web_kit/input/test_pre_data_json.py index b81af4df..23d1b42e 100644 --- a/tests/llm_web_kit/input/test_pre_data_json.py +++ b/tests/llm_web_kit/input/test_pre_data_json.py @@ -392,6 +392,7 @@ def test_pre_data_json_key_constants(self): assert hasattr(PreDataJsonKey, 'TYPICAL_DICT_HTML') assert hasattr(PreDataJsonKey, 'DYNAMIC_ID_ENABLE') assert hasattr(PreDataJsonKey, 'DYNAMIC_CLASSID_ENABLE') + assert hasattr(PreDataJsonKey, 'DYNAMIC_CLASSID_SIM_THRESH') assert hasattr(PreDataJsonKey, 'MORE_NOISE_ENABLE') # Check actual values diff --git a/tests/llm_web_kit/main_html_parser/parser/test_layout_parser.py b/tests/llm_web_kit/main_html_parser/parser/test_layout_parser.py index 81912780..f6c26f68 100644 --- a/tests/llm_web_kit/main_html_parser/parser/test_layout_parser.py +++ b/tests/llm_web_kit/main_html_parser/parser/test_layout_parser.py @@ -280,3 +280,41 @@ def test_more_noise_enable(self): parts = parser.parse(pre_data) main_html_body = parts[PreDataJsonKey.MAIN_HTML_BODY] assert 'test more noise' in main_html_body + + def test_classid_with_first_class(self): + """测试动态classid失败的情况下，采取first classid方案判断 e.g. "class-main" vs "class- + main middle-content"只取第一个classid来判断.""" + # 构造测试html + html_source = base_dir.joinpath('assets/input_layout_batch_parser/www.wdi.it.html').read_text( + encoding='utf-8') + template_source = re.sub( + 'clearfix post post-37041 type-post status-publish format-standard hentry category-economia ' + + 'category-societa tag-camera tag-commercio tag-cosenza tag-diritto tag-economia-2 tag-imprese tag-libro ' + + 'tag-usi item-wrap', + 'post-classid', html_source) + template_source = re.sub('post-37041', '', template_source) + expand_source = re.sub('post-classid', 'post-classid template-classid', template_source) + # 简化网页 + simplified_html, typical_raw_tag_html, _ = simplify_html(template_source) + # 模型结果格式改写 + llm_path = base_dir.joinpath(TEST_CASES[0]['input'][2][0]) + llm_response = json.loads(llm_path.read_text(encoding='utf-8')) + for key in llm_response.keys(): + llm_response[key] = 1 if llm_response[key] == 'Yes' else 0 + pre_data = {'typical_raw_tag_html': typical_raw_tag_html, 'typical_raw_html': template_source, + 'llm_response': llm_response} + pre_data = PreDataJson(pre_data) + # 映射 + parser = MapItemToHtmlTagsParser({}) + pre_data = parser.parse(pre_data) + element_dict = pre_data.get(PreDataJsonKey.HTML_ELEMENT_DICT, {}) + # 推广 + pre_data[PreDataJsonKey.HTML_SOURCE] = expand_source + pre_data[PreDataJsonKey.DYNAMIC_ID_ENABLE] = True + pre_data[PreDataJsonKey.DYNAMIC_CLASSID_ENABLE] = True + pre_data[PreDataJsonKey.MORE_NOISE_ENABLE] = True + pre_data[PreDataJsonKey.DYNAMIC_CLASSID_SIM_THRESH] = 1 + parser = LayoutBatchParser(element_dict) + parts = parser.parse(pre_data) + main_html_body = parts[PreDataJsonKey.MAIN_HTML_BODY] + assert 'Permalink link a questo articolo' not in main_html_body and 'Con la stesura di un' in main_html_body