diff --git a/llm_web_kit/input/pre_data_json.py b/llm_web_kit/input/pre_data_json.py index 7902d093..8eec7cc2 100644 --- a/llm_web_kit/input/pre_data_json.py +++ b/llm_web_kit/input/pre_data_json.py @@ -34,6 +34,8 @@ class PreDataJsonKey: TYPICAL_DICT_HTML = 'typical_dict_html' # 动态id开关 DYNAMIC_ID_ENABLE = 'dynamic_id_enable' + # 动态classid开关 + DYNAMIC_CLASSID_ENABLE = 'dynamic_classid_enable' # 推广原网页 HTML_SOURCE = 'html_source' # 推广网页提取正文成功标签, bool类型 diff --git a/llm_web_kit/main_html_parser/parser/layout_batch_parser.py b/llm_web_kit/main_html_parser/parser/layout_batch_parser.py index de7038f6..d4641f17 100644 --- a/llm_web_kit/main_html_parser/parser/layout_batch_parser.py +++ b/llm_web_kit/main_html_parser/parser/layout_batch_parser.py @@ -65,8 +65,10 @@ def parse(self, pre_data: PreDataJson) -> PreDataJson: layer = self.__get_max_width_layer(template_data) feature1 = get_feature(template_main_html) feature2 = get_feature(body) - sim = similarity(feature1, feature2, layer_n=layer) - if sim < SIMILARITY_THRESHOLD: + sim = None + if feature1 is not None and feature2 is not None: + sim = similarity(feature1, feature2, layer_n=layer) + if sim is None or sim < SIMILARITY_THRESHOLD: pre_data[PreDataJsonKey.MAIN_HTML_SUCCESS] = False else: pre_data[PreDataJsonKey.MAIN_HTML_SUCCESS] = True diff --git a/llm_web_kit/main_html_parser/parser/tag_mapping.py b/llm_web_kit/main_html_parser/parser/tag_mapping.py index 91b074d1..8d525281 100644 --- a/llm_web_kit/main_html_parser/parser/tag_mapping.py +++ b/llm_web_kit/main_html_parser/parser/tag_mapping.py @@ -52,10 +52,12 @@ def parse(self, pre_data: PreDataJson) -> PreDataJson: feature1 = get_feature(template_raw_html) feature2 = get_feature(template_extract_html) layer = self.__get_max_width_layer(element_dict) - template_sim = similarity(feature1, feature2, layer_n=layer) + template_sim = None + if feature1 is not None and feature2 is not None: + template_sim = similarity(feature1, feature2, layer_n=layer) # 比较模版正文html与原html相似度 - if template_sim > SIMILAR_THRESHOLD: + if template_sim is None or template_sim > SIMILAR_THRESHOLD: pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML_SUCCESS] = False else: pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML_SUCCESS] = True diff --git a/tests/llm_web_kit/input/test_pre_data_json.py b/tests/llm_web_kit/input/test_pre_data_json.py index 502ee37c..5371fee9 100644 --- a/tests/llm_web_kit/input/test_pre_data_json.py +++ b/tests/llm_web_kit/input/test_pre_data_json.py @@ -391,6 +391,7 @@ def test_pre_data_json_key_constants(self): assert hasattr(PreDataJsonKey, 'MAIN_HTML_SUCCESS') assert hasattr(PreDataJsonKey, 'TYPICAL_DICT_HTML') assert hasattr(PreDataJsonKey, 'DYNAMIC_ID_ENABLE') + assert hasattr(PreDataJsonKey, 'DYNAMIC_CLASSID_ENABLE') # Check actual values assert PreDataJsonKey.DOMAIN_NAME == 'domain_name' diff --git a/tests/llm_web_kit/main_html_parser/parser/test_layout_parser.py b/tests/llm_web_kit/main_html_parser/parser/test_layout_parser.py index 50836314..af2b6aec 100644 --- a/tests/llm_web_kit/main_html_parser/parser/test_layout_parser.py +++ b/tests/llm_web_kit/main_html_parser/parser/test_layout_parser.py @@ -167,7 +167,7 @@ def test_dynamic_id(self): # 推广 pre_data[PreDataJsonKey.HTML_SOURCE] = expand_source pre_data[PreDataJsonKey.DYNAMIC_ID_ENABLE] = True - pre_data['dynamic_classid_enable'] = True + pre_data[PreDataJsonKey.DYNAMIC_CLASSID_ENABLE] = True parser = LayoutBatchParser(element_dict) parts = parser.parse(pre_data) main_html_body = parts[PreDataJsonKey.MAIN_HTML_BODY] @@ -207,7 +207,7 @@ def test_dynamic_classid(self): # 推广 pre_data[PreDataJsonKey.HTML_SOURCE] = expand_source pre_data[PreDataJsonKey.DYNAMIC_ID_ENABLE] = True - pre_data['dynamic_classid_enable'] = True + pre_data[PreDataJsonKey.DYNAMIC_CLASSID_ENABLE] = True parser = LayoutBatchParser(element_dict) parts = parser.parse(pre_data) main_html_body = parts[PreDataJsonKey.MAIN_HTML_BODY] @@ -230,7 +230,7 @@ def test_dynamic_classid(self): # 推广 pre_data[PreDataJsonKey.HTML_SOURCE] = expand_source2 pre_data[PreDataJsonKey.DYNAMIC_ID_ENABLE] = True - pre_data['dynamic_classid_enable'] = True + pre_data[PreDataJsonKey.DYNAMIC_CLASSID_ENABLE] = True parser = LayoutBatchParser(element_dict) parts = parser.parse(pre_data) main_html_body = parts[PreDataJsonKey.MAIN_HTML_BODY]