Skip to content

bench: fix MagicHTMLFIleFormatorExtractor #471

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yogacc33 merged 2 commits into from
Jun 24, 2025
Merged

bench: fix MagicHTMLFIleFormatorExtractor #471

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a24c601
bench: fix MagicHTMLFIleFormatorExtractor
e06084 Jun 24, 2025
4bea48e
x
e06084 Jun 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion bench/config/ours_config.jsonc
View file
Open in desktop
Comment thread
yogacc33 marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
"extractor": [
{
"enable": true,
"python_class": "llm_web_kit.extractor.html.extractor.HTMLFileFormatExtractor",
"python_class": "llm_web_kit.extractor.html.extractor.MagicHTMLFIleFormatorExtractor",
"class_init_kwargs": {}
}
],
Expand Down
135 changes: 66 additions & 69 deletions bench/eval/ours.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
import json
import os
from pathlib import Path
from typing import Dict, List, Tuple

from llm_web_kit.extractor.extractor_chain import ExtractSimpleFactory
Expand All @@ -22,69 +19,69 @@ def eval_ours_extract_html(chain_config: dict, test_data: dict) -> Tuple[str, Li
return content, content_list._get_data(), statics


if __name__ == '__main__':
root = Path(__file__).parent.parent.parent
from llm_web_kit.dataio.filebase import (FileBasedDataReader,
FileBasedDataWriter)
reader = FileBasedDataReader('')
writer = FileBasedDataWriter('')

# 确保输出目录存在
output_dir = f'{root}/bench/output/ours'
os.makedirs(output_dir, exist_ok=True)

with open(f'{root}/bench/config/ours_config.jsonc', 'r') as f:
chain_config = json.load(f)

# 循环处理每一行数据
with open(f'{root}/bench/config/data_math_config.jsonl', 'r') as f:
for line in f:
test_data = json.loads(line.strip())
content, content_list, statics = eval_ours_extract_html(
chain_config,
test_data
)
print('处理数据:', test_data.get('track_id'))
print('URL:', test_data.get('url'))
print('统计信息:', statics)

# 读取html
html_content = reader.read(
f'{root}/bench/{test_data.get("path")}'
).decode('utf-8')

# 提取main_html
from llm_web_kit.extractor.html.extractor import \
HTMLFileFormatExtractor
htmlExtractor = HTMLFileFormatExtractor(chain_config)
main_html, method, title = htmlExtractor._extract_main_html(
html_content, test_data.get('url', ''), test_data.get('page_layout_type', 'article')
)

out = {
'url': test_data.get('url'),
'content': content,
'main_html': main_html,
'content_list': content_list,
'html': html_content,
'statics': statics
}

# 获取path的前两级目录
path = test_data.get('path', '')
path_parts = path.split('/')
if len(path_parts) >= 2:
output_subdir = '/'.join(path_parts[:2])
else:
output_subdir = 'unknown'

# 创建对应的输出目录
output_dir = f'{root}/bench/output/ours/{output_subdir}'

# 追加写入结果
output_file = f'{output_dir}.jsonl'
writer.append_write(
output_file,
json.dumps(out).encode('utf-8') + b'\n'
)
print(f'结果已追加到: {output_file}')
# if __name__ == '__main__':
# root = Path(__file__).parent.parent.parent
# from llm_web_kit.dataio.filebase import (FileBasedDataReader,
# FileBasedDataWriter)
# reader = FileBasedDataReader('')
# writer = FileBasedDataWriter('')

# # 确保输出目录存在
# output_dir = f'{root}/bench/output/ours'
# os.makedirs(output_dir, exist_ok=True)

# with open(f'{root}/bench/config/ours_config.jsonc', 'r') as f:
# chain_config = json.load(f)

# # 循环处理每一行数据
# with open(f'{root}/bench/config/data_math_config.jsonl', 'r') as f:
# for line in f:
# test_data = json.loads(line.strip())
# content, content_list, statics = eval_ours_extract_html(
# chain_config,
# test_data
# )
# print('处理数据:', test_data.get('track_id'))
# print('URL:', test_data.get('url'))
# print('统计信息:', statics)

# # 读取html
# html_content = reader.read(
# f'{root}/bench/{test_data.get('path')}'
# ).decode('utf-8')

# # 提取main_html
# from llm_web_kit.extractor.html.extractor import \
# MagicHTMLFIleFormatorExtractor
# htmlExtractor = MagicHTMLFIleFormatorExtractor(chain_config)
# main_html, method, title = htmlExtractor._extract_main_html(
# html_content, test_data.get('url', ''), test_data.get('page_layout_type', 'article')
# )

# out = {
# 'url': test_data.get('url'),
# 'content': content,
# 'main_html': main_html,
# 'content_list': content_list,
# 'html': html_content,
# 'statics': statics
# }

# # 获取path的前两级目录
# path = test_data.get('path', '')
# path_parts = path.split('/')
# if len(path_parts) >= 2:
# output_subdir = '/'.join(path_parts[:2])
# else:
# output_subdir = 'unknown'

# # 创建对应的输出目录
# output_dir = f'{root}/bench/output/ours/{output_subdir}'

# # 追加写入结果
# output_file = f'{output_dir}.jsonl'
# writer.append_write(
# output_file,
# json.dumps(out).encode('utf-8') + b'\n'
# )
# print(f'结果已追加到: {output_file}')
91 changes: 40 additions & 51 deletions bench/run.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from bench.eval.ours import eval_ours_extract_html
from llm_web_kit.dataio.filebase import (FileBasedDataReader,
FileBasedDataWriter)
from llm_web_kit.extractor.html.extractor import HTMLFileFormatExtractor
from llm_web_kit.extractor.html.extractor import MagicHTMLFIleFormatorExtractor
from llm_web_kit.libs.statics import Statics


Expand All @@ -18,13 +18,11 @@ def parse_arguments():
parser = argparse.ArgumentParser(description='HTML提取与评估工具')
parser.add_argument('--input', type=str, help='HTML文件路径')
parser.add_argument('--output', type=str, help='输出文件路径')
parser.add_argument(
'--tool',
type=str,
choices=['ours', 'magic_html', 'unstructured'],
help='抽取工具',
default='ours'
)
parser.add_argument('--tool',
type=str,
choices=['ours', 'magic_html', 'unstructured'],
help='抽取工具',
default='ours')
return parser.parse_args()


Expand All @@ -41,7 +39,8 @@ def setup_paths():
return paths


def run_ours(config_path, data_path, output_path, statics_pre, reader, writer, summary, detail):
def run_ours(config_path, data_path, output_path, statics_pre, reader, writer,
summary, detail):
"""运行我们的提取模型.

Args:
Expand All @@ -62,6 +61,11 @@ def run_ours(config_path, data_path, output_path, statics_pre, reader, writer, s
print(f'数据路径: {data_path_str}')
print(f'输出路径: {output_path_str}')

# 加载配置文件
import commentjson
with open(config_path_str, 'r', encoding='utf-8') as f:
chain_config = commentjson.load(f)

with open(data_path_str, 'r', encoding='utf-8') as f:
for line_num, line in enumerate(f, 1):
try:
Expand All @@ -74,8 +78,7 @@ def run_ours(config_path, data_path, output_path, statics_pre, reader, writer, s

# 执行评估
content, content_list, statics = eval_ours_extract_html(
config_path_str, data_json
)
chain_config, data_json)

# 获取路径并进行安全处理
path = data_json.get('path', '')
Expand All @@ -88,18 +91,20 @@ def run_ours(config_path, data_path, output_path, statics_pre, reader, writer, s
html_content = ''
if file_path and os.path.exists(file_path):
try:
html_content = reader.read(file_path).decode('utf-8')
html_content = reader.read(file_path).decode(
'utf-8')
print(f'成功读取HTML，长度: {len(html_content)}')
except Exception as e:
print(f'读取HTML文件失败: {e}')
else:
print(f'文件不存在或路径为空: {file_path}')

# 提取main_html
htmlExtractor = HTMLFileFormatExtractor(config_path_str)
htmlExtractor = MagicHTMLFIleFormatorExtractor(
chain_config)
main_html, method, title = htmlExtractor._extract_main_html(
html_content, data_json.get('url', ''), data_json.get('page_layout_type', 'article')
)
html_content, data_json.get('url', ''),
data_json.get('page_layout_type', 'article'))

# 准备输出内容
out = {
Expand Down Expand Up @@ -130,7 +135,7 @@ def run_ours(config_path, data_path, output_path, statics_pre, reader, writer, s
print(f'成功写入结果到: {output_file}')
summary.total += 1
except Exception as e:
summary.error_count += 1
summary.error_summary['count'] += 1
import traceback
print(f'处理单条数据时出错: {e}')
print(traceback.format_exc())
Expand Down Expand Up @@ -165,8 +170,7 @@ def run_magic_html(html, url, file_name, output_path, writer):

writer.write(
os.path.join(output_path, 'magic_html', f'{file_name}.jsonl'),
json.dumps(out, ensure_ascii=False).encode('utf-8') + b'\n'
)
json.dumps(out, ensure_ascii=False).encode('utf-8') + b'\n')
except Exception as e:
print(f'运行magic_html评估时出错: {e}')

Expand Down Expand Up @@ -195,8 +199,7 @@ def run_unstructured(html, url, file_name, output_path, writer):

writer.write(
os.path.join(output_path, 'unstructured', f'{file_name}.jsonl'),
json.dumps(out, ensure_ascii=False).encode('utf-8') + b'\n'
)
json.dumps(out, ensure_ascii=False).encode('utf-8') + b'\n')
except Exception as e:
print(f'运行unstructured评估时出错: {e}')

Expand All @@ -218,13 +221,11 @@ def main():
output_path = os.path.join(paths['output'], task_id)

# 创建评测结果概览
summary = Result_Summary.create(
task_id=task_id,
output_path=output_path,
total=0,
result_summary={},
error_count=0
)
summary = Result_Summary.create(task_id=task_id,
output_path=output_path,
total=0,
result_summary={},
error_count=0)

# 创建评测结果详情
detail = Result_Detail.create(
Expand All @@ -239,16 +240,11 @@ def main():

# 如果是ours工具，直接运行ours评估
if args.tool == 'ours':
summary, detail, statics_pre = run_ours(
paths['pipeline_config'],
paths['pipeline_data'],
paths['output'],
statics_pre,
reader,
writer,
summary,
detail
)
summary, detail, statics_pre = run_ours(paths['pipeline_config'],
paths['pipeline_data'],
paths['output'], statics_pre,
reader, writer, summary,
detail)
else:
# 读取HTML文件
try:
Expand All @@ -261,8 +257,7 @@ def main():
url = file_data.get('url', '')
origin_filepath = file_data.get('origin_filepath', '')
groundtruth_filepath = file_data.get(
'groundtruth_filepath', ''
)
'groundtruth_filepath', '')
layout_type = file_data.get('layout_type', '')

print(f'处理: {file_name}, 类型: {layout_type}')
Expand All @@ -275,27 +270,21 @@ def main():
try:
html = reader.read(str(html_path)).decode('utf-8')
groundtruth_data = reader.read(
str(groundtruth_path)
).decode('utf-8')
str(groundtruth_path)).decode('utf-8')
groundtruth = json.loads(groundtruth_data)
statics_gt.merge_statics(
groundtruth.get('statics', {})
)
groundtruth.get('statics', {}))
except Exception as e:
print(f'读取文件失败: {e}')
continue

# 根据工具类型运行不同的评估
if args.tool == 'magic_html':
run_magic_html(
html, url, file_name,
paths['output'], writer
)
run_magic_html(html, url, file_name,
paths['output'], writer)
elif args.tool == 'unstructured':
run_unstructured(
html, url, file_name,
paths['output'], writer
)
run_unstructured(html, url, file_name,
paths['output'], writer)
except Exception as e:
print(f'处理文件 {file_name} 时出错: {e}')
except Exception as e:
Expand Down
3 changes: 2 additions & 1 deletion jupyter/domain_clustering/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,5 +93,6 @@ offset = 257877031
record_count = 10

# Read records from specified offset and count
rows = list(read_s3_rows_from_offset(filepath, offset=offset, limit=record_count))
for i, row in enumerate(read_s3_rows_from_offset(filepath, offset=offset, limit=record_count)):
data = json.loads(row.value)
```
3 changes: 2 additions & 1 deletion jupyter/domain_clustering/README_ZH.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,5 +93,6 @@ offset = 257877031
record_count = 10

# 读取指定偏移量和数量的记录
rows = list(read_s3_rows_from_offset(filepath, offset=offset, limit=record_count))
for i, row in enumerate(read_s3_rows_from_offset(filepath, offset=offset, limit=record_count)):
data = json.loads(row.value)
```
Loading