From a24c601044fa5bbb95ae3483278e8f4caad02747 Mon Sep 17 00:00:00 2001 From: chupei Date: Tue, 24 Jun 2025 10:35:02 +0800 Subject: [PATCH 1/2] bench: fix MagicHTMLFIleFormatorExtractor --- bench/config/ours_config.jsonc | 2 +- bench/eval/ours.py | 6 +- bench/run.py | 91 ++++++-------- jupyter/domain_clustering/README.md | 3 +- jupyter/domain_clustering/README_ZH.md | 3 +- jupyter/html-pre-dedup/cc_dedup_fir.ipynb | 63 ++-------- jupyter/html-pre-dedup/cc_dedup_sec.ipynb | 143 ++++------------------ jupyter/html-pre-dedup/cc_dedup_thr.ipynb | 117 ++++-------------- jupyter/layout-clustering/layout.ipynb | 46 ++----- 9 files changed, 123 insertions(+), 351 deletions(-) diff --git a/bench/config/ours_config.jsonc b/bench/config/ours_config.jsonc index e602e8bb..6fdc9281 100644 --- a/bench/config/ours_config.jsonc +++ b/bench/config/ours_config.jsonc @@ -37,7 +37,7 @@ "extractor": [ { "enable": true, - "python_class": "llm_web_kit.extractor.html.extractor.HTMLFileFormatExtractor", + "python_class": "llm_web_kit.extractor.html.extractor.MagicHTMLFIleFormatorExtractor", "class_init_kwargs": {} } ], diff --git a/bench/eval/ours.py b/bench/eval/ours.py index 12e01fb3..dbe756c8 100644 --- a/bench/eval/ours.py +++ b/bench/eval/ours.py @@ -50,13 +50,13 @@ def eval_ours_extract_html(chain_config: dict, test_data: dict) -> Tuple[str, Li # 读取html html_content = reader.read( - f'{root}/bench/{test_data.get("path")}' + f'{root}/bench/{test_data.get('path')}' ).decode('utf-8') # 提取main_html from llm_web_kit.extractor.html.extractor import \ - HTMLFileFormatExtractor - htmlExtractor = HTMLFileFormatExtractor(chain_config) + MagicHTMLFIleFormatorExtractor + htmlExtractor = MagicHTMLFIleFormatorExtractor(chain_config) main_html, method, title = htmlExtractor._extract_main_html( html_content, test_data.get('url', ''), test_data.get('page_layout_type', 'article') ) diff --git a/bench/run.py b/bench/run.py index b591bd24..13c05ee5 100644 --- a/bench/run.py +++ b/bench/run.py @@ -9,7 +9,7 @@ from bench.eval.ours import eval_ours_extract_html from llm_web_kit.dataio.filebase import (FileBasedDataReader, FileBasedDataWriter) -from llm_web_kit.extractor.html.extractor import HTMLFileFormatExtractor +from llm_web_kit.extractor.html.extractor import MagicHTMLFIleFormatorExtractor from llm_web_kit.libs.statics import Statics @@ -18,13 +18,11 @@ def parse_arguments(): parser = argparse.ArgumentParser(description='HTML提取与评估工具') parser.add_argument('--input', type=str, help='HTML文件路径') parser.add_argument('--output', type=str, help='输出文件路径') - parser.add_argument( - '--tool', - type=str, - choices=['ours', 'magic_html', 'unstructured'], - help='抽取工具', - default='ours' - ) + parser.add_argument('--tool', + type=str, + choices=['ours', 'magic_html', 'unstructured'], + help='抽取工具', + default='ours') return parser.parse_args() @@ -41,7 +39,8 @@ def setup_paths(): return paths -def run_ours(config_path, data_path, output_path, statics_pre, reader, writer, summary, detail): +def run_ours(config_path, data_path, output_path, statics_pre, reader, writer, + summary, detail): """运行我们的提取模型. Args: @@ -62,6 +61,11 @@ def run_ours(config_path, data_path, output_path, statics_pre, reader, writer, s print(f'数据路径: {data_path_str}') print(f'输出路径: {output_path_str}') + # 加载配置文件 + import commentjson + with open(config_path_str, 'r', encoding='utf-8') as f: + chain_config = commentjson.load(f) + with open(data_path_str, 'r', encoding='utf-8') as f: for line_num, line in enumerate(f, 1): try: @@ -74,8 +78,7 @@ def run_ours(config_path, data_path, output_path, statics_pre, reader, writer, s # 执行评估 content, content_list, statics = eval_ours_extract_html( - config_path_str, data_json - ) + chain_config, data_json) # 获取路径并进行安全处理 path = data_json.get('path', '') @@ -88,7 +91,8 @@ def run_ours(config_path, data_path, output_path, statics_pre, reader, writer, s html_content = '' if file_path and os.path.exists(file_path): try: - html_content = reader.read(file_path).decode('utf-8') + html_content = reader.read(file_path).decode( + 'utf-8') print(f'成功读取HTML，长度: {len(html_content)}') except Exception as e: print(f'读取HTML文件失败: {e}') @@ -96,10 +100,11 @@ def run_ours(config_path, data_path, output_path, statics_pre, reader, writer, s print(f'文件不存在或路径为空: {file_path}') # 提取main_html - htmlExtractor = HTMLFileFormatExtractor(config_path_str) + htmlExtractor = MagicHTMLFIleFormatorExtractor( + chain_config) main_html, method, title = htmlExtractor._extract_main_html( - html_content, data_json.get('url', ''), data_json.get('page_layout_type', 'article') - ) + html_content, data_json.get('url', ''), + data_json.get('page_layout_type', 'article')) # 准备输出内容 out = { @@ -130,7 +135,7 @@ def run_ours(config_path, data_path, output_path, statics_pre, reader, writer, s print(f'成功写入结果到: {output_file}') summary.total += 1 except Exception as e: - summary.error_count += 1 + summary.error_summary['count'] += 1 import traceback print(f'处理单条数据时出错: {e}') print(traceback.format_exc()) @@ -165,8 +170,7 @@ def run_magic_html(html, url, file_name, output_path, writer): writer.write( os.path.join(output_path, 'magic_html', f'{file_name}.jsonl'), - json.dumps(out, ensure_ascii=False).encode('utf-8') + b'\n' - ) + json.dumps(out, ensure_ascii=False).encode('utf-8') + b'\n') except Exception as e: print(f'运行magic_html评估时出错: {e}') @@ -195,8 +199,7 @@ def run_unstructured(html, url, file_name, output_path, writer): writer.write( os.path.join(output_path, 'unstructured', f'{file_name}.jsonl'), - json.dumps(out, ensure_ascii=False).encode('utf-8') + b'\n' - ) + json.dumps(out, ensure_ascii=False).encode('utf-8') + b'\n') except Exception as e: print(f'运行unstructured评估时出错: {e}') @@ -218,13 +221,11 @@ def main(): output_path = os.path.join(paths['output'], task_id) # 创建评测结果概览 - summary = Result_Summary.create( - task_id=task_id, - output_path=output_path, - total=0, - result_summary={}, - error_count=0 - ) + summary = Result_Summary.create(task_id=task_id, + output_path=output_path, + total=0, + result_summary={}, + error_count=0) # 创建评测结果详情 detail = Result_Detail.create( @@ -239,16 +240,11 @@ def main(): # 如果是ours工具，直接运行ours评估 if args.tool == 'ours': - summary, detail, statics_pre = run_ours( - paths['pipeline_config'], - paths['pipeline_data'], - paths['output'], - statics_pre, - reader, - writer, - summary, - detail - ) + summary, detail, statics_pre = run_ours(paths['pipeline_config'], + paths['pipeline_data'], + paths['output'], statics_pre, + reader, writer, summary, + detail) else: # 读取HTML文件 try: @@ -261,8 +257,7 @@ def main(): url = file_data.get('url', '') origin_filepath = file_data.get('origin_filepath', '') groundtruth_filepath = file_data.get( - 'groundtruth_filepath', '' - ) + 'groundtruth_filepath', '') layout_type = file_data.get('layout_type', '') print(f'处理: {file_name}, 类型: {layout_type}') @@ -275,27 +270,21 @@ def main(): try: html = reader.read(str(html_path)).decode('utf-8') groundtruth_data = reader.read( - str(groundtruth_path) - ).decode('utf-8') + str(groundtruth_path)).decode('utf-8') groundtruth = json.loads(groundtruth_data) statics_gt.merge_statics( - groundtruth.get('statics', {}) - ) + groundtruth.get('statics', {})) except Exception as e: print(f'读取文件失败: {e}') continue # 根据工具类型运行不同的评估 if args.tool == 'magic_html': - run_magic_html( - html, url, file_name, - paths['output'], writer - ) + run_magic_html(html, url, file_name, + paths['output'], writer) elif args.tool == 'unstructured': - run_unstructured( - html, url, file_name, - paths['output'], writer - ) + run_unstructured(html, url, file_name, + paths['output'], writer) except Exception as e: print(f'处理文件 {file_name} 时出错: {e}') except Exception as e: diff --git a/jupyter/domain_clustering/README.md b/jupyter/domain_clustering/README.md index 1a5c5808..24c46f93 100644 --- a/jupyter/domain_clustering/README.md +++ b/jupyter/domain_clustering/README.md @@ -93,5 +93,6 @@ offset = 257877031 record_count = 10 # Read records from specified offset and count -rows = list(read_s3_rows_from_offset(filepath, offset=offset, limit=record_count)) +for i, row in enumerate(read_s3_rows_from_offset(filepath, offset=offset, limit=record_count)): + data = json.loads(row.value) ``` diff --git a/jupyter/domain_clustering/README_ZH.md b/jupyter/domain_clustering/README_ZH.md index 44363768..3f87cf4c 100644 --- a/jupyter/domain_clustering/README_ZH.md +++ b/jupyter/domain_clustering/README_ZH.md @@ -93,5 +93,6 @@ offset = 257877031 record_count = 10 # 读取指定偏移量和数量的记录 -rows = list(read_s3_rows_from_offset(filepath, offset=offset, limit=record_count)) +for i, row in enumerate(read_s3_rows_from_offset(filepath, offset=offset, limit=record_count)): + data = json.loads(row.value) ``` diff --git a/jupyter/html-pre-dedup/cc_dedup_fir.ipynb b/jupyter/html-pre-dedup/cc_dedup_fir.ipynb index c4346155..a0d68007 100644 --- a/jupyter/html-pre-dedup/cc_dedup_fir.ipynb +++ b/jupyter/html-pre-dedup/cc_dedup_fir.ipynb @@ -3,16 +3,8 @@ { "cell_type": "code", "execution_count": null, - "id": "6a86ea9b-beca-4324-b101-5684b3ed0955", - "metadata": { - "execution": { - "iopub.execute_input": "2025-04-28T02:21:07.893806Z", - "iopub.status.busy": "2025-04-28T02:21:07.893523Z", - "iopub.status.idle": "2025-04-28T02:21:19.180935Z", - "shell.execute_reply": "2025-04-28T02:21:19.180129Z", - "shell.execute_reply.started": "2025-04-28T02:21:07.893790Z" - } - }, + "id": "0", + "metadata": {}, "outputs": [], "source": [ "from pyspark.sql import Row\n", @@ -42,7 +34,7 @@ }, { "cell_type": "markdown", - "id": "d61f7baa-669f-43ed-8597-dbe17ffb6e6f", + "id": "1", "metadata": {}, "source": [ "# html source" @@ -51,16 +43,8 @@ { "cell_type": "code", "execution_count": null, - "id": "b0846950-4d76-4446-b7a8-7eea3a097a32", - "metadata": { - "execution": { - "iopub.execute_input": "2025-04-28T02:21:19.217067Z", - "iopub.status.busy": "2025-04-28T02:21:19.216842Z", - "iopub.status.idle": "2025-04-28T02:32:01.584069Z", - "shell.execute_reply": "2025-04-28T02:32:01.583574Z", - "shell.execute_reply.started": "2025-04-28T02:21:19.217052Z" - } - }, + "id": "2", + "metadata": {}, "outputs": [], "source": [ "# 获取 cc warc path list\n", @@ -73,16 +57,9 @@ { "cell_type": "code", "execution_count": null, - "id": "4e659d5d-ce3e-4578-b525-b97d5d65b835", + "id": "3", "metadata": { "editable": true, - "execution": { - "iopub.execute_input": "2025-04-28T02:32:01.585248Z", - "iopub.status.busy": "2025-04-28T02:32:01.584870Z", - "iopub.status.idle": "2025-04-28T02:32:01.589941Z", - "shell.execute_reply": "2025-04-28T02:32:01.589505Z", - "shell.execute_reply.started": "2025-04-28T02:32:01.585230Z" - }, "slideshow": { "slide_type": "" }, @@ -127,16 +104,8 @@ { "cell_type": "code", "execution_count": null, - "id": "07f785ee-b51b-4f6b-a9da-8abd7286c1d0", - "metadata": { - "execution": { - "iopub.execute_input": "2025-04-28T02:32:01.591272Z", - "iopub.status.busy": "2025-04-28T02:32:01.590889Z", - "iopub.status.idle": "2025-04-28T04:12:44.895573Z", - "shell.execute_reply": "2025-04-28T04:12:44.894960Z", - "shell.execute_reply.started": "2025-04-28T02:32:01.591255Z" - } - }, + "id": "4", + "metadata": {}, "outputs": [], "source": [ "# mapPartitions 对 warc path 并行解析数据\n", @@ -149,7 +118,7 @@ }, { "cell_type": "markdown", - "id": "ff889c66-6c1d-4160-9704-f26f085c01d6", + "id": "5", "metadata": {}, "source": [ "# 写出s3" @@ -158,16 +127,8 @@ { "cell_type": "code", "execution_count": null, - "id": "6a058b7f-6c11-4b34-9178-8d6c58bb3d3b", - "metadata": { - "execution": { - "iopub.execute_input": "2025-04-24T03:43:00.145735Z", - "iopub.status.busy": "2025-04-24T03:43:00.145323Z", - "iopub.status.idle": "2025-04-24T08:01:20.893351Z", - "shell.execute_reply": "2025-04-24T08:01:20.892740Z", - "shell.execute_reply.started": "2025-04-24T03:43:00.145716Z" - } - }, + "id": "6", + "metadata": {}, "outputs": [], "source": [ "config[\"skip_output_version\"] = True\n", @@ -178,7 +139,7 @@ { "cell_type": "code", "execution_count": null, - "id": "697a3eae-a6e1-4815-9697-5783d77ba29b", + "id": "7", "metadata": {}, "outputs": [], "source": [] diff --git a/jupyter/html-pre-dedup/cc_dedup_sec.ipynb b/jupyter/html-pre-dedup/cc_dedup_sec.ipynb index c9061d86..70f17d7a 100644 --- a/jupyter/html-pre-dedup/cc_dedup_sec.ipynb +++ b/jupyter/html-pre-dedup/cc_dedup_sec.ipynb @@ -3,16 +3,8 @@ { "cell_type": "code", "execution_count": null, - "id": "6a86ea9b-beca-4324-b101-5684b3ed0955", - "metadata": { - "execution": { - "iopub.execute_input": "2025-04-25T07:54:27.433409Z", - "iopub.status.busy": "2025-04-25T07:54:27.433041Z", - "iopub.status.idle": "2025-04-25T07:54:39.114913Z", - "shell.execute_reply": "2025-04-25T07:54:39.114213Z", - "shell.execute_reply.started": "2025-04-25T07:54:27.433388Z" - } - }, + "id": "0", + "metadata": {}, "outputs": [], "source": [ "from xinghe.spark import *\n", @@ -42,7 +34,7 @@ }, { "cell_type": "markdown", - "id": "d61f7baa-669f-43ed-8597-dbe17ffb6e6f", + "id": "1", "metadata": {}, "source": [ "# 未去重的hash data" @@ -51,16 +43,8 @@ { "cell_type": "code", "execution_count": null, - "id": "c593add8-6678-43ae-9bca-8fae5d5d70a3", - "metadata": { - "execution": { - "iopub.execute_input": "2025-04-25T07:54:39.146127Z", - "iopub.status.busy": "2025-04-25T07:54:39.145279Z", - "iopub.status.idle": "2025-04-25T07:54:39.152579Z", - "shell.execute_reply": "2025-04-25T07:54:39.152019Z", - "shell.execute_reply.started": "2025-04-25T07:54:39.146096Z" - } - }, + "id": "2", + "metadata": {}, "outputs": [], "source": [ "dump_paths = []\n", @@ -72,16 +56,8 @@ { "cell_type": "code", "execution_count": null, - "id": "a0b5d748-af2d-4d91-8162-59b0bf6a8b69", - "metadata": { - "execution": { - "iopub.execute_input": "2025-04-25T07:54:39.153953Z", - "iopub.status.busy": "2025-04-25T07:54:39.153802Z", - "iopub.status.idle": "2025-04-25T07:57:41.861896Z", - "shell.execute_reply": "2025-04-25T07:57:41.861011Z", - "shell.execute_reply.started": "2025-04-25T07:54:39.153940Z" - } - }, + "id": "3", + "metadata": {}, "outputs": [], "source": [ "input_df = spark.read.format(\"json\").load(dump_paths)" @@ -89,7 +65,7 @@ }, { "cell_type": "markdown", - "id": "e1cb8715-6052-4589-87e9-d89f6414fc73", + "id": "4", "metadata": {}, "source": [ "# 已去重的ID" @@ -98,16 +74,8 @@ { "cell_type": "code", "execution_count": null, - "id": "92043e9e-a936-41a5-b914-47160a6b9043", - "metadata": { - "execution": { - "iopub.execute_input": "2025-04-25T07:57:41.863420Z", - "iopub.status.busy": "2025-04-25T07:57:41.863247Z", - "iopub.status.idle": "2025-04-25T08:04:55.067717Z", - "shell.execute_reply": "2025-04-25T08:04:55.066825Z", - "shell.execute_reply.started": "2025-04-25T07:57:41.863403Z" - } - }, + "id": "5", + "metadata": {}, "outputs": [], "source": [ "already_exist_id_v_df = read_any_path(spark, already_exist_id_path, config)" @@ -116,16 +84,8 @@ { "cell_type": "code", "execution_count": null, - "id": "90522101-aae7-460d-b1a3-23ec16ab44df", - "metadata": { - "execution": { - "iopub.execute_input": "2025-04-25T08:04:55.069168Z", - "iopub.status.busy": "2025-04-25T08:04:55.068981Z", - "iopub.status.idle": "2025-04-25T08:04:55.223173Z", - "shell.execute_reply": "2025-04-25T08:04:55.222285Z", - "shell.execute_reply.started": "2025-04-25T08:04:55.069150Z" - } - }, + "id": "6", + "metadata": {}, "outputs": [], "source": [ "# 定义 Schema\n", @@ -139,7 +99,7 @@ }, { "cell_type": "markdown", - "id": "968593e6-550c-4a18-8745-12eb4b8246ce", + "id": "7", "metadata": {}, "source": [ "# filter 历史未去重的剩余id" @@ -148,16 +108,8 @@ { "cell_type": "code", "execution_count": null, - "id": "8efab25d-5f75-4f73-9a95-6681fd904395", - "metadata": { - "execution": { - "iopub.execute_input": "2025-04-25T08:04:55.225065Z", - "iopub.status.busy": "2025-04-25T08:04:55.224885Z", - "iopub.status.idle": "2025-04-25T08:40:50.086052Z", - "shell.execute_reply": "2025-04-25T08:40:50.085226Z", - "shell.execute_reply.started": "2025-04-25T08:04:55.225046Z" - } - }, + "id": "8", + "metadata": {}, "outputs": [], "source": [ "join_df = input_df.join(already_exist_id_df, on=\"hash_html\", how=\"left\")\n", @@ -166,17 +118,8 @@ }, { "cell_type": "markdown", - "id": "c73b1b0f-01d8-4e34-be57-234889486ca6", - "metadata": { - "execution": { - "iopub.execute_input": "2024-12-17T02:52:53.668493Z", - "iopub.status.busy": "2024-12-17T02:52:53.668115Z", - "iopub.status.idle": "2024-12-17T02:52:53.727246Z", - "shell.execute_reply": "2024-12-17T02:52:53.726736Z", - "shell.execute_reply.started": "2024-12-17T02:52:53.668472Z" - }, - "scrolled": true - }, + "id": "9", + "metadata": {}, "source": [ "# 分区去重-->全局去重" ] @@ -184,16 +127,8 @@ { "cell_type": "code", "execution_count": null, - "id": "90da0774-60e4-4c17-97b7-36c2a27cf326", - "metadata": { - "execution": { - "iopub.execute_input": "2025-04-25T08:40:50.087637Z", - "iopub.status.busy": "2025-04-25T08:40:50.087211Z", - "iopub.status.idle": "2025-04-25T09:19:07.990695Z", - "shell.execute_reply": "2025-04-25T09:19:07.989734Z", - "shell.execute_reply.started": "2025-04-25T08:40:50.087608Z" - } - }, + "id": "10", + "metadata": {}, "outputs": [], "source": [ "def deduplicate_partition(partition):\n", @@ -205,16 +140,8 @@ { "cell_type": "code", "execution_count": null, - "id": "f29512c4-eb4a-433f-84c6-05af75f4fc56", - "metadata": { - "execution": { - "iopub.execute_input": "2025-04-25T09:19:07.992398Z", - "iopub.status.busy": "2025-04-25T09:19:07.991998Z", - "iopub.status.idle": "2025-04-25T09:26:22.593043Z", - "shell.execute_reply": "2025-04-25T09:26:22.592114Z", - "shell.execute_reply.started": "2025-04-25T09:19:07.992377Z" - } - }, + "id": "11", + "metadata": {}, "outputs": [], "source": [ "dedup_df = dedup_part_df.dropDuplicates([\"hash_html\"])" @@ -222,7 +149,7 @@ }, { "cell_type": "markdown", - "id": "ff889c66-6c1d-4160-9704-f26f085c01d6", + "id": "12", "metadata": {}, "source": [ "# 写出s3" @@ -231,16 +158,8 @@ { "cell_type": "code", "execution_count": null, - "id": "480ad3b5-93c2-4ec5-8a7b-c0b5c6474fa6", - "metadata": { - "execution": { - "iopub.execute_input": "2025-04-25T09:26:22.609121Z", - "iopub.status.busy": "2025-04-25T09:26:22.608653Z", - "iopub.status.idle": "2025-04-25T09:26:22.641888Z", - "shell.execute_reply": "2025-04-25T09:26:22.641315Z", - "shell.execute_reply.started": "2025-04-25T09:26:22.609100Z" - } - }, + "id": "13", + "metadata": {}, "outputs": [], "source": [ "struct_col = struct(dedup_df[\"track_id\"],dedup_df[\"sub_path\"],dedup_df[\"hash_html\"],)\n", @@ -250,16 +169,8 @@ { "cell_type": "code", "execution_count": null, - "id": "6a058b7f-6c11-4b34-9178-8d6c58bb3d3b", - "metadata": { - "execution": { - "iopub.execute_input": "2025-04-25T09:26:22.642766Z", - "iopub.status.busy": "2025-04-25T09:26:22.642622Z", - "iopub.status.idle": "2025-04-25T09:59:13.305437Z", - "shell.execute_reply": "2025-04-25T09:59:13.304628Z", - "shell.execute_reply.started": "2025-04-25T09:26:22.642752Z" - } - }, + "id": "14", + "metadata": {}, "outputs": [], "source": [ "config[\"skip_output_version\"] = True\n", @@ -270,7 +181,7 @@ { "cell_type": "code", "execution_count": null, - "id": "1502e9f0-0b45-4f9b-aeb6-3463673b5c7b", + "id": "15", "metadata": {}, "outputs": [], "source": [] diff --git a/jupyter/html-pre-dedup/cc_dedup_thr.ipynb b/jupyter/html-pre-dedup/cc_dedup_thr.ipynb index 8bb1952d..97b262d1 100644 --- a/jupyter/html-pre-dedup/cc_dedup_thr.ipynb +++ b/jupyter/html-pre-dedup/cc_dedup_thr.ipynb @@ -3,16 +3,8 @@ { "cell_type": "code", "execution_count": null, - "id": "6a86ea9b-beca-4324-b101-5684b3ed0955", - "metadata": { - "execution": { - "iopub.execute_input": "2025-04-25T10:30:19.574024Z", - "iopub.status.busy": "2025-04-25T10:30:19.573692Z", - "iopub.status.idle": "2025-04-25T10:30:31.101849Z", - "shell.execute_reply": "2025-04-25T10:30:31.101215Z", - "shell.execute_reply.started": "2025-04-25T10:30:19.574004Z" - } - }, + "id": "0", + "metadata": {}, "outputs": [], "source": [ "from pyspark.sql import Row\n", @@ -46,16 +38,8 @@ { "cell_type": "code", "execution_count": null, - "id": "1e87f6d1-8559-4f75-a2e3-cbb549bffcda", - "metadata": { - "execution": { - "iopub.execute_input": "2025-04-25T10:30:31.110540Z", - "iopub.status.busy": "2025-04-25T10:30:31.110404Z", - "iopub.status.idle": "2025-04-25T10:30:31.127547Z", - "shell.execute_reply": "2025-04-25T10:30:31.127075Z", - "shell.execute_reply.started": "2025-04-25T10:30:31.110528Z" - } - }, + "id": "1", + "metadata": {}, "outputs": [], "source": [ "def compress_and_decompress_str(input_data: Union[str, bytes], compress: bool = True, base: bool = False) -> Union[str, bytes]:\n", @@ -97,7 +81,7 @@ }, { "cell_type": "markdown", - "id": "d61f7baa-669f-43ed-8597-dbe17ffb6e6f", + "id": "2", "metadata": {}, "source": [ "# html source" @@ -106,14 +90,8 @@ { "cell_type": "code", "execution_count": null, - "id": "c593add8-6678-43ae-9bca-8fae5d5d70a3", - "metadata": { - "execution": { - "execution_failed": "2025-04-25T10:32:44.641Z", - "iopub.execute_input": "2025-04-25T10:30:31.128562Z", - "iopub.status.busy": "2025-04-25T10:30:31.128423Z" - } - }, + "id": "3", + "metadata": {}, "outputs": [], "source": [ "warc_paths = []\n", @@ -125,12 +103,8 @@ { "cell_type": "code", "execution_count": null, - "id": "8f77e0d6-2704-4a8b-8437-7b8efc17bfad", - "metadata": { - "execution": { - "execution_failed": "2025-04-25T10:32:44.641Z" - } - }, + "id": "4", + "metadata": {}, "outputs": [], "source": [ "def parse_path_to_html(iter):\n", @@ -151,12 +125,8 @@ { "cell_type": "code", "execution_count": null, - "id": "67a264ce-7aff-4c1b-bed0-441b8fe776c2", - "metadata": { - "execution": { - "execution_failed": "2025-04-25T10:32:44.641Z" - } - }, + "id": "5", + "metadata": {}, "outputs": [], "source": [ "html_schema = StructType([\n", @@ -169,17 +139,8 @@ }, { "cell_type": "markdown", - "id": "03cdc3be-02f9-4790-af49-393b56f7818d", - "metadata": { - "execution": { - "iopub.execute_input": "2024-12-17T02:52:53.668493Z", - "iopub.status.busy": "2024-12-17T02:52:53.668115Z", - "iopub.status.idle": "2024-12-17T02:52:53.727246Z", - "shell.execute_reply": "2024-12-17T02:52:53.726736Z", - "shell.execute_reply.started": "2024-12-17T02:52:53.668472Z" - }, - "scrolled": true - }, + "id": "6", + "metadata": {}, "source": [ "# unique id" ] @@ -187,12 +148,8 @@ { "cell_type": "code", "execution_count": null, - "id": "a9e9ff32-9770-4af0-97fc-f73d4a9e36f6", - "metadata": { - "execution": { - "execution_failed": "2025-04-25T10:32:44.641Z" - } - }, + "id": "7", + "metadata": {}, "outputs": [], "source": [ "input_paths = [f\"{base_unique_path}{i}/\" for i in DUMPS]\n", @@ -202,12 +159,8 @@ { "cell_type": "code", "execution_count": null, - "id": "c0dc5088-7528-4495-af62-176cc10379b4", - "metadata": { - "execution": { - "execution_failed": "2025-04-25T10:32:44.641Z" - } - }, + "id": "8", + "metadata": {}, "outputs": [], "source": [ "unique_schema = StructType([\n", @@ -221,12 +174,8 @@ { "cell_type": "code", "execution_count": null, - "id": "903c4560-a5a4-4b02-8039-326a2fec37e0", - "metadata": { - "execution": { - "execution_failed": "2025-04-25T10:32:44.641Z" - } - }, + "id": "9", + "metadata": {}, "outputs": [], "source": [ "inner_df = dump_html_df.join(unique_id_v_df, on='track_id', how='inner')" @@ -234,7 +183,7 @@ }, { "cell_type": "markdown", - "id": "ff889c66-6c1d-4160-9704-f26f085c01d6", + "id": "10", "metadata": {}, "source": [ "# write gz" @@ -243,16 +192,8 @@ { "cell_type": "code", "execution_count": null, - "id": "8234de52-3b83-442c-9039-76959b38f7c9", - "metadata": { - "execution": { - "iopub.execute_input": "2025-02-03T07:27:31.218856Z", - "iopub.status.busy": "2025-02-03T07:27:31.218575Z", - "iopub.status.idle": "2025-02-03T07:27:31.234740Z", - "shell.execute_reply": "2025-02-03T07:27:31.234249Z", - "shell.execute_reply.started": "2025-02-03T07:27:31.218838Z" - } - }, + "id": "11", + "metadata": {}, "outputs": [], "source": [ "output_df = inner_df.select(\"value\")" @@ -261,16 +202,8 @@ { "cell_type": "code", "execution_count": null, - "id": "67797c2a-19d5-4a7e-b98d-1bb96a43bfd7", - "metadata": { - "execution": { - "iopub.execute_input": "2025-02-03T04:35:05.291642Z", - "iopub.status.busy": "2025-02-03T04:35:05.291371Z", - "iopub.status.idle": "2025-02-03T06:25:09.748760Z", - "shell.execute_reply": "2025-02-03T06:25:09.748256Z", - "shell.execute_reply.started": "2025-02-03T04:35:05.291626Z" - } - }, + "id": "12", + "metadata": {}, "outputs": [], "source": [ "config[\"skip_output_version\"] = True\n", @@ -281,7 +214,7 @@ { "cell_type": "code", "execution_count": null, - "id": "697a3eae-a6e1-4815-9697-5783d77ba29b", + "id": "13", "metadata": {}, "outputs": [], "source": [] diff --git a/jupyter/layout-clustering/layout.ipynb b/jupyter/layout-clustering/layout.ipynb index ff935385..de0b28d1 100644 --- a/jupyter/layout-clustering/layout.ipynb +++ b/jupyter/layout-clustering/layout.ipynb @@ -3,16 +3,8 @@ { "cell_type": "code", "execution_count": null, - "id": "6a86ea9b-beca-4324-b101-5684b3ed0955", - "metadata": { - "execution": { - "iopub.execute_input": "2025-06-10T06:10:55.155678Z", - "iopub.status.busy": "2025-06-10T06:10:55.155250Z", - "iopub.status.idle": "2025-06-10T06:10:59.865678Z", - "shell.execute_reply": "2025-06-10T06:10:59.864800Z", - "shell.execute_reply.started": "2025-06-10T06:10:55.155655Z" - } - }, + "id": "0", + "metadata": {}, "outputs": [], "source": [ "from xinghe.spark import *\n", @@ -75,7 +67,7 @@ }, { "cell_type": "markdown", - "id": "8471645e-61ff-4d8f-9ba9-eb837826892e", + "id": "1", "metadata": { "jp-MarkdownHeadingCollapsed": true }, @@ -86,16 +78,8 @@ { "cell_type": "code", "execution_count": null, - "id": "eecf4c96-b0eb-4a0b-84f1-b6318332b5c2", - "metadata": { - "execution": { - "iopub.execute_input": "2025-06-10T06:10:59.881928Z", - "iopub.status.busy": "2025-06-10T06:10:59.881771Z", - "iopub.status.idle": "2025-06-10T06:10:59.884994Z", - "shell.execute_reply": "2025-06-10T06:10:59.884476Z", - "shell.execute_reply.started": "2025-06-10T06:10:59.881912Z" - } - }, + "id": "2", + "metadata": {}, "outputs": [], "source": [ "# 异常日志\n", @@ -600,7 +584,7 @@ }, { "cell_type": "markdown", - "id": "88f44a2f-8c65-4146-8a7b-a415f0a2ab21", + "id": "3", "metadata": {}, "source": [ "# main func" @@ -609,16 +593,8 @@ { "cell_type": "code", "execution_count": null, - "id": "8315c329-619c-483d-806c-9be915e23ea4", - "metadata": { - "execution": { - "iopub.execute_input": "2025-06-10T06:10:59.923082Z", - "iopub.status.busy": "2025-06-10T06:10:59.922942Z", - "iopub.status.idle": "2025-06-10T06:10:59.939642Z", - "shell.execute_reply": "2025-06-10T06:10:59.939010Z", - "shell.execute_reply.started": "2025-06-10T06:10:59.923068Z" - } - }, + "id": "4", + "metadata": {}, "outputs": [], "source": [ "\"\"\"第一步 choose domain\"\"\"\n", @@ -982,7 +958,7 @@ }, { "cell_type": "markdown", - "id": "3dc2976b-eeaa-424f-870f-64d99505f0ba", + "id": "5", "metadata": {}, "source": [ "# start" @@ -991,7 +967,7 @@ { "cell_type": "code", "execution_count": null, - "id": "86cbb2b6-66e7-4095-9ad6-9e5890f8b8e5", + "id": "6", "metadata": {}, "outputs": [], "source": [ @@ -1004,7 +980,7 @@ { "cell_type": "code", "execution_count": null, - "id": "1ef3163d-9bd0-4e69-9a0e-36af884c9e01", + "id": "7", "metadata": {}, "outputs": [], "source": [] From 4bea48e70961f1435eb4300006b264d8101402ad Mon Sep 17 00:00:00 2001 From: chupei Date: Tue, 24 Jun 2025 11:14:59 +0800 Subject: [PATCH 2/2] x --- bench/eval/ours.py | 135 ++++++++++++++++++++++----------------------- 1 file changed, 66 insertions(+), 69 deletions(-) diff --git a/bench/eval/ours.py b/bench/eval/ours.py index dbe756c8..26e0bc51 100644 --- a/bench/eval/ours.py +++ b/bench/eval/ours.py @@ -1,6 +1,3 @@ -import json -import os -from pathlib import Path from typing import Dict, List, Tuple from llm_web_kit.extractor.extractor_chain import ExtractSimpleFactory @@ -22,69 +19,69 @@ def eval_ours_extract_html(chain_config: dict, test_data: dict) -> Tuple[str, Li return content, content_list._get_data(), statics -if __name__ == '__main__': - root = Path(__file__).parent.parent.parent - from llm_web_kit.dataio.filebase import (FileBasedDataReader, - FileBasedDataWriter) - reader = FileBasedDataReader('') - writer = FileBasedDataWriter('') - - # 确保输出目录存在 - output_dir = f'{root}/bench/output/ours' - os.makedirs(output_dir, exist_ok=True) - - with open(f'{root}/bench/config/ours_config.jsonc', 'r') as f: - chain_config = json.load(f) - - # 循环处理每一行数据 - with open(f'{root}/bench/config/data_math_config.jsonl', 'r') as f: - for line in f: - test_data = json.loads(line.strip()) - content, content_list, statics = eval_ours_extract_html( - chain_config, - test_data - ) - print('处理数据:', test_data.get('track_id')) - print('URL:', test_data.get('url')) - print('统计信息:', statics) - - # 读取html - html_content = reader.read( - f'{root}/bench/{test_data.get('path')}' - ).decode('utf-8') - - # 提取main_html - from llm_web_kit.extractor.html.extractor import \ - MagicHTMLFIleFormatorExtractor - htmlExtractor = MagicHTMLFIleFormatorExtractor(chain_config) - main_html, method, title = htmlExtractor._extract_main_html( - html_content, test_data.get('url', ''), test_data.get('page_layout_type', 'article') - ) - - out = { - 'url': test_data.get('url'), - 'content': content, - 'main_html': main_html, - 'content_list': content_list, - 'html': html_content, - 'statics': statics - } - - # 获取path的前两级目录 - path = test_data.get('path', '') - path_parts = path.split('/') - if len(path_parts) >= 2: - output_subdir = '/'.join(path_parts[:2]) - else: - output_subdir = 'unknown' - - # 创建对应的输出目录 - output_dir = f'{root}/bench/output/ours/{output_subdir}' - - # 追加写入结果 - output_file = f'{output_dir}.jsonl' - writer.append_write( - output_file, - json.dumps(out).encode('utf-8') + b'\n' - ) - print(f'结果已追加到: {output_file}') +# if __name__ == '__main__': +# root = Path(__file__).parent.parent.parent +# from llm_web_kit.dataio.filebase import (FileBasedDataReader, +# FileBasedDataWriter) +# reader = FileBasedDataReader('') +# writer = FileBasedDataWriter('') + +# # 确保输出目录存在 +# output_dir = f'{root}/bench/output/ours' +# os.makedirs(output_dir, exist_ok=True) + +# with open(f'{root}/bench/config/ours_config.jsonc', 'r') as f: +# chain_config = json.load(f) + +# # 循环处理每一行数据 +# with open(f'{root}/bench/config/data_math_config.jsonl', 'r') as f: +# for line in f: +# test_data = json.loads(line.strip()) +# content, content_list, statics = eval_ours_extract_html( +# chain_config, +# test_data +# ) +# print('处理数据:', test_data.get('track_id')) +# print('URL:', test_data.get('url')) +# print('统计信息:', statics) + +# # 读取html +# html_content = reader.read( +# f'{root}/bench/{test_data.get('path')}' +# ).decode('utf-8') + +# # 提取main_html +# from llm_web_kit.extractor.html.extractor import \ +# MagicHTMLFIleFormatorExtractor +# htmlExtractor = MagicHTMLFIleFormatorExtractor(chain_config) +# main_html, method, title = htmlExtractor._extract_main_html( +# html_content, test_data.get('url', ''), test_data.get('page_layout_type', 'article') +# ) + +# out = { +# 'url': test_data.get('url'), +# 'content': content, +# 'main_html': main_html, +# 'content_list': content_list, +# 'html': html_content, +# 'statics': statics +# } + +# # 获取path的前两级目录 +# path = test_data.get('path', '') +# path_parts = path.split('/') +# if len(path_parts) >= 2: +# output_subdir = '/'.join(path_parts[:2]) +# else: +# output_subdir = 'unknown' + +# # 创建对应的输出目录 +# output_dir = f'{root}/bench/output/ours/{output_subdir}' + +# # 追加写入结果 +# output_file = f'{output_dir}.jsonl' +# writer.append_write( +# output_file, +# json.dumps(out).encode('utf-8') + b'\n' +# ) +# print(f'结果已追加到: {output_file}')