Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion deepsearch/docs/feature/algorithm/report-generation.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
## 可见行为

- 报告正文以 Markdown 输出,并清理标题编号和过深标题。
- 最终报告在标题与摘要之间生成一级目录;目录以无项目符号的独立链接行列出正文一级章节,不展开子标题,也不包含摘要、结论和参考文章。
- 子报告根据章节计划和候选资料生成,失败时使用统一错误格式。
- 候选资料会先去重、按 step 分桶和按评分均衡筛选,再进入 LLM 分类。
- 表格 caption 会被标准化为稳定的“表 N”或英文对应格式,避免引用错位。
Expand Down Expand Up @@ -67,7 +68,7 @@
4. 子报告 Prompt 根据章节契约、选中文档和历史上下文生成 Markdown。
5. 报告工具清理标题编号、规范化表格标题,并按报告类型生成摘要、结论或建议。
6. 可视化内容如需插入,先抽取结构化数据并校验 schema,再生成 Mermaid 或交给图表模块。
7. 最终报告、classified contents、sidecar 和引用相关元数据交给后续溯源和前端展示。
7. 总报告按“标题 → 一级目录 → 摘要 → 正文章节 → 结论 → 参考文章”拼接,再将最终报告、classified contents、sidecar 和引用相关元数据交给后续溯源和前端展示。

## 数据契约与依赖

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,11 @@
## 可见行为

- HTML 导出会生成完整 HTML 文件并注入报告 CSS。
- HTML 导出保留报告目录的 `#chapter-N` 链接与章节锚点,普通页面和美化页面均可点击跳转。
- HTML 中数学公式通过 KaTeX 脚本(`katex.min.js` + `auto-render.min.js` + `katex.min.css`,版本固定 0.16.11)渲染,使用 `$...$` / `$$...$$` 作为定界符;`\bm` 宏映射为 `\boldsymbol{#1}`,`throwOnError=false` 保证无法解析的公式不打断页面渲染。
- HTML 在 KaTeX 渲染前会做"货币美元保护":遍历正文文本节点,把 `$` + 数字开头且不像公式的片段替换为全角 `$`(U+FF04)占位符,渲染完成后还原为 `$`,避免 `$4`、`$1,200.50` 等金额被 KaTeX 误配对为公式定界符。
- DOCX 导出使用纯 Python 流水线从 Markdown 生成 Word 文件。
- DOCX 导出将 `#chapter-N` 目录链接转换为 Word 内部超链接,并将对应章节锚点转换为书签;普通外部链接行为不变。
- HTML 路径通过 `conversion_utils.protect_math_spans`、DOCX 路径通过 `word_utils._iter_math_spans` 切分公式段;两者复用 `conversion_utils` 中的公式判别函数 `_is_likely_inline_math` / 货币判别函数 `_is_currency_start` / `_find_inline_math_end` / `_is_escaped` / `_is_double_dollar`,保证两侧对"哪些 `$...$` 是公式、哪些是货币或纯文本"的判定一致。
- DOCX 超链接文本中若包含 `$...$` 或 `$$...$$`,会把公式段单独切出并转为 OMML 公式 run,其余文本保持为普通文本 run;HTML 实体先经 `html.unescape` 解码再进入公式处理。
- DOCX 列表嵌套按列表类型显式分支:同类型嵌套(`ul→ul` / `ol→ol`)沿用同一编号并增加缩进层级;异类型嵌套(如 `ul` 内含 `ol`)创建独立编号,避免编号串号。
Expand Down
84 changes: 83 additions & 1 deletion deepsearch/openjiuwen_deepsearch/algorithm/report/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -1075,10 +1075,16 @@ async def generate_report(self, gen_report_context: dict) -> Tuple[bool, str]:
_outline_title = current_outline.get("title", "")
else:
_outline_title = getattr(current_outline, "title", "")
sub_reports_content = self._add_table_of_contents_anchors(sub_reports_content)
table_of_contents = self._build_table_of_contents(
sub_reports_content,
gen_report_context["language"],
)
report_content = (
f"{'# ' + _outline_title}\n\n" # Use outline title directly for report title
f"{table_of_contents}\n\n"
f"{self._post_process_abstract(abstract)}\n\n"
f"{sub_report_res.get('sub_reports_content')}\n\n"
f"{sub_reports_content}\n\n"
f"{self._post_process_conclusion(conclusion)}\n\n"
f"{ArticlePart.get_title('reference', gen_report_context['language'])}"
f"{sub_report_res.get('sub_references')}\n\n"
Expand Down Expand Up @@ -1479,6 +1485,82 @@ async def _generate_with_llm(self, task_type, prompt, content):
)
return llm_output.get("content")

@staticmethod
def _build_table_of_contents(sub_reports_content: str, language: str) -> str:
"""Build a clickable level-one TOC from the final body headings."""
headings = Reporter._extract_level_one_headings(sub_reports_content)
toc_title = ArticlePart.get_title("toc", language).strip()
if not headings:
return toc_title

toc_entries = "\n\n".join(
"[{0}](#chapter-{1})".format(heading["title"], index)
for index, heading in enumerate(headings, start=1)
)
return f"{toc_title}\n\n{toc_entries}"

@staticmethod
def _extract_level_one_headings(sub_reports_content: str) -> list[dict]:
"""Extract real Markdown H1 headings while ignoring fenced code blocks."""
headings = []
fence_char = ""
fence_length = 0

offset = 0
for line in (sub_reports_content or "").splitlines(keepends=True):
content_line = line.rstrip("\r\n")
if fence_char:
closing_fence = re.match(r"^\s{0,3}(`{3,}|~{3,})\s*$", content_line)
if closing_fence:
marker = closing_fence.group(1)
if marker[0] == fence_char and len(marker) >= fence_length:
fence_char = ""
fence_length = 0
offset += len(line)
continue

opening_fence = re.match(r"^\s{0,3}(`{3,}|~{3,})", content_line)
if opening_fence:
marker = opening_fence.group(1)
fence_char = marker[0]
fence_length = len(marker)
offset += len(line)
continue

heading_match = re.match(r"^\s{0,3}#(?!#)\s+(.+?)\s*$", content_line)
if heading_match:
heading = re.sub(
r"[ \t]+#+[ \t]*$", "", heading_match.group(1)
).strip()
if heading:
headings.append({"title": heading, "offset": offset})
offset += len(line)

return headings

@staticmethod
def _add_table_of_contents_anchors(sub_reports_content: str) -> str:
"""Add stable HTML anchors immediately before each real H1 chapter."""
anchor_line_pattern = re.compile(
r'(?m)^[ \t]*<a id="chapter-\d+"></a>[ \t]*(?:\r?\n|$)'
)
normalized = anchor_line_pattern.sub("", sub_reports_content or "")
headings = Reporter._extract_level_one_headings(normalized)
if not headings:
return normalized

newline = "\r\n" if "\r\n" in normalized else "\n"
for index, heading in reversed(list(enumerate(headings, start=1))):
anchor = '<a id="chapter-{}"></a>'.format(index)
position = heading["offset"]
normalized = (
normalized[:position]
+ anchor
+ newline
+ normalized[position:]
)
return normalized

def _post_process_abstract(self, content: str) -> str:
language = self.gen_report_context["language"]
if content is None or content == "":
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ class ArticlePart:
},
}
titles = {
"toc": {CHINESE: "# 目录\n\n", ENGLISH: "# Table of Contents\n\n"},
"abstract": {CHINESE: "# 摘要\n\n", ENGLISH: "# Abstract\n\n"},
"conclusion": {CHINESE: "# 结论\n\n", ENGLISH: "# Conclusion\n\n"},
"reference": {CHINESE: "# 参考文章\n\n", ENGLISH: "# Reference Articles\n\n"},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -414,17 +414,14 @@ def _add_hyperlink(paragraph, url, text, *, style_r_fonts=None, superscript: boo
若文本含 LaTeX 定界符($$...$$ 或 $...$),将其转为 OMML 公式
插入超链接内部。HTML 实体会先解码再进行 LaTeX 处理。
"""
# 创建关系 id
part = paragraph.part
r_id = part.relate_to(
url,
HYPERLINK_URI,
is_external=True
)

# 创建 <w:hyperlink>
hyperlink = OxmlElement("w:hyperlink")
hyperlink.set(qn("r:id"), r_id)
if url.startswith("#") and len(url) > 1:
hyperlink.set(qn("w:anchor"), _word_bookmark_name(url[1:]))
hyperlink.set(qn("w:history"), "1")
else:
r_id = paragraph.part.relate_to(url, HYPERLINK_URI, is_external=True)
hyperlink.set(qn("r:id"), r_id)

# 解码 HTML 实体(处理 &amp;#92; → &#92; → \ 等双重转义)
text = html.unescape(text)
Expand Down Expand Up @@ -467,6 +464,47 @@ def _add_hyperlink(paragraph, url, text, *, style_r_fonts=None, superscript: boo
_docx_paragraph_p(paragraph).append(hyperlink)


def _word_bookmark_name(value: str) -> str:
"""Convert an HTML anchor into a valid, stable Word bookmark name."""
name = re.sub(r"[^A-Za-z0-9_]", "_", unquote(value).strip())
if not name or name[0].isdigit():
name = f"_{name}"
return name[:40]


def _prepare_heading_bookmarks(container) -> None:
"""Move standalone HTML anchors onto their following heading elements."""
bookmark_id = 0
for anchor in list(container.select("a[id]:not([href])")):
if anchor.get_text(strip=True):
continue
heading = anchor.find_next(lambda tag: tag.name in HEADING_TAGS)
if heading is None:
continue

heading["data-docx-bookmark-name"] = _word_bookmark_name(anchor["id"])
heading["data-docx-bookmark-id"] = str(bookmark_id)
bookmark_id += 1

parent = anchor.parent
if parent is not None and parent.name == "p" and not parent.get_text(strip=True):
parent.decompose()
else:
anchor.decompose()


def _add_bookmark(paragraph, name: str, bookmark_id: str) -> None:
paragraph_element = _docx_paragraph_p(paragraph)
start = OxmlElement("w:bookmarkStart")
start.set(qn("w:id"), bookmark_id)
start.set(qn("w:name"), name)
end = OxmlElement("w:bookmarkEnd")
end.set(qn("w:id"), bookmark_id)

paragraph_element.insert(1 if paragraph_element.pPr is not None else 0, start)
paragraph_element.append(end)


def _is_relative_to(path: Path, base_path: Path) -> bool:
try:
path.relative_to(base_path)
Expand Down Expand Up @@ -661,6 +699,11 @@ def _add_para_and_apply_style(doc, element, context: HtmlToDocContext):
for child in element.contents:
_process_inline(p, child, replace(context, style_r_fonts=style_r_fonts))

bookmark_name = element.get("data-docx-bookmark-name")
bookmark_id = element.get("data-docx-bookmark-id")
if bookmark_name is not None and bookmark_id is not None:
_add_bookmark(p, bookmark_name, bookmark_id)


def _insert_omml(paragraph, omml_xml: str):
"""向段落中插入 OMML 公式"""
Expand Down Expand Up @@ -1069,6 +1112,7 @@ def html_to_doc(doc, html_content, style_dict, base_path: str | Path | None = No
container = soup.find("div", class_="report-container")
if container is None:
container = soup.body or soup
_prepare_heading_bookmarks(container)

resolved_base_path = Path(base_path).resolve() if base_path is not None else None
max_image_width = _get_available_page_width(doc)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import json
import zipfile
from pathlib import Path
from xml.etree import ElementTree

from docx import Document
import pytest
Expand Down Expand Up @@ -106,6 +107,61 @@ def test_render_markdown_html_fragment_preserves_shared_semantics() -> None:
assert '<div class="table-wrap">' in fragment


def test_docx_export_converts_report_toc_to_internal_links(tmp_path: Path) -> None:
"""DOCX 目录应链接到章节书签,而不是创建伪外部链接。"""
from openjiuwen_deepsearch.algorithm.report_export.docx_export import convert_md_to_docx

markdown_path = tmp_path / "report.md"
docx_path = tmp_path / "report.docx"
markdown_path.write_text(
"# 报告\n\n"
"# 目录\n\n"
"[第一章](#chapter-1)\n\n"
"[第二章](#chapter-2)\n\n"
'<a id="chapter-1"></a>\n'
"# 第一章\n\n正文。\n\n"
'<a id="chapter-2"></a>\n'
"# 第二章\n\n[外部链接](https://example.com)\n",
encoding="utf-8",
)

convert_md_to_docx(markdown_path, docx_path)

with zipfile.ZipFile(docx_path) as archive:
document = ElementTree.fromstring(archive.read("word/document.xml"))
relationships = archive.read("word/_rels/document.xml.rels").decode("utf-8")

word_ns = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
rel_ns = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"
bookmark_names = {
node.get(f"{{{word_ns}}}name")
for node in document.findall(f".//{{{word_ns}}}bookmarkStart")
}
internal_links = [
node
for node in document.findall(f".//{{{word_ns}}}hyperlink")
if node.get(f"{{{word_ns}}}anchor")
]

assert {"chapter_1", "chapter_2"}.issubset(bookmark_names)
assert {
node.get(f"{{{word_ns}}}anchor") for node in internal_links
} == {"chapter_1", "chapter_2"}
assert all(node.get(f"{{{rel_ns}}}id") is None for node in internal_links)
toc_paragraphs = [
paragraph
for paragraph in document.findall(f".//{{{word_ns}}}p")
if paragraph.find(f".//{{{word_ns}}}hyperlink[@{{{word_ns}}}anchor]") is not None
]
assert all(
paragraph.find(f".//{{{word_ns}}}numPr") is None
for paragraph in toc_paragraphs
)
assert "#chapter-1" not in relationships
assert "#chapter-2" not in relationships
assert "https://example.com" in relationships


def test_exporters_do_not_preprocess_mermaid_again_in_renderer(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,3 +46,37 @@ def test_html_export_variants_share_mermaid_rendering_and_limit_styled_dom(tmp_p
assert styled.select_one(".mermaid-wrap svg.chart-svg") is not None
assert standard.select_one("main.report-shell") is None
assert styled.select_one("main.report-shell") is not None


def test_html_export_variants_preserve_clickable_report_toc(tmp_path) -> None:
"""普通与美化 HTML 都应保留目录链接及对应章节锚点。"""
from openjiuwen_deepsearch.algorithm.report_export.html_export import (
ConvertOptions,
convert_md_to_html,
)

source = tmp_path / "report.md"
source.write_text(
"# 报告\n\n"
"# 目录\n\n"
"[第一章](#chapter-1)\n\n"
"[第二章](#chapter-2)\n\n"
'<a id="chapter-1"></a>\n'
"# 第一章\n\n正文。\n\n"
'<a id="chapter-2"></a>\n'
"# 第二章\n",
encoding="utf-8",
)

variants = (
(tmp_path / "standard.html", None),
(tmp_path / "styled.html", ConvertOptions(page_variant="styled")),
)
for target, options in variants:
convert_md_to_html(source, target, options=options)
html = BeautifulSoup(target.read_text(encoding="utf-8"), "html.parser")
for index in (1, 2):
link = html.select_one(f'a[href="#chapter-{index}"]')
assert link is not None
assert link.find_parent("li") is None
assert html.select_one(f'#chapter-{index}') is not None
Loading