diff --git a/backend/app/core/ask_prompts.py b/backend/app/core/ask_prompts.py index 757daa82..4bff0190 100644 --- a/backend/app/core/ask_prompts.py +++ b/backend/app/core/ask_prompts.py @@ -13,132 +13,27 @@ - Multi-step research planning (Q&A is direct) """ +from app.prompts import load_prompt + from datetime import datetime # ============================================================================= # WORKFLOW INSTRUCTIONS # ============================================================================= -ASK_WORKFLOW_INSTRUCTIONS = """# Ask Tool — Agentic Q&A - -You are a code expert answering questions about a software repository. -Today's date is {date}. - -## Your Goal - -Answer the user's question **accurately and concisely**, citing specific files -and symbols. You have access to tools that search the codebase at different -levels of detail. Use them strategically. - -## Conversation Follow-ups - -If a `## Conversation History` section is present in the user's message, the -current question may be a follow-up to a prior exchange. Use the history to: -- Resolve pronouns and references ("it", "that function", "the same module") -- Avoid re-explaining things already covered -- Focus on what is *new* or *different* in the current question - -Do NOT summarise prior answers — just use them as context. - -## Mandatory Workflow (FOLLOW THIS ORDER) - -1. **Discover** — call `search_symbols` (or `query_graph` for precise filters) - to find relevant classes, functions, and modules. This is cheap (~50 tokens/result). - Call multiple searches in parallel when investigating different aspects. - -2. **Connect** — call `get_relationships` for the 2-3 most relevant symbols to - understand how they relate (callers, callees, inheritance, imports). - In project mode, if `find_cross_repo_links` is available, use it for questions - about cross-repo integration, API contracts, shared DTOs, clients/servers, - FFI/ABI, protobuf/gRPC, GraphQL, BDD, or CLI boundaries. Treat those direct - project links as high-confidence evidence. - -3. **Read selectively** — call `get_code` ONLY for the 2-4 symbols you truly need - to read. This is expensive (~200-2000 tokens). Do NOT dump every search result - into get_code. - -4. **Documentation** — call `search_docs` if the question is about project - setup, configuration, or high-level architecture described in documentation - (README, guides, etc.). Skip this for pure code questions. - -5. **Answer** — synthesize your findings into a clear, concise response. - Reference files by path and symbols by name. - -## Iteration Budget - -You have a maximum of **{max_iterations} tool calls**. Most questions need 3-6. -If you've used 5+ tool calls and haven't found what you need, synthesize what -you have and note the gaps. - -## When to Stop Searching - -Stop and answer when: -- You have code evidence for your claims -- Your last 2 searches returned similar/overlapping results -- You can answer the specific question asked (don't over-research) - -## Citation Style (MANDATORY) - -Reference code **naturally by file path and symbol name**. Do NOT use [N] numeric citations. - -In prose: -- "The `authenticate()` function in `auth/auth_manager.py` handles token validation" -- "Session management is implemented in `services/session.py`" - -In code blocks — always include the file path as a comment header: -```python -# auth/auth_manager.py -def authenticate(token: str) -> bool: - ... -``` - -At the end of your answer, include: ---- -**Files Referenced:** -- `path/to/file.py` — Brief description -""" +ASK_WORKFLOW_INSTRUCTIONS: str = load_prompt("ask/workflow_instructions.md") # ============================================================================= # TOOL USAGE GUIDANCE # ============================================================================= -ASK_TOOL_INSTRUCTIONS = """# Tool Tips - -Each tool has its own description — read it. Below are cost/strategy hints only. - -| Tool | Cost | When to use | -|------|------|-------------| -| `search_symbols` | Cheap | First step — discover classes, functions, modules | -| `find_cross_repo_links` | Cheap | Project mode only — direct API-surface/contracts between repos | -| `get_relationships` | Medium | After finding key symbols — see callers, callees, inheritance | -| `get_code` | **Expensive** | Only 2-4 key symbols — reads full source | -| `search_docs` | Medium | README / config / architecture questions only | -| `query_graph` | Cheap | Precise JQL filters (`type:class file:src/auth/*`) | -| `think` | Free | 1-2 sentences to decide next step | - -Call multiple `search_symbols` in parallel for different aspects. -Do NOT call `get_code` on every search result — be selective. -""" +ASK_TOOL_INSTRUCTIONS: str = load_prompt("ask/tool_instructions.md") # ============================================================================= # OUTPUT FORMAT # ============================================================================= -ASK_OUTPUT_INSTRUCTIONS = """# Output Format - -Your response should be: -1. **Direct** — Answer the question first, then provide supporting details -2. **Structured** — Use headers (##) for sections, bullets for lists, code blocks with file headers -3. **Concise** — Focus on what was asked. Don't explain tangential systems -4. **Evidence-based** — Every claim references a specific file/symbol -5. **Honest** — If context is insufficient, say so clearly - -Do NOT: -- Generate a "Research Report" with executive summaries -- List every file you searched -- Explain your search process -- Use [N] numeric citations -""" +ASK_OUTPUT_INSTRUCTIONS: str = load_prompt("ask/output_instructions.md") # ============================================================================= # COMBINED SYSTEM PROMPT diff --git a/backend/app/core/ask_tool.py b/backend/app/core/ask_tool.py index 3e93b82f..d9b0033e 100644 --- a/backend/app/core/ask_tool.py +++ b/backend/app/core/ask_tool.py @@ -21,6 +21,7 @@ from langchain_core.documents import Document from langchain_core.messages import HumanMessage, SystemMessage +from app.prompts import load_prompt from app.services.llm_factory import is_bedrock_llm, make_cached_system_message from .repository_analysis_store import ( @@ -65,113 +66,14 @@ class AskResponse: # System prompt for query optimization (uses repository context like wiki structure generation) -QUERY_OPTIMIZATION_SYSTEM_PROMPT = """You are a search query optimizer for code repositories. - -Your task is to transform user questions into optimized retrieval queries that will find the most relevant code and documentation in a vector store. - -**QUERY OPTIMIZATION STRATEGY:** -- Identify key technical terms, component names, and concepts from the repository context -- Include specific file names, folder paths, and symbol names when relevant -- Combine topic keywords with implementation-specific terms -- Add related functionality and patterns that may appear in code -- Use both high-level concepts and low-level implementation details - -**OUTPUT FORMAT:** -Return ONLY the optimized search query - a space-separated list of relevant terms. -Do NOT include explanations, quotes, or any other text.""" - -QUERY_OPTIMIZATION_USER_PROMPT = """## Repository Context -{repository_context} - -## User Question -"{question}" - -**TASK:** Generate an optimized retrieval query that will find the most relevant code and documentation. - -**QUERY OPTIMIZATION EXAMPLE:** -For a question about "How does authentication work?": -Generated Query: "authentication login logout session token JWT user validation security middleware auth_manager password hashing access control authorization user_service security_config" - -This query combines topic keywords with file-specific terms and related functionality. - -**Return ONLY the optimized query (space-separated terms), nothing else:**""" - +QUERY_OPTIMIZATION_SYSTEM_PROMPT: str = load_prompt("ask/query_optimization_system.md") +QUERY_OPTIMIZATION_USER_PROMPT: str = load_prompt("ask/query_optimization_user.md") # System prompt for answer generation with natural code citations -ANSWER_SYSTEM_PROMPT = """You are a helpful assistant answering questions about a code repository. - -You have access to the repository's documentation, code, and architecture analysis. -Use the provided context to answer questions accurately and comprehensively. - -## CITATION STYLE (NATURAL FILE REFERENCES) - -**Reference code naturally by file path and symbol name - do NOT use [N] numeric citations.** - -### In Prose: -- "The `authenticate()` function in `auth/auth_manager.py` handles token validation" -- "Session management is implemented via `SessionStore` class in `services/session.py`" -- "The configuration is loaded from `config/settings.py`" - -### In Code Blocks: -Always include the file path as a comment header: -```python -# auth/auth_manager.py -def authenticate(token: str) -> bool: - ... -``` - -### Multiple File References: -- "The authentication flow spans `auth/login.py` → `auth/auth_manager.py` → `services/session.py`" -- "Both `UserModel` in `models/user.py` and `UserService` in `services/user.py` are involved" - -## RESPONSE GUIDELINES - -1. **Be Accurate**: Only state facts supported by the provided context -2. **Be Specific**: Reference actual file paths, function names, and class names naturally in text -3. **Be Structured**: Use headers, bullet points, and code blocks for clarity -4. **Acknowledge Gaps**: If context is insufficient, say so clearly -5. **Show Code**: Include relevant code snippets with file path headers - -## FORMAT - -Use clear markdown: -- Headers (##) for major sections -- Code blocks with language tags and file path comments -- Bullet points for lists -- `backticks` for inline code/file references -- Natural file path mentions (not [N] citations) - -## SOURCES SUMMARY - -At the end of your response, include a brief summary: ---- -**📁 Files Referenced:** -- `path/to/file.py` — Brief description of what was used from this file -""" - - -ANSWER_USER_PROMPT = """## Retrieved Code Context - -{context} - -## Available Sources -{sources_reference} - -## User Question - -{question} - ---- +ANSWER_SYSTEM_PROMPT: str = load_prompt("ask/answer_system.md") -**INSTRUCTIONS:** -1. Answer the question comprehensively based on the code context above -2. Reference files naturally by path (e.g., "in `path/to/file.py`") - do NOT use [N] citations -3. Include relevant code snippets with file path headers as comments -4. Reference specific symbol names (classes, functions, methods) -5. End with a "📁 Files Referenced" summary -6. If the context doesn't contain enough information, state this clearly -**Provide your answer:**""" +ANSWER_USER_PROMPT: str = load_prompt("ask/answer_user.md") class AskTool: diff --git a/backend/app/core/deep_research/research_prompts.py b/backend/app/core/deep_research/research_prompts.py index e47fbf81..29a174df 100644 --- a/backend/app/core/deep_research/research_prompts.py +++ b/backend/app/core/deep_research/research_prompts.py @@ -11,227 +11,34 @@ built-in tools (filesystem, todos). These prompts focus on repo-specific workflow. """ +from app.prompts import load_prompt + from datetime import datetime # ============================================================================= # MAIN RESEARCH WORKFLOW INSTRUCTIONS # ============================================================================= -RESEARCH_WORKFLOW_INSTRUCTIONS = """# Deep Research Workflow - -You are a Deep Research Agent specialized in analyzing software repositories. -Today's date is {date}. - -## Todo Progress Tracking - -Use `write_todos` only when it helps (i.e., the work is genuinely multi-step). -If you create todos, keep them short, actionable, and scoped, and update them as you progress: -- Prefer a single task as "in_progress" (unless you're intentionally doing parallel work) -- Mark tasks as "completed" when done - -## CRITICAL: Context Management for Token Efficiency - -**Keep context lean to avoid token limits.** - -Use filesystem offloading when it helps (large outputs, multi-step investigations, or delegation). -If you offload, remember: `write_file` creates a NEW file and fails if the path already exists; use `edit_file` to update. - -Avoid accumulating many large tool outputs in the conversation context. - -## Your Workflow (FOLLOW THIS ORDER) - -1. **Optional Save Request**: If you expect a long multi-step run, `write_file('/request.md', 'Question: ...')` -2. **Optional Todos**: `write_todos([...])` - Create a minimal set of focused tasks (only if useful) -3. **Research Loop** (for each todo task, if you created any): - a. Update todos to show current task "in_progress" - b. Call appropriate research tool - c. Use `think` to extract key insights (keep brief) - d. If needed, `write_file('/findings/topic_N.md', ...)` - Save key findings for later synthesis - e. Update todos to mark task "completed" -4. **Synthesize**: `ls('/findings/')` then read only what you need -5. **Answer**: Return the full report directly in your final assistant message - -## Filesystem Tools for Context Offloading - -**Writing (use when helpful):** -- `write_file('/findings/search_1.md', content)` - Save a search result snapshot (new file only) -- `write_file('/findings/analysis.md', content)` - Save analysis notes (new file only) -- `write_file('/context/for_subagent.md', content)` - Save context for delegation (new file only) -- If you need to update an existing file, use `edit_file`. - -**Reading (use sparingly, with pagination):** -- `read_file('/findings/search_1.md', offset=0, limit=50)` - Read first 50 lines -- `ls('/findings/')` - List what you've saved -- `grep('pattern', '/findings/*.md')` - Search your findings - -**Directory structure:** -``` -/request.md # Original question -/findings/ # Intermediate research results - search_1.md - search_2.md - relationships.md - overview.md -``` - -## Token-Efficient Patterns - -**DO:** -- Offload large/important tool outputs when you’ll need to reference them later -- Use `think` for brief reflection (2-3 sentences max) -- Read files with pagination (offset/limit) -- Return the complete answer directly in your final assistant message - -**DON'T:** -- Hoard large tool outputs in the conversation -- Write long reflections in `think` calls -- Read entire files without pagination -- Accumulate context across multiple tool calls - -## Working Directory (Agent Scratch Space ONLY) - -Your filesystem tools (`ls`, `glob`, `read_file`, etc.) operate on an -**in-memory scratch space**, NOT on the repository source code. - -## Source File Access (when available) - -If `read_source_file` and `list_repo_files` appear in your tool list, use them to: -- Read full source files: `read_source_file('src/auth/manager.py')` -- Browse directories: `list_repo_files('src/auth')` -- Read config files, Dockerfiles, scripts, etc. that aren't in the search index -- See full context around a snippet returned by `search_codebase` - -**These tools may not always be available.** If they are not in your tool list, -rely on `search_codebase` for all code discovery. - -## Project Cross-Repo Links (when available) - -If `find_cross_repo_links` appears in your tool list, you are in project mode. -Use it for questions about how repositories integrate or depend on each other. -Its results are direct evidence links from API-surface matching: REST endpoints, -DTO/object shapes, FFI/ABI, protobuf/gRPC, GraphQL, BDD steps, CLI commands, -and similar repo-to-repo contracts. Treat these links as high-confidence starting -points, then use `get_relationships` and `get_code` to inspect the exact symbols. -""" +RESEARCH_WORKFLOW_INSTRUCTIONS: str = load_prompt("research/workflow_instructions.md") # ============================================================================= # TOOL USAGE GUIDELINES # ============================================================================= -TOOL_USAGE_INSTRUCTIONS = """# Custom Tool Guidelines - -## `search_codebase` - Semantic Code Search - -**Your primary research tool.** Returns semantically relevant code snippets. - -**Search patterns:** -- Start broad: "authentication system" -- Then narrow: "token validation AuthService" -- Use specific symbols found: "validateToken function" - -**Parallel search example:** -If investigating auth, call these in parallel: -- search_codebase("authentication login") -- search_codebase("session management") -- search_codebase("token validation") - -## `get_symbol_relationships` - Code Graph Analysis - -Use AFTER finding symbols via search to understand connections: -- What calls this function? -- What does this class inherit from? -- What depends on this module? - -## `find_cross_repo_links` - Direct Project Integration Evidence (if available) - -Use this in project mode to find explicit links between repos by API contract, -shared DTO/object shape, FFI/ABI binding, protobuf/gRPC service, GraphQL field, -BDD step, CLI command, or similar surface. Call it with either a broad query or -an exact symbol/node id found by `search_symbols`. - -## `read_source_file` - Direct File Access (if available) - -If `read_source_file` is in your tool list, use it to read raw source files: -- `read_source_file('src/auth/manager.py')` — read full file -- `read_source_file('src/auth/manager.py', offset=50, limit=30)` — read lines 51-80 - -Use for: config files, scripts, full file context, files not in the search index. -**Only use this tool if it appears in your available tools.** - -## `list_repo_files` - Browse Repository (if available) - -If `list_repo_files` is in your tool list, use it to explore the repo: -- `list_repo_files()` — list root directory -- `list_repo_files('src/auth', pattern='*.py')` — list Python files in a directory - -**Only use this tool if it appears in your available tools.** - -## `think` - Strategic Reflection - -**Use after tool calls** to briefly reflect (2-3 sentences max): -- What did I find? -- What's still missing? -- Should I search more or synthesize? -""" +TOOL_USAGE_INSTRUCTIONS: str = load_prompt("research/tool_instructions.md") # ============================================================================= # STOPPING CRITERIA # ============================================================================= -STOPPING_CRITERIA = """# When to Stop Researching - -**Stop searching when:** -- You can answer the user's question comprehensively -- You have 3+ relevant code examples/sources -- Your last 2 searches returned similar/redundant information -- You've hit the search limit (5-8 calls depending on complexity) - -**Write your final report when:** -- You have code evidence for your claims -- You can explain the "why" not just the "what" -""" +STOPPING_CRITERIA: str = load_prompt("research/stopping_criteria.md") # ============================================================================= # OUTPUT FORMAT # ============================================================================= -OUTPUT_FORMAT_INSTRUCTIONS = """# Output Format - -## During Research: Reflect and Offload (When Helpful) - -After each search/analysis tool call: -1. Brief `think` (2-3 sentences max) -2. If the output is large or you’ll need it later, offload key parts to a NEW file under `/findings/` -3. Continue - -## Final Response: Return Report in Chat - -Return the comprehensive report directly in your final assistant message. -Do not write the final report to `/final_report.md` by default. - -```markdown -# Research Report: [Question Summary] - -## Executive Summary -[2-3 sentence answer] - -## Key Findings - -### Finding 1: [Title] -[Explanation with code evidence] -**Code:** `file/path.py:lines` - -### Finding 2: [Title] -... - -## Recommendations -[If applicable] - -## Files Examined -- `/path/file.py` - [what was found] -``` -""" +OUTPUT_FORMAT_INSTRUCTIONS: str = load_prompt("research/output_format.md") # ============================================================================= # MAIN RESEARCH INSTRUCTIONS (COMBINED) diff --git a/backend/app/core/extractors/image.py b/backend/app/core/extractors/image.py index 8e342353..79f23988 100644 --- a/backend/app/core/extractors/image.py +++ b/backend/app/core/extractors/image.py @@ -28,6 +28,7 @@ from typing import TYPE_CHECKING from app.core.extractors.protocol import ExtractedDocument +from app.prompts import load_prompt if TYPE_CHECKING: from langchain_core.language_models import BaseChatModel @@ -35,16 +36,7 @@ logger = logging.getLogger(__name__) -_DESCRIBE_PROMPT = ( - "You are indexing this image for a technical documentation wiki. " - "Describe what the image contains in detail — visible text, diagrams, " - "charts, code snippets, UI elements, architecture, data flow. " - "Include any text you can read verbatim. Aim for a complete textual " - "representation that a search engine can match against natural-" - "language queries. Do not add commentary about the image's quality " - "or style; describe content only. If the image is blank or contains " - "no useful content, respond with exactly: EMPTY_IMAGE." -) +_DESCRIBE_PROMPT: str = load_prompt("extractors/image_describe.md") _MIME_BY_SUFFIX = { diff --git a/backend/app/core/extractors/pdf.py b/backend/app/core/extractors/pdf.py index 9b0f00ba..19c14d6c 100644 --- a/backend/app/core/extractors/pdf.py +++ b/backend/app/core/extractors/pdf.py @@ -29,6 +29,7 @@ from app.core.extractors.image import _extract_text, _token_usage from app.core.extractors.protocol import ExtractedDocument +from app.prompts import load_prompt if TYPE_CHECKING: from langchain_core.language_models import BaseChatModel @@ -36,15 +37,7 @@ logger = logging.getLogger(__name__) -_PER_PAGE_PROMPT = ( - "You are indexing this page of a PDF for a technical documentation " - "wiki. Describe everything on the page in detail — body text " - "verbatim where readable, tables row by row, diagrams, charts, " - "figures, code blocks, footnotes. Preserve the document's logical " - "structure (headings, lists, table rows) using markdown. Do not " - "add commentary about the page's quality or style. If the page is " - "blank, respond with exactly: BLANK_PAGE." -) +_PER_PAGE_PROMPT: str = load_prompt("extractors/pdf_page_describe.md") # pypdfium2 returns pixels per inch via a scale factor; 2.0 gives ~144 DPI diff --git a/backend/app/core/prompts/surgical_edit_prompts.py b/backend/app/core/prompts/surgical_edit_prompts.py index aaae4a8d..38a83e25 100644 --- a/backend/app/core/prompts/surgical_edit_prompts.py +++ b/backend/app/core/prompts/surgical_edit_prompts.py @@ -14,56 +14,10 @@ from __future__ import annotations +from app.prompts import load_prompt -SURGICAL_EDIT_SYSTEM = """You are a documentation editor with one job: -update an existing wiki page so its prose accurately describes the -*newly modified versions* of specific code symbols, while leaving every -other part of the page byte-identical. - -Hard rules: - -1. **Do not change the page title or its section headings.** They feed - stable URL anchors that other pages link to. -2. **Do not rewrite sections that aren't about the changed symbols.** - If a paragraph mentions an unchanged symbol, leave it alone. -3. **Preserve every ```` block exactly,** unless - the path is in the ``moved_paths`` map below (in which case rewrite - the path attribute only; the body is unchanged). -4. **Match the original tone, voice, and approximate length.** A two- - paragraph description should remain a two-paragraph description. -5. **Do not invent new code examples or API references.** If you can't - describe the new symbol behavior from the diff alone, say so in a - single sentence rather than fabricating details. - -Output the full revised page markdown. No commentary, no surrounding -fences, no leading title.""" - - -SURGICAL_EDIT_USER_TEMPLATE = """## Page being edited - -Title: {page_title} -Primary symbol: {primary_symbol_id} - -## Symbols whose source changed in this regen - -{symbol_diffs} - -## File paths that moved (rewrite attributes only) - -{moved_paths} - -## Current page markdown - -The full current body follows between the ```` markers. -Replace ONLY the prose about the changed symbols above. Everything else -must be preserved byte-for-byte. - - -{current_content} - - -Now return the revised page markdown. Start with the first heading and -end with the last line of body; no fences, no explanation.""" +SURGICAL_EDIT_SYSTEM: str = load_prompt("wiki/surgical_edit_system.md") +SURGICAL_EDIT_USER_TEMPLATE: str = load_prompt("wiki/surgical_edit_user.md") #: Per-source cap for the surgical-edit prompt. A typical changed diff --git a/backend/app/core/prompts/wiki_prompts_enhanced.py b/backend/app/core/prompts/wiki_prompts_enhanced.py index 60ce06a2..1dfa9df2 100644 --- a/backend/app/core/prompts/wiki_prompts_enhanced.py +++ b/backend/app/core/prompts/wiki_prompts_enhanced.py @@ -4,6 +4,8 @@ Updated prompts for creating comprehensive, diagram-rich, location-aware documentation. """ +from app.prompts import load_prompt as _load_prompt + # Enhanced Creative Content Generation Prompt with Technical Excellence ENHANCED_CONTENT_GENERATION_PROMPT = """ You are an expert technical writer and architect known for creating exceptionally clear, scannable, and engaging documentation. Your hallmark is **structural variety**; you masterfully break down complex topics to prevent "walls of text" and guide the reader's eye, exercising creative freedom through thoughtful organization. @@ -3853,3 +3855,16 @@ def example_function(): - [ ] Code citations included for all referenced files - [ ] Content serves both technical and non-technical audiences """ + +# ── Override actively-used prompts with MD-file-backed versions ─────────────── +# The inline definitions above are kept for historical reference and any callers +# that import the old variant names. The four constants imported by +# wiki_graph_optimized.py are now loaded from MD files so prompt text has a +# single source of truth. + +ENHANCED_CONTENT_GENERATION_PROMPT_V3_TONE_ADJUSTED = _load_prompt( + "wiki/content_generation_v3_tone.md" +) +STRUCTURED_REPO_ANALYSIS_PROMPT = _load_prompt("wiki/repo_analysis_structured.md") +ENHANCED_REPO_ANALYSIS_PROMPT = _load_prompt("wiki/repo_analysis_enhanced.md") +ENHANCED_WIKI_STRUCTURE_PROMPT = _load_prompt("wiki/wiki_structure.md") diff --git a/backend/app/core/wiki_content_writer/diagram_generator.py b/backend/app/core/wiki_content_writer/diagram_generator.py index 028d23e0..4d172107 100644 --- a/backend/app/core/wiki_content_writer/diagram_generator.py +++ b/backend/app/core/wiki_content_writer/diagram_generator.py @@ -16,7 +16,6 @@ logger = logging.getLogger(__name__) -_MAX_NODES = 15 # Relationship types to include in the diagram (structural + call graph). _INTRA_CLUSTER_REL_TYPES = { "calls", "imports", "references", "creates", @@ -44,14 +43,27 @@ def _sanitize_label(name: str) -> str: return s or _safe_id(name) +def _sanitize_member(text: str) -> str: + """Sanitize a classDiagram member line. + + Preserves ``->`` return-type arrows (only strips bare ``<``/``>`` that + would break Mermaid's parser, not the ``->`` sequence used for return types). + """ + s = text.replace('"', "'") + # Strip < not preceded by - and > not preceded by - (preserves -> arrows). + s = re.sub(r"(?", "", s) + s = re.sub(r"[{}|\\]", "", s) + s = re.sub(r"\s+", " ", s).strip() + return s or text + + def _validate_mermaid(diagram: str) -> tuple[bool, str]: """Lightweight structural validation for a Mermaid diagram block. Returns (is_valid, error_reason). Checks: - Fenced code block with mermaid tag - - Valid graph/flowchart header - - All node-declaration lines use quoted labels - - No obvious unclosed quotes + - Valid diagram header (graph/flowchart, classDiagram, or erDiagram) + - No obvious unclosed double-quotes """ lines = diagram.strip().splitlines() if not lines or lines[0].strip() != "```mermaid": @@ -64,7 +76,12 @@ def _validate_mermaid(diagram: str) -> tuple[bool, str]: return False, "empty diagram body" first = body[0].strip() - if not re.match(r"^(graph|flowchart)\s+(LR|TD|TB|RL|BT)$", first): + if not re.match( + r"^(graph|flowchart)\s+(LR|TD|TB|RL|BT)$" + r"|^classDiagram$|^erDiagram$" + r"|^stateDiagram(-v2)?$", + first, + ): return False, f"invalid header: {first!r}" for line in body[1:]: @@ -96,6 +113,29 @@ def _node_name(node: dict) -> str: "member_uses", "inheritance", "composition", } +# Symbol types considered "class-like" for classDiagram. +_CLASS_LIKE_TYPES = {"class", "interface", "struct", "enum", "trait"} + +# Symbol types considered member-level (rendered inside the class body). +_MEMBER_TYPES = {"method", "function", "property", "field"} + +# Source-text patterns that identify TRUE database/persistence model classes. +# Bug 3 fix: BaseModel and @dataclass are removed — they match Pydantic schemas +# (request/response DTOs) which are NOT ER data models. Only SQLAlchemy ORM +# constructs, Django models, TypeORM entities, GORM structs, and similar +# database-level declarations qualify. +_DATA_MODEL_PATTERNS = ( + "Column(", # SQLAlchemy Column() + "mapped_column(", # SQLAlchemy 2.x mapped_column() + "relationship(", # SQLAlchemy relationship() + "DeclarativeBase", # SQLAlchemy DeclarativeBase + "db.Model", # Flask-SQLAlchemy + "Table(", # SQLAlchemy Table construct + "@Entity", # TypeORM / Java JPA entity + "@Column", # TypeORM / Java JPA column + 'gorm:"', # Go GORM struct tag +) + class DiagramGenerator: def __init__(self, storage: Any, llm_client: Any = None) -> None: @@ -232,6 +272,196 @@ def generate_cluster_diagram(self, cluster_id: int, page_spec: Any) -> str: return f"## Architecture\n\n{explanation}\n\n{mermaid}" return f"## Architecture\n\n{mermaid}" + # ── Class diagram helper (internal) ──────────────────────────────────────── + + def _build_class_diagram(self, cluster_id: int) -> str: + all_nodes = self.storage.get_nodes_by_cluster(cluster_id) + if not all_nodes: + return "" + + # Split into class-like nodes and potential members. + class_nodes = [ + n for n in all_nodes + if (n.get("symbol_type") or n.get("kind", "")) in _CLASS_LIKE_TYPES + ] + if len(class_nodes) < 2: + return "" + + class_node_ids: set[str] = {n["node_id"] for n in class_nodes} + class_name_to_id: dict[str, str] = { + _node_name(n): n["node_id"] for n in class_nodes + } + + # Collect member nodes (methods, properties, fields) whose parent_symbol + # matches one of the selected class nodes. + # Bug 2 fix: DB stores parent_symbol as "module.ClassName"; normalize to + # the short name (after the last dot) before matching against class_names. + class_names: set[str] = set(class_name_to_id) + member_map: dict[str, list[dict]] = {_node_name(n): [] for n in class_nodes} + for n in all_nodes: + if (n.get("symbol_type") or n.get("kind", "")) in _MEMBER_TYPES: + parent = n.get("parent_symbol") + if parent: + parent_short = parent.rsplit(".", 1)[-1] if "." in parent else parent + if parent_short in class_names: + member_map[parent_short].append(n) + + # Collect inheritance edges within the selected class nodes. + inheritance_edges: list[tuple[str, str]] = [] + seen_inh: set[tuple[str, str]] = set() + for n in class_nodes: + nid = n["node_id"] + try: + raw = self.storage.get_edges_from(nid, rel_types=["inheritance"]) + except Exception: + raw = [] + for e in raw: + tgt_id = e.get("target_id", "") + if tgt_id in class_node_ids: + pair = (nid, tgt_id) + if pair not in seen_inh: + seen_inh.add(pair) + inheritance_edges.append(pair) + + id_to_name: dict[str, str] = {n["node_id"]: _node_name(n) for n in class_nodes} + + return _render_class_diagram(class_nodes, member_map, inheritance_edges, id_to_name) + + # ── ER diagram helper (internal) ──────────────────────────────────────────── + + def _build_data_model_diagram(self, cluster_id: int) -> str: + all_nodes = self.storage.get_nodes_by_cluster(cluster_id) + if not all_nodes: + return "" + + # Filter to struct/class nodes that look like data models. + model_nodes = [ + n for n in all_nodes + if (n.get("symbol_type") or n.get("kind", "")) in {"class", "struct"} + and _is_data_model(n) + ] + if len(model_nodes) < 2: + return "" + + model_node_ids: set[str] = {n["node_id"] for n in model_nodes} + + # Collect composition/references edges between model nodes. + rel_edges: list[tuple[str, str]] = [] + seen_rel: set[tuple[str, str]] = set() + for n in model_nodes: + nid = n["node_id"] + try: + raw = self.storage.get_edges_from(nid, rel_types=["composition", "references"]) + except Exception: + raw = [] + for e in raw: + tgt_id = e.get("target_id", "") + if tgt_id in model_node_ids: + pair = (nid, tgt_id) + if pair not in seen_rel: + seen_rel.add(pair) + rel_edges.append(pair) + + id_to_name: dict[str, str] = {n["node_id"]: _node_name(n) for n in model_nodes} + return _render_er_diagram(model_nodes, rel_edges, id_to_name) + + def generate_cluster_context_diagram( + self, + cluster_id: int, + cluster_title: str, + all_cluster_ids: list[int], + cluster_titles: dict[int, str], + ) -> str: + """Generate a Mermaid graph LR showing the cluster as a subgraph. + + The cluster's own architectural nodes are rendered inside a ``subgraph`` + block. When *all_cluster_ids* and *cluster_titles* are provided, + cross-cluster edges are detected and neighbouring clusters are rendered + as plain external nodes. + + Returns a fenced ``graph TD`` block, or ``''`` on empty cluster / + storage error. + """ + if self.storage is None: + return "" + try: + return self._build_cluster_context_diagram( + cluster_id, cluster_title, all_cluster_ids, cluster_titles + ) + except Exception as exc: + logger.debug( + "generate_cluster_context_diagram cluster_id=%d error: %s", cluster_id, exc + ) + return "" + + def _build_cluster_context_diagram( + self, + cluster_id: int, + cluster_title: str, + all_cluster_ids: list[int], + cluster_titles: dict[int, str], + ) -> str: + nodes = self.storage.get_nodes_by_cluster(cluster_id) + if not nodes: + return "" + + nodes = _select_nodes(nodes) + if not nodes: + return "" + + node_ids: set[str] = {n["node_id"] for n in nodes} + id_to_name: dict[str, str] = {n["node_id"]: _node_name(n) for n in nodes} + + # Collect intra-cluster edges. + intra_edges: list[tuple[str, str]] = [] + seen_intra: set[tuple[str, str]] = set() + + # Build a node_id → external cluster_id lookup if cross-cluster info given. + node_to_ext_cluster: dict[str, int] = {} + if all_cluster_ids and cluster_titles: + for ext_cid in all_cluster_ids: + if ext_cid == cluster_id: + continue + try: + ext_nodes = self.storage.get_nodes_by_cluster(ext_cid) + except Exception: + ext_nodes = [] + for en in ext_nodes: + node_to_ext_cluster[en["node_id"]] = ext_cid + + cross_cluster_edges: list[tuple[str, int]] = [] # (src_node_id, tgt_cluster_id) + seen_cross: set[tuple[str, int]] = set() + + for n in nodes: + nid = n["node_id"] + try: + raw = self.storage.get_edges_from(nid, rel_types=list(_CROSS_CLUSTER_REL_TYPES)) + except Exception: + raw = [] + for e in raw: + tgt_id = e.get("target_id", "") + if tgt_id in node_ids: + pair = (nid, tgt_id) + if pair not in seen_intra: + seen_intra.add(pair) + intra_edges.append(pair) + elif tgt_id in node_to_ext_cluster: + ext_cid = node_to_ext_cluster[tgt_id] + cross_pair = (nid, ext_cid) + if cross_pair not in seen_cross: + seen_cross.add(cross_pair) + cross_cluster_edges.append(cross_pair) + + return _render_cluster_context( + cluster_id=cluster_id, + cluster_title=cluster_title, + nodes=nodes, + id_to_name=id_to_name, + intra_edges=intra_edges, + cross_cluster_edges=cross_cluster_edges, + cluster_titles=cluster_titles, + ) + def _fix_mermaid(self, mermaid: str, reason: str) -> str: """Ask the LLM to fix invalid Mermaid syntax. Returns fixed diagram or ''.""" if self._llm is None: @@ -293,6 +523,8 @@ def _build_diagram(self, cluster_id: int) -> str: return "" nodes = _select_nodes(nodes) + if not nodes: + return "" node_ids: set[str] = {n["node_id"] for n in nodes} id_to_name: dict[str, str] = {n["node_id"]: _node_name(n) for n in nodes} @@ -320,14 +552,24 @@ def _build_diagram(self, cluster_id: int) -> str: # ── helpers ──────────────────────────────────────────────────────────────────── +_DOC_TYPES: frozenset[str] = frozenset({ + "config_document", + "markdown_document", + "toml_document", + "module_doc", + "file_doc", + "rst_document", + "html_document", + "asciidoc_document", +}) + + def _select_nodes(nodes: list[dict]) -> list[dict]: - """Filter to ≤ 15 nodes; prefer architectural ones when > 15 candidates.""" - if len(nodes) <= _MAX_NODES: - return nodes - arch = [n for n in nodes if n.get("is_architectural")] - if len(arch) <= _MAX_NODES: - return arch - return arch[:_MAX_NODES] + """Filter document-type nodes from a node list, keeping all code nodes.""" + return [ + n for n in nodes + if (n.get("symbol_type") or n.get("kind", "")) not in _DOC_TYPES + ] def _render(id_to_name: dict[str, str], edges: list[tuple[str, str]]) -> str: @@ -350,6 +592,187 @@ def _render(id_to_name: dict[str, str], edges: list[tuple[str, str]]) -> str: return "\n".join(lines) +def _select_class_nodes(nodes: list[dict]) -> list[dict]: + """Return all class-like nodes; prefer architectural ones when filtering.""" + return nodes + + +def _is_data_model(node: dict) -> bool: + """Return True when a node's source_text looks like a data-model class.""" + source = node.get("source_text") or "" + return any(pattern in source for pattern in _DATA_MODEL_PATTERNS) + + +def _is_abstract(node: dict) -> bool: + """Heuristic: does the class appear to be abstract?""" + src = node.get("source_text") or "" + return "abstract" in src or "ABC" in src or "@abstractmethod" in src + + +def _method_visibility(name: str) -> str: + """Return Mermaid visibility prefix: + public, - private.""" + if name.startswith("_"): + return "-" + return "+" + + +def _method_signature_line(member: dict) -> str: + """Build a Mermaid member line like ``+process(x int) str``.""" + raw_name = _node_name(member) + # Strip leading path segments (keep only the part after the last dot). + display_name = raw_name.rsplit(".", 1)[-1] + visibility = _method_visibility(display_name) + + sig = (member.get("signature") or "").strip() + if sig: + # Strip the leading 'def ' / 'func ' / 'fn ' keyword if present. + sig = re.sub(r"^(def|func|fn|function)\s+", "", sig) + # Strip leading path prefix from the signature name as well. + sig = re.sub(r"^[A-Za-z0-9_.]+\.", "", sig) + return f"{visibility}{_sanitize_member(sig)}" + + params = (member.get("parameters") or "").strip() + ret = (member.get("return_type") or "").strip() + if ret: + return f"{visibility}{_sanitize_member(display_name)}({_sanitize_member(params)}) {_sanitize_member(ret)}" + return f"{visibility}{_sanitize_member(display_name)}({_sanitize_member(params)})" + + +def _render_class_diagram( + class_nodes: list[dict], + member_map: dict[str, list[dict]], + inheritance_edges: list[tuple[str, str]], + id_to_name: dict[str, str], +) -> str: + """Build a Mermaid classDiagram block.""" + lines: list[str] = ["```mermaid", "classDiagram"] + + for n in class_nodes: + cname = _node_name(n) + sym_type = n.get("symbol_type") or n.get("kind", "class") + lines.append(f" class {_safe_id(cname)} {{") + # Stereotype annotation. + if sym_type == "interface": + lines.append(f" <>") + elif sym_type == "enum": + lines.append(f" <>") + elif _is_abstract(n): + lines.append(f" <>") + # Member lines. + for member in member_map.get(cname, []): + lines.append(f" {_method_signature_line(member)}") + lines.append(" }") + + # Inheritance edges: Child --|> Parent + for src_id, tgt_id in inheritance_edges: + src_name = _safe_id(id_to_name[src_id]) + tgt_name = _safe_id(id_to_name[tgt_id]) + lines.append(f" {src_name} --|> {tgt_name} : implements") + + lines.append("```") + return "\n".join(lines) + + +def _render_er_diagram( + model_nodes: list[dict], + rel_edges: list[tuple[str, str]], + id_to_name: dict[str, str], +) -> str: + """Build a Mermaid erDiagram block.""" + lines: list[str] = ["```mermaid", "erDiagram"] + + for n in model_nodes: + entity_name = _safe_id(_node_name(n)) + lines.append(f" {entity_name} {{") + lines.append(" }") + + for src_id, tgt_id in rel_edges: + src = _safe_id(id_to_name[src_id]) + tgt = _safe_id(id_to_name[tgt_id]) + lines.append(f' {src} ||--o{{ {tgt} : "has"') + + lines.append("```") + return "\n".join(lines) + + +def _render_cluster_context( + cluster_id: int, + cluster_title: str, + nodes: list[dict], + id_to_name: dict[str, str], + intra_edges: list[tuple[str, str]], + cross_cluster_edges: list[tuple[str, int]], + cluster_titles: dict[int, str], +) -> str: + """Build a Mermaid graph TD with a subgraph for the cluster. + + Only nodes that participate in at least one edge are rendered — isolated + nodes add visual clutter without conveying relationships. Returns '' when + fewer than 2 connected nodes exist (nothing useful to show). + """ + # Collect node IDs that appear in at least one edge. + connected: set[str] = set() + for src_id, tgt_id in intra_edges: + connected.add(src_id) + connected.add(tgt_id) + for src_nid, _ in cross_cluster_edges: + connected.add(src_nid) + + # Filter to connected nodes only; bail if fewer than 2. + connected_nodes = [n for n in nodes if n["node_id"] in connected] + if len(connected_nodes) < 2 and not cross_cluster_edges: + return "" + + lines: list[str] = ["```mermaid", "graph TD"] + + safe_title = _sanitize_label(cluster_title) + lines.append(f' subgraph "{safe_title}"') + for n in connected_nodes: + name = _node_name(n) + safe = _safe_id(name) + label = _sanitize_label(name) + lines.append(f' {safe}["{label}"]') + lines.append(" end") + + # External cluster nodes (one node per neighbouring cluster). + # Build a stable cid → safe_id map first to ensure edges reference the + # same de-collided ID used in the node declaration. + internal_safe_ids: set[str] = {_safe_id(_node_name(n)) for n in nodes} + ext_cid_to_safe: dict[int, str] = {} + seen_ext_safe: set[str] = set(internal_safe_ids) + seen_ext: set[int] = set() + for _src_nid, ext_cid in cross_cluster_edges: + if ext_cid not in seen_ext: + seen_ext.add(ext_cid) + ext_title = cluster_titles.get(ext_cid) or f"cluster_{ext_cid}" + ext_safe = _safe_id(ext_title[:_MAX_LABEL_CHARS]) + # Guarantee uniqueness against internal nodes and other external nodes. + if ext_safe in seen_ext_safe: + ext_safe = f"{ext_safe}_{ext_cid}" + seen_ext_safe.add(ext_safe) + ext_cid_to_safe[ext_cid] = ext_safe + ext_label = _sanitize_label(ext_title[:_MAX_LABEL_CHARS]) + lines.append(f' {ext_safe}["{ext_label}"]') + + # Intra-cluster edges. + for src_id, tgt_id in intra_edges: + src = _safe_id(id_to_name[src_id]) + tgt = _safe_id(id_to_name[tgt_id]) + lines.append(f" {src} --> {tgt}") + + # Cross-cluster edges: source node → external cluster node. + for src_nid, ext_cid in cross_cluster_edges: + src_name = id_to_name.get(src_nid) + ext_safe = ext_cid_to_safe.get(ext_cid) + if src_name is None or ext_safe is None: + continue + src_safe = _safe_id(src_name) + lines.append(f" {src_safe} --> {ext_safe}") + + lines.append("```") + return "\n".join(lines) + + def _render_repo_clusters( cluster_ids: list[int], cluster_titles: dict[int, str], diff --git a/backend/app/core/wiki_content_writer/page_formatter.py b/backend/app/core/wiki_content_writer/page_formatter.py index ade124c4..0721fb5e 100644 --- a/backend/app/core/wiki_content_writer/page_formatter.py +++ b/backend/app/core/wiki_content_writer/page_formatter.py @@ -42,6 +42,8 @@ import re from typing import Any +from app.prompts import load_prompt + logger = logging.getLogger(__name__) # ── Citation token patterns ─────────────────────────────────────────────────── @@ -73,23 +75,7 @@ # ── LLM formatting prompt ───────────────────────────────────────────────────── -_FORMAT_PROMPT = """\ -You are reorganizing a wiki page for human readability. Output ONLY the reorganized Markdown. - -RULES: -1. Do NOT add new facts. -2. Preserve every [^N] footnote ref exactly as written. -3. Copy the ## References block exactly. -4. Output only Markdown — no explanations or commentary. -5. Do NOT emit [path:line] or [path:lo-hi] raw citation tokens — use only the existing [^N] refs. - -Produce: -- A brief 1-2 sentence Overview paragraph at the top. -- 2-5 ## section headings grouping related paragraphs. -- Smooth prose connections within sections. - -INPUT: -{content}""" +_FORMAT_PROMPT = load_prompt("wiki/page_format.md") class PageFormatter: diff --git a/backend/app/core/wiki_content_writer/repo_explorer.py b/backend/app/core/wiki_content_writer/repo_explorer.py index 480e50c1..b2e8b6a1 100644 --- a/backend/app/core/wiki_content_writer/repo_explorer.py +++ b/backend/app/core/wiki_content_writer/repo_explorer.py @@ -14,57 +14,15 @@ from langchain_core.messages import HumanMessage, ToolMessage from app.core.agent_tools import WIKI_TOOL_SCHEMAS +from app.prompts import load_prompt from app.services.llm_factory import is_bedrock_llm, make_cached_system_message, safe_bind_tools from .writer_tools import WriterTools logger = logging.getLogger(__name__) -_SYSTEM_PROMPT_TEMPLATE = """\ -You are a senior software architect doing a first-day deep dive on a new codebase. -Your goal: produce a comprehensive architectural analysis document. - -You have tools to read the actual files. Use them systematically: -1. Read entry points (main.py, app.py, __init__.py, run.py) -2. Follow imports to understand the application structure -3. Read router/controller files to understand the API surface -4. Read main service/domain files to understand business logic -5. Read model/schema files to understand data contracts -6. Read config files to understand external dependencies - -REQUIRED OUTPUT STRUCTURE (fill all sections): -# Repository Analysis: {repo_name} - -## Purpose -What does this software do? What problem does it solve? - -## Target Users & Use Cases -Who uses this? How? - -## Architecture -Key components and their responsibilities. - -## Main Data Flows -Step-by-step description of 2-3 key flows. - -## Technology Stack -Languages, frameworks, databases, external services. - -## Key Design Decisions -Notable patterns, architectural choices, trade-offs. - -## Integration Points -APIs exposed, external services consumed. - -Use your tool budget ({budget} calls) wisely. Start broad (entry points), -go deep on the most important files. -""" - -_BUDGET_EXHAUSTED_PROMPT = ( - "You have used all available tool calls. Based on the files you explored, " - "write the complete architectural analysis now following the REQUIRED OUTPUT STRUCTURE. " - "Include all seven sections. Do NOT call any more tools." -) +_SYSTEM_PROMPT_TEMPLATE = load_prompt("repo/explorer_system.md") +_BUDGET_EXHAUSTED_PROMPT = load_prompt("repo/explorer_budget_exhausted.md") class RepoExplorer: diff --git a/backend/app/core/wiki_content_writer/repo_overview.py b/backend/app/core/wiki_content_writer/repo_overview.py index 1d44227d..cc484823 100644 --- a/backend/app/core/wiki_content_writer/repo_overview.py +++ b/backend/app/core/wiki_content_writer/repo_overview.py @@ -19,6 +19,8 @@ if TYPE_CHECKING: from langchain_core.language_models import BaseChatModel +from app.prompts import load_prompt + logger = logging.getLogger(__name__) # Maximum number of pages whose markdown is read in Step 1 @@ -27,40 +29,7 @@ # Maximum characters of markdown per page (context budget) _MAX_CHARS_PER_PAGE = 2000 -_OVERVIEW_SYSTEM_PROMPT = """\ -You are a technical writer creating a concise, engaging repository overview page for a software wiki. - -You will be given: -1. An architectural analysis of the codebase (background context). -2. Excerpts from up to {max_pages} generated wiki pages (the actual content). - -Your task: synthesise this into a well-structured overview page with ALL of the following sections: - -## Purpose -What does this software do? What problem does it solve? (2-3 sentences) - -## How It Works -Step-by-step narrative of the main data flow or request lifecycle. (3-5 sentences) - -## Key Capabilities -Bullet list of 5-8 key features or capabilities. - -## Architecture -Brief description of the main components and how they relate. (3-5 sentences) - -## Getting Started -Pointers to the most important wiki pages for a new engineer. -Use [[Page Title]] wikilinks (e.g. [[Authentication]], [[API Reference]]). - -## Wiki Pages -List every page as a wikilink bullet with a one-sentence description. -Format: - [[Page Title]] — description - -Rules: -- Do NOT fabricate claims about the codebase. -- Be concise. Aim for 500-800 words total. -- Output raw markdown only — no code fences, no preamble. -""" +_OVERVIEW_SYSTEM_PROMPT_TEMPLATE = load_prompt("repo/overview_system.md") def _get_page_attr(page: Any, attr: str, default: str = "") -> str: @@ -173,7 +142,7 @@ def generate_repo_overview( for p in pages ) - system_prompt = _OVERVIEW_SYSTEM_PROMPT.format(max_pages=_MAX_PAGES_TO_READ) + system_prompt = _OVERVIEW_SYSTEM_PROMPT_TEMPLATE.format(max_pages=_MAX_PAGES_TO_READ) # Prepend the architectural analysis to the system prompt for caching. if repo_analysis: diff --git a/backend/app/core/wiki_content_writer/writer_agent.py b/backend/app/core/wiki_content_writer/writer_agent.py index 6e0758e6..66ce79c6 100644 --- a/backend/app/core/wiki_content_writer/writer_agent.py +++ b/backend/app/core/wiki_content_writer/writer_agent.py @@ -37,6 +37,9 @@ from app.core.agent_tools import WIKI_TOOL_SCHEMAS as _WRITER_TOOL_SCHEMAS from app.services.llm_factory import is_bedrock_llm, make_cached_system_message, safe_bind_tools +from app.prompts import load_prompt + +from .diagram_generator import DiagramGenerator from .prompts import CITATION_FORMAT_DESCRIPTION, CITATION_PROMPT_RULES from .source_gate import ToolCall from .writer_tools import WriterTools @@ -118,142 +121,30 @@ def compute_page_budget( # System prompt prefix — matches the role-detection prefix in integration tests. _SYSTEM_PROMPT_PREFIX = "You are a technical documentation writer." -_SYSTEM_PROMPT_TEMPLATE = """{prefix} - -Your job is to write a detailed, accurate wiki page in Markdown based on the -page specification and your tool results. - -## Citation contract - -{format_description} - -Rules: -{rules} - -### Why citations matter - -The citation verifier checks every paragraph against the actual source code. -A paragraph with no citation is treated as ungrounded prose and is discarded -entirely — it will not appear in the final wiki page. A citation pointing to a -line range that does not exist in the repository is also discarded. Only -paragraphs with citations that match real, readable code survive into the -final output. This means: if you write a sentence without a citation, it will -be silently removed. If you cite a line range that you did not actually read -with read_file, the verifier may reject it. The only safe approach is to call -read_file, observe the line numbers in the returned content, and then cite -those exact lines. Never guess a line number. Never cite a file you did not -open with read_file or confirm with get_signature. - -### Citation rule details - -Rule 1 — Every claim needs a citation. The consequence of omitting a citation -is that the whole paragraph is stripped. Write claims in the form: -"The scheduler uses a one-second poll interval [src/scheduler.py:88]." -Append the citation token immediately after the claim, before the sentence's -closing punctuation. Do not move it to the end of the paragraph if the -paragraph covers multiple files — cite each claim individually. - -Rule 2 — README content must be verified. If you intend to reference something -described in a README, you must call read_file on the README first and cite -the specific lines. Paraphrasing README text without a matching citation is -treated as an uncited claim and will be stripped. - -Rule 3 — Identifiers must appear in your tool trace. Do not name a function, -class, or environment variable that did not appear in the output of read_file, -get_signature, get_callers, get_callees, grep, or list_doc_chunks. If you are -uncertain whether a symbol exists, call get_signature or grep before writing -about it. Mentioning a non-existent symbol causes the paragraph to be flagged -and may cause the entire page to fail verification. - -## Available tools - -**read_file(path, start_line?, end_line?)** — Read lines from a repo-relative file. -Returns: numbered lines in the form `: ` (one per line). - Example: `47: class JobQueue:\\n48: def __init__(self, max_size: int = 1000):` -Use: read files containing the symbols in target_symbols. The line numbers -in the output are the exact numbers to use in citations: read line 47 → cite -[path:47] or [path:47-48] for a range. -Error: `[error] ...` — try a different path or call get_signature first. - -**get_signature(symbol)** — Look up a symbol's definition location and signature. -Returns: `file_path: signature (layer)\\ndocstring` - Example: `src/workers/queue.py: class JobQueue (infrastructure)\\nAsyncIO job queue.` -Use: confirm a class or function exists and find its file path before calling -read_file. Does NOT return line numbers — use read_file after get_signature -to get the exact lines for citations. -Not found: `[not found] symbol` — do not mention this symbol in the page. - -**grep(pattern)** — Full-text search across source files. -Returns: `file_path:line_number: line_text` (up to 20 matches) -Use: find where a concept or identifier is used across the codebase. Use grep -to locate configuration values, string literals, or identifiers you know exist -but whose file path you are unsure of. -No matches: `[no matches]` — the pattern does not appear in source files. - -**get_callers(symbol)** — List symbols that call the given symbol. -Returns: `file_path: symbol_name` per caller -Use: document who uses a function or class (integration context). Useful for -writing the "used by" section of a public API page. Shows the call graph from -the perspective of dependents. - -**get_callees(symbol)** — List symbols the given symbol calls. -Returns: `file_path: symbol_name` per callee -Use: document what a function or class depends on (dependency chain). Useful -for showing what a component orchestrates or delegates to. - -**list_doc_chunks(doc_path)** — Return documentation sections for a doc file. -Returns: `[N] heading\\ntext` per chunk -Use: incorporate doc context into pages that cover documented modules. Prefer -this over read_file when you need the logical structure of a markdown document -rather than raw line content. - -## Citation examples - -Good — claim immediately followed by a bracketed citation token: - "The worker polls with a 1-second timeout [src/workers/worker.py:62-63]." - "Default queue size is 1000 items [src/workers/queue.py:57]." - "Startup recovery loads pending jobs [src/workers/queue.py:121-133]." - "Config is read from environment [config.py:5-12] and merged with file - defaults [config.py:40-48]." - -Bad — NEVER write these (they are discarded by the verifier): - "The worker polls with a 1-second timeout." ← no citation, discarded - "See worker.py for the timeout logic." ← path in prose not in brackets, discarded - "The worker polls (worker.py:62)." ← parens not brackets, citation not parsed - "The queue has configurable size [queue.py:57]." ← missing src/ prefix, - may not resolve to a real path — always use the exact path from read_file - "The scheduler module handles retries." ← whole paragraph with no citation, - discarded in full regardless of how accurate the claim is - -## Budget guidance - -You have a finite number of tool calls available for this page. Use them to -read the files and symbols listed in the page spec before writing. Prioritise -reading symbols from target_symbols first, then explore related callers or -callees if the budget allows. Once you have read enough to ground every claim, -write the complete page. Do not call tools after you have started writing. - -IMPORTANT: every substantive paragraph MUST end with at least one citation in -the format shown above. A paragraph with no citation will be stripped by the -verifier. Start by calling tools to read the relevant files, then write the -page with inline citations after every claim. -""" - -_USER_PROMPT_TEMPLATE = """\ -## Page to write +_SYSTEM_PROMPT_TEMPLATE = load_prompt("writer/system.md") +_USER_PROMPT_TEMPLATE = load_prompt("writer/user_template.md") -**Title:** {title} -**Description:** {description} -**Retrieval query:** {retrieval_query} -**Target symbols:** {symbols} -**Target folders:** {folders} -**Target docs:** {docs} +def _extract_llm_text(response: Any) -> str: + """Extract plain text from an LLM response regardless of provider format. -First, read the most relevant files from target symbols / folders using the -available tools. Then write the complete wiki page in Markdown with inline -`[path:N]` or `[path:lo-hi]` citations after every claim. -""" + Anthropic/Bedrock returns ``response.content`` as a list of content blocks + (``[{'type': 'text', 'text': '...'}, {'type': 'tool_use', ...}]``). + Other providers return a plain string. This helper normalises both. + """ + content = response.content if hasattr(response, "content") else str(response) + if isinstance(content, str): + return content + if isinstance(content, list): + parts: list[str] = [] + for block in content: + if isinstance(block, dict): + if block.get("type") == "text": + parts.append(block.get("text", "")) + elif hasattr(block, "type") and getattr(block, "type", None) == "text": + parts.append(getattr(block, "text", "")) + return "\n".join(parts) + return str(content) class WikiContentWriter: @@ -465,17 +356,10 @@ def generate_page( # Budget exhausted without a final write — explicitly instruct the # LLM to write the page and cite every claim using the file paths # and line numbers already in the message history (#313). - messages.append(HumanMessage(content=( - "You have used all available tool calls. Based on the evidence " - "you gathered, write the complete wiki page in Markdown now.\n\n" - "IMPORTANT: cite every claim using the file path and line numbers " - "from your tool results. Format: [path/to/file.py:N] or " - "[path/to/file.py:lo-hi]. A paragraph without a citation will be " - "discarded. Do NOT call any more tools." - ))) + messages.append(HumanMessage(content=load_prompt("writer/budget_exhausted.md"))) try: response = effective_llm.invoke(messages) - final_content = response.content if isinstance(response.content, str) else str(response.content) + final_content = _extract_llm_text(response) # If LLM still returned tool_calls (stubborn), ignore and fall back. final_tool_calls = getattr(response, "tool_calls", None) or [] if final_content and not final_tool_calls: @@ -670,13 +554,7 @@ def _generate_chapter_index( if tool_calls_made >= self._CHAPTER_INDEX_BUDGET: # Budget exhausted — force final write. - messages.append(HumanMessage(content=( - "You have used all available tool calls. Based on the " - "evidence you gathered, write the complete chapter index " - "in Markdown now. Follow the structure exactly: " - "## Overview, ## Key Components, ## Sub-pages. " - "Do NOT call any more tools." - ))) + messages.append(HumanMessage(content=load_prompt("writer/chapter_index_budget_exhausted.md"))) response = llm_with_tools.invoke(messages) final_tc = getattr(response, "tool_calls", None) or [] if not final_tc: @@ -833,6 +711,25 @@ async def _generate_one(sp: "SubPageSpec", sl: "EvidenceSlice") -> str: ), ) + # ── 4. Inject cluster diagrams into the chapter index ───────── + # Collect file paths from evidence so _inject can bypass the + # skeleton cluster_id → DB macro_cluster mismatch. + _evidence_paths: list[str] = [] + for _sl in evidence_slices: + _evidence_paths.extend(getattr(_sl, "file_paths", [])) + + # Run in executor: _inject_cluster_diagrams does sync disk IO and LLM + # calls that would block the event loop if called directly. + chapter_index_md = await loop.run_in_executor( + None, + lambda: self._inject_cluster_diagrams( + chapter_index_md, + chapter_spec.cluster_id, + chapter_spec.chapter_title, + file_hints=_evidence_paths, + ), + ) + logger.info( "[WRITER] generate_chapter done: title=%r subpages=%d index_chars=%d", chapter_spec.chapter_title, @@ -841,6 +738,494 @@ async def _generate_one(sp: "SubPageSpec", sl: "EvidenceSlice") -> str: ) return chapter_index_md, subpage_mds + def _inject_cluster_diagrams( + self, + chapter_index_md: str, + cluster_id: int, + chapter_title: str, + file_hints: "list[str] | None" = None, + ) -> str: + """Inject Mermaid diagrams into a chapter _index.md using agentic file reading. + + Flow: + 1. Deterministic architecture diagram from graph (no LLM needed). + 2. LLM-based class diagram from actual source files. + 3. LLM-based data model diagram from actual source files. + + *file_hints* are relative paths from evidence slices — used to resolve + the correct DB ``macro_cluster`` when skeleton cluster_ids diverge from + stored ``macro_cluster`` values (they use different numbering systems). + + Sections are injected before ``## Key Components`` (or ``## Sub-pages`` + when Key Components is absent). Returns original markdown unchanged + when no diagrams are generated or on any error. + """ + storage = self.tools.storage + if storage is None: + return chapter_index_md + + try: + effective_llm = self.llm_client + sections: list[str] = [] + + db_cluster_id = self._resolve_db_cluster(cluster_id, storage, file_hints) + logger.debug("[DIAGRAMS] cluster_id=%s → db_cluster_id=%s title=%r", cluster_id, db_cluster_id, chapter_title) + + # All three diagrams are generated by an agentic loop: the LLM + # sees the available files and reads whichever it needs before + # outputting each diagram type. + if effective_llm is not None: + available_files = self._list_cluster_files( + db_cluster_id, storage, file_hints=file_hints + ) + if available_files: + arch = self._agentic_diagram( + available_files, chapter_title, "architecture" + ) + if arch: + sections.append(f"## Architecture\n\n{arch}") + + class_diag = self._agentic_diagram( + available_files, chapter_title, "class" + ) + if class_diag: + sections.append(f"## Class Structure\n\n{class_diag}") + + data_diag = self._agentic_diagram( + available_files, chapter_title, "datamodel" + ) + if data_diag: + sections.append(f"## Data Model\n\n{data_diag}") + + if not sections: + return chapter_index_md + + injected = "\n\n".join(sections) + for marker in ("## Key Components", "## Sub-pages"): + pos = chapter_index_md.find(marker) + if pos >= 0: + return ( + chapter_index_md[:pos].rstrip() + + f"\n\n{injected}\n\n" + + chapter_index_md[pos:] + ) + return chapter_index_md.rstrip() + f"\n\n{injected}" + + except Exception as exc: + logger.debug("[WRITER] _inject_cluster_diagrams failed: %s", exc) + return chapter_index_md + + def _resolve_db_cluster( + self, + cluster_id: int, + storage: Any, + file_hints: "list[str] | None", + ) -> int: + """Return the DB macro_cluster value that corresponds to this chapter. + + Skeleton cluster_ids (assigned 1-based by enumerate in structure_skeleton.py) + differ from the 0-based macro_cluster values stored in the DB by the + Leiden clustering step. Probe using file_hints paths — the first node + found for any hint path gives the true macro_cluster. Falls back to + cluster_id unchanged when probing fails or hints are absent. + """ + if not file_hints or storage is None: + return cluster_id + try: + from collections import Counter # noqa: PLC0415 + hint_set = set(file_hints[:8]) + votes: Counter = Counter() + # Query by directory but only count nodes whose rel_path is one of + # the exact hint paths — avoids sibling subdirectory contamination. + seen_dirs: set[str] = set() + for path in file_hints[:8]: + dir_prefix = path.rsplit("/", 1)[0] if "/" in path else path + if dir_prefix in seen_dirs: + continue + seen_dirs.add(dir_prefix) + try: + nodes = storage.get_nodes_by_path_prefix(dir_prefix, limit=200) + except Exception: + continue + for n in nodes: + if n.get("rel_path") in hint_set: + mc = n.get("macro_cluster") + if mc is not None: + votes[int(mc)] += 1 + if votes: + return votes.most_common(1)[0][0] + except Exception as exc: + logger.debug("[DIAGRAMS] _resolve_db_cluster failed: %s", exc) + return cluster_id + + def _gather_cluster_source( + self, + cluster_id: int, + storage: Any, + max_files: int = 5, + chars_per_file: int = 3000, + file_hints: "list[str] | None" = None, + ) -> str: + """Read source files for cluster_id; return concatenated content for LLM context. + + Prefers *file_hints* (paths from evidence slices) over a DB cluster + lookup, since skeleton cluster_ids don't always match DB macro_cluster + values. Falls back to cluster node query when hints are absent or + produce no readable files. + """ + _DOC_TYPES = { + "config_document", "markdown_document", "toml_document", + "module_doc", "file_doc", "rst_document", "html_document", + "asciidoc_document", + } + + def _read_paths(paths: list[str]) -> list[str]: + parts: list[str] = [] + for path in paths[:max_files]: + try: + fc = self.tools.read_file(path) + if fc.error or not fc.lines: + continue + content = "\n".join(fc.lines)[:chars_per_file] + parts.append(f"=== {path} ===\n{content}") + except Exception: + continue + return parts + + # Primary: use evidence file paths directly (avoids cluster_id mismatch). + if file_hints: + unique_hints = list(dict.fromkeys(p for p in file_hints if p)) + parts = _read_paths(unique_hints) + if parts: + return "\n\n".join(parts) + + # Fallback: query DB by cluster_id. + try: + nodes = storage.get_nodes_by_cluster(cluster_id) if storage else [] + except Exception: + nodes = [] + + code_nodes = [n for n in nodes if n.get("symbol_type") not in _DOC_TYPES] + path_scores: dict[str, int] = {} + for n in code_nodes: + p = n.get("rel_path", "") + if p: + path_scores[p] = path_scores.get(p, 0) + (1 if n.get("is_architectural") else 0) + + sorted_paths = sorted(path_scores, key=lambda p: path_scores[p], reverse=True) + return "\n\n".join(_read_paths(sorted_paths)) + + def _list_cluster_files( + self, + db_cluster_id: int, + storage: Any, + file_hints: "list[str] | None" = None, + ) -> list[str]: + """Return deduplicated source file paths for this cluster. + + Prefers file_hints (from evidence slices) because they represent + exactly what the planner considered relevant. Falls back to querying + the DB by macro_cluster and scoring by architectural node count. + """ + _DOC_TYPES = { + "config_document", "markdown_document", "toml_document", + "module_doc", "file_doc", "rst_document", "html_document", + "asciidoc_document", + } + + # Primary: evidence paths are already the right files. + if file_hints: + unique = list(dict.fromkeys(p for p in file_hints if p)) + if unique: + return unique + + # Fallback: query DB. + try: + nodes = storage.get_nodes_by_cluster(db_cluster_id) if storage else [] + except Exception: + return [] + code_nodes = [n for n in nodes if n.get("symbol_type") not in _DOC_TYPES] + scores: dict[str, int] = {} + for n in code_nodes: + p = n.get("rel_path", "") + if p: + scores[p] = scores.get(p, 0) + (1 if n.get("is_architectural") else 0) + return sorted(scores, key=lambda p: scores[p], reverse=True) + + _DIAGRAM_BUDGET = 6 # max tool calls the LLM may make to read files + + _DIAGRAM_PROMPT_FILES: dict[str, str] = { + "architecture": "diagrams/architecture.md", + "class": "diagrams/class_structure.md", + "datamodel": "diagrams/data_model.md", + } + + _DIAGRAM_EXPECTED_HEADER: dict[str, tuple[str, ...]] = { + "architecture": ("stateDiagram",), # stateDiagram-v2 for functional flows + "class": ("classDiagram",), + "datamodel": ("erDiagram",), + } + + def _agentic_diagram( + self, + available_files: list[str], + chapter_title: str, + diagram_type: str, + ) -> str: + """Run a tool-augmented loop to generate one Mermaid diagram. + + The LLM receives the list of available source files and calls + ``read_file`` on whichever it needs (budget: _DIAGRAM_BUDGET calls). + After reading it outputs the diagram. Returns '' on failure/EMPTY. + """ + if self.llm_client is None: + return "" + + from langchain_core.messages import ( # noqa: PLC0415 + HumanMessage as _HM, SystemMessage as _SM, ToolMessage as _TM, + ) + import re as _re # noqa: PLC0415 + from .diagram_generator import _validate_mermaid # noqa: PLC0415 + + from app.prompts import load_prompt # noqa: PLC0415 + prompt_file = self._DIAGRAM_PROMPT_FILES.get(diagram_type, "") + system_prompt = load_prompt(prompt_file) if prompt_file else "" + expected_headers = self._DIAGRAM_EXPECTED_HEADER.get(diagram_type, ()) + + file_list = "\n".join(f"- {p}" for p in available_files) + user_msg = ( + f"Chapter: {chapter_title}\n\n" + f"Available source files:\n{file_list}\n\n" + "Read the files you need and then output the diagram." + ) + + try: + llm_with_tools = safe_bind_tools(self.llm_client, _WRITER_TOOL_SCHEMAS) + messages: list[Any] = [ + _SM(content=system_prompt), + _HM(content=user_msg), + ] + + for _round in range(self._DIAGRAM_BUDGET + 1): + response = llm_with_tools.invoke(messages) + tool_calls = getattr(response, "tool_calls", None) or [] + + if not tool_calls: + # No more tool calls — extract the diagram from the response. + text = _extract_llm_text(response) + if not text or text.strip() == "EMPTY": + return "" + m = _re.search(r"```mermaid\n.*?```", text, _re.DOTALL) + if not m: + return "" + candidate = m.group(0) + ok, _ = _validate_mermaid(candidate) + if not ok: + return "" + second_line = candidate.splitlines()[1] if len(candidate.splitlines()) > 1 else "" + if not any(h in second_line for h in expected_headers): + return "" + return candidate + + if _round >= self._DIAGRAM_BUDGET: + # Budget exhausted — force output now. + messages.append(response) + messages.append(_HM(content=( + "Budget reached. Output the diagram now based on what you have read. " + "Do not call any more tools." + ))) + continue + + # Execute tool calls. + messages.append(response) + for tc in tool_calls: + tc_name = tc.get("name", "") if isinstance(tc, dict) else getattr(tc, "name", "") + tc_args = tc.get("args", {}) if isinstance(tc, dict) else getattr(tc, "args", {}) + tc_id = tc.get("id", str(_round)) if isinstance(tc, dict) else getattr(tc, "id", str(_round)) + if isinstance(tc_args, str): + try: + import json as _json; tc_args = _json.loads(tc_args) # noqa: PLC0415,E702 + except Exception: + tc_args = {} + result = self._dispatch_tool(tc_name, tc_args) + messages.append(_TM(content=result, tool_call_id=tc_id, name=tc_name)) + + except Exception as exc: + logger.debug("[WRITER] _agentic_diagram(%s) failed: %s", diagram_type, exc) + + return "" + + def _diagram_llm_invoke(self, prompt: str) -> str: + """Invoke the LLM for diagram generation with tool calls suppressed. + + The writer's llm_client has WIKI_TOOL_SCHEMAS pre-bound so the model + tends to call read_file instead of generating. This helper: + 1. Tries to bind tool_choice='none' (Anthropic/OpenAI) to stop tool calls. + 2. Prepends a SystemMessage to reinforce the no-tools constraint. + 3. Falls back to a plain invoke if binding fails. + """ + from langchain_core.messages import HumanMessage as _HM, SystemMessage as _SM # noqa: PLC0415 + + system_msg = _SM(content=( + "You are a diagram generator. Output ONLY the requested Mermaid code block. " + "Do not call any tools or functions. Do not request to read any files. " + "Use only the source code provided in the user message." + )) + user_msg = _HM(content=prompt) + + # Try to suppress tool calls at the API level. + llm = self.llm_client + try: + llm = llm.bind(tool_choice="none") + except Exception: + pass + + response = llm.invoke([system_msg, user_msg]) + return _extract_llm_text(response) + + def _llm_architecture_diagram(self, source_ctx: str, chapter_title: str) -> str: + """Generate a Mermaid flowchart showing the architectural structure of this cluster. + + The LLM reads the source code and identifies meaningful components + (services, handlers, repositories, clients, queues — not individual + data fields or response model classes) and their relationships. + Returns a validated fenced graph TD block, or '' on failure / empty output. + """ + if self.llm_client is None: + return "" + + prompt = ( + "The source code below is complete. Read it directly — do NOT call any tools " + "or request additional files. Generate a Mermaid architecture diagram.\n\n" + "Rules:\n" + "- Show 4-8 meaningful architectural components: services, handlers, " + "repositories, clients, queues, workers, routers\n" + "- DO NOT include: individual response/request models, data classes, " + "enums, constants, or utility functions\n" + "- Label each edge with the relationship: calls, reads, writes, publishes, " + "subscribes, manages, depends on\n" + "- Use graph TD (top-down) layout\n" + "- If the code has no meaningful architectural components (e.g. it is purely " + "data models or utilities), return the string \"EMPTY\"\n" + "- Return ONLY the ```mermaid\\ngraph TD\\n...``` block or the string \"EMPTY\"\n\n" + f"Chapter: {chapter_title}\n\n" + f"Source code:\n{source_ctx}" + ) + + try: + text = self._diagram_llm_invoke(prompt) + if not text or text.strip() == "EMPTY": + return "" + import re as _re # noqa: PLC0415 + m = _re.search(r"```mermaid\n.*?```", text, _re.DOTALL) + if not m: + return "" + from .diagram_generator import _validate_mermaid # noqa: PLC0415 + candidate = m.group(0) + ok, _ = _validate_mermaid(candidate) + # Accept graph or flowchart headers only — reject classDiagram/erDiagram. + second_line = candidate.splitlines()[1] if len(candidate.splitlines()) > 1 else "" + if ok and not any(kw in second_line for kw in ("graph", "flowchart")): + return "" + return candidate if ok else "" + except Exception as exc: + logger.debug("[WRITER] _llm_architecture_diagram failed: %s", exc) + return "" + + def _llm_class_diagram(self, source_ctx: str, chapter_title: str) -> str: + """Use LLM to generate a Mermaid classDiagram from source code context. + + Returns a validated fenced classDiagram block, or '' on failure / empty output. + """ + if self.llm_client is None: + return "" + + prompt = ( + "The source code below is complete. Read it directly — do NOT call any tools " + "or request additional files. Generate a Mermaid classDiagram immediately.\n\n" + "Rules:\n" + "- Include all significant classes, interfaces, abstract classes, enums, and structs\n" + "- Show class members: +publicMethod(param: type): returnType, -_privateField, #protected\n" + "- Show inheritance: ChildClass --|> ParentClass\n" + "- Show composition: ClassA *-- ClassB (when A contains B as a field)\n" + "- Add stereotypes: <>, <>, <>\n" + "- Skip test classes, private helpers, and trivial single-method classes\n" + "- Cap at 10 classes maximum; prefer the most important ones\n" + "- If there are fewer than 2 meaningful classes, return the string \"EMPTY\"\n" + "- Return ONLY the ```mermaid classDiagram ... ``` block or the string \"EMPTY\"\n\n" + f"Chapter: {chapter_title}\n\n" + f"Source code:\n{source_ctx}" + ) + + try: + text = self._diagram_llm_invoke(prompt) + if not text or text.strip() == "EMPTY": + return "" + import re as _re # noqa: PLC0415 + m = _re.search(r"```mermaid\n.*?```", text, _re.DOTALL) + if not m: + return "" + from .diagram_generator import _validate_mermaid # noqa: PLC0415 + candidate = m.group(0) + ok, _ = _validate_mermaid(candidate) + if ok and "classDiagram" not in candidate.splitlines()[1]: + return "" + return candidate if ok else "" + except Exception as exc: + logger.debug("[WRITER] _llm_class_diagram failed: %s", exc) + return "" + + def _llm_data_model_diagram(self, source_ctx: str, chapter_title: str) -> str: + """Use LLM to generate a Mermaid erDiagram from source code context. + + Only produces a diagram for real database/persistence models (SQLAlchemy, + Django, TypeORM, GORM, etc.) — not Pydantic schemas or plain dataclasses. + Returns a validated fenced erDiagram block, or '' on failure / empty output. + """ + if self.llm_client is None: + return "" + + prompt = ( + "The source code below is complete. Read it directly — do NOT call any tools " + "or request additional files. Identify ONLY true database/persistence " + "models — not API response schemas.\n\n" + "Database models include:\n" + "- SQLAlchemy ORM models (classes with Column(), relationship(), mapped_column())\n" + "- Django models (models.Model subclasses)\n" + "- TypeORM entities (@Entity decorator)\n" + "- Go GORM structs (gorm: tags)\n" + "- Prisma schema definitions\n\n" + "DO NOT include: Pydantic response/request models, dataclasses used as DTOs " + "or value objects, plain TypeScript interfaces, configuration classes.\n\n" + "If you find 2 or more database models, produce a Mermaid erDiagram showing:\n" + "- Entity names and their fields with types (int, str, uuid, etc.)\n" + "- Mark primary keys with PK, foreign keys with FK\n" + "- Relationships between entities (||--o{, }|--|{, etc.)\n\n" + "If fewer than 2 database models exist, return the string \"EMPTY\".\n" + "Return ONLY the ```mermaid erDiagram ... ``` block or the string \"EMPTY\".\n\n" + f"Chapter: {chapter_title}\n\n" + f"Source code:\n{source_ctx}" + ) + + try: + text = self._diagram_llm_invoke(prompt) + if not text or text.strip() == "EMPTY": + return "" + import re as _re # noqa: PLC0415 + m = _re.search(r"```mermaid\n.*?```", text, _re.DOTALL) + if not m: + return "" + from .diagram_generator import _validate_mermaid # noqa: PLC0415 + candidate = m.group(0) + ok, _ = _validate_mermaid(candidate) + # Enforce erDiagram header — reject graph LR or classDiagram blocks. + if ok and "erDiagram" not in candidate.splitlines()[1]: + return "" + return candidate if ok else "" + except Exception as exc: + logger.debug("[WRITER] _llm_data_model_diagram failed: %s", exc) + return "" + # ── Tool dispatch ───────────────────────────────────────────────────── def _dispatch_tool(self, name: str, args: dict[str, Any]) -> str: diff --git a/backend/app/core/wiki_structure_planner/coverage_ledger.py b/backend/app/core/wiki_structure_planner/coverage_ledger.py index 24c89e19..afb93d9e 100644 --- a/backend/app/core/wiki_structure_planner/coverage_ledger.py +++ b/backend/app/core/wiki_structure_planner/coverage_ledger.py @@ -22,6 +22,8 @@ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Set, Tuple +from app.prompts import load_prompt + from ..constants import ( DOC_CLUSTER_SYMBOLS, PAGE_IDENTITY_SYMBOLS, @@ -243,31 +245,7 @@ def _find_overlap_pairs(self) -> List[Tuple[int, int, int]]: # ── Prompt template ───────────────────────────────────────────────── -REFINER_SYSTEM_PROMPT = """\ -You are a wiki page naming assistant. Given a section's symbols and quality \ -flags, output a JSON object with page naming and actions. - -Rules: -- Use capability-based names (not file/class names) -- Each page should cover one coherent capability -- Use the quality flags to decide whether to keep, merge, or split pages -- Output ONLY valid JSON, no explanation - -Output schema: -{ - "section_name": "string", - "section_description": "string", - "page_actions": [ - { - "action": "keep|merge|split|promote_docs|demote", - "micro_id": , - "name": "Page Name", - "description": "1-2 sentence description", - "retrieval_query": "search query for content retrieval" - } - ] -} -""" +REFINER_SYSTEM_PROMPT: str = load_prompt("planner/refiner_system.md") def build_refiner_prompt( diff --git a/backend/app/core/wiki_structure_planner/planner_prompts.py b/backend/app/core/wiki_structure_planner/planner_prompts.py index aa9cd125..6c42044c 100644 --- a/backend/app/core/wiki_structure_planner/planner_prompts.py +++ b/backend/app/core/wiki_structure_planner/planner_prompts.py @@ -17,164 +17,11 @@ from app.core.wiki_structure_planner.evidence import EvidencePack from app.core.wiki_structure_planner.structure_skeleton import Cluster +from app.prompts import load_prompt # ── System prompt ───────────────────────────────────────────────────────────── -PLANNER_SYSTEM_PROMPT: str = """\ -You are a wiki structure planner. Your job is to name wiki pages for software \ -repository clusters. - -## Inputs you receive - -For each cluster you will receive: -- A **cluster header** describing the cluster id, kind, directories, and languages. -- An **evidence pack** with pre-fetched content: symbol signatures, file heads, \ -README excerpts, and SQL schema blocks. - -## How to work - -1. **Read the evidence pack first.** It contains ~1 000 tokens of pre-fetched \ - evidence that is usually sufficient to name the page accurately. -2. **Use tools sparingly.** You have a hard total tool budget across ALL clusters. \ - Call `read_file`, `get_signature`, or `grep` ONLY when: - - The evidence pack contradicts what you expect from the directory names. - - A critical symbol is mentioned but its purpose is unclear from the pack. - - The pack is empty (rare). -3. **Never invent symbols or paths.** If the evidence is insufficient and \ - the budget is exhausted, produce a reasonable name from the directory names. - -## Symbol summaries - -Each symbol in the evidence pack may include a `summary` field — a 2-3 sentence \ -description of its role. Use these summaries to choose accurate page titles and \ -descriptions. Do not invent functionality not reflected in the summaries. - -## Evidence pack fields - -Each cluster's evidence pack contains the following fields. Understanding what -each field means helps you interpret the evidence correctly. - -- **artifact_names**: Architectural symbol names — classes, major functions, and - modules that the clustering algorithm identified as the primary identifiers in - this cluster. These are real identifiers in the code; use them to name - target_symbols in the page spec. Do not invent symbols not present here. -- **path_leaves**: File stem names without extension (e.g., "scheduler", - "queue_manager"). Use these as hints for the page's URL slug and title. A - cluster whose path_leaves are ["scheduler", "worker", "queue"] likely - implements a job-queue subsystem. -- **dir_paths**: Directory paths covered by this cluster. Use to identify the - module boundary. A cluster confined to a single directory (e.g., - "src/workers/") is likely a focused subsystem; one spanning many directories - may be a cross-cutting concern. -- **call_graph_edges**: (caller, callee) pairs within the cluster. Shows the - internal dependency order. A symbol that appears only as a callee is likely - a utility; one that appears only as a caller is likely an entry point. -- **cross_cluster_edges**: (symbol, foreign_cluster_id) pairs — what this - cluster imports from or is imported by other clusters. Use these to populate - the target_symbols list with symbols this cluster needs to reference, and to - identify which foreign cluster pages should be listed under Related Pages. - Do not duplicate the content of the foreign cluster — reference it by name. -- **doc_snippets**: Excerpts from attached documentation files (markdown, - RST, etc.). Use these to understand the documented purpose of the cluster - and to produce an accurate description and retrieval_query. -- **summary** (when present): A 2-3 sentence LLM-generated description of what - the cluster's primary symbol does. Trust this over raw symbol names for page - titles and descriptions. If summary is present, use it as the basis for the - description field in your output. - -## Example output format - -For a cluster covering src/workers/: - -```json -{ - "cluster_id": 2, - "title": "Test Result Ingestion Job Processing", - "description": "Implements an AsyncIO job queue with PostgreSQL backing and a configurable pool of parallel workers that process test result ingestion jobs. Provides startup recovery of pending jobs.", - "retrieval_query": "async job queue worker pool job processing ingestion", - "target_symbols": ["WorkerPool", "Worker", "JobQueue"], - "target_folders": ["src/onetest/receiver/workers/"], - "target_docs": [] -} -``` - -Note: "title" describes WHAT the cluster does, not just what it IS. -"Test Result Ingestion Job Processing" beats "Workers Module". The title -should be a capability-focused phrase that would make sense to a new -engineer reading the wiki table of contents. - -## Cross-cluster links - -cross_cluster_edges tell you what this cluster depends on or what depends on -it. When selecting target_symbols for the page spec, include symbols from -cross_cluster_edges that are architecturally significant — the writer will -read about them from the other cluster's page. - -Do NOT duplicate page content from another cluster's page. Instead, reference -the other cluster by title in the description or in the retrieval_query. The -writer agent will link to the referenced cluster's page automatically via the -wiki-link resolver. Use cross_cluster_edges to understand the import and -dependency structure, not to copy content across pages. - -## Page title quality guidelines - -A good page title describes the capability implemented by the cluster, not the -technical artifact that implements it. The following contrasts illustrate the -distinction: - - Good titles (capability-focused): - - "HTTP Request Routing and Middleware Pipeline" - - "User Authentication and Session Management" - - "Async Job Queue with Worker Pool" - - "Repository Cloning and Filesystem Indexing" - - "Dense and Sparse Search with Ensemble Retrieval" - - Bad titles (artifact-focused — avoid): - - "routes.py" — names the file, not the capability - - "AuthMiddleware" — names the class, not what it does - - "workers/" — names the directory, not the subsystem - - "db_utils" — names a module, describes nothing - - "Utils Module" — completely uninformative - -A new engineer reading the wiki table of contents should be able to understand -what capability each page covers without opening it. Prefer titles that answer -"what does this cluster DO?" not "what is this cluster CALLED?". - -## Retrieval query guidelines - -The retrieval_query is used by the writer agent to find related documentation -and code snippets via dense and sparse retrieval. A good retrieval query: - -- Is 3-8 keywords that describe the cluster's core capability. -- Includes both technical terms (class names, patterns) and conceptual terms - (what the cluster does for the user or system). -- Avoids stop words (the, and, of) — these do not help retrieval. -- Is specific enough to distinguish this cluster from adjacent ones. - -Example: for a cluster implementing async job processing with PostgreSQL: - Good: "async job queue worker pool postgresql ingestion retry" - Bad: "workers and jobs" — too vague, no technical signal - -## Output format - -For each cluster, respond with a **single JSON object** on its own line: - -```json -{"cluster_id": , "title": "", \ -"description": "<1-2 sentences describing the capability>", \ -"retrieval_query": ""} -``` - -Rules: -- `title` must be a capability-focused phrase, NOT a symbol or file name. - - Good: "Raft Consensus Protocol" Bad: "raft_group_manager.py" -- `description` must be 1-2 sentences summarising what the cluster implements \ - or documents (not what files it contains). -- `retrieval_query` must be 3-8 keywords useful for dense/sparse retrieval. -- Produce EXACTLY one JSON object for the current cluster shown in the - message. Do not reason about, reference, or emit pages for other clusters. -- Output ONLY the JSON object — no markdown, no extra text. -""" +PLANNER_SYSTEM_PROMPT: str = load_prompt("planner/system.md") # ── Kind-specific framing ───────────────────────────────────────────────────── @@ -283,51 +130,4 @@ def build_user_prompt( # -- Chaptered planner system prompt ------------------------------------------ -CHAPTERED_PLANNER_SYSTEM_PROMPT: str = """You are a wiki structure planner. Your job is to organise a software repository cluster into a chapter with focused sub-pages. - -## Inputs you receive - -For each cluster you will receive: -- A **cluster header** describing the cluster id, kind, directories, and languages. -- An **evidence pack** with pre-fetched content: symbol signatures, file heads, README excerpts, and SQL schema blocks. - -## How to work - -1. **Read the evidence pack first.** It usually contains enough context. -2. **Use tools sparingly.** You have a hard total tool budget across ALL clusters. -3. **Never invent symbols or paths.** Produce reasonable names from dir names. - -## Output format - -Respond with a **single JSON object** on its own line: - -```json -{ - "cluster_id": , - "chapter_title": "", - "chapter_description": "<1-2 sentences>", - "subpages": [ - { - "title": "", - "description": "<1-2 sentences>", - "page_order": , - "retrieval_query": "<3-8 keywords>", - "target_symbols": [], - "target_folders": [], - "target_docs": [], - "evidence_filter_symbols": [], - "evidence_filter_paths": [] - } - ] -} -``` - -Rules: -- `chapter_title` must be a capability-focused phrase, NOT a symbol or file name. -- Emit **3-5 sub-pages** covering distinct subtopics of the cluster. -- If the cluster has <=5 symbols, emit exactly **1 sub-page**. -- **Never emit more than 6 sub-pages** — cap strictly at 6. -- `page_order` is 1-indexed reading sequence within the chapter. -- `retrieval_query` must be 3-8 keywords useful for dense/sparse retrieval. -- Output ONLY the JSON object — no markdown, no extra text. -""" +CHAPTERED_PLANNER_SYSTEM_PROMPT: str = load_prompt("planner/chaptered_system.md") diff --git a/backend/app/prompts/__init__.py b/backend/app/prompts/__init__.py new file mode 100644 index 00000000..ebbe9c03 --- /dev/null +++ b/backend/app/prompts/__init__.py @@ -0,0 +1,61 @@ +"""Prompt loader — reads Markdown prompt files from this package. + +Usage +----- + from app.prompts import load_prompt + + # Static prompt (no substitutions) + text = load_prompt("writer/system.md") + + # Template prompt (Python .format()-style placeholders) + text = load_prompt("planner/system.md").format(n_clusters=5) + +Design +------ +- All prompt files live under ``backend/app/prompts/`` as ``*.md`` files. +- ``load_prompt`` resolves paths relative to this file's directory. +- Results are cached via ``functools.lru_cache`` — no repeated I/O. +- Path separators use forward-slashes; works on all platforms. +""" + +from __future__ import annotations + +import functools +from pathlib import Path + +_PROMPTS_ROOT: Path = Path(__file__).parent + + +@functools.lru_cache(maxsize=256) +def load_prompt(relative_path: str) -> str: + """Return the text of a prompt Markdown file. + + Parameters + ---------- + relative_path: + Path relative to ``backend/app/prompts/``, e.g. + ``"writer/system.md"`` or ``"planner/user_template.md"``. + Use forward slashes regardless of OS. + + Returns + ------- + str + File contents, stripped of leading/trailing whitespace. + + Raises + ------ + FileNotFoundError + When the file does not exist at the resolved path. + """ + path = (_PROMPTS_ROOT / relative_path).resolve() + # Guard against path traversal (e.g. "../../etc/passwd"). + if not path.is_relative_to(_PROMPTS_ROOT.resolve()): + raise ValueError( + f"Prompt path escapes prompts directory: {relative_path!r}" + ) + if not path.is_file(): + raise FileNotFoundError( + f"Prompt file not found: {path} " + f"(resolved from relative_path={relative_path!r})" + ) + return path.read_text(encoding="utf-8").strip() diff --git a/backend/app/prompts/ask/answer_system.md b/backend/app/prompts/ask/answer_system.md new file mode 100644 index 00000000..7c302c04 --- /dev/null +++ b/backend/app/prompts/ask/answer_system.md @@ -0,0 +1,49 @@ +You are a helpful assistant answering questions about a code repository. + +You have access to the repository's documentation, code, and architecture analysis. +Use the provided context to answer questions accurately and comprehensively. + +## CITATION STYLE (NATURAL FILE REFERENCES) + +**Reference code naturally by file path and symbol name - do NOT use [N] numeric citations.** + +### In Prose: +- "The `authenticate()` function in `auth/auth_manager.py` handles token validation" +- "Session management is implemented via `SessionStore` class in `services/session.py`" +- "The configuration is loaded from `config/settings.py`" + +### In Code Blocks: +Always include the file path as a comment header: +```python +# auth/auth_manager.py +def authenticate(token: str) -> bool: + ... +``` + +### Multiple File References: +- "The authentication flow spans `auth/login.py` → `auth/auth_manager.py` → `services/session.py`" +- "Both `UserModel` in `models/user.py` and `UserService` in `services/user.py` are involved" + +## RESPONSE GUIDELINES + +1. **Be Accurate**: Only state facts supported by the provided context +2. **Be Specific**: Reference actual file paths, function names, and class names naturally in text +3. **Be Structured**: Use headers, bullet points, and code blocks for clarity +4. **Acknowledge Gaps**: If context is insufficient, say so clearly +5. **Show Code**: Include relevant code snippets with file path headers + +## FORMAT + +Use clear markdown: +- Headers (##) for major sections +- Code blocks with language tags and file path comments +- Bullet points for lists +- `backticks` for inline code/file references +- Natural file path mentions (not [N] citations) + +## SOURCES SUMMARY + +At the end of your response, include a brief summary: +--- +**📁 Files Referenced:** +- `path/to/file.py` — Brief description of what was used from this file diff --git a/backend/app/prompts/ask/answer_user.md b/backend/app/prompts/ask/answer_user.md new file mode 100644 index 00000000..66963897 --- /dev/null +++ b/backend/app/prompts/ask/answer_user.md @@ -0,0 +1,22 @@ +## Retrieved Code Context + +{context} + +## Available Sources +{sources_reference} + +## User Question + +{question} + +--- + +**INSTRUCTIONS:** +1. Answer the question comprehensively based on the code context above +2. Reference files naturally by path (e.g., "in `path/to/file.py`") - do NOT use [N] citations +3. Include relevant code snippets with file path headers as comments +4. Reference specific symbol names (classes, functions, methods) +5. End with a "📁 Files Referenced" summary +6. If the context doesn't contain enough information, state this clearly + +**Provide your answer:** diff --git a/backend/app/prompts/ask/output_instructions.md b/backend/app/prompts/ask/output_instructions.md new file mode 100644 index 00000000..8bca7f92 --- /dev/null +++ b/backend/app/prompts/ask/output_instructions.md @@ -0,0 +1,14 @@ +# Output Format + +Your response should be: +1. **Direct** — Answer the question first, then provide supporting details +2. **Structured** — Use headers (##) for sections, bullets for lists, code blocks with file headers +3. **Concise** — Focus on what was asked. Don't explain tangential systems +4. **Evidence-based** — Every claim references a specific file/symbol +5. **Honest** — If context is insufficient, say so clearly + +Do NOT: +- Generate a "Research Report" with executive summaries +- List every file you searched +- Explain your search process +- Use [N] numeric citations diff --git a/backend/app/prompts/ask/query_optimization_system.md b/backend/app/prompts/ask/query_optimization_system.md new file mode 100644 index 00000000..c519c7d5 --- /dev/null +++ b/backend/app/prompts/ask/query_optimization_system.md @@ -0,0 +1,14 @@ +You are a search query optimizer for code repositories. + +Your task is to transform user questions into optimized retrieval queries that will find the most relevant code and documentation in a vector store. + +**QUERY OPTIMIZATION STRATEGY:** +- Identify key technical terms, component names, and concepts from the repository context +- Include specific file names, folder paths, and symbol names when relevant +- Combine topic keywords with implementation-specific terms +- Add related functionality and patterns that may appear in code +- Use both high-level concepts and low-level implementation details + +**OUTPUT FORMAT:** +Return ONLY the optimized search query - a space-separated list of relevant terms. +Do NOT include explanations, quotes, or any other text. diff --git a/backend/app/prompts/ask/query_optimization_user.md b/backend/app/prompts/ask/query_optimization_user.md new file mode 100644 index 00000000..e4160af4 --- /dev/null +++ b/backend/app/prompts/ask/query_optimization_user.md @@ -0,0 +1,15 @@ +## Repository Context +{repository_context} + +## User Question +"{question}" + +**TASK:** Generate an optimized retrieval query that will find the most relevant code and documentation. + +**QUERY OPTIMIZATION EXAMPLE:** +For a question about "How does authentication work?": +Generated Query: "authentication login logout session token JWT user validation security middleware auth_manager password hashing access control authorization user_service security_config" + +This query combines topic keywords with file-specific terms and related functionality. + +**Return ONLY the optimized query (space-separated terms), nothing else:** diff --git a/backend/app/prompts/ask/tool_instructions.md b/backend/app/prompts/ask/tool_instructions.md new file mode 100644 index 00000000..36df69e0 --- /dev/null +++ b/backend/app/prompts/ask/tool_instructions.md @@ -0,0 +1,16 @@ +# Tool Tips + +Each tool has its own description — read it. Below are cost/strategy hints only. + +| Tool | Cost | When to use | +|------|------|-------------| +| `search_symbols` | Cheap | First step — discover classes, functions, modules | +| `find_cross_repo_links` | Cheap | Project mode only — direct API-surface/contracts between repos | +| `get_relationships` | Medium | After finding key symbols — see callers, callees, inheritance | +| `get_code` | **Expensive** | Only 2-4 key symbols — reads full source | +| `search_docs` | Medium | README / config / architecture questions only | +| `query_graph` | Cheap | Precise JQL filters (`type:class file:src/auth/*`) | +| `think` | Free | 1-2 sentences to decide next step | + +Call multiple `search_symbols` in parallel for different aspects. +Do NOT call `get_code` on every search result — be selective. diff --git a/backend/app/prompts/ask/workflow_instructions.md b/backend/app/prompts/ask/workflow_instructions.md new file mode 100644 index 00000000..941289e7 --- /dev/null +++ b/backend/app/prompts/ask/workflow_instructions.md @@ -0,0 +1,77 @@ +# Ask Tool — Agentic Q&A + +You are a code expert answering questions about a software repository. +Today's date is {date}. + +## Your Goal + +Answer the user's question **accurately and concisely**, citing specific files +and symbols. You have access to tools that search the codebase at different +levels of detail. Use them strategically. + +## Conversation Follow-ups + +If a `## Conversation History` section is present in the user's message, the +current question may be a follow-up to a prior exchange. Use the history to: +- Resolve pronouns and references ("it", "that function", "the same module") +- Avoid re-explaining things already covered +- Focus on what is *new* or *different* in the current question + +Do NOT summarise prior answers — just use them as context. + +## Mandatory Workflow (FOLLOW THIS ORDER) + +1. **Discover** — call `search_symbols` (or `query_graph` for precise filters) + to find relevant classes, functions, and modules. This is cheap (~50 tokens/result). + Call multiple searches in parallel when investigating different aspects. + +2. **Connect** — call `get_relationships` for the 2-3 most relevant symbols to + understand how they relate (callers, callees, inheritance, imports). + In project mode, if `find_cross_repo_links` is available, use it for questions + about cross-repo integration, API contracts, shared DTOs, clients/servers, + FFI/ABI, protobuf/gRPC, GraphQL, BDD, or CLI boundaries. Treat those direct + project links as high-confidence evidence. + +3. **Read selectively** — call `get_code` ONLY for the 2-4 symbols you truly need + to read. This is expensive (~200-2000 tokens). Do NOT dump every search result + into get_code. + +4. **Documentation** — call `search_docs` if the question is about project + setup, configuration, or high-level architecture described in documentation + (README, guides, etc.). Skip this for pure code questions. + +5. **Answer** — synthesize your findings into a clear, concise response. + Reference files by path and symbols by name. + +## Iteration Budget + +You have a maximum of **{max_iterations} tool calls**. Most questions need 3-6. +If you've used 5+ tool calls and haven't found what you need, synthesize what +you have and note the gaps. + +## When to Stop Searching + +Stop and answer when: +- You have code evidence for your claims +- Your last 2 searches returned similar/overlapping results +- You can answer the specific question asked (don't over-research) + +## Citation Style (MANDATORY) + +Reference code **naturally by file path and symbol name**. Do NOT use [N] numeric citations. + +In prose: +- "The `authenticate()` function in `auth/auth_manager.py` handles token validation" +- "Session management is implemented in `services/session.py`" + +In code blocks — always include the file path as a comment header: +```python +# auth/auth_manager.py +def authenticate(token: str) -> bool: + ... +``` + +At the end of your answer, include: +--- +**Files Referenced:** +- `path/to/file.py` — Brief description diff --git a/backend/app/prompts/diagrams/architecture.md b/backend/app/prompts/diagrams/architecture.md new file mode 100644 index 00000000..f10b1040 --- /dev/null +++ b/backend/app/prompts/diagrams/architecture.md @@ -0,0 +1,52 @@ +You generate Mermaid stateDiagram-v2 diagrams showing the functional flow and state transitions of a software cluster. + +## Available tools — use them aggressively +- **read_file(path)** — read source files to understand the code +- **grep(pattern)** — find patterns across the codebase (e.g. grep for `async def`, `@router`, handler names) +- **get_signature(symbol)** — get a function/class signature without reading the whole file +- **get_callers(symbol)** — discover what calls a function (entry points, triggers) +- **get_callees(symbol)** — discover what a function depends on + +## What to generate +Produce a `stateDiagram-v2` diagram capturing the **functional flow** of this cluster: +- States are the major phases, stages, or conditions the system passes through +- Transitions show how the system moves between states (triggered by actions, events, conditions) +- Use notes to annotate key actions or data at important transitions +- Include error/failure paths and their recovery or termination states +- Composite states (`state X { ... }`) for complex sub-flows + +## Mermaid stateDiagram-v2 syntax reference + +``` +stateDiagram-v2 + [*] --> Idle + Idle --> Processing : job received + Processing --> Success : completed + Processing --> Failed : error + Success --> [*] + Failed --> Idle : retry + + state Processing { + [*] --> Validating + Validating --> Executing + Executing --> [*] + } + + note right of Failed + Writes failure status to DB + end note +``` + +Key syntax: +- `[*]` — start or end state +- `StateA --> StateB : label` — transition with label +- `state "Long Name" as ShortId` — alias for long state names +- `state X { ... }` — composite/nested state +- `note left/right of State` — annotation + +## Output rules +- Discover the flow by reading files and grepping for handlers, state machines, queues, event loops +- Show 5-12 states — enough to understand the flow without overwhelming +- Use descriptive transition labels (what triggers the change) +- If there is no meaningful flow (pure data model cluster, utility helpers), return "EMPTY" +- Return ONLY the ```mermaid\nstateDiagram-v2\n...``` block or "EMPTY" diff --git a/backend/app/prompts/diagrams/class_structure.md b/backend/app/prompts/diagrams/class_structure.md new file mode 100644 index 00000000..52209abc --- /dev/null +++ b/backend/app/prompts/diagrams/class_structure.md @@ -0,0 +1,65 @@ +You generate Mermaid classDiagram diagrams showing the object-oriented structure of a software cluster. + +## Available tools — use them aggressively +- **read_file(path)** — read source files +- **grep(pattern)** — find class definitions, inheritance, method patterns (e.g. grep for `class `, `def __init__`, `@dataclass`) +- **get_signature(symbol)** — get a class or method signature with parameters and return type +- **get_callers(symbol)** — find what uses a class or method + +## What to generate +Produce a `classDiagram` showing: +- All significant classes, interfaces, abstract classes, enums, structs +- Member visibility: `+public`, `-private`, `#protected`, `~package` +- Method signatures: `+methodName(param: Type): ReturnType` +- Relationships: `--|>` inheritance, `*--` composition, `o--` aggregation, `-->` association, `..>` dependency +- Stereotypes: `<>`, `<>`, `<>`, `<>`, `<>` +- Skip test classes and trivial single-method helpers + +## Mermaid classDiagram syntax reference + +``` +classDiagram + class Animal { + <> + +String name + +int age + +makeSound()* String + +move() void + } + class Dog { + +String breed + +bark() void + } + class Cat { + -bool indoor + +purr() void + } + class Owner { + +String name + +List~Animal~ pets + +addPet(animal: Animal) void + } + + Animal <|-- Dog : extends + Animal <|-- Cat : extends + Owner "1" *-- "many" Animal : owns +``` + +Key syntax: +- `class Foo { ... }` — declare class with members +- `class Foo~GenericType~` — generic class +- `<>`, `<>`, `<>` — stereotypes inside the class or on a separate line +- `+`, `-`, `#`, `~` — visibility prefixes +- `methodName()* Type` — abstract method (asterisk) +- `fieldName$ Type` — static member (dollar sign) +- `A --|> B` — A inherits B +- `A *-- B` — A composes B (filled diamond) +- `A o-- B` — A aggregates B (empty diamond) +- `A "1" --> "n" B : label` — association with multiplicity +- `note for ClassName "text"` — annotation + +## Output rules +- Read files and grep for class definitions before generating +- Show all classes that matter for understanding this cluster's design +- If fewer than 2 meaningful classes exist, return "EMPTY" +- Return ONLY the ```mermaid\nclassDiagram\n...``` block or "EMPTY" diff --git a/backend/app/prompts/diagrams/data_model.md b/backend/app/prompts/diagrams/data_model.md new file mode 100644 index 00000000..e7637749 --- /dev/null +++ b/backend/app/prompts/diagrams/data_model.md @@ -0,0 +1,71 @@ +You generate Mermaid erDiagram diagrams showing the data model and entity relationships of a software cluster. + +## Available tools — use them aggressively +- **read_file(path)** — read source files +- **grep(pattern)** — find ORM models: grep for `Column(`, `relationship(`, `mapped_column`, `@Entity`, `BaseModel`, `db.Model`, `DeclarativeBase`, `gorm:"` +- **get_signature(symbol)** — get a model class definition including fields + +## What to identify — ONLY models DEFINED in these files +Include ONLY entities whose class/struct body is physically present in the files you read: +- SQLAlchemy ORM models (classes with `Column()`, `relationship()`, `mapped_column()`) +- Django models (`models.Model` subclasses) +- TypeORM entities (`@Entity` decorator) +- GORM structs (struct tags with `gorm:`) +- Prisma schema model blocks + +DO NOT include: +- Any model that is only *imported* (`from db.models import X`) — even if referenced heavily +- Pydantic response/request schemas (even if they extend `BaseModel`) +- Plain dataclasses used as DTOs or value objects +- Configuration classes + +If the files you read only *use* models defined elsewhere, return "EMPTY". + +## Mermaid erDiagram syntax reference + +``` +erDiagram + CUSTOMER ||--o{ ORDER : "places" + CUSTOMER { + uuid id PK + string name + string email UK + datetime created_at + } + ORDER ||--|{ LINE_ITEM : "contains" + ORDER { + uuid id PK + uuid customer_id FK + string status + decimal total + } + LINE_ITEM { + uuid id PK + uuid order_id FK + uuid product_id FK + int quantity + decimal unit_price + } + PRODUCT ||--o{ LINE_ITEM : "appears in" + PRODUCT { + uuid id PK + string name + decimal price + } +``` + +Key syntax: +- `ENTITY { type fieldName PK/FK/UK }` — entity with typed attributes +- `||--o{` — one-to-many (exact one to zero-or-many) +- `||--|{` — one-to-many (exact one to one-or-many) +- `}|--|{` — many-to-many +- `||--||` — one-to-one +- `..` instead of `--` for non-identifying relationships +- Attribute types: `int`, `string`, `uuid`, `datetime`, `decimal`, `bool`, `json` +- `PK` primary key, `FK` foreign key, `UK` unique key + +## Output rules +- Read files and grep for ORM patterns before deciding which entities to include +- Show all foreign key relationships as edges +- If fewer than 2 true DB entities exist, return "EMPTY" +- Return ONLY the ```mermaid\nerDiagram\n...``` block or "EMPTY" diff --git a/backend/app/prompts/diagrams/references/classDiagram.md b/backend/app/prompts/diagrams/references/classDiagram.md new file mode 100644 index 00000000..ffef1e5c --- /dev/null +++ b/backend/app/prompts/diagrams/references/classDiagram.md @@ -0,0 +1,1024 @@ +> **Warning** +> +> ## THIS IS AN AUTOGENERATED FILE. DO NOT EDIT. +> +> ## Please edit the corresponding file in [/packages/mermaid/src/docs/syntax/classDiagram.md](../../packages/mermaid/src/docs/syntax/classDiagram.md). + +# Class diagrams + +> "In software engineering, a class diagram in the Unified Modeling Language (UML) is a type of static structure diagram that describes the structure of a system by showing the system's classes, their attributes, operations (or methods), and the relationships among objects." +> +> -Wikipedia + +The class diagram is the main building block of object-oriented modeling. It is used for general conceptual modeling of the structure of the application, and for detailed modeling to translate the models into programming code. Class diagrams can also be used for data modeling. The classes in a class diagram represent both the main elements, interactions in the application, and the classes to be programmed. + +Mermaid can render class diagrams. + +```mermaid-example +--- +title: Animal example +--- +classDiagram + note "From Duck till Zebra" + Animal <|-- Duck + note for Duck "can fly
can swim
can dive
can help in debugging" + Animal <|-- Fish + Animal <|-- Zebra + Animal : +int age + Animal : +String gender + Animal: +isMammal() + Animal: +mate() + class Duck{ + +String beakColor + +swim() + +quack() + } + class Fish{ + -int sizeInFeet + -canEat() + } + class Zebra{ + +bool is_wild + +run() + } +``` + +```mermaid +--- +title: Animal example +--- +classDiagram + note "From Duck till Zebra" + Animal <|-- Duck + note for Duck "can fly
can swim
can dive
can help in debugging" + Animal <|-- Fish + Animal <|-- Zebra + Animal : +int age + Animal : +String gender + Animal: +isMammal() + Animal: +mate() + class Duck{ + +String beakColor + +swim() + +quack() + } + class Fish{ + -int sizeInFeet + -canEat() + } + class Zebra{ + +bool is_wild + +run() + } +``` + +## Syntax + +### Class + +UML provides mechanisms to represent class members, such as attributes and methods, and additional information about them. +A single instance of a class in the diagram contains three compartments: + +- The top compartment contains the name of the class. It is printed in bold and centered, and the first letter is capitalized. It may also contain optional annotation text describing the nature of the class. +- The middle compartment contains the attributes of the class. They are left-aligned and the first letter is lowercase. +- The bottom compartment contains the operations the class can execute. They are also left-aligned and the first letter is lowercase. + +```mermaid-example +--- +title: Bank example +--- +classDiagram + class BankAccount + BankAccount : +String owner + BankAccount : +Bigdecimal balance + BankAccount : +deposit(amount) + BankAccount : +withdrawal(amount) + +``` + +```mermaid +--- +title: Bank example +--- +classDiagram + class BankAccount + BankAccount : +String owner + BankAccount : +Bigdecimal balance + BankAccount : +deposit(amount) + BankAccount : +withdrawal(amount) + +``` + +## Define a class + +There are two ways to define a class: + +- Explicitly using keyword **class** like `class Animal` which would define the Animal class. +- Via a **relationship** which defines two classes at a time along with their relationship. For instance, `Vehicle <|-- Car`. + +```mermaid-example +classDiagram + class Animal + Vehicle <|-- Car +``` + +```mermaid +classDiagram + class Animal + Vehicle <|-- Car +``` + +Naming convention: a class name should be composed only of alphanumeric characters (including unicode), underscores, and dashes (-). + +### Class labels + +In case you need to provide a label for a class, you can use the following syntax: + +```mermaid-example +classDiagram + class Animal["Animal with a label"] + class Car["Car with *! symbols"] + Animal --> Car +``` + +```mermaid +classDiagram + class Animal["Animal with a label"] + class Car["Car with *! symbols"] + Animal --> Car +``` + +You can also use backticks to escape special characters in the label: + +```mermaid-example +classDiagram + class `Animal Class!` + class `Car Class` + `Animal Class!` --> `Car Class` +``` + +```mermaid +classDiagram + class `Animal Class!` + class `Car Class` + `Animal Class!` --> `Car Class` +``` + +## Defining Members of a class + +UML provides mechanisms to represent class members such as attributes and methods, as well as additional information about them. + +Mermaid distinguishes between attributes and functions/methods based on if the **parenthesis** `()` are present or not. The ones with `()` are treated as functions/methods, and all others as attributes. + +There are two ways to define the members of a class, and regardless of whichever syntax is used to define the members, the output will still be same. The two different ways are : + +- Associate a member of a class using **:** (colon) followed by member name, useful to define one member at a time. For example: + +```mermaid-example +classDiagram +class BankAccount +BankAccount : +String owner +BankAccount : +BigDecimal balance +BankAccount : +deposit(amount) +BankAccount : +withdrawal(amount) +``` + +```mermaid +classDiagram +class BankAccount +BankAccount : +String owner +BankAccount : +BigDecimal balance +BankAccount : +deposit(amount) +BankAccount : +withdrawal(amount) +``` + +- Associate members of a class using **{}** brackets, where members are grouped within curly brackets. Suitable for defining multiple members at once. For example: + +```mermaid-example +classDiagram +class BankAccount{ + +String owner + +BigDecimal balance + +deposit(amount) + +withdrawal(amount) +} +``` + +```mermaid +classDiagram +class BankAccount{ + +String owner + +BigDecimal balance + +deposit(amount) + +withdrawal(amount) +} +``` + +#### Return Type + +Optionally you can end a method/function definition with the data type that will be returned (note: there must be a space between the final `)` and the return type). An example: + +```mermaid-example +classDiagram +class BankAccount{ + +String owner + +BigDecimal balance + +deposit(amount) bool + +withdrawal(amount) int +} +``` + +```mermaid +classDiagram +class BankAccount{ + +String owner + +BigDecimal balance + +deposit(amount) bool + +withdrawal(amount) int +} +``` + +#### Generic Types + +Generics can be represented as part of a class definition, and for class members/return types. In order to denote an item as generic, you enclose that type within `~` (**tilde**). **Nested** type declarations such as `List>` are supported, though generics that include a comma are currently not supported. (such as `List>`) + +> _note_ when a generic is used within a class definition, the generic type is NOT considered part of the class name. i.e.: for any syntax which required you to reference the class name, you need to drop the type part of the definition. This also means that mermaid does not currently support having two classes with the same name, but different generic types. + +```mermaid-example +classDiagram +class Square~Shape~{ + int id + List~int~ position + setPoints(List~int~ points) + getPoints() List~int~ +} + +Square : -List~string~ messages +Square : +setMessages(List~string~ messages) +Square : +getMessages() List~string~ +Square : +getDistanceMatrix() List~List~int~~ +``` + +```mermaid +classDiagram +class Square~Shape~{ + int id + List~int~ position + setPoints(List~int~ points) + getPoints() List~int~ +} + +Square : -List~string~ messages +Square : +setMessages(List~string~ messages) +Square : +getMessages() List~string~ +Square : +getDistanceMatrix() List~List~int~~ +``` + +#### Visibility + +To describe the visibility (or encapsulation) of an attribute or method/function that is a part of a class (i.e. a class member), optional notation may be placed before that members' name: + +- `+` Public +- `-` Private +- `#` Protected +- `~` Package/Internal + +> _note_ you can also include additional _classifiers_ to a method definition by adding the following notation to the _end_ of the method, i.e.: after the `()` or after the return type: +> +> - `*` Abstract e.g.: `someAbstractMethod()*` or `someAbstractMethod() int*` +> - `$` Static e.g.: `someStaticMethod()$` or `someStaticMethod() String$` + +> _note_ you can also include additional _classifiers_ to a field definition by adding the following notation to the very end: +> +> - `$` Static e.g.: `String someField$` + +## Defining Relationship + +A relationship is a general term covering the specific types of logical connections found on class and object diagrams. + +``` +[classA][Arrow][ClassB] +``` + +There are eight different types of relations defined for classes under UML which are currently supported: + +| Type | Description | +| ------- | ------------- | +| `<\|--` | Inheritance | +| `*--` | Composition | +| `o--` | Aggregation | +| `-->` | Association | +| `--` | Link (Solid) | +| `..>` | Dependency | +| `..\|>` | Realization | +| `..` | Link (Dashed) | + +```mermaid-example +classDiagram +classA <|-- classB +classC *-- classD +classE o-- classF +classG <-- classH +classI -- classJ +classK <.. classL +classM <|.. classN +classO .. classP + +``` + +```mermaid +classDiagram +classA <|-- classB +classC *-- classD +classE o-- classF +classG <-- classH +classI -- classJ +classK <.. classL +classM <|.. classN +classO .. classP + +``` + +We can use the labels to describe the nature of the relation between two classes. Also, arrowheads can be used in the opposite direction as well: + +```mermaid-example +classDiagram +classA --|> classB : Inheritance +classC --* classD : Composition +classE --o classF : Aggregation +classG --> classH : Association +classI -- classJ : Link(Solid) +classK ..> classL : Dependency +classM ..|> classN : Realization +classO .. classP : Link(Dashed) + +``` + +```mermaid +classDiagram +classA --|> classB : Inheritance +classC --* classD : Composition +classE --o classF : Aggregation +classG --> classH : Association +classI -- classJ : Link(Solid) +classK ..> classL : Dependency +classM ..|> classN : Realization +classO .. classP : Link(Dashed) + +``` + +### Labels on Relations + +It is possible to add label text to a relation: + +``` +[classA][Arrow][ClassB]:LabelText +``` + +```mermaid-example +classDiagram +classA <|-- classB : implements +classC *-- classD : composition +classE o-- classF : aggregation +``` + +```mermaid +classDiagram +classA <|-- classB : implements +classC *-- classD : composition +classE o-- classF : aggregation +``` + +### Two-way relations + +Relations can logically represent an N:M association: + +```mermaid-example +classDiagram + Animal <|--|> Zebra +``` + +```mermaid +classDiagram + Animal <|--|> Zebra +``` + +Here is the syntax: + +``` +[Relation Type][Link][Relation Type] +``` + +Where `Relation Type` can be one of: + +| Type | Description | +| ----- | ----------- | +| `<\|` | Inheritance | +| `\*` | Composition | +| `o` | Aggregation | +| `>` | Association | +| `<` | Association | +| `\|>` | Realization | + +And `Link` can be one of: + +| Type | Description | +| ---- | ----------- | +| -- | Solid | +| .. | Dashed | + +### Lollipop Interfaces + +Classes can also be given a special relation type that defines a lollipop interface on the class. A lollipop interface is defined using the following syntax: + +- `bar ()-- foo` +- `foo --() bar` + +The interface (bar) with the lollipop connects to the class (foo). + +Note: Each interface that is defined is unique and is meant to not be shared between classes / have multiple edges connecting to it. + +```mermaid-example +classDiagram + bar ()-- foo +``` + +```mermaid +classDiagram + bar ()-- foo +``` + +```mermaid-example +classDiagram + class Class01 { + int amount + draw() + } + Class01 --() bar + Class02 --() bar + + foo ()-- Class01 +``` + +```mermaid +classDiagram + class Class01 { + int amount + draw() + } + Class01 --() bar + Class02 --() bar + + foo ()-- Class01 +``` + +## Define Namespace + +A namespace groups classes. + +```mermaid-example +classDiagram +namespace BaseShapes { + class Triangle + class Rectangle { + double width + double height + } +} +``` + +```mermaid +classDiagram +namespace BaseShapes { + class Triangle + class Rectangle { + double width + double height + } +} +``` + +## Cardinality / Multiplicity on relations + +Multiplicity or cardinality in class diagrams indicates the number of instances of one class that can be linked to an instance of the other class. For example, each company will have one or more employees (not zero), and each employee currently works for zero or one companies. + +Multiplicity notations are placed near the end of an association. + +The different cardinality options are : + +- `1` Only 1 +- `0..1` Zero or One +- `1..*` One or more +- `*` Many +- `n` n (where n>1) +- `0..n` zero to n (where n>1) +- `1..n` one to n (where n>1) + +Cardinality can be easily defined by placing the text option within quotes `"` before or after a given arrow. For example: + +``` +[classA] "cardinality1" [Arrow] "cardinality2" [ClassB]:LabelText +``` + +```mermaid-example +classDiagram + Customer "1" --> "*" Ticket + Student "1" --> "1..*" Course + Galaxy --> "many" Star : Contains +``` + +```mermaid +classDiagram + Customer "1" --> "*" Ticket + Student "1" --> "1..*" Course + Galaxy --> "many" Star : Contains +``` + +## Annotations on classes + +It is possible to annotate classes with markers to provide additional metadata about the class. This can give a clearer indication about its nature. Some common annotations include: + +- `<>` To represent an Interface class +- `<>` To represent an abstract class +- `<>` To represent a service class +- `<>` To represent an enum + +Annotations are defined within the opening `<<` and closing `>>`. There are two ways to add an annotation to a class, and either way the output will be same: + +> **Tip:**\ +> In Mermaid class diagrams, annotations like `<>` can be attached in two ways: +> +> - **Inline with the class definition** (Recommended for consistency): +> +> ```mermaid-example +> classDiagram +> class Shape <> +> ``` +> +> ```mermaid +> classDiagram +> class Shape <> +> ``` +> +> - **Separate line after the class definition**: +> +> ```mermaid-example +> classDiagram +> class Shape +> <> Shape +> ``` +> +> ```mermaid +> classDiagram +> class Shape +> <> Shape +> ``` +> +> Both methods are fully supported and produce identical diagrams.\ +> However, it is recommended to use the **inline style** for better readability and consistent formatting across diagrams. + +- In a **_separate line_** after a class is defined: + +```mermaid-example +classDiagram +class Shape +<> Shape +Shape : noOfVertices +Shape : draw() +``` + +```mermaid +classDiagram +class Shape +<> Shape +Shape : noOfVertices +Shape : draw() +``` + +- In a **_nested structure_** along with the class definition: + +```mermaid-example +classDiagram +class Shape{ + <> + noOfVertices + draw() +} +class Color{ + <> + RED + BLUE + GREEN + WHITE + BLACK +} + +``` + +```mermaid +classDiagram +class Shape{ + <> + noOfVertices + draw() +} +class Color{ + <> + RED + BLUE + GREEN + WHITE + BLACK +} + +``` + +## Comments + +Comments can be entered within a class diagram, which will be ignored by the parser. Comments need to be on their own line, and must be prefaced with `%%` (double percent signs). Any text until the next newline will be treated as a comment, including any class diagram syntax. + +```mermaid-example +classDiagram +%% This whole line is a comment classDiagram class Shape <> +class Shape{ + <> + noOfVertices + draw() +} +``` + +```mermaid +classDiagram +%% This whole line is a comment classDiagram class Shape <> +class Shape{ + <> + noOfVertices + draw() +} +``` + +## Setting the direction of the diagram + +With class diagrams you can use the direction statement to set the direction in which the diagram will render: + +```mermaid-example +classDiagram + direction RL + class Student { + -idCard : IdCard + } + class IdCard{ + -id : int + -name : string + } + class Bike{ + -id : int + -name : string + } + Student "1" --o "1" IdCard : carries + Student "1" --o "1" Bike : rides +``` + +```mermaid +classDiagram + direction RL + class Student { + -idCard : IdCard + } + class IdCard{ + -id : int + -name : string + } + class Bike{ + -id : int + -name : string + } + Student "1" --o "1" IdCard : carries + Student "1" --o "1" Bike : rides +``` + +## Interaction + +It is possible to bind a click event to a node. The click can lead to either a javascript callback or to a link which will be opened in a new browser tab. **Note**: This functionality is disabled when using `securityLevel='strict'` and enabled when using `securityLevel='loose'`. + +You would define these actions on a separate line after all classes have been declared. + +``` +action className "reference" "tooltip" +click className call callback() "tooltip" +click className href "url" "tooltip" +``` + +- _action_ is either `link` or `callback`, depending on which type of interaction you want to have called +- _className_ is the id of the node that the action will be associated with +- _reference_ is either the url link, or the function name for callback. +- (_optional_) tooltip is a string to be displayed when hovering over element (note: The styles of the tooltip are set by the class .mermaidTooltip.) +- note: callback function will be called with the nodeId as parameter. + +## Notes + +It is possible to add notes on the diagram using `note "line1\nline2"`. A note can be added for a specific class using `note for "line1\nline2"`. + +### Examples + +```mermaid-example +classDiagram + note "This is a general note" + note for MyClass "This is a note for a class" + class MyClass{ + } +``` + +```mermaid +classDiagram + note "This is a general note" + note for MyClass "This is a note for a class" + class MyClass{ + } +``` + +_URL Link:_ + +```mermaid-example +classDiagram +class Shape +link Shape "https://www.github.com" "This is a tooltip for a link" +class Shape2 +click Shape2 href "https://www.github.com" "This is a tooltip for a link" +``` + +```mermaid +classDiagram +class Shape +link Shape "https://www.github.com" "This is a tooltip for a link" +class Shape2 +click Shape2 href "https://www.github.com" "This is a tooltip for a link" +``` + +_Callback:_ + +```mermaid-example +classDiagram +class Shape +callback Shape "callbackFunction" "This is a tooltip for a callback" +class Shape2 +click Shape2 call callbackFunction() "This is a tooltip for a callback" +``` + +```mermaid +classDiagram +class Shape +callback Shape "callbackFunction" "This is a tooltip for a callback" +class Shape2 +click Shape2 call callbackFunction() "This is a tooltip for a callback" +``` + +```html + +``` + +```mermaid-example +classDiagram + class Class01 + class Class02 + callback Class01 "callbackFunction" "Callback tooltip" + link Class02 "https://www.github.com" "This is a link" + class Class03 + class Class04 + click Class03 call callbackFunction() "Callback tooltip" + click Class04 href "https://www.github.com" "This is a link" +``` + +```mermaid +classDiagram + class Class01 + class Class02 + callback Class01 "callbackFunction" "Callback tooltip" + link Class02 "https://www.github.com" "This is a link" + class Class03 + class Class04 + click Class03 call callbackFunction() "Callback tooltip" + click Class04 href "https://www.github.com" "This is a link" +``` + +> **Success** The tooltip functionality and the ability to link to urls are available from version 0.5.2. + +Beginner's tip—a full example using interactive links in an HTML page: + +```html + + 
+    classDiagram
+    Animal <|-- Duck
+    Animal <|-- Fish
+    Animal <|-- Zebra
+    Animal : +int age
+    Animal : +String gender
+    Animal: +isMammal()
+    Animal: +mate()
+    class Duck{
+      +String beakColor
+      +swim()
+      +quack()
+      }
+    class Fish{
+      -int sizeInFeet
+      -canEat()
+      }
+    class Zebra{
+      +bool is_wild
+      +run()
+      }
+
+      callback Duck "callback" "Tooltip"
+      link Zebra "https://www.github.com" "This is a link"
+
+ + + +``` + +## Styling + +### Styling a node + +It is possible to apply specific styles such as a thicker border or a different background color to an individual node using the `style` keyword. + +Note that notes and namespaces cannot be styled individually but do support themes. + +```mermaid-example +classDiagram + class Animal + class Mineral + style Animal fill:#f9f,stroke:#333,stroke-width:4px + style Mineral fill:#bbf,stroke:#f66,stroke-width:2px,color:#fff,stroke-dasharray: 5 5 +``` + +```mermaid +classDiagram + class Animal + class Mineral + style Animal fill:#f9f,stroke:#333,stroke-width:4px + style Mineral fill:#bbf,stroke:#f66,stroke-width:2px,color:#fff,stroke-dasharray: 5 5 +``` + +#### Classes + +More convenient than defining the style every time is to define a class of styles and attach this class to the nodes that +should have a different look. + +A class definition looks like the example below: + +``` +classDef className fill:#f9f,stroke:#333,stroke-width:4px; +``` + +Also, it is possible to define style to multiple classes in one statement: + +``` +classDef firstClassName,secondClassName font-size:12pt; +``` + +Attachment of a class to a node is done as per below: + +``` +cssClass "nodeId1" className; +``` + +It is also possible to attach a class to a list of nodes in one statement: + +``` +cssClass "nodeId1,nodeId2" className; +``` + +A shorter form of adding a class is to attach the classname to the node using the `:::` operator: + +```mermaid-example +classDiagram + class Animal:::someclass + classDef someclass fill:#f96 +``` + +```mermaid +classDiagram + class Animal:::someclass + classDef someclass fill:#f96 +``` + +Or: + +```mermaid-example +classDiagram + class Animal:::someclass { + -int sizeInFeet + -canEat() + } + classDef someclass fill:#f96 +``` + +```mermaid +classDiagram + class Animal:::someclass { + -int sizeInFeet + -canEat() + } + classDef someclass fill:#f96 +``` + +### Default class + +If a class is named default it will be applied to all nodes. Specific styles and classes should be defined afterwards to override the applied default styling. + +``` +classDef default fill:#f9f,stroke:#333,stroke-width:4px; +``` + +```mermaid-example +classDiagram + class Animal:::pink + class Mineral + + classDef default fill:#f96,color:red + classDef pink color:#f9f +``` + +```mermaid +classDiagram + class Animal:::pink + class Mineral + + classDef default fill:#f96,color:red + classDef pink color:#f9f +``` + +### CSS Classes + +It is also possible to predefine classes in CSS styles that can be applied from the graph definition as in the example +below: + +**Example style** + +```html + +``` + +**Example definition** + +```mermaid-example +classDiagram + class Animal:::styleClass +``` + +```mermaid +classDiagram + class Animal:::styleClass +``` + +> cssClasses cannot be added using this shorthand method at the same time as a relation statement. + +## Configuration + +### Members Box + +It is possible to hide the empty members box of a class node. + +This is done by changing the **hideEmptyMembersBox** value of the class diagram configuration. For more information on how to edit the Mermaid configuration see the [configuration page.](https://mermaid.js.org/config/configuration.html) + +```mermaid-example +--- + config: + class: + hideEmptyMembersBox: true +--- +classDiagram + class Duck +``` + +```mermaid +--- + config: + class: + hideEmptyMembersBox: true +--- +classDiagram + class Duck +``` diff --git a/backend/app/prompts/diagrams/references/entityRelationshipDiagram.md b/backend/app/prompts/diagrams/references/entityRelationshipDiagram.md new file mode 100644 index 00000000..e31a00eb --- /dev/null +++ b/backend/app/prompts/diagrams/references/entityRelationshipDiagram.md @@ -0,0 +1,670 @@ +> **Warning** +> +> ## THIS IS AN AUTOGENERATED FILE. DO NOT EDIT. +> +> ## Please edit the corresponding file in [/packages/mermaid/src/docs/syntax/entityRelationshipDiagram.md](../../packages/mermaid/src/docs/syntax/entityRelationshipDiagram.md). + +# Entity Relationship Diagrams + +> An entity–relationship model (or ER model) describes interrelated things of interest in a specific domain of knowledge. A basic ER model is composed of entity types (which classify the things of interest) and specifies relationships that can exist between entities (instances of those entity types) [Wikipedia](https://en.wikipedia.org/wiki/Entity%E2%80%93relationship_model). + +Note that practitioners of ER modelling almost always refer to _entity types_ simply as _entities_. For example the `CUSTOMER` entity _type_ would be referred to simply as the `CUSTOMER` entity. This is so common it would be inadvisable to do anything else, but technically an entity is an abstract _instance_ of an entity type, and this is what an ER diagram shows - abstract instances, and the relationships between them. This is why entities are always named using singular nouns. + +Mermaid can render ER diagrams + +```mermaid-example +--- +title: Order example +--- +erDiagram + CUSTOMER ||--o{ ORDER : places + ORDER ||--|{ LINE-ITEM : contains + CUSTOMER }|..|{ DELIVERY-ADDRESS : uses +``` + +```mermaid +--- +title: Order example +--- +erDiagram + CUSTOMER ||--o{ ORDER : places + ORDER ||--|{ LINE-ITEM : contains + CUSTOMER }|..|{ DELIVERY-ADDRESS : uses +``` + +Entity names are often capitalised, although there is no accepted standard on this, and it is not required in Mermaid. + +Relationships between entities are represented by lines with end markers representing cardinality. Mermaid uses the most popular crow's foot notation. The crow's foot intuitively conveys the possibility of many instances of the entity that it connects to. + +ER diagrams can be used for various purposes, ranging from abstract logical models devoid of any implementation details, through to physical models of relational database tables. It can be useful to include attribute definitions on ER diagrams to aid comprehension of the purpose and meaning of entities. These do not necessarily need to be exhaustive; often a small subset of attributes is enough. Mermaid allows them to be defined in terms of their _type_ and _name_. + +```mermaid-example +erDiagram + CUSTOMER ||--o{ ORDER : places + CUSTOMER { + string name + string custNumber + string sector + } + ORDER ||--|{ LINE-ITEM : contains + ORDER { + int orderNumber + string deliveryAddress + } + LINE-ITEM { + string productCode + int quantity + float pricePerUnit + } +``` + +```mermaid +erDiagram + CUSTOMER ||--o{ ORDER : places + CUSTOMER { + string name + string custNumber + string sector + } + ORDER ||--|{ LINE-ITEM : contains + ORDER { + int orderNumber + string deliveryAddress + } + LINE-ITEM { + string productCode + int quantity + float pricePerUnit + } +``` + +When including attributes on ER diagrams, you must decide whether to include foreign keys as attributes. This probably depends on how closely you are trying to represent relational table structures. If your diagram is a _logical_ model which is not meant to imply a relational implementation, then it is better to leave these out because the associative relationships already convey the way that entities are associated. For example, a JSON data structure can implement a one-to-many relationship without the need for foreign key properties, using arrays. Similarly an object-oriented programming language may use pointers or references to collections. Even for models that are intended for relational implementation, you might decide that inclusion of foreign key attributes duplicates information already portrayed by the relationships, and does not add meaning to entities. Ultimately, it's your choice. + +## Syntax + +### Entities and Relationships + +Mermaid syntax for ER diagrams is compatible with PlantUML, with an extension to label the relationship. Each statement consists of the following parts: + +``` + [ : ] +``` + +Where: + +- `first-entity` is the name of an entity. Names support any unicode characters and can include spaces if surrounded by double quotes (e.g. "name with space"). +- `relationship` describes the way that both entities inter-relate. See below. +- `second-entity` is the name of the other entity. +- `relationship-label` describes the relationship from the perspective of the first entity. + +For example: + +``` + PROPERTY ||--|{ ROOM : contains +``` + +This statement can be read as _a property contains one or more rooms, and a room is part of one and only one property_. You can see that the label here is from the first entity's perspective: a property contains a room, but a room does not contain a property. When considered from the perspective of the second entity, the equivalent label is usually very easy to infer. (Some ER diagrams label relationships from both perspectives, but this is not supported here, and is usually superfluous). + +Only the `first-entity` part of a statement is mandatory. This makes it possible to show an entity with no relationships, which can be useful during iterative construction of diagrams. If any other parts of a statement are specified, then all parts are mandatory. + +#### Unicode text + +Entity names, relationships, and attributes all support unicode text. + +```mermaid-example +erDiagram + "This ❤ Unicode" +``` + +```mermaid +erDiagram + "This ❤ Unicode" +``` + +#### Markdown formatting + +Markdown formatting and text is also supported. + +```mermaid-example +erDiagram + "This **is** _Markdown_" +``` + +```mermaid +erDiagram + "This **is** _Markdown_" +``` + +### Relationship Syntax + +The `relationship` part of each statement can be broken down into three sub-components: + +- the cardinality of the first entity with respect to the second +- whether the relationship confers identity on a 'child' entity +- the cardinality of the second entity with respect to the first + +Cardinality is a property that describes how many elements of another entity can be related to the entity in question. In the above example a `PROPERTY` can have one or more `ROOM` instances associated to it, whereas a `ROOM` can only be associated with one `PROPERTY`. In each cardinality marker there are two characters. The outermost character represents a maximum value, and the innermost character represents a minimum value. The table below summarises possible cardinalities. + +| Value (left) | Value (right) | Meaning | +| :----------: | :-----------: | ----------------------------- | +| `\|o` | `o\|` | Zero or one | +| `\|\|` | `\|\|` | Exactly one | +| `}o` | `o{` | Zero or more (no upper limit) | +| `}\|` | `\|{` | One or more (no upper limit) | + +**Aliases** + +| Value (left) | Value (right) | Alias for | +| :----------: | :-----------: | ------------ | +| one or zero | one or zero | Zero or one | +| zero or one | zero or one | Zero or one | +| one or more | one or more | One or more | +| one or many | one or many | One or more | +| many(1) | many(1) | One or more | +| 1+ | 1+ | One or more | +| zero or more | zero or more | Zero or more | +| zero or many | zero or many | Zero or more | +| many(0) | many(0) | Zero or more | +| 0+ | 0+ | Zero or more | +| only one | only one | Exactly one | +| 1 | 1 | Exactly one | + +### Identification + +Relationships may be classified as either _identifying_ or _non-identifying_ and these are rendered with either solid or dashed lines respectively. This is relevant when one of the entities in question cannot have independent existence without the other. For example a firm that insures people to drive cars might need to store data on `NAMED-DRIVER`s. In modelling this we might start out by observing that a `CAR` can be driven by many `PERSON` instances, and a `PERSON` can drive many `CAR`s - both entities can exist without the other, so this is a non-identifying relationship that we might specify in Mermaid as: `PERSON }|..|{ CAR : "driver"`. Note the two dots in the middle of the relationship that will result in a dashed line being drawn between the two entities. But when this many-to-many relationship is resolved into two one-to-many relationships, we observe that a `NAMED-DRIVER` cannot exist without both a `PERSON` and a `CAR` - the relationships become identifying and would be specified using hyphens, which translate to a solid line: + +| Value | Alias for | +| :---: | :---------------: | +| -- | _identifying_ | +| .. | _non-identifying_ | + +**Aliases** + +| Value | Alias for | +| :-----------: | :---------------: | +| to | _identifying_ | +| optionally to | _non-identifying_ | + +```mermaid-example +erDiagram + CAR ||--o{ NAMED-DRIVER : allows + PERSON }o..o{ NAMED-DRIVER : is +``` + +```mermaid +erDiagram + CAR ||--o{ NAMED-DRIVER : allows + PERSON }o..o{ NAMED-DRIVER : is +``` + +```mermaid-example +erDiagram + CAR 1 to zero or more NAMED-DRIVER : allows + PERSON many(0) optionally to 0+ NAMED-DRIVER : is +``` + +```mermaid +erDiagram + CAR 1 to zero or more NAMED-DRIVER : allows + PERSON many(0) optionally to 0+ NAMED-DRIVER : is +``` + +### Attributes + +Attributes can be defined for entities by specifying the entity name followed by a block containing multiple `type name` pairs, where a block is delimited by an opening `{` and a closing `}`. The attributes are rendered inside the entity boxes. For example: + +```mermaid-example +erDiagram + CAR ||--o{ NAMED-DRIVER : allows + CAR { + string registrationNumber + string make + string model + } + PERSON ||--o{ NAMED-DRIVER : is + PERSON { + string firstName + string lastName + int age + } +``` + +```mermaid +erDiagram + CAR ||--o{ NAMED-DRIVER : allows + CAR { + string registrationNumber + string make + string model + } + PERSON ||--o{ NAMED-DRIVER : is + PERSON { + string firstName + string lastName + int age + } +``` + +The `type` values must begin with an alphabetic character and may contain digits, hyphens, underscores, parentheses and square brackets. The `name` values follow a similar format to `type`, but may start with an asterisk as another option to indicate an attribute is a primary key. Other than that, there are no restrictions, and there is no implicit set of valid data types. + +### Entity Name Aliases + +An alias can be added to an entity using square brackets. If provided, the alias will be showed in the diagram instead of the entity name. Alias names follow all of the same rules as entity names. + +```mermaid-example +erDiagram + p[Person] { + string firstName + string lastName + } + a["Customer Account"] { + string email + } + p ||--o| a : has +``` + +```mermaid +erDiagram + p[Person] { + string firstName + string lastName + } + a["Customer Account"] { + string email + } + p ||--o| a : has +``` + +#### Attribute Keys and Comments + +Attributes may also have a `key` or comment defined. Keys can be `PK`, `FK` or `UK`, for Primary Key, Foreign Key or Unique Key (markdown formatting and unicode is not supported for keys). To specify multiple key constraints on a single attribute, separate them with a comma (e.g., `PK, FK`). A `comment` is defined by double quotes at the end of an attribute. Comments themselves cannot have double-quote characters in them. + +```mermaid-example +erDiagram + CAR ||--o{ NAMED-DRIVER : allows + CAR { + string registrationNumber PK + string make + string model + string[] parts + } + PERSON ||--o{ NAMED-DRIVER : is + PERSON { + string driversLicense PK "The license #" + string(99) firstName "Only 99 characters are allowed" + string lastName + string phone UK + int age + } + NAMED-DRIVER { + string carRegistrationNumber PK, FK + string driverLicence PK, FK + } + MANUFACTURER only one to zero or more CAR : makes +``` + +```mermaid +erDiagram + CAR ||--o{ NAMED-DRIVER : allows + CAR { + string registrationNumber PK + string make + string model + string[] parts + } + PERSON ||--o{ NAMED-DRIVER : is + PERSON { + string driversLicense PK "The license #" + string(99) firstName "Only 99 characters are allowed" + string lastName + string phone UK + int age + } + NAMED-DRIVER { + string carRegistrationNumber PK, FK + string driverLicence PK, FK + } + MANUFACTURER only one to zero or more CAR : makes +``` + +### Direction + +The direction statement declares the direction of the diagram. + +This declares that the diagram is oriented from top to bottom (`TB`). This can be reversed to be oriented from bottom to top (`BT`). + +```mermaid-example +erDiagram + direction TB + CUSTOMER ||--o{ ORDER : places + CUSTOMER { + string name + string custNumber + string sector + } + ORDER ||--|{ LINE-ITEM : contains + ORDER { + int orderNumber + string deliveryAddress + } + LINE-ITEM { + string productCode + int quantity + float pricePerUnit + } +``` + +```mermaid +erDiagram + direction TB + CUSTOMER ||--o{ ORDER : places + CUSTOMER { + string name + string custNumber + string sector + } + ORDER ||--|{ LINE-ITEM : contains + ORDER { + int orderNumber + string deliveryAddress + } + LINE-ITEM { + string productCode + int quantity + float pricePerUnit + } +``` + +This declares that the diagram is oriented from left to right (`LR`). This can be reversed to be oriented from right to left (`RL`). + +```mermaid-example +erDiagram + direction LR + CUSTOMER ||--o{ ORDER : places + CUSTOMER { + string name + string custNumber + string sector + } + ORDER ||--|{ LINE-ITEM : contains + ORDER { + int orderNumber + string deliveryAddress + } + LINE-ITEM { + string productCode + int quantity + float pricePerUnit + } +``` + +```mermaid +erDiagram + direction LR + CUSTOMER ||--o{ ORDER : places + CUSTOMER { + string name + string custNumber + string sector + } + ORDER ||--|{ LINE-ITEM : contains + ORDER { + int orderNumber + string deliveryAddress + } + LINE-ITEM { + string productCode + int quantity + float pricePerUnit + } +``` + +Possible diagram orientations are: + +- TB - Top to bottom +- BT - Bottom to top +- RL - Right to left +- LR - Left to right + +### Styling a node + +It is possible to apply specific styles such as a thicker border or a different background color to a node. + +```mermaid-example +erDiagram + id1||--||id2 : label + style id1 fill:#f9f,stroke:#333,stroke-width:4px + style id2 fill:#bbf,stroke:#f66,stroke-width:2px,color:#fff,stroke-dasharray: 5 5 +``` + +```mermaid +erDiagram + id1||--||id2 : label + style id1 fill:#f9f,stroke:#333,stroke-width:4px + style id2 fill:#bbf,stroke:#f66,stroke-width:2px,color:#fff,stroke-dasharray: 5 5 +``` + +It is also possible to attach styles to a list of nodes in one statement: + +``` + style nodeId1,nodeId2 styleList +``` + +#### Classes + +More convenient than defining the style every time is to define a class of styles and attach this class to the nodes that +should have a different look. + +A class definition looks like the example below: + +``` + classDef className fill:#f9f,stroke:#333,stroke-width:4px +``` + +It is also possible to define multiple classes in one statement: + +``` + classDef firstClassName,secondClassName font-size:12pt +``` + +Attachment of a class to a node is done as per below: + +``` + class nodeId1 className +``` + +It is also possible to attach a class to a list of nodes in one statement: + +``` + class nodeId1,nodeId2 className +``` + +Multiple classes can be attached at the same time as well: + +``` + class nodeId1,nodeId2 className1,className2 +``` + +A shorter form of adding a class is to attach the classname to the node using the `:::`operator as per below: + +```mermaid-example +erDiagram + direction TB + CAR:::someclass { + string registrationNumber + string make + string model + } + PERSON:::someclass { + string firstName + string lastName + int age + } + HOUSE:::someclass + + classDef someclass fill:#f96 +``` + +```mermaid +erDiagram + direction TB + CAR:::someclass { + string registrationNumber + string make + string model + } + PERSON:::someclass { + string firstName + string lastName + int age + } + HOUSE:::someclass + + classDef someclass fill:#f96 +``` + +This form can be used when declaring relationships between entities: + +```mermaid-example +erDiagram + CAR { + string registrationNumber + string make + string model + } + PERSON { + string firstName + string lastName + int age + } + PERSON:::foo ||--|| CAR : owns + PERSON o{--|| HOUSE:::bar : has + + classDef foo stroke:#f00 + classDef bar stroke:#0f0 + classDef foobar stroke:#00f +``` + +```mermaid +erDiagram + CAR { + string registrationNumber + string make + string model + } + PERSON { + string firstName + string lastName + int age + } + PERSON:::foo ||--|| CAR : owns + PERSON o{--|| HOUSE:::bar : has + + classDef foo stroke:#f00 + classDef bar stroke:#0f0 + classDef foobar stroke:#00f +``` + +Similar to the class statement, the shorthand syntax can also apply multiple classes at once: + +``` + nodeId:::className1,className2 +``` + +### Default class + +If a class is named default it will be assigned to all classes without specific class definitions. + +``` + classDef default fill:#f9f,stroke:#333,stroke-width:4px; +``` + +> **Note:** Custom styles from style or other class statements take priority and will overwrite the default styles. (e.g. The `default` class gives nodes a background color of pink but the `blue` class will give that node a background color of blue if applied.) + +```mermaid-example +erDiagram + CAR { + string registrationNumber + string make + string model + } + PERSON { + string firstName + string lastName + int age + } + PERSON:::foo ||--|| CAR : owns + PERSON o{--|| HOUSE:::bar : has + + classDef default fill:#f9f,stroke-width:4px + classDef foo stroke:#f00 + classDef bar stroke:#0f0 + classDef foobar stroke:#00f +``` + +```mermaid +erDiagram + CAR { + string registrationNumber + string make + string model + } + PERSON { + string firstName + string lastName + int age + } + PERSON:::foo ||--|| CAR : owns + PERSON o{--|| HOUSE:::bar : has + + classDef default fill:#f9f,stroke-width:4px + classDef foo stroke:#f00 + classDef bar stroke:#0f0 + classDef foobar stroke:#00f +``` + +## Configuration + +### Layout + +The layout of the diagram is handled by [`render()`](../config/setup/mermaid/interfaces/Mermaid.md#render). The default layout is dagre. + +For larger or more-complex diagrams, you can alternatively apply the ELK (Eclipse Layout Kernel) layout using your YAML frontmatter's `config`. For more information, see [Customizing ELK Layout](../intro/syntax-reference.md#customizing-elk-layout). + +```yaml +--- +config: + layout: elk +--- +``` + +Your Mermaid code should be similar to the following: + +```mermaid-example +--- +title: Order example +config: + layout: elk +--- +erDiagram + CUSTOMER ||--o{ ORDER : places + ORDER ||--|{ LINE-ITEM : contains + CUSTOMER }|..|{ DELIVERY-ADDRESS : uses +``` + +```mermaid +--- +title: Order example +config: + layout: elk +--- +erDiagram + CUSTOMER ||--o{ ORDER : places + ORDER ||--|{ LINE-ITEM : contains + CUSTOMER }|..|{ DELIVERY-ADDRESS : uses +``` + +> **Note** +> Note that the site needs to use mermaid version 9.4+ for this to work and have this featured enabled in the lazy-loading configuration. + + diff --git a/backend/app/prompts/diagrams/references/stateDiagram.md b/backend/app/prompts/diagrams/references/stateDiagram.md new file mode 100644 index 00000000..c9ca956f --- /dev/null +++ b/backend/app/prompts/diagrams/references/stateDiagram.md @@ -0,0 +1,672 @@ +> **Warning** +> +> ## THIS IS AN AUTOGENERATED FILE. DO NOT EDIT. +> +> ## Please edit the corresponding file in [/packages/mermaid/src/docs/syntax/stateDiagram.md](../../packages/mermaid/src/docs/syntax/stateDiagram.md). + +# State diagrams + +> "A state diagram is a type of diagram used in computer science and related fields to describe the behavior of systems. +> State diagrams require that the system described is composed of a finite number of states; sometimes, this is indeed the +> case, while at other times this is a reasonable abstraction." Wikipedia + +Mermaid can render state diagrams. The syntax tries to be compliant with the syntax used in plantUml as this will make +it easier for users to share diagrams between mermaid and plantUml. + +```mermaid-example +--- +title: Simple sample +--- +stateDiagram-v2 + [*] --> Still + Still --> [*] + + Still --> Moving + Moving --> Still + Moving --> Crash + Crash --> [*] +``` + +```mermaid +--- +title: Simple sample +--- +stateDiagram-v2 + [*] --> Still + Still --> [*] + + Still --> Moving + Moving --> Still + Moving --> Crash + Crash --> [*] +``` + +Older renderer: + +```mermaid-example +stateDiagram + [*] --> Still + Still --> [*] + + Still --> Moving + Moving --> Still + Moving --> Crash + Crash --> [*] +``` + +```mermaid +stateDiagram + [*] --> Still + Still --> [*] + + Still --> Moving + Moving --> Still + Moving --> Crash + Crash --> [*] +``` + +In state diagrams systems are described in terms of _states_ and how one _state_ can change to another _state_ via +a _transition._ The example diagram above shows three states: **Still**, **Moving** and **Crash**. You start in the +**Still** state. From **Still** you can change to the **Moving** state. From **Moving** you can change either back to the **Still** state or to +the **Crash** state. There is no transition from **Still** to **Crash**. (You can't crash if you're still.) + +## States + +A state can be declared in multiple ways. The simplest way is to define a state with just an id: + +```mermaid-example +stateDiagram-v2 + stateId +``` + +```mermaid +stateDiagram-v2 + stateId +``` + +Another way is by using the state keyword with a description as per below: + +```mermaid-example +stateDiagram-v2 + state "This is a state description" as s2 +``` + +```mermaid +stateDiagram-v2 + state "This is a state description" as s2 +``` + +Another way to define a state with a description is to define the state id followed by a colon and the description: + +```mermaid-example +stateDiagram-v2 + s2 : This is a state description +``` + +```mermaid +stateDiagram-v2 + s2 : This is a state description +``` + +## Transitions + +Transitions are path/edges when one state passes into another. This is represented using text arrow, "-->". + +When you define a transition between two states and the states are not already defined, the undefined states are defined +with the id from the transition. You can later add descriptions to states defined this way. + +```mermaid-example +stateDiagram-v2 + s1 --> s2 +``` + +```mermaid +stateDiagram-v2 + s1 --> s2 +``` + +It is possible to add text to a transition to describe what it represents: + +```mermaid-example +stateDiagram-v2 + s1 --> s2: A transition +``` + +```mermaid +stateDiagram-v2 + s1 --> s2: A transition +``` + +## Start and End + +There are two special states indicating the start and stop of the diagram. These are written with the \[\*] syntax and +the direction of the transition to it defines it either as a start or a stop state. + +```mermaid-example +stateDiagram-v2 + [*] --> s1 + s1 --> [*] +``` + +```mermaid +stateDiagram-v2 + [*] --> s1 + s1 --> [*] +``` + +## Composite states + +In a real world use of state diagrams you often end up with diagrams that are multidimensional as one state can +have several internal states. These are called composite states in this terminology. + +In order to define a composite state you need to use the state keyword followed by an id and the body of the composite +state between {}. You can name a composite state on a separate line just like a simple state. See the example below: + +```mermaid-example +stateDiagram-v2 + [*] --> First + state First { + [*] --> second + second --> [*] + } + + [*] --> NamedComposite + NamedComposite: Another Composite + state NamedComposite { + [*] --> namedSimple + namedSimple --> [*] + namedSimple: Another simple + } +``` + +```mermaid +stateDiagram-v2 + [*] --> First + state First { + [*] --> second + second --> [*] + } + + [*] --> NamedComposite + NamedComposite: Another Composite + state NamedComposite { + [*] --> namedSimple + namedSimple --> [*] + namedSimple: Another simple + } +``` + +You can do this in several layers: + +```mermaid-example +stateDiagram-v2 + [*] --> First + + state First { + [*] --> Second + + state Second { + [*] --> second + second --> Third + + state Third { + [*] --> third + third --> [*] + } + } + } +``` + +```mermaid +stateDiagram-v2 + [*] --> First + + state First { + [*] --> Second + + state Second { + [*] --> second + second --> Third + + state Third { + [*] --> third + third --> [*] + } + } + } +``` + +You can also define transitions also between composite states: + +```mermaid-example +stateDiagram-v2 + [*] --> First + First --> Second + First --> Third + + state First { + [*] --> fir + fir --> [*] + } + state Second { + [*] --> sec + sec --> [*] + } + state Third { + [*] --> thi + thi --> [*] + } +``` + +```mermaid +stateDiagram-v2 + [*] --> First + First --> Second + First --> Third + + state First { + [*] --> fir + fir --> [*] + } + state Second { + [*] --> sec + sec --> [*] + } + state Third { + [*] --> thi + thi --> [*] + } +``` + +_You cannot define transitions between internal states belonging to different composite states_ + +## Choice + +Sometimes you need to model a choice between two or more paths, you can do so using <\>. + +```mermaid-example +stateDiagram-v2 + state if_state <> + [*] --> IsPositive + IsPositive --> if_state + if_state --> False: if n < 0 + if_state --> True : if n >= 0 +``` + +```mermaid +stateDiagram-v2 + state if_state <> + [*] --> IsPositive + IsPositive --> if_state + if_state --> False: if n < 0 + if_state --> True : if n >= 0 +``` + +## Forks + +It is possible to specify a fork in the diagram using <\> <\>. + +```mermaid-example + stateDiagram-v2 + state fork_state <> + [*] --> fork_state + fork_state --> State2 + fork_state --> State3 + + state join_state <> + State2 --> join_state + State3 --> join_state + join_state --> State4 + State4 --> [*] +``` + +```mermaid + stateDiagram-v2 + state fork_state <> + [*] --> fork_state + fork_state --> State2 + fork_state --> State3 + + state join_state <> + State2 --> join_state + State3 --> join_state + join_state --> State4 + State4 --> [*] +``` + +## Notes + +Sometimes nothing says it better than a Post-it note. That is also the case in state diagrams. + +Here you can choose to put the note to the _right of_ or to the _left of_ a node. + +```mermaid-example + stateDiagram-v2 + State1: The state with a note + note right of State1 + Important information! You can write + notes. + end note + State1 --> State2 + note left of State2 : This is the note to the left. +``` + +```mermaid + stateDiagram-v2 + State1: The state with a note + note right of State1 + Important information! You can write + notes. + end note + State1 --> State2 + note left of State2 : This is the note to the left. +``` + +## Concurrency + +As in plantUml you can specify concurrency using the -- symbol. + +```mermaid-example +stateDiagram-v2 + [*] --> Active + + state Active { + [*] --> NumLockOff + NumLockOff --> NumLockOn : EvNumLockPressed + NumLockOn --> NumLockOff : EvNumLockPressed + -- + [*] --> CapsLockOff + CapsLockOff --> CapsLockOn : EvCapsLockPressed + CapsLockOn --> CapsLockOff : EvCapsLockPressed + -- + [*] --> ScrollLockOff + ScrollLockOff --> ScrollLockOn : EvScrollLockPressed + ScrollLockOn --> ScrollLockOff : EvScrollLockPressed + } +``` + +```mermaid +stateDiagram-v2 + [*] --> Active + + state Active { + [*] --> NumLockOff + NumLockOff --> NumLockOn : EvNumLockPressed + NumLockOn --> NumLockOff : EvNumLockPressed + -- + [*] --> CapsLockOff + CapsLockOff --> CapsLockOn : EvCapsLockPressed + CapsLockOn --> CapsLockOff : EvCapsLockPressed + -- + [*] --> ScrollLockOff + ScrollLockOff --> ScrollLockOn : EvScrollLockPressed + ScrollLockOn --> ScrollLockOff : EvScrollLockPressed + } +``` + +## Setting the direction of the diagram + +With state diagrams you can use the direction statement to set the direction which the diagram will render like in this +example. + +```mermaid-example +stateDiagram + direction LR + [*] --> A + A --> B + B --> C + state B { + direction LR + a --> b + } + B --> D +``` + +```mermaid +stateDiagram + direction LR + [*] --> A + A --> B + B --> C + state B { + direction LR + a --> b + } + B --> D +``` + +## Comments + +Comments can be entered within a state diagram chart, which will be ignored by the parser. Comments need to be on their +own line, and must be prefaced with `%%` (double percent signs). Any text after the start of the comment to the next +newline will be treated as a comment, including any diagram syntax + +```mermaid-example +stateDiagram-v2 + [*] --> Still + Still --> [*] +%% this is a comment + Still --> Moving + Moving --> Still %% another comment + Moving --> Crash + Crash --> [*] +``` + +```mermaid +stateDiagram-v2 + [*] --> Still + Still --> [*] +%% this is a comment + Still --> Moving + Moving --> Still %% another comment + Moving --> Crash + Crash --> [*] +``` + +## Styling with classDefs + +As with other diagrams (like flowcharts), you can define a style in the diagram itself and apply that named style to a +state or states in the diagram. + +**These are the current limitations with state diagram classDefs:** + +1. Cannot be applied to start or end states +2. Cannot be applied to or within composite states + +_These are in development and will be available in a future version._ + +You define a style using the `classDef` keyword, which is short for "class definition" (where "class" means something +like a _CSS class_) +followed by _a name for the style,_ +and then one or more _property-value pairs_. Each _property-value pair_ is +a _[valid CSS property name](https://www.w3.org/TR/CSS/#properties)_ followed by a colon (`:`) and then a _value._ + +Here is an example of a classDef with just one property-value pair: + +```txt +classDef movement font-style:italic; +``` + +where + +- the _name_ of the style is `movement` +- the only _property_ is `font-style` and its _value_ is `italic` + +If you want to have more than one _property-value pair_ then you put a comma (`,`) between each _property-value pair._ + +Here is an example with three property-value pairs: + +```txt +classDef badBadEvent fill:#f00,color:white,font-weight:bold,stroke-width:2px,stroke:yellow +``` + +where + +- the _name_ of the style is `badBadEvent` +- the first _property_ is `fill` and its _value_ is `#f00` +- the second _property_ is `color` and its _value_ is `white` +- the third _property_ is `font-weight` and its _value_ is `bold` +- the fourth _property_ is `stroke-width` and its _value_ is `2px` +- the fifth _property_ is `stroke` and its _value_ is `yellow` + +### Apply classDef styles to states + +There are two ways to apply a `classDef` style to a state: + +1. use the `class` keyword to apply a classDef style to one or more states in a single statement, or +2. use the `:::` operator to apply a classDef style to a state as it is being used in a transition statement (e.g. with an arrow + to/from another state) + +#### 1. `class` statement + +A `class` statement tells Mermaid to apply the named classDef to one or more classes. The form is: + +```txt +class [one or more state names, separated by commas] [name of a style defined with classDef] +``` + +Here is an example applying the `badBadEvent` style to a state named `Crash`: + +```txt +class Crash badBadEvent +``` + +Here is an example applying the `movement` style to the two states `Moving` and `Crash`: + +```txt +class Moving, Crash movement +``` + +Here is a diagram that shows the examples in use. Note that the `Crash` state has two classDef styles applied: `movement` +and `badBadEvent` + +```mermaid-example + stateDiagram + direction TB + + accTitle: This is the accessible title + accDescr: This is an accessible description + + classDef notMoving fill:white + classDef movement font-style:italic + classDef badBadEvent fill:#f00,color:white,font-weight:bold,stroke-width:2px,stroke:yellow + + [*]--> Still + Still --> [*] + Still --> Moving + Moving --> Still + Moving --> Crash + Crash --> [*] + + class Still notMoving + class Moving, Crash movement + class Crash badBadEvent + class end badBadEvent +``` + +```mermaid + stateDiagram + direction TB + + accTitle: This is the accessible title + accDescr: This is an accessible description + + classDef notMoving fill:white + classDef movement font-style:italic + classDef badBadEvent fill:#f00,color:white,font-weight:bold,stroke-width:2px,stroke:yellow + + [*]--> Still + Still --> [*] + Still --> Moving + Moving --> Still + Moving --> Crash + Crash --> [*] + + class Still notMoving + class Moving, Crash movement + class Crash badBadEvent + class end badBadEvent +``` + +#### 2. `:::` operator to apply a style to a state + +You can apply a classDef style to a state using the `:::` (three colons) operator. The syntax is + +```txt +[state]:::[style name] +``` + +You can use this in a diagram within a statement using a class. This includes the start and end states. For example: + +```mermaid-example +stateDiagram + direction TB + + accTitle: This is the accessible title + accDescr: This is an accessible description + + classDef notMoving fill:white + classDef movement font-style:italic; + classDef badBadEvent fill:#f00,color:white,font-weight:bold,stroke-width:2px,stroke:yellow + + [*] --> Still:::notMoving + Still --> [*] + Still --> Moving:::movement + Moving --> Still + Moving --> Crash:::movement + Crash:::badBadEvent --> [*] +``` + +```mermaid +stateDiagram + direction TB + + accTitle: This is the accessible title + accDescr: This is an accessible description + + classDef notMoving fill:white + classDef movement font-style:italic; + classDef badBadEvent fill:#f00,color:white,font-weight:bold,stroke-width:2px,stroke:yellow + + [*] --> Still:::notMoving + Still --> [*] + Still --> Moving:::movement + Moving --> Still + Moving --> Crash:::movement + Crash:::badBadEvent --> [*] +``` + +## Spaces in state names + +Spaces can be added to a state by first defining the state with an id and then referencing the id later. + +In the following example there is a state with the id **yswsii** and description **Your state with spaces in it**. +After it has been defined, **yswsii** is used in the diagram in the first transition (`[*] --> yswsii`) +and also in the transition to **YetAnotherState** (`yswsii --> YetAnotherState`). +(**yswsii** has been styled so that it is different from the other states.) + +```mermaid-example +stateDiagram + classDef yourState font-style:italic,font-weight:bold,fill:white + + yswsii: Your state with spaces in it + [*] --> yswsii:::yourState + [*] --> SomeOtherState + SomeOtherState --> YetAnotherState + yswsii --> YetAnotherState + YetAnotherState --> [*] +``` + +```mermaid +stateDiagram + classDef yourState font-style:italic,font-weight:bold,fill:white + + yswsii: Your state with spaces in it + [*] --> yswsii:::yourState + [*] --> SomeOtherState + SomeOtherState --> YetAnotherState + yswsii --> YetAnotherState + YetAnotherState --> [*] +``` + + diff --git a/backend/app/prompts/extractors/image_describe.md b/backend/app/prompts/extractors/image_describe.md new file mode 100644 index 00000000..53a127ee --- /dev/null +++ b/backend/app/prompts/extractors/image_describe.md @@ -0,0 +1,8 @@ +You are indexing this image for a technical documentation wiki. +Describe what the image contains in detail — visible text, diagrams, +charts, code snippets, UI elements, architecture, data flow. +Include any text you can read verbatim. Aim for a complete textual +representation that a search engine can match against natural- +language queries. Do not add commentary about the image's quality +or style; describe content only. If the image is blank or contains +no useful content, respond with exactly: EMPTY_IMAGE. diff --git a/backend/app/prompts/extractors/pdf_page_describe.md b/backend/app/prompts/extractors/pdf_page_describe.md new file mode 100644 index 00000000..be326cad --- /dev/null +++ b/backend/app/prompts/extractors/pdf_page_describe.md @@ -0,0 +1,7 @@ +You are indexing this page of a PDF for a technical documentation +wiki. Describe everything on the page in detail — body text +verbatim where readable, tables row by row, diagrams, charts, +figures, code blocks, footnotes. Preserve the document's logical +structure (headings, lists, table rows) using markdown. Do not +add commentary about the page's quality or style. If the page is +blank, respond with exactly: BLANK_PAGE. diff --git a/backend/app/prompts/planner/chaptered_system.md b/backend/app/prompts/planner/chaptered_system.md new file mode 100644 index 00000000..4e8344df --- /dev/null +++ b/backend/app/prompts/planner/chaptered_system.md @@ -0,0 +1,47 @@ +You are a wiki structure planner. Your job is to organise a software repository cluster into a chapter with focused sub-pages. + +## Inputs you receive + +For each cluster you will receive: +- A **cluster header** describing the cluster id, kind, directories, and languages. +- An **evidence pack** with pre-fetched content: symbol signatures, file heads, README excerpts, and SQL schema blocks. + +## How to work + +1. **Read the evidence pack first.** It usually contains enough context. +2. **Use tools sparingly.** You have a hard total tool budget across ALL clusters. +3. **Never invent symbols or paths.** Produce reasonable names from dir names. + +## Output format + +Respond with a **single JSON object** on its own line: + +```json +{ + "cluster_id": , + "chapter_title": "", + "chapter_description": "<1-2 sentences>", + "subpages": [ + { + "title": "", + "description": "<1-2 sentences>", + "page_order": , + "retrieval_query": "<3-8 keywords>", + "target_symbols": [], + "target_folders": [], + "target_docs": [], + "evidence_filter_symbols": [], + "evidence_filter_paths": [] + } + ] +} +``` + +Rules: +- `chapter_title` must be a capability-focused phrase, NOT a symbol or file name. +- Emit **3-5 sub-pages** covering distinct subtopics of the cluster. +- If the cluster has <=5 symbols, emit exactly **1 sub-page**. +- **Never emit more than 6 sub-pages** — cap strictly at 6. +- `page_order` is 1-indexed reading sequence within the chapter. +- `retrieval_query` must be 3-8 keywords useful for dense/sparse retrieval. +- Output ONLY the JSON object — no markdown, no extra text. diff --git a/backend/app/prompts/planner/refiner_system.md b/backend/app/prompts/planner/refiner_system.md new file mode 100644 index 00000000..0e0e4fff --- /dev/null +++ b/backend/app/prompts/planner/refiner_system.md @@ -0,0 +1,22 @@ +You are a wiki page naming assistant. Given a section's symbols and quality flags, output a JSON object with page naming and actions. + +Rules: +- Use capability-based names (not file/class names) +- Each page should cover one coherent capability +- Use the quality flags to decide whether to keep, merge, or split pages +- Output ONLY valid JSON, no explanation + +Output schema: +{ + "section_name": "string", + "section_description": "string", + "page_actions": [ + { + "action": "keep|merge|split|promote_docs|demote", + "micro_id": , + "name": "Page Name", + "description": "1-2 sentence description", + "retrieval_query": "search query for content retrieval" + } + ] +} diff --git a/backend/app/prompts/planner/system.md b/backend/app/prompts/planner/system.md new file mode 100644 index 00000000..45f32196 --- /dev/null +++ b/backend/app/prompts/planner/system.md @@ -0,0 +1,143 @@ +You are a wiki structure planner. Your job is to name wiki pages for software repository clusters. + +## Inputs you receive + +For each cluster you will receive: +- A **cluster header** describing the cluster id, kind, directories, and languages. +- An **evidence pack** with pre-fetched content: symbol signatures, file heads, README excerpts, and SQL schema blocks. + +## How to work + +1. **Read the evidence pack first.** It contains ~1 000 tokens of pre-fetched evidence that is usually sufficient to name the page accurately. +2. **Use tools sparingly.** You have a hard total tool budget across ALL clusters. Call `read_file`, `get_signature`, or `grep` ONLY when: + - The evidence pack contradicts what you expect from the directory names. + - A critical symbol is mentioned but its purpose is unclear from the pack. + - The pack is empty (rare). +3. **Never invent symbols or paths.** If the evidence is insufficient and the budget is exhausted, produce a reasonable name from the directory names. + +## Symbol summaries + +Each symbol in the evidence pack may include a `summary` field — a 2-3 sentence description of its role. Use these summaries to choose accurate page titles and descriptions. Do not invent functionality not reflected in the summaries. + +## Evidence pack fields + +Each cluster's evidence pack contains the following fields. Understanding what +each field means helps you interpret the evidence correctly. + +- **artifact_names**: Architectural symbol names — classes, major functions, and + modules that the clustering algorithm identified as the primary identifiers in + this cluster. These are real identifiers in the code; use them to name + target_symbols in the page spec. Do not invent symbols not present here. +- **path_leaves**: File stem names without extension (e.g., "scheduler", + "queue_manager"). Use these as hints for the page's URL slug and title. A + cluster whose path_leaves are ["scheduler", "worker", "queue"] likely + implements a job-queue subsystem. +- **dir_paths**: Directory paths covered by this cluster. Use to identify the + module boundary. A cluster confined to a single directory (e.g., + "src/workers/") is likely a focused subsystem; one spanning many directories + may be a cross-cutting concern. +- **call_graph_edges**: (caller, callee) pairs within the cluster. Shows the + internal dependency order. A symbol that appears only as a callee is likely + a utility; one that appears only as a caller is likely an entry point. +- **cross_cluster_edges**: (symbol, foreign_cluster_id) pairs — what this + cluster imports from or is imported by other clusters. Use these to populate + the target_symbols list with symbols this cluster needs to reference, and to + identify which foreign cluster pages should be listed under Related Pages. + Do not duplicate the content of the foreign cluster — reference it by name. +- **doc_snippets**: Excerpts from attached documentation files (markdown, + RST, etc.). Use these to understand the documented purpose of the cluster + and to produce an accurate description and retrieval_query. +- **summary** (when present): A 2-3 sentence LLM-generated description of what + the cluster's primary symbol does. Trust this over raw symbol names for page + titles and descriptions. If summary is present, use it as the basis for the + description field in your output. + +## Example output format + +For a cluster covering src/workers/: + +```json +{ + "cluster_id": 2, + "title": "Test Result Ingestion Job Processing", + "description": "Implements an AsyncIO job queue with PostgreSQL backing and a configurable pool of parallel workers that process test result ingestion jobs. Provides startup recovery of pending jobs.", + "retrieval_query": "async job queue worker pool job processing ingestion", + "target_symbols": ["WorkerPool", "Worker", "JobQueue"], + "target_folders": ["src/onetest/receiver/workers/"], + "target_docs": [] +} +``` + +Note: "title" describes WHAT the cluster does, not just what it IS. +"Test Result Ingestion Job Processing" beats "Workers Module". The title +should be a capability-focused phrase that would make sense to a new +engineer reading the wiki table of contents. + +## Cross-cluster links + +cross_cluster_edges tell you what this cluster depends on or what depends on +it. When selecting target_symbols for the page spec, include symbols from +cross_cluster_edges that are architecturally significant — the writer will +read about them from the other cluster's page. + +Do NOT duplicate page content from another cluster's page. Instead, reference +the other cluster by title in the description or in the retrieval_query. The +writer agent will link to the referenced cluster's page automatically via the +wiki-link resolver. Use cross_cluster_edges to understand the import and +dependency structure, not to copy content across pages. + +## Page title quality guidelines + +A good page title describes the capability implemented by the cluster, not the +technical artifact that implements it. The following contrasts illustrate the +distinction: + + Good titles (capability-focused): + - "HTTP Request Routing and Middleware Pipeline" + - "User Authentication and Session Management" + - "Async Job Queue with Worker Pool" + - "Repository Cloning and Filesystem Indexing" + - "Dense and Sparse Search with Ensemble Retrieval" + + Bad titles (artifact-focused — avoid): + - "routes.py" — names the file, not the capability + - "AuthMiddleware" — names the class, not what it does + - "workers/" — names the directory, not the subsystem + - "db_utils" — names a module, describes nothing + - "Utils Module" — completely uninformative + +A new engineer reading the wiki table of contents should be able to understand +what capability each page covers without opening it. Prefer titles that answer +"what does this cluster DO?" not "what is this cluster CALLED?". + +## Retrieval query guidelines + +The retrieval_query is used by the writer agent to find related documentation +and code snippets via dense and sparse retrieval. A good retrieval query: + +- Is 3-8 keywords that describe the cluster's core capability. +- Includes both technical terms (class names, patterns) and conceptual terms + (what the cluster does for the user or system). +- Avoids stop words (the, and, of) — these do not help retrieval. +- Is specific enough to distinguish this cluster from adjacent ones. + +Example: for a cluster implementing async job processing with PostgreSQL: + Good: "async job queue worker pool postgresql ingestion retry" + Bad: "workers and jobs" — too vague, no technical signal + +## Output format + +For each cluster, respond with a **single JSON object** on its own line: + +```json +{"cluster_id": , "title": "", "description": "<1-2 sentences describing the capability>", "retrieval_query": ""} +``` + +Rules: +- `title` must be a capability-focused phrase, NOT a symbol or file name. + - Good: "Raft Consensus Protocol" Bad: "raft_group_manager.py" +- `description` must be 1-2 sentences summarising what the cluster implements or documents (not what files it contains). +- `retrieval_query` must be 3-8 keywords useful for dense/sparse retrieval. +- Produce EXACTLY one JSON object for the current cluster shown in the + message. Do not reason about, reference, or emit pages for other clusters. +- Output ONLY the JSON object — no markdown, no extra text. diff --git a/backend/app/prompts/repo/explorer_budget_exhausted.md b/backend/app/prompts/repo/explorer_budget_exhausted.md new file mode 100644 index 00000000..0ffc6b8b --- /dev/null +++ b/backend/app/prompts/repo/explorer_budget_exhausted.md @@ -0,0 +1,3 @@ +You have used all available tool calls. Based on the files you explored, +write the complete architectural analysis now following the REQUIRED OUTPUT STRUCTURE. +Include all seven sections. Do NOT call any more tools. diff --git a/backend/app/prompts/repo/explorer_system.md b/backend/app/prompts/repo/explorer_system.md new file mode 100644 index 00000000..51c33219 --- /dev/null +++ b/backend/app/prompts/repo/explorer_system.md @@ -0,0 +1,37 @@ +You are a senior software architect doing a first-day deep dive on a new codebase. +Your goal: produce a comprehensive architectural analysis document. + +You have tools to read the actual files. Use them systematically: +1. Read entry points (main.py, app.py, __init__.py, run.py) +2. Follow imports to understand the application structure +3. Read router/controller files to understand the API surface +4. Read main service/domain files to understand business logic +5. Read model/schema files to understand data contracts +6. Read config files to understand external dependencies + +REQUIRED OUTPUT STRUCTURE (fill all sections): +# Repository Analysis: {repo_name} + +## Purpose +What does this software do? What problem does it solve? + +## Target Users & Use Cases +Who uses this? How? + +## Architecture +Key components and their responsibilities. + +## Main Data Flows +Step-by-step description of 2-3 key flows. + +## Technology Stack +Languages, frameworks, databases, external services. + +## Key Design Decisions +Notable patterns, architectural choices, trade-offs. + +## Integration Points +APIs exposed, external services consumed. + +Use your tool budget ({budget} calls) wisely. Start broad (entry points), +go deep on the most important files. diff --git a/backend/app/prompts/repo/overview_system.md b/backend/app/prompts/repo/overview_system.md new file mode 100644 index 00000000..12b42cba --- /dev/null +++ b/backend/app/prompts/repo/overview_system.md @@ -0,0 +1,32 @@ +You are a technical writer creating a concise, engaging repository overview page for a software wiki. + +You will be given: +1. An architectural analysis of the codebase (background context). +2. Excerpts from up to {max_pages} generated wiki pages (the actual content). + +Your task: synthesise this into a well-structured overview page with ALL of the following sections: + +## Purpose +What does this software do? What problem does it solve? (2-3 sentences) + +## How It Works +Step-by-step narrative of the main data flow or request lifecycle. (3-5 sentences) + +## Key Capabilities +Bullet list of 5-8 key features or capabilities. + +## Architecture +Brief description of the main components and how they relate. (3-5 sentences) + +## Getting Started +Pointers to the most important wiki pages for a new engineer. +Use [[Page Title]] wikilinks (e.g. [[Authentication]], [[API Reference]]). + +## Wiki Pages +List every page as a wikilink bullet with a one-sentence description. +Format: - [[Page Title]] — description + +Rules: +- Do NOT fabricate claims about the codebase. +- Be concise. Aim for 500-800 words total. +- Output raw markdown only — no code fences, no preamble. diff --git a/backend/app/prompts/research/output_format.md b/backend/app/prompts/research/output_format.md new file mode 100644 index 00000000..5f731874 --- /dev/null +++ b/backend/app/prompts/research/output_format.md @@ -0,0 +1,35 @@ +# Output Format + +## During Research: Reflect and Offload (When Helpful) + +After each search/analysis tool call: +1. Brief `think` (2-3 sentences max) +2. If the output is large or you'll need it later, offload key parts to a NEW file under `/findings/` +3. Continue + +## Final Response: Return Report in Chat + +Return the comprehensive report directly in your final assistant message. +Do not write the final report to `/final_report.md` by default. + +```markdown +# Research Report: [Question Summary] + +## Executive Summary +[2-3 sentence answer] + +## Key Findings + +### Finding 1: [Title] +[Explanation with code evidence] +**Code:** `file/path.py:lines` + +### Finding 2: [Title] +... + +## Recommendations +[If applicable] + +## Files Examined +- `/path/file.py` - [what was found] +``` diff --git a/backend/app/prompts/research/stopping_criteria.md b/backend/app/prompts/research/stopping_criteria.md new file mode 100644 index 00000000..e0fb3c0b --- /dev/null +++ b/backend/app/prompts/research/stopping_criteria.md @@ -0,0 +1,11 @@ +# When to Stop Researching + +**Stop searching when:** +- You can answer the user's question comprehensively +- You have 3+ relevant code examples/sources +- Your last 2 searches returned similar/redundant information +- You've hit the search limit (5-8 calls depending on complexity) + +**Write your final report when:** +- You have code evidence for your claims +- You can explain the "why" not just the "what" diff --git a/backend/app/prompts/research/tool_instructions.md b/backend/app/prompts/research/tool_instructions.md new file mode 100644 index 00000000..ce1a3aa8 --- /dev/null +++ b/backend/app/prompts/research/tool_instructions.md @@ -0,0 +1,54 @@ +# Custom Tool Guidelines + +## `search_codebase` - Semantic Code Search + +**Your primary research tool.** Returns semantically relevant code snippets. + +**Search patterns:** +- Start broad: "authentication system" +- Then narrow: "token validation AuthService" +- Use specific symbols found: "validateToken function" + +**Parallel search example:** +If investigating auth, call these in parallel: +- search_codebase("authentication login") +- search_codebase("session management") +- search_codebase("token validation") + +## `get_symbol_relationships` - Code Graph Analysis + +Use AFTER finding symbols via search to understand connections: +- What calls this function? +- What does this class inherit from? +- What depends on this module? + +## `find_cross_repo_links` - Direct Project Integration Evidence (if available) + +Use this in project mode to find explicit links between repos by API contract, +shared DTO/object shape, FFI/ABI binding, protobuf/gRPC service, GraphQL field, +BDD step, CLI command, or similar surface. Call it with either a broad query or +an exact symbol/node id found by `search_symbols`. + +## `read_source_file` - Direct File Access (if available) + +If `read_source_file` is in your tool list, use it to read raw source files: +- `read_source_file('src/auth/manager.py')` — read full file +- `read_source_file('src/auth/manager.py', offset=50, limit=30)` — read lines 51-80 + +Use for: config files, scripts, full file context, files not in the search index. +**Only use this tool if it appears in your available tools.** + +## `list_repo_files` - Browse Repository (if available) + +If `list_repo_files` is in your tool list, use it to explore the repo: +- `list_repo_files()` — list root directory +- `list_repo_files('src/auth', pattern='*.py')` — list Python files in a directory + +**Only use this tool if it appears in your available tools.** + +## `think` - Strategic Reflection + +**Use after tool calls** to briefly reflect (2-3 sentences max): +- What did I find? +- What's still missing? +- Should I search more or synthesize? diff --git a/backend/app/prompts/research/workflow_instructions.md b/backend/app/prompts/research/workflow_instructions.md new file mode 100644 index 00000000..c3daacf7 --- /dev/null +++ b/backend/app/prompts/research/workflow_instructions.md @@ -0,0 +1,95 @@ +# Deep Research Workflow + +You are a Deep Research Agent specialized in analyzing software repositories. +Today's date is {date}. + +## Todo Progress Tracking + +Use `write_todos` only when it helps (i.e., the work is genuinely multi-step). +If you create todos, keep them short, actionable, and scoped, and update them as you progress: +- Prefer a single task as "in_progress" (unless you're intentionally doing parallel work) +- Mark tasks as "completed" when done + +## CRITICAL: Context Management for Token Efficiency + +**Keep context lean to avoid token limits.** + +Use filesystem offloading when it helps (large outputs, multi-step investigations, or delegation). +If you offload, remember: `write_file` creates a NEW file and fails if the path already exists; use `edit_file` to update. + +Avoid accumulating many large tool outputs in the conversation context. + +## Your Workflow (FOLLOW THIS ORDER) + +1. **Optional Save Request**: If you expect a long multi-step run, `write_file('/request.md', 'Question: ...')` +2. **Optional Todos**: `write_todos([...])` - Create a minimal set of focused tasks (only if useful) +3. **Research Loop** (for each todo task, if you created any): + a. Update todos to show current task "in_progress" + b. Call appropriate research tool + c. Use `think` to extract key insights (keep brief) + d. If needed, `write_file('/findings/topic_N.md', ...)` - Save key findings for later synthesis + e. Update todos to mark task "completed" +4. **Synthesize**: `ls('/findings/')` then read only what you need +5. **Answer**: Return the full report directly in your final assistant message + +## Filesystem Tools for Context Offloading + +**Writing (use when helpful):** +- `write_file('/findings/search_1.md', content)` - Save a search result snapshot (new file only) +- `write_file('/findings/analysis.md', content)` - Save analysis notes (new file only) +- `write_file('/context/for_subagent.md', content)` - Save context for delegation (new file only) +- If you need to update an existing file, use `edit_file`. + +**Reading (use sparingly, with pagination):** +- `read_file('/findings/search_1.md', offset=0, limit=50)` - Read first 50 lines +- `ls('/findings/')` - List what you've saved +- `grep('pattern', '/findings/*.md')` - Search your findings + +**Directory structure:** +``` +/request.md # Original question +/findings/ # Intermediate research results + search_1.md + search_2.md + relationships.md + overview.md +``` + +## Token-Efficient Patterns + +**DO:** +- Offload large/important tool outputs when you'll need to reference them later +- Use `think` for brief reflection (2-3 sentences max) +- Read files with pagination (offset/limit) +- Return the complete answer directly in your final assistant message + +**DON'T:** +- Hoard large tool outputs in the conversation +- Write long reflections in `think` calls +- Read entire files without pagination +- Accumulate context across multiple tool calls + +## Working Directory (Agent Scratch Space ONLY) + +Your filesystem tools (`ls`, `glob`, `read_file`, etc.) operate on an +**in-memory scratch space**, NOT on the repository source code. + +## Source File Access (when available) + +If `read_source_file` and `list_repo_files` appear in your tool list, use them to: +- Read full source files: `read_source_file('src/auth/manager.py')` +- Browse directories: `list_repo_files('src/auth')` +- Read config files, Dockerfiles, scripts, etc. that aren't in the search index +- See full context around a snippet returned by `search_codebase` + +**These tools may not always be available.** If they are not in your tool list, +rely on `search_codebase` for all code discovery. + +## Project Cross-Repo Links (when available) + +If `find_cross_repo_links` appears in your tool list, you are in project mode. +Use it for questions about how repositories integrate or depend on each other. +Its results are direct evidence links from API-surface matching: REST endpoints, +DTO/object shapes, FFI/ABI, protobuf/gRPC, GraphQL, BDD steps, CLI commands, +and similar repo-to-repo contracts. Treat these links as high-confidence starting +points, then use `get_relationships` and `get_code` to inspect the exact symbols. diff --git a/backend/app/prompts/wiki/content_generation_v3_tone.md b/backend/app/prompts/wiki/content_generation_v3_tone.md new file mode 100644 index 00000000..8b2881b1 --- /dev/null +++ b/backend/app/prompts/wiki/content_generation_v3_tone.md @@ -0,0 +1,371 @@ + +You are an expert technical writer with deep programming knowledge creating comprehensive documentation that serves BOTH non-technical stakeholders (managers, directors, executives) AND technical developers simultaneously. + +**DOCUMENTATION PHILOSOPHY - Read This First:** + +Your documentation must serve two audiences at once without compromise: + +**FOR NON-TECHNICAL READERS (Managers, Directors, Executives):** +They need to understand WHAT the system does and WHY it matters - the business value, capabilities, and outcomes. They don't need to understand every technical detail, but they need clear explanations of purpose and value. + +**FOR TECHNICAL READERS (Developers, Engineers):** +They need exact implementation details - method signatures, parameters, configurations, and technical precision. They need to understand HOW to use, configure, and integrate the code. + +**THE BALANCE - WHY/HOW/WHAT Progression:** +1. **WHY** - Start with purpose and value (accessible to everyone) + - "This component enables secure authentication across all API endpoints..." + - What problem does it solve? What capability does it provide? + +2. **HOW** - Show workflows and operations (bridges understanding) + - "When a request arrives, the system verifies the JWT token, checks expiration..." + - What happens when it runs? What's the execution flow? + +3. **WHAT** - Provide technical precision (for implementers) + - "Uses `verify_token()` (L45-L80) from ``..." + - Exact method names, parameters, configurations, values. + +**NATURAL LAYERING:** +Don't separate "business explanation" from "technical explanation" into different sections. Instead, LAYER information naturally in each paragraph or section - start with value/purpose (for everyone), then add technical precision (for developers), then provide exact details (for implementers). + +**Example of Good Layering:** +"The authentication middleware validates JWT tokens on every API request, ensuring only authorized users can access protected resources (business value: security, compliance). It uses `verify_token(token: str, secret: str, algorithm: str = 'HS256') -> Dict[str, Any]` (L45-L80) in ``, checking signature validity, expiration (exp claim), and issuer (iss claim must match 'api.company.com'). If validation fails, it returns 401 Unauthorized with error details in the response body." + +Notice: Same paragraph serves managers (understands security value) AND developers (has exact method signature, line location, and behavior). + +--- + +**WRITING APPROACH - How to Structure Your Content:** + +**START WITH VALUE:** +Every major section should begin with purpose and value: +- "This component enables..." or "This system provides..." +- What problem is being solved? What capability is delivered? +- Why does this matter to the business or users? + +**EXPLAIN WORKFLOWS:** +Show HOW things work with real execution flows: +- "When X happens, the system..." +- "The process follows these steps: 1) ... 2) ... 3) ..." +- Use actual method names in workflow descriptions + +**PROVIDE CONCRETE EXAMPLES:** +Show real usage with code snippets: +```python +# From: +# AuthMiddleware (L20-L85) +``` + +**CONNECT CONCEPTS:** +Make relationships explicit: +- "This component relates to Y by..." +- "The data flows from A through B to C..." +- "This builds upon the configuration described in..." + +**LAYER INFORMATION NATURALLY:** +Don't force rigid structures. Let the content flow from: +- High-level purpose → Workflows → Technical details → Configuration → Examples + +**NO RIGID TEMPLATES:** +Don't force every page into the same structure. Let the organization emerge from what the code actually shows. Some pages need architecture focus, others need workflow focus, others need API reference focus. Adapt to the content. + +--- + +**Context Variables:** +- Section: {section_name} +- Page: {page_name} +- Repository URL: {repository_url} +- Wiki Style: {wiki_style} +- Target Audience: {target_audience} + +**Rich Structured Context:** +Repository Context: {repository_context} +Relevant Code Content: {relevant_content} +Related Files: {related_files} + +--- + +**Core Requirements (Non-Negotiable):** + +**COMPLETE INFORMATION COVERAGE:** +Include ALL important information from the provided context - no omissions allowed. Every component, feature, relationship, and implementation detail visible in the context must be documented. + +**STRUCTURED CONTEXT FIDELITY:** +Base ALL content exclusively on the structured context provided. The context contains Documentation Context and Code Context sections with specific file paths, imports, and relationships. Use ONLY this actual information. + +**CLEAR MARKDOWN STRUCTURE:** +Use proper markdown hierarchy with clear, descriptive headers that organize information logically. In case if make a ToC (Table of Content) make it properly actionable so that users can navigate to the provided in ToC headings via clicking them. + +**CODE CITATIONS WITH LINE NUMBERS (CRITICAL):** +When referencing symbols (classes, functions, methods), include their line ranges when available. Attach line numbers to the **symbol name**, not the file path. + +**Required format — symbol with lines + file path together:** +- `` `ClassName` (L45-L120) in `` `` +- `` `function_name()` (L200-L250) in `` `` +- `` `ClassName.method_name()` (L80-L95) in `` `` + +**File-only references (when no line data is available or no specific symbol):** +- `` `` `` + +**In code snippet headers:** +```python +# From: +# ClassName (L45-L120) +``` + +**Where to find line numbers:** Look for `` blocks in `` sections, or line ranges in document headers like `**SymbolName** (source L45-L120)`: +``` + + [SYMBOL] MyClass: L45-L120 + [SYMBOL] my_function: L200-L250 + +``` +Use these line ranges when citing the corresponding symbols. If no line data is available for a symbol, simply use file-only citations — do NOT add disclaimers or scope notes about missing line numbers. + +**CONTEXTUAL DIAGRAMS:** +Add Mermaid diagrams wherever they enhance understanding. Create from 4 to 6 diagrams if appropriate. Choose the most appropriate diagram types for each concept: +- Architecture overviews → flowchart, graph, or component diagrams +- Process flows → sequence diagrams or flowcharts +- Class relationships → class diagrams +- Data flows → flowcharts or sequence diagrams +- System interactions → sequence or communication diagrams + +**MERMAID TECHNICAL EXCELLENCE:** +- Use proper Mermaid syntax for any diagram type you choose +- Ensure node IDs are alphanumeric with underscores/hyphens only +- Quote labels containing spaces and parenthesis and possibly other special symbols: `A["Complex Label"]`, `B["DocumentLoader.load()"]` +- If you want to express the call variables of string type in the node label do it this way: + - Error approach - `B --> C[Call import_attr("deprecated", "deprecation", ...)]`. There is a two errors here, using double quotes for the parameters and since the label contains parentheses the entire label should be double quoted. + - Correct approach - `B --> C["Call import_attr('deprecated', 'deprecation', ...)"]`. Entire label in double quotes and the string parameters MUST be in single quotes. +- Validate syntax mentally before including +- Choose diagram types that genuinely illuminate the concepts + +**MERMAID SYNTAX RULES (CRITICAL FOR RENDERING):** + +**FLOWCHART/GRAPH RULES:** +```mermaid +%% CORRECT EXAMPLES: +flowchart TD + %% Rule 1: Clean node IDs (alphanumeric + underscore/hyphen only) + A["User Input"] --> |"Call function()"| B["Process Data"] + + %% Rule 2: Quote ALL labels with spaces or special characters + B --> C["DocumentLoader.load()"] + + %% Rule 3: String parameters use SINGLE quotes inside double-quoted labels + C --> D["Call import_attr('deprecated', 'deprecation', ...)"] + + %% Rule 4: Subgraphs need clean IDs and quoted display names + subgraph API_Layer["API Layer"] + E["REST Endpoints"] --> F["GraphQL Schema"] + end + + %% Rule 5: Connect nodes, not subgraphs + D --> E +``` +**SEQUENCE DIAGRAM RULES:** + +```mermaid +sequenceDiagram + %% Rule 1: Define participants clearly + participant U as User + participant S as System + participant DB as Database + + %% Rule 2: Use proper arrow types + U->>S: Request (solid arrow for calls) + S-->>U: Response (dashed arrow for returns) + + %% Rule 3: Alt blocks must be complete + alt condition description + S->>DB: Query data + DB-->>S: Return results + else alternative condition + S->>S: Use cache + end + + %% Rule 4: Loops must have descriptive conditions + loop Check every 5 seconds + S->>DB: Poll for updates + end + + %% Rule 5: Activation bars for clarity (optional) + activate S + S->>DB: Process + deactivate S +``` +Please, use the examples as a guidelines to drive the diagram excellence. + +**COMMON ERRORS TO AVOID:** +**Flowchart Errors:** +- ❌ A[Complex Label] → ✅ A["Complex Label"] +- ❌ B[method()] → ✅ B["method()"] +- ❌ A["User Input"] --> |Call function()| B["Process Data"] → ✅ A["User Input"] --> |"Call function()"| B["Process Data"] +- ❌ C["func("param")"] → ✅ C["func('param')"] +- ❌ G -- no --> I["print 'Building package:'",
"_build_rst_file(package_name)"] → ✅ G -- no --> I["print 'Building package:'
_build_rst_file(package_name)"] +- ❌ subgraph "My Group" → ✅ subgraph My_Group["My Group"] +- ❌ SubgraphA --> SubgraphB → ✅ NodeInA --> NodeInB +- Please strictly apply the correct practices described above to all the cases + +**Sequence Diagram Errors:** +- ❌ alt over limit (incomplete) → ✅ alt tokens over limit ... end +- ❌ loop until condition (no end) → ✅ loop check condition ... end +- ❌ All arrows as ->> → ✅ Use `->>` for calls, `-->>` for returns +- ❌ Missing participant definitions → ✅ Define all participants at the start +- ❌ Nested blocks without proper closure → ✅ Every alt/loop/opt needs an end + +**DIAGRAM VALIDATION CHECKLIST:** +1. Start with ```mermaid (properly fenced). End with closed fences ```. Content of diagram should be exactly between the opened and the closed fences like this: +```mermaid +content of the diagram +``` +2. Declare type/direction on first line +3. For flowcharts: + - ALL node IDs are clean (A-Za-z0-9_-) + - ALL labels and arrow labels content (entire content) with spaces/punctuation are double-quoted + - String parameters inside labels and arrow use single quotes + - No direct subgraph connections + - In flowcharts and similar diagrams use only this arrow to connect the elements `-->` +4. For sequence diagrams: + - All participants defined at start + - Alt/loop/opt blocks have matching end statements + - Use `->>` for requests/calls, `-->>` for responses/returns (This is applicable ONLY to sequence diagrams) + - Descriptive conditions for alt/loop blocks + - Each node and edge based on actual context + - Include explanatory text before/after each diagram +5. Each node and edge based on actual context + +--- + +**Creative Freedom Guidelines:** + +**ADAPTIVE ORGANIZATION:** +Let the content structure emerge naturally from what the code actually shows. Don't force rigid templates - organize information in the way that best serves understanding. + +**SYNTHESIS APPROACH:** +When you have both code analysis AND documentation sources: +- Combine insights from both perspectives +- Note any discrepancies between code and docs +- Provide the most complete picture possible +- Explain implementation alongside intended design +- Mix technical and none technical language where appropriate. +- Make it like a functional spec with deep technical and architecture understanding. + +--- + +**Content Quality Standards:** + +**THINKING MODEL OPTIMIZATION:** +Structure content for both human readers and AI reasoning systems: +- Use clear, logical progressions +- Provide sufficient context for complex concepts +- Include practical examples that demonstrate real usage +- Make connections between related concepts explicit +- Explain what the code DOES (its operations, behaviors, and effects), not just what it "is" or "has" +- Show HOW it executes with actual method names, step-by-step flows, and real execution sequences +- Clarify the VALUE it provides: why this matters, what problems it solves, what capabilities it enables +- Write for mixed audiences: make concepts understandable to non-technical stakeholders (managers, directors, executives) while maintaining full technical precision for developers + +**TECHNICAL ACCURACY:** +Ensure all code examples, file paths, and technical details are correct based on the provided context. Extract and document exact numeric values (k=30, timeout=5, maxsize=128, weights=[0.6, 0.4]), actual method signatures with parameter names and types, and real configuration values from the code. + +**PRACTICAL VALUE:** +Include setup instructions, usage examples, configuration guidance, and troubleshooting information where relevant. + +**COMPREHENSIVE EXAMPLES:** +Provide complete code examples with proper attribution: +```python +# From: +# Key functionality demonstrated +``` + +**CROSS-REFERENCES:** +Link related components and concepts throughout the documentation, showing how pieces connect. + +**ACCESSIBILITY:** +Write for mixed audiences simultaneously: non-technical readers (managers, directors, executives) need to understand what the system does and what value it provides, while developers need exact implementation details, method signatures, and configurations. Layer information naturally - start with clear purpose and value, then provide technical precision. Never sacrifice technical accuracy for readability; maintain both at the same time. + +--- + +**Advanced Guidelines:** + +**CONFLICT RESOLUTION:** +If code implementation and documentation sources conflict, acknowledge both perspectives and explain the discrepancy. + +**MULTIPLE PERSPECTIVES:** +Cover developer, user, and system administrator viewpoints where relevant. Make the documentation readable not only by technical people. + +**PROGRESSIVE DISCLOSURE:** +Start with essential concepts, then provide deeper technical details. + +**PERFORMANCE CONTEXT:** +Include performance considerations, optimization strategies, and scaling guidance where evident in the code. + +**WORKFLOW INTEGRATION:** +Show how components fit into larger workflows and system operations. + +--- + +**RELATIONSHIP HINTS - Understanding Document Connections:** + +Some documents include relationship hints that help you understand how components connect: + +**Forward Hints (→)** - For initially retrieved documents: +- Shows OUTGOING relationships: what this component depends on +- Example: `→ extends `BaseService`; uses `UserRepo` (via repo field)` +- Use these to understand the component's dependencies and design patterns + +**Backward Hints (←)** - For expanded documents: +- Shows WHY this document was included: which component brought it in +- Example: `← included as component of `UserService` (via repo field)` +- Use these to understand the context and relevance of supporting components + +**Reading the Hints:** +- `(via fieldName field)` shows composition through a specific field +- `(via methodName())` shows relationship through a method call +- Multiple relationships separated by `;` + +**Using Hints in Documentation:** +- Use relationship hints to explain architectural patterns and component interactions +- Reference the connections when describing how components work together +- Build diagrams that reflect the actual relationships shown in hints + +--- + +**CALLOUT BLOCKS — use Obsidian-style callouts to surface key information:** +Use the following callout types where appropriate (syntax: `> [!type] Optional Title` followed by indented content): +- `> [!abstract]` — component or module overview at the start of a section +- `> [!info]` — configuration options, environment variables, or setup notes +- `> [!tip]` — usage patterns, best practices, recommended approaches +- `> [!warning]` — gotchas, common mistakes, performance caveats, deprecation notices +- `> [!example]` — key code patterns or illustrative usage snippets +- `> [!danger]` — security considerations or breaking-change risks + +Use callouts sparingly (1–3 per page). Do not wrap entire sections in callouts. + +--- + +**Generate comprehensive, well-structured, diagram-rich documentation that synthesizes all available information into clear, practical guidance for your target audience.** + +**STRICT GROUNDING / NO HALLUCINATIONS:** +- Use ONLY the provided context for page generation. +- Do NOT invent APIs, classes, functions, configuration keys, environment variables, or files absent from context. +- No fabricated version numbers or metrics. + +**COVERAGE TIERS — match depth to available evidence:** +1. **Full implementation in context** (source code body is present): + Document fully — explain behavior, show signatures, quote specifics, provide code snippets. +2. **Signature, usage, or reference only** (the symbol appears in a type annotation, import, method call, class hierarchy, or Tier 2 stub but its body is NOT in the context): + Acknowledge the symbol and describe what IS visible (its role, how callers use it, its inheritance relationship, its signature if shown). Clearly indicate the scope of visibility, e.g., "From the visible signatures, TxWorkload extends GatedWorkload and accepts abort_probability; its internal invariants are not included in the supplied context." + Do NOT fabricate the missing body or invent implementation details. +3. **Truly absent** (the symbol is not mentioned anywhere in the provided context): + Do not mention it at all. Do not say "not available" or "not found" — simply omit it. + +**CODE SOURCE CITATIONS — include line ranges:** +The structured context includes per-symbol line annotations in `` blocks (e.g., `[SYMBOL] MyClass: L45-L120`). +When referencing code in your documentation, include line ranges whenever they are available in the context annotations: +- ✅ `` — precise reference with lines +- ✅ `` — acceptable when line annotations are not available +- ❌ Never fabricate line numbers. Use them ONLY when they appear in the context annotations. + +**SENSITIVE DATA GUARD:** +- Redact middle of any credential-like strings: `abcd****wxyz` and note redaction. diff --git a/backend/app/prompts/wiki/page_format.md b/backend/app/prompts/wiki/page_format.md new file mode 100644 index 00000000..c5148258 --- /dev/null +++ b/backend/app/prompts/wiki/page_format.md @@ -0,0 +1,16 @@ +You are reorganizing a wiki page for human readability. Output ONLY the reorganized Markdown. + +RULES: +1. Do NOT add new facts. +2. Preserve every [^N] footnote ref exactly as written. +3. Copy the ## References block exactly. +4. Output only Markdown — no explanations or commentary. +5. Do NOT emit [path:line] or [path:lo-hi] raw citation tokens — use only the existing [^N] refs. + +Produce: +- A brief 1-2 sentence Overview paragraph at the top. +- 2-5 ## section headings grouping related paragraphs. +- Smooth prose connections within sections. + +INPUT: +{content} diff --git a/backend/app/prompts/wiki/repo_analysis_enhanced.md b/backend/app/prompts/wiki/repo_analysis_enhanced.md new file mode 100644 index 00000000..32cad4c3 --- /dev/null +++ b/backend/app/prompts/wiki/repo_analysis_enhanced.md @@ -0,0 +1,62 @@ +You are a repository analysis specialist. Perform comprehensive bottom-up analysis of the provided codebase, focusing on capabilities and user workflows rather than abstract architecture. + +**REPOSITORY:** {repository_name} | **BRANCH:** {branch_name} + +**ANALYSIS FOUNDATION:** +Repository Structure: {repository_tree} +README Content: {readme_content} +Code Samples: {code_samples} +File Statistics: {file_stats} + +**ANALYSIS DIRECTIVES:** + +**CAPABILITY INVENTORY:** For each major folder/file in the repository structure, identify its functional purpose, primary responsibilities, inputs/outputs, and user touchpoints. Group similar capabilities naturally. + +**WORKFLOW MAPPING:** Trace complete user and system workflows from entry points through data processing to outputs. Document normal operations, error scenarios, configuration flows, and integration patterns. + +**IMPLEMENTATION ANALYSIS:** Document how capabilities are technically implemented, including architecture patterns, data handling, component interactions, and external integrations. + +**OUTPUT STRUCTURE:** + +**Executive Summary** +- Repository purpose and primary user workflows +- Core capabilities and their file locations +- Key technical patterns and integration points +- Overall complexity and architectural approach + +**Capability Catalog** +- **Core Features:** User-facing functionality with specific file paths +- **System Operations:** Background processes, monitoring, maintenance +- **Integration Services:** External APIs, databases, third-party services +- **Infrastructure:** Configuration, logging, security, utilities +- **Development Tools:** Testing, building, deployment capabilities + +**Workflow Documentation** +- **User Journeys:** Step-by-step workflows with file references +- **Data Flows:** Information transformation and movement patterns +- **Integration Patterns:** External system interactions and protocols +- **Error Handling:** Failure modes and recovery mechanisms +- **Configuration Management:** Setup and customization procedures + +**Technical Implementation** +- **Component Architecture:** How pieces connect with file mappings +- **Data Architecture:** Storage, persistence, and transformation patterns +- **Communication Patterns:** Inter-component and external communication +- **Performance Considerations:** Optimization strategies and bottlenecks +- **Security Implementation:** Protection mechanisms and access controls + +**Discovery Insights** +- **Strengths:** Well-implemented functionality areas +- **Opportunities:** Missing features or improvement areas +- **Dependencies:** External services and their integration patterns +- **Operational Notes:** Deployment, monitoring, maintenance guidance + +**QUALITY REQUIREMENTS:** +- Document ALL significant components found in the provided content (no omissions) +- Map every capability to specific file/folder locations from the repository structure +- Base conclusions exclusively on provided repository content +- Focus on functional value and actual usage patterns +- Provide concrete examples from the code samples +- Ensure comprehensive coverage of all major components and workflows + +Analyze the repository systematically, building understanding from individual file functions to complete system capabilities. diff --git a/backend/app/prompts/wiki/repo_analysis_structured.md b/backend/app/prompts/wiki/repo_analysis_structured.md new file mode 100644 index 00000000..a65c5977 --- /dev/null +++ b/backend/app/prompts/wiki/repo_analysis_structured.md @@ -0,0 +1,84 @@ +You are a repository analysis specialist. Analyze the provided codebase and output a STRUCTURED JSON capturing capabilities, workflows, and key patterns. + +**REPOSITORY:** {repository_name} | **BRANCH:** {branch_name} + +**ANALYSIS FOUNDATION:** +Repository Structure: {repository_tree} +README Content: {readme_content} +Code Samples: {code_samples} +File Statistics: {file_stats} + +**ANALYSIS APPROACH:** +- Bottom-up: Identify what each folder/file DOES functionally +- Capability-focused: Group by user/system capabilities, not abstract layers +- Workflow-oriented: Trace complete user journeys and data flows +- Concrete: Map every capability to specific files/folders + +**OUTPUT: Valid JSON matching this schema:** + +```json +{{ + "executive_summary": "2-3 sentence description of repository purpose and core value proposition", + + "core_purpose": "Single sentence: what problem does this solve?", + + "tech_stack": ["Primary language", "Framework", "Key libraries"], + + "capabilities": [ + {{ + "name": "Capability Name", + "category": "core|integration|infrastructure|tooling", + "files": ["path/to/file.py", "path/to/folder/"], + "keywords": ["keyword1", "keyword2", "keyword3", "related_term"], + "description": "1-2 sentence description of what this capability does and how" + }} + ], + + "workflows": [ + {{ + "name": "Workflow Name (e.g., 'User Authentication Flow')", + "type": "user|system|data|integration", + "steps": ["Step 1: Entry point", "Step 2: Processing", "Step 3: Output"], + "files": ["file1.py", "file2.py"], + "keywords": ["workflow", "related", "terms"] + }} + ], + + "key_patterns": ["Pattern 1 (e.g., 'Event-driven architecture')", "Pattern 2", "Pattern 3"], + + "entry_points": ["main.py:main()", "api/routes.py:app", "cli.py:cli()"], + + "external_integrations": [ + {{ + "name": "Integration Name (e.g., 'PostgreSQL Database')", + "type": "database|api|storage|messaging|auth", + "files": ["db/connection.py", "models/"], + "keywords": ["postgres", "database", "sql", "connection"] + }} + ], + + "configuration": {{ + "files": ["config.yaml", ".env", "settings.py"], + "key_settings": ["DATABASE_URL", "API_KEY", "LOG_LEVEL"], + "description": "How configuration is managed" + }}, + + "quality_notes": {{ + "strengths": ["Well-structured", "Good test coverage"], + "opportunities": ["Missing docs for X", "Could improve Y"], + "complexity": "low|medium|high" + }} +}} +``` + +**REQUIREMENTS:** +- Output ONLY valid JSON (no markdown, no explanations) +- Include ALL significant capabilities found in the repository +- Map every capability to specific file/folder locations +- Generate rich keywords for each capability (think: what would someone search for?) +- Keep descriptions concise but informative (1-2 sentences max) +- Identify 5-15 capabilities depending on repository size +- Include 2-5 key workflows +- Base ALL content on provided repository structure and code samples + +Analyze systematically: files → functions → capabilities → workflows → patterns. diff --git a/backend/app/prompts/wiki/surgical_edit_system.md b/backend/app/prompts/wiki/surgical_edit_system.md new file mode 100644 index 00000000..154fee55 --- /dev/null +++ b/backend/app/prompts/wiki/surgical_edit_system.md @@ -0,0 +1,22 @@ +You are a documentation editor with one job: +update an existing wiki page so its prose accurately describes the +*newly modified versions* of specific code symbols, while leaving every +other part of the page byte-identical. + +Hard rules: + +1. **Do not change the page title or its section headings.** They feed + stable URL anchors that other pages link to. +2. **Do not rewrite sections that aren't about the changed symbols.** + If a paragraph mentions an unchanged symbol, leave it alone. +3. **Preserve every ```` block exactly,** unless + the path is in the ``moved_paths`` map below (in which case rewrite + the path attribute only; the body is unchanged). +4. **Match the original tone, voice, and approximate length.** A two- + paragraph description should remain a two-paragraph description. +5. **Do not invent new code examples or API references.** If you can't + describe the new symbol behavior from the diff alone, say so in a + single sentence rather than fabricating details. + +Output the full revised page markdown. No commentary, no surrounding +fences, no leading title. diff --git a/backend/app/prompts/wiki/surgical_edit_user.md b/backend/app/prompts/wiki/surgical_edit_user.md new file mode 100644 index 00000000..eebf7768 --- /dev/null +++ b/backend/app/prompts/wiki/surgical_edit_user.md @@ -0,0 +1,25 @@ +## Page being edited + +Title: {page_title} +Primary symbol: {primary_symbol_id} + +## Symbols whose source changed in this regen + +{symbol_diffs} + +## File paths that moved (rewrite attributes only) + +{moved_paths} + +## Current page markdown + +The full current body follows between the ```` markers. +Replace ONLY the prose about the changed symbols above. Everything else +must be preserved byte-for-byte. + + +{current_content} + + +Now return the revised page markdown. Start with the first heading and +end with the last line of body; no fences, no explanation. diff --git a/backend/app/prompts/wiki/wiki_structure.md b/backend/app/prompts/wiki/wiki_structure.md new file mode 100644 index 00000000..6ea9f743 --- /dev/null +++ b/backend/app/prompts/wiki/wiki_structure.md @@ -0,0 +1,91 @@ +You are a documentation architect creating comprehensive wiki structure based on repository capability analysis. Focus on functional organization and user workflows rather than abstract architectural concepts. + +Repository Information: +- Repository Tree: {repository_tree} +- README Content: {readme_content} +- Analysis: {repo_analysis} +- Target Audience: {target_audience} +- Wiki Type: {wiki_type} + +**DOCUMENTATION STRUCTURE DIRECTIVES:** + +**CAPABILITY-DRIVEN ORGANIZATION:** Structure documentation around what users actually DO with the codebase. Group related capabilities that support complete user workflows and system operations. + +**COMPREHENSIVE COVERAGE:** Document ALL significant components identified in the repository analysis without omissions. Let repository complexity naturally determine documentation scope. + +**WORKFLOW-FOCUSED SECTIONS:** Create sections that support complete user journeys from setup through advanced usage, maintenance, and integration. + +**OUTPUT STRUCTURE:** + +Create a comprehensive JSON structure that covers all repository capabilities identified in the analysis. Each page must have: +- Clear functional purpose derived from repository analysis +- Specific file/folder mappings from actual codebase structure +- Comprehensive retrieval query for optimal vector store content gathering + +**RETRIEVAL QUERY GENERATION:** + +For each page, generate a comprehensive retrieval query that combines: +- Page topic and capabilities focus +- Specific folder/file context +- Related functionality and workflow patterns +- Technical implementation details needed + +**RETRIEVAL QUERY EXAMPLE:** +For a page about "Authentication System Implementation": +- Topic: Authentication, user management, security +- Folders: ["auth/", "middleware/", "config/"] +- Files: ["auth_manager.py", "user_service.py", "security_config.py"] +- Generated Query: "authentication system user management security middleware auth_manager user_service login logout session token validation authorization middleware security configuration password hashing JWT session management user roles permissions access control authentication flow" + +This query combines topic keywords with file-specific terms and related functionality to ensure comprehensive content retrieval. + +**QUALITY REQUIREMENTS:** +- Map ALL components from repository analysis to specific pages +- Ensure comprehensive coverage without arbitrary omissions +- Generate retrieval queries that capture both functional context and file-specific implementation details +- Organize content around actual user workflows and system capabilities +- Base ALL decisions on provided repository analysis content + +**REQUIRED JSON FORMAT:** + +Return a comprehensive JSON structure with this exact format: + +{{ + "wiki_title": "Repository-specific title based on actual analysis", + "overview": "Comprehensive overview that references specific repository folders and components from the analysis", + "sections": [ + {{ + "section_name": "Section name that naturally emerges from repository analysis", + "section_order": 1, + "description": "Description based on actual repository characteristics", + "rationale": "Why this section is essential based on the specific repository structure and complexity", + "pages": [ + {{ + "page_name": "Page name that reflects actual repository concepts", + "page_order": 1, + "description": "Description based on actual repository needs", + "content_focus": "Focus areas derived from actual repository analysis", + "rationale": "Why this page is needed based on specific repository complexity and structure", + "target_folders": ["Actual folders from repository analysis"], + "key_files": ["Actual files from repository analysis"], + "retrieval_query": "Comprehensive query combining page topic, folder/file context, related functionality, and implementation details for optimal vector store retrieval" + }} + ] + }} + ], + "total_pages": "Actual count based on repository complexity" +}} + + +**STRUCTURE REQUIREMENTS:** +- Create complete documentation structure covering ALL repository components without omission +- Each page provides substantial coverage of assigned components with comprehensive retrieval queries +- Include ALL necessary pages based on repository analysis findings +- Ensure hierarchical structure matches repository logical organization +- Generate retrieval queries that combine functional context with specific file/folder targeting +- Base ALL content organization on actual repository capabilities and user workflows +- Do not create duplicate pages covering the same set of files; merge instead and broaden content_focus. + +Analyze the repository systematically and create documentation structure that emerges organically from the capability analysis. + +**GROUNDING / NO HALLUCINATIONS:** Only reference folders/files present in repository tree or analysis. If uncertain, omit and note uncertainty. diff --git a/backend/app/prompts/writer/budget_exhausted.md b/backend/app/prompts/writer/budget_exhausted.md new file mode 100644 index 00000000..ab4fe0d7 --- /dev/null +++ b/backend/app/prompts/writer/budget_exhausted.md @@ -0,0 +1,7 @@ +You have used all available tool calls. Based on the evidence +you gathered, write the complete wiki page in Markdown now. + +IMPORTANT: cite every claim using the file path and line numbers +from your tool results. Format: [path/to/file.py:N] or +[path/to/file.py:lo-hi]. A paragraph without a citation will be +discarded. Do NOT call any more tools. diff --git a/backend/app/prompts/writer/chapter_index_budget_exhausted.md b/backend/app/prompts/writer/chapter_index_budget_exhausted.md new file mode 100644 index 00000000..09cff0dd --- /dev/null +++ b/backend/app/prompts/writer/chapter_index_budget_exhausted.md @@ -0,0 +1,5 @@ +You have used all available tool calls. Based on the +evidence you gathered, write the complete chapter index +in Markdown now. Follow the structure exactly: +## Overview, ## Key Components, ## Sub-pages. +Do NOT call any more tools. diff --git a/backend/app/prompts/writer/system.md b/backend/app/prompts/writer/system.md new file mode 100644 index 00000000..bb1aa5db --- /dev/null +++ b/backend/app/prompts/writer/system.md @@ -0,0 +1,119 @@ +{prefix} + +Your job is to write a detailed, accurate wiki page in Markdown based on the +page specification and your tool results. + +## Citation contract + +{format_description} + +Rules: +{rules} + +### Why citations matter + +The citation verifier checks every paragraph against the actual source code. +A paragraph with no citation is treated as ungrounded prose and is discarded +entirely — it will not appear in the final wiki page. A citation pointing to a +line range that does not exist in the repository is also discarded. Only +paragraphs with citations that match real, readable code survive into the +final output. This means: if you write a sentence without a citation, it will +be silently removed. If you cite a line range that you did not actually read +with read_file, the verifier may reject it. The only safe approach is to call +read_file, observe the line numbers in the returned content, and then cite +those exact lines. Never guess a line number. Never cite a file you did not +open with read_file or confirm with get_signature. + +### Citation rule details + +Rule 1 — Every claim needs a citation. The consequence of omitting a citation +is that the whole paragraph is stripped. Write claims in the form: +"The scheduler uses a one-second poll interval [src/scheduler.py:88]." +Append the citation token immediately after the claim, before the sentence's +closing punctuation. Do not move it to the end of the paragraph if the +paragraph covers multiple files — cite each claim individually. + +Rule 2 — README content must be verified. If you intend to reference something +described in a README, you must call read_file on the README first and cite +the specific lines. Paraphrasing README text without a matching citation is +treated as an uncited claim and will be stripped. + +Rule 3 — Identifiers must appear in your tool trace. Do not name a function, +class, or environment variable that did not appear in the output of read_file, +get_signature, get_callers, get_callees, grep, or list_doc_chunks. If you are +uncertain whether a symbol exists, call get_signature or grep before writing +about it. Mentioning a non-existent symbol causes the paragraph to be flagged +and may cause the entire page to fail verification. + +## Available tools + +**read_file(path, start_line?, end_line?)** — Read lines from a repo-relative file. +Returns: numbered lines in the form `: ` (one per line). + Example: `47: class JobQueue:\n48: def __init__(self, max_size: int = 1000):` +Use: read files containing the symbols in target_symbols. The line numbers +in the output are the exact numbers to use in citations: read line 47 → cite +[path:47] or [path:47-48] for a range. +Error: `[error] ...` — try a different path or call get_signature first. + +**get_signature(symbol)** — Look up a symbol's definition location and signature. +Returns: `file_path: signature (layer)\ndocstring` + Example: `src/workers/queue.py: class JobQueue (infrastructure)\nAsyncIO job queue.` +Use: confirm a class or function exists and find its file path before calling +read_file. Does NOT return line numbers — use read_file after get_signature +to get the exact lines for citations. +Not found: `[not found] symbol` — do not mention this symbol in the page. + +**grep(pattern)** — Full-text search across source files. +Returns: `file_path:line_number: line_text` (up to 20 matches) +Use: find where a concept or identifier is used across the codebase. Use grep +to locate configuration values, string literals, or identifiers you know exist +but whose file path you are unsure of. +No matches: `[no matches]` — the pattern does not appear in source files. + +**get_callers(symbol)** — List symbols that call the given symbol. +Returns: `file_path: symbol_name` per caller +Use: document who uses a function or class (integration context). Useful for +writing the "used by" section of a public API page. Shows the call graph from +the perspective of dependents. + +**get_callees(symbol)** — List symbols the given symbol calls. +Returns: `file_path: symbol_name` per callee +Use: document what a function or class depends on (dependency chain). Useful +for showing what a component orchestrates or delegates to. + +**list_doc_chunks(doc_path)** — Return documentation sections for a doc file. +Returns: `[N] heading\ntext` per chunk +Use: incorporate doc context into pages that cover documented modules. Prefer +this over read_file when you need the logical structure of a markdown document +rather than raw line content. + +## Citation examples + +Good — claim immediately followed by a bracketed citation token: + "The worker polls with a 1-second timeout [src/workers/worker.py:62-63]." + "Default queue size is 1000 items [src/workers/queue.py:57]." + "Startup recovery loads pending jobs [src/workers/queue.py:121-133]." + "Config is read from environment [config.py:5-12] and merged with file + defaults [config.py:40-48]." + +Bad — NEVER write these (they are discarded by the verifier): + "The worker polls with a 1-second timeout." ← no citation, discarded + "See worker.py for the timeout logic." ← path in prose not in brackets, discarded + "The worker polls (worker.py:62)." ← parens not brackets, citation not parsed + "The queue has configurable size [queue.py:57]." ← missing src/ prefix, + may not resolve to a real path — always use the exact path from read_file + "The scheduler module handles retries." ← whole paragraph with no citation, + discarded in full regardless of how accurate the claim is + +## Budget guidance + +You have a finite number of tool calls available for this page. Use them to +read the files and symbols listed in the page spec before writing. Prioritise +reading symbols from target_symbols first, then explore related callers or +callees if the budget allows. Once you have read enough to ground every claim, +write the complete page. Do not call tools after you have started writing. + +IMPORTANT: every substantive paragraph MUST end with at least one citation in +the format shown above. A paragraph with no citation will be stripped by the +verifier. Start by calling tools to read the relevant files, then write the +page with inline citations after every claim. diff --git a/backend/app/prompts/writer/user_template.md b/backend/app/prompts/writer/user_template.md new file mode 100644 index 00000000..6dfaf052 --- /dev/null +++ b/backend/app/prompts/writer/user_template.md @@ -0,0 +1,13 @@ +## Page to write + +**Title:** {title} +**Description:** {description} +**Retrieval query:** {retrieval_query} + +**Target symbols:** {symbols} +**Target folders:** {folders} +**Target docs:** {docs} + +First, read the most relevant files from target symbols / folders using the +available tools. Then write the complete wiki page in Markdown with inline +`[path:N]` or `[path:lo-hi]` citations after every claim. diff --git a/backend/app/services/wiki_service.py b/backend/app/services/wiki_service.py index 3cc2e263..c726a373 100644 --- a/backend/app/services/wiki_service.py +++ b/backend/app/services/wiki_service.py @@ -928,6 +928,21 @@ async def _finalize_generation( invocation.pages_total = len(generated_pages) invocation.pages_completed = 0 + # Snapshot existing wiki_pages/*.md artifact keys BEFORE any uploads + # so we can remove stale ones after the new generation is fully written. + # Must happen before any writes so the set reflects the prior state only. + try: + existing_artifacts = await self.storage.list_artifacts( + "wiki_artifacts", prefix=invocation.wiki_id + ) + existing_wiki_page_files: set[str] = { + a for a in existing_artifacts + if a.endswith(".md") and "wiki_pages" in a + } + except Exception as _e: + logger.warning("Could not list existing artifacts for stale cleanup: %s", _e) + existing_wiki_page_files = set() + for page_id, content in generated_pages.items(): try: data = content.encode("utf-8") if isinstance(content, str) else content @@ -946,14 +961,35 @@ async def _finalize_generation( except Exception as e: logger.warning("Failed to index pages for FTS: %s", e) - # Store export artifacts (index, summary, etc.) + # Store export artifacts (index, summary, etc.). The ArtifactExporter + # produces the wiki_pages/*.md files that the viewer actually reads; collect + # their keys so we know which files the new generation owns. artifacts = result.get("artifacts", []) + new_wiki_page_keys: set[str] = set() for artifact in artifacts: name = artifact.get("name", "unknown") data = artifact.get("data", b"") if isinstance(data, str): data = data.encode("utf-8") - await self.storage.upload("wiki_artifacts", f"{invocation.wiki_id}/{name}", data) + key = f"{invocation.wiki_id}/{name}" + await self.storage.upload("wiki_artifacts", key, data) + if name.endswith(".md") and "wiki_pages" in name: + new_wiki_page_keys.add(key) + + # Remove stale wiki_pages/*.md files — only AFTER all new files are + # safely written, so users never see a partially-updated wiki. + if existing_wiki_page_files and new_wiki_page_keys: + stale_page_files = existing_wiki_page_files - new_wiki_page_keys + for stale in stale_page_files: + try: + await self.storage.delete("wiki_artifacts", stale) + except Exception as _e: + logger.debug("Could not delete stale page file %r: %s", stale, _e) + if stale_page_files: + logger.info( + "[WIKI] Removed %d stale page file(s) for wiki %s", + len(stale_page_files), invocation.wiki_id, + ) page_count = invocation.pages_completed if self.wiki_management and request is not None: diff --git a/backend/tests/unit/test_prompt_loader.py b/backend/tests/unit/test_prompt_loader.py new file mode 100644 index 00000000..8ae58b5d --- /dev/null +++ b/backend/tests/unit/test_prompt_loader.py @@ -0,0 +1,200 @@ +"""Unit tests for app.prompts.load_prompt (#346 migration). + +Verifies that: +- load_prompt resolves paths relative to the prompts package directory. +- Missing files raise FileNotFoundError with a helpful message. +- Results are strings with real content. +- caching works (same object on second call). +- All migrated prompt files are accessible and non-empty. +- The migrated constants in Python modules match the MD files. +""" + +from __future__ import annotations + +import pytest + +from app.prompts import load_prompt + + +# ── Basic loader behaviour ──────────────────────────────────────────────────── + +def test_load_prompt_returns_string(): + """load_prompt returns a non-empty str for a known file.""" + text = load_prompt("writer/system.md") + assert isinstance(text, str) + assert len(text) > 100 + + +def test_load_prompt_missing_file_raises_filenotfounderror(): + """Non-existent paths raise FileNotFoundError, not a generic exception.""" + with pytest.raises(FileNotFoundError, match="not found"): + load_prompt("nonexistent/path.md") + + +def test_load_prompt_is_cached(): + """Two calls with the same path return the identical string object (lru_cache).""" + a = load_prompt("writer/system.md") + b = load_prompt("writer/system.md") + assert a is b + + +def test_load_prompt_strips_whitespace(): + """load_prompt strips leading/trailing whitespace from file contents.""" + text = load_prompt("writer/system.md") + assert text == text.strip() + + +# ── Existence check for all migrated prompt files ───────────────────────────── + +MIGRATED_FILES = [ + # writer + "writer/system.md", + "writer/user_template.md", + "writer/budget_exhausted.md", + "writer/chapter_index_budget_exhausted.md", + # planner + "planner/system.md", + "planner/chaptered_system.md", + "planner/refiner_system.md", + # ask + "ask/workflow_instructions.md", + "ask/tool_instructions.md", + "ask/output_instructions.md", + "ask/query_optimization_system.md", + "ask/query_optimization_user.md", + "ask/answer_system.md", + "ask/answer_user.md", + # research + "research/workflow_instructions.md", + "research/tool_instructions.md", + "research/stopping_criteria.md", + "research/output_format.md", + # extractors + "extractors/image_describe.md", + "extractors/pdf_page_describe.md", + # repo + "repo/explorer_system.md", + "repo/explorer_budget_exhausted.md", + "repo/overview_system.md", + # wiki + "wiki/surgical_edit_system.md", + "wiki/surgical_edit_user.md", + "wiki/page_format.md", + "wiki/content_generation_v3_tone.md", + "wiki/repo_analysis_structured.md", + "wiki/repo_analysis_enhanced.md", + "wiki/wiki_structure.md", +] + + +@pytest.mark.parametrize("path", MIGRATED_FILES) +def test_migrated_file_is_non_empty(path: str): + """Each migrated MD file can be loaded and contains non-trivial content.""" + text = load_prompt(path) + assert isinstance(text, str) + assert len(text) >= 30, f"{path} has suspiciously short content: {text!r}" + + +# ── Template variable placeholders ──────────────────────────────────────────── + +def test_writer_system_has_format_placeholders(): + """Writer system prompt is a template with {prefix}, {format_description}, {rules}.""" + text = load_prompt("writer/system.md") + assert "{prefix}" in text + assert "{format_description}" in text + assert "{rules}" in text + + +def test_writer_user_template_has_format_placeholders(): + """Writer user template has the expected {title}, {description}, etc. placeholders.""" + text = load_prompt("writer/user_template.md") + assert "{title}" in text + assert "{description}" in text + assert "{symbols}" in text + + +def test_planner_system_has_no_format_placeholders(): + """Planner system prompt is static — no {variable} substitutions needed.""" + text = load_prompt("planner/system.md") + # The planner system prompt contains example JSON with {cluster_id} etc. in it, + # but those are inside code fences not format placeholders. + # The template is passed as-is to the LLM without .format(). + assert "You are a wiki structure planner" in text + + +def test_ask_workflow_has_date_and_iterations_placeholders(): + """Ask workflow instructions template uses {date} and {max_iterations}.""" + text = load_prompt("ask/workflow_instructions.md") + assert "{date}" in text + assert "{max_iterations}" in text + + +def test_repo_explorer_system_has_repo_and_budget_placeholders(): + """Repo explorer system prompt uses {repo_name} and {budget}.""" + text = load_prompt("repo/explorer_system.md") + assert "{repo_name}" in text + assert "{budget}" in text + + +def test_repo_overview_system_has_max_pages_placeholder(): + """Repo overview system prompt uses {max_pages}.""" + text = load_prompt("repo/overview_system.md") + assert "{max_pages}" in text + + +def test_surgical_edit_user_has_required_placeholders(): + """Surgical edit user prompt has all required template placeholders.""" + text = load_prompt("wiki/surgical_edit_user.md") + assert "{page_title}" in text + assert "{symbol_diffs}" in text + assert "{current_content}" in text + + +# ── Integration: Python modules export the loaded prompts ───────────────────── + +def test_surgical_edit_prompts_module_exports_match_files(): + """surgical_edit_prompts constants match the MD file content.""" + from app.core.prompts.surgical_edit_prompts import ( + SURGICAL_EDIT_SYSTEM, + SURGICAL_EDIT_USER_TEMPLATE, + ) + assert SURGICAL_EDIT_SYSTEM == load_prompt("wiki/surgical_edit_system.md") + assert SURGICAL_EDIT_USER_TEMPLATE == load_prompt("wiki/surgical_edit_user.md") + + +def test_planner_prompts_module_exports_match_files(): + """planner_prompts constants match the MD file content.""" + from app.core.wiki_structure_planner.planner_prompts import ( + CHAPTERED_PLANNER_SYSTEM_PROMPT, + PLANNER_SYSTEM_PROMPT, + ) + assert PLANNER_SYSTEM_PROMPT == load_prompt("planner/system.md") + assert CHAPTERED_PLANNER_SYSTEM_PROMPT == load_prompt("planner/chaptered_system.md") + + +def test_ask_tool_prompts_match_files(): + """AskTool prompt constants match the MD file content.""" + from app.core.ask_tool import ( + ANSWER_SYSTEM_PROMPT, + ANSWER_USER_PROMPT, + QUERY_OPTIMIZATION_SYSTEM_PROMPT, + QUERY_OPTIMIZATION_USER_PROMPT, + ) + assert QUERY_OPTIMIZATION_SYSTEM_PROMPT == load_prompt("ask/query_optimization_system.md") + assert QUERY_OPTIMIZATION_USER_PROMPT == load_prompt("ask/query_optimization_user.md") + assert ANSWER_SYSTEM_PROMPT == load_prompt("ask/answer_system.md") + assert ANSWER_USER_PROMPT == load_prompt("ask/answer_user.md") + + +def test_research_prompts_module_exports_match_files(): + """research_prompts constants match the MD file content.""" + from app.core.deep_research.research_prompts import ( + OUTPUT_FORMAT_INSTRUCTIONS, + RESEARCH_WORKFLOW_INSTRUCTIONS, + STOPPING_CRITERIA, + TOOL_USAGE_INSTRUCTIONS, + ) + assert RESEARCH_WORKFLOW_INSTRUCTIONS == load_prompt("research/workflow_instructions.md") + assert TOOL_USAGE_INSTRUCTIONS == load_prompt("research/tool_instructions.md") + assert STOPPING_CRITERIA == load_prompt("research/stopping_criteria.md") + assert OUTPUT_FORMAT_INSTRUCTIONS == load_prompt("research/output_format.md") diff --git a/backend/tests/unit/wiki_content_writer/test_diagram_generator.py b/backend/tests/unit/wiki_content_writer/test_diagram_generator.py index d9ffd75c..cadee46a 100644 --- a/backend/tests/unit/wiki_content_writer/test_diagram_generator.py +++ b/backend/tests/unit/wiki_content_writer/test_diagram_generator.py @@ -71,6 +71,42 @@ def _get_edges_from(node_id, rel_types=None): return storage +def _class_node( + node_id: str, + symbol_name: str, + symbol_type: str = "class", + is_architectural: int = 1, + source_text: str = "", + parent_symbol: str | None = None, + signature: str = "", + parameters: str = "", + return_type: str = "", + macro_cluster: int = 1, + micro_cluster: int = 0, +) -> dict[str, Any]: + """Create a node dict matching the real storage API schema.""" + return { + "node_id": node_id, + "symbol_name": symbol_name, + "symbol_type": symbol_type, + "is_architectural": is_architectural, + "source_text": source_text, + "parent_symbol": parent_symbol, + "signature": signature, + "parameters": parameters, + "return_type": return_type, + "macro_cluster": macro_cluster, + "micro_cluster": micro_cluster, + # Legacy aliases so _node_name() works for both formats. + "name": symbol_name, + } + + +def _storage_edge(target_id: str, rel_type: str = "inheritance") -> dict[str, Any]: + """Edge dict as returned by get_edges_from.""" + return {"target_id": target_id, "rel_type": rel_type} + + # ── Scenario 1: 3 nodes, 2 edges ────────────────────────────────────────────── @@ -152,41 +188,26 @@ def _make_nodes(self, count: int, is_arch_up_to: int) -> list[dict]: for i in range(count) ] - def test_non_architectural_nodes_excluded_when_over_limit(self): - """With 20 nodes (5 arch, 15 non-arch), only arch nodes appear.""" + def test_all_code_nodes_included_no_arch_filter(self): + """All non-doc nodes appear — no is_architectural filter or count cap.""" nodes = self._make_nodes(20, is_arch_up_to=5) storage = _make_storage(nodes) dg = DiagramGenerator(storage) diagram = dg.generate_cluster_diagram(cluster_id=1, page_spec=None) - # Only arch nodes (Symbol0..Symbol4) should appear - for i in range(5): + # All 20 nodes should appear (arch filter removed) + for i in range(20): assert f"Symbol{i}" in diagram - # Non-arch nodes should NOT appear - for i in range(5, 20): - assert f"Symbol{i}" not in diagram - - def test_max_15_nodes_even_if_all_architectural(self): - """With 25 architectural nodes, at most 15 appear in the diagram.""" - nodes = self._make_nodes(25, is_arch_up_to=25) - storage = _make_storage(nodes) - dg = DiagramGenerator(storage) - diagram = dg.generate_cluster_diagram(cluster_id=1, page_spec=None) - # Count occurrences of "Symbol" — each node name appears once - # (either in a node definition or an edge) - present = sum(1 for i in range(25) if f"Symbol{i}" in diagram) - assert present <= 15 - - def test_with_exactly_15_nodes_all_included(self): - """Exactly 15 architectural nodes → all included.""" - nodes = self._make_nodes(15, is_arch_up_to=15) + def test_all_architectural_nodes_included_no_cap(self): + """All architectural nodes appear — no hard cap is applied.""" + nodes = self._make_nodes(20, is_arch_up_to=20) storage = _make_storage(nodes) dg = DiagramGenerator(storage) diagram = dg.generate_cluster_diagram(cluster_id=1, page_spec=None) - for i in range(15): - assert f"Symbol{i}" in diagram + present = sum(1 for i in range(20) if f"Symbol{i}" in diagram) + assert present == 20 # ── Scenario 4: isolated nodes (no edges) ──────────────────────────────────── @@ -290,3 +311,445 @@ def test_node_id_has_no_spaces(self): # A valid Mermaid node id with spaces is quoted: A["My Worker Class"] # If the name appears only inside quotes it's fine assert '["My Worker Class"]' in stripped or "My_Worker_Class" in stripped + + +# ═══════════════════════════════════════════════════════════════════════════════ +# Bug 1: _select_nodes doc-type filtering +# ═══════════════════════════════════════════════════════════════════════════════ + + +class TestSelectNodesDocTypeFiltering: + """_select_nodes must exclude document-type nodes from architecture diagrams.""" + + def _doc_class_node(self, node_id: str, name: str, symbol_type: str) -> dict: + return { + "node_id": node_id, + "symbol_name": name, + "name": name, + "symbol_type": symbol_type, + "is_architectural": 1, + } + + def test_markdown_document_excluded(self): + from app.core.wiki_content_writer.diagram_generator import _select_nodes + nodes = [self._doc_class_node("d1", "README", "markdown_document")] + result = _select_nodes(nodes) + assert result == [] + + def test_config_document_excluded(self): + from app.core.wiki_content_writer.diagram_generator import _select_nodes + nodes = [self._doc_class_node("d1", "pyproject", "config_document")] + result = _select_nodes(nodes) + assert result == [] + + def test_toml_document_excluded(self): + from app.core.wiki_content_writer.diagram_generator import _select_nodes + nodes = [self._doc_class_node("d1", "Cargo", "toml_document")] + result = _select_nodes(nodes) + assert result == [] + + def test_module_doc_excluded(self): + from app.core.wiki_content_writer.diagram_generator import _select_nodes + nodes = [self._doc_class_node("d1", "MyModule", "module_doc")] + result = _select_nodes(nodes) + assert result == [] + + def test_code_nodes_kept(self): + from app.core.wiki_content_writer.diagram_generator import _select_nodes + nodes = [ + _class_node("c1", "MyClass", "class"), + _class_node("c2", "MyFunc", "function"), + ] + result = _select_nodes(nodes) + assert len(result) == 2 + + def test_mixed_nodes_only_code_returned(self): + from app.core.wiki_content_writer.diagram_generator import _select_nodes + nodes = [ + _class_node("c1", "MyClass", "class"), + self._doc_class_node("d1", "README", "markdown_document"), + _class_node("c2", "MyFunc", "function"), + self._doc_class_node("d2", "Config", "config_document"), + ] + result = _select_nodes(nodes) + names = [n["symbol_name"] for n in result] + assert "MyClass" in names + assert "MyFunc" in names + assert "README" not in names + assert "Config" not in names + + def test_generate_cluster_context_diagram_empty_when_all_doc_nodes(self): + """generate_cluster_context_diagram returns '' when cluster has only doc nodes.""" + storage = MagicMock() + storage.get_nodes_by_cluster.return_value = [ + { + "node_id": "d1", "symbol_name": "README", "name": "README", + "symbol_type": "markdown_document", "is_architectural": 1, + }, + { + "node_id": "d2", "symbol_name": "CHANGELOG", "name": "CHANGELOG", + "symbol_type": "markdown_document", "is_architectural": 0, + }, + ] + storage.get_edges_from.return_value = [] + dg = DiagramGenerator(storage) + result = dg.generate_cluster_context_diagram( + cluster_id=1, cluster_title="Docs", all_cluster_ids=[], cluster_titles={} + ) + assert result == "" + + +# ═══════════════════════════════════════════════════════════════════════════════ +# Bug 2: parent_symbol format normalization +# ═══════════════════════════════════════════════════════════════════════════════ + + +class TestParentSymbolNormalization: + """_build_class_diagram must match dotted parent_symbol to class short name.""" + + def _make_dg(self, nodes, edges_map=None): + storage = MagicMock() + storage.get_nodes_by_cluster.return_value = nodes + storage.get_edges_from.side_effect = ( + lambda nid, rel_types=None: (edges_map or {}).get(nid, []) + ) + return DiagramGenerator(storage) + + def test_dotted_parent_symbol_method_appears_in_class(self): + """Method with parent_symbol='module.ClassName' should appear under ClassName.""" + nodes = [ + _class_node("c1", "WorkerPool", "class"), + _class_node("c2", "JobQueue", "class"), + # Method with dotted parent_symbol as stored in DB + _class_node( + "m1", "submit", "method", + parent_symbol="worker.WorkerPool", + signature="submit(job: Job) -> None", + ), + ] + dg = self._make_dg(nodes) + result = dg._build_class_diagram(cluster_id=1) + # The method 'submit' must appear in the diagram under WorkerPool + assert "submit" in result + + def test_simple_parent_symbol_still_works(self): + """Method with simple (non-dotted) parent_symbol still matches.""" + nodes = [ + _class_node("c1", "Foo", "class"), + _class_node("c2", "Bar", "class"), + _class_node("m1", "do_it", "method", parent_symbol="Foo"), + ] + dg = self._make_dg(nodes) + result = dg._build_class_diagram(cluster_id=1) + assert "do_it" in result + + def test_deeply_dotted_parent_symbol_normalized(self): + """Method with 'a.b.c.ClassName' parent_symbol resolves to 'ClassName'.""" + nodes = [ + _class_node("c1", "MyClass", "class"), + _class_node("c2", "Other", "class"), + _class_node( + "m1", "run", "method", + parent_symbol="pkg.subpkg.module.MyClass", + ), + ] + dg = self._make_dg(nodes) + result = dg._build_class_diagram(cluster_id=1) + assert "run" in result + + +# ═══════════════════════════════════════════════════════════════════════════════ +# Bug 3: _is_data_model only detects real DB models +# ═══════════════════════════════════════════════════════════════════════════════ + + +class TestIsDataModelStrict: + """_is_data_model must NOT fire on BaseModel-only or @dataclass classes.""" + + def test_pydantic_basemodel_not_detected(self): + """Pydantic request/response schemas must NOT trigger erDiagram.""" + from app.core.wiki_content_writer.diagram_generator import _is_data_model + node = _class_node( + "m1", "LaunchResponse", "class", + source_text="class LaunchResponse(BaseModel):\n id: str\n status: str", + ) + assert _is_data_model(node) is False + + def test_dataclass_not_detected(self): + """@dataclass DTOs must NOT trigger erDiagram.""" + from app.core.wiki_content_writer.diagram_generator import _is_data_model + node = _class_node( + "m1", "JobStatus", "class", + source_text="@dataclass\nclass JobStatus:\n id: str\n state: str", + ) + assert _is_data_model(node) is False + + def test_sqlalchemy_column_detected(self): + from app.core.wiki_content_writer.diagram_generator import _is_data_model + node = _class_node( + "m1", "User", "class", + source_text="class User(Base):\n id = Column(Integer, primary_key=True)", + ) + assert _is_data_model(node) is True + + def test_sqlalchemy_relationship_detected(self): + from app.core.wiki_content_writer.diagram_generator import _is_data_model + node = _class_node( + "m1", "Post", "class", + source_text="class Post(Base):\n user_id = Column(Integer)\n user = relationship('User')", + ) + assert _is_data_model(node) is True + + def test_mapped_column_detected(self): + from app.core.wiki_content_writer.diagram_generator import _is_data_model + node = _class_node( + "m1", "Article", "class", + source_text="class Article(Base):\n id: Mapped[int] = mapped_column(primary_key=True)", + ) + assert _is_data_model(node) is True + + def test_declarative_base_detected(self): + from app.core.wiki_content_writer.diagram_generator import _is_data_model + node = _class_node( + "m1", "Base", "class", + source_text="class Base(DeclarativeBase):\n pass", + ) + assert _is_data_model(node) is True + + def test_db_model_detected(self): + from app.core.wiki_content_writer.diagram_generator import _is_data_model + node = _class_node( + "m1", "User", "class", + source_text="class User(db.Model):\n id = db.Column(db.Integer)", + ) + assert _is_data_model(node) is True + + def test_plain_class_not_detected(self): + from app.core.wiki_content_writer.diagram_generator import _is_data_model + node = _class_node( + "m1", "Util", "class", + source_text="class Util:\n def run(self): pass", + ) + assert _is_data_model(node) is False + + def test_generate_data_model_diagram_skips_basemodel_only(self): + """generate_cluster_context_diagram skips Pydantic-only clusters.""" + storage = MagicMock() + storage.get_nodes_by_cluster.return_value = [ + _class_node( + "m1", "LaunchResponse", "class", + source_text="class LaunchResponse(BaseModel):\n id: str", + ), + _class_node( + "m2", "JobStatus", "class", + source_text="class JobStatus(BaseModel):\n state: str", + ), + ] + storage.get_edges_from.return_value = [] + dg = DiagramGenerator(storage) + result = dg._build_data_model_diagram(cluster_id=1) + assert result == "" + + def test_sqlalchemy_nodes_produce_er_diagram(self): + storage = MagicMock() + storage.get_nodes_by_cluster.return_value = [ + _class_node( + "m1", "User", "class", + source_text="class User(Base):\n id = Column(Integer)", + ), + _class_node( + "m2", "Post", "class", + source_text="class Post(Base):\n id = Column(Integer)\n user = relationship('User')", + ), + ] + storage.get_edges_from.return_value = [] + dg = DiagramGenerator(storage) + result = dg._build_data_model_diagram(cluster_id=1) + assert "erDiagram" in result + + +# ═══════════════════════════════════════════════════════════════════════════════ +# NEW: generate_cluster_context_diagram +# ═══════════════════════════════════════════════════════════════════════════════ + + +class TestGenerateClusterContextDiagram: + """Unit tests for DiagramGenerator.generate_cluster_context_diagram.""" + + def _make_dg(self, nodes, edges_map=None): + def _side_effect(node_id, rel_types=None): + if edges_map: + return edges_map.get(node_id, []) + return [] + + storage = MagicMock() + storage.get_nodes_by_cluster.return_value = nodes + storage.get_edges_from.side_effect = _side_effect + return DiagramGenerator(storage) + + # ── Intra-cluster nodes rendered in subgraph ─────────────────────────── + + def test_connected_nodes_appear_in_subgraph(self): + """Only nodes participating in edges appear — isolated nodes are omitted.""" + nodes = [ + _class_node("n1", "AuthService", "class"), + _class_node("n2", "TokenParser", "class"), + ] + edges_map = {"n1": [_storage_edge("n2", "calls")], "n2": []} + dg = self._make_dg(nodes, edges_map) + result = dg.generate_cluster_context_diagram( + cluster_id=1, cluster_title="Auth", all_cluster_ids=[], cluster_titles={} + ) + assert "subgraph" in result + assert "AuthService" in result + assert "TokenParser" in result + + def test_isolated_nodes_only_returns_empty(self): + """Cluster with no edges produces an empty string — nothing to show.""" + nodes = [ + _class_node("n1", "AuthService", "class"), + _class_node("n2", "TokenParser", "class"), + ] + dg = self._make_dg(nodes) # no edges_map → no edges + result = dg.generate_cluster_context_diagram( + cluster_id=1, cluster_title="Auth", all_cluster_ids=[], cluster_titles={} + ) + assert result == "" + + def test_diagram_has_mermaid_fence(self): + nodes = [ + _class_node("n1", "A", "class"), + _class_node("n2", "B", "class"), + ] + edges_map = {"n1": [_storage_edge("n2", "calls")], "n2": []} + dg = self._make_dg(nodes, edges_map) + result = dg.generate_cluster_context_diagram( + cluster_id=1, cluster_title="Test", all_cluster_ids=[], cluster_titles={} + ) + assert "```mermaid" in result + assert result.strip().endswith("```") + + def test_graph_td_header(self): + """Layout switched to TD for readability.""" + nodes = [ + _class_node("n1", "X", "class"), + _class_node("n2", "Y", "class"), + ] + edges_map = {"n1": [_storage_edge("n2", "calls")], "n2": []} + dg = self._make_dg(nodes, edges_map) + result = dg.generate_cluster_context_diagram( + cluster_id=1, cluster_title="X", all_cluster_ids=[], cluster_titles={} + ) + assert "graph TD" in result + + # ── Empty cluster → empty string ────────────────────────────────────── + + def test_empty_cluster_returns_empty(self): + storage = MagicMock() + storage.get_nodes_by_cluster.return_value = [] + storage.get_edges_from.return_value = [] + dg = DiagramGenerator(storage) + result = dg.generate_cluster_context_diagram( + cluster_id=1, cluster_title="Empty", all_cluster_ids=[], cluster_titles={} + ) + assert result == "" + + def test_none_storage_returns_empty(self): + dg = DiagramGenerator(None) + result = dg.generate_cluster_context_diagram( + cluster_id=1, cluster_title="X", all_cluster_ids=[], cluster_titles={} + ) + assert result == "" + + # ── Intra-cluster edges ──────────────────────────────────────────────── + + def test_intra_cluster_edges_rendered(self): + nodes = [ + _class_node("n1", "A", "class"), + _class_node("n2", "B", "class"), + ] + edges_map = { + "n1": [_storage_edge("n2", "calls")], + "n2": [], + } + dg = self._make_dg(nodes, edges_map) + result = dg.generate_cluster_context_diagram( + cluster_id=1, cluster_title="MyCluster", all_cluster_ids=[], cluster_titles={} + ) + assert "-->" in result + + # ── Cross-cluster external nodes ─────────────────────────────────────── + + def test_cross_cluster_edge_creates_external_node(self): + # n1 is in cluster 1; n_ext is in cluster 2 + nodes_c1 = [_class_node("n1", "AuthService", "class")] + nodes_c2 = [_class_node("n_ext", "WikiService", "class")] + + # Storage returns different nodes for different cluster IDs + def _get_nodes_by_cluster(cluster_id): + if cluster_id == 1: + return nodes_c1 + elif cluster_id == 2: + return nodes_c2 + return [] + + edges_map = { + "n1": [_storage_edge("n_ext", "calls")], + } + + storage = MagicMock() + storage.get_nodes_by_cluster.side_effect = _get_nodes_by_cluster + storage.get_edges_from.side_effect = lambda nid, rel_types=None: edges_map.get(nid, []) + + dg = DiagramGenerator(storage) + result = dg.generate_cluster_context_diagram( + cluster_id=1, + cluster_title="Auth Service", + all_cluster_ids=[1, 2], + cluster_titles={1: "Auth Service", 2: "Wiki Service"}, + ) + # The external cluster title should appear as a node + assert "Wiki Service" in result or "Wiki_Service" in result + + def test_cross_cluster_shown_as_single_node_not_individual_members(self): + """External cluster appears as a single labeled node, not individual nodes.""" + nodes_c1 = [_class_node("n1", "AuthService", "class")] + nodes_c2 = [ + _class_node("n_ext1", "WikiService", "class"), + _class_node("n_ext2", "PageRenderer", "class"), + ] + + def _get_nodes_by_cluster(cluster_id): + if cluster_id == 1: + return nodes_c1 + return nodes_c2 + + edges_map = { + "n1": [_storage_edge("n_ext1", "calls"), _storage_edge("n_ext2", "calls")], + } + + storage = MagicMock() + storage.get_nodes_by_cluster.side_effect = _get_nodes_by_cluster + storage.get_edges_from.side_effect = lambda nid, rel_types=None: edges_map.get(nid, []) + + dg = DiagramGenerator(storage) + result = dg.generate_cluster_context_diagram( + cluster_id=1, + cluster_title="Auth", + all_cluster_ids=[1, 2], + cluster_titles={1: "Auth", 2: "Wiki"}, + ) + # WikiService individual nodes should NOT appear — only the cluster label "Wiki" + assert "WikiService" not in result + assert "PageRenderer" not in result + + # ── Storage error → graceful empty ──────────────────────────────────── + + def test_storage_error_returns_empty(self): + storage = MagicMock() + storage.get_nodes_by_cluster.side_effect = RuntimeError("db down") + dg = DiagramGenerator(storage) + result = dg.generate_cluster_context_diagram( + cluster_id=1, cluster_title="X", all_cluster_ids=[], cluster_titles={} + ) + assert result == "" diff --git a/backend/tests/unit/wiki_content_writer/test_writer_diagrams.py b/backend/tests/unit/wiki_content_writer/test_writer_diagrams.py new file mode 100644 index 00000000..85f10e0f --- /dev/null +++ b/backend/tests/unit/wiki_content_writer/test_writer_diagrams.py @@ -0,0 +1,378 @@ +"""Unit tests for LLM-based diagram injection in WikiContentWriter. + +Covers: +- _gather_cluster_source: mock storage + tools, file path gathering +- _llm_class_diagram: valid diagram, EMPTY response, invalid Mermaid +- _llm_data_model_diagram: same pattern +- _inject_cluster_diagrams: end-to-end, injection before ## Key Components +""" +from __future__ import annotations + +from typing import Any +from unittest.mock import MagicMock + +from app.core.wiki_content_writer.writer_agent import WikiContentWriter +from app.core.wiki_content_writer.writer_tools import FileContent, WriterTools + + +# ── Helpers ──────────────────────────────────────────────────────────────────── + + +def _make_writer( + storage=None, + llm=None, + repo_root="/repo", +) -> WikiContentWriter: + """Build a WikiContentWriter with a minimal mock WriterTools.""" + tools = MagicMock(spec=WriterTools) + tools.storage = storage + tools.repo_root = repo_root + writer = WikiContentWriter(tools=tools, llm_client=llm) + return writer + + +def _code_node( + node_id: str, + symbol_name: str, + symbol_type: str = "class", + rel_path: str = "src/mod.py", + is_architectural: int = 1, +) -> dict[str, Any]: + return { + "node_id": node_id, + "symbol_name": symbol_name, + "symbol_type": symbol_type, + "rel_path": rel_path, + "is_architectural": is_architectural, + } + + +def _doc_node(node_id: str, symbol_name: str, rel_path: str = "README.md") -> dict[str, Any]: + return { + "node_id": node_id, + "symbol_name": symbol_name, + "symbol_type": "markdown_document", + "rel_path": rel_path, + "is_architectural": 0, + } + + +VALID_CLASS_DIAGRAM = """\ +```mermaid +classDiagram + class Foo { + +bar() str + } + class Baz { + -_qux int + } + Foo --|> Baz : implements +```""" + +VALID_ER_DIAGRAM = """\ +```mermaid +erDiagram + User { + int id PK + } + Post { + int id PK + int user_id FK + } + User ||--o{ Post : "has" +```""" + + +# ── _gather_cluster_source ────────────────────────────────────────────────── + + +class TestGatherClusterSource: + """_gather_cluster_source reads files from storage nodes.""" + + def test_returns_empty_when_storage_is_none(self): + writer = _make_writer(storage=None) + result = writer._gather_cluster_source(cluster_id=1, storage=None, max_files=5, chars_per_file=3000) + assert result == "" + + def test_returns_empty_when_no_nodes(self): + storage = MagicMock() + storage.get_nodes_by_cluster.return_value = [] + writer = _make_writer(storage=storage) + result = writer._gather_cluster_source(1, storage, max_files=5, chars_per_file=3000) + assert result == "" + + def test_doc_nodes_are_excluded(self): + storage = MagicMock() + storage.get_nodes_by_cluster.return_value = [ + _doc_node("d1", "README", "README.md"), + ] + # tools.read_file should never be called — no code nodes + writer = _make_writer(storage=storage) + result = writer._gather_cluster_source(1, storage, max_files=5, chars_per_file=3000) + writer.tools.read_file.assert_not_called() + assert result == "" + + def test_code_nodes_trigger_read_file(self): + storage = MagicMock() + nodes = [_code_node("n1", "Foo", rel_path="src/foo.py")] + storage.get_nodes_by_cluster.return_value = nodes + + # tools.read_file returns content for src/foo.py + writer = _make_writer(storage=storage) + writer.tools.read_file.return_value = FileContent( + path="src/foo.py", + lines=["class Foo:", " pass"], + total_lines=2, + ) + + result = writer._gather_cluster_source(1, storage, max_files=5, chars_per_file=3000) + assert "src/foo.py" in result + assert "class Foo" in result + + def test_max_files_respected(self): + storage = MagicMock() + # 10 nodes with distinct paths + nodes = [_code_node(f"n{i}", f"Class{i}", rel_path=f"src/mod{i}.py") for i in range(10)] + storage.get_nodes_by_cluster.return_value = nodes + + writer = _make_writer(storage=storage) + writer.tools.read_file.return_value = FileContent( + path="x", + lines=["content"], + total_lines=1, + ) + + writer._gather_cluster_source(1, storage, max_files=3, chars_per_file=3000) + # read_file should be called at most 3 times + assert writer.tools.read_file.call_count <= 3 + + def test_architectural_paths_preferred(self): + storage = MagicMock() + # 1 architectural node + 5 non-architectural nodes + nodes = [ + _code_node("arch1", "CoreClass", rel_path="src/core.py", is_architectural=1), + ] + [ + _code_node(f"n{i}", f"Other{i}", rel_path=f"src/other{i}.py", is_architectural=0) + for i in range(5) + ] + storage.get_nodes_by_cluster.return_value = nodes + + writer = _make_writer(storage=storage) + writer.tools.read_file.return_value = FileContent( + path="x", + lines=["content"], + total_lines=1, + ) + + # Only max_files=1, should pick the architectural one + writer._gather_cluster_source(1, storage, max_files=1, chars_per_file=3000) + call_args = [c[0][0] for c in writer.tools.read_file.call_args_list] + assert "src/core.py" in call_args + + def test_chars_per_file_truncates_content(self): + storage = MagicMock() + nodes = [_code_node("n1", "Foo", rel_path="src/foo.py")] + storage.get_nodes_by_cluster.return_value = nodes + + long_content = "x" * 10000 + writer = _make_writer(storage=storage) + writer.tools.read_file.return_value = FileContent( + path="src/foo.py", + lines=[long_content], + total_lines=1, + ) + + result = writer._gather_cluster_source(1, storage, max_files=5, chars_per_file=100) + # The content for this file should be truncated to 100 chars + file_section = result.split("=== src/foo.py ===")[-1] + assert len(file_section) <= 101 # +1 for the leading newline + + def test_read_file_error_skips_file(self): + storage = MagicMock() + nodes = [ + _code_node("n1", "Foo", rel_path="src/foo.py"), + _code_node("n2", "Bar", rel_path="src/bar.py"), + ] + storage.get_nodes_by_cluster.return_value = nodes + + writer = _make_writer(storage=storage) + + def _side_effect(path, **kwargs): + if path == "src/foo.py": + return FileContent(path=path, lines=[], total_lines=0, error="not found") + return FileContent(path=path, lines=["class Bar: pass"], total_lines=1) + + writer.tools.read_file.side_effect = _side_effect + + result = writer._gather_cluster_source(1, storage, max_files=5, chars_per_file=3000) + assert "src/bar.py" in result + assert "src/foo.py" not in result + + +# ── _llm_class_diagram ───────────────────────────────────────────────────────── + + +class TestAgenticDiagram: + """_agentic_diagram: tool loop produces validated Mermaid block.""" + + def _make_llm(self, final_text: str): + """Mock LLM that immediately returns final_text with no tool calls.""" + llm = MagicMock() + resp = MagicMock() + resp.content = final_text + resp.tool_calls = [] + llm.invoke.return_value = resp + llm.bind_tools = MagicMock(return_value=llm) + return llm + + def test_valid_class_diagram_returned(self): + llm = self._make_llm(VALID_CLASS_DIAGRAM) + writer = _make_writer(llm=llm) + result = writer._agentic_diagram(["src/foo.py"], "Foo Module", "class") + assert "classDiagram" in result + assert "```mermaid" in result + + def test_valid_arch_diagram_returned(self): + arch = "```mermaid\nstateDiagram-v2\n [*] --> Idle\n Idle --> Running\n```" + llm = self._make_llm(arch) + writer = _make_writer(llm=llm) + result = writer._agentic_diagram(["src/foo.py"], "Foo", "architecture") + assert "stateDiagram-v2" in result + + def test_empty_sentinel_returns_empty(self): + llm = self._make_llm("EMPTY") + writer = _make_writer(llm=llm) + result = writer._agentic_diagram(["src/foo.py"], "Foo", "class") + assert result == "" + + def test_no_mermaid_block_returns_empty(self): + llm = self._make_llm("Here is some prose without a diagram.") + writer = _make_writer(llm=llm) + result = writer._agentic_diagram(["src/foo.py"], "Foo", "class") + assert result == "" + + def test_wrong_header_rejected(self): + """classDiagram expected but LLM returns graph TD → rejected.""" + bad = "```mermaid\ngraph TD\n A --> B\n```" + llm = self._make_llm(bad) + writer = _make_writer(llm=llm) + result = writer._agentic_diagram(["src/foo.py"], "Foo", "class") + assert result == "" + + def test_llm_none_returns_empty(self): + writer = _make_writer(llm=None) + result = writer._agentic_diagram(["src/foo.py"], "Foo", "class") + assert result == "" + + def test_llm_exception_returns_empty(self): + llm = MagicMock() + llm.bind_tools = MagicMock(side_effect=RuntimeError("no tools")) + writer = _make_writer(llm=llm) + result = writer._agentic_diagram(["src/foo.py"], "Foo", "class") + assert result == "" + + +# ── _inject_cluster_diagrams ─────────────────────────────────────────────────── + + +class TestInjectClusterDiagrams: + """_inject_cluster_diagrams: assembles sections from _agentic_diagram calls.""" + + _ARCH = "```mermaid\nstateDiagram-v2\n [*] --> Idle\n Idle --> Running\n Running --> [*]\n```" + _CLASS = VALID_CLASS_DIAGRAM + _ER = VALID_ER_DIAGRAM + + def _writer_with_agentic(self, arch="", class_d="", data_d=""): + """Writer whose _agentic_diagram is mocked to return preset diagrams.""" + storage = MagicMock() + storage.get_nodes_by_cluster.return_value = [ + _code_node("n1", "Foo", rel_path="src/foo.py") + ] + storage.get_nodes_by_path_prefix.return_value = [ + {"symbol_name": "Foo", "macro_cluster": 1, "rel_path": "src/foo.py"} + ] + writer = _make_writer(storage=storage, llm=MagicMock()) + + def _fake_agentic(files, title, diagram_type): + return {"architecture": arch, "class": class_d, "datamodel": data_d}.get(diagram_type, "") + + writer._agentic_diagram = _fake_agentic + return writer + + def test_no_diagrams_returns_original(self): + writer = self._writer_with_agentic() + original = "## Overview\nFoo.\n\n## Key Components\n- Foo" + result = writer._inject_cluster_diagrams(original, cluster_id=1, chapter_title="Foo") + assert result == original + + def test_arch_diagram_injected_before_key_components(self): + writer = self._writer_with_agentic(arch=self._ARCH) + md = "## Overview\nText.\n\n## Key Components\n- Foo" + result = writer._inject_cluster_diagrams(md, cluster_id=1, chapter_title="Foo") + assert "## Architecture" in result + assert result.find("## Architecture") < result.find("## Key Components") + + def test_sections_injected_before_sub_pages_when_no_key_components(self): + writer = self._writer_with_agentic(arch=self._ARCH) + md = "## Overview\nText.\n\n## Sub-pages\n- [[Foo]]" + result = writer._inject_cluster_diagrams(md, cluster_id=1, chapter_title="Foo") + assert "## Architecture" in result + assert result.find("## Architecture") < result.find("## Sub-pages") + + def test_class_diagram_injected_when_available(self): + writer = self._writer_with_agentic(class_d=self._CLASS) + md = "## Overview\nText.\n\n## Key Components\n- Foo" + result = writer._inject_cluster_diagrams(md, cluster_id=1, chapter_title="Foo") + assert "## Class Structure" in result + + def test_all_three_sections_when_all_available(self): + writer = self._writer_with_agentic(arch=self._ARCH, class_d=self._CLASS, data_d=self._ER) + md = "## Overview\nText.\n\n## Key Components\n- Foo" + result = writer._inject_cluster_diagrams(md, cluster_id=1, chapter_title="Foo") + assert "## Architecture" in result + assert "## Class Structure" in result + assert "## Data Model" in result + + def test_no_storage_returns_original(self): + writer = _make_writer(storage=None, llm=None) + original = "## Overview\nFoo.\n\n## Key Components\n- Foo" + result = writer._inject_cluster_diagrams(original, cluster_id=1, chapter_title="Foo") + assert result == original + + def test_appended_to_end_when_no_marker_found(self): + writer = self._writer_with_agentic(arch=self._ARCH) + md = "## Overview\nJust overview text with no markers." + result = writer._inject_cluster_diagrams(md, cluster_id=1, chapter_title="Foo") + assert "## Architecture" in result + assert "stateDiagram-v2" in result + + +# ── _validate_mermaid accepts classDiagram and erDiagram ────────────────────── + + +class TestValidateMermaidHeaders: + """_validate_mermaid must accept classDiagram and erDiagram headers.""" + + def test_class_diagram_is_valid(self): + from app.core.wiki_content_writer.diagram_generator import _validate_mermaid + diagram = "```mermaid\nclassDiagram\n class Foo {\n +bar() str\n }\n```" + ok, reason = _validate_mermaid(diagram) + assert ok, f"Expected valid, got: {reason}" + + def test_er_diagram_is_valid(self): + from app.core.wiki_content_writer.diagram_generator import _validate_mermaid + diagram = "```mermaid\nerDiagram\n User {\n int id PK\n }\n```" + ok, reason = _validate_mermaid(diagram) + assert ok, f"Expected valid, got: {reason}" + + def test_graph_lr_still_valid(self): + from app.core.wiki_content_writer.diagram_generator import _validate_mermaid + diagram = "```mermaid\ngraph LR\n A[\"Node\"]\n```" + ok, reason = _validate_mermaid(diagram) + assert ok, f"Expected valid, got: {reason}" + + def test_invalid_header_rejected(self): + from app.core.wiki_content_writer.diagram_generator import _validate_mermaid + diagram = "```mermaid\nsequenceDiagram\n A ->> B: Hello\n```" + ok, reason = _validate_mermaid(diagram) + assert not ok diff --git a/web/src/spa/components/MermaidDiagram.tsx b/web/src/spa/components/MermaidDiagram.tsx index 8e04a080..a67fdcca 100644 --- a/web/src/spa/components/MermaidDiagram.tsx +++ b/web/src/spa/components/MermaidDiagram.tsx @@ -1,6 +1,15 @@ import { useEffect, useRef, useState } from 'react'; -import { Alert, Box, Typography } from '@mui/material'; +import { Alert, Box, Dialog, IconButton, Tooltip, Typography } from '@mui/material'; import mermaid from 'mermaid'; +import ArrowBackIcon from '@mui/icons-material/ArrowBack'; +import ArrowDownwardIcon from '@mui/icons-material/ArrowDownward'; +import ArrowForwardIcon from '@mui/icons-material/ArrowForward'; +import ArrowUpwardIcon from '@mui/icons-material/ArrowUpward'; +import CloseIcon from '@mui/icons-material/Close'; +import FitScreenIcon from '@mui/icons-material/FitScreen'; +import FullscreenIcon from '@mui/icons-material/Fullscreen'; +import ZoomInIcon from '@mui/icons-material/ZoomIn'; +import ZoomOutIcon from '@mui/icons-material/ZoomOut'; interface MermaidDiagramProps { chart: string; @@ -9,11 +18,34 @@ interface MermaidDiagramProps { let mermaidInitialized = false; +const PAN_STEP = 80; +const SCALE_STEP = 0.15; +const MIN_SCALE = 0.25; +const MAX_SCALE = 10; +const INLINE_MIN_SCALE = 0.25; +const INLINE_MAX_SCALE = 10; + export function MermaidDiagram({ chart, mode = 'dark' }: MermaidDiagramProps) { const containerRef = useRef(null); const [svg, setSvg] = useState(null); + const [naturalWidth, setNaturalWidth] = useState(null); const [error, setError] = useState(null); + // Inline zoom + const [scale, setScale] = useState(1); + + // Hover state for controls + const [hovered, setHovered] = useState(false); + + // Modal state + const [modalOpen, setModalOpen] = useState(false); + const [modalScale, setModalScale] = useState(1); + const [pan, setPan] = useState({ x: 0, y: 0 }); + + // Drag state (refs to avoid re-renders during drag) + const isDragging = useRef(false); + const dragStart = useRef({ x: 0, y: 0 }); + useEffect(() => { mermaid.initialize({ startOnLoad: false, @@ -35,7 +67,11 @@ export function MermaidDiagram({ chart, mode = 'dark' }: MermaidDiagramProps) { await mermaid.parse(chart); const id = `mermaid-${Math.random().toString(36).slice(2, 11)}`; const { svg: rendered } = await mermaid.render(id, chart); - setSvg(rendered); + // Extract natural width from Mermaid's inline style before stripping it. + const widthMatch = rendered.match(/max-width:\s*([0-9.]+)px/); + setNaturalWidth(widthMatch ? parseFloat(widthMatch[1]) : null); + // Remove Mermaid's inline max-width so CSS width can control scaling. + setSvg(rendered.replace(/max-width:[^;]+;?\s*/g, '')); } catch (err) { setSvg(null); setError(err instanceof Error ? err.message : 'Failed to render diagram'); @@ -45,6 +81,72 @@ export function MermaidDiagram({ chart, mode = 'dark' }: MermaidDiagramProps) { render(); }, [chart, mode]); + const openModal = () => { + setModalScale(1); + setPan({ x: 0, y: 0 }); + setModalOpen(true); + }; + + const closeModal = () => setModalOpen(false); + + // Inline zoom handlers + const handleInlineZoomIn = (e: React.MouseEvent) => { + e.stopPropagation(); + setScale((s) => Math.min(INLINE_MAX_SCALE, parseFloat((s + SCALE_STEP).toFixed(2)))); + }; + + const handleInlineZoomOut = (e: React.MouseEvent) => { + e.stopPropagation(); + setScale((s) => Math.max(INLINE_MIN_SCALE, parseFloat((s - SCALE_STEP).toFixed(2)))); + }; + + const handleOpenFullscreen = (e: React.MouseEvent) => { + e.stopPropagation(); + openModal(); + }; + + // Modal zoom handlers + const handleModalZoomIn = () => + setModalScale((s) => Math.min(MAX_SCALE, parseFloat((s + SCALE_STEP).toFixed(2)))); + + const handleModalZoomOut = () => + setModalScale((s) => Math.max(MIN_SCALE, parseFloat((s - SCALE_STEP).toFixed(2)))); + + const handleModalReset = () => { + setModalScale(1); + setPan({ x: 0, y: 0 }); + }; + + // Pan handlers + const handlePanUp = () => setPan((p) => ({ ...p, y: p.y + PAN_STEP })); + const handlePanDown = () => setPan((p) => ({ ...p, y: p.y - PAN_STEP })); + const handlePanLeft = () => setPan((p) => ({ ...p, x: p.x + PAN_STEP })); + const handlePanRight = () => setPan((p) => ({ ...p, x: p.x - PAN_STEP })); + + // Drag-to-pan in modal + const handleMouseDown = (e: React.MouseEvent) => { + isDragging.current = true; + dragStart.current = { x: e.clientX - pan.x, y: e.clientY - pan.y }; + e.preventDefault(); + }; + + const handleMouseMove = (e: React.MouseEvent) => { + if (!isDragging.current) return; + setPan({ x: e.clientX - dragStart.current.x, y: e.clientY - dragStart.current.y }); + }; + + const handleMouseUp = () => { + isDragging.current = false; + }; + + // Scroll-to-zoom in modal + const handleWheel = (e: React.WheelEvent) => { + e.preventDefault(); + setModalScale((s) => + Math.max(MIN_SCALE, Math.min(MAX_SCALE, parseFloat((s + (e.deltaY < 0 ? SCALE_STEP : -SCALE_STEP)).toFixed(2)))) + ); + }; + if (error) { return ( @@ -64,20 +166,268 @@ export function MermaidDiagram({ chart, mode = 'dark' }: MermaidDiagramProps) { if (!svg) return null; + const pillBg = 'rgba(0,0,0,0.55)'; + const pillColor = '#fff'; + return ( - + <> + {/* Diagram card with hover controls */} + setHovered(true)} + onMouseLeave={() => setHovered(false)} + onClick={openModal} + sx={{ + position: 'relative', + my: 3, + p: 2, + borderRadius: 2, + bgcolor: mode === 'dark' ? 'rgba(255,255,255,0.02)' : 'rgba(0,0,0,0.02)', + border: '1px solid', + borderColor: 'divider', + overflow: 'auto', + cursor: 'zoom-in', + // Use width (layout-affecting) instead of zoom/transform:scale so + // the card expands correctly in all browsers including Firefox. + '& .mermaid-svg-wrap svg': { + display: 'block', + width: naturalWidth ? `${naturalWidth * scale}px` : '100%', + height: 'auto', + }, + }} + > + {/* SVG — zoom applied via .mermaid-svg-wrap selector above */} + + + {/* Floating hover controls pill */} + + + + + + + + + = INLINE_MAX_SCALE} + > + + + + + + + + + + + + { e.stopPropagation(); setScale(1); }} + sx={{ color: pillColor, p: 0.5 }} + disabled={scale === 1} + > + + + + + + + {/* Fullscreen modal */} + + {/* Close button */} + + + + + + + + + {/* Zoom % indicator */} + + {Math.round(modalScale * 100)}% + + + {/* Pannable / zoomable SVG canvas — fills the whole dialog */} + + + + + {/* GitHub-style control pad — bottom-center */} + + {/* Row 1: [empty] [up] [zoom+] */} + + + + + + + + = MAX_SCALE} + > + + + + + {/* Row 2: [left] [reset] [right] */} + + + + + + + + + + + + + + + + + {/* Row 3: [empty] [down] [zoom-] */} + + + + + + + + + + + + + + ); }