diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 5c342d14..4bf5c649 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -5,8 +5,8 @@ "email": "daymadev89@gmail.com" }, "metadata": { - "description": "Professional Claude Code skills for GitHub operations, document conversion, diagram generation, statusline customization, Teams communication, repomix utilities, skill creation, CLI demo generation, LLM icon access, Cloudflare troubleshooting, UI design system extraction, professional presentation creation, YouTube video downloading, secure repomix packaging, ASR transcription correction, video comparison quality analysis, comprehensive QA testing infrastructure, prompt optimization with EARS methodology, session history recovery, documentation cleanup, format-controlled deep research report generation with evidence tracking, PDF generation with Chinese font support, CLAUDE.md progressive disclosure optimization, CCPM skill registry search and management, Promptfoo LLM evaluation framework, iOS app development with XcodeGen and SwiftUI, fact-checking with automated corrections, Twitter/X content fetching, intelligent macOS disk space recovery, skill quality review and improvement, GitHub contribution strategy, complete internationalization/localization setup, plugin/skill troubleshooting with diagnostic tools, evidence-based competitor analysis with source citations, Windows Remote Desktop (AVD/W365) connection quality diagnosis with transport protocol analysis and log parsing, Tailscale+proxy conflict diagnosis with SSH tunnel SOP for remote development, multi-path parallel product analysis with cross-model test-time compute scaling, real financial data collection for US equities with validation and yfinance pitfall handling, advanced Excel automation for formatted workbook generation and complex xlsm parsing, and macOS programmatic window screenshot capture workflows", - "version": "1.37.0", + "description": "Professional Claude Code skills for GitHub operations, document conversion, diagram generation, statusline customization, Teams communication, repomix utilities, skill creation, CLI demo generation, LLM icon access, Cloudflare troubleshooting, UI design system extraction, professional presentation creation, YouTube video downloading, secure repomix packaging, ASR transcription correction, video comparison quality analysis, comprehensive QA testing infrastructure, prompt optimization with EARS methodology, session history recovery, local Claude session continuation from `.claude` artifacts, documentation cleanup, format-controlled deep research report generation with evidence tracking, PDF generation with Chinese font support, CLAUDE.md progressive disclosure optimization, CCPM skill registry search and management, Promptfoo LLM evaluation framework, iOS app development with XcodeGen and SwiftUI, fact-checking with automated corrections, Twitter/X content fetching, intelligent macOS disk space recovery, skill quality review and improvement, GitHub contribution strategy, complete internationalization/localization setup, plugin/skill troubleshooting with diagnostic tools, evidence-based competitor analysis with source citations, Windows Remote Desktop (AVD/W365) connection quality diagnosis with transport protocol analysis and log parsing, Tailscale+proxy conflict diagnosis with SSH tunnel SOP for remote development, multi-path parallel product analysis with cross-model test-time compute scaling, real financial data collection for US equities with validation and yfinance pitfall handling, advanced Excel automation for formatted workbook generation and complex xlsm parsing, macOS programmatic window screenshot capture workflows, and verified Scrapling CLI installation and web extraction workflows", + "version": "1.40.0", "homepage": "https://github.com/daymade/claude-code-skills" }, "plugins": [ @@ -15,7 +15,7 @@ "description": "Essential meta-skill for creating effective Claude Code skills with initialization scripts, validation, packaging, marketplace registration, and privacy best practices", "source": "./", "strict": false, - "version": "1.4.1", + "version": "1.6.0", "category": "developer-tools", "keywords": [ "skill-creation", @@ -50,25 +50,25 @@ ] }, { - "name": "markdown-tools", - "description": "Convert documents (PDFs, Word, PowerPoint) to high-quality markdown with multi-tool orchestration. Supports Quick Mode (fast, single tool) and Heavy Mode (best quality, multi-tool merge with segment-level selection). Features PyMuPDF4LLM for LLM-optimized PDF conversion, pandoc for DOCX/PPTX structure preservation, quality validation with HTML reports, and image extraction with metadata", + "name": "doc-to-markdown", + "description": "Converts DOCX/PDF/PPTX to high-quality Markdown. Pandoc engine + 8 post-processing fixes: grid/simple tables to pipe tables, CJK bold spacing, JSON pretty-print, image path flattening, pandoc attribute cleanup, code block detection, bracket fixes. Benchmarked 7.6/10 (best-in-class vs Docling/MarkItDown/Mammoth). 31 unit tests. Trigger on \"convert document\", \"docx to markdown\", \"parse word\", \"doc to markdown\", \"解析word\", \"转换文档\".", "source": "./", "strict": false, - "version": "1.2.0", + "version": "2.1.0", "category": "document-conversion", "keywords": [ "markdown", - "pdf", "docx", + "pdf", "pptx", - "pymupdf4llm", + "converter", "pandoc", - "markitdown", - "heavy-mode", - "quality-validation" + "document", + "cjk", + "chinese" ], "skills": [ - "./markdown-tools" + "./doc-to-markdown" ] }, { @@ -289,10 +289,10 @@ }, { "name": "transcript-fixer", - "description": "Corrects speech-to-text (ASR/STT) transcription errors in meeting notes, lecture recordings, interviews, and voice memos through dictionary-based rules and AI corrections. Supports Chinese domain names, AI fallback to Claude Code, and iterative dictionary building. Use when users mention transcript correction, ASR errors, speech-to-text mistakes, homophone errors, or working with transcription files", + "description": "Corrects speech-to-text (ASR/STT) transcription errors using dictionary rules and native Claude AI corrections (no API key needed by default). Supports intelligent paragraph breaks, filler word reduction, interactive review, Chinese domain names, and iterative dictionary building. Use when users mention transcript correction, ASR errors, speech-to-text mistakes, homophone errors, or working with transcription files", "source": "./", "strict": false, - "version": "1.1.0", + "version": "1.3.0", "category": "productivity", "keywords": [ "transcription", @@ -414,15 +414,17 @@ }, { "name": "pdf-creator", - "description": "Create PDF documents from markdown with proper Chinese font support using weasyprint. Use when converting markdown to PDF, generating formal documents (legal filings, trademark applications, reports), or when Chinese typography is required. Triggers include convert to PDF, generate PDF, markdown to PDF, or printable documents", + "description": "Create PDF documents from markdown with Chinese font support. Supports theme system (default for formal docs, warm-terra for training materials) and dual backend (weasyprint or Chrome). Triggers include convert to PDF, generate PDF, markdown to PDF, or printable documents", "source": "./", "strict": false, - "version": "1.0.0", + "version": "1.2.0", "category": "document-conversion", "keywords": [ "pdf", "markdown", "weasyprint", + "chrome", + "themes", "chinese-fonts", "document-generation", "legal", @@ -500,7 +502,7 @@ "description": "Develops iOS applications with XcodeGen, SwiftUI, and SPM. Use when configuring XcodeGen project.yml, resolving SPM dependency issues, deploying to devices, handling code signing, debugging camera/AVFoundation, iOS version compatibility issues, or fixing Library not loaded @rpath framework errors. Includes state machine testing patterns for @MainActor classes", "source": "./", "strict": false, - "version": "1.1.1", + "version": "1.1.0", "category": "developer-tools", "keywords": [ "ios", @@ -565,7 +567,7 @@ "description": "Intelligent macOS disk space analysis and cleanup with safety-first philosophy. Use when users report disk space issues, need to clean their Mac, or want to understand storage consumption. Analyzes system caches, application remnants, large files, and development environments (Docker, Homebrew, npm, pip) with risk categorization (Safe/Caution/Keep) and requires explicit user confirmation before any deletions. Includes Mole visual tool integration for hybrid workflow", "source": "./", "strict": false, - "version": "1.1.1", + "version": "1.1.0", "category": "utilities", "keywords": [ "macos", @@ -608,7 +610,7 @@ "description": "Strategic guide for becoming an effective GitHub contributor. Covers opportunity discovery, project selection, high-quality PR creation, and reputation building. Use when looking to contribute to open-source projects, building GitHub presence, or learning contribution best practices", "source": "./", "strict": false, - "version": "1.0.2", + "version": "1.0.3", "category": "developer-tools", "keywords": [ "github", @@ -670,7 +672,7 @@ }, { "name": "meeting-minutes-taker", - "description": "Transform meeting transcripts into high-fidelity, structured meeting minutes with iterative review. Features speaker identification via feature analysis (word count, speaking style, topic focus) with context.md team directory mapping, intelligent file naming from content, integration with markdown-tools and transcript-fixer for pre-processing, evidence-based recording with speaker quotes, Mermaid diagrams for architecture discussions, and multi-turn parallel generation with UNION merge", + "description": "Transform meeting transcripts into high-fidelity, structured meeting minutes with iterative review. Features speaker identification via feature analysis (word count, speaking style, topic focus) with context.md team directory mapping, intelligent file naming from content, integration with doc-to-markdown and transcript-fixer for pre-processing, evidence-based recording with speaker quotes, Mermaid diagrams for architecture discussions, and multi-turn parallel generation with UNION merge", "source": "./", "strict": false, "version": "1.1.0", @@ -691,10 +693,10 @@ }, { "name": "deep-research", - "description": "Generate format-controlled research reports with evidence tracking, citations, and iterative review. Use when users request research reports, literature reviews, market or industry analysis, competitive landscapes, policy or technical briefs, or strict report templates and section formatting", + "description": "Generate format-controlled research reports with evidence tracking, source governance, and multi-pass synthesis. V6.1 adds: source accessibility (circular verification forbidden, exclusive advantage encouraged). Enterprise Research Mode: six-dimension data collection, SWOT/barrier/risk frameworks, and three-level quality control for company research", "source": "./", "strict": false, - "version": "1.0.0", + "version": "2.4.0", "category": "documentation", "keywords": [ "research", @@ -704,7 +706,10 @@ "market-research", "citations", "evidence", - "deepresearch" + "deepresearch", + "enterprise", + "company-research", + "due-diligence" ], "skills": [ "./deep-research" @@ -735,7 +740,7 @@ "description": "Diagnoses and fixes conflicts between Tailscale and proxy/VPN tools (Shadowrocket, Clash, Surge) on macOS. Covers five conflict layers: route hijacking, HTTP proxy env var interception, system proxy bypass, SSH ProxyCommand double tunneling, and VM/container proxy propagation. Includes an automated quick-diagnose script plus SOP for remote development via SSH tunnels with proxy-safe Makefile patterns. Use when Tailscale ping works but SSH/HTTP times out, when browser returns 503 but curl works, when local vanity domains fail behind proxy, when git push fails with failed to begin relaying via HTTP, when Docker pull times out behind TUN/VPN, when setting up Tailscale SSH to WSL instances, or when bootstrapping remote dev environments over Tailscale", "source": "./", "strict": false, - "version": "1.3.0", + "version": "1.4.0", "category": "developer-tools", "keywords": [ "tailscale", @@ -839,7 +844,7 @@ "description": "Programmatic screenshot capture on macOS. Get window IDs via Swift CGWindowListCopyWindowInfo, capture specific windows with screencapture -l, and control application windows via AppleScript. Supports multi-shot workflows for capturing different sections of the same window. Use when taking automated screenshots, capturing application windows, or creating visual documentation", "source": "./", "strict": false, - "version": "1.0.0", + "version": "1.0.1", "category": "utilities", "keywords": [ "screenshot", @@ -876,6 +881,70 @@ "skills": [ "./financial-data-collector" ] + }, + { + "name": "continue-claude-work", + "description": "Recover actionable context from local `.claude` session artifacts and continue interrupted work without running `claude --resume`. Extracts compact boundary summaries, subagent workflow state, session end reason, and workspace drift via bundled Python script. Use when a user provides a Claude session ID, asks to continue prior work from local history, or wants to inspect `.claude` files before resuming implementation", + "source": "./", + "strict": false, + "version": "1.1.1", + "category": "developer-tools", + "keywords": [ + "claude-code", + "session-resume", + "context-recovery", + "jsonl", + "compaction", + "subagent", + "history", + "workflow-continuation", + "local-artifacts" + ], + "skills": [ + "./continue-claude-work" + ] + }, + { + "name": "scrapling-skill", + "description": "Install, troubleshoot, and use Scrapling CLI for extracting HTML, Markdown, or text from webpages. Diagnoses missing extras, Playwright browser runtime issues, TLS verification failures, and WeChat public article extraction patterns. Use when users mention Scrapling, `scrapling extract`, `uv tool install scrapling`, or need to decide between static and browser-backed fetching", + "source": "./", + "strict": false, + "version": "1.0.0", + "category": "developer-tools", + "keywords": [ + "scrapling", + "web-scraping", + "html", + "markdown", + "playwright", + "wechat", + "extraction", + "cli" + ], + "skills": [ + "./scrapling-skill" + ] + }, + { + "name": "asr-transcribe-to-text", + "description": "Transcribe audio and video files to text using a remote ASR service (Qwen3-ASR or OpenAI-compatible endpoint). Extracts audio from video, sends to configurable ASR endpoint, outputs clean text. Use when the user wants to transcribe recordings, convert audio/video to text, do speech-to-text, or mentions ASR, Qwen ASR, 转录, 语音转文字, 录音转文字, or has a meeting recording, lecture, interview, or screen recording to transcribe.", + "source": "./", + "strict": false, + "version": "1.0.0", + "category": "productivity", + "keywords": [ + "asr", + "transcription", + "speech-to-text", + "qwen", + "audio", + "video", + "vllm", + "ffmpeg" + ], + "skills": [ + "./asr-transcribe-to-text" + ] } ] } diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 00000000..89d8979a --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,53 @@ +#!/bin/bash +# Pre-commit hook: scan staged changes for sensitive data +# Install: git config core.hooksPath .githooks + +set -euo pipefail + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +echo "🔍 Scanning staged changes for sensitive data..." + +FAILED=0 + +# Layer 1: gitleaks (if available) +if command -v gitleaks &>/dev/null; then + if ! gitleaks protect --staged --config .gitleaks.toml --no-banner 2>/dev/null; then + echo -e "${RED}❌ gitleaks found secrets in staged changes${NC}" + FAILED=1 + fi +else + echo -e "${YELLOW}⚠ gitleaks not installed (brew install gitleaks), falling back to pattern scan${NC}" +fi + +# Layer 2: fast regex scan (always runs, catches what gitleaks config might miss) +STAGED_DIFF=$(git diff --cached --diff-filter=ACDMR) + +PATTERNS=( + '/Users/[a-zA-Z][a-zA-Z0-9_-]+/' + '/home/[a-zA-Z][a-zA-Z0-9_-]+/' + 'C:\\Users\\[a-zA-Z]' + 'songtiansheng' + 'tiansheng' + '15366[0-9]+' +) + +for pattern in "${PATTERNS[@]}"; do + MATCHES=$(echo "$STAGED_DIFF" | grep -nE "^\+" | grep -E "$pattern" | grep -v "^+++\|\.gitleaks\.toml\|\.githooks/\|\.gitignore\|placeholder\|example\|CLAUDE\.md" || true) + if [ -n "$MATCHES" ]; then + echo -e "${RED}❌ Found sensitive pattern '${pattern}':${NC}" + echo "$MATCHES" | head -5 + FAILED=1 + fi +done + +if [ $FAILED -eq 1 ]; then + echo "" + echo -e "${RED}Commit blocked. Fix the issues above, or use --no-verify to bypass (not recommended).${NC}" + exit 1 +fi + +echo -e "${GREEN}✅ No sensitive data found in staged changes.${NC}" diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 51966ef5..7de86f63 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -16,7 +16,7 @@ Which skill is affected? - [ ] skill-creator - [ ] github-ops -- [ ] markdown-tools +- [ ] doc-to-markdown - [ ] mermaid-tools - [ ] statusline-generator - [ ] teams-channel-post-writer diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index edaf3704..050c3e22 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -20,7 +20,7 @@ Which skill would this enhance? - [ ] skill-creator - [ ] github-ops -- [ ] markdown-tools +- [ ] doc-to-markdown - [ ] mermaid-tools - [ ] statusline-generator - [ ] teams-channel-post-writer diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 27a4c902..54d20160 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -33,7 +33,7 @@ Which skills are affected by this PR? - [ ] skill-creator - [ ] github-ops -- [ ] markdown-tools +- [ ] doc-to-markdown - [ ] mermaid-tools - [ ] statusline-generator - [ ] teams-channel-post-writer diff --git a/.gitignore b/.gitignore index 27c1cf97..64c8d1c0 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,7 @@ Thumbs.db *.tgz *.rar *.7z +*.skill # Build artifacts *.o @@ -81,3 +82,14 @@ INSTALLATION.md # Private/commercial skills (moved to claude-code-skills-pro) seo-expert/ video-creator/ +/jsonl-viewer/ + +# Research output (may contain sensitive data) +deep-research-output/ +recovered_deep_research/ + +# OpenCLI cache +.opencli/ + +# Eval workspaces (contain test data with personal info) +douban-skill-workspace/ diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 00000000..b1b1df73 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,53 @@ +# Gitleaks custom rules for claude-code-skills repo +# Catches personal info that shouldn't be in an open source repo + +title = "claude-code-skills sensitive data rules" + +[extend] +useDefault = true + +# Global allowlist: files that are allowed to contain patterns +# (the config file itself, hooks, and contribution guides) +[allowlist] +paths = [ + '''\.gitleaks\.toml$''', + '''\.githooks/''', + '''CONTRIBUTING\.md$''', + '''CLAUDE\.md$''', +] + +[[rules]] +id = "absolute-user-path-macos" +description = "Hardcoded macOS user home directory path" +regex = '''/Users/[a-zA-Z][a-zA-Z0-9_-]+/''' +tags = ["pii", "path"] + +[[rules]] +id = "absolute-user-path-linux" +description = "Hardcoded Linux home directory path" +regex = '''/home/[a-zA-Z][a-zA-Z0-9_-]+/''' +tags = ["pii", "path"] + +[[rules]] +id = "windows-user-path" +description = "Hardcoded Windows user profile path" +regex = '''C:\\Users\\[a-zA-Z][a-zA-Z0-9_-]+\\''' +tags = ["pii", "path"] + +[[rules]] +id = "phone-number-cn" +description = "Chinese mobile phone number" +regex = '''1[3-9]\d{9}''' +tags = ["pii", "phone"] + +[[rules]] +id = "douban-user-id-literal" +description = "Hardcoded Douban user ID" +regex = '''songtiansheng''' +tags = ["pii", "username"] + +[[rules]] +id = "email-personal" +description = "Personal email address" +regex = '''[a-zA-Z0-9._%+-]+@(gmail|qq|163|126|outlook|hotmail|yahoo|icloud|foxmail)\.[a-zA-Z]{2,}''' +tags = ["pii", "email"] diff --git a/CHANGELOG.md b/CHANGELOG.md index 14c1c277..b9558d97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,54 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- **Renamed**: `markdown-tools` → `doc-to-markdown` — clearer name for DOCX/PDF/PPTX → Markdown conversion +- **doc-to-markdown**: Added 8 DOCX post-processing fixes (grid tables, simple tables, CJK bold spacing, JSON pretty-print, image path flattening, pandoc attribute cleanup, code block detection, bracket fixes) +- **doc-to-markdown**: Added 31 unit tests (`test_convert.py`) +- **doc-to-markdown**: Added 5-tool benchmark report (`references/benchmark-2026-03-22.md`) + +## [1.39.0] - 2026-03-18 + ### Added -- None +- **New Skill**: scrapling-skill v1.0.0 - Reliable Scrapling CLI installation, troubleshooting, and extraction workflows for HTML, Markdown, and text output + - Bundled `diagnose_scrapling.py` script to verify CLI health, detect missing extras, inspect Playwright browser runtime, and run real smoke tests + - Static-first workflow for choosing between `extract get`, `extract fetch`, and `stealthy-fetch` + - Verified WeChat public article extraction pattern using `#js_content` + - Verified recovery path for local TLS trust-store failures via `--no-verify` + - Bundled troubleshooting reference covering extras, browser runtime, and output validation + +### Changed +- **skill-creator** v1.5.0 → v1.5.1: Fixed `scripts/package_skill.py` so it works when invoked directly from the repository root instead of only via `python -m` +- **continue-claude-work** v1.1.0 → v1.1.1: Replaced newer Python-only type syntax in `extract_resume_context.py` so the script runs under the local `python3` environment +- Updated marketplace skills/plugins count from 42 to 43 +- Updated marketplace version from 1.38.0 to 1.39.0 +- Updated marketplace metadata description to include Scrapling CLI extraction workflows +- Updated README.md and README.zh-CN.md badges, installation commands, skill listings, use cases, quick links, and requirements +- Updated CLAUDE.md counts, version reference, and Available Skills list (added #43) + +## [1.38.0] - 2026-03-07 + +### Added +- **New Skill**: continue-claude-work v1.1.0 - Recover local `.claude` session context and continue interrupted work without `claude --resume` + - Bundled Python script (`extract_resume_context.py`) for one-call context extraction + - Compact-boundary-aware extraction using `isCompactSummary` flag (highest-signal context from session compaction summaries) + - Subagent workflow recovery — parses `subagents/` directory to report completed vs interrupted agents with last outputs + - Session end reason detection — classifies clean exit, interrupted (ctrl-c), error cascade, or abandoned + - Size-adaptive reading strategy based on file size and compaction count + - Noise filtering — skips progress/queue-operation/api_error (37-53% of session lines) + - Self-session exclusion, stale index fallback, ghost session warnings + - MEMORY.md and session-memory integration, git workspace state fusion + +### Changed +- **skill-creator** v1.4.1 → v1.5.0: SKILL.md rewrite, added eval benchmarking system (run_eval, run_loop, aggregate_benchmark), agents (analyzer, comparator, grader), eval-viewer, and improve_description script +- **transcript-fixer** v1.1.0 → v1.2.0: `--domain` defaults to all domains, added `get_domain_stats()`, cross-domain listing, and zero-match hints +- **tunnel-doctor** v1.3.0 → v1.4.0: Added Step 2C-1 for local vanity domain proxy interception, bundled `quick_diagnose.py` automated diagnostic script +- **pdf-creator** v1.0.0 → v1.1.0: Replaced Python `markdown` library with pandoc for MD→HTML conversion, removed `_ensure_list_spacing` workaround +- **github-contributor** v1.0.2 → v1.0.3: Fixed gh CLI field name (`stargazersCount` → `stargazerCount`), added Prerequisites section +- Updated marketplace skills/plugins count from 41 to 42 +- Updated marketplace version from 1.37.0 to 1.38.0 +- Updated README.md and README.zh-CN.md badges, installation commands, skill listings, use cases, quick links, and requirements +- Updated CLAUDE.md counts, version reference, and Available Skills list (added #42) ## [1.37.0] - 2026-03-02 diff --git a/CLAUDE.md b/CLAUDE.md index 93c52c7f..8c6d2678 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Repository Overview -This is a Claude Code skills marketplace containing 41 production-ready skills organized in a plugin marketplace structure. Each skill is a self-contained package that extends Claude's capabilities with specialized knowledge, workflows, and bundled resources. +This is a Claude Code skills marketplace containing 43 production-ready skills organized in a plugin marketplace structure. Each skill is a self-contained package that extends Claude's capabilities with specialized knowledge, workflows, and bundled resources. **Essential Skill**: `skill-creator` is the most important skill in this marketplace - it's a meta-skill that enables users to create their own skills. Always recommend it first for users interested in extending Claude Code. @@ -115,13 +115,22 @@ description: Clear description with activation triggers. This skill should be us --- ``` -### Privacy and Path Guidelines +### Privacy and Path Guidelines (Enforced by Pre-commit Hook) Skills for public distribution must NOT contain: - Absolute paths to user directories (`/home/username/`, `/Users/username/`) - Personal usernames, company names, product names +- Phone numbers, personal email addresses - OneDrive paths or environment-specific absolute paths -- Use relative paths within skill bundle or standard placeholders +- Use relative paths within skill bundle or standard placeholders (`~/workspace/`, ``) + +**Three-layer defense system:** +1. **CLAUDE.md rules** (this section) — Claude avoids generating sensitive content +2. **Pre-commit hook** (`.githooks/pre-commit`) — blocks commits with sensitive patterns +3. **gitleaks** (`.gitleaks.toml`) — deep scan with custom rules for this repo + +The pre-commit hook is auto-activated via `git config core.hooksPath .githooks`. +If it fires, fix the issue — do NOT use `--no-verify` to bypass. ### Content Organization @@ -134,7 +143,7 @@ Skills for public distribution must NOT contain: ## Marketplace Configuration The marketplace is configured in `.claude-plugin/marketplace.json`: - - Contains 41 plugins, each mapping to one skill +- Contains 43 plugins, each mapping to one skill - Each plugin has: name, description, version, category, keywords, skills array - Marketplace metadata: name, owner, version, homepage @@ -144,7 +153,7 @@ The marketplace is configured in `.claude-plugin/marketplace.json`: 1. **Marketplace Version** (`.claude-plugin/marketplace.json` → `metadata.version`) - Tracks the marketplace catalog as a whole - - Current: v1.37.0 + - Current: v1.39.0 - Bump when: Adding/removing skills, major marketplace restructuring - Semantic versioning: MAJOR.MINOR.PATCH @@ -179,7 +188,7 @@ This applies when you change ANY file under a skill directory: 1. **skill-creator** ⭐ - **Essential meta-skill** for creating your own skills (with init/validate/package scripts) 2. **github-ops** - GitHub operations via gh CLI and API -3. **markdown-tools** - Document conversion with WSL path handling +3. **doc-to-markdown** - DOCX/PDF/PPTX → Markdown conversion with CJK post-processing 4. **mermaid-tools** - Diagram extraction and PNG generation 5. **statusline-generator** - Claude Code statusline customization 6. **teams-channel-post-writer** - Teams communication templates @@ -212,27 +221,20 @@ This applies when you change ANY file under a skill directory: 33. **meeting-minutes-taker** - Transform meeting transcripts into structured minutes with multi-pass generation, speaker quotes, and iterative human review 34. **deep-research** - Generate format-controlled research reports with evidence mapping, citations, and multi-pass synthesis 35. **competitors-analysis** - Evidence-based competitor tracking and analysis with source citations (file:line_number format) - 36. **tunnel-doctor** - Diagnose and fix Tailscale + proxy/VPN conflicts (four layers: route, HTTP env, system proxy, SSH ProxyCommand) on macOS with WSL SSH support - 37. **windows-remote-desktop-connection-doctor** - Diagnose AVD/W365 connection quality issues with transport protocol analysis and Windows App log parsing - 38. **product-analysis** - Perform structured product audits across UX, API, architecture, and compare mode to produce prioritized optimization recommendations - 39. **financial-data-collector** - Collect real financial data for US public companies via yfinance with validation, NaN detection, and NO FALLBACK principle - 40. **excel-automation** - Create formatted Excel files, parse complex xlsm models, and control Excel windows on macOS via AppleScript - 41. **capture-screen** - Programmatically capture macOS application windows using Swift window ID discovery and screencapture workflows +36. **tunnel-doctor** - Diagnose and fix Tailscale + proxy/VPN conflicts (four layers: route, HTTP env, system proxy, SSH ProxyCommand) on macOS with WSL SSH support +37. **windows-remote-desktop-connection-doctor** - Diagnose AVD/W365 connection quality issues with transport protocol analysis and Windows App log parsing +38. **product-analysis** - Perform structured product audits across UX, API, architecture, and compare mode to produce prioritized optimization recommendations +39. **financial-data-collector** - Collect real financial data for US public companies via yfinance with validation, NaN detection, and NO FALLBACK principle +40. **excel-automation** - Create formatted Excel files, parse complex xlsm models, and control Excel windows on macOS via AppleScript +41. **capture-screen** - Programmatically capture macOS application windows using Swift window ID discovery and screencapture workflows +42. **continue-claude-work** - Recover local `.claude` session context via compact-boundary extraction, subagent workflow recovery, and session end reason detection, then continue interrupted work without `claude --resume` +43. **scrapling-skill** - Install, troubleshoot, and use Scrapling CLI for static/dynamic web extraction, WeChat article capture, and verified output validation **Recommendation**: Always suggest `skill-creator` first for users interested in creating skills or extending Claude Code. ## YouTube Downloader SOP (Internal) -Use this SOP to avoid common yt-dlp failures and confusion: - -1. Quote YouTube URLs in shell commands (zsh treats `?` as glob). Example: `'https://www.youtube.com/watch?v=VIDEO_ID'`. -2. Ensure proxy is active for both yt-dlp and PO Token providers (HTTP_PROXY/HTTPS_PROXY/ALL_PROXY). -3. If you see “Sign in to confirm you’re not a bot”, request cookie permission and use browser cookies. -4. Start the PO Token provider before downloading. Prefer Docker bgutil; fall back to browser-based WPC when Docker is unavailable or fails. -5. Use `web_safari` client when cookies are present; otherwise use `mweb` for PO tokens. -6. Keep the browser window open while WPC is minting tokens and make sure it can reach YouTube through the same proxy. -7. If you see “Only images are available” or “Requested format is not available”, treat it as PO token failure and retry after fixing provider/browser state. -8. If you see SSL EOF or fragment errors, treat it as proxy instability. Retry with progressive formats or switch to a more stable proxy. +See [youtube-downloader/references/internal-sop.md](./youtube-downloader/references/internal-sop.md) for yt-dlp troubleshooting steps (PO tokens, proxy, cookies, etc.). ## Python Development @@ -266,326 +268,35 @@ When creating a new skill: ## Adding a New Skill to Marketplace -**CRITICAL**: When adding a skill to this marketplace, you MUST update all of these files in the correct order. Missing any file will result in incomplete integration. +For the full step-by-step guide with templates and examples, see [references/new-skill-guide.md](./references/new-skill-guide.md). -### Step-by-Step Process +**Files to update** (all required): -#### 1. Refine the Skill (if needed) -```bash -# Ensure skill follows best practices -# - SKILL.md uses imperative/infinitive form -# - Third-person description in YAML frontmatter -# - Progressive disclosure (details in references/) -# - Security scan passed - -cd skill-creator -python3 scripts/security_scan.py ../skill-name --verbose -``` +| File | Locations to update | +|------|-------------------| +| `.claude-plugin/marketplace.json` | metadata.version + metadata.description + new plugin entry | +| `CHANGELOG.md` | New version entry | +| `README.md` | 7 locations: badges, description, install cmd, skill section, use case, docs link, requirements | +| `README.zh-CN.md` | 7 locations: same as above, translated | +| `CLAUDE.md` | 3 locations: overview count, marketplace config count, Available Skills list | +| `skill-name/` | The actual skill directory + packaged .zip | -#### 2. Package the Skill +**Quick workflow**: ```bash -cd skill-creator +# 1. Validate & package +cd skill-creator && python3 scripts/security_scan.py ../skill-name --verbose python3 scripts/package_skill.py ../skill-name -# This will: -# - Validate skill structure -# - Check security scan status -# - Create skill-name.zip in skill-creator/ -# - Move zip to skill-name/ directory -``` - -#### 3. Update CHANGELOG.md ⚠️ REQUIRED - -Add new version entry at the top (after [Unreleased]): - -```markdown -## [X.Y.0] - YYYY-MM-DD - -### Added -- **New Skill**: skill-name - Brief description - - Feature 1 - - Feature 2 - - Feature 3 - - Bundled scripts/references/assets - - Key capabilities - -### Changed -- Updated marketplace skills count from N to N+1 -- Updated marketplace version from X.(Y-1).0 to X.Y.0 -- Updated README.md badges (skills count, version) -- Updated README.md to include skill-name in skills listing -- Updated README.zh-CN.md badges (skills count, version) -- Updated README.zh-CN.md to include skill-name in skills listing -- Updated CLAUDE.md skills count from N to N+1 -- Added skill-name use case section to README.md -- Added skill-name use case section to README.zh-CN.md -- Added dependencies to requirements section (if any, both EN and ZH) -``` - -**Version numbering**: Increment MINOR version (e.g., 1.8.0 → 1.9.0) when adding a skill. - -#### 4. Update README.md ⚠️ REQUIRED - -**a. Update badges (top of file):** -```markdown -[![Skills](https://img.shields.io/badge/skills-N-blue.svg)] -[![Version](https://img.shields.io/badge/version-X.Y.0-green.svg)] -``` - -**b. Update description:** -```markdown -Professional Claude Code skills marketplace featuring N production-ready skills... -``` - -**c. Add installation command:** -```markdown -# Brief description -claude plugin install skill-name@daymade-skills -``` - -**d. Add skill section (### N. **skill-name**):** -```markdown -### N. **skill-name** - One-line Title - -Brief description paragraph. - -**When to use:** -- Use case 1 -- Use case 2 -- Use case 3 - -**Key features:** -- Feature 1 -- Feature 2 -- Feature 3 - -**Example usage:** -\`\`\`bash -# Example commands -\`\`\` - -**🎬 Live Demo** - -*Coming soon* (or add demo GIF) - -📚 **Documentation**: See [skill-name/references/](./skill-name/references/)... - -**Requirements**: Dependencies (e.g., Python 3.8+, FFmpeg, etc.) -``` - -**e. Add use case section:** -```markdown -### For [Use Case Category] -Use **skill-name** to [describe primary use case]. Combine with **other-skill** to [describe integration]. -``` - -**f. Add documentation quick link:** -```markdown -- **skill-name**: See `skill-name/references/...` for ... -``` - -**g. Update requirements section (if needed):** -```markdown -- **Tool Name** (for skill-name): `install command` -``` - -#### 5. Update CLAUDE.md ⚠️ REQUIRED - -**a. Update repository overview:** -```markdown -This is a Claude Code skills marketplace containing N production-ready skills... -``` - -**b. Update marketplace configuration:** -```markdown -The marketplace is configured in `.claude-plugin/marketplace.json`: -- Contains N plugins, each mapping to one skill -``` - -**c. Update marketplace version:** -```markdown -1. **Marketplace Version** (`.claude-plugin/marketplace.json` → `metadata.version`) - - Tracks the marketplace catalog as a whole - - Current: vX.Y.0 -``` - -**d. Add skill to Available Skills list:** -```markdown -N. **skill-name** - Brief description with key feature -``` - -#### 6. Update .claude-plugin/marketplace.json ⚠️ CRITICAL - -**MOST IMPORTANT FILE** - This file makes the skill installable! - -**a. Update metadata.description:** -```json -"description": "Professional Claude Code skills for ..., and [new skill capability]" -``` - -**b. Update metadata.version:** -```json -"version": "X.Y.0" -``` - -**c. Add new plugin entry to plugins array:** -```json -{ - "name": "skill-name", - "description": "Clear description with trigger conditions. Use when [scenarios]", - "source": "./", - "strict": false, - "version": "1.0.0", - "category": "appropriate-category", - "keywords": ["keyword1", "keyword2", "keyword3", ...], - "skills": ["./skill-name"] -} -``` - -**Categories:** `developer-tools`, `document-conversion`, `documentation`, `customization`, `communication`, `utilities`, `assets`, `design`, `productivity`, `security`, `media` - -**d. Validate JSON syntax:** -```bash -python3 -m json.tool .claude-plugin/marketplace.json > /dev/null -``` - -#### 7. Update README.zh-CN.md ⚠️ REQUIRED - -**CRITICAL**: Chinese documentation must be kept in sync with English version. - -**a. Update badges (top of file):** -```markdown -[![Skills](https://img.shields.io/badge/skills-N-blue.svg)] -[![Version](https://img.shields.io/badge/version-X.Y.0-green.svg)] -``` - -**b. Update description:** -```markdown -专业的 Claude Code 技能市场，提供 N 个生产就绪的技能，用于增强开发工作流。 -``` +# 2. Update all files listed above (see references/new-skill-guide.md for details) -**c. Add installation command:** -```markdown -# 简短描述 -claude plugin install skill-name@daymade-skills -``` - -**d. Add skill section (### N. **skill-name** - Chinese Title):** -- Translate all content from English README -- Include: 使用场景 (When to use), 主要功能 (Key features), 示例用法 (Example usage) -- Maintain same structure as English version -- Include documentation links and requirements - -**e. Add use case section:** -```markdown -### [Use Case Category in Chinese] -使用 **skill-name** [describe use case in Chinese]. 与 **other-skill** 结合使用以 [describe integration]. -``` - -**f. Add documentation quick link:** -```markdown -- **skill-name**：参见 `skill-name/references/...` 了解 ... -``` - -**g. Update requirements section (if needed):** -```markdown -- **Tool Name**（用于 skill-name）：`install command` -``` - -**Translation tips:** -- Use professional technical Chinese -- Maintain consistency with existing translations -- Keep code examples in English (don't translate variable names, function names) -- Translate user-facing descriptions, features, and use cases - -#### 8. Verification Checklist - -Before committing, verify: - -- [ ] CHANGELOG.md has new version entry -- [ ] README.md badges updated (skills count + version) -- [ ] README.md has skill section with number -- [ ] README.md has use case section -- [ ] README.md has documentation link -- [ ] README.md requirements updated (if needed) -- [ ] README.zh-CN.md badges updated (skills count + version) ⚠️ NEW -- [ ] README.zh-CN.md has skill section with number ⚠️ NEW -- [ ] README.zh-CN.md has use case section ⚠️ NEW -- [ ] README.zh-CN.md has documentation link ⚠️ NEW -- [ ] README.zh-CN.md requirements updated (if needed) ⚠️ NEW -- [ ] README.zh-CN.md installation command added ⚠️ NEW -- [ ] CLAUDE.md skill count updated in 3 places -- [ ] CLAUDE.md has skill in Available Skills list -- [ ] marketplace.json metadata.version updated -- [ ] marketplace.json metadata.description updated -- [ ] marketplace.json has new plugin entry -- [ ] marketplace.json validates (python3 -m json.tool) -- [ ] skill-name.zip package exists -- [ ] Security scan passed - -### Common Mistakes to Avoid - -1. **Forgetting marketplace.json** ⚠️ - The most critical file! Without this, the skill cannot be installed via `claude plugin install` -2. **Forgetting Chinese documentation** ⚠️ - README.zh-CN.md must be updated in sync with README.md (6 locations) -3. **Inconsistent version numbers** - CHANGELOG, README badges (both EN and ZH), CLAUDE.md, and marketplace.json must all match -4. **Inconsistent skill counts** - README description (both EN and ZH), badges, CLAUDE.md must all have same count -5. **Missing skill number in README** - Skills must be numbered sequentially (1, 2, 3, ...) in both EN and ZH versions -6. **Invalid JSON syntax** - Always validate marketplace.json after editing -7. **Forgetting dependencies** - Update README requirements section (both EN and ZH) if skill needs external tools -8. **Incomplete Chinese translation** - Must translate all sections: description, use cases, features, use case section, docs link - -### File Update Summary Template - -When adding a skill, this is the complete file list: - -``` -Files to Update: -✅ CHANGELOG.md (Add version entry) -✅ README.md (7 locations: badges, description, install, skill section, use case, docs link, requirements) -✅ README.zh-CN.md (7 locations: badges, description, install, skill section, use case, docs link, requirements) ⚠️ CRITICAL -✅ CLAUDE.md (3 locations: overview, marketplace config, available skills) -✅ .claude-plugin/marketplace.json (CRITICAL: metadata + new plugin entry) -✅ skill-name/ (The actual skill directory) -✅ skill-name/skill-name.zip (Packaged skill) +# 3. Validate, commit, push, release +cd .. && python3 -m json.tool .claude-plugin/marketplace.json > /dev/null +git add -A && git commit -m "Release vX.Y.0: Add skill-name" +git push +gh release create vX.Y.0 --title "Release vX.Y.0: Add skill-name" --notes "..." ``` -**IMPORTANT**: README.zh-CN.md is MANDATORY. Do not skip Chinese documentation updates! - -### Version Numbering Convention - -- **MAJOR.MINOR.PATCH** (Semantic Versioning) -- Increment **MINOR** when adding a new skill: 1.8.0 → 1.9.0 -- Increment **PATCH** for bug fixes or small updates: 1.9.0 → 1.9.1 -- Increment **MAJOR** for breaking changes or major restructuring: 1.9.0 → 2.0.0 - -### Quick Reference Commands - -```bash -# 1. Refine and validate skill -cd skill-creator -python3 scripts/security_scan.py ../skill-name --verbose - -# 2. Package skill -python3 scripts/package_skill.py ../skill-name - -# 3. Validate marketplace.json -cd .. -python3 -m json.tool .claude-plugin/marketplace.json > /dev/null && echo "✅ Valid" - -# 4. Check what needs committing -git status - -# 5. View specific file changes -git diff CHANGELOG.md -git diff README.md -git diff README.zh-CN.md -git diff CLAUDE.md -git diff .claude-plugin/marketplace.json - -# 6. Verify Chinese documentation is in sync -grep "skills-[0-9]*" README.md README.zh-CN.md -grep "version-[0-9.]*" README.md README.zh-CN.md -``` +**Top mistakes**: Forgetting to push to GitHub, forgetting README.zh-CN.md, inconsistent version numbers across files. ## Chinese User Support @@ -599,950 +310,19 @@ See README.md section "🇨🇳 中文用户指南" for details. ## Handling Third-Party Marketplace Promotion Requests -This repository is a **personal curated marketplace**, NOT a community directory or ecosystem hub. All requests to add third-party marketplace links, skill collection references, or "Community Marketplaces" sections should be declined. - -### Policy - -**DO NOT accept:** -- PRs adding "Related Resources" or "Community Marketplaces" sections linking to third-party skill collections -- Issues requesting promotion of external marketplaces -- PRs adding links to other skill repositories in README.md - -**Rationale:** -1. **Scope creep**: Shifts repository purpose from curated skills to ecosystem directory -2. **Implicit endorsement**: Listing implies quality/security review we cannot maintain -3. **Maintenance burden**: Would need to track and vet external projects over time -4. **Precedent setting**: Accepting one creates obligation to accept others - -### Response Template - -When declining, use this approach: - -```markdown -Hi @{username}, - -Thank you for your interest and for sharing {project-name}! {Brief positive acknowledgment of their project}. - -However, I'm keeping this repository focused as a **personal curated marketplace** rather than a directory of external skill collections. Adding third-party references would: - -1. Shift the repository's scope from curated skills to ecosystem directory -2. Create implicit endorsement expectations I can't maintain -3. Set precedent for similar requests (reference other declined requests if applicable) - -**What you can do instead:** - -1. **Standalone marketplace** - Your repo already works as an independent marketplace: - ``` - /plugin marketplace add {owner}/{repo} - ``` - -2. **Community channels** - Promote through: - - Claude Code GitHub discussions/issues (Anthropic's official repo) - - Developer communities (Reddit, Discord, etc.) - - Your own blog/social media - -3. **Official registry** - If/when Anthropic launches an official skill registry, that would be the appropriate place for ecosystem-wide discovery. - -Your marketplace can succeed on its own merits. Good luck with {project-name}! -``` - -### Workflow - -1. **Review the request** - Confirm it's a third-party promotion (not a legitimate contribution) -2. **Add polite comment** - Use template above, customize for their specific project -3. **Close with reason** - Use "not planned" for issues, just close for PRs -4. **Reference precedent** - Link to previously declined requests for consistency (e.g., #7, PR #5) - -### Examples - -- **Issue #7**: "Add Community Marketplaces section - Protocol Thunderdome" → Declined, closed as "not planned" -- **PR #5**: "Add Trail of Bits Security Skills to Related Resources" → Declined, closed - -## Release Workflow - -When adding a new skill or creating a marketplace release: - -### 1. Create the Skill -```bash -# Develop skill in its directory -skill-name/ -├── SKILL.md (no version history!) -├── scripts/ -└── references/ - -# Validate -./skill-creator/scripts/quick_validate.py skill-name - -# Package -./skill-creator/scripts/package_skill.py skill-name -``` - -### 2. Update Marketplace Configuration - -Edit `.claude-plugin/marketplace.json`: - -```json -{ - "metadata": { - "version": "1.x.0" // Bump minor version for new skill - }, - "plugins": [ - { - "name": "new-skill", - "version": "1.0.0", // Skill's initial version - "description": "...", - "category": "...", - "keywords": [...], - "skills": ["./new-skill"] - } - ] -} -``` - -### 3. Update Documentation - -**README.md:** -- Update badges (skills count, marketplace version) -- Add skill description and features -- Create demo GIF using cli-demo-generator -- Add use case section -- Add documentation references -- Add requirements (if applicable) - -**CLAUDE.md:** -- Update skill count in Repository Overview -- Add skill to Available Skills list -- Update Marketplace Configuration count - -### 4. Generate Demo (Optional but Recommended) - -```bash -# Use cli-demo-generator to create demo GIF -./cli-demo-generator/scripts/auto_generate_demo.py \ - -c "command1" \ - -c "command2" \ - -o demos/skill-name/demo-name.gif \ - --title "Skill Demo" \ - --theme "Dracula" -``` - -### 5. Commit and Release - -```bash -# Commit marketplace update -git add .claude-plugin/marketplace.json skill-name/ -git commit -m "Release vX.Y.0: Add skill-name - -- Add skill-name vX.Y.Z -- Update marketplace to vX.Y.0 -..." - -# Commit documentation -git add README.md CLAUDE.md demos/ -git commit -m "docs: Update README for vX.Y.0 with skill-name" - -# Push -git push - -# Create GitHub release -gh release create vX.Y.0 \ - --title "Release vX.Y.0: Add skill-name - Description" \ - --notes "$(cat <<'EOF' -## New Skill: skill-name - -Features: -- Feature 1 -- Feature 2 - -Installation: -```bash -claude plugin install skill-name@daymade-skills -``` - -Changelog: ... -EOF -)" -``` - -### Version Bumping Guide - -**Marketplace version (metadata.version):** -- **MAJOR** (2.0.0): Breaking changes, incompatible marketplace structure -- **MINOR** (1.5.0): New skill added, significant feature addition -- **PATCH** (1.4.1): Bug fixes, documentation updates, skill updates - -**Skill version (plugins[].version):** -- **MAJOR** (2.0.0): Breaking API changes for the skill -- **MINOR** (1.2.0): New features in the skill -- **PATCH** (1.1.1): Bug fixes in the skill - -### Example: v1.5.0 Release (ppt-creator) - -```bash -# 1. Created ppt-creator skill -# 2. Updated marketplace.json: 1.4.0 → 1.5.0 -# 3. Added ppt-creator plugin entry (version: 1.0.0) -# 4. Updated README.md (badges, description, demo) -# 5. Generated demo GIF with cli-demo-generator -# 6. Committed changes -# 7. Created GitHub release with gh CLI -``` +Decline all third-party marketplace promotion requests. For policy, response template, and precedents, see [references/promotion-policy.md](./references/promotion-policy.md). ## Best Practices Reference Always consult Anthropic's skill authoring best practices before creating or updating skills: https://docs.claude.com/en/docs/agents-and-tools/agent-skills/best-practices.md -- remember this release workflow in claude.md ## Plugin and Skill Architecture -This section explains the architecture of Claude Code's extension system and how different components work together. - -### Core Concepts - -#### 1. Skills - -**What**: Functional units that extend Claude's capabilities with specialized knowledge and workflows. - -**Structure**: -``` -skill-name/ -├── SKILL.md (required) # YAML frontmatter + Markdown instructions -├── scripts/ (optional) # Executable code (Python/Bash) -├── references/ (optional) # Documentation loaded as needed -└── assets/ (optional) # Templates and resources -``` - -**Loading mechanism** (Progressive Disclosure): -1. **Metadata** (~100 tokens): Always in context (name + description from YAML frontmatter) -2. **SKILL.md body** (<5k tokens): Loaded when Claude determines the skill applies -3. **Bundled resources**: Loaded only as needed by Claude - -**Location**: -- **Personal**: `~/.claude/skills/` (user-specific, not shared) -- **Project**: `.claude/skills/` (checked into git, shared with team) -- **Plugin cache**: `~/.claude/plugins/cache/{marketplace}/{plugin}/{version}/{skill}/` - -**Example**: When you ask "analyze my disk space", Claude loads the `macos-cleaner` skill's SKILL.md, then reads `references/cleanup_targets.md` as needed. - -#### 2. Plugins - -**What**: Distribution units that package one or more skills for installation via marketplaces. - -**Purpose**: Plugins enable: -- One-command installation (`claude plugin install skill-name@marketplace-name`) -- Version management -- Dependency tracking -- Marketplace distribution - -**Relationship to Skills**: -``` -Plugin (marketplace.json entry) -├── Skill 1 (./skill-name-1/) -├── Skill 2 (./skill-name-2/) -└── Skill 3 (./skill-name-3/) -``` - -**Configuration** (in `.claude-plugin/marketplace.json`): -```json -{ - "name": "my-plugin", - "description": "Use when...", - "version": "1.0.0", - "category": "utilities", - "keywords": ["keyword1", "keyword2"], - "skills": ["./skill-1", "./skill-2"] -} -``` - -**Example**: The `skill-creator` plugin contains one skill (`./skill-creator`), while a hypothetical `developer-tools` plugin might contain multiple skills like `./git-helper`, `./code-reviewer`, `./test-runner`. - -#### 3. Agents (Subagents) - -**What**: Specialized autonomous agents invoked via the `Task` tool for complex, multi-step operations. - -**Types**: -- **Bash**: Command execution specialist -- **general-purpose**: Research, search, multi-step tasks -- **Explore**: Fast codebase exploration -- **Plan**: Software architecture planning -- **skill-creator**: Meta-agent for creating skills -- **Custom**: Domain-specific agents (e.g., `test-runner`, `build-validator`) - -**When to use**: -- Tasks requiring multiple rounds of tool calls -- Open-ended exploration (finding files, searching code) -- Planning before implementation -- Autonomous execution without user intervention - -**Example**: -```python -# Instead of manually searching multiple times: -Task( - subagent_type="Explore", - description="Find error handling code", - prompt="Search the codebase for error handling patterns and list all files that handle HTTP errors" -) -``` - -#### 4. Commands - -**What**: Slash commands (e.g., `/commit`, `/review-pr`) that trigger skills. - -**Relationship**: Commands are shortcuts to invoke skills. -- `/commit` → invokes `commit` skill -- `/review-pr` → invokes `review-pr` skill - -**Configuration**: Defined in plugin's `commands/` directory or skill metadata. - -### Architecture Diagram - -``` -Marketplace (GitHub) - ↓ (git clone) -~/.claude/plugins/marketplaces/{marketplace-name}/ - ↓ (plugin install) -~/.claude/plugins/cache/{marketplace-name}/{plugin}/{version}/ - ├── skill-1/ - │ ├── SKILL.md - │ ├── scripts/ - │ └── references/ - └── skill-2/ - └── SKILL.md - ↓ (Claude loads) -Claude Code Context - ├── Metadata (always loaded) - ├── SKILL.md (loaded when relevant) - └── Resources (loaded as needed) -``` - -### Installation Flow - -#### Step 1: User initiates installation -```bash -claude plugin install macos-cleaner@daymade-skills -``` - -#### Step 2: CLI locates marketplace -```bash -# Check ~/.claude/plugins/marketplaces/daymade-skills/ -# If not exists, git clone from GitHub -``` - -#### Step 3: Read marketplace.json -```json -{ - "plugins": [ - { - "name": "macos-cleaner", - "version": "1.0.0", - "skills": ["./macos-cleaner"] - } - ] -} -``` - -#### Step 4: Download to cache -```bash -# Clone entire marketplace repo to: -~/.claude/plugins/cache/daymade-skills/macos-cleaner/1.0.0/ - -# Extract skill to: -~/.claude/plugins/cache/daymade-skills/macos-cleaner/1.0.0/macos-cleaner/ -``` - -#### Step 5: Record installation -```json -// ~/.claude/plugins/installed_plugins.json -{ - "plugins": { - "macos-cleaner@daymade-skills": [{ - "scope": "user", - "installPath": "~/.claude/plugins/cache/daymade-skills/macos-cleaner/1.0.0", - "version": "1.0.0", - "installedAt": "2026-01-11T08:03:46.593Z" - }] - } -} -``` - -#### Step 6: Claude Code loads skill -``` -When user asks: "My Mac is running out of space" - ↓ -Claude scans installed plugins metadata - ↓ -Finds "macos-cleaner" description matches - ↓ -Loads SKILL.md into context - ↓ -Executes workflow (analyze → report → confirm → cleanup) - ↓ -Loads references/scripts as needed -``` - -### Key Files and Locations - -#### Configuration Files - -| File | Location | Purpose | -|------|----------|---------| -| `marketplace.json` | `~/.claude/plugins/marketplaces/{name}/.claude-plugin/` | Defines available plugins | -| `installed_plugins.json` | `~/.claude/plugins/` | Tracks installed plugins | -| `known_marketplaces.json` | `~/.claude/plugins/` | Lists registered marketplaces | - -#### Directory Structure - -``` -~/.claude/ -├── skills/ # Personal skills (not from marketplace) -├── plugins/ -│ ├── marketplaces/ # Marketplace clones -│ │ ├── daymade-skills/ # Marketplace name -│ │ │ └── .claude-plugin/ -│ │ │ └── marketplace.json -│ │ └── anthropic-agent-skills/ -│ ├── cache/ # Installed plugins -│ │ └── daymade-skills/ -│ │ └── macos-cleaner/ -│ │ └── 1.0.0/ # Version -│ │ └── macos-cleaner/ # Skill directory -│ │ ├── SKILL.md -│ │ ├── scripts/ -│ │ └── references/ -│ ├── installed_plugins.json # Installation registry -│ └── known_marketplaces.json # Marketplace registry -``` - -### Data Flow - -#### Skill Activation -``` -User message - ↓ -Claude analyzes installed plugin metadata - ↓ -Matches description to user intent - ↓ -Loads SKILL.md (progressive disclosure) - ↓ -Executes instructions - ↓ -Loads bundled resources (scripts, references) as needed - ↓ -Generates response -``` - -#### Plugin Update -``` -Local changes to skill - ↓ -git add & commit - ↓ -git push to GitHub - ↓ -User runs: claude plugin marketplace update {marketplace-name} - ↓ -CLI pulls latest from GitHub - ↓ -Updates ~/.claude/plugins/marketplaces/{marketplace-name}/ - ↓ -User runs: claude plugin update {plugin-name@marketplace} - ↓ -Re-downloads to cache with new version number - ↓ -Updates installed_plugins.json -``` - -### Common Misconceptions - -#### ❌ Myth 1: "Updating local files immediately updates the plugin" -**Reality**: Plugins are distributed via GitHub. Local changes require `git push` before users can install updates. - -#### ❌ Myth 2: "Skills and plugins are the same thing" -**Reality**: Skills are functional units (SKILL.md + resources). Plugins are distribution packages (can contain multiple skills). - -#### ❌ Myth 3: "marketplace.json is just metadata" -**Reality**: marketplace.json is the **source of truth** for plugin discovery. Without correct configuration here, `claude plugin install` will fail with "Plugin not found". - -#### ❌ Myth 4: "Cache is just for performance" -**Reality**: Cache (`~/.claude/plugins/cache/`) is where installed plugins actually live. Deleting cache uninstalls all plugins. - -#### ❌ Myth 5: "Skills in ~/.claude/skills/ work the same as plugin skills" -**Reality**: -- `~/.claude/skills/` = Personal skills (manual management, no versioning) -- Plugin cache = Managed by CLI (versioned, updateable, shareable) - -### Best Practices - -#### For Skill Authors - -1. **Clear metadata**: Description should clearly state "Use when..." to help Claude match user intent -2. **Progressive disclosure**: Keep SKILL.md lean, move details to `references/` -3. **Test locally first**: Copy to `~/.claude/skills/` for testing before packaging -4. **Version properly**: Use semver (MAJOR.MINOR.PATCH) in marketplace.json -5. **Document bundled resources**: All scripts and references should be mentioned in SKILL.md - -#### For Marketplace Maintainers - -1. **Git workflow**: Always `git push` after updating marketplace.json -2. **Validate JSON**: Run `python -m json.tool marketplace.json` before committing -3. **Update cache**: Remind users to run `claude plugin marketplace update` after releases -4. **Version consistency**: Marketplace version ≠ plugin versions (they track independently) - -#### For Users - -1. **Update marketplaces**: Run `claude plugin marketplace update {name}` periodically -2. **Check installed plugins**: Inspect `~/.claude/plugins/installed_plugins.json` -3. **Clear cache on issues**: `rm -rf ~/.claude/plugins/cache/{marketplace-name}` then reinstall -4. **Understand scopes**: - - `--scope user`: Only you (default) - - `--scope project`: Shared with team via `.claude/plugins/` - - `--scope local`: Gitignored, local only +For full architecture documentation (core concepts, installation flow, data flow, common misconceptions, best practices), see [references/plugin-architecture.md](./references/plugin-architecture.md). ## Plugin and Skill Troubleshooting -This section provides systematic debugging steps for common plugin and skill installation issues. - -### Understanding the Architecture First - -**CRITICAL**: Before troubleshooting, understand that Claude Code's plugin system is **GitHub-based**, not local-file-based. - -``` -GitHub Repository (source of truth) - ↓ (git clone / git pull) -~/.claude/plugins/marketplaces/{marketplace-name}/ - ↓ (claude plugin install) -~/.claude/plugins/cache/{marketplace-name}/{plugin}/{version}/ - ↓ (Claude Code loads) -Active skill in Claude's context -``` - -**Key insight**: Local file changes are NOT visible to `claude plugin install` until pushed to GitHub. - -### Common Error 1: "Plugin not found in marketplace" - -**Error message**: -``` -Installing plugin "skill-name@marketplace-name"... -✘ Failed to install plugin: Plugin "skill-name" not found in marketplace "marketplace-name" -``` - -**Root causes** (in order of likelihood): - -#### Cause 1.1: Local changes not pushed to GitHub ⭐ **MOST COMMON** - -**Symptoms**: -- `git status` shows modified files or untracked directories -- marketplace.json updated locally but install fails -- All documentation updated but plugin not found - -**Diagnosis**: -```bash -# Check if you have uncommitted changes -git status - -# Check last commit vs remote -git log origin/main..HEAD - -# Verify GitHub has latest marketplace.json -# Open: https://github.com/{owner}/{repo}/blob/main/.claude-plugin/marketplace.json -``` - -**Solution**: -```bash -# 1. Commit all changes -git add -A -git commit -m "Release vX.Y.Z: Add {skill-name}" - -# 2. Push to GitHub -git push - -# 3. Clear local marketplace cache -rm -rf ~/.claude/plugins/cache/{marketplace-name} - -# 4. Update marketplace from GitHub -claude plugin marketplace update {marketplace-name} - -# 5. Retry installation -claude plugin install {skill-name}@{marketplace-name} -``` - -**Why this happens**: You updated marketplace.json locally, but Claude CLI pulls from GitHub, not your local filesystem. - -#### Cause 1.2: marketplace.json configuration error - -**Symptoms**: -- Git is up-to-date but install still fails -- Other plugins from same marketplace install fine - -**Diagnosis**: -```bash -# 1. Check marketplace.json syntax -python3 -m json.tool .claude-plugin/marketplace.json > /dev/null - -# 2. Verify plugin entry exists -cat .claude-plugin/marketplace.json | jq '.plugins[] | select(.name == "skill-name")' - -# 3. Check spelling matches exactly -# Plugin name in marketplace.json MUST match install command -``` - -**Common mistakes**: -```json -// ❌ Wrong: name mismatch -{ - "name": "macos_cleaner", // Underscore - "skills": ["./macos-cleaner"] // Dash -} - -// ✅ Correct: consistent naming -{ - "name": "macos-cleaner", - "skills": ["./macos-cleaner"] -} -``` - -**Solution**: Fix marketplace.json, then commit and push. - -#### Cause 1.3: Skill directory missing - -**Symptoms**: -- marketplace.json has entry -- Git is up-to-date -- Plugin installs but skills don't load - -**Diagnosis**: -```bash -# Check if skill directory exists -ls -la {skill-name}/ - -# Verify SKILL.md exists -ls -la {skill-name}/SKILL.md -``` - -**Solution**: Ensure skill directory and SKILL.md are committed to git. - -### Common Error 2: Marketplace cache is stale - -**Symptoms**: -- GitHub has latest changes -- Install finds plugin but gets old version -- Newly added plugins not visible - -**Diagnosis**: -```bash -# Check cache timestamp -ls -la ~/.claude/plugins/cache/{marketplace-name}/ - -# Compare with last git push -git log -1 --format="%ai" -``` - -**Solution**: -```bash -# Option 1: Update marketplace cache -claude plugin marketplace update {marketplace-name} - -# Option 2: Clear cache and re-fetch -rm -rf ~/.claude/plugins/cache/{marketplace-name} -claude plugin marketplace update {marketplace-name} -``` - -### Common Error 3: JSON syntax error - -**Error message**: -``` -Error parsing marketplace manifest -``` - -**Diagnosis**: -```bash -# Validate JSON syntax -python3 -m json.tool .claude-plugin/marketplace.json - -# Check for common issues: -# - Missing commas -# - Trailing commas (in arrays/objects) -# - Unescaped quotes in strings -# - Missing closing braces -``` - -**Solution**: Fix JSON syntax, validate, commit, push. - -### Systematic Debugging Process - -When facing ANY plugin/skill issue, follow this checklist: - -#### Step 1: Verify marketplace registration - -```bash -# List registered marketplaces -claude plugin marketplace list - -# Expected output should include your marketplace -``` - -If missing, register: -```bash -claude plugin marketplace add https://github.com/{owner}/{repo} -``` - -#### Step 2: Check Git status - -```bash -# Are there uncommitted changes? -git status - -# Is local ahead of remote? -git log origin/main..HEAD - -# Push if needed -git push -``` - -#### Step 3: Verify GitHub has latest - -```bash -# Open in browser or use gh CLI -gh browse .claude-plugin/marketplace.json - -# Or check with curl -curl https://raw.githubusercontent.com/{owner}/{repo}/main/.claude-plugin/marketplace.json | jq '.plugins[] | .name' -``` - -#### Step 4: Clear and update cache - -```bash -# Remove stale cache -rm -rf ~/.claude/plugins/cache/{marketplace-name} - -# Re-fetch from GitHub -claude plugin marketplace update {marketplace-name} -``` - -#### Step 5: Validate configuration - -```bash -# Check marketplace.json is valid JSON -python3 -m json.tool .claude-plugin/marketplace.json > /dev/null && echo "✅ Valid JSON" - -# Check plugin entry exists -cat .claude-plugin/marketplace.json | jq '.plugins[] | select(.name == "skill-name")' || echo "❌ Plugin not found" -``` - -#### Step 6: Inspect installation state - -```bash -# Check if plugin is installed -cat ~/.claude/plugins/installed_plugins.json | jq -r '.plugins | keys[]' | grep "skill-name" - -# If installed, check details -cat ~/.claude/plugins/installed_plugins.json | jq '.plugins["skill-name@marketplace-name"]' - -# Verify files exist -ls ~/.claude/plugins/cache/{marketplace-name}/{skill-name}/{version}/ -``` - -### Debugging Commands Reference - -| Purpose | Command | -|---------|---------| -| List marketplaces | `claude plugin marketplace list` | -| Update marketplace | `claude plugin marketplace update {name}` | -| Install plugin | `claude plugin install {plugin}@{marketplace}` | -| Uninstall plugin | `claude plugin uninstall {plugin}@{marketplace}` | -| Update plugin | `claude plugin update {plugin}@{marketplace}` | -| Validate manifest | `claude plugin validate {path}` | -| Check installed plugins | `cat ~/.claude/plugins/installed_plugins.json \| jq '.plugins \| keys'` | -| Inspect plugin details | `cat ~/.claude/plugins/installed_plugins.json \| jq '.plugins["{plugin}@{marketplace}"]'` | -| Clear marketplace cache | `rm -rf ~/.claude/plugins/cache/{marketplace-name}` | -| Verify JSON syntax | `python3 -m json.tool {file.json}` | - -### Understanding File Locations - -```bash -# Marketplace clones (git repositories) -~/.claude/plugins/marketplaces/{marketplace-name}/ - -# Installed plugin cache -~/.claude/plugins/cache/{marketplace-name}/{plugin-name}/{version}/ - -# Installation registry -~/.claude/plugins/installed_plugins.json - -# Personal skills (not from marketplace) -~/.claude/skills/ - -# Project-scoped skills (shared with team) -.claude/skills/ -``` - -### Common Pitfalls - -#### Pitfall 1: Confusing local skills with plugin skills - -```bash -# ❌ Wrong: Copying skill to personal directory doesn't make it installable -cp -r my-skill ~/.claude/skills/my-skill # Works, but not managed by plugin system - -# ✅ Correct: Install via marketplace for version management -claude plugin install my-skill@my-marketplace -``` - -#### Pitfall 2: Forgetting to push after updating marketplace.json - -```bash -# ❌ Wrong workflow -vim .claude-plugin/marketplace.json # Add new plugin -git add .claude-plugin/marketplace.json -git commit -m "Add plugin" -# FORGOT TO PUSH! -claude plugin install new-plugin@my-marketplace # ❌ Fails: not found - -# ✅ Correct workflow -vim .claude-plugin/marketplace.json -git add -A -git commit -m "Add new plugin" -git push # ← CRITICAL STEP -claude plugin marketplace update my-marketplace -claude plugin install new-plugin@my-marketplace # ✅ Works -``` - -#### Pitfall 3: Expecting instant propagation - -```bash -# ❌ Wrong expectation -git push # Push changes -claude plugin install new-plugin@my-marketplace # ❌ Fails: cache is stale - -# ✅ Correct: Update cache first -git push -claude plugin marketplace update my-marketplace # Fetch latest from GitHub -claude plugin install new-plugin@my-marketplace # ✅ Works -``` - -#### Pitfall 4: Inconsistent naming - -```json -// marketplace.json -{ - "name": "my_plugin", // Underscore - "skills": ["./my-plugin"] // Dash - will cause confusion -} -``` - -```bash -# User tries to install -claude plugin install my-plugin@marketplace # ❌ Not found (name has underscore) -claude plugin install my_plugin@marketplace # ✅ Works, but confusing -``` - -**Best practice**: Use consistent kebab-case for both plugin name and skill directory. - -### Real-World Example: macos-cleaner Installation Issue - -**Scenario**: After creating macos-cleaner skill and updating all documentation, `claude plugin install macos-cleaner@daymade-skills` failed with "Plugin not found". - -**Investigation**: -```bash -# 1. Check git status -git status -# Output: 16 modified/untracked files - -# 2. Check marketplace cache -ls -la ~/.claude/plugins/cache/daymade-skills/ -# Output: Last modified Dec 20 (weeks old!) - -# 3. Check GitHub -# marketplace.json on GitHub: version 1.20.0 (old) -# Local marketplace.json: version 1.21.0 (new) -``` - -**Root cause**: Local changes weren't pushed to GitHub. CLI was pulling from GitHub, not local files. - -**Solution**: -```bash -# 1. Commit and push -git add -A -git commit -m "Release v1.21.0: Add macos-cleaner" -git push - -# 2. Update marketplace -claude plugin marketplace update daymade-skills - -# 3. Install -claude plugin install macos-cleaner@daymade-skills -# ✔ Successfully installed plugin: macos-cleaner@daymade-skills -``` - -**Verification**: -```bash -cat ~/.claude/plugins/installed_plugins.json | jq '.plugins["macos-cleaner@daymade-skills"]' -# Output: Installation details with correct version - -ls ~/.claude/plugins/cache/daymade-skills/macos-cleaner/1.0.0/ -# Output: All skill files present -``` - -**Lesson**: Always remember the GitHub-based architecture. Local changes are invisible until pushed. - -### Advanced: Manual Cache Inspection - -If automated commands don't reveal the issue, manually inspect: - -```bash -# 1. Check marketplace clone -cat ~/.claude/plugins/marketplaces/{marketplace-name}/.claude-plugin/marketplace.json | jq '.metadata.version' - -# 2. Check what's in cache -ls -R ~/.claude/plugins/cache/{marketplace-name}/ - -# 3. Compare with GitHub -curl -s https://raw.githubusercontent.com/{owner}/{repo}/main/.claude-plugin/marketplace.json | jq '.metadata.version' - -# 4. Check installation record -cat ~/.claude/plugins/installed_plugins.json | jq '.plugins' | grep -i "{skill-name}" -``` - -### When All Else Fails - -1. **Complete cache reset**: - ```bash - rm -rf ~/.claude/plugins/cache/* - claude plugin marketplace update {marketplace-name} - ``` - -2. **Re-register marketplace**: - ```bash - # Remove marketplace - # (No direct command, manual edit ~/.claude/plugins/known_marketplaces.json) - - # Re-add - claude plugin marketplace add https://github.com/{owner}/{repo} - ``` - -3. **Check Claude Code version**: - ```bash - claude --version - # Plugins require Claude Code v2.0.12+ - ``` - -4. **Enable verbose logging** (if available): - ```bash - CLAUDE_DEBUG=1 claude plugin install {plugin}@{marketplace} - ``` - -### Getting Help - -If you're still stuck: - -1. **Check GitHub issues**: https://github.com/anthropics/claude-code/issues -2. **Verify marketplace.json**: Run `claude plugin validate .claude-plugin/marketplace.json` -3. **Compare with working marketplace**: Study anthropic's official marketplace structure -4. **Document your debugging**: Include output from all diagnostic commands above +For systematic debugging steps (common errors, debugging process, pitfalls, real-world examples), see [references/plugin-troubleshooting.md](./references/plugin-troubleshooting.md). -For this project specifically, refer to: -- [Plugin and Skill Architecture](#plugin-and-skill-architecture) - Architecture overview (this document) -- [skill-creator/SKILL.md](./skill-creator/SKILL.md) - Skill creation guide -- [CONTRIBUTING.md](./CONTRIBUTING.md) - Development workflow +**Quick fix for most issues**: Commit → push → `claude plugin marketplace update daymade-skills` → retry install. diff --git a/QUICKSTART.md b/QUICKSTART.md index ec254187..2f3abb85 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -122,7 +122,7 @@ claude plugin marketplace add https://github.com/daymade/claude-code-skills # In Claude Code use `/plugin ...`; in your terminal use `claude plugin ...` # Step 2: Install skills you need claude plugin install github-ops@daymade-skills -claude plugin install markdown-tools@daymade-skills +claude plugin install doc-to-markdown@daymade-skills # ... add more as needed # Step 3: Restart Claude Code @@ -136,7 +136,7 @@ This table is a quick starter list. See [README.md](./README.md) for the full ca |-------|-------------|-------------| | **skill-creator** ⭐ | Create your own skills | Building custom workflows | | **github-ops** | GitHub operations | Managing PRs, issues, workflows | -| **markdown-tools** | Document conversion | Converting docs to markdown | +| **doc-to-markdown** | Document conversion | Converting docs to markdown | | **mermaid-tools** | Diagram generation | Creating PNG diagrams | | **statusline-generator** | Statusline customization | Customizing Claude Code UI | | **teams-channel-post-writer** | Teams communication | Writing professional posts | diff --git a/QUICKSTART.zh-CN.md b/QUICKSTART.zh-CN.md index a188f556..7b865a96 100644 --- a/QUICKSTART.zh-CN.md +++ b/QUICKSTART.zh-CN.md @@ -122,7 +122,7 @@ claude plugin marketplace add https://github.com/daymade/claude-code-skills # 在 Claude Code 内使用 `/plugin ...`，在终端中使用 `claude plugin ...` # 步骤 2：安装你需要的技能 claude plugin install github-ops@daymade-skills -claude plugin install markdown-tools@daymade-skills +claude plugin install doc-to-markdown@daymade-skills # ... 根据需要添加更多 # 步骤 3：重启 Claude Code @@ -136,7 +136,7 @@ claude plugin install markdown-tools@daymade-skills |-------|-------------|-------------| | **skill-creator** ⭐ | 创建你自己的技能 | 构建自定义工作流 | | **github-ops** | GitHub 操作 | 管理 PR、问题、工作流 | -| **markdown-tools** | 文档转换 | 将文档转换为 markdown | +| **doc-to-markdown** | 文档转换 | 将文档转换为 markdown | | **mermaid-tools** | 图表生成 | 创建 PNG 图表 | | **statusline-generator** | 状态栏定制 | 自定义 Claude Code UI | | **teams-channel-post-writer** | Teams 通信 | 编写专业帖子 | diff --git a/README.md b/README.md index 015a5f52..ea13f11b 100644 --- a/README.md +++ b/README.md @@ -6,15 +6,15 @@ [![简体中文](https://img.shields.io/badge/语言-简体中文-red)](./README.zh-CN.md) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -[![Skills](https://img.shields.io/badge/skills-41-blue.svg)](https://github.com/daymade/claude-code-skills) -[![Version](https://img.shields.io/badge/version-1.37.0-green.svg)](https://github.com/daymade/claude-code-skills) +[![Skills](https://img.shields.io/badge/skills-43-blue.svg)](https://github.com/daymade/claude-code-skills) +[![Version](https://img.shields.io/badge/version-1.39.0-green.svg)](https://github.com/daymade/claude-code-skills) [![Claude Code](https://img.shields.io/badge/Claude%20Code-2.0.13+-purple.svg)](https://claude.com/code) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](./CONTRIBUTING.md) [![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://github.com/daymade/claude-code-skills/graphs/commit-activity) -Professional Claude Code skills marketplace featuring 41 production-ready skills for enhanced development workflows. +Professional Claude Code skills marketplace featuring 43 production-ready skills for enhanced development workflows. ## 📑 Table of Contents @@ -38,13 +38,35 @@ Professional Claude Code skills marketplace featuring 41 production-ready skills The `skill-creator` is the **meta-skill** that enables you to build, validate, and package your own Claude Code skills. It's the most important tool in this marketplace because it empowers you to extend Claude Code with your own specialized workflows. -### Why skill-creator First? +### Why This skill-creator? -- **🎯 Foundation**: Learn how skills work by creating your own -- **🛠️ Complete Toolkit**: Initialization, validation, and packaging scripts included -- **📖 Best Practices**: Learn from production-ready examples -- **🚀 Quick Start**: Generate skill templates in seconds -- **✅ Quality Assurance**: Built-in validation ensures your skills meet standards +This is a **production-hardened fork** of [Anthropic's official skill-creator](https://github.com/anthropics/skills/tree/main/skills/skill-creator), born from building real skills and hitting every wall the official version doesn't warn you about. + +**The official skill-creator tells you _what_ to build. Ours also tells you _what not to try_ — and why.** + +| You're trying to... | Official | This Fork | +|---------------------|----------|-----------| +| Research before building | "Check available MCPs" (5 lines) | 8-channel search protocol with decision matrix: Adopt / Extend / Build | +| Create a skill interactively | Prose-based instructions | 9 structured AskUserQuestion checkpoints — user never loses context | +| Avoid common mistakes | No guidance | Cache edit warnings, prerequisite checks, security scan gate | +| Know the architecture options | Not mentioned | Inline vs Fork decision guide with examples (choosing wrong silently breaks your skill) | +| Validate before shipping | Basic YAML check | Expanded validator (all frontmatter fields, path reference integrity, whitespace issues) | +| Catch security issues | No tooling | `security_scan.py` with gitleaks integration — hard gate before packaging | +| Learn from real failures | No failure cases | Battle-tested methodology with documented failure patterns and gotchas | + +**Quality comparison** (independent audit, 8 dimensions): + +| Dimension | Official | This Fork | +|-----------|----------|-----------| +| Actionability | 7 | 9 | +| Error Prevention | 5 | 9 | +| Prior Art Research | 4 | 9 | +| Counter Review Process | 4 | 8 | +| Real-World Lessons | 3 | 8 | +| User Experience | 4 | 9 | +| **Total (out of 80)** | **42** | **65** | + +> Full methodology: [skill-creator/references/skill-development-methodology.md](./skill-creator/references/skill-development-methodology.md) ### Quick Install @@ -146,7 +168,7 @@ claude plugin install skill-creator@daymade-skills claude plugin install github-ops@daymade-skills # Document conversion -claude plugin install markdown-tools@daymade-skills +claude plugin install doc-to-markdown@daymade-skills # Diagram generation claude plugin install mermaid-tools@daymade-skills @@ -237,6 +259,12 @@ claude plugin install excel-automation@daymade-skills # Programmatic macOS screenshot capture workflows claude plugin install capture-screen@daymade-skills + +# Resume interrupted Claude work from local session artifacts +claude plugin install continue-claude-work@daymade-skills + +# Scrapling CLI extraction and troubleshooting +claude plugin install scrapling-skill@daymade-skills ``` Each skill can be installed independently - choose only what you need! @@ -288,7 +316,7 @@ Comprehensive GitHub operations using gh CLI and GitHub API. --- -### 2. **markdown-tools** - Document Conversion Suite +### 2. **doc-to-markdown** - Document Conversion Suite Converts documents to markdown with Windows/WSL path handling and PDF image extraction. @@ -307,7 +335,7 @@ Converts documents to markdown with Windows/WSL path handling and PDF image extr **🎬 Live Demo** -![Markdown Tools Demo](./demos/markdown-tools/convert-docs.gif) +![Markdown Tools Demo](./demos/doc-to-markdown/convert-docs.gif) --- @@ -1749,6 +1777,79 @@ claude plugin install capture-screen@daymade-skills --- +### 42. **continue-claude-work** - Resume Interrupted Claude Work + +Recover actionable context from local `~/.claude` session artifacts and continue implementation without reopening the old interactive session. Uses a bundled Python script for intelligent context extraction. + +**When to use:** +- A user provides a Claude session ID and wants the task continued +- You need to inspect local `.claude` JSONL files instead of running `claude --resume` +- A previous session was interrupted and the next concrete step must be reconstructed +- A multi-agent workflow was interrupted and you need to know which subagents completed + +**Key features:** +- Compact-boundary-aware extraction — reads Claude's own session compaction summaries as highest-signal context +- Subagent workflow recovery — reports completed vs. interrupted subagents with last outputs +- Session end reason detection — classifies clean exit, interrupted (ctrl-c), error cascade, or abandoned +- Size-adaptive strategy — different reading approaches for small (<500KB) vs. large (>5MB) sessions +- Noise filtering — skips progress/queue-operation/api_error messages (37-53% of session lines) +- Self-session exclusion, stale index fallback, MEMORY.md integration, git workspace state + +**Example usage:** +```bash +# Install the skill +claude plugin install continue-claude-work@daymade-skills + +# Then ask Claude to resume from local artifacts +"continue work from session 123e4567-e89b-12d3-a456-426614174000" +"don't resume, just read the .claude files and continue" +"check what I was working on in the last session and keep going" +``` + +📚 **Documentation**: See [continue-claude-work/SKILL.md](./continue-claude-work/SKILL.md). + +**Requirements**: Python 3.8+, `git` for workspace reconciliation. + +--- + +### 43. **scrapling-skill** - Reliable Scrapling CLI Workflows + +Install, troubleshoot, and use Scrapling CLI with a verified static-first workflow for extracting HTML, Markdown, or text from webpages. Includes a diagnostic script for broken extras installs, Playwright browser runtime checks, and smoke tests against real URLs. + +**When to use:** +- Users mention Scrapling, `uv tool install scrapling`, or `scrapling extract` +- You need to choose between static and browser-backed fetching +- You need to extract article bodies from WeChat public pages (`mp.weixin.qq.com`) +- A Scrapling install works partially but fails on missing extras, browser runtime, or TLS verification + +**Key features:** +- Bundled `diagnose_scrapling.py` script for CLI, browser runtime, and live URL smoke tests +- Verified default path: start with `extract get`, escalate to `extract fetch` only when needed +- WeChat extraction pattern using `#js_content` for clean article Markdown +- Troubleshooting guidance for missing `click`, Playwright runtime setup, and `curl: (60)` trust-store failures +- Output validation workflow using file size and content checks instead of exit-code assumptions + +**Example usage:** +```bash +# Install the skill +claude plugin install scrapling-skill@daymade-skills + +# Then ask Claude to work through Scrapling for you +"Install Scrapling CLI and verify the setup" +"Extract this WeChat article into Markdown with Scrapling" +"Decide whether this page needs static or browser-backed fetching" +``` + +**🎬 Live Demo** + +*Coming soon* + +📚 **Documentation**: See [scrapling-skill/SKILL.md](./scrapling-skill/SKILL.md) and [scrapling-skill/references/troubleshooting.md](./scrapling-skill/references/troubleshooting.md). + +**Requirements**: Python 3.6+, `uv`, Scrapling CLI, and Playwright browser runtime for browser-backed fetches. + +--- + ## 🎬 Interactive Demo Gallery Want to see all demos in one place with click-to-enlarge functionality? Check out our [interactive demo gallery](./demos/index.html) or browse the [demos directory](./demos/). @@ -1759,7 +1860,7 @@ Want to see all demos in one place with click-to-enlarge functionality? Check ou Use **github-ops** to streamline PR creation, issue management, and API operations. ### For Documentation -Combine **markdown-tools** for document conversion and **mermaid-tools** for diagram generation to create comprehensive documentation. Use **llm-icon-finder** to add brand icons. +Combine **doc-to-markdown** for document conversion and **mermaid-tools** for diagram generation to create comprehensive documentation. Use **llm-icon-finder** to add brand icons. ### For Research & Analysis Use **deep-research** to produce format-controlled research reports with evidence tables and citations. Combine with **fact-checker** to validate claims or with **twitter-reader** for social-source collection. @@ -1812,6 +1913,12 @@ Use **prompt-optimizer** to transform vague feature requests into precise EARS s ### For Session History & File Recovery Use **claude-code-history-files-finder** to recover deleted files from previous Claude Code sessions, search for specific implementations across conversation history, or track file evolution over time. Essential for recovering accidentally deleted code or finding that feature implementation you remember but can't locate. +### For Resuming Interrupted Claude Sessions +Use **continue-claude-work** to recover the last actionable request from local `~/.claude` artifacts and continue implementation without reopening the original session. Combine with **claude-code-history-files-finder** when you need broader cross-session search, statistics, or deleted-file recovery. + +### For Web Extraction & WeChat Articles +Use **scrapling-skill** to install and validate Scrapling CLI, choose between static and browser-backed fetching, and extract clean Markdown from sites like `mp.weixin.qq.com`. Combine with **deep-research** to turn extracted sources into structured reports or with **docs-cleaner** to normalize captured article content. + ### For Documentation Maintenance Use **docs-cleaner** to consolidate redundant documentation while preserving valuable content. Perfect for cleaning up documentation sprawl after rapid development phases or merging overlapping docs into authoritative sources. @@ -1831,7 +1938,7 @@ Use **iOS-APP-developer** to configure XcodeGen projects, resolve SPM dependency Use **macos-cleaner** to intelligently analyze and reclaim disk space on macOS with safety-first approach. Unlike one-click cleaners that blindly delete, macos-cleaner explains what each file is, categorizes by risk level (🟢/🟡/🔴), and requires explicit confirmation before any deletion. Perfect for developers dealing with Docker/Homebrew/npm/pip cache bloat, users wanting to understand storage consumption, or anyone who values transparency over automation. Combines script-based precision with optional Mole visual tool integration for hybrid workflow. ### For Twitter/X Content Research -Use **twitter-reader** to fetch tweet content without JavaScript rendering or authentication. Perfect for documenting social media discussions, archiving threads, analyzing tweet content, or gathering reference material from Twitter/X. Combine with **markdown-tools** to convert fetched content into other formats, or with **repomix-safe-mixer** to package research collections securely. +Use **twitter-reader** to fetch tweet content without JavaScript rendering or authentication. Perfect for documenting social media discussions, archiving threads, analyzing tweet content, or gathering reference material from Twitter/X. Combine with **doc-to-markdown** to convert fetched content into other formats, or with **repomix-safe-mixer** to package research collections securely. ### For Skill Quality & Open-Source Contributions Use **skill-reviewer** to validate your own skills against best practices before publishing, or to review and improve others' skill repositories. Combine with **github-contributor** to find high-impact open-source projects, create professional PRs, and build your contributor reputation. Perfect for developers who want to contribute to the Claude Code ecosystem or any GitHub project systematically. @@ -1862,7 +1969,7 @@ Each skill includes: ### Quick Links - **github-ops**: See `github-ops/references/api_reference.md` for API documentation -- **markdown-tools**: See `markdown-tools/references/conversion-examples.md` for conversion scenarios +- **doc-to-markdown**: See `doc-to-markdown/references/conversion-examples.md` for conversion scenarios - **mermaid-tools**: See `mermaid-tools/references/setup_and_troubleshooting.md` for setup guide - **statusline-generator**: See `statusline-generator/references/color_codes.md` for customization - **teams-channel-post-writer**: See `teams-channel-post-writer/references/writing-guidelines.md` for quality standards @@ -1899,13 +2006,15 @@ Each skill includes: - **product-analysis**: See `product-analysis/SKILL.md` for workflow and `product-analysis/references/synthesis_methodology.md` for cross-agent weighting and recommendation logic - **excel-automation**: See `excel-automation/SKILL.md` for create/parse/control workflows and `excel-automation/references/formatting-reference.md` for formatting standards - **capture-screen**: See `capture-screen/SKILL.md` for CGWindowID-based screenshot workflows on macOS +- **continue-claude-work**: See `continue-claude-work/SKILL.md` for local artifact recovery, drift checks, and resume workflow +- **scrapling-skill**: See `scrapling-skill/SKILL.md` for the CLI workflow and `scrapling-skill/references/troubleshooting.md` for verified Scrapling failure modes ## 🛠️ Requirements - **Claude Code** 2.0.13 or higher - **Python 3.6+** (for scripts in multiple skills) - **gh CLI** (for github-ops) -- **markitdown** (for markdown-tools) +- **markitdown** (for doc-to-markdown) - **mermaid-cli** (for mermaid-tools) - **yt-dlp** (for youtube-downloader): `brew install yt-dlp` or `pip install yt-dlp` - **FFmpeg/FFprobe** (for video-comparer): `brew install ffmpeg`, `apt install ffmpeg`, or `winget install ffmpeg` @@ -1924,6 +2033,8 @@ Each skill includes: - **Codex CLI** (optional, for product-analysis multi-model mode) - **uv + openpyxl** (for excel-automation): `uv run --with openpyxl ...` - **macOS** (for capture-screen and excel-automation AppleScript control workflows) +- **Python 3.8+** (for continue-claude-work): bundled script for session extraction (no external dependencies) +- **uv + Scrapling CLI** (for scrapling-skill): `uv tool install 'scrapling[shell]'` and `scrapling install` for browser-backed fetches ## ❓ FAQ diff --git a/README.zh-CN.md b/README.zh-CN.md index 5404eaaf..29bfc1a2 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -6,15 +6,15 @@ [![简体中文](https://img.shields.io/badge/语言-简体中文-red)](./README.zh-CN.md) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -[![Skills](https://img.shields.io/badge/skills-41-blue.svg)](https://github.com/daymade/claude-code-skills) -[![Version](https://img.shields.io/badge/version-1.37.0-green.svg)](https://github.com/daymade/claude-code-skills) +[![Skills](https://img.shields.io/badge/skills-43-blue.svg)](https://github.com/daymade/claude-code-skills) +[![Version](https://img.shields.io/badge/version-1.39.0-green.svg)](https://github.com/daymade/claude-code-skills) [![Claude Code](https://img.shields.io/badge/Claude%20Code-2.0.13+-purple.svg)](https://claude.com/code) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](./CONTRIBUTING.md) [![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://github.com/daymade/claude-code-skills/graphs/commit-activity) -专业的 Claude Code 技能市场，提供 41 个生产就绪的技能，用于增强开发工作流。 +专业的 Claude Code 技能市场，提供 43 个生产就绪的技能，用于增强开发工作流。 ## 📑 目录 @@ -38,13 +38,35 @@ `skill-creator` 是一个**元技能**，它使你能够构建、验证和打包自己的 Claude Code 技能。它是这个市场中最重要的工具，因为它赋予你用自己的专业工作流扩展 Claude Code 的能力。 -### 为什么首选 skill-creator？ +### 为什么选这个 skill-creator？ -- **🎯 基础工具**：通过创建自己的技能来学习技能的工作原理 -- **🛠️ 完整工具包**：包含初始化、验证和打包脚本 -- **📖 最佳实践**：从生产就绪的示例中学习 -- **🚀 快速启动**：在几秒钟内生成技能模板 -- **✅ 质量保证**：内置验证确保你的技能符合标准 +这是 [Anthropic 官方 skill-creator](https://github.com/anthropics/skills/tree/main/skills/skill-creator) 的**生产强化版 fork**——从真实 skill 开发中踩过的坑里长出来的。 + +**官方告诉你"做什么"。我们还告诉你"别试什么"——以及为什么。** + +| 你想要... | 官方版 | 本 Fork | +|----------|--------|---------| +| 造之前先调研 | "Check available MCPs"（5 行） | 8 渠道搜索协议 + Adopt/Extend/Build 决策矩阵 | +| 交互式创建 skill | 纯文字指令 | 9 个结构化 AskUserQuestion 检查点——用户永远不丢上下文 | +| 避免常见错误 | 无指引 | 缓存编辑警告、前置依赖检查、安全扫描门禁 | +| 了解架构选项 | 未提及 | Inline vs Fork 决策指南（选错会静默破坏你的 skill） | +| 发布前验证 | 基本 YAML 检查 | 扩展验证器（全部 frontmatter 字段、路径引用完整性、空白字符问题） | +| 安全审查 | 无工具 | `security_scan.py` + gitleaks 集成——打包前硬门禁 | +| 从真实失败中学习 | 无失败案例 | 实战方法论 + 文档化的失败模式和踩坑记录 | + +**质量对比**（独立审计，8 个维度）： + +| 维度 | 官方版 | 本 Fork | +|------|--------|---------| +| 可操作性 | 7 | 9 | +| 错误预防 | 5 | 9 | +| 前置调研 | 4 | 9 | +| 对抗性审查 | 4 | 8 | +| 实战经验 | 3 | 8 | +| 用户体验 | 4 | 9 | +| **总分（/80）** | **42** | **65** | + +> 完整方法论：[skill-creator/references/skill-development-methodology.md](./skill-creator/references/skill-development-methodology.md) ### 快速安装 @@ -146,7 +168,7 @@ claude plugin install skill-creator@daymade-skills claude plugin install github-ops@daymade-skills # 文档转换 -claude plugin install markdown-tools@daymade-skills +claude plugin install doc-to-markdown@daymade-skills # 图表生成 claude plugin install mermaid-tools@daymade-skills @@ -240,6 +262,12 @@ claude plugin install excel-automation@daymade-skills # macOS 程序化窗口截图工作流 claude plugin install capture-screen@daymade-skills + +# 基于本地会话产物续做中断的 Claude 工作 +claude plugin install continue-claude-work@daymade-skills + +# Scrapling CLI 抽取与故障排查 +claude plugin install scrapling-skill@daymade-skills ``` 每个技能都可以独立安装 - 只选择你需要的！ @@ -313,7 +341,7 @@ CC-Switch 支持以下中国 AI 服务提供商： --- -### 2. **markdown-tools** - 文档转换套件 +### 2. **doc-to-markdown** - 文档转换套件 将文档转换为 markdown，支持 Windows/WSL 路径处理和 PDF 图片提取。 @@ -332,7 +360,7 @@ CC-Switch 支持以下中国 AI 服务提供商： **🎬 实时演示** -![Markdown 工具演示](./demos/markdown-tools/convert-docs.gif) +![Markdown 工具演示](./demos/doc-to-markdown/convert-docs.gif) --- @@ -1791,6 +1819,79 @@ claude plugin install capture-screen@daymade-skills --- +### 42. **continue-claude-work** - 续做中断的 Claude 工作 + +从本地 `~/.claude` 会话产物中恢复可执行上下文，并在不重新打开旧交互会话的前提下继续实现工作。内置 Python 脚本实现智能上下文提取。 + +**使用场景：** +- 用户提供 Claude 会话 ID，希望继续上次的任务 +- 需要直接检查本地 `.claude` JSONL 文件，而不是运行 `claude --resume` +- 上一次会话被中断，需要重建下一步具体动作 +- 多 agent 工作流被中断，需要了解哪些 subagent 已完成 + +**主要功能：** +- Compact-boundary 感知提取 — 读取 Claude 自身的会话压缩摘要作为最高信噪比上下文 +- Subagent 工作流恢复 — 报告已完成与被中断的 subagent 及其最后输出 +- 会话结束原因检测 — 区分正常退出、中断（ctrl-c）、错误级联、废弃会话 +- 大小自适应策略 — 对小型（<500KB）和大型（>5MB）会话采用不同读取方式 +- 噪声过滤 — 跳过 progress/queue-operation/api_error 消息（占会话行数的 37-53%） +- 自会话排除、过期索引回退、MEMORY.md 集成、git 工作区状态 + +**示例用法：** +```bash +# 安装技能 +claude plugin install continue-claude-work@daymade-skills + +# 然后让 Claude 基于本地产物续做 +"continue work from session 123e4567-e89b-12d3-a456-426614174000" +"不用真的 resume，去 .claude 里找上下文继续做" +"查看上次会话做了什么，然后继续" +``` + +📚 **文档**：参见 [continue-claude-work/SKILL.md](./continue-claude-work/SKILL.md)。 + +**要求**：Python 3.8+，用于工作区核对的 `git`。 + +--- + +### 43. **scrapling-skill** - 可靠的 Scrapling CLI 工作流 + +围绕 Scrapling CLI 提供经过验证的安装、排障与网页抽取工作流，用于从网页输出 HTML、Markdown 或纯文本。内置诊断脚本，可检查 extras 安装问题、Playwright 浏览器运行时，以及真实 URL 的烟测结果。 + +**使用场景：** +- 用户提到 Scrapling、`uv tool install scrapling` 或 `scrapling extract` +- 需要判断应该使用静态抓取还是浏览器抓取 +- 需要从微信公众号页面（`mp.weixin.qq.com`）提取正文 +- Scrapling 安装看似成功，但在 extras、浏览器运行时或 TLS 校验上失败 + +**主要功能：** +- 内置 `diagnose_scrapling.py`，检查 CLI、浏览器运行时与真实 URL 烟测 +- 经过验证的默认路径：先用 `extract get`，只有必要时再升级到 `extract fetch` +- 针对微信公众号文章的 `#js_content` 提取模式 +- 覆盖缺少 `click`、Playwright 运行时缺失、`curl: (60)` 证书问题等真实故障 +- 用文件大小和内容验证结果，而不是只看退出码 + +**示例用法：** +```bash +# 安装技能 +claude plugin install scrapling-skill@daymade-skills + +# 然后让 Claude 代你跑 Scrapling +"安装 Scrapling CLI 并验证配置" +"用 Scrapling 把这篇微信公众号文章提取成 Markdown" +"判断这个页面应不应该走浏览器抓取" +``` + +**🎬 实时演示** + +*即将推出* + +📚 **文档**：参见 [scrapling-skill/SKILL.md](./scrapling-skill/SKILL.md) 和 [scrapling-skill/references/troubleshooting.md](./scrapling-skill/references/troubleshooting.md)。 + +**要求**：Python 3.6+、`uv`、Scrapling CLI；如需浏览器抓取，还需要 Playwright 浏览器运行时。 + +--- + ## 🎬 交互式演示画廊 想要在一个地方查看所有演示并具有点击放大功能？访问我们的[交互式演示画廊](./demos/index.html)或浏览[演示目录](./demos/)。 @@ -1801,7 +1902,7 @@ claude plugin install capture-screen@daymade-skills 使用 **github-ops** 简化 PR 创建、问题管理和 API 操作。 ### 文档处理 -结合 **markdown-tools** 进行文档转换和 **mermaid-tools** 进行图表生成，创建全面的文档。使用 **llm-icon-finder** 添加品牌图标。 +结合 **doc-to-markdown** 进行文档转换和 **mermaid-tools** 进行图表生成，创建全面的文档。使用 **llm-icon-finder** 添加品牌图标。 ### 调研与分析 使用 **deep-research** 生成格式可控的调研报告，包含证据表与引用。与 **fact-checker** 结合用于验证关键结论，或与 **twitter-reader** 结合收集社媒资料。 @@ -1854,6 +1955,12 @@ claude plugin install capture-screen@daymade-skills ### 会话历史与文件恢复 使用 **claude-code-history-files-finder** 从之前的 Claude Code 会话中恢复已删除的文件、在对话历史中搜索特定实现，或跟踪文件随时间的演变。对于恢复意外删除的代码或查找你记得但找不到的功能实现至关重要。 +### 续做中断的 Claude 会话 +使用 **continue-claude-work** 从本地 `~/.claude` 产物中恢复最后一个可执行请求，并在不重新打开原始会话的情况下继续实现。若还需要跨会话搜索、统计分析或恢复已删除文件，可与 **claude-code-history-files-finder** 配合使用。 + +### 网页提取与微信公众号文章 +使用 **scrapling-skill** 安装并验证 Scrapling CLI，判断应使用静态抓取还是浏览器抓取，并从 `mp.weixin.qq.com` 等页面提取干净的 Markdown。可与 **deep-research** 配合，将抓取内容整理为结构化报告，或与 **docs-cleaner** 配合清理抽取后的文章内容。 + ### 文档维护 使用 **docs-cleaner** 在保留有价值内容的同时整合冗余文档。非常适合在快速开发阶段后清理文档扩散或将重叠的文档合并为权威来源。 @@ -1867,7 +1974,7 @@ claude plugin install capture-screen@daymade-skills 使用 **iOS-APP-developer** 配置 XcodeGen 项目，处理 SPM 依赖、签名与部署问题。 ### Twitter/X 内容研究 -使用 **twitter-reader** 无需 JavaScript 渲染或身份验证即可获取推文内容。非常适合记录社交媒体讨论、归档话题、分析推文内容或从 Twitter/X 收集参考资料。与 **markdown-tools** 结合可将获取的内容转换为其他格式，或与 **repomix-safe-mixer** 结合安全地打包研究集合。 +使用 **twitter-reader** 无需 JavaScript 渲染或身份验证即可获取推文内容。非常适合记录社交媒体讨论、归档话题、分析推文内容或从 Twitter/X 收集参考资料。与 **doc-to-markdown** 结合可将获取的内容转换为其他格式，或与 **repomix-safe-mixer** 结合安全地打包研究集合。 ### macOS 系统维护与磁盘空间恢复 使用 **macos-cleaner** 以安全优先的方式智能分析和恢复 macOS 上的磁盘空间。与盲目删除的一键清理工具不同，macos-cleaner 解释每个文件是什么、按风险级别分类（🟢/🟡/🔴），并在任何删除前需要明确确认。非常适合处理 Docker/Homebrew/npm/pip 缓存膨胀的开发者、希望了解存储空间消耗的用户，或任何重视透明度而非自动化的人。结合基于脚本的精度和可选的 Mole 可视化工具集成以实现混合工作流。 @@ -1904,7 +2011,7 @@ claude plugin install capture-screen@daymade-skills ### 快速链接 - **github-ops**：参见 `github-ops/references/api_reference.md` 了解 API 文档 -- **markdown-tools**：参见 `markdown-tools/references/conversion-examples.md` 了解转换场景 +- **doc-to-markdown**：参见 `doc-to-markdown/references/conversion-examples.md` 了解转换场景 - **mermaid-tools**：参见 `mermaid-tools/references/setup_and_troubleshooting.md` 了解设置指南 - **statusline-generator**：参见 `statusline-generator/references/color_codes.md` 了解自定义 - **teams-channel-post-writer**：参见 `teams-channel-post-writer/references/writing-guidelines.md` 了解质量标准 @@ -1941,13 +2048,15 @@ claude plugin install capture-screen@daymade-skills - **product-analysis**：参见 `product-analysis/SKILL.md` 了解工作流，参见 `product-analysis/references/synthesis_methodology.md` 了解跨代理加权与推荐逻辑 - **excel-automation**：参见 `excel-automation/SKILL.md` 了解创建/解析/控制工作流，参见 `excel-automation/references/formatting-reference.md` 了解格式规范 - **capture-screen**：参见 `capture-screen/SKILL.md` 了解基于 CGWindowID 的 macOS 截图流程 +- **continue-claude-work**：参见 `continue-claude-work/SKILL.md` 了解本地会话产物恢复、漂移检查与续做流程 +- **scrapling-skill**：参见 `scrapling-skill/SKILL.md` 了解 CLI 工作流，参见 `scrapling-skill/references/troubleshooting.md` 了解已验证的 Scrapling 故障模式 ## 🛠️ 系统要求 - **Claude Code** 2.0.13 或更高版本 - **Python 3.6+**（用于多个技能中的脚本） - **gh CLI**（用于 github-ops） -- **markitdown**（用于 markdown-tools） +- **markitdown**（用于 doc-to-markdown） - **mermaid-cli**（用于 mermaid-tools） - **VHS**（用于 cli-demo-generator）：`brew install vhs` - **asciinema**（可选，用于 cli-demo-generator 交互式录制） @@ -1963,6 +2072,8 @@ claude plugin install capture-screen@daymade-skills - **Mole**（可选，用于 macos-cleaner 可视化清理）：从 https://github.com/tw93/Mole 下载 - **uv + openpyxl**（用于 excel-automation）：`uv run --with openpyxl ...` - **macOS**（用于 capture-screen 与 excel-automation 的 AppleScript 控制流程） +- **Python 3.8+**（用于 continue-claude-work）：内置脚本进行会话提取（无外部依赖） +- **uv + Scrapling CLI**（用于 scrapling-skill）：`uv tool install 'scrapling[shell]'`，浏览器抓取前运行 `scrapling install` ## ❓ 常见问题 diff --git a/asr-transcribe-to-text/.security-scan-passed b/asr-transcribe-to-text/.security-scan-passed new file mode 100644 index 00000000..d93e23f4 --- /dev/null +++ b/asr-transcribe-to-text/.security-scan-passed @@ -0,0 +1,4 @@ +Security scan passed +Scanned at: 2026-03-22T23:58:59.508059 +Tool: gitleaks + pattern-based validation +Content hash: 5dbbc175b8bfd6c6c8ab97f4112bf1f48eb489ef2a55375f88266deade644cd4 diff --git a/asr-transcribe-to-text/SKILL.md b/asr-transcribe-to-text/SKILL.md new file mode 100644 index 00000000..15af8f58 --- /dev/null +++ b/asr-transcribe-to-text/SKILL.md @@ -0,0 +1,197 @@ +--- +name: asr-transcribe-to-text +description: Transcribe audio and video files to text using a remote ASR service (Qwen3-ASR or OpenAI-compatible endpoint). Extracts audio from video, sends to configurable ASR endpoint, outputs clean text. Use when the user wants to transcribe recordings, convert audio/video to text, do speech-to-text, or mentions ASR, Qwen ASR, 转录, 语音转文字, 录音转文字, or has a meeting recording, lecture, interview, or screen recording to transcribe. +argument-hint: [audio-or-video-file-path] +--- + +# ASR Transcribe to Text + +Transcribe audio/video files to text using a configurable ASR endpoint (default: Qwen3-ASR-1.7B via vLLM). Configuration persists across sessions in `${CLAUDE_PLUGIN_DATA}/config.json`. + +## Step 0: Load or Initialize Configuration + +```bash +cat "${CLAUDE_PLUGIN_DATA}/config.json" 2>/dev/null +``` + +**If config exists**, read the values and proceed to Step 1. + +**If config does not exist** (first run), use **AskUserQuestion**: + +``` +First-time setup for ASR transcription. +I need to know where your ASR service is running so I can send audio to it. + +RECOMMENDATION: Use the defaults below if you have Qwen3-ASR on a 4090 via Tailscale. + +Q1: ASR Endpoint URL? + A) http://workstation-4090-wsl:8002/v1/audio/transcriptions (Default — Qwen3-ASR vLLM via Tailscale) + B) http://localhost:8002/v1/audio/transcriptions (Local machine) + C) Let me enter a custom URL + +Q2: Does your network have an HTTP proxy that might intercept LAN/Tailscale traffic? + A) Yes — add --noproxy to bypass it (Recommended if you use Shadowrocket/Clash/corporate proxy) + B) No — direct connection is fine +``` + +Save the config: +```bash +mkdir -p "${CLAUDE_PLUGIN_DATA}" +python3 -c " +import json +config = { + 'endpoint': 'USER_PROVIDED_ENDPOINT', + 'model': 'USER_PROVIDED_MODEL_OR_DEFAULT', + 'noproxy': True, # or False based on user answer + 'max_timeout': 900 +} +with open('${CLAUDE_PLUGIN_DATA}/config.json', 'w') as f: + json.dump(config, f, indent=2) +print('Config saved.') +" +``` + +## Step 1: Validate Input and Check Service Health + +Read config and health-check in a single command (shell variables don't persist across Bash calls): + +```bash +python3 -c " +import json, subprocess, sys +with open('${CLAUDE_PLUGIN_DATA}/config.json') as f: + cfg = json.load(f) +base = cfg['endpoint'].rsplit('/audio/', 1)[0] +noproxy = ['--noproxy', '*'] if cfg.get('noproxy', True) else [] +result = subprocess.run( + ['curl', '-s', '--max-time', '10'] + noproxy + [f'{base}/models'], + capture_output=True, text=True +) +if result.returncode != 0 or not result.stdout.strip(): + print(f'HEALTH CHECK FAILED', file=sys.stderr) + print(f'Endpoint: {base}/models', file=sys.stderr) + print(f'stdout: {result.stdout[:200]}', file=sys.stderr) + print(f'stderr: {result.stderr[:200]}', file=sys.stderr) + sys.exit(1) +else: + print(f'Service healthy: {base}') + print(f'Model: {cfg[\"model\"]}') +" +``` + +**If health check fails**, use **AskUserQuestion**: + +``` +ASR service at [endpoint] is not responding. + +Options: +A) Diagnose — check network, Tailscale, and service status step by step +B) Reconfigure — the endpoint URL might be wrong, let me re-enter it +C) Try anyway — send the transcription request and see what happens +D) Abort — I'll fix the service manually and come back later +``` + +For option A, diagnose in order: +1. Network: `ping -c 1 HOST` or `tailscale status | grep HOST` +2. Service: `tailscale ssh USER@HOST "curl -s localhost:PORT/v1/models"` +3. Proxy: retry with `--noproxy '*'` toggled + +## Step 2: Extract Audio (if input is video) + +For video files (mp4, mov, mkv, avi, webm), extract audio as 16kHz mono MP3: + +```bash +ffmpeg -i INPUT_VIDEO -vn -acodec libmp3lame -q:a 4 -ar 16000 -ac 1 OUTPUT.mp3 -y +``` + +For audio files (mp3, wav, m4a, flac, ogg), use directly — no conversion needed. + +Get duration for progress estimation: +```bash +ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 INPUT_FILE +``` + +## Step 3: Transcribe — Single Request First + +**Always try full-length single request first.** Chunking causes sentence truncation at every split boundary — the model forces the last sentence to close and loses words. Single request = zero truncation + fastest speed. + +The Qwen3-ASR paper's "20-minute limit" is a training benchmark, not an inference hard limit. Empirically verified: 55 minutes transcribed in a single 76-second request on 4090 24GB. + +```bash +python3 -c " +import json, subprocess, sys, os, tempfile +with open('${CLAUDE_PLUGIN_DATA}/config.json') as f: + cfg = json.load(f) +noproxy = ['--noproxy', '*'] if cfg.get('noproxy', True) else [] +timeout = str(cfg.get('max_timeout', 900)) +audio_file = 'AUDIO_FILE_PATH' # replace with actual path +output_json = tempfile.mktemp(suffix='.json', prefix='asr_') + +result = subprocess.run( + ['curl', '-s', '--max-time', timeout] + noproxy + [ + cfg['endpoint'], + '-F', f'file=@{audio_file}', + '-F', f'model={cfg[\"model\"]}', + '-o', output_json + ], capture_output=True, text=True +) + +with open(output_json) as f: + data = json.load(f) +if 'text' not in data: + print(f'ERROR: {json.dumps(data)[:300]}', file=sys.stderr) + sys.exit(1) +text = data['text'] +duration = data.get('usage', {}).get('seconds', 0) +print(f'Transcribed: {len(text)} chars, {duration}s audio', file=sys.stderr) +print(text) +os.unlink(output_json) +" > OUTPUT.txt +``` + +**Performance reference**: ~400 characters per minute for Chinese speech; rates vary by language. Qwen3-ASR supports 52 languages including Chinese dialects, English, Japanese, Korean, and more. + +## Step 4: Verify and Confirm Output + +After transcription, verify quality: +1. Confirm the response contains a `text` field (not an error message) +2. Check character count is plausible for the audio duration (~400 chars/min for Chinese) +3. Show the user the first ~200 characters as a preview + +If the output looks wrong (empty, garbled, or error), use **AskUserQuestion**: + +``` +Transcription may have an issue: +- Expected: ~[N] chars for [M] minutes of audio +- Got: [actual chars] chars +- Preview: "[first 100 chars...]" + +Options: +A) Save as-is — the output looks fine to me +B) Retry with fallback — split into chunks and merge (handles long audio / OOM) +C) Reconfigure — try a different model or endpoint +D) Abort — something is wrong with the service +``` + +If output is good, save as `.txt` alongside the original file or to user-specified location. + +## Step 5: Fallback — Overlap-Merge for Very Long Audio + +If single request fails (timeout, OOM, HTTP error), fall back to chunked transcription with overlap merging: + +```bash +python3 ${CLAUDE_PLUGIN_ROOT}/scripts/overlap_merge_transcribe.py \ + --config "${CLAUDE_PLUGIN_DATA}/config.json" \ + INPUT_AUDIO OUTPUT.txt +``` + +This splits into 18-minute chunks with 2-minute overlap, then merges using punctuation-stripped fuzzy matching. See [references/overlap_merge_strategy.md](references/overlap_merge_strategy.md) for the algorithm details. + +## Reconfigure + +To change the ASR endpoint, model, or proxy settings: + +```bash +rm "${CLAUDE_PLUGIN_DATA}/config.json" +``` + +Then re-run Step 0 to collect new values via AskUserQuestion. diff --git a/asr-transcribe-to-text/references/overlap_merge_strategy.md b/asr-transcribe-to-text/references/overlap_merge_strategy.md new file mode 100644 index 00000000..e005be0e --- /dev/null +++ b/asr-transcribe-to-text/references/overlap_merge_strategy.md @@ -0,0 +1,56 @@ +# Overlap-Merge Strategy: Why and How + +## The Problem with Naive Chunking + +When ASR transcribes audio in chunks, each chunk's last sentence gets **forcibly truncated**. The model closes the sentence at the chunk boundary even if the speaker is mid-sentence. + +**Real example from testing (5-minute chunks):** + +| Version | Text at boundary | +|---------|-----------------| +| 5min chunk ending | "...靠的就。" (truncated) | +| Continuous 10min | "...靠的就是其中一次战略会。" (complete) | +| Next 5min chunk start | "如果不这么啃，这个业务..." (picks up but gap exists) | + +Concatenating these chunks produces: "靠的就。如果不这么啃..." — losing "是其中一次战略会" entirely. + +## Why Exact String Matching Fails + +The same 2-minute audio segment transcribed in two different contexts (end of chunk A vs start of chunk B) produces **different punctuation**: + +- Chunk A: "可能三年AI的进化" +- Chunk B: "可能。三年AI的进化" + +The words are identical, but punctuation differs because the model's sentence boundary decisions depend on surrounding context. Exact string matching finds zero overlap. + +## The Solution: Punctuation-Stripped Fuzzy Matching + +1. Strip all punctuation from both the tail of chunk A and the head of chunk B +2. Find the longest common substring in the stripped versions +3. Use chunk B's version at the merge point (because chunk A's ending is truncated while chunk B has the complete sentence) + +## Optimal Parameters (Empirically Determined) + +| Parameter | Value | Rationale | +|-----------|-------|-----------| +| Chunk duration | 18 min (1080s) | Safe margin under 20min paper benchmark; 4090 24GB handles much more | +| Overlap duration | 2 min (120s) | ~800 chars overlap region; enough for reliable fuzzy matching | +| Min match length | 15 chars | Filters false positives while catching real overlaps | +| Search window | 600 chars | Covers the overlap region with margin | + +## When to Use This Fallback + +Only use overlap-merge when the single full-length request fails. Reasons it might fail: +- Audio longer than ~2 hours (untested territory, may OOM on 24GB VRAM) +- GPU memory pressure from other processes +- Network timeout (curl max-time exceeded) + +For audio under 1 hour, always try single request first — it's faster, simpler, and produces the best quality. + +## Empirical Comparison (55-minute AI course recording) + +| Strategy | Segments | Boundaries | Total chars | Quality | +|----------|----------|------------|-------------|---------| +| 12x5min direct concat | 12 | 11 cuts | 23,060 | 11 truncated sentences | +| 4x18min overlap merge | 4 | 3 merges | 22,781 | Zero truncation | +| 1x55min single request | 1 | 0 | 22,889 | Perfect (best) | diff --git a/asr-transcribe-to-text/scripts/overlap_merge_transcribe.py b/asr-transcribe-to-text/scripts/overlap_merge_transcribe.py new file mode 100644 index 00000000..b7305756 --- /dev/null +++ b/asr-transcribe-to-text/scripts/overlap_merge_transcribe.py @@ -0,0 +1,200 @@ +# /// script +# requires-python = ">=3.9" +# /// +""" +Overlap-merge transcription for long audio files. + +Splits audio into 18-minute chunks with 2-minute overlap, transcribes each chunk +via a configurable ASR endpoint, then merges using punctuation-stripped fuzzy +matching to eliminate sentence truncation at boundaries. + +Usage: + python3 scripts/overlap_merge_transcribe.py INPUT_AUDIO OUTPUT.txt --config CONFIG.json + python3 scripts/overlap_merge_transcribe.py INPUT_AUDIO OUTPUT.txt --endpoint URL --model MODEL +""" + +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile + + +def get_duration(audio_path: str) -> float: + result = subprocess.run( + ["ffprobe", "-v", "error", "-show_entries", "format=duration", + "-of", "default=noprint_wrappers=1:nokey=1", audio_path], + capture_output=True, text=True + ) + return float(result.stdout.strip()) + + +def split_audio(audio_path: str, chunk_dir: str, chunk_duration: int, overlap: int) -> list[tuple[int, int, str]]: + """Split audio into overlapping chunks. Returns list of (start_sec, duration_sec, chunk_path).""" + total = int(get_duration(audio_path)) + chunks = [] + start = 0 + + while start < total: + duration = min(chunk_duration, total - start) + chunk_path = os.path.join(chunk_dir, f"chunk_{len(chunks):02d}.mp3") + + subprocess.run( + ["ffmpeg", "-i", audio_path, "-ss", str(start), "-t", str(duration), + "-acodec", "copy", chunk_path, "-y"], + capture_output=True + ) + chunks.append((start, duration, chunk_path)) + print(f" Chunk {len(chunks)-1}: {start//60}:{start%60:02d} - {(start+duration)//60}:{(start+duration)%60:02d}", file=sys.stderr) + + start += duration - overlap + if start + duration >= total and duration == chunk_duration: + start = total - duration # ensure last chunk covers the end + if start <= chunks[-1][0]: + break + + return chunks + + +def transcribe(audio_path: str, endpoint: str, model: str, noproxy: bool = True) -> str: + """Send audio to ASR endpoint and return text.""" + noproxy_args = ["--noproxy", "*"] if noproxy else [] + result = subprocess.run( + ["curl", "-s", "--max-time", "600"] + noproxy_args + [ + endpoint, + "-F", f"file=@{audio_path}", + "-F", f"model={model}" + ], + capture_output=True, text=True + ) + data = json.loads(result.stdout) + return data["text"] + + +def strip_punct(text: str) -> str: + """Remove all punctuation, keep only CJK chars, letters, and digits.""" + return re.sub(r'[^\w\u4e00-\u9fff]', '', text) + + +def fuzzy_merge(text_a: str, text_b: str, search_chars: int = 600, min_match: int = 15) -> str: + """ + Merge two overlapping transcription segments using punctuation-stripped fuzzy matching. + + The ASR model produces slightly different punctuation for the same audio segment + across different runs, so exact string matching fails. By stripping punctuation + before matching, we find the true overlap region reliably. + + Uses text_b's version at the merge point because text_a truncates its final sentence + while text_b has the complete version. + """ + tail_a_clean = strip_punct(text_a[-search_chars:]) + text_b_clean = strip_punct(text_b) + + best_match_len = 0 + best_b_clean_end = 0 + + # Search for longest matching substring (punctuation-stripped) + for start in range(len(tail_a_clean)): + substr = tail_a_clean[start:start + min_match] + if len(substr) < min_match: + break + pos = text_b_clean.find(substr) + if pos >= 0: + # Extend the match as far as possible + match_len = min_match + while (start + match_len < len(tail_a_clean) + and pos + match_len < len(text_b_clean) + and tail_a_clean[start + match_len] == text_b_clean[pos + match_len]): + match_len += 1 + + if match_len > best_match_len: + best_match_len = match_len + best_b_clean_end = pos + match_len + best_a_clean_start = start + + if best_match_len >= min_match: + # Map clean positions back to raw text positions + # For text_a: find where the match starts in raw text + a_offset = len(text_a) - search_chars + clean_count = 0 + a_cut_pos = len(text_a) + for idx, ch in enumerate(text_a[-search_chars:]): + if strip_punct(ch): + clean_count += 1 + if clean_count > best_a_clean_start: + a_cut_pos = a_offset + idx + break + + # For text_b: find where the match ends in raw text + clean_count = 0 + b_start_pos = 0 + for idx, ch in enumerate(text_b): + if strip_punct(ch): + clean_count += 1 + if clean_count >= best_b_clean_end: + b_start_pos = idx + 1 + break + + print(f" Merged: {best_match_len} chars matched (punct-stripped)", file=sys.stderr) + return text_a[:a_cut_pos] + text_b[b_start_pos:] + else: + print(f" Warning: no overlap found ({best_match_len} chars), concatenating directly", file=sys.stderr) + return text_a + text_b + + +def main(): + parser = argparse.ArgumentParser(description="Overlap-merge ASR transcription") + parser.add_argument("input", help="Input audio/video file") + parser.add_argument("output", help="Output text file") + parser.add_argument("--config", help="Path to config.json (from CLAUDE_PLUGIN_DATA)") + parser.add_argument("--endpoint", default="http://workstation-4090-wsl:8002/v1/audio/transcriptions", help="ASR endpoint URL") + parser.add_argument("--model", default="Qwen/Qwen3-ASR-1.7B", help="Model name") + parser.add_argument("--noproxy", action="store_true", default=True, help="Use --noproxy with curl") + parser.add_argument("--chunk-duration", type=int, default=1080, help="Chunk duration in seconds (default: 1080 = 18min)") + parser.add_argument("--overlap", type=int, default=120, help="Overlap duration in seconds (default: 120 = 2min)") + args = parser.parse_args() + + # Load config from file if provided, otherwise use CLI args + if args.config and os.path.exists(args.config): + with open(args.config) as f: + cfg = json.load(f) + args.endpoint = cfg.get("endpoint", args.endpoint) + args.model = cfg.get("model", args.model) + args.noproxy = cfg.get("noproxy", args.noproxy) + + print(f"Input: {args.input}", file=sys.stderr) + total_duration = get_duration(args.input) + print(f"Duration: {total_duration:.0f}s ({total_duration/60:.1f}min)", file=sys.stderr) + + with tempfile.TemporaryDirectory() as chunk_dir: + # Split + print(f"\nSplitting into {args.chunk_duration}s chunks with {args.overlap}s overlap...", file=sys.stderr) + chunks = split_audio(args.input, chunk_dir, args.chunk_duration, args.overlap) + print(f"Created {len(chunks)} chunks\n", file=sys.stderr) + + # Transcribe each chunk + texts = [] + for i, (start, dur, path) in enumerate(chunks): + print(f"Transcribing chunk {i} ({start//60}:{start%60:02d})...", end=" ", file=sys.stderr, flush=True) + text = transcribe(path, args.endpoint, args.model, args.noproxy) + texts.append(text) + print(f"{len(text)} chars", file=sys.stderr) + + # Merge + print(f"\nMerging {len(texts)} segments...", file=sys.stderr) + merged = texts[0] + for i in range(1, len(texts)): + print(f" Merging chunk {i-1} + {i}:", file=sys.stderr) + merged = fuzzy_merge(merged, texts[i]) + + # Save + with open(args.output, "w", encoding="utf-8") as f: + f.write(merged) + + print(f"\nDone! {len(merged)} chars saved to {args.output}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/capture-screen/SKILL.md b/capture-screen/SKILL.md index ab88d264..8cf693ba 100644 --- a/capture-screen/SKILL.md +++ b/capture-screen/SKILL.md @@ -227,6 +227,65 @@ These methods were tested and confirmed to fail on macOS: | Python `import Quartz` (PyObjC) | `ModuleNotFoundError` | PyObjC not installed in system Python; don't attempt to install it — use Swift instead | | `osascript` window id | Wrong format | Returns AppleScript window index, not CGWindowID needed by `screencapture -l` | +## Permission Troubleshooting + +`swift scripts/get_window_id.swift` reads on-screen windows via CoreGraphics, so it needs Screen Recording permission on macOS. + +Use this order: + +1. Confirm trigger +2. Confirm target identity +3. Add/enable exact app in Settings + +If the command fails with `ERROR: Failed to enumerate windows`, do this: + +```bash +open "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture" +``` + +Or print the same checklist directly from the script: + +```bash +swift scripts/get_window_id.swift --permission-hint screen +swift scripts/get_window_id.swift --permission-hint microphone +``` + +Then: + +1. In Privacy & Security → Screen Recording, enable the target app. +2. If your app is missing from the list: + - Ensure you granted permission to the real app bundle (not `swift` / terminal helpers). + - For CLI tools, build/run as a packaged `.app` during permission verification. + - Click `+` and add the `.app` manually from `/Applications`. +3. Re-run the command after restarting the app. +4. If this is a CLI workflow, also check whether the launcher is a helper binary: + - In most cases the entry shown in TCC is the helper process (`swift`, `Terminal`, `iTerm`, etc.), not the business app. + - Permission still works after helper-level grant, but it is not ideal for final UX. + +For mic-access-related prompts, use the same pattern with the microphone pane: + +```bash +open "x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone" +``` + +The same rule still applies: the system can only show permissions for a concrete `.app` bundle. If the request is made by a helper binary, the settings list can be misleading or empty for your product app. + +### Quick Check Template + +```text +1) Error: permission denied +2) Open target pane +3) Verify identity shown by OS = identity you granted +4) If not matched, use the script-reported candidate identities and grant the launcher process +5) Reopen/restart and verify +``` + +For production apps, avoid requesting permissions via `swift`/`python` entry points; always route permission checks in the packaged app process so users only see one target. + +If you maintain another macOS permission-related flow, reuse this standardized triage template: + +- [permission-triage-template.md](references/permission-triage-template.md) + ## Supported Applications | Application | Window ID | AppleScript Control | Notes | diff --git a/capture-screen/references/permission-triage-template.md b/capture-screen/references/permission-triage-template.md new file mode 100644 index 00000000..16c3919a --- /dev/null +++ b/capture-screen/references/permission-triage-template.md @@ -0,0 +1,42 @@ +# macOS 权限排障模板（Screen Recording / 麦克风） + +## 排障目标 +- 在系统设置里找不到目标应用 +- 权限拒绝但设置项看起来已打开 +- 通过终端/脚本入口触发时，用户不知道该给谁授权 + +## 标准排查顺序（必须按序执行） + +1. 确认触发点 + - 明确是哪个权限被拒绝（Screen Recording / 麦克风）。 +2. 确认 TCC 实体 + - 不是脚本文件名。 + - 先确认“当前触发进程”与“最终应用体”是否一致。 + - 关注脚本输出里的候选身份列表（invoker/runtime）并逐项核验。 +3. 确认设置面板 + - 直接跳转到对应隐私面板 + - 允许该进程/应用 + - 重启进程后复验 + +## 通用动作模板 + +```bash +# Screen Recording +open "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture" + +# Microphone +open "x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone" +``` + +## 不在列表时处理 + +- 优先确认请求来自真实 .app Bundle（签名、打包） +- 如果当前为 CLI/脚本入口，先给宿主进程授权（Terminal/iTerm/swift/python） +- 在设置面板点击 `+` 手工添加目标 `.app` +- 变更后退出并重启应用，重新测试 + +## 验收标准（用户侧） + +- 用户能看到一条明确的“应授权对象” +- 错误提示中有“找不到对象时下一步该做什么” +- 无需反复猜测在设置里要点击什么 diff --git a/capture-screen/scripts/get_window_id.swift b/capture-screen/scripts/get_window_id.swift index c19ef7fd..8ccef145 100755 --- a/capture-screen/scripts/get_window_id.swift +++ b/capture-screen/scripts/get_window_id.swift @@ -4,9 +4,11 @@ // Enumerate on-screen windows and print their Window IDs. // // Usage: -// swift get_window_id.swift # List all windows -// swift get_window_id.swift Excel # Filter by keyword -// swift get_window_id.swift "Chrome" # Filter by app name +// swift scripts/get_window_id.swift # List all windows +// swift scripts/get_window_id.swift Excel # Filter by keyword +// swift scripts/get_window_id.swift "Chrome" # Filter by app name +// swift scripts/get_window_id.swift --permission-hint screen # Print Screen Recording triage +// swift scripts/get_window_id.swift --permission-hint microphone # Print Microphone triage // // Output format: // WID=12345 | App=Microsoft Excel | Title=workbook.xlsx @@ -15,10 +17,192 @@ // import CoreGraphics +import Foundation -let keyword = CommandLine.arguments.count > 1 - ? CommandLine.arguments[1] - : "" +enum PermissionKind: String { + case screen + case microphone +} + +let invocationPath = (CommandLine.arguments.first ?? "") +let invokerName = URL(fileURLWithPath: invocationPath).lastPathComponent +let runtimeProcessName = ProcessInfo.processInfo.processName +let invokerIsBundle = invocationPath.hasSuffix(".app") || invocationPath.contains(".app/") +let scriptPath: String? = invocationPath.hasSuffix(".swift") ? invocationPath : nil +let helperBinaries: Set = [ + "swift", + "swift-frontend", + "python", + "python3", + "node", + "uv", + "npm", + "bun", + "pnpm", + "yarn", + "bash", + "zsh", + "sh", + "osascript", + "Terminal", + "iTerm2", + "iTerm" +] + +let invokerCandidates: [String] = { + var candidates = [String]() + var seen = Set() + func append(_ value: String) { + guard !value.isEmpty, !seen.contains(value) else { return } + seen.insert(value) + candidates.append(value) + } + if let scriptPath = scriptPath { + append(scriptPath) + } + if !invokerName.isEmpty { + append(invokerName) + } + if !runtimeProcessName.isEmpty && runtimeProcessName != invokerName { + append(runtimeProcessName) + } + return candidates +}() + +let args = Array(CommandLine.arguments.dropFirst()) + +var permissionHintTarget: PermissionKind? +var keyword = "" +var expectPermissionTarget = false + +func printUsage() { + fputs("Usage:\n", stderr) + fputs(" swift scripts/get_window_id.swift [keyword]\n", stderr) + fputs(" swift scripts/get_window_id.swift --permission-hint [screen|microphone]\n", stderr) + fputs("\n", stderr) + fputs("Options:\n", stderr) + fputs(" --permission-hint [screen|microphone] Print permission triage instructions\n", stderr) + fputs(" -h, --help Show this help\n", stderr) + fputs("\n", stderr) + fputs("Examples:\n", stderr) + fputs(" swift scripts/get_window_id.swift Excel\n", stderr) + fputs(" swift scripts/get_window_id.swift --permission-hint screen\n", stderr) + fputs(" swift scripts/get_window_id.swift --permission-hint microphone\n", stderr) +} + +for arg in args { + if expectPermissionTarget { + if let kind = PermissionKind(rawValue: arg.lowercased()) { + permissionHintTarget = kind + expectPermissionTarget = false + continue + } + fputs("Unknown permission target: \(arg)\n", stderr) + printUsage() + exit(2) + } + + if arg == "-h" || arg == "--help" { + printUsage() + exit(0) + } else if arg == "--permission-hint" { + permissionHintTarget = .screen + expectPermissionTarget = true + } else if arg.hasPrefix("--permission-hint=") { + let target = String(arg.dropFirst("--permission-hint=".count)).lowercased() + guard let kind = PermissionKind(rawValue: target) else { + fputs("Unknown permission target: \(target)\n", stderr) + printUsage() + exit(2) + } + permissionHintTarget = kind + } else if arg == "--permission-hint-screen" { + permissionHintTarget = .screen + } else if arg == "--permission-hint-microphone" || arg == "--permission-hint-mic" { + permissionHintTarget = .microphone + } else if arg.hasPrefix("-") { + fputs("Unknown option: \(arg)\n", stderr) + printUsage() + exit(2) + } else if keyword.isEmpty { + keyword = arg + } +} + +if expectPermissionTarget { + fputs("Missing permission hint target after --permission-hint.\n", stderr) + printUsage() + exit(2) +} + +if let kind = permissionHintTarget { + switch kind { + case .screen: + fputs("Screen Recording permission required.\n", stderr) + printCommonPermissionHint( + pane: "Privacy_ScreenCapture", + label: "Screen Recording" + ) + printPermissionContextHint() + case .microphone: + fputs("Microphone permission required.\n", stderr) + printCommonPermissionHint( + pane: "Privacy_Microphone", + label: "Microphone" + ) + printPermissionContextHint() + } + exit(0) +} + +func printCommonPermissionHint(pane: String, label: String, missing: Bool = true) { + let openCommand = "x-apple.systempreferences:com.apple.preference.security?\(pane)" + fputs("Troubleshooting:\n", stderr) + fputs(" - Open Settings: `open \"\(openCommand)\"`\n", stderr) + fputs(" - In Privacy & Security → \(label), enable the target application.\n", stderr) + if missing { + fputs(" - If the target app is not in the list:\n", stderr) + fputs(" - Granting happens by real .app bundle, not by helper/terminal scripts.\n", stderr) + fputs(" - For CLI workflows, grant to the host app you launch from (Terminal, iTerm, iTerm2, Swift, etc.) if no dedicated .app exists yet.\n", stderr) + fputs(" - Click `+` and add the actual `.app` from `/Applications`.\n", stderr) + } + fputs(" - If permission status does not refresh, quit/reopen terminal/app and retry.\n", stderr) + if helperBinaries.contains(invokerName) || helperBinaries.contains(runtimeProcessName) { + fputs(" - Current launcher is a helper/runtime process (`\(runtimeProcessName)`) -> OS may show this entry instead of the tool name.\n", stderr) + } else if invokerIsBundle { + fputs(" - The launcher path looks like a bundled app, which is the preferred state for permissions.\n", stderr) + } + if let scriptPath = scriptPath { + fputs(" - Script entry: `\(scriptPath)`\n", stderr) + } +} + +func printPermissionContextHint() { + fputs(" - Invoker path: `\(invocationPath)`\n", stderr) + fputs(" - Runtime process: `\(runtimeProcessName)`\n", stderr) + if !invokerCandidates.isEmpty { + fputs(" - Candidate identities in System Settings:\n", stderr) + for identity in invokerCandidates { + fputs(" - \(identity)\n", stderr) + } + } + if invokerIsBundle { + fputs(" This looks like a bundled app path, so the setting should match the app identity.\n", stderr) + } else { + fputs(" If this is not your final app binary, permissions can be inconsistent.\n", stderr) + } + fputs(" - Recommended for production: keep permission requests inside your signed `.app` process.\n", stderr) + if let scriptPath = scriptPath { + fputs(" - Script entry currently used: `\(scriptPath)`.\n", stderr) + } +} + +func printScreenRecordingPermissionHint() { + fputs("Screen Recording permission required.\n", stderr) + fputs("Troubleshooting:\n", stderr) + printCommonPermissionHint(pane: "Privacy_ScreenCapture", label: "Screen Recording") + printPermissionContextHint() +} guard let windowList = CGWindowListCopyWindowInfo( .optionOnScreenOnly, kCGNullWindowID @@ -27,6 +211,7 @@ guard let windowList = CGWindowListCopyWindowInfo( fputs("Possible causes:\n", stderr) fputs(" - No applications with visible windows are running\n", stderr) fputs(" - Screen Recording permission not granted (System Settings → Privacy & Security → Screen Recording)\n", stderr) + printScreenRecordingPermissionHint() exit(1) } diff --git a/claude-export-txt-better/SKILL.md b/claude-export-txt-better/SKILL.md new file mode 100644 index 00000000..3caa42d2 --- /dev/null +++ b/claude-export-txt-better/SKILL.md @@ -0,0 +1,101 @@ +--- +name: fixing-claude-export-conversations +description: > + Fixes broken line wrapping in Claude Code exported conversation files (.txt), + reconstructing tables, paragraphs, paths, and tool calls that were hard-wrapped + at fixed column widths. Includes an automated validation suite (generic, file-agnostic checks). + Triggers when the user has a Claude Code export file with broken formatting, + mentions "fix export", "fix conversation", "exported conversation", "make export + readable", references a file matching YYYY-MM-DD-HHMMSS-*.txt, or has a .txt + file with broken tables, split paths, or mangled tool output from Claude Code. +--- + +# Fixing Claude Code Export Conversations + +Reconstruct broken line wrapping in Claude Code exported `.txt` files. + +## Quick Start + +```bash +# Fix and show stats +uv run /scripts/fix-claude-export.py --stats + +# Custom output +uv run /scripts/fix-claude-export.py -o fixed.txt + +# Validate the result (53 automated checks) +uv run /scripts/validate-claude-export-fix.py fixed.txt +``` + +Replace `` with the resolved path to this skill's directory. Find it with: +```bash +find ~/.claude -path "*/fixing-claude-export-conversations/scripts" -type d 2>/dev/null +``` + +## Workflow + +Copy this checklist and track progress: + +``` +- [ ] Step 1: Locate the exported .txt file +- [ ] Step 2: Run fix script with --stats +- [ ] Step 3: Run validation suite +- [ ] Step 4: Spot-check output (tables, CJK paragraphs, tool results) +- [ ] Step 5: Deliver fixed file to user +``` + +**Step 1: Locate the file.** Claude Code exports use the naming pattern `YYYY-MM-DD-HHMMSS-.txt`. + +**Step 2: Run the fix script.** +```bash +uv run /scripts/fix-claude-export.py -o --stats +``` +Review the stats output — typical results: 20-25% line reduction, 80+ table borders fixed, 160+ table cells fixed. + +**Step 3: Run the validation suite.** +```bash +uv run /scripts/validate-claude-export-fix.py +``` +All checks must pass. If any fail, investigate before delivering. Use `--verbose` for full details on passing checks too. + +**Step 4: Spot-check.** Open the output and verify: +- Tables have intact borders (box-drawing characters on single lines) +- CJK/English mixed text has pangu spacing (`Portal 都需要`, not `Portal都需要`) +- Tool result blocks (`⎿`) have complete content on joined lines +- Diff output within tool results has each line number on its own line + +**Step 5: Deliver** the fixed file to the user. + +## What Gets Fixed + +The script handles 10 content types using a state-machine with next-line look-ahead: + +- **User prompts** (`❯` prefix, dw=76 padding) — paragraph joins with pangu spacing +- **Claude responses** (`●` prefix) — narrative, bullet, and numbered list joins +- **Claude paragraphs** (2-space indent) — next-line look-ahead via `_is_continuation_fragment` +- **Tables** — border reconstruction, cell re-padding with pipe-count tracking +- **Tool calls** (`● Bash(` etc.) — path and argument reconstruction +- **Tool results** (`⎿` prefix) — continuation joins including deeper-indented fragments +- **Plan text** (5-space indent) — next-line look-ahead via `_is_plan_continuation_fragment` +- **Agent tree** (`├─`/`└─`) — preserved structure +- **Separators** (`────`, `---`) — never joined +- **Tree connectors** (standalone `│`) — preserved + +## Key Design Decisions + +**Next-line look-ahead** (not dw thresholds): Instead of asking "was this line wrapped?" (fragile threshold), the script asks "does the next line look like a continuation?" by examining its content patterns — lowercase start, CJK ideograph start, opening bracket, hyphen/slash/underscore continuation. + +**Pangu spacing**: Inserts spaces between ASCII alphanumeric characters and CJK ideographs at join boundaries. Also triggers for `%`, `#`, `+`, `:` adjacent to CJK. + +**Mid-token detection**: Joins without space when boundaries indicate identifiers (`BASE_` + `URL`), paths (`documents` + `/05-team`), or hyphenated names (`ready` + `-together`). Exception: `--` prefix gets a space (`run` + `--headed`). + +## Safety + +- Never modifies the original file +- Marker counts verified: `❯`, `●`, `✻`, `⎿`, `…` must match input/output +- Runaway join detection: warns if any line exceeds 500 display-width +- Strict UTF-8 encoding — no silent fallbacks + +## Dependencies + +Python 3.10+ via `uv run` — zero external packages (stdlib only: `unicodedata`, `argparse`, `re`, `pathlib`, `dataclasses`). \ No newline at end of file diff --git a/claude-export-txt-better/evals/evals.json b/claude-export-txt-better/evals/evals.json new file mode 100644 index 00000000..b0d7dc4e --- /dev/null +++ b/claude-export-txt-better/evals/evals.json @@ -0,0 +1,32 @@ +{ + "skill_name": "fixing-claude-export-conversations", + "evals": [ + { + "id": 1, + "prompt": "Fix this exported conversation file: — it was exported from Claude Code and has broken line wrapping everywhere. Output the fixed version next to the original.", + "expected_output": "A *-fixed.txt file with correct line wrapping, intact table borders, matching marker counts, and fewer total lines than the input.", + "files": [], + "assertions": [ + {"name": "output_file_exists", "description": "The fixed output file was created successfully"}, + {"name": "line_count_reduced", "description": "Output has fewer lines than input. Reasonable reduction is 10-25%."}, + {"name": "marker_counts_match", "description": "The count of ❯, ●, ✻, ⎿, and … characters in output matches input exactly"}, + {"name": "no_broken_table_borders", "description": "Every line starting with ┌/├/└ also contains the matching right border on the same line"}, + {"name": "no_runaway_joins", "description": "No output line exceeds 500 display-width characters"} + ] + }, + { + "id": 2, + "prompt": "我从 Claude Code 导出了一个对话记录 ，表格和段落的换行全部乱了。帮我修复一下，顺便看看统计数据——修了多少行？", + "expected_output": "Fixed file created with --stats output showing join counts per content type. Tables should be properly reconstructed with correct column padding.", + "files": [], + "assertions": [ + {"name": "output_file_exists", "description": "The fixed output file was created successfully"}, + {"name": "line_count_reduced", "description": "Output has fewer lines than input. Reasonable reduction is 10-25%."}, + {"name": "marker_counts_match", "description": "The count of ❯, ●, ✻, ⎿, and … characters in output matches input exactly"}, + {"name": "no_broken_table_borders", "description": "Every line starting with ┌/├/└ also contains the matching right border on the same line"}, + {"name": "no_runaway_joins", "description": "No output line exceeds 500 display-width characters"}, + {"name": "stats_output_present", "description": "The run log contains statistics output with join counts (e.g., 'Table rows merged', 'Tool results fixed')"} + ] + } + ] +} diff --git a/claude-export-txt-better/scripts/fix-claude-export.py b/claude-export-txt-better/scripts/fix-claude-export.py new file mode 100644 index 00000000..27c50d6e --- /dev/null +++ b/claude-export-txt-better/scripts/fix-claude-export.py @@ -0,0 +1,2065 @@ +#!/usr/bin/env python3 +"""Fix broken line wrapping in Claude Code exported conversation files. + +Claude Code exports hard-wrap lines at fixed column widths, breaking tables, +paragraphs, and paths. This script reconstructs the original logical lines +using a state-machine + lookahead merge approach. + +Usage: + uv run scripts/fix-claude-export.py + uv run scripts/fix-claude-export.py -o + uv run scripts/fix-claude-export.py --stats --dry-run +""" + +from __future__ import annotations + +import argparse +import re +import sys +import unicodedata +from dataclasses import dataclass +from pathlib import Path + +# --------------------------------------------------------------------------- +# Display-width helpers +# --------------------------------------------------------------------------- + +def display_width(s: str) -> int: + """Calculate display width accounting for CJK double-width characters.""" + w = 0 + for ch in s: + eaw = unicodedata.east_asian_width(ch) + w += 2 if eaw in ("W", "F") else 1 + return w + + +def is_wide_char(ch: str) -> bool: + """Return True if *ch* occupies two display columns (CJK full-width).""" + return unicodedata.east_asian_width(ch) in ("W", "F") + + +def _is_cjk_ideograph(ch: str) -> bool: + """Return True if *ch* is a CJK ideograph (not punctuation/symbol). + + CJK ideographs have Unicode category ``Lo`` (Letter, other) — e.g. + ``你``, ``好``, ``接``. CJK punctuation (``。，！？；：「」（）``) has + categories ``Ps``, ``Pe``, ``Po``, etc. and should NOT match. + + This distinction matters for pangu spacing: a space is inserted between + ASCII alphanumeric characters and CJK ideographs, but NOT between + CJK punctuation and anything. + """ + return is_wide_char(ch) and unicodedata.category(ch) == "Lo" + + +# --------------------------------------------------------------------------- +# Join helpers +# --------------------------------------------------------------------------- + +def smart_join(left: str, right_content: str) -> str: + """Join text with CJK-aware spacing (pangu style). + + Spacing rules at the join boundary: + + - **CJK ↔ CJK**: no space (both characters wide). + - **ASCII alnum ↔ CJK ideograph**: insert one space. This is "pangu + spacing" — the standard practice of separating Han characters from + Latin letters / digits in mixed CJK/English text. The original + content almost always has these spaces; they are lost when Claude + hard-wraps at the column boundary. + - **CJK punctuation ↔ anything**: no space. Punctuation like ``，`` + ``。`` ``）`` ``（`` clings to its neighbor. + - **ASCII ↔ ASCII**: insert one space (English word boundary). + """ + left_s = left.rstrip() + right_s = right_content.lstrip() + if not left_s or not right_s: + return left_s + right_s + last_ch = left_s[-1] + first_ch = right_s[0] + + last_wide = is_wide_char(last_ch) + first_wide = is_wide_char(first_ch) + + if last_wide and first_wide: + # Both CJK — no space. + return left_s + right_s + + if last_wide or first_wide: + # Mixed CJK/ASCII boundary — apply pangu spacing. + # In addition to alnum, certain symbols that attach to numbers + # or abbreviations (%, #, +, :) also trigger pangu spacing because + # they're part of the same "token" as the adjacent number/word. + _PANGU_SYMBOLS = "%#+:" + if last_wide: + # CJK → ASCII: space if CJK ideograph + ASCII alnum/symbol. + if _is_cjk_ideograph(last_ch) and (first_ch.isalnum() or first_ch in _PANGU_SYMBOLS): + return left_s + " " + right_s + return left_s + right_s + else: + # ASCII → CJK: space if ASCII alnum/symbol + CJK ideograph. + if (last_ch.isalnum() or last_ch in _PANGU_SYMBOLS) and _is_cjk_ideograph(first_ch): + return left_s + " " + right_s + return left_s + right_s + + # Both ASCII — mid-token detection before adding space. + # Path continuation: alnum + / (e.g., "documents" + "/05-team") + # Hyphen continuation: alnum + - (e.g., "ready" + "-together") + # Underscore continuation: _ + alnum (e.g., "BASE_" + "URL") + # Exception: -- prefix is a CLI flag (e.g., "run" + "--headed"). + if last_ch.isalnum() and first_ch in ("-", "/"): + if first_ch == "-" and right_s.startswith("--"): + pass # fall through to default (add space) + else: + return left_s + right_s + if last_ch in ("-", "/") and first_ch.isalnum(): + return left_s + right_s + # Underscore at identifier boundary (e.g., "E2E_PO_BASE_" + "URL") + if last_ch == "_" and first_ch.isalnum(): + return left_s + right_s + if last_ch.isalnum() and first_ch == "_": + return left_s + right_s + + # Default: word boundary → add space. + return left_s + " " + right_s + + +def raw_join(left: str, right: str) -> str: + """Strip trailing whitespace from *left*, leading spaces from *right*, concat.""" + return left.rstrip() + right.lstrip() + + +def _table_cell_content_join( + left: str, right: str, *, left_filled: bool = False +) -> str: + """Join two cell-content fragments from a multi-row table cell. + + Claude wraps long cell content across multiple physical rows at fixed + column widths. This function reassembles the original content by + detecting mid-word breaks (e.g. ``Contr`` + ``oller``) vs word + boundaries (e.g. ``Backend`` + ``Risk``). + + When *left_filled* is True, the left fragment filled its entire column + (≤1 trailing space before the ``│`` delimiter), meaning the split was + forced at the column boundary — almost certainly mid-word. This is the + strongest signal and overrides other heuristics. + + Heuristic priority: + 1. left_filled → raw join (column-boundary split) + 2. Continuation punctuation at boundary (``- _ . /``) → raw join + 3. Left ends with letter, right starts with lowercase → mid-word + 4. Left ends with digit, right starts with digit → mid-number + 5. Otherwise → smart_join (CJK-aware spacing) + """ + left_s = left.rstrip() + right_s = right.lstrip() + if not left_s or not right_s: + return left_s + right_s + + last_ch = left_s[-1] + first_ch = right_s[0] + + # Continuation punctuation at boundary — usually concatenate directly. + # Exception: double-hyphen at right boundary (``--flag``) is a CLI + # argument prefix, not a continuation of the left content. + if last_ch in ("-", "_", ".", "/") or first_ch in ("-", "_", ".", "/"): + if first_ch == "-" and right_s.startswith("--"): + pass # fall through to other checks + else: + return left_s + right_s + + if left_filled: + # Column-boundary split: content filled the cell width. + # The break is at a fixed position and likely mid-token, but we + # verify with character-class checks to avoid false positives + # like "Behavioral" + "Spec" (complete word at column edge). + if last_ch.isalpha() and first_ch.islower(): + return left_s + right_s + if last_ch.isupper() and first_ch.isupper(): + return left_s + right_s + if last_ch.isdigit() and first_ch.isdigit(): + return left_s + right_s + if last_ch.isalnum() and first_ch.isdigit(): + return left_s + right_s + if last_ch.isdigit() and first_ch.islower(): + # Hex hash continuation: c3df79 + b → c3df79b + return left_s + right_s + # Filled but no mid-word evidence → fall through to smart_join. + + # Everything else: word-boundary or ambiguous → CJK-aware spacing. + return smart_join(left_s, right_s) + + +def table_cell_join(left: str, right: str) -> str: + """Join table cell continuation, preserving column spacing. + + Unlike raw_join, this strips only the table indent (matching the left + side's indent) from the right side, keeping internal cell padding. + It also ensures a space between ``│`` and adjacent cell content. + """ + ls = left.rstrip() + if not ls: + return right.rstrip() + # Determine indent from left side (typically 5 spaces for plan tables) + indent = len(ls) - len(ls.lstrip()) + rs = right.rstrip() + # Strip exactly the indent, keep remaining whitespace (cell padding) + if len(rs) >= indent and rs[:indent].strip() == "": + rs = rs[indent:] + else: + rs = rs.lstrip() + if not rs: + return ls + # Ensure space between │ and non-│/non-space content + if ls[-1] == "│" and rs[0] not in " │": + return ls + " " + rs + if ls[-1] not in " │" and rs[0] == "│": + return ls + " " + rs + return ls + rs + + +def boundary_aware_join(left: str, right: str) -> str: + """Join with heuristic for mid-word vs word-boundary splits. + + If *left* had a trailing space before stripping, the hard wrap was at a + word boundary -- preserve one space. Otherwise the wrap split mid-word + -- concatenate directly (no space). + """ + # Check if left had trailing whitespace (word-boundary wrap). + left_had_trailing_space = left != left.rstrip() + left_s = left.rstrip() + right_s = right.lstrip() + if not left_s or not right_s: + return left_s + right_s + if left_had_trailing_space: + # Word-boundary wrap: preserve spacing via smart_join which + # handles pangu spacing (ASCII alnum ↔ CJK ideograph). + return smart_join(left_s, right_s) + # Mid-word wrap: no space. + return left_s + right_s + + +def _dw_aware_join(left: str, right: str, left_dw: int) -> str: + """Join with display-width-aware mid-word detection. + + Claude drops the trailing space at wrap points, making it impossible to + distinguish word-boundary from mid-word breaks by whitespace alone. + This function uses the display width of the *left* physical line to + resolve the ambiguity: + + - **dw < 74**: the line ended well below the wrap column (~76) — the + break was at a natural word boundary → smart_join. + - **dw >= 75**: the line was forced to break near the column limit. + Use character-class heuristics: alpha→lower = likely mid-word (raw + join); all other transitions = word boundary (smart_join). + """ + if left_dw < 75: + return smart_join(left, right) + + # Near wrap limit — check trailing space first. + left_had_trailing = left != left.rstrip() + left_s = left.rstrip() + right_s = right.lstrip() + if not left_s or not right_s: + return left_s + right_s + + if left_had_trailing: + return smart_join(left_s, right_s) + + # No trailing space, near wrap limit. + # Character-class heuristics for mid-word detection. + last_ch = left_s[-1] + first_ch = right_s[0] + # alpha→lower: mid-word (e.g., "Backgrou" + "nd") + if last_ch.isalpha() and first_ch.islower(): + return left_s + right_s + # Path/hyphenated-name continuations: slash or hyphen at boundary + # (e.g., ".claude/skills" + "/generating-..." or "ready" + "-together") + if last_ch.isalnum() and first_ch in ("-", "/"): + return left_s + right_s + if last_ch in ("-", "/") and first_ch.isalnum(): + return left_s + right_s + # digit→alpha (e.g., "md-e" + "2e-section" — hex/version fragments) + if last_ch.isalpha() and first_ch.isdigit(): + return left_s + right_s + + # All other cases: treat as word boundary. + return smart_join(left_s, right_s) + + +def _bullet_join(left: str, right: str) -> str: + """Join a bullet line with its wrapped continuation. + + Like smart_join, but with mid-word detection: when left ends with an + ASCII letter and right (after stripping) starts with a lowercase ASCII + letter, concatenate directly (the hard-wrap split a word mid-token, + e.g. ``RiskModelAss`` + ``ignment``). Otherwise delegate to smart_join + for CJK-aware spacing. + """ + left_s = left.rstrip() + right_s = right.lstrip() + if not left_s or not right_s: + return left_s + right_s + last_ch = left_s[-1] + first_ch = right_s[0] + # Mid-word split: left ends with a letter, right starts with lowercase. + if last_ch.isalpha() and first_ch.islower(): + return left_s + right_s + # Hyphenated names / paths split at boundary. + # e.g. "ready" + "-together-project" or "skills" + "/generating" + if last_ch.isalnum() and first_ch in ("-", "/"): + return left_s + right_s + if last_ch in ("-", "/") and first_ch.isalnum(): + return left_s + right_s + return smart_join(left_s, right_s) + + +# --------------------------------------------------------------------------- +# Line classification helpers +# --------------------------------------------------------------------------- + +# Markers that ALWAYS start a new logical line (never join TO these). +_USER_PROMPT_RE = re.compile(r"^❯ ") +_CLAUDE_ACTION_RE = re.compile(r"^● ") +_THINKING_RE = re.compile(r"^✻ ") +_HR_RE = re.compile(r"^ ---") +_BOX_HR_RE = re.compile(r"^ ────") +_AGENT_TREE_RE = re.compile(r"^ [├└]─") +_TOOL_RESULT_RE = re.compile(r"^ ⎿") +_BULLET_RE = re.compile(r"^ - ") +_NUMBERED_RE = re.compile(r"^ \d+\. ") + +# Indented bullets / numbered items inside plan blocks (5-space indent). +_PLAN_BULLET_RE = re.compile(r"^ - ") +_PLAN_NUMBERED_RE = re.compile(r"^ \d+\. ") + +# Tool call openers (● ToolName( ...). +_TOOL_CALL_RE = re.compile( + r"^● (?:Bash|Read|Write|Glob|Grep|Edit|Update|Searched|NotebookEdit)\(" +) + +# Table box-drawing characters. +_TABLE_CORNERS = set("┐┤┘") + + +def _is_truly_empty(line: str) -> bool: + """A truly empty line (zero length after stripping the newline).""" + return len(line) == 0 + + +def _is_structural_break(line: str) -> bool: + """Return True if *line* is a structural marker that must never be joined TO.""" + if _is_truly_empty(line): + return True + if _USER_PROMPT_RE.match(line): + return True + if _CLAUDE_ACTION_RE.match(line): + return True + if _THINKING_RE.match(line): + return True + if _HR_RE.match(line): + return True + if _BOX_HR_RE.match(line): + return True + if _AGENT_TREE_RE.match(line): + return True + if _TOOL_RESULT_RE.match(line): + return True + if _BULLET_RE.match(line): + return True + if _NUMBERED_RE.match(line): + return True + return False + + +# Regex for CJK labels like 模块:, 输出文件:, 状态:, 覆盖范围: +_CJK_LABEL_RE = re.compile(r"[\u4e00-\u9fff]{1,6}[:：]") +# Regex for English labelled list items: "Phase 1:", "Step 2:", "Layer 3:" +_LABELLED_ITEM_RE = re.compile(r"[A-Z]\w+ \d+[:.] ") + + +def _is_continuation_fragment(nl: str, acc: str) -> bool: + """Return True if *nl* looks like a wrapped continuation of *acc*. + + This is the core predicate for joining 2-space-indented paragraph text. + Instead of asking "was the current line wrapped?" (fragile dw threshold), + it asks "does the NEXT line look like a continuation fragment?" + + A continuation fragment has NO structural identity — it is not a new + bullet, numbered item, labelled field, or structural marker. It + typically starts with a lowercase letter, CJK ideograph, or is a short + uppercase fragment of a sentence that wrapped mid-phrase. + """ + # Must be 2-space indent (not deeper, not tool result). + if not nl.startswith(" "): + return False + if nl.startswith(" "): # 5+ space = plan/tool block + return False + if nl.startswith(" ⎿"): + return False + if _is_structural_break(nl): + return False + + stripped = nl.lstrip() + if not stripped: + return False + + # --- New-item patterns (NOT a continuation) --- + if stripped.startswith("- "): + return False + if re.match(r"\d+[.)] ", stripped): + return False + if _LABELLED_ITEM_RE.match(stripped): + return False + # CJK labels: 模块:, 输出文件:, 状态:, 覆盖范围: etc. + if _CJK_LABEL_RE.match(stripped): + return False + # Column layout (side-by-side comparison with internal spacing). + if " " in stripped: # 8+ internal spaces + return False + + nl_dw = display_width(nl.rstrip()) + # A "continuation" line that is itself full-width is likely independent. + if nl_dw >= 76: + return False + + first_ch = stripped[0] + + # --- Strong continuation signals --- + # Lowercase → mid-sentence continuation. + if first_ch.islower(): + return True + # CJK ideograph → continuing Chinese text. + if _is_cjk_ideograph(first_ch): + return True + # CJK/fullwidth punctuation → continues previous CJK content. + if is_wide_char(first_ch) and not _is_cjk_ideograph(first_ch): + return True + # Opening bracket → e.g. "(c67e5ded-..." UUID, list in parens. + if first_ch in ("(", "[", "{", "（", "「", "【"): + return True + # Hyphen/slash continuation: acc ends with alnum, next starts with -//. + # (e.g. "ready" + "-together-project", "skills" + "/generating") + if first_ch in ("-", "/"): + _acc_s = acc.rstrip() + if _acc_s and _acc_s[-1].isalnum(): + return True + + # --- Check if accumulated text signals continuation --- + acc_stripped = acc.rstrip() + if acc_stripped: + last_acc = acc_stripped[-1] + # Continuation operators at end of acc → next line must continue. + if last_acc in (",", "+", "→", "=", "&", "|", "、"): + return True + # Acc ends with sentence-terminal → next line is a new sentence. + if last_acc in "。.！!？?；;": + return False + + # --- Uppercase start: ambiguous --- + # Short fragment (< 55 dw) is likely a sentence fragment. + if nl_dw < 55: + return True + return False + + +def _is_plan_continuation_fragment(nl: str, acc: str) -> bool: + """Return True if *nl* looks like a wrapped continuation in 5-space plan text. + + Same design philosophy as _is_continuation_fragment: examine the NEXT + line's content to decide if it is a new item or a continuation fragment. + """ + if not nl.startswith(" "): + return False + if nl.startswith(" "): # 7+ space = deeper indent, separate item + return False + if _is_truly_empty(nl): + return False + if _is_structural_break(nl): + return False + if _is_plan_structural(nl): + return False + + stripped = nl.lstrip() + if not stripped: + return False + + # CJK labels (模块:, 输出文件:, 状态:) + if _CJK_LABEL_RE.match(stripped): + return False + # English labelled items with number (Phase 1:, Step 2:) + if _LABELLED_ITEM_RE.match(stripped): + return False + # English labels without number (Plan:, Context:, Summary:) + if re.match(r"[A-Z][a-zA-Z]+: ", stripped): + return False + # ASCII terminal labels (error:, hint:, remote:) + if re.match(r"[a-z]+: ", stripped): + return False + # Diff output line numbers (e.g., "600 - **Test Data**:") + # Pattern: 1-5 digits followed by 2+ spaces (diff line number format). + if re.match(r"\d{1,5}\s{2,}", stripped): + return False + # Column layout + if " " in stripped: + return False + + nl_dw = display_width(nl.rstrip()) + if nl_dw >= 76: + return False + + first_ch = stripped[0] + + # --- New-item patterns that start with lowercase --- + # ASCII labels like "error:", "hint:", "remote:" are terminal output + # lines, not continuations. They start a new message. + if re.match(r"[a-z]+: ", stripped): + return False + + # --- Strong continuation signals --- + if first_ch.islower(): + return True + if _is_cjk_ideograph(first_ch): + return True + if is_wide_char(first_ch) and not _is_cjk_ideograph(first_ch): + return True + if first_ch in ("(", "[", "{", "（", "「", "【"): + return True + # Hyphen/slash continuation (compound names, paths). + if first_ch in ("-", "/"): + _acc_s = acc.rstrip() + if _acc_s and _acc_s[-1].isalnum(): + return True + + # Non-alnum, non-CJK, non-bracket starts (!, #, >, *, etc.) + # are structural markers in terminal output, not continuations. + if not first_ch.isalnum(): + return False + + # --- Check accumulated text ending --- + acc_stripped = acc.rstrip() + if acc_stripped: + last_acc = acc_stripped[-1] + if last_acc in (",", "+", "→", "=", "&", "|", "、"): + return True + if last_acc in "。.！!？?；;": + return False + + # Uppercase start, ambiguous: short fragment = likely continuation. + if nl_dw < 55: + return True + return False + + +def _is_plan_structural(line: str) -> bool: + """Structural markers within 5-space-indented plan blocks.""" + if _PLAN_BULLET_RE.match(line): + return True + if _PLAN_NUMBERED_RE.match(line): + return True + stripped = line.lstrip() + if stripped.startswith("##"): + return True + # Box-drawing separators within plan blocks (5-space indent). + # The 2-space variant is caught by _BOX_HR_RE in _is_structural_break, + # but 5-space-indented ones slip through. + if stripped.startswith("────"): + return True + # Markdown HR within plan blocks (5-space indent). + if stripped == "---": + return True + # Tree connector (standalone │ used in ASCII dependency diagrams). + # Must not be confused with table data rows (which have 2+ │ chars). + if stripped == "│": + return True + # File-tree lines (├── or └── patterns) within plan blocks. + if stripped.startswith("├──") or stripped.startswith("└──"): + return True + # Expansion indicators ("… +N lines (ctrl+o to expand)"). + if stripped.startswith("…"): + return True + return False + + +def _has_continuation_signal(line: str) -> bool: + """Detect if *line* was almost certainly hard-wrapped mid-content. + + Lines ending with a trailing comma, a CJK character, or an unclosed + bracket are continuation signals — they indicate the content continues + on the next line regardless of how narrow the display width is. + """ + stripped = line.rstrip() + if not stripped: + return False + last_ch = stripped[-1] + if last_ch == ",": + return True + if is_wide_char(last_ch): + return True + if last_ch in ("(", "[", "{"): + return True + return False + + +def _has_unclosed_bracket(line: str) -> bool: + """Detect if *line* contains an opening bracket with no matching close. + + An unclosed bracket means the parenthetical/list continues on the next + physical line — a strong continuation signal regardless of display width. + This catches cases like ``Requirements（P0/P1 分层，每个 Feature`` where + Claude wraps at a word boundary well below the column limit because the + remaining CJK text would push past it. + """ + _PAIRS = (("(", ")"), ("[", "]"), ("{", "}"), + ("（", "）"), ("「", "」"), ("【", "】")) + for open_ch, close_ch in _PAIRS: + if open_ch in line and close_ch not in line: + return True + return False + + +def _looks_like_mid_word(left: str, right: str) -> bool: + """Heuristic: detect if a tool-call wrap split a token mid-character. + + Returns True when both sides appear to be fragments of one continuous + filesystem path or identifier. This is deliberately conservative -- + when in doubt, return False so a space gets inserted. + """ + if not left or not right: + return False + lc = left[-1] + rc = right[0] + # Mid-path: one side has '/' at the boundary. + if lc == "/" or rc == "/": + return True + # Mid-word with hyphen/underscore continuation + # (e.g., "/skills/gener" + "ating-e2e-test-suite"). + # Must distinguish from command-argument boundaries + # (e.g., "git add" + "test-cases/...") by checking the character + # preceding the left side's last token. + if lc.isalpha() and rc.isalpha() and lc.islower() and rc.islower(): + first_token = "" + for ch in right: + if ch.isalnum() or ch in "-_.": + first_token += ch + else: + break + if "-" in first_token or "_" in first_token: + # Find start of last token on the left side + last_token_start = len(left) + while last_token_start > 0 and ( + left[last_token_start - 1].isalnum() + or left[last_token_start - 1] in "-_." + ): + last_token_start -= 1 + # If preceded by '/', it might be mid-path -- but check if the + # token is a complete filename (has file extension like .md/.py). + if last_token_start > 0 and left[last_token_start - 1] == "/": + last_token = left[last_token_start:] + if re.search(r"\.\w{1,5}$", last_token): + return False # Complete filename, not a fragment + return True # Mid-path fragment (e.g., /skills/gener) + # If preceded by space or start-of-string, it's a word + # boundary (e.g., "git add" + "test-cases") -> insert space + return False + return False + + +# --------------------------------------------------------------------------- +# Table detection helpers +# --------------------------------------------------------------------------- + +def is_table_border_split(current: str, next_line: str) -> bool: + """Detect a table border that was split across two lines.""" + cs = current.rstrip() + if not cs or cs[-1] != "─": + return False + ns = next_line.lstrip() + if not ns: + return False + return ns[0] == "─" and ns.rstrip()[-1] in _TABLE_CORNERS + + +def _is_table_border_line(line: str) -> bool: + """Check if line is a table border (├─, └─, ┌─ patterns).""" + stripped = line.lstrip() + if not stripped: + return False + return stripped[0] in "├└┌" and "─" in stripped + + +def _count_expected_pipes(border_line: str) -> int: + """Count expected │ per data row from a ┌ or ├ border line. + + A border like ``┌──┬──┬──┐`` has 2 ``┬`` → 3 columns → 4 ``│`` per row. + """ + return border_line.count("┬") + 2 + + +def _parse_column_widths(border_line: str) -> list[int]: + """Extract column display widths from a ┌ or ├ border line. + + Splits by column separators (┬ or ┼) and counts ``─`` chars per segment. + Returns a list of column widths (inner content width, not including │). + """ + stripped = border_line.strip() + if len(stripped) < 2: + return [] + inner = stripped[1:-1] # Remove corner chars (┌/┐ or ├/┤) + segments = re.split("[┬┼]", inner) + return [len(seg) for seg in segments] + + +def _repad_table_row(row: str, col_widths: list[int]) -> str: + """Re-pad each cell in a table data row to match the column widths. + + Preserves existing left padding and content; only adds right-padding + so each cell reaches the correct display width from the border. + """ + parts = row.split("│") + # parts[0] = indent before first │; parts[-1] = after last │ (empty) + if len(parts) < 3: + return row + indent = parts[0] + cells = parts[1:-1] + if len(cells) != len(col_widths): + return row # Column count mismatch — leave unchanged + new_cells = [] + for cell, width in zip(cells, col_widths): + cell_dw = display_width(cell) + if cell_dw < width: + new_cells.append(cell + " " * (width - cell_dw)) + else: + new_cells.append(cell) + return indent + "│" + "│".join(new_cells) + "│" + + +# --------------------------------------------------------------------------- +# Statistics +# --------------------------------------------------------------------------- + +@dataclass +class Stats: + input_lines: int = 0 + output_lines: int = 0 + user_lines_joined: int = 0 + claude_lines_joined: int = 0 + table_borders_fixed: int = 0 + table_cells_fixed: int = 0 + tool_calls_fixed: int = 0 + tool_results_fixed: int = 0 + agent_tree_fixed: int = 0 + bullet_text_joined: int = 0 + plan_text_joined: int = 0 + table_multirow_merged: int = 0 + table_borders_realigned: int = 0 + box_rows_merged: int = 0 + + def summary(self) -> str: + lines = [ + "--- Statistics ---", + f" Input lines: {self.input_lines}", + f" Output lines: {self.output_lines}", + f" User lines joined: {self.user_lines_joined}", + f" Claude lines joined: {self.claude_lines_joined}", + f" Table borders fixed: {self.table_borders_fixed}", + f" Table cells fixed: {self.table_cells_fixed}", + f" Table rows merged: {self.table_multirow_merged}", + f" Borders realigned: {self.table_borders_realigned}", + f" Box rows merged: {self.box_rows_merged}", + f" Tool calls fixed: {self.tool_calls_fixed}", + f" Tool results fixed: {self.tool_results_fixed}", + f" Agent tree fixed: {self.agent_tree_fixed}", + f" Bullet text joined: {self.bullet_text_joined}", + f" Plan text joined: {self.plan_text_joined}", + ] + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Main processing logic +# --------------------------------------------------------------------------- + + +def process(lines: list[str], stats: Stats) -> list[str]: + """Process all *lines* and return the list of fixed output lines.""" + stats.input_lines = len(lines) + output: list[str] = [] + i = 0 + n = len(lines) + + def peek(offset: int = 1) -> str | None: + idx = i + offset + return lines[idx] if idx < n else None + + while i < n: + line = lines[i] + + # --------------------------------------------------------------- + # 1) User prompt blocks (❯ at column 0, continuations at dw=76) + # --------------------------------------------------------------- + if _USER_PROMPT_RE.match(line): + i = _process_user_block(lines, i, n, output, stats) + continue + + # --------------------------------------------------------------- + # 2) Table border split → join, then enter table region + # --------------------------------------------------------------- + next_line = peek() + if next_line is not None and is_table_border_split(line, next_line): + acc = raw_join(line, next_line) + stats.table_borders_fixed += 1 + i += 2 + while i < n and is_table_border_split(acc, lines[i]): + acc = raw_join(acc, lines[i]) + stats.table_borders_fixed += 1 + i += 1 + output.append(acc) + # If this was a ┌ border, process the full table body + if "┌" in acc: + expected_pipes = _count_expected_pipes(acc) + col_widths = _parse_column_widths(acc) + i = _process_table_body( + lines, i, n, output, stats, expected_pipes, col_widths, + ) + continue + + # --------------------------------------------------------------- + # 2b) Non-split ┌ border → enter table body processor + # --------------------------------------------------------------- + stripped_for_border = line.lstrip() + if ( + stripped_for_border.startswith("┌") + and "─" in stripped_for_border + and "┐" in stripped_for_border + ): + expected_pipes = _count_expected_pipes(line) + col_widths = _parse_column_widths(line) + output.append(line) + i += 1 + i = _process_table_body( + lines, i, n, output, stats, expected_pipes, col_widths, + ) + continue + + # --------------------------------------------------------------- + # 3) Table cell row (│ in line, outside tracked table region) + # Fallback for tables whose ┌ border was not split and was + # already emitted before we entered this logic. + # --------------------------------------------------------------- + if "│" in line: + stripped = line.lstrip() + if stripped.startswith("│") or stripped.endswith("│"): + # Likely a table row. Check if PREVIOUS output line was a + # ┌ or ├ border to determine expected_pipes. + expected_pipes = 0 + fallback_col_widths: list[int] = [] + for prev in reversed(output): + ps = prev.strip() + if ps and (ps[0] in "┌├"): + expected_pipes = _count_expected_pipes(prev) + fallback_col_widths = _parse_column_widths(prev) + break + if ps and ps[0] not in "│": + break + if expected_pipes > 0: + acc = line.rstrip() + pipe_count = acc.count("│") + i += 1 + while pipe_count < expected_pipes and i < n: + nl = lines[i] + if _is_truly_empty(nl): + break + if _is_table_border_line(nl): + break + # Check for border split on this line + if i + 1 < n and is_table_border_split(nl, lines[i + 1]): + break + acc = table_cell_join(acc, nl) + pipe_count = acc.count("│") + stats.table_cells_fixed += 1 + i += 1 + if fallback_col_widths: + acc = _repad_table_row(acc, fallback_col_widths) + output.append(acc) + continue + + # --------------------------------------------------------------- + # 4) Tool call continuation (● Bash(, ● Read(, etc.) + # Continuations are 6-space indented. + # --------------------------------------------------------------- + if _TOOL_CALL_RE.match(line): + acc = line + i += 1 + while i < n: + nl = lines[i] + # 6-space continuation (tool call argument wrapping) + if nl.startswith(" ") and not _is_structural_break(nl): + # Strip the 6-char continuation indent, preserving + # any additional whitespace from the original. + right_part = nl[6:] + left_s = acc.rstrip() + if right_part and right_part[0] == " ": + # Extra space means the original had whitespace + # at this position -- preserve it. + acc = left_s + right_part + elif left_s and _looks_like_mid_word(left_s, right_part): + # Mid-word/mid-path split: concatenate directly. + acc = left_s + right_part + else: + # Argument boundary where the space was consumed + # by wrapping. Restore it. + acc = left_s + " " + right_part + stats.tool_calls_fixed += 1 + i += 1 + else: + break + output.append(acc) + continue + + # --------------------------------------------------------------- + # 5) Tool result continuation ( ⎿ ... at dw>=74) + # --------------------------------------------------------------- + if _TOOL_RESULT_RE.match(line): + last_raw_dw = display_width(line.rstrip()) + acc = line + i += 1 + # Phase 1: join 5-space continuations of the ⎿ line itself. + # Only join when the PREVIOUS raw line was near the wrap limit + # (dw >= 74). Lines well below the limit ended naturally — + # subsequent 5-space lines are separate output lines (e.g. + # git log entries), not wrapped continuations. + # Character-class check: alpha→lower = mid-word (raw join); + # all other transitions = word boundary (smart_join). + while last_raw_dw >= 74 and i < n: + nl = lines[i] + if nl.startswith(" ") and not _is_structural_break(nl) and not _is_plan_structural(nl): + prev_dw = last_raw_dw + last_raw_dw = display_width(nl.rstrip()) + acc = _dw_aware_join(acc, nl, prev_dw) + stats.tool_results_fixed += 1 + i += 1 + else: + break + # Phase 1 fallback: if ⎿ line ends with trailing space + # (word-boundary wrap just below the 74 threshold), join + # short continuation fragments. This catches lines like + # "Plan saved to: ... · /plan to " + "edit" (dw=72). + while i < n and acc != acc.rstrip(): + nl = lines[i] + if nl.startswith(" ") and not nl.startswith(" "): + if _is_plan_continuation_fragment(nl, acc): + # Trailing space = word-boundary wrap → smart_join + # (not _bullet_join, which would raw-join alpha→lower + # like "to" + "edit" → "toedit"). + acc = smart_join(acc, nl) + stats.tool_results_fixed += 1 + i += 1 + continue + break + output.append(acc) + + # Phase 2: handle remaining tool output lines at 6-space indent. + # After the ⎿ line, tool output continues at 6-space indent. + # Each such line may itself be wrapped, with 5-space continuations. + # We emit each output line individually, joining only its + # wrapped fragments. + while i < n: + nl = lines[i] + if _is_truly_empty(nl): + break + if _is_structural_break(nl): + break + # Tool output lines use 6-space indent (5-space lines that + # aren't continuations would be plan text, etc.) + if not nl.startswith(" ") or nl.startswith(" "): + break + acc2 = nl + last_raw_dw2 = display_width(nl.rstrip()) + i += 1 + # Join continuations of this output line. + # Phase 2a: high-dw continuations at 5-6 space indent. + while last_raw_dw2 >= 74 and i < n: + nl2 = lines[i] + if nl2.startswith(" ") and not _is_structural_break(nl2) and not _is_plan_structural(nl2): + prev_dw2 = last_raw_dw2 + last_raw_dw2 = display_width(nl2.rstrip()) + acc2 = _dw_aware_join(acc2, nl2, prev_dw2) + stats.tool_results_fixed += 1 + i += 1 + else: + break + # Phase 2b: deeper-indented continuations when the + # output line ends with a continuation signal: + # - comma (list continuation) + # - trailing space (word-boundary wrap) + # - underscore (identifier split, e.g. E2E_PO_BASE_ + URL) + while i < n: + _acc2_s = acc2.rstrip() + if not _acc2_s: + break + _last2 = _acc2_s[-1] + _has_trailing = acc2 != _acc2_s + if _last2 not in (",", "_") and not _has_trailing: + break # no continuation signal + nl2 = lines[i] + # Only accept DEEPER-indented continuations (7+ spaces). + # Same-indent lines (6 spaces) are sibling entries in the + # tool output — e.g. separate diff lines that happen to + # have trailing padding spaces. + if not nl2.startswith(" ") or _is_structural_break(nl2): + break + nl2_dw = display_width(nl2.rstrip()) + if nl2_dw >= 76: + break # full-width = independent line + acc2 = smart_join(acc2, nl2) + last_raw_dw2 = nl2_dw + stats.tool_results_fixed += 1 + i += 1 + output.append(acc2) + continue + + # --------------------------------------------------------------- + # 6) Agent tree continuation (├─ or └─ at 3-space indent) + # --------------------------------------------------------------- + if _AGENT_TREE_RE.match(line): + dw = display_width(line.rstrip()) + acc = line + i += 1 + if dw >= 70: + while i < n: + nl = lines[i] + if _is_structural_break(nl): + break + nl_stripped = nl.lstrip() + # Never join lines that are sub-results (contain ⎿) + # or new tree nodes (├─, └─, │) + if "⎿" in nl_stripped: + break + if nl_stripped.startswith("├─") or nl_stripped.startswith("└─"): + break + if nl_stripped.startswith("│"): + break + # Same-indent non-structural continuation + if nl.startswith(" "): + acc = raw_join(acc, nl) + stats.agent_tree_fixed += 1 + i += 1 + else: + break + output.append(acc) + continue + + # --------------------------------------------------------------- + # 7) Claude narrative (● text with dw>=55+, NOT tool call/short marker) + # Threshold lowered from 77 to 55 because CJK-heavy lines end + # content well before the 77-80 column wrap limit (CJK chars take + # 2 columns each, so word boundaries fall earlier). + # --------------------------------------------------------------- + if _CLAUDE_ACTION_RE.match(line) and not _TOOL_CALL_RE.match(line): + dw = display_width(line.rstrip()) + acc = line + i += 1 + if dw >= 55 or _has_continuation_signal(line): + while i < n: + nl = lines[i] + if _is_structural_break(nl): + break + nl_dw = display_width(nl.rstrip()) + # 2-space continuation, short, not structural + if nl.startswith(" ") and not nl.startswith(" ⎿") and nl_dw < 82: + nl_stripped = nl.lstrip() + if nl_stripped.startswith("- "): + break + if re.match(r"\d+\. ", nl_stripped): + break + acc = smart_join(acc, nl) + stats.claude_lines_joined += 1 + i += 1 + else: + break + output.append(acc) + continue + + # --------------------------------------------------------------- + # 7b) Bullet item in Claude response ( - text) + # When a bullet line has high dw or ends with a continuation + # signal (CJK char, comma, etc.), its wrapped continuation + # on the next line (2-space indent, NOT a bullet/numbered) + # must be joined. Uses _bullet_join to handle mid-word + # breaks (e.g. "RiskModelAss" + "ignment") correctly. + # --------------------------------------------------------------- + if _BULLET_RE.match(line): + dw = display_width(line.rstrip()) + acc = line + i += 1 + if dw >= 55 or _has_continuation_signal(line) or _has_unclosed_bracket(line): + while i < n: + nl = lines[i] + if not _is_continuation_fragment(nl, acc): + break + acc = _bullet_join(acc, nl) + stats.bullet_text_joined += 1 + i += 1 + # Peek-ahead for plan-context bullets (5+ space indent). + # Short plan bullets (dw < 55) whose text wraps to the base + # 5-space indent won't enter the join loop above. Check if + # the next line is a non-structural 5-space continuation. + elif line.startswith(" ") and i < n: + nl = lines[i] + if ( + nl.startswith(" ") + and not _is_structural_break(nl) + and not _is_plan_structural(nl) + ): + nl_dw = display_width(nl.rstrip()) + if nl_dw < 70: + acc = smart_join(acc, nl) + stats.bullet_text_joined += 1 + i += 1 + output.append(acc) + continue + + # --------------------------------------------------------------- + # 8) Numbered list item in Claude response ( N. text) + # --------------------------------------------------------------- + if _NUMBERED_RE.match(line): + dw = display_width(line.rstrip()) + acc = line + i += 1 + if dw >= 55 or _has_continuation_signal(line): + while i < n: + nl = lines[i] + if _is_structural_break(nl): + break + # Numbered list continuation is at 2-space indent + if nl.startswith(" ") and not nl.startswith(" ⎿"): + nl_stripped = nl.lstrip() + if nl_stripped.startswith("- "): + break + if re.match(r"\d+\. ", nl_stripped): + break + dw_nl = display_width(nl.rstrip()) + if dw_nl < 82: + acc = _bullet_join(acc, nl) + stats.claude_lines_joined += 1 + i += 1 + else: + break + else: + break + # Peek-ahead for plan-context numbered items (5+ space indent). + elif line.startswith(" ") and i < n: + nl = lines[i] + if ( + nl.startswith(" ") + and not _is_structural_break(nl) + and not _is_plan_structural(nl) + ): + nl_dw = display_width(nl.rstrip()) + if nl_dw < 70: + acc = _bullet_join(acc, nl) + stats.claude_lines_joined += 1 + i += 1 + output.append(acc) + continue + + # --------------------------------------------------------------- + # 8c) Claude paragraph text (2-space indent, standalone) + # Text paragraphs within Claude response blocks that aren't + # preceded by a ● marker — they appear after tables, bullet + # lists, or between structural elements. + # + # DESIGN: Instead of measuring the current line's dw to guess + # whether it was wrapped (fragile threshold), we examine the + # NEXT line via _is_continuation_fragment() to decide whether + # it is a new item or a continuation of the current paragraph. + # --------------------------------------------------------------- + if line.startswith(" ") and not line.startswith(" "): + acc = line + i += 1 + # Skip column-layout lines (side-by-side comparison format). + _content = line.strip() + if " " not in _content: # 8+ internal spaces = layout + while i < n: + nl = lines[i] + if _is_continuation_fragment(nl, acc): + acc = _bullet_join(acc, nl) + stats.claude_lines_joined += 1 + i += 1 + else: + break + output.append(acc) + continue + + # --------------------------------------------------------------- + # 9) Plan / indented text (5+ space indent) + # + # DESIGN: Like step 8c, uses next-line look-ahead via + # _is_plan_continuation_fragment() instead of dw thresholds. + # The join function (_dw_aware_join) still uses the last + # segment's dw to decide mid-word vs word-boundary joins. + # --------------------------------------------------------------- + if line.startswith(" "): + acc = line + last_seg_dw = display_width(line.rstrip()) + i += 1 + while i < n: + nl = lines[i] + if _is_plan_continuation_fragment(nl, acc): + prev_dw = last_seg_dw + last_seg_dw = display_width(nl.rstrip()) + acc = _dw_aware_join(acc, nl, prev_dw) + stats.plan_text_joined += 1 + i += 1 + else: + break + output.append(acc) + continue + + # --------------------------------------------------------------- + # 10) Default: emit as-is + # --------------------------------------------------------------- + output.append(line) + i += 1 + + # Post-processing: merge multi-row table cells. + # After the main loop, each physical data row is complete (correct pipe + # count, re-padded). But a single logical row may still span multiple + # physical rows when Claude wrapped cell content at column width. + # This pass collapses those into one row per logical cell. + output = _merge_multirow_table_cells(output, stats) + + # Post-processing: realign table borders. + # After merging, merged data rows may exceed original column widths. + # This pass recalculates border widths to match the widest data cells. + output = _realign_table_borders(output, stats) + + # Post-processing: merge wrapped text within single-column box items. + # Single-column boxes (exactly 2 │ per row) contain numbered/bulleted + # items that may wrap across multiple rows. This pass merges continuation + # lines back into their parent item. + output = _merge_singlecol_box_rows(output, stats) + + stats.output_lines = len(output) + return output + + +# --------------------------------------------------------------------------- +# Post-processing: multi-row table cell merge +# --------------------------------------------------------------------------- + +def _is_table_data_row(stripped: str) -> bool: + """Check if *stripped* (leading-whitespace-removed) is a multi-column table data row. + + A data row starts and ends with ``│`` and contains no ``─`` (which + would make it a border row). Requires at least 3 ``│`` (i.e. 2+ + columns) so that single-column boxes (which have only 2 ``│``) + are excluded — their rows are independent items, not wrapped cells. + """ + return ( + len(stripped) >= 5 + and stripped[0] == "│" + and stripped[-1] == "│" + and "─" not in stripped + and stripped.count("│") >= 3 + ) + + +def _merge_multirow_table_cells(lines: list[str], stats: Stats) -> list[str]: + """Collapse consecutive table data rows into single logical rows. + + Between border/separator rows (``┌├└``), Claude may emit multiple + physical data rows for one logical row when cell content exceeds the + column width. This function detects such groups and merges them, + joining cell content with ``_table_cell_content_join`` which handles + mid-word and CJK boundaries. + """ + result: list[str] = [] + i = 0 + n = len(lines) + + while i < n: + line = lines[i] + stripped = line.lstrip() + + if not _is_table_data_row(stripped): + result.append(line) + i += 1 + continue + + # Collect consecutive data rows (same table region). + group = [line] + j = i + 1 + while j < n: + next_stripped = lines[j].lstrip() + if _is_table_data_row(next_stripped): + group.append(lines[j]) + j += 1 + else: + break + + if len(group) == 1: + result.append(line) + i = j + continue + + # Multiple physical rows → merge into one logical row. + merged = _merge_row_group(group, stats) + result.append(merged) + i = j + + return result + + +def _merge_row_group(rows: list[str], stats: Stats) -> str: + """Merge a group of physical table data rows into one logical row.""" + first = rows[0] + indent = first[: len(first) - len(first.lstrip())] + + # Split each row into cell contents — keep both raw and stripped forms. + # The raw form preserves trailing spaces, which we use to detect whether + # the content filled the entire column (≤1 trailing space → mid-word split). + all_cells: list[list[str]] = [] + all_raw: list[list[str]] = [] + for row in rows: + parts = row.strip().split("│") + # parts[0] is empty (before first │), parts[-1] is empty (after last │) + raw_cells = parts[1:-1] + cells = [c.strip() for c in raw_cells] + all_cells.append(cells) + all_raw.append(raw_cells) + + num_cols = max(len(cells) for cells in all_cells) + + # Merge each column's fragments. + merged_cells: list[str] = [] + for col_idx in range(num_cols): + fragments: list[str] = [] + raw_fragments: list[str] = [] + for row_idx, row_cells in enumerate(all_cells): + if col_idx < len(row_cells) and row_cells[col_idx]: + fragments.append(row_cells[col_idx]) + raw_fragments.append(all_raw[row_idx][col_idx]) + + if not fragments: + merged_cells.append("") + elif len(fragments) == 1: + merged_cells.append(fragments[0]) + else: + acc = fragments[0] + for k in range(1, len(fragments)): + # Determine if the previous fragment filled its column. + prev_raw = raw_fragments[k - 1] + trailing_spaces = len(prev_raw) - len(prev_raw.rstrip()) + left_filled = trailing_spaces <= 1 + acc = _table_cell_content_join( + acc, fragments[k], left_filled=left_filled + ) + merged_cells.append(acc) + + # Reconstruct the row with 1-space padding per cell. + cell_parts = [f" {cell} " if cell else " " for cell in merged_cells] + merged = indent + "│" + "│".join(cell_parts) + "│" + + stats.table_multirow_merged += len(rows) - 1 + return merged + + +# --------------------------------------------------------------------------- +# Post-processing: merge wrapped text within single-column box items +# --------------------------------------------------------------------------- + +def _is_singlecol_data_row(line: str) -> bool: + """Check if *line* is a single-column box data row. + + A single-column data row starts and ends with ``│``, has exactly 2 + ``│`` characters, and contains no ``─`` (which would indicate a border). + """ + stripped = line.lstrip() + return ( + len(stripped) >= 3 + and stripped[0] == "│" + and stripped[-1] == "│" + and "─" not in stripped + and stripped.count("│") == 2 + ) + + +def _is_singlecol_border(line: str) -> bool: + """Check if *line* is a single-column box border (┌─┐, ├─┤, or └─┘).""" + stripped = line.lstrip() + if not stripped: + return False + return ( + stripped[0] in "┌├└" + and "─" in stripped + and stripped[-1] in "┐┤┘" + and "┬" not in stripped + and "┼" not in stripped + and "┴" not in stripped + ) + + +_ITEM_START_RE = re.compile(r"^\s*\d+\.\s") +_BULLET_START_RE = re.compile(r"^\s*[-*]\s") + + +def _merge_singlecol_box_rows(lines: list[str], stats: Stats) -> list[str]: + """Merge wrapped text within single-column box items. + + Single-column boxes (2 ``│`` per row) contain numbered/bulleted items + that may wrap across multiple rows. This function merges continuation + lines back into their parent item while keeping separate items on + separate rows. + + Title boxes (a single data row between borders) are left untouched. + """ + result: list[str] = [] + i = 0 + n = len(lines) + + while i < n: + line = lines[i] + + # Look for the start of a single-column box (┌ border with no ┬). + if not _is_singlecol_border(line) or not line.lstrip().startswith("┌"): + result.append(line) + i += 1 + continue + + # Found a ┌ border. Collect the entire box (border + data + └). + box_lines: list[str] = [line] + j = i + 1 + found_close = False + while j < n: + if _is_singlecol_data_row(lines[j]): + box_lines.append(lines[j]) + j += 1 + elif _is_singlecol_border(lines[j]): + box_lines.append(lines[j]) + if lines[j].lstrip().startswith("└"): + found_close = True + j += 1 + break + elif lines[j].lstrip().startswith("├"): + j += 1 + else: + j += 1 + break + else: + break # Non-box line — box ended unexpectedly + + if not found_close: + # Incomplete box — emit as-is. + for bl in box_lines: + result.append(bl) + i = j + continue + + # Extract data rows (skip borders). + data_indices = [ + idx for idx, bl in enumerate(box_lines) if _is_singlecol_data_row(bl) + ] + + # Title boxes: single data row between borders → skip merging. + if len(data_indices) <= 1: + for bl in box_lines: + result.append(bl) + i = j + continue + + # Group data rows into logical items and merge continuations. + data_rows = [box_lines[idx] for idx in data_indices] + merged_contents = _merge_box_items(data_rows, stats) + + # Determine whether box borders need to grow. + indent = line[: len(line) - len(line.lstrip())] + + # Get current box inner width from the ┌ border. + top_border = line.lstrip() + border_inner_dw = display_width(top_border) - 2 + + # Compute max content width after merge. + max_content_dw = 0 + for content in merged_contents: + # Content needs 2 spaces padding (1 left + 1 right minimum). + content_dw = display_width(content) + 2 + max_content_dw = max(max_content_dw, content_dw) + + new_inner_width = max(border_inner_dw, max_content_dw) + + # Rebuild the box: borders + merged data rows. + # Emit ┌ border (potentially wider). + result.append( + _rebuild_singlecol_border(top_border, new_inner_width, indent) + ) + + # Emit merged data rows. + for content in merged_contents: + padded = _pad_singlecol_content( + " " + content + " ", new_inner_width, + ) + result.append(indent + "│" + padded + "│") + + # Emit any ├ and └ borders (potentially wider). + for idx, bl in enumerate(box_lines): + if idx == 0: + continue # Already emitted ┌ + stripped_bl = bl.lstrip() + if stripped_bl and stripped_bl[0] in "├└" and "─" in stripped_bl: + result.append( + _rebuild_singlecol_border(stripped_bl, new_inner_width, indent) + ) + + i = j + + return result + + +def _merge_box_items( + data_rows: list[str], stats: Stats, +) -> list[str]: + """Merge continuation rows within each logical item of a single-column box. + + Returns a list of merged content strings (one per logical item). + """ + # Parse content from each row (strip │ and whitespace). + contents: list[str] = [] + for row in data_rows: + stripped = row.lstrip() + inner = stripped[1:-1] # Remove │ ... │ + contents.append(inner.strip()) + + # Group into logical items. + items: list[list[str]] = [] + for content in contents: + if _ITEM_START_RE.match(content) or _BULLET_START_RE.match(content): + items.append([content]) + elif not items: + items.append([content]) + else: + items[-1].append(content) + + # Merge fragments within each item. + merged: list[str] = [] + for fragments in items: + if len(fragments) == 1: + merged.append(fragments[0]) + else: + acc = fragments[0] + for frag in fragments[1:]: + acc = _table_cell_content_join(acc, frag) + stats.box_rows_merged += 1 + merged.append(acc) + + return merged + + +def _pad_singlecol_content(content: str, inner_width: int) -> str: + """Pad content to fill *inner_width* display columns inside a box.""" + content_dw = display_width(content) + if content_dw >= inner_width: + return content + return content + " " * (inner_width - content_dw) + + +def _rebuild_singlecol_border( + stripped: str, inner_width: int, indent: str, +) -> str: + """Rebuild a single-column box border to the given inner width. + + Preserves styled headers like ``┌─── Title ───┐`` by detecting + embedded text and re-centering it. + """ + left_corner = stripped[0] + right_corner = stripped[-1] + + # Check for embedded title text (e.g., ┌─── Pre-Suite ───┐). + inner = stripped[1:-1] + title_match = re.search(r"([^─]+)", inner) + if title_match: + title_text = title_match.group(1).strip() + if title_text: + # Styled header border: ─── Title ─── + title_with_spaces = f" {title_text} " + title_dw = display_width(title_with_spaces) + remaining = inner_width - title_dw + left_dashes = max(remaining // 2, 3) + right_dashes = max(remaining - left_dashes, 3) + return ( + indent + + left_corner + + "─" * left_dashes + + title_with_spaces + + "─" * right_dashes + + right_corner + ) + + # Plain border (all ─). + return indent + left_corner + "─" * inner_width + right_corner + + +# --------------------------------------------------------------------------- +# Post-processing: realign table borders after multi-row merge +# --------------------------------------------------------------------------- + +def _realign_table_borders(lines: list[str], stats: Stats) -> list[str]: + """Recalculate border widths to match (potentially wider) merged data rows. + + After ``_merge_multirow_table_cells`` collapses multi-row cells, the + merged content may exceed the original column widths encoded in the + border rows. This pass: + + 1. Identifies contiguous table *regions* (runs of border + data rows). + 2. For each region, computes the maximum display-width per column + across all data rows. + 3. Regenerates every border row with the correct ``─`` widths. + 4. Re-pads every data row so cells align with the new borders. + """ + result: list[str] = [] + i = 0 + n = len(lines) + + while i < n: + line = lines[i] + stripped = line.lstrip() + + # Detect start of a table region (┌ border). + if stripped.startswith("┌") and "─" in stripped and "┐" in stripped: + # Collect every line in this table region. + region_start = i + region: list[str] = [line] + i += 1 + while i < n: + s = lines[i].lstrip() + if _is_table_data_row(s): + region.append(lines[i]) + i += 1 + elif s and s[0] in "├└" and "─" in s: + region.append(lines[i]) + i += 1 + if s[0] == "└": + break # End of table + else: + break # Non-table line — region ended unexpectedly + + realigned = _realign_table_region(region, stats) + result.extend(realigned) + continue + + result.append(line) + i += 1 + + return result + + +def _realign_table_region(region: list[str], stats: Stats) -> list[str]: + """Realign borders and data rows within a single table region.""" + # Separate border and data rows; determine indent from first line. + first = region[0] + indent = first[: len(first) - len(first.lstrip())] + + # Collect data rows and their per-cell display widths. + data_rows: list[tuple[int, list[str]]] = [] # (index, cells) + border_indices: list[int] = [] + + for idx, row in enumerate(region): + stripped = row.lstrip() + if _is_table_data_row(stripped): + parts = stripped.split("│") + # parts[0] = '' (before first │), parts[-1] = '' (after last │) + cells = parts[1:-1] + data_rows.append((idx, cells)) + elif stripped and stripped[0] in "┌├└": + border_indices.append(idx) + + if not data_rows: + return region # No data rows — nothing to realign + + # Determine column count from data rows. + num_cols = max(len(cells) for _, cells in data_rows) + if num_cols == 0: + return region + + # Compute max display-width per column across all data rows. + # Each cell has 1-space padding on each side, so content width is what + # we see between the padding. But we measure the full cell (including + # padding) to get the column width that the border must span. + max_widths: list[int] = [0] * num_cols + for _, cells in data_rows: + for col_idx, cell in enumerate(cells): + if col_idx < num_cols: + cw = display_width(cell) + if cw > max_widths[col_idx]: + max_widths[col_idx] = cw + + # Ensure minimum width of 3 (1 space + 1 char + 1 space). + max_widths = [max(w, 3) for w in max_widths] + + # Check if any realignment is needed by comparing with current border. + current_widths = _parse_column_widths(region[border_indices[0]]) + if current_widths == max_widths: + return region # Already aligned + + # Rebuild the region. + rebuilt: list[str] = [] + for idx, row in enumerate(region): + stripped = row.lstrip() + if idx in border_indices: + rebuilt.append(_rebuild_border(stripped[0], stripped[-1], max_widths, indent)) + stats.table_borders_realigned += 1 + elif _is_table_data_row(stripped): + rebuilt.append(_repad_table_row_to_widths(row, max_widths, indent)) + else: + rebuilt.append(row) + + return rebuilt + + +def _rebuild_border( + left_corner: str, right_corner: str, col_widths: list[int], indent: str, +) -> str: + """Build a border row from corner chars and column widths. + + Maps corner pairs: + ┌ ┐ → separator ┬ + ├ ┤ → separator ┼ + └ ┘ → separator ┴ + """ + sep_map = {"┌": "┬", "├": "┼", "└": "┴"} + separator = sep_map.get(left_corner, "┼") + segments = ["─" * w for w in col_widths] + return indent + left_corner + separator.join(segments) + right_corner + + +def _repad_table_row_to_widths( + row: str, col_widths: list[int], indent: str, +) -> str: + """Re-pad a data row so each cell matches the given column widths.""" + stripped = row.lstrip() + parts = stripped.split("│") + if len(parts) < 3: + return row + cells = parts[1:-1] + if len(cells) != len(col_widths): + return row # Column count mismatch — leave unchanged + + new_cells: list[str] = [] + for cell, width in zip(cells, col_widths): + # Strip existing padding, then re-pad. + content = cell.strip() + content_dw = display_width(content) + # Target: 1 space left + content + right padding to fill width. + # Total cell display-width must equal `width`. + # Cell = " " + content + " " * (width - 1 - content_dw) + # But if content_dw + 2 > width, just use " content " (overflow). + if content: + right_pad = max(width - 1 - content_dw, 1) + new_cells.append(" " + content + " " * right_pad) + else: + new_cells.append(" " * width) + + return indent + "│" + "│".join(new_cells) + "│" + + +def _process_table_body( + lines: list[str], + start: int, + n: int, + output: list[str], + stats: Stats, + expected_pipes: int, + col_widths: list[int] | None = None, +) -> int: + """Process lines inside a table body (after ┌ border, until └ border). + + Uses pipe-count accumulation: each data row is accumulated until the + ``│`` count reaches *expected_pipes*, then emitted. Border lines + (├─, └─) are emitted directly (with split joining if needed). + + When *col_widths* is provided, each completed data row is re-padded + so every cell matches the column width from the border. + + Returns the next line index to process after the table ends. + """ + i = start + while i < n: + line = lines[i] + + # --- Empty line: table ended unexpectedly --- + if _is_truly_empty(line): + break + + # --- Border line (├ or └): join splits, emit, maybe exit --- + stripped = line.lstrip() + if stripped and stripped[0] in "├└" and "─" in stripped: + acc = line + i += 1 + # Join border split if needed + if acc.rstrip()[-1] == "─" and i < n: + nl = lines[i] + ns = nl.lstrip() + if ns and ns[0] == "─" and ns.rstrip()[-1] in _TABLE_CORNERS: + acc = raw_join(acc, nl) + stats.table_borders_fixed += 1 + i += 1 + while i < n and is_table_border_split(acc, lines[i]): + acc = raw_join(acc, lines[i]) + stats.table_borders_fixed += 1 + i += 1 + output.append(acc) + # └ border: table ends + if "└" in acc: + return i + continue + + # --- Data row: accumulate until pipe count matches --- + if "│" in line or (stripped and stripped[-1] == "│"): + acc = line.rstrip() + pipe_count = acc.count("│") + i += 1 + while pipe_count < expected_pipes and i < n: + nl = lines[i] + if _is_truly_empty(nl): + break + # Stop at border lines + nl_s = nl.lstrip() + if nl_s and nl_s[0] in "├└┌" and "─" in nl_s: + break + acc = table_cell_join(acc, nl) + pipe_count = acc.count("│") + stats.table_cells_fixed += 1 + i += 1 + # Re-pad cells to match column widths from border + if col_widths: + acc = _repad_table_row(acc, col_widths) + output.append(acc) + continue + + # --- Non-table line (shouldn't happen but be safe) --- + output.append(line) + i += 1 + + return i + + +def _process_user_block( + lines: list[str], + start: int, + n: int, + output: list[str], + stats: Stats, +) -> int: + """Process a user prompt block starting at *start*. + + User blocks begin with ``❯ `` and continue with lines that have + display_width == 76 (right-padded with trailing spaces). All content + lines in the block share this fixed width. + + Within the block: + - Blank lines (76 spaces) are paragraph separators. + - Lines whose stripped content starts with ``- `` are bullet items. + - Lines whose stripped content starts with ``\\d+. `` are numbered items. + - ````` ``` ````` toggles code-fence mode (content preserved as-is). + - Everything else is a continuation of the current paragraph/item. + + Returns the index of the first line AFTER the block. + """ + acc = lines[start] + i = start + 1 + + # Check if the first line itself is at dw=76 (padded). + first_dw = display_width(lines[start]) + if first_dw != 76: + # Short user prompt, no wrapping. Emit as-is. + output.append(acc) + return i + + in_code_fence = False + + while i < n: + line = lines[i] + dw = display_width(line) + + # The user block boundary: ALL lines inside have dw==76. + if dw != 76: + break + + stripped = line.rstrip() + + # --- Code fence toggle --- + # Detect ``` anywhere in the stripped content. + if stripped.lstrip().startswith("```"): + # Flush current accumulator before the fence marker. + if acc: + output.append(acc) + acc = "" + output.append(line) + in_code_fence = not in_code_fence + i += 1 + continue + + # Inside code fences: emit lines as-is (no joining). + if in_code_fence: + output.append(line) + i += 1 + continue + + # --- Blank line (paragraph separator) --- + if not stripped: + if acc: + output.append(acc) + acc = "" + output.append("") + i += 1 + continue + + # --- Structural item detection --- + content_no_indent = stripped.lstrip() + + # Bullet: stripped content starts with "- " + is_bullet = content_no_indent.startswith("- ") + + # Numbered list: stripped content starts with digit+". " + is_numbered = bool(re.match(r"\d+\. ", content_no_indent)) + + if is_bullet or is_numbered: + # Start a new logical line for this item. + if acc: + output.append(acc) + acc = line + i += 1 + continue + + # --- Normal continuation --- + if acc: + # Extract the content portion (skip the 2-space continuation + # indent that the export prepends). + join_content = stripped[2:] if stripped.startswith(" ") else stripped + acc = smart_join(acc, join_content) + stats.user_lines_joined += 1 + else: + # After a paragraph break (acc was reset to ""), this is the + # first line of a new paragraph. + acc = line + i += 1 + + # Flush remaining accumulated text. + if acc: + output.append(acc) + + return i + + +# --------------------------------------------------------------------------- +# Marker-count verification +# --------------------------------------------------------------------------- + +def _count_markers(lines: list[str]) -> dict[str, int]: + """Count occurrences of structural markers.""" + counts: dict[str, int] = {"●": 0, "❯": 0, "✻": 0} + for line in lines: + for marker in counts: + if line.startswith(marker): + counts[marker] += 1 + return counts + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main() -> None: + parser = argparse.ArgumentParser( + description="Fix broken line wrapping in Claude Code export files.", + ) + parser.add_argument("input", type=Path, help="Input .txt file") + parser.add_argument( + "-o", + "--output", + type=Path, + default=None, + help="Output path (default: -fixed.txt)", + ) + parser.add_argument( + "--stats", action="store_true", help="Print statistics to stderr" + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Process without writing output file", + ) + args = parser.parse_args() + + input_path: Path = args.input.resolve() + if not input_path.is_file(): + print(f"ERROR: Input file not found: {input_path}", file=sys.stderr) + sys.exit(1) + + if args.output is not None: + output_path: Path = args.output.resolve() + else: + output_path = input_path.with_stem(input_path.stem + "-fixed") + + if output_path == input_path: + print( + "ERROR: Output path must differ from input path. " + "Use -o to specify a different output file.", + file=sys.stderr, + ) + sys.exit(1) + + # Read input (strict UTF-8, no fallback). + text = input_path.read_text(encoding="utf-8", errors="strict") + raw_lines = text.split("\n") + # Remove trailing empty element from final newline (if present). + if raw_lines and raw_lines[-1] == "": + raw_lines.pop() + + stats = Stats() + result = process(raw_lines, stats) + + # --------------------------------------------------------------- + # Safety: marker-count verification + # --------------------------------------------------------------- + input_markers = _count_markers(raw_lines) + output_markers = _count_markers(result) + for marker, in_count in input_markers.items(): + out_count = output_markers.get(marker, 0) + if in_count != out_count: + print( + f"WARNING: Marker '{marker}' count mismatch: " + f"input={in_count}, output={out_count}", + file=sys.stderr, + ) + + # Safety: runaway join detection + for idx, line in enumerate(result): + if display_width(line) > 500: + print( + f"WARNING: Line {idx + 1} has display width " + f"{display_width(line)} (>500) — possible runaway join", + file=sys.stderr, + ) + + # --------------------------------------------------------------- + # Output + # --------------------------------------------------------------- + if args.dry_run: + print( + f"Dry run complete. Would write {len(result)} lines to {output_path}", + file=sys.stderr, + ) + else: + output_path.write_text( + "\n".join(result) + "\n", encoding="utf-8", errors="strict" + ) + print(f"Written: {output_path}", file=sys.stderr) + + if args.stats: + print(stats.summary(), file=sys.stderr) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/claude-export-txt-better/scripts/validate-claude-export-fix.py b/claude-export-txt-better/scripts/validate-claude-export-fix.py new file mode 100644 index 00000000..553ee70c --- /dev/null +++ b/claude-export-txt-better/scripts/validate-claude-export-fix.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +"""Automated validation for fix-claude-export.py output. + +Runs a comprehensive suite of checks against a fixed file and its original, +reporting PASS/FAIL for each with evidence. Designed to be run after every +fix iteration as a quality gate. + +Usage: + uv run scripts/validate-claude-export-fix.py + uv run scripts/validate-claude-export-fix.py --verbose +""" + +from __future__ import annotations + +import argparse +import re +import sys +import unicodedata +from dataclasses import dataclass, field +from pathlib import Path + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def display_width(s: str) -> int: + return sum(2 if unicodedata.east_asian_width(ch) in ("W", "F") else 1 for ch in s) + + +def is_cjk_ideograph(ch: str) -> bool: + return unicodedata.east_asian_width(ch) in ("W", "F") and unicodedata.category(ch) == "Lo" + + +# --------------------------------------------------------------------------- +# Check infrastructure +# --------------------------------------------------------------------------- + +@dataclass +class CheckResult: + name: str + passed: bool + detail: str + category: str = "" + + +@dataclass +class ValidationReport: + results: list[CheckResult] = field(default_factory=list) + + def add(self, name: str, passed: bool, detail: str, category: str = ""): + self.results.append(CheckResult(name, passed, detail, category)) + + @property + def passed(self) -> int: + return sum(1 for r in self.results if r.passed) + + @property + def failed(self) -> int: + return sum(1 for r in self.results if not r.passed) + + def print_report(self, verbose: bool = False): + cats = {} + for r in self.results: + cats.setdefault(r.category or "General", []).append(r) + + for cat, checks in cats.items(): + print(f"\n{'=' * 60}") + print(f" {cat}") + print(f"{'=' * 60}") + for r in checks: + icon = "✓" if r.passed else "✗" + print(f" {icon} {r.name}") + if verbose or not r.passed: + for line in r.detail.split("\n"): + print(f" {line}") + + total = len(self.results) + print(f"\n{'=' * 60}") + print(f" TOTAL: {self.passed}/{total} passed, {self.failed} failed") + print(f"{'=' * 60}") + + +# --------------------------------------------------------------------------- +# Check implementations +# --------------------------------------------------------------------------- + +def check_marker_counts(orig: str, fixed: str, report: ValidationReport): + """Verify structural markers are preserved exactly.""" + markers = [ + ("❯", "User prompts"), + ("●", "Claude actions"), + ("✻", "Stars/crunched"), + ("⎿", "Tool results"), + ("…", "Expansion indicators"), + ] + for marker, name in markers: + orig_count = orig.count(marker) + fixed_count = fixed.count(marker) + report.add( + f"Marker {marker} ({name}): {orig_count}", + orig_count == fixed_count, + f"orig={orig_count} fixed={fixed_count}", + "Structural Integrity", + ) + + +def check_table_borders(fixed: str, report: ValidationReport): + """Verify table border corners are balanced.""" + for ch, name in [("┌", "top-left"), ("┐", "top-right"), ("┘", "bottom-right")]: + count = fixed.count(ch) + report.add( + f"Table corner {ch} ({name}): {count}", + True, # Just record the count + f"count={count}", + "Structural Integrity", + ) + tl = fixed.count("┌") + tr = fixed.count("┐") + br = fixed.count("┘") + report.add( + "Table corners balanced (┌ = ┐ = ┘)", + tl == tr == br, + f"┌={tl} ┐={tr} ┘={br}", + "Structural Integrity", + ) + + +def check_line_reduction(orig: str, fixed: str, report: ValidationReport): + """Output should have fewer lines than input (joins happened).""" + orig_lines = orig.count("\n") + fixed_lines = fixed.count("\n") + report.add( + f"Line reduction: {orig_lines} → {fixed_lines}", + fixed_lines < orig_lines, + f"delta={orig_lines - fixed_lines} ({(orig_lines - fixed_lines) / orig_lines * 100:.1f}% reduction)", + "Structural Integrity", + ) + + +def check_table_border_completeness(fixed: str, report: ValidationReport): + """Verify table border lines have matching left and right ends.""" + lines = fixed.split("\n") + broken = [] + for i, line in enumerate(lines): + stripped = line.strip() + if not stripped: + continue + # Lines starting with a left border char should have a right border char + if stripped[0] == "┌" and "┐" not in stripped: + broken.append((i + 1, "┌ without ┐", stripped[:80])) + elif stripped[0] == "├" and "┤" not in stripped: + broken.append((i + 1, "├ without ┤", stripped[:80])) + elif stripped[0] == "└" and "┘" not in stripped: + broken.append((i + 1, "└ without ┘", stripped[:80])) + + report.add( + f"Table borders complete: {len(broken)} broken", + len(broken) == 0, + "\n".join(f" L{ln}: {desc}" for ln, desc, _ in broken[:5]) + if broken else "all borders have matching ends", + "Structural Integrity", + ) + + +def check_phase_separation(fixed: str, report: ValidationReport): + """Verify Phase N: items are on separate lines.""" + lines = fixed.split("\n") + multi_phase_lines = [] + for i, line in enumerate(lines): + # Count "Phase N:" occurrences on this line + matches = re.findall(r"Phase \d+:", line) + if len(matches) >= 2: + # Allow pipeline diagrams with arrows (legitimate multi-phase) + if "→" in line: + continue + # Allow status updates like "Phase 3 进度: 3/5" + if line.strip().startswith("●"): + continue + multi_phase_lines.append((i + 1, matches, line[:80])) + + report.add( + "Phase items on separate lines", + len(multi_phase_lines) == 0, + f"{len(multi_phase_lines)} violations" + ( + "\n" + "\n".join(f" L{ln}: {m}" for ln, m, _ in multi_phase_lines[:5]) + if multi_phase_lines else "" + ), + "Over-Join Prevention", + ) + + +def check_runaway_joins(fixed: str, report: ValidationReport): + """Flag lines with very high display width that might be runaway joins.""" + lines = fixed.split("\n") + runaways = [] + for i, line in enumerate(lines): + dw = display_width(line) + if dw > 500: + # Check if it's a legitimate long line (user prompt) + if line.startswith("❯ "): + continue + runaways.append((i + 1, dw, line[:60])) + + report.add( + f"No runaway joins (dw > 500): {len(runaways)} found", + len(runaways) == 0, + "\n".join(f" L{ln}: dw={dw} [{preview}...]" for ln, dw, preview in runaways[:5]) + if runaways else "none", + "Over-Join Prevention", + ) + + +def check_en_cjk_no_space(fixed: str, report: ValidationReport): + """Count remaining EN-CJK adjacency without space (potential pangu misses).""" + # Only check at join boundaries (lines that were modified), not all text + pattern_alnum_cjk = re.compile(r"[a-zA-Z0-9][一-龥]") + pattern_cjk_alnum = re.compile(r"[一-龥][a-zA-Z0-9]") + + violations_ac = len(pattern_alnum_cjk.findall(fixed)) + violations_ca = len(pattern_cjk_alnum.findall(fixed)) + total = violations_ac + violations_ca + + # This is informational — some violations are in original content, code, etc. + report.add( + f"EN-CJK adjacency count: {total}", + True, # Informational + f"ASCII→CJK: {violations_ac}, CJK→ASCII: {violations_ca} (includes original content)", + "Pangu Spacing", + ) + + +def check_diff_lines_separate(fixed: str, report: ValidationReport): + """Verify diff output lines aren't merged (line numbers should be separate).""" + lines = fixed.split("\n") + merged_diffs = [] + for i, line in enumerate(lines): + # Look for two diff line numbers on the same line + matches = re.findall(r"\b(\d{3})\s{2,}[-+]?\s", line) + if len(matches) >= 2: + merged_diffs.append((i + 1, matches, line[:80])) + + report.add( + "Diff lines separate", + len(merged_diffs) == 0, + f"{len(merged_diffs)} violations" + ( + "\n" + "\n".join(f" L{ln}: numbers={m}" for ln, m, _ in merged_diffs[:5]) + if merged_diffs else "" + ), + "Over-Join Prevention", + ) + + +def check_cjk_label_separation(fixed: str, report: ValidationReport): + """Verify CJK label fields (模块:, 输出文件:, 状态:) are on separate lines.""" + lines = fixed.split("\n") + merged_labels = [] + # Pattern: field-label style "Label1: value1 Label2: value2" where + # each label starts at a position that looks like a separate field. + # Only flag when labels are at field boundaries (preceded by whitespace + # or start of line), not mid-sentence. + cjk_field_re = re.compile(r"(?:^|\s)([\u4e00-\u9fff]{1,4}[:：])\s") + for i, line in enumerate(lines): + stripped = line.strip() + matches = cjk_field_re.findall(stripped) + if len(matches) >= 3: # 3+ field labels = likely over-joined fields + merged_labels.append((i + 1, matches, stripped[:80])) + + report.add( + "CJK labels on separate lines", + len(merged_labels) == 0, + f"{len(merged_labels)} violations" + ( + "\n" + "\n".join(f" L{ln}: labels={m}" for ln, m, _ in merged_labels[:5]) + if merged_labels else "" + ), + "Over-Join Prevention", + ) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("original", type=Path, help="Original exported file") + parser.add_argument("fixed", type=Path, help="Fixed output file") + parser.add_argument("--verbose", "-v", action="store_true", help="Show all details") + args = parser.parse_args() + + orig = args.original.read_text(encoding="utf-8") + fixed = args.fixed.read_text(encoding="utf-8") + + report = ValidationReport() + + # Run all checks + check_marker_counts(orig, fixed, report) + check_table_borders(fixed, report) + check_table_border_completeness(fixed, report) + check_line_reduction(orig, fixed, report) + check_phase_separation(fixed, report) + check_runaway_joins(fixed, report) + check_diff_lines_separate(fixed, report) + check_cjk_label_separation(fixed, report) + check_en_cjk_no_space(fixed, report) + + report.print_report(verbose=args.verbose) + + sys.exit(0 if report.failed == 0 else 1) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/continue-claude-work/.security-scan-passed b/continue-claude-work/.security-scan-passed new file mode 100644 index 00000000..4e1d205b --- /dev/null +++ b/continue-claude-work/.security-scan-passed @@ -0,0 +1,4 @@ +Security scan passed +Scanned at: 2026-03-18T23:02:18.627209 +Tool: gitleaks + pattern-based validation +Content hash: 62e456422cabfe74e5757a802044dac45d1662341b2e90dc943685db81d8f659 diff --git a/continue-claude-work/SKILL.md b/continue-claude-work/SKILL.md new file mode 100644 index 00000000..0e3edbe5 --- /dev/null +++ b/continue-claude-work/SKILL.md @@ -0,0 +1,141 @@ +--- +name: continue-claude-work +description: Recover actionable context from local `.claude` session artifacts and continue interrupted work without running `claude --resume`. This skill should be used when the user provides a Claude session ID, asks to continue prior work from local history, or wants to inspect `.claude` files before resuming implementation. +argument-hint: [session-id] +--- + +# Continue Claude Work + +## Overview + +Recover actionable context from a prior Claude Code session and continue execution in the current conversation. Use local session files as the source of truth, then continue with concrete edits and checks — not just summarizing. + +**Why this exists instead of `claude --resume`**: `claude --resume` replays the full session transcript into the context window. For long sessions this wastes tokens on resolved issues and stale state. This skill **selectively reconstructs** only actionable context — the latest compact summary, pending work, known errors, and current workspace state — giving a fresh start with prior knowledge. + +## File Structure Reference + +For directory layout, JSONL schemas, and compaction block format, see `references/file_structure.md`. + +## Workflow + +### Step 1: Extract Context (single script call) + +Run the bundled extraction script. It handles session discovery, compact-boundary parsing, noise filtering, and workspace state in one call: + +```bash +# Latest session for current project +python3 scripts/extract_resume_context.py + +# Specific session by ID +python3 scripts/extract_resume_context.py --session + +# Search by topic +python3 scripts/extract_resume_context.py --query "auth feature" + +# List recent sessions +python3 scripts/extract_resume_context.py --list +``` + +The script outputs a structured Markdown **briefing** containing: +- **Session metadata** from `sessions-index.json` +- **Compact summary** — Claude's own distilled summary from the last compaction boundary (highest-signal context) +- **Last user requests** — the most recent explicit asks +- **Last assistant responses** — what was claimed done +- **Errors encountered** — tool failures and error outputs +- **Unresolved tool calls** — indicates interrupted session +- **Subagent workflow state** — which subagents completed, which were interrupted, their last outputs +- **Session end reason** — clean exit, interrupted (ctrl-c), error cascade, or abandoned +- **Files touched** — files created/edited/read during the session +- **MEMORY.md** — persistent cross-session notes +- **Git state** — current status, branch, recent log + +The script automatically skips the currently active session (modified < 60s ago) to avoid self-extraction. + +### Step 2: Branch by Session End Reason + +The briefing includes a **Session end reason**. Use it to choose the right continuation strategy: + +| End Reason | Strategy | +|-----------|----------| +| **Clean exit** | Session completed normally. Read the last user request that was addressed. Continue from pending work if any. | +| **Interrupted** | Tool calls were dispatched but never got results (likely ctrl-c or timeout). Retry the interrupted tool calls or assess whether they are still needed. | +| **Error cascade** | Multiple API errors caused the session to fail. Do not retry blindly — diagnose the root cause first. | +| **Abandoned** | User sent a message but got no response. Treat the last user message as the current request. | + +If the briefing has a **Subagent Workflow** section with interrupted subagents, check what each was doing and whether to retry or skip. + +### Step 3: Reconcile and Continue + +Before making changes: +1. Confirm the current directory matches the session's project. +2. If the git branch has changed from the session's branch, note this and decide whether to switch. +3. Inspect files related to pending work — verify old claims still hold. +4. Do not assume old claims are valid without checking. + +Then: +- Implement the next concrete step aligned with the latest user request. +- Run deterministic verification (tests, type-checks, build). +- If blocked, state the exact blocker and propose one next action. + +### Step 4: Report + +Respond concisely: +- **Context recovered**: which session, key findings from the briefing +- **Work executed**: files changed, commands run, test results +- **Remaining**: pending tasks, if any + +## How the Script Works + +### Compact-Boundary-Aware Extraction + +The script finds the **last** compact boundary in the session JSONL and extracts its summary. This is the single highest-signal piece of context in any long session -- Claude's own distilled understanding of the entire conversation up to that point. For details on compaction format and JSONL schemas, see `references/file_structure.md`. + +### Size-Adaptive Strategy + +| Session size | Strategy | +|-------------|----------| +| Has compactions | Read last compact summary + all post-compact messages | +| < 500 KB, no compactions | Read last 60% of messages | +| 500 KB - 5 MB | Read last 30% of messages | +| > 5 MB | Read last 15% of messages | + +### Subagent Context Extraction + +When a session has subagent directories (`/subagents/`), the script parses each subagent's JSONL to extract agent type, completion status, and last text output. This enables recovery of multi-agent workflows -- e.g., if a 32-subagent evaluation pipeline was interrupted, the briefing shows which agents completed and which need retry. + +### Session End Reason Detection + +The script classifies how the session ended: +- **completed** -- assistant had the last word (clean exit) +- **interrupted** -- unresolved tool calls (ctrl-c or timeout) +- **error_cascade** -- 3+ API errors +- **abandoned** -- user sent a message with no response + +### Noise Filtering + +These message types are skipped (37-53% of lines in real sessions): +- `progress`, `queue-operation`, `file-history-snapshot` -- operational noise +- `api_error`, `turn_duration`, `stop_hook_summary` -- system subtypes +- ``, `` -- filtered from user text extraction + +## Guardrails + +- Do not run `claude --resume` or `claude --continue` — this skill provides context recovery within the current session. +- Do not treat compact summaries as complete truth — they are lossy. Always verify claims against current workspace. +- Do not overwrite unrelated working-tree changes. +- Do not load the full session file into context — always use the script. + +## Limitations + +- Cannot recover sessions whose `.jsonl` files have been deleted from `~/.claude/projects/`. +- Cannot access sessions from other machines (files are local only). +- Edit tool operations show deltas, not full file content — use `claude-code-history-files-finder` for full file recovery. +- Compact summaries are lossy — early conversation details may be missing. +- `sessions-index.json` can be stale (entries pointing to deleted files). The script falls back to filesystem-based discovery. + +## Example Trigger Phrases + +- "continue work from session `abc123-...`" +- "don't resume, just read the .claude files and continue" +- "check what I was working on in the last session and keep going" +- "search my sessions for the PR review work" diff --git a/continue-claude-work/references/file_structure.md b/continue-claude-work/references/file_structure.md new file mode 100644 index 00000000..c4772407 --- /dev/null +++ b/continue-claude-work/references/file_structure.md @@ -0,0 +1,249 @@ +# Claude Code Local File Structure + +Ground-truth reference for `~/.claude/` directory layout and JSONL session format. + +## Directory Layout + +``` +~/.claude/ + projects/ # Per-project session storage (primary) + / + sessions-index.json # Master index of all sessions + .jsonl # Session transcript + / # Session subdirectory (optional) + subagents/ + agent-.meta.json # Agent metadata + agent-.jsonl # Agent transcript + tool-results/ + toolu_.txt # Large tool outputs + memory/ # Persistent memory files (MEMORY.md, etc.) + history.jsonl # Global prompt history (no session IDs) + tasks/ # Task tracking (per-session lock/highwatermark) + plans/ # Plan documents (random-name.md) + debug/ # Per-session debug logs (.txt) + transcripts/ # Global tool operation logs (ses_.jsonl) + file-history/ # File modification backups + todos/ # Todo items +``` + +## Path Normalization + +Project paths are encoded by replacing `/` with `-`: + +| Original | Normalized | +|----------|-----------| +| `/path/to/project` | `-path-to-project` | +| `/another/workspace/app` | `-another-workspace-app` | + +## sessions-index.json Schema + +```json +{ + "version": 1, + "entries": [ + { + "sessionId": "20089b2a-e3dd-48b8-809c-0647128bf3b8", + "fullPath": "~/.claude/projects/-path-to-project/20089b2a-....jsonl", + "fileMtime": 1741327503477, + "firstPrompt": "fix the login bug", + "summary": "Fixed authentication redirect...", + "messageCount": 42, + "created": "2026-03-07T03:25:03.477Z", + "modified": "2026-03-07T12:21:43.806Z", + "gitBranch": "main", + "projectPath": "/path/to/project", + "isSidechain": false + } + ], + "originalPath": "/path/to/project" +} +``` + +Key fields for session identification: +- `sessionId` — UUID v4 format +- `firstPrompt` — first user message (best for topic matching) +- `summary` — auto-generated summary of the session +- `modified` — last activity timestamp (ISO 8601) +- `gitBranch` — git branch at session time +- `isSidechain` — `false` for main conversations + +## Compaction in Session Files + +Claude Code uses [server-side compaction](https://platform.claude.com/docs/en/build-with-claude/compaction). When context fills up, two consecutive lines appear: + +### Line 1: compact_boundary marker + +```json +{ + "type": "system", + "subtype": "compact_boundary", + "parentUuid": null, + "logicalParentUuid": "prev-uuid", + "compactMetadata": { + "trigger": "input_tokens", + "preTokens": 180000 + } +} +``` + +### Line 2: Compact summary (special user message) + +```json +{ + "type": "user", + "isCompactSummary": true, + "isVisibleInTranscriptOnly": true, + "message": { + "role": "user", + "content": "This session is being continued from a previous conversation that ran out of context. The summary below covers the earlier portion of the conversation.\n\nAnalysis:\n1. **Initial Request**: User asked to...\n2. **Progress**: Completed X, Y, Z...\n3. **Current state**: Working on..." + } +} +``` + +Key properties: +- **`isCompactSummary: true`** — most reliable way to identify compact summaries +- **`isVisibleInTranscriptOnly: true`** — not sent to the API, only stored in the transcript +- Summary is always a plain string in `.message.content` (not an array) +- Typically 12K-31K characters (high-density information) +- A long session may have multiple compact boundaries (4+ is common for 10MB+ sessions) +- The **last** compact boundary's summary reflects the most recent state +- Messages after the last boundary are the "hot zone" — they were in Claude's live context + +### Default compaction prompt (from API docs) + +> "You have written a partial transcript for the initial task above. Please write a summary of the transcript. The purpose of this summary is to provide continuity so you can continue to make progress towards solving the task in a future context, where the raw history above may not be accessible and will be replaced with this summary." + +## Session JSONL Message Types + +Each `.jsonl` file has one JSON object per line. Common types: + +### file-history-snapshot (always first line) + +```json +{ + "type": "file-history-snapshot", + "messageId": "uuid", + "snapshot": { "trackedFileBackups": {}, "timestamp": "..." }, + "isSnapshotUpdate": false +} +``` + +### User message + +```json +{ + "parentUuid": "prev-uuid or null", + "isSidechain": false, + "cwd": "/path/to/project", + "sessionId": "session-uuid", + "version": "2.1.71", + "gitBranch": "main", + "type": "user", + "message": { + "role": "user", + "content": "fix the login bug" + }, + "uuid": "msg-uuid", + "timestamp": "2026-03-07T03:25:03.477Z" +} +``` + +**Important**: `.message.content` can be: +- A **string** for plain text user messages +- An **array** of content blocks for tool results and multi-part messages: + ```json + "content": [ + { "type": "tool_result", "tool_use_id": "toolu_...", "content": "..." }, + { "type": "text", "text": "now do X" } + ] + ``` + +### Assistant message + +```json +{ + "type": "assistant", + "message": { + "role": "assistant", + "model": "claude-opus-4-6", + "content": [ + { "type": "thinking", "thinking": "internal reasoning..." }, + { "type": "text", "text": "visible response text" }, + { "type": "tool_use", "id": "toolu_...", "name": "Bash", "input": { "command": "..." } } + ] + } +} +``` + +Content block types in assistant messages: +- `thinking` — internal reasoning (skip when extracting actionable context) +- `text` — visible response to user (extract this) +- `tool_use` — tool invocations (useful for understanding what was done) + +### Tool result (user message with tool output) + +```json +{ + "type": "user", + "message": { + "role": "user", + "content": [ + { "type": "tool_result", "tool_use_id": "toolu_...", "content": "command output..." } + ] + } +} +``` + +## history.jsonl Schema + +Global prompt log. Does NOT contain session IDs — only useful for finding when a prompt was issued and in which project: + +```json +{ + "display": "/init ", + "pastedContents": {}, + "timestamp": 1758996122446, + "project": "/path/to/project" +} +``` + +## Extraction Patterns + +### List recent sessions for current project + +```bash +cat ~/.claude/projects//sessions-index.json \ + | jq '.entries | sort_by(.modified) | reverse | .[:5] | .[] | {sessionId, firstPrompt, summary, messageCount, modified, gitBranch}' +``` + +### Extract user text from session tail + +Handles both string and array content formats: + +```bash +tail -n 200 .jsonl \ + | jq -r 'select(.type == "user" and .message.role == "user") + | .message.content + | if type == "string" then . + elif type == "array" then map(select(.type == "text") | .text) | join("\n") + else empty end' \ + | tail -n 80 +``` + +### Extract assistant text (excluding thinking/tool_use) + +```bash +tail -n 200 .jsonl \ + | jq -r 'select(.message.role == "assistant") + | .message.content + | if type == "array" then map(select(.type == "text") | .text) | join("\n") + else empty end' \ + | tail -n 120 +``` + +### Find sessions by keyword in firstPrompt + +```bash +cat ~/.claude/projects//sessions-index.json \ + | jq -r '.entries[] | select(.firstPrompt | test("keyword"; "i")) | "\(.modified) \(.sessionId) \(.firstPrompt)"' +``` diff --git a/continue-claude-work/scripts/extract_resume_context.py b/continue-claude-work/scripts/extract_resume_context.py new file mode 100755 index 00000000..fad677bc --- /dev/null +++ b/continue-claude-work/scripts/extract_resume_context.py @@ -0,0 +1,841 @@ +#!/usr/bin/env python3 +""" +Extract actionable resume context from Claude Code session files. + +Produces a structured Markdown briefing by fusing: +- Session index metadata (sessions-index.json) +- Compact boundary summaries (highest-signal context) +- Post-compact user/assistant messages (the "hot zone") +- Subagent workflow state (multi-agent recovery) +- Session end reason detection +- Git workspace state +- MEMORY.md persistent context +- Interrupted tool-call detection + +Usage: + # Extract context from latest session for current project + python3 extract_resume_context.py + + # Extract context from a specific session + python3 extract_resume_context.py --session + + # Search sessions by topic + python3 extract_resume_context.py --query "auth feature" + + # List recent sessions + python3 extract_resume_context.py --list + + # Specify project path explicitly + python3 extract_resume_context.py --project /path/to/project +""" + +import argparse +import json +import os +import re +import subprocess +import sys +import time +from pathlib import Path +from typing import Dict, List, Optional + +CLAUDE_DIR = Path.home() / ".claude" +PROJECTS_DIR = CLAUDE_DIR / "projects" + +# Message types that are noise — skip when extracting context +NOISE_TYPES = {"progress", "queue-operation", "file-history-snapshot", "last-prompt"} +# System message subtypes that are noise +NOISE_SUBTYPES = {"api_error", "turn_duration", "stop_hook_summary"} + +# Patterns that indicate system/internal content, not real user requests +NOISE_USER_PATTERNS = [ + "This session is being continued", + "", + "", +] + + +def normalize_path(project_path: str) -> str: + """Convert absolute path to Claude's normalized directory name.""" + return project_path.replace("/", "-") + + +def find_project_dir(project_path: str) -> Optional[Path]: + """Find the Claude projects directory for a given project path.""" + abs_path = os.path.abspath(project_path) + + # If the path is already inside ~/.claude/projects/, use it directly + projects_str = str(PROJECTS_DIR) + "/" + if abs_path.startswith(projects_str): + candidate = Path(abs_path) + if candidate.is_dir(): + return candidate + rel = abs_path[len(projects_str):] + top_dir = PROJECTS_DIR / rel.split("/")[0] + if top_dir.is_dir(): + return top_dir + + normalized = normalize_path(abs_path) + candidate = PROJECTS_DIR / normalized + if candidate.is_dir(): + return candidate + # Fallback: search for partial match + for d in PROJECTS_DIR.iterdir(): + if d.is_dir() and normalized in d.name: + return d + return None + + +def load_sessions_index(project_dir: Path) -> List[Dict]: + """Load and parse sessions-index.json, sorted by modified desc.""" + index_file = project_dir / "sessions-index.json" + if not index_file.exists(): + return [] + with open(index_file, encoding="utf-8") as f: + data = json.load(f) + entries = data.get("entries", []) + entries.sort(key=lambda e: e.get("modified", ""), reverse=True) + return entries + + +def search_sessions(entries: List[Dict], query: str) -> List[Dict]: + """Search sessions by keyword in firstPrompt and summary.""" + query_lower = query.lower() + results = [] + for entry in entries: + first_prompt = (entry.get("firstPrompt") or "").lower() + summary = (entry.get("summary") or "").lower() + if query_lower in first_prompt or query_lower in summary: + results.append(entry) + return results + + +def format_session_entry(entry: Dict, file_exists: bool = True) -> str: + """Format a session index entry for display.""" + sid = entry.get("sessionId", "?") + modified = entry.get("modified", "?") + msgs = entry.get("messageCount", "?") + branch = entry.get("gitBranch", "?") + prompt = (entry.get("firstPrompt") or "")[:80] + ghost = "" if file_exists else " [file missing]" + return f" {sid} [{branch}] {msgs} msgs {modified}{ghost}\n {prompt}" + + +# ── Session file parsing ──────────────────────────────────────────── + + +def parse_session_structure(session_file: Path) -> Dict: + """Parse a session JSONL file and return structured data.""" + file_size = session_file.stat().st_size + total_lines = 0 + + # First pass: find compact boundaries and count lines + compact_boundaries = [] # (line_num, summary_text) + with open(session_file, encoding="utf-8") as f: + prev_boundary_line = None + for i, raw_line in enumerate(f): + total_lines += 1 + + # Detect compact summary via isCompactSummary flag (most reliable) + if '"isCompactSummary"' in raw_line: + try: + obj = json.loads(raw_line) + if obj.get("isCompactSummary"): + content = obj.get("message", {}).get("content", "") + if isinstance(content, str): + boundary_line = prev_boundary_line if prev_boundary_line is not None else max(0, i - 1) + compact_boundaries.append((boundary_line, content)) + prev_boundary_line = None + continue + except json.JSONDecodeError: + pass + + # Detect compact_boundary marker + if '"compact_boundary"' in raw_line and '"subtype"' in raw_line: + try: + obj = json.loads(raw_line) + if obj.get("subtype") == "compact_boundary": + prev_boundary_line = i + continue + except json.JSONDecodeError: + pass + + # Fallback: if prev line was boundary and this is a user message with long string content + if prev_boundary_line is not None: + try: + obj = json.loads(raw_line) + content = obj.get("message", {}).get("content", "") + if isinstance(content, str) and len(content) > 100: + compact_boundaries.append((prev_boundary_line, content)) + except (json.JSONDecodeError, AttributeError): + compact_boundaries.append((prev_boundary_line, "")) + prev_boundary_line = None + + # Determine hot zone: everything after last compact boundary + its summary + if compact_boundaries: + last_boundary_line = compact_boundaries[-1][0] + hot_zone_start = last_boundary_line + 2 # skip boundary + summary + else: + # No compact boundaries: use size-adaptive strategy + if file_size < 500_000: # <500KB: read last 60% + hot_zone_start = max(0, int(total_lines * 0.4)) + elif file_size < 5_000_000: # <5MB: read last 30% + hot_zone_start = max(0, int(total_lines * 0.7)) + else: # >5MB: read last 15% + hot_zone_start = max(0, int(total_lines * 0.85)) + + # Second pass: extract hot zone messages + messages = [] + unresolved_tool_calls = {} # tool_use_id -> tool_use_info + errors = [] + files_touched = set() + last_message_role = None + error_count = 0 + + with open(session_file, encoding="utf-8") as f: + for i, raw_line in enumerate(f): + if i < hot_zone_start: + continue + try: + obj = json.loads(raw_line) + except json.JSONDecodeError: + continue + + msg_type = obj.get("type", "") + if msg_type in NOISE_TYPES: + continue + if msg_type == "system": + subtype = obj.get("subtype", "") + if subtype in NOISE_SUBTYPES: + if subtype == "api_error": + error_count += 1 + continue + if subtype == "compact_boundary": + continue + + # Track tool calls and results + msg = obj.get("message", {}) + role = msg.get("role", "") + content = msg.get("content", "") + + # Extract tool_use from assistant messages + if role == "assistant" and isinstance(content, list): + for block in content: + if not isinstance(block, dict) or block.get("type") != "tool_use": + continue + tool_id = block.get("id", "") + tool_name = block.get("name", "?") + inp = block.get("input", {}) + unresolved_tool_calls[tool_id] = { + "name": tool_name, + "input_preview": str(inp)[:200], + } + # Track file operations + if tool_name in ("Write", "Edit", "Read"): + fp = inp.get("file_path", "") + if fp: + files_touched.add(fp) + elif tool_name == "Bash": + cmd = inp.get("command", "") + for match in re.findall(r'(? str: + """Detect why the session ended.""" + if unresolved: + return "interrupted" # Tool calls dispatched but no results — likely ctrl-c + if error_count >= 3: + return "error_cascade" # Multiple API errors suggest systemic failure + if last_role == "assistant": + return "completed" # Assistant had the last word — clean end + if last_role == "user": + return "abandoned" # User sent a message but got no response + return "unknown" + + +def _is_noise_user_text(text: str) -> bool: + """Check if user text is system noise rather than a real request.""" + for pattern in NOISE_USER_PATTERNS: + if text.startswith(pattern) or pattern in text[:200]: + return True + return False + + +def extract_user_text(messages: List[Dict], limit: int = 5) -> List[str]: + """Extract the last N user text messages (not tool results or system noise).""" + user_texts = [] + for msg_obj in reversed(messages): + if msg_obj.get("isCompactSummary"): + continue + msg = msg_obj.get("message", {}) + if msg.get("role") != "user": + continue + content = msg.get("content", "") + if isinstance(content, str) and content.strip(): + if _is_noise_user_text(content): + continue + user_texts.append(content.strip()) + elif isinstance(content, list): + texts = [ + b.get("text", "") + for b in content + if isinstance(b, dict) and b.get("type") == "text" + ] + combined = "\n".join(t for t in texts if t.strip()) + if combined and not _is_noise_user_text(combined): + user_texts.append(combined) + if len(user_texts) >= limit: + break + user_texts.reverse() + return user_texts + + +def extract_assistant_text(messages: List[Dict], limit: int = 3) -> List[str]: + """Extract the last N assistant text responses (no thinking/tool_use).""" + assistant_texts = [] + for msg_obj in reversed(messages): + msg = msg_obj.get("message", {}) + if msg.get("role") != "assistant": + continue + content = msg.get("content", "") + if isinstance(content, list): + texts = [ + b.get("text", "") + for b in content + if isinstance(b, dict) and b.get("type") == "text" + ] + combined = "\n".join(t for t in texts if t.strip()) + if combined: + assistant_texts.append(combined[:2000]) + if len(assistant_texts) >= limit: + break + assistant_texts.reverse() + return assistant_texts + + +# ── Subagent extraction ────────────────────────────────────────────── + + +def extract_subagent_context(session_file: Path) -> List[Dict]: + """Extract subagent summaries from session subdirectories. + + Returns list of {name, type, status, last_text, is_interrupted}. + """ + session_dir = session_file.parent / session_file.stem + subagents_dir = session_dir / "subagents" + if not subagents_dir.is_dir(): + return [] + + # Group by agent ID: find meta.json and .jsonl pairs + agent_ids = set() + for f in subagents_dir.iterdir(): + if f.suffix == ".jsonl": + agent_ids.add(f.stem) + + results = [] + for agent_id in sorted(agent_ids): + jsonl_file = subagents_dir / f"{agent_id}.jsonl" + meta_file = subagents_dir / f"{agent_id}.meta.json" + + # Parse agent type from ID (format: agent-a- or agent-a) + agent_type = "unknown" + if meta_file.exists(): + try: + with open(meta_file, encoding="utf-8") as f: + meta = json.load(f) + agent_type = meta.get("type", meta.get("subagent_type", "unknown")) + except (json.JSONDecodeError, OSError): + pass + + if agent_type == "unknown": + # Infer from ID pattern: agent-a- + match = re.match(r'agent-a(compact|prompt_suggestion|[a-z_]+)-', agent_id) + if match: + agent_type = match.group(1) + + # Skip compact and prompt_suggestion agents (internal, not user work) + if agent_type in ("compact", "prompt_suggestion"): + continue + + # Read last few lines for final output + last_text = "" + is_interrupted = False + line_count = 0 + + if jsonl_file.exists(): + try: + lines = jsonl_file.read_text(encoding="utf-8").strip().split("\n") + line_count = len(lines) + has_tool_use_pending = False + # Check last 10 lines for final assistant text + for raw_line in reversed(lines[-10:]): + try: + obj = json.loads(raw_line) + msg = obj.get("message", {}) + role = msg.get("role", "") + content = msg.get("content", "") + + if role == "assistant" and isinstance(content, list): + for block in content: + if not isinstance(block, dict): + continue + block_type = block.get("type") + if block_type == "tool_use": + has_tool_use_pending = True + elif block_type == "text": + text = block.get("text", "") + if text.strip() and not last_text: + last_text = text.strip()[:500] + + if role == "user" and isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_result": + has_tool_use_pending = False + except json.JSONDecodeError: + continue + + is_interrupted = has_tool_use_pending + except OSError: + pass + + results.append({ + "id": agent_id, + "type": agent_type, + "last_text": last_text, + "is_interrupted": is_interrupted, + "lines": line_count, + }) + + return results + + +# ── Context sources ────────────────────────────────────────────────── + + +def get_git_state(project_path: str) -> str: + """Get current git status and recent log.""" + parts = [] + try: + branch = subprocess.run( + ["git", "branch", "--show-current"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, cwd=project_path, timeout=5, + ) + if branch.stdout.strip(): + parts.append(f"**Current branch**: `{branch.stdout.strip()}`") + except (subprocess.TimeoutExpired, FileNotFoundError): + pass + + try: + status = subprocess.run( + ["git", "status", "--short"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, cwd=project_path, timeout=10, + ) + if status.stdout.strip(): + parts.append(f"### git status\n```\n{status.stdout.strip()}\n```") + else: + parts.append("### git status\nClean working tree.") + except (subprocess.TimeoutExpired, FileNotFoundError): + parts.append("### git status\n(unavailable)") + + try: + log = subprocess.run( + ["git", "log", "--oneline", "-5"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, cwd=project_path, timeout=10, + ) + if log.stdout.strip(): + parts.append(f"### git log (last 5)\n```\n{log.stdout.strip()}\n```") + except (subprocess.TimeoutExpired, FileNotFoundError): + pass + + return "\n\n".join(parts) + + +def get_memory_md(project_dir: Path) -> Optional[str]: + """Read MEMORY.md if it exists in the project's memory directory.""" + memory_dir = project_dir / "memory" + memory_file = memory_dir / "MEMORY.md" + if memory_file.exists(): + content = memory_file.read_text(encoding="utf-8").strip() + if content: + return content[:3000] + return None + + +def get_session_memory(session_file: Path) -> Optional[str]: + """Read session-memory/summary.md if it exists (newer CC versions).""" + session_dir = session_file.parent / session_file.stem + summary = session_dir / "session-memory" / "summary.md" + if summary.exists(): + content = summary.read_text(encoding="utf-8").strip() + if content: + return content[:3000] + return None + + +# ── Output formatting ──────────────────────────────────────────────── + + +END_REASON_LABELS = { + "completed": "Clean exit (assistant completed response)", + "interrupted": "Interrupted (unresolved tool calls — likely ctrl-c or timeout)", + "error_cascade": "Error cascade (multiple API errors)", + "abandoned": "Abandoned (user message with no response)", + "unknown": "Unknown", +} + + +def build_briefing( + session_entry: Optional[Dict], + parsed: Dict, + project_path: str, + project_dir: Path, + session_file: Path, +) -> str: + """Build the structured Markdown briefing.""" + sections = [] + + # Header + sections.append("# Resume Context Briefing\n") + + # Session metadata + if session_entry: + sid = session_entry.get("sessionId", "?") + modified = session_entry.get("modified", "?") + branch = session_entry.get("gitBranch", "?") + msg_count = session_entry.get("messageCount", "?") + first_prompt = session_entry.get("firstPrompt", "") + summary = session_entry.get("summary", "") + + sections.append("## Session Info\n") + sections.append(f"- **ID**: `{sid}`") + sections.append(f"- **Last active**: {modified}") + sections.append(f"- **Branch**: `{branch}`") + sections.append(f"- **Messages**: {msg_count}") + sections.append(f"- **First prompt**: {first_prompt}") + if summary: + sections.append(f"- **Summary**: {summary[:300]}") + + # File stats + end reason + file_mb = parsed["file_size"] / 1_000_000 + end_label = END_REASON_LABELS.get(parsed["end_reason"], parsed["end_reason"]) + sections.append(f"\n**Session file**: {file_mb:.1f} MB, {parsed['total_lines']} lines, " + f"{len(parsed['compact_boundaries'])} compaction(s)") + sections.append(f"**Session end reason**: {end_label}") + if parsed["error_count"] > 0: + sections.append(f"**API errors**: {parsed['error_count']}") + + # Session memory (newer CC versions generate this automatically) + session_mem = get_session_memory(session_file) + if session_mem: + sections.append("\n## Session Memory (auto-generated by Claude Code)\n") + sections.append(session_mem) + + # Compact summary (highest-signal context) + if parsed["compact_boundaries"]: + last_summary = parsed["compact_boundaries"][-1][1] + if last_summary: + # Allow up to 8K chars — this is the highest-signal content + display = last_summary[:8000] + if len(last_summary) > 8000: + display += f"\n\n... (truncated, full summary: {len(last_summary)} chars)" + sections.append("\n## Compact Summary (auto-generated by previous session)\n") + sections.append(display) + + # Last user requests + user_texts = extract_user_text(parsed["messages"]) + if user_texts: + sections.append("\n## Last User Requests\n") + for i, text in enumerate(user_texts, 1): + display = text[:500] + if len(text) > 500: + display += "..." + sections.append(f"### Request {i}\n{display}\n") + + # Last assistant responses + assistant_texts = extract_assistant_text(parsed["messages"]) + if assistant_texts: + sections.append("\n## Last Assistant Responses\n") + for i, text in enumerate(assistant_texts, 1): + display = text[:1000] + if len(text) > 1000: + display += "..." + sections.append(f"### Response {i}\n{display}\n") + + # Errors encountered + if parsed["errors"]: + sections.append("\n## Errors Encountered\n") + seen = set() + for err in parsed["errors"]: + short = err[:200] + if short not in seen: + seen.add(short) + sections.append(f"```\n{err}\n```\n") + + # Unresolved tool calls (interrupted session) + if parsed["unresolved_tool_calls"]: + sections.append("\n## Unresolved Tool Calls (session was interrupted)\n") + for tool_id, info in parsed["unresolved_tool_calls"].items(): + sections.append(f"- **{info['name']}**: `{tool_id}`") + sections.append(f" Input: {info['input_preview']}") + + # Subagent context (the "nobody has done this" feature) + subagents = extract_subagent_context(session_file) + if subagents: + interrupted = [s for s in subagents if s["is_interrupted"]] + completed = [s for s in subagents if not s["is_interrupted"]] + sections.append(f"\n## Subagent Workflow ({len(completed)} completed, {len(interrupted)} interrupted)\n") + if interrupted: + sections.append("### Interrupted Subagents\n") + for sa in interrupted: + sections.append(f"- **{sa['type']}** (`{sa['id']}`, {sa['lines']} lines)") + if sa["last_text"]: + sections.append(f" Last output: {sa['last_text'][:300]}") + if completed: + sections.append("\n### Completed Subagents\n") + for sa in completed: + sections.append(f"- **{sa['type']}** (`{sa['id']}`, {sa['lines']} lines)") + if sa["last_text"]: + sections.append(f" Last output: {sa['last_text'][:200]}") + + # Files touched in session + if parsed["files_touched"]: + sections.append("\n## Files Touched in Session\n") + for fp in sorted(parsed["files_touched"])[:30]: + sections.append(f"- `{fp}`") + + # MEMORY.md + memory = get_memory_md(project_dir) + if memory: + sections.append("\n## Persistent Memory (MEMORY.md)\n") + sections.append(memory) + + # Git state + sections.append("\n## Current Workspace State\n") + sections.append(get_git_state(project_path)) + + return "\n".join(sections) + + +# ── CLI ────────────────────────────────────────────────────────────── + + +def _check_session_files(entries: List[Dict], project_dir: Path) -> Dict[str, bool]: + """Check which index entries have actual files on disk.""" + status = {} + for entry in entries: + sid = entry.get("sessionId", "") + session_file = project_dir / f"{sid}.jsonl" + if session_file.exists(): + status[sid] = True + else: + full_path = entry.get("fullPath", "") + status[sid] = bool(full_path and Path(full_path).exists()) + return status + + +def main(): + parser = argparse.ArgumentParser( + description="Extract actionable resume context from Claude Code sessions.", + ) + parser.add_argument( + "--project", "-p", + default=os.getcwd(), + help="Project path (default: current directory)", + ) + parser.add_argument( + "--session", "-s", + default=None, + help="Session ID to extract context from", + ) + parser.add_argument( + "--query", "-q", + default=None, + help="Search sessions by keyword in firstPrompt/summary", + ) + parser.add_argument( + "--list", "-l", + action="store_true", + help="List recent sessions", + ) + parser.add_argument( + "--limit", "-n", + type=int, + default=10, + help="Number of sessions to list (default: 10)", + ) + parser.add_argument( + "--exclude-current", + default=None, + help="Session ID to exclude (typically the currently active session)", + ) + args = parser.parse_args() + + project_path = os.path.abspath(args.project) + project_dir = find_project_dir(project_path) + + if not project_dir: + print(f"Error: no Claude session data found for {project_path}", file=sys.stderr) + print(f"Looked in: {PROJECTS_DIR / normalize_path(project_path)}", file=sys.stderr) + sys.exit(1) + + entries = load_sessions_index(project_dir) + + # ── List mode ── + if args.list: + # Show both index entries and actual files, with file-exists status + file_status = _check_session_files(entries, project_dir) + existing = sum(1 for v in file_status.values() if v) + missing = sum(1 for v in file_status.values() if not v) + + if not entries and not list(project_dir.glob("*.jsonl")): + print("No sessions found.") + sys.exit(0) + + print(f"Sessions for {project_path}:\n") + if missing > 0: + print(f" (index has {len(entries)} entries, {existing} with files, {missing} with missing files)\n") + + for entry in entries[:args.limit]: + sid = entry.get("sessionId", "") + exists = file_status.get(sid, False) + print(format_session_entry(entry, file_exists=exists)) + print() + + # Also show files NOT in index + indexed_ids = {e.get("sessionId") for e in entries} + orphan_files = [ + f for f in sorted(project_dir.glob("*.jsonl"), key=lambda f: f.stat().st_mtime, reverse=True) + if f.stem not in indexed_ids + ] + if orphan_files: + print(f" Files not in index ({len(orphan_files)}):") + for f in orphan_files[:5]: + size_kb = f.stat().st_size / 1000 + print(f" {f.stem} ({size_kb:.0f} KB)") + print() + + sys.exit(0) + + # ── Query mode ── + if args.query: + results = search_sessions(entries, args.query) + if not results: + print(f"No sessions matching '{args.query}'.", file=sys.stderr) + sys.exit(1) + print(f"Sessions matching '{args.query}' ({len(results)} found):\n") + for entry in results[: args.limit]: + print(format_session_entry(entry)) + print() + if len(results) == 1: + args.session = results[0]["sessionId"] + else: + sys.exit(0) + + # ── Extract mode ── + session_id = args.session + session_entry = None + + if session_id: + for entry in entries: + if entry.get("sessionId") == session_id: + session_entry = entry + break + session_file = project_dir / f"{session_id}.jsonl" + if not session_file.exists(): + if session_entry: + full_path = session_entry.get("fullPath", "") + if full_path and Path(full_path).exists(): + session_file = Path(full_path) + if not session_file.exists(): + for jsonl in project_dir.glob("*.jsonl"): + if session_id in jsonl.name: + session_file = jsonl + break + else: + print(f"Error: session file not found for {session_id}", file=sys.stderr) + sys.exit(1) + else: + # Use latest session — prefer actual files, skip current session + jsonl_files = sorted( + project_dir.glob("*.jsonl"), + key=lambda f: f.stat().st_mtime, + reverse=True, + ) + if args.exclude_current: + jsonl_files = [f for f in jsonl_files if f.stem != args.exclude_current] + if not jsonl_files: + print("Error: no session files found.", file=sys.stderr) + sys.exit(1) + + # Skip the most recent file if it's likely the current active session + # (modified within the last 60 seconds and no explicit session requested) + if len(jsonl_files) > 1: + newest = jsonl_files[0] + age_seconds = time.time() - newest.stat().st_mtime + if age_seconds < 60: + print(f"Skipping active session {newest.stem} (modified {age_seconds:.0f}s ago)", + file=sys.stderr) + jsonl_files = jsonl_files[1:] + + session_file = jsonl_files[0] + session_id = session_file.stem + for entry in entries: + if entry.get("sessionId") == session_id: + session_entry = entry + break + + # Parse and build briefing + print(f"Parsing session {session_id} ({session_file.stat().st_size / 1_000_000:.1f} MB)...", + file=sys.stderr) + + parsed = parse_session_structure(session_file) + briefing = build_briefing(session_entry, parsed, project_path, project_dir, session_file) + + print(briefing) + + +if __name__ == "__main__": + main() diff --git a/deep-research/SKILL.md b/deep-research/SKILL.md index f15212e5..b89b0ffc 100644 --- a/deep-research/SKILL.md +++ b/deep-research/SKILL.md @@ -1,183 +1,479 @@ --- name: deep-research description: | - Generate format-controlled research reports with evidence tracking, citations, and iterative review. This skill should be used when users request a research report, literature review, market or industry analysis, competitive landscape, policy or technical brief, or require a strict report template and section formatting that a single deepresearch pass cannot reliably enforce. + Generate format-controlled research reports with evidence tracking, citations, source governance, and multi-pass synthesis. + This skill should be used when users request a research report, literature review, market or industry analysis, + competitive landscape, policy or technical brief. Triggers: "帮我调研一下", "深度研究", "综述报告", "深入分析", + "research this topic", "write a report on", "survey the literature on", "competitive analysis of", + "技术选型分析", "竞品研究", "政策分析", "行业报告". + V6 adds: source-type governance, AS_OF freshness checks, mandatory counter-review, and citation registry. V6.1 adds: source accessibility (circular verification forbidden, exclusive advantage encouraged). --- # Deep Research -Create high-fidelity research reports with strict format control, evidence mapping, and multi-pass synthesis. +Create high-fidelity research reports with strict format control, evidence mapping, source governance, and multi-pass synthesis. -## Quick Start +## Architecture: Lead Agent + Subagents -1. Clarify the report spec and format contract -2. Build a research plan and query set -3. Collect evidence with the deepresearch tool (multi-pass if needed) -4. Triage sources and build an evidence table -5. Draft the full report in multiple complete passes (parallel subagents) -6. UNION merge, enforce format compliance, verify citations -7. Present draft for human review and iterate +``` +Lead Agent (coordinator — minimizes raw search context) + | + P0: Environment + source policy setup + | + P1: Research Task Board (roles, queries, parallel groups) + | + Dispatch ──→ Subagent A ──→ writes task-a.md ──┐ + ──→ Subagent B ──→ writes task-b.md ──┤ (parallel) + ──→ Subagent C ──→ writes task-c.md ──┘ + | | + | research-notes/ <────────────────────────┘ + | + P2: Build citation registry with source_type + as_of + authority + P3: Evidence-mapped outline with counter-claim flags + P4: Draft from notes (never from raw search results) + P5: Counter-review (claims, confidence, alternatives) + P6: Verify (every [n] in registry, traceability check) + P7: Polish → final report with confidence markers +``` -## Core Workflow +**Context efficiency:** Subagents' raw search results stay in their context and are discarded. Lead agent sees only distilled notes (~60-70% context reduction). -Copy this checklist and track progress: +## Mode Selection -``` -Deep Research Progress: -- [ ] Step 1: Intake and format contract -- [ ] Step 2: Research plan and query set -- [ ] Step 3: Evidence collection (deepresearch tool) -- [ ] Step 4: Source triage and evidence table -- [ ] Step 5: Outline and section map -- [ ] Step 6: Multi-pass full drafting (parallel subagents) -- [ ] Step 7: UNION merge and format compliance -- [ ] Step 8: Evidence and citation verification -- [ ] Step 9: Present draft for human review and iterate -``` +Determine the research mode before starting: + +| Dimension | Options | +|-----------|---------| +| **Topic Mode** | Enterprise Research (company/corporation) OR General Research (industry/policy/tech) | +| **Depth Mode** | Standard (5-6 tasks, 3000-8000 words) OR Lightweight (3-4 tasks, 2000-4000 words) | + +- **Enterprise Research Mode**: Six-dimension data collection with structured analysis frameworks (SWOT, risk matrix, competitive barrier quantification) +- **General Research Mode**: Standard P0-P7 research pipeline with source governance +- **Depth Selection**: Lightweight for single entity/concept < 30 words; Standard for multi-entity comparison or "深入"/"comprehensive" requests + +## Source Governance (V6) + +### Source Accessibility Classification + +**CRITICAL RULE**: Every source must be classified by accessibility: + +| Accessibility | Definition | Examples | Usage Rule | +|--------------|------------|----------|------------| +| `public` | Available to any external researcher without authentication | Public websites, news articles, WHOIS (without privacy), academic papers | ✅ Always allowed | +| `semi-public` | Requires registration or limited access | LinkedIn profiles, Crunchbase basic, industry reports (free tier) | ✅ Allowed with disclosure | +| `exclusive-user-provided` | User's paid subscriptions, private APIs, proprietary databases | Crunchbase Pro, PitchBook, private data feeds, internal databases | ✅ **ALLOWED** for third-party research | +| `private-user-owned` | User's own accounts when researching themselves | User's registrar for user's own company, user's bank for user's own finances | ❌ **FORBIDDEN** - circular verification | + +**⚠️ CIRCULAR VERIFICATION BAN**: You must NOT: +- Use user's private data to "discover" what they already know about themselves +- Research user's own company by accessing user's private accounts +- Present user's private knowledge as "research findings" + +**✅ EXCLUSIVE INFORMATION ADVANTAGE**: You SHOULD: +- Use user's Crunchbase Pro to research competitors +- Use user's proprietary databases for market research +- Use user's private APIs for investment analysis +- Leverage any exclusive source user provides for third-party research + +### Source Type Labels -### Step 1: Intake and Format Contract +Every source MUST also be tagged with: -Establish the report requirements before any research: +| Label | Definition | Examples | +|-------|------------|----------| +| `official` | Primary source, official documentation | Company SEC filings, government reports, official blog | +| `academic` | Peer-reviewed research | Journal articles, conference papers, dissertations | +| `secondary-industry` | Professional analysis | Industry reports, analyst coverage, trade publications | +| `journalism` | News reporting | Reputable media outlets, investigative journalism | +| `community` | User-generated content | Forums, reviews, social media, Q&A sites | +| `other` | Uncategorized or mixed | Aggregators, unverified sources | -- Confirm audience, purpose, scope, time range, and geography -- Lock output format: Markdown, DOCX, slides, or user-provided template -- Capture required sections and exact formatting rules -- Confirm citation style (footnotes, inline, numbered, APA, etc.) -- Confirm length targets per section -- Ask for any existing style guide or sample report +**Quality Gates:** +- Standard mode: ≥30% official sources in final approved set +- Lightweight mode: ≥20% official sources +- Maximum single-source share: ≤25% (Standard), ≤30% (Lightweight) +- Minimum unique domains: 5 (Standard), 3 (Lightweight) -Create a concise report spec file: +## AS_OF Date Policy + +Set `AS_OF` date explicitly at P0. For all time-sensitive claims: +- Include source publication date with every citation +- Downgrade confidence if source is older than relevant horizon +- Flag stale sources in registry (studies >3 years, news >6 months for fast-moving topics) + +## P0: Environment & Policy Setup + +Check capabilities before starting: + +| Check | Requirement | Impact if Missing | +|-------|-------------|-------------------| +| web_search available | Required | Stop - cannot proceed | +| web_fetch available | Required for DEEP tasks | SCAN-only mode | +| Subagent dispatch | Preferred | Degrade to sequential | +| Filesystem writable | Required | In-memory notes only | + +Set policy variables: +- `AS_OF`: Today's date (YYYY-MM-DD) - mandatory for timed topics +- `MODE`: Standard (default) or Lightweight +- `SOURCE_TYPE_POLICY`: Enforce official/academic/secondary/journalism/community/other labels +- `COUNTER_REVIEW_PLAN`: What opposing interpretation to test + +Report: `[P0 complete] Subagent: {yes/no}. Mode: {standard/lightweight}. AS_OF: {YYYY-MM-DD}.` + +When researching a specific company/enterprise, follow this specialized workflow that ensures six-dimension coverage, quantified analysis frameworks, and three-level quality control. + +### Enterprise Workflow Overview ``` -Report Spec: -- Audience: -- Purpose: -- Scope: -- Time Range: -- Geography: -- Required Sections: -- Section Formatting Rules: -- Citation Style: -- Output Format: -- Length Targets: -- Tone: -- Must-Include Sources: -- Must-Exclude Topics: +Enterprise Research Progress: +- [ ] E1: Intake — confirm company entity, research depth, format contract +- [ ] E2: Six-dimension data collection (parallel where possible) + - [ ] D1: Company fundamentals (entity, founding, funding, ownership) + - [ ] D2: Business & products (segments, products, revenue structure) + - [ ] D3: Competitive position (industry rank, competitors, barriers) + - [ ] D4: Financial & operations (3-year financials, efficiency metrics) + - [ ] D5: Recent developments (6-month events, strategic signals) + - [ ] D6: Internal/proprietary sources (or note limitation) +- [ ] E3: Structured analysis frameworks + - [ ] SWOT analysis (evidence-backed, 4 quadrants × 3-5 entries) + - [ ] Competitive barrier quantification (7 dimensions, weighted score) + - [ ] Risk matrix (8 categories, probability × impact) + - [ ] Comprehensive scorecard (6 dimensions, weighted total) +- [ ] E4: L1/L2/L3 quality checks at each stage transition +- [ ] E5: Draft report using 7-chapter enterprise template +- [ ] E6: Multi-pass drafting + UNION merge (same as general Step 6-7) +- [ ] E7: Present draft for human review and iterate ``` -If a user provides a template or an example report, treat it as a hard constraint and mirror the structure. +## P1: Research Task Board + +Decompose the research question into 4-6 investigation tasks (Standard) or 3-4 tasks (Lightweight). + +Each task assignment includes: +- **Expert Role**: Specialist persona (e.g., "Policy Historian", "Ecosystem Mapper") +- **Objective**: One-sentence investigation goal +- **Queries**: 2-3 pre-planned search queries +- **Depth**: DEEP (fetch 2-3 full articles) or SCAN (snippets sufficient) +- **Output**: Path to research notes file +- **Parallel Group**: Group A (independent) or Group B (depends on Group A) + +### Task Decomposition Rules + +1. Each task covers one coherent sub-topic a specialist would own +2. Group A tasks must be independent and source-diverse +3. Max 3 tasks per parallel group (concurrency limit) +4. Every task must flag time-sensitive claims and expected citation aging risk + +### Enterprise Research Integration + +When in Enterprise Research Mode, task board maps to six dimensions: +- Task A: Company fundamentals (entity, founding, funding, ownership) +- Task B: Business & products (segments, products, revenue structure) +- Task C: Competitive position (industry rank, competitors, barriers) +- Task D: Financial & operations (3-year financials, efficiency metrics) +- Task E: Recent developments (6-month events, strategic signals) +- Task F: Internal/proprietary sources (or document limitation) + +Report: `[P1 complete] {N} tasks in {M} groups. Dispatching Group A.` + +--- + +## Enterprise Research Mode (Specialized Pipeline) + +When researching a specific company/enterprise, follow this specialized workflow that ensures six-dimension coverage, quantified analysis frameworks, and three-level quality control. + +### E1: Intake + +Same as P0/P1 above, plus: +- Confirm the exact legal entity being researched (parent vs subsidiary) +- Select research depth: Quick scan (3-5 pages) / Standard (10-20 pages) / Deep (20-40 pages) +- Identify any specific comparison targets (benchmark companies) + +## P2: Dispatch + Investigate + +Subagents execute tasks using [references/subagent_prompt.md](references/subagent_prompt.md) and output to [references/research_notes_format.md](references/research_notes_format.md). + +### With Subagents (Claude Code / Cowork / DeerFlow) + +1. Dispatch Group A tasks in parallel (max 3 concurrent) +2. Each subagent searches, fetches, and tags source types +3. Every source line includes `Source-Type` and `As Of` +4. Wait for Group A completion +5. Dispatch Group B (can read Group A notes) + +### Subagent Output Requirements + +Each task-{id}.md must contain: +- **Sources section**: URLs from actual search results with Source-Type, As Of, Authority (1-10) +- **Findings section**: Max 10 one-sentence facts with source numbers +- **Deep Read Notes** (DEEP tasks): 2-3 sources read in full with key data/insights +- **Gaps section**: What was searched but NOT found, alternative interpretations + +### Without Subagents (Degraded Mode) + +Lead agent executes tasks sequentially, acting as each specialist. Raw search results are discarded after writing notes. + +### Enterprise Research: Six-Dimension Collection + +Follow [references/enterprise_research_methodology.md](references/enterprise_research_methodology.md) for: +- Detailed collection workflow per dimension (query strategies, data fields, validation) +- Data source priority matrix (P0-P3 ranking) +- Cross-validation rules (min sources, max deviation thresholds) + +**Key principles**: +- Evidence-driven: every conclusion must trace to a citable source +- Multi-source validation: key data requires ≥2 independent sources +- Restrained judgment: mark speculation explicitly, avoid unsubstantiated claims +- Structured presentation: complex information via tables, lists, hierarchies + +Run L1 quality check after completing each dimension (see enterprise_quality_checklist.md). + +Status per task: `[P2 task-{id} complete] {N} sources, {M} findings.` +Status all: `[P2 complete] {N} tasks done, {M} total sources. Building registry.` + +### E3: Structured Analysis Frameworks + +Apply frameworks from [references/enterprise_analysis_frameworks.md](references/enterprise_analysis_frameworks.md) in order: +1. **SWOT analysis** — each entry with evidence + source + impact assessment +2. **Competitive barrier quantification** — 7 dimensions with weighted scoring → A+/A/B+/B/C+/C rating +3. **Risk matrix** — 8 mandatory categories, probability × impact → Red/Yellow/Green +4. **Comprehensive scorecard** — 6-dimension weighted total → X/10 + +Run L2 quality check after analysis is complete. -### Step 2: Research Plan and Query Set +### E4: Quality Control -Define the research strategy before calling tools: +Three-level checks from [references/enterprise_quality_checklist.md](references/enterprise_quality_checklist.md): +- **L1 (Data)**: Source count, attribution, cross-validation, timeliness +- **L2 (Analysis)**: SWOT completeness, risk coverage, barrier scoring, conclusion support +- **L3 (Document)**: Structure compliance, format consistency, readability, appendices -- Break the main question into 3-7 subquestions -- Define key entities, keywords, and synonyms -- Identify primary sources vs secondary sources -- Define disqualifiers (outdated, low quality, opinion-only) -- Assemble a query set per section +### E5: Draft Using Enterprise Template -Use [references/research_plan_checklist.md](references/research_plan_checklist.md) for guidance. +Use the 7-chapter enterprise report template from enterprise_quality_checklist.md: +1. Company Overview +2. Business & Product Structure +3. Market & Competitive Position +4. Financial & Operations Analysis +5. Risks & Concerns +6. Recent Developments +7. Comprehensive Assessment & Conclusion -### Step 3: Evidence Collection (Deepresearch Tool) +Plus appendices: Data Source Index, Glossary, Disclaimer. -Use the deepresearch tool to collect evidence and citations. +### E3-E7: Enterprise Analysis, Drafting, and Review -- Run multiple complete passes if coverage is uncertain -- Vary query phrasing to reduce blind spots -- Preserve raw tool output in files for traceability +- **E3: Structured Analysis** — Apply frameworks from [references/enterprise_analysis_frameworks.md](references/enterprise_analysis_frameworks.md) +- **E4: Quality Control** — Run L1/L2/L3 checks per [references/enterprise_quality_checklist.md](references/enterprise_quality_checklist.md) +- **E5: Draft** — Use 7-chapter enterprise template +- **E6-E7: Multi-Pass Drafting and Review** — Same as P4-P7 below + +--- + +## P3: Citation Registry + Source Governance + +Lead agent reads all task notes and builds unified registry. + +### Registry Process + +1. Read every task file's `## Sources` section +2. Merge all sources, deduplicate by URL +3. Assign sequential [n] numbers by first appearance +4. Tag: source_type, as_of date, authority score (1-10), task id +5. **Apply quality gates:** + - Standard: ≥12 approved sources, ≥5 unique domains, ≥30% official + - Lightweight: ≥6 approved sources, ≥3 unique domains, ≥20% official + - Max single-source share: ≤25% (Standard), ≤30% (Lightweight) +6. **Drop sources** below threshold and list them explicitly + +### Registry Output Format -**File structure (recommended):** ``` -/research// - deepresearch_pass1.md - deepresearch_pass2.md - deepresearch_pass3.md +CITATION REGISTRY + +Approved: +[1] Author/Org — Title | URL | Source-Type: official | Accessibility: public | Date: 2026-03-01 | Auth: 8 | task-a +[2] ... + +Dropped: +x Source | URL | Source-Type: community | Accessibility: privileged | Auth: 3 | Reason: PRIVILEGED SOURCE - NOT ALLOWED + +Stats: {approved}/{total}, {N} domains, official_share {xx}% +Privileged sources rejected: {N} ``` -If deepresearch is unavailable, rely on user-provided sources only and state limitations explicitly. +**Critical rule:** These [n] are FINAL. P5 may only cite from Approved list. Dropped sources never reappear. -### Step 4: Source Triage and Evidence Table +**Circular verification handling**: When researching the user's own company/assets, if you discover data in user's private accounts (e.g., user's domain registrar showing they own domains), you MUST: +1. Reject it from the registry (user already knows this) +2. Note it as "CIRCULAR - USER ALREADY KNOWS" in Dropped +3. Search for equivalent PUBLIC sources (e.g., public WHOIS, news articles) +4. Report from external investigator perspective only -Normalize and score sources before drafting: +**Exclusive source handling**: When user EXPLICITLY PROVIDES their paid subscriptions or private APIs for third-party research (e.g., "Use my Crunchbase Pro to research competitors"), you SHOULD: +1. Accept it as "exclusive-user-provided" accessibility +2. Use it as competitive advantage +3. Cite it properly in registry +4. If no public equivalent exists, mark as [unverified] or omit the claim -- De-duplicate sources across passes -- Score sources using [references/source_quality_rubric.md](references/source_quality_rubric.md) -- Build an evidence table mapping claims to sources +Report: `[P3 complete] {approved}/{total} sources. {N} domains. Official share: {xx}%. Privileged rejected: {N}.` -Evidence table minimum columns: +### Handling Information Black Box -- Source ID -- Title -- Publisher -- Date -- URL or reference -- Quality tier (A/B/C) -- Notes +When researching entities with no public footprint (like the "字节跳动子公司" example): -### Step 5: Outline and Section Map +**What an external researcher would find:** +- WHOIS: Privacy protected → No owner info +- Web search: No news, no press releases +- Social media: No company pages +- Business registries: No public API or requires local access +- Result: **Complete information black box** -Create an outline that enforces the format contract: +**Correct response:** +``` +Findings: NO PUBLIC INFORMATION AVAILABLE -- Use the template in [references/research_report_template.md](references/research_report_template.md) -- Produce a section map with required elements per section -- Confirm ordering and headings match the report spec +Sources checked: +- WHOIS (public): Privacy protected [failed] +- Company registry (public): Access denied/No API [failed] +- News media: No coverage [failed] +- Corporate website: Placeholder only [minimal] -### Step 6: Multi-Pass Full Drafting (Parallel Subagents) +Verdict: UNABLE TO VERIFY COMPANY EXISTENCE from external perspective +Sources found: 0 (or minimal, e.g., only WHOIS showing domain exists) +Confidence: N/A - Insufficient evidence +``` -Avoid single-pass drafting; generate multiple complete reports, then merge. +**DO NOT:** +- ❌ Use user's own credentials to "fill in the gaps" +- ❌ Assume the company exists based on domain registration alone +- ❌ Fill missing data with speculation +- ❌ Claim to have "verified" information you accessed through privileged means -#### Preferred Strategy: Parallel Subagents (Complete Draft Each) +**DO:** +- ✅ Clearly state what an external researcher can/cannot verify +- ✅ Document all failed search attempts +- ✅ Mark claims as [unverified] or omit entirely +- ✅ Downgrade mode to Lightweight or stop if insufficient public sources +- ✅ Recommend direct contact for due diligence -Use the Task tool to spawn parallel subagents with isolated context. Each subagent must: +--- -- Load the report spec, outline, and evidence table -- Draft the FULL report (all sections) -- Enforce formatting rules and citation style +## P4: Evidence-Mapped Outline -**Implementation pattern:** +Lead agent reads notes + registry to build outline. + +1. Identify cross-task patterns +2. Design sections topic-first, not task-order-first +3. Map each section to specific findings with source numbers +4. Flag sections needing counter-review +5. Mark recency-sensitive claims with AS_OF checks + +Outline format: ``` -Task(subagent_type="general-purpose", prompt="Draft complete report ...", run_in_background=false) -> version1.md -Task(subagent_type="general-purpose", prompt="Draft complete report ...", run_in_background=false) -> version2.md -Task(subagent_type="general-purpose", prompt="Draft complete report ...", run_in_background=false) -> version3.md +## N. {Section Title} +Sources: [1][3][7] from tasks a, b +Claims: {claim from task-a finding 3}, {claim from task-b finding 1} +Counter-claim candidates: {alternative explanations} +Recency checks: {source dates + AS_OF} +Gaps: {limited official evidence} ``` -**Write drafts to files, not conversation context:** -``` -/intermediate//version1.md -/intermediate//version2.md -/intermediate//version3.md +--- + +## P5: Draft from Notes + +Write section by section using [references/report_template_v6.md](references/report_template_v6.md). + +**Rules:** +- Every factual claim needs citation [n] +- Numbers/percentages must have source +- Add **confidence marker** per section: High/Medium/Low with rationale +- Add **counter-claim sentence** when evidence conflicts +- No new sources may be introduced +- Use [unverified] for unsupported statements + +**Anti-hallucination:** +- Lead agent never invents URLs — only from subagent notes +- Lead agent never fabricates data — mark [unverified] if number not in notes + +Status: `[P5 in progress] {N}/{M} sections, ~{words} words.` + +--- + +## P6: Counter-Review (Mandatory) + +For each major conclusion, perform opposite-view checks: + +1. **Could the conclusion be wrong?** +2. **Which high-impact claims depend on a single source?** +3. **Which claims lack official/academic support?** +4. **Are stale sources used for time-sensitive claims?** +5. **Find ≥3 issues** (re-examine if 0 found) + +### Using Counter-Review Team (Recommended) + +For comprehensive parallel review, use the Counter-Review Team: + +```bash +# 1. Prepare inputs +counter-review-inputs/ + ├── draft_report.md + ├── citation_registry.md + ├── task-notes/ + └── p0_config.md + +# 2. Dispatch to 4 specialist agents in parallel +SendMessage to: claim-validator +SendMessage to: source-diversity-checker +SendMessage to: recency-validator +SendMessage to: contradiction-finder + +# 3. Wait for all specialists to complete + +# 4. Send to coordinator for synthesis +SendMessage to: counter-review-coordinator + inputs: [4 specialist reports] + +# 5. Receive final P6 Counter-Review Report ``` -### Step 7: UNION Merge and Format Compliance +See [references/counter_review_team_guide.md](references/counter_review_team_guide.md) for detailed usage. -Merge using UNION, never remove content without evidence-based justification: +### Manual Counter-Review (Fallback) -- Keep all unique findings from all versions -- Consolidate duplicates while preserving the most detailed phrasing -- Ensure every claim in the merged draft has a cited source -- Enforce the exact section order, headings, and formatting -- Re-run formatting rules from [references/formatting_rules.md](references/formatting_rules.md) +If Counter-Review Team is unavailable, perform manual checks: +- Verify every high-confidence claim has ≥2 sources +- Check official/academic backing for key claims +- Verify AS_OF dates on time-sensitive claims +- Document opposing interpretations + +### Output + +Include in final report: +``` +## 核心争议 / Key Controversies +- **争议 1:** [主张 A 与反向证据 B 对比] [n][m] +- **争议 2:** ... +``` -### Step 8: Evidence and Citation Verification +Report: `[P6 complete] {N} issues found: {critical} critical, {high} high, {medium} medium.` -Verify traceability: +--- -- Every numeric claim has at least one source -- Every recommendation references supporting evidence -- No orphan claims without citations -- Dates and time ranges are consistent -- Conflicts are explicitly called out with both sources +## P7: Verify -Use [references/completeness_review_checklist.md](references/completeness_review_checklist.md). +Cross-check before finalization: -### Step 9: Present Draft for Human Review and Iterate +1. **Registry cross-check:** List every [n] in report vs approved registry +2. **Spot-check 5+ claims:** Trace to task notes +3. **Remove/fix non-traceable claims** +4. **Validate no dropped source resurrected** +5. **Check source concentration** for key claims -Present the draft as a reviewable version: +Report: `[P7 complete] {N} spot-checks, {M} violations fixed.` -- Emphasize that format compliance and factual accuracy need human review -- Accept edits to format, structure, and scope -- If the user provides another AI output, cross-compare and UNION merge +--- ## Output Requirements @@ -188,6 +484,18 @@ Present the draft as a reviewable version: ## Reference Files +### Core V6 Pipeline References + +| File | When to Load | +| --- | --- | +| [source_accessibility_policy.md](references/source_accessibility_policy.md) | **P0 (CRITICAL)**: Source classification rules - read first | +| [subagent_prompt.md](references/subagent_prompt.md) | P2: Task dispatch to subagents | +| [research_notes_format.md](references/research_notes_format.md) | P2: Subagent output format | +| [report_template_v6.md](references/report_template_v6.md) | P5: Draft with confidence markers and counter-review | +| [quality_gates.md](references/quality_gates.md) | All phases: Quality thresholds and anti-hallucination checks | + +### General Research References + | File | When to Load | | --- | --- | | [research_report_template.md](references/research_report_template.md) | Build outline and draft structure | @@ -196,6 +504,14 @@ Present the draft as a reviewable version: | [research_plan_checklist.md](references/research_plan_checklist.md) | Build research plan and query set | | [completeness_review_checklist.md](references/completeness_review_checklist.md) | Review for coverage, citations, and compliance | +### Enterprise Research References (load when in Enterprise Research Mode) + +| File | When to Load | +| --- | --- | +| [enterprise_research_methodology.md](references/enterprise_research_methodology.md) | Six-dimension data collection workflow, source priority, cross-validation rules | +| [enterprise_analysis_frameworks.md](references/enterprise_analysis_frameworks.md) | SWOT template, competitive barrier quantification, risk matrix, comprehensive scoring | +| [enterprise_quality_checklist.md](references/enterprise_quality_checklist.md) | L1/L2/L3 quality checks, per-dimension checklists, 7-chapter report template | + ## Anti-Patterns - Single-pass drafting without parallel complete passes @@ -205,3 +521,10 @@ Present the draft as a reviewable version: - Mixing conflicting dates without calling out discrepancies - Copying external AI output without verification - Deleting intermediate drafts or raw research outputs +- **Lead agent reading raw search results** — only read subagent notes +- **Inventing URLs** — only use URLs from actual search results +- **Resurrecting dropped sources** — dropped in P3 never reappear +- **Missing AS_OF for time-sensitive claims** — always include source date +- **Skipping counter-review** — mandatory P6 must find ≥3 issues +- **CIRCULAR VERIFICATION** — never use user's private data to "discover" what they already know about themselves +- **IGNORING EXCLUSIVE SOURCES** — when user provides Crunchbase Pro etc. for competitor research, USE IT diff --git a/deep-research/references/V6_1_improvements.md b/deep-research/references/V6_1_improvements.md new file mode 100644 index 00000000..c04722a6 --- /dev/null +++ b/deep-research/references/V6_1_improvements.md @@ -0,0 +1,112 @@ +# Deep Research Skill V6.1 Improvements + +**Date**: 2026-04-03 +**Version**: 2.3.0 → 2.4.0 +**Based on**: User feedback and "字节跳动" case study + +--- + +## Summary of Changes + +### 1. Source Accessibility Policy - Critical Correction + +**Problem Identified**: +Previously, we incorrectly banned all "privileged" sources. This was wrong because it prevented users from leveraging their competitive information advantages. + +**The Real Issue**: +The problem is not using user's private information—it's **circular verification**: using user's data to "discover" what they already know about themselves. + +**Example of the Error**: +``` +User: "Research my company 字节跳动子公司" +❌ WRONG: Access user's Spaceship → "You own 25 domains" + → This is circular: user already knows they own these domains + +✅ RIGHT: Check public WHOIS → "Privacy protected, ownership not visible" + → This is external research perspective +``` + +**Correct Classification**: + +| Accessibility | For Self-Research | For Third-Party Research | +|--------------|-------------------|-------------------------| +| `public` | ✅ Use | ✅ Use | +| `semi-public` | ✅ Use | ✅ Use | +| `exclusive-user-provided` | ⚠️ Careful* | ✅ **ENCOURAGED** | +| `private-user-owned` | ❌ **FORBIDDEN** | N/A | + +\* When user provides exclusive sources for their own company, evaluate if it's circular + +### 2. Counter-Review Team V2 + +**Created**: 5-agent parallel review team +- 🔵 claim-validator: Claim validation +- 🟢 source-diversity-checker: Source diversity analysis +- 🟡 recency-validator: Recency/freshness checks +- 🟣 contradiction-finder: Contradiction and bias detection +- 🟠 counter-review-coordinator: Synthesis and reporting + +**Usage**: +```bash +# 1. Dispatch to 4 specialists in parallel +SendMessage to: claim-validator +SendMessage to: source-diversity-checker +SendMessage to: recency-validator +SendMessage to: contradiction-finder + +# 2. Send to coordinator for synthesis +SendMessage to: counter-review-coordinator +``` + +### 3. Methodology Clarifications + +#### When Researching User's Own Company +- **Approach**: External investigator perspective +- **Use**: Public sources only +- **Do NOT use**: User's private accounts (creates circular verification) +- **Report**: "From public perspective: X, Y, Z gaps" + +#### When User Provides Exclusive Sources for Third-Party Research +- **Approach**: Leverage competitive advantage +- **Use**: User's paid subscriptions, private APIs, proprietary databases +- **Cite**: Mark as `exclusive-user-provided` +- **Report**: "Per user's exclusive source [Crunchbase Pro], competitor X raised $Y" + +### 4. Registry Format Update + +**Added fields**: +- `Accessibility`: public / semi-public / exclusive-user-provided / private-user-owned +- `Circular rejection tracking`: Note when sources are rejected for circular verification + +**Updated anti-patterns**: +- ❌ **CIRCULAR VERIFICATION**: Never use user's private data to "discover" what they already know +- ✅ **USE EXCLUSIVE SOURCES**: When user provides Crunchbase Pro etc. for competitor research, USE IT + +### 5. Documentation Updates + +**New/Updated Files**: +- `source_accessibility_policy.md`: Complete rewrite explaining circular vs. competitive advantage distinction +- `counter_review_team_guide.md`: Usage guide for the 5-agent team +- `SKILL.md`: Updated Source Governance section with correct classification +- `marketplace.json`: Updated description + +--- + +## Key Principles Summary + +1. **Circular Verification is Bad**: Don't use user's data to tell them what they already know +2. **Exclusive Information Advantage is Good**: Use user's paid tools to research competitors +3. **External Perspective for Self-Research**: When researching user's own company, act like an external investigator +4. **Leverage Everything for Third-Party**: When researching others, use every advantage user provides + +--- + +## Version History + +| Version | Changes | +|---------|---------| +| 2.0.0 | Initial Enterprise Research Mode | +| 2.1.0 | V6 features: source governance, AS_OF, counter-review | +| 2.2.0 | Counter-Review Team | +| 2.3.0 | Source accessibility (initial, incorrect ban on privileged) | +| **2.4.0** | **Corrected: circular vs. exclusive advantage distinction** | diff --git a/deep-research/references/counter_review_team_guide.md b/deep-research/references/counter_review_team_guide.md new file mode 100644 index 00000000..3b85627a --- /dev/null +++ b/deep-research/references/counter_review_team_guide.md @@ -0,0 +1,181 @@ +# Counter-Review Team 使用指南 + +Deep Research V6 P6 阶段的专用 Agent Team，并行执行多维度审查。 + +## Team 架构 + +``` +counter-review-coordinator (协调者) + ├── claim-validator (声明验证器) + ├── source-diversity-checker (来源多样性检查器) + ├── recency-validator (时效性验证器) + └── contradiction-finder (矛盾发现器) +``` + +## Agent 职责 + +| Agent | 职责 | 输出 | +|-------|------|------| +| **claim-validator** | 验证声明准确性，识别无证据/弱证据声明 | Claim Validation Report | +| **source-diversity-checker** | 检查单一来源依赖，source-type 分布 | Source Diversity Report | +| **recency-validator** | 验证时敏声明的新鲜度，AS_OF 合规 | Recency Validation Report | +| **contradiction-finder** | 发现内部矛盾，缺失的反向观点 | Contradiction and Bias Report | +| **counter-review-coordinator** | 整合所有报告，生成最终 P6 报告 | P6 Counter-Review Report | + +## 使用流程 + +### 1. 准备输入材料 + +在 P5 (Draft) 完成后，收集以下材料： + +``` +inputs/ +├── draft_report.md # P5 起草的报告 +├── citation_registry.md # P3 的引用注册表 +├── task-notes/ +│ ├── task-a.md # 子代理研究笔记 +│ ├── task-b.md +│ └── ... +└── p0_config.md # P0 配置 (AS_OF 日期, Mode 等) +``` + +### 2. 并行分发任务 + +向 4 个 specialist agent 同时发送任务： + +```bash +# 向 claim-validator 发送 +SendMessage to: claim-validator + 输入: draft_report.md + citation_registry.md + task-notes/ + 指令: 验证所有声明的证据支持 + +# 向 source-diversity-checker 发送 +SendMessage to: source-diversity-checker + 输入: draft_report.md + citation_registry.md + 指令: 检查来源多样性和单一来源依赖 + +# 向 recency-validator 发送 +SendMessage to: recency-validator + 输入: draft_report.md + citation_registry.md + p0_config.md + 指令: 验证时敏声明的新鲜度 + +# 向 contradiction-finder 发送 +SendMessage to: contradiction-finder + 输入: draft_report.md + task-notes/ + citation_registry.md + 指令: 发现矛盾和缺失的反向观点 +``` + +### 3. 协调汇总 + +等待 4 个 specialist 完成后，发送给 coordinator： + +```bash +SendMessage to: counter-review-coordinator + 输入: + - Claim Validation Report + - Source Diversity Report + - Recency Validation Report + - Contradiction and Bias Report + 指令: 整合所有报告，生成最终 P6 Counter-Review Report +``` + +### 4. 获取最终输出 + +Coordinator 输出包含： +- 问题汇总（必须 ≥3 个） +- 关键争议部分（可直接复制到最终报告） +- 强制修复清单 +- 质量门状态 + +## 质量门要求 + +| 检查项 | 标准模式 | 轻量模式 | 失败处理 | +|--------|---------|---------|---------| +| 发现问题数 | ≥3 | ≥3 | 重新审查 | +| 关键声明单来源 | 0 | 0 | 补充来源或降级 | +| 官方来源占比 | ≥30% | ≥20% | 补充官方来源 | +| AS_OF 日期完整 | 100% | 100% | 补充日期 | +| 核心争议文档化 | 必填 | 必填 | 补充争议部分 | + +## 输出示例 + +### Coordinator 最终报告结构 + +```markdown +# P6 Counter-Review Report + +## Executive Summary +- Total issues found: 7 (critical: 2, high: 3, medium: 2) +- Must-fix before publish: 2 +- Recommended improvements: 5 + +## Critical Issues (Block Publish) +| # | Issue | Location | Source | Fix Required | +|---|-------|----------|--------|--------------| +| 1 | 市场份额声明无来源 | 3.2节 | 无 | 补充来源或删除 | +| 2 | 单一社区来源支持收入数据 | 4.1节 | [12] community | 找官方来源替代 | + +## 核心争议 / Key Controversies + +- **争议 1:** 公司声称增长 50% vs 分析师报告增长 30% + - 证据强度: official(公司财报) vs academic(第三方研究) + - 建议: 并列呈现两种数据，说明差异原因 + +## Mandatory Fixes Checklist +- [ ] 补充 3.2 节市场份额来源 +- [ ] 替换 4.1 节收入数据来源 +- [ ] 添加 AS_OF: 2026-04-03 到所有时敏声明 + +## Quality Gates Status +| Gate | Status | Notes | +|------|--------|-------| +| P6 ≥3 issues found | ✅ | 发现 7 个问题 | +| No critical claim single-sourced | ❌ | 2 个问题待修复 | +| AS_OF dates present | ❌ | 3 处缺失 | +| Counter-claims documented | ✅ | 已添加 | +``` + +## 集成到 SKILL.md 工作流 + +在 SKILL.md 的 P6 阶段，添加以下指令： + +```markdown +## P6: Counter-Review (Mandatory) + +**使用 Counter-Review Team 执行并行审查：** + +1. **准备材料**: draft_report.md, citation_registry.md, task-notes/, p0_config.md +2. **并行分发**: 同时发送给 4 个 specialist agent +3. **等待完成**: 收集 4 份 specialist 报告 +4. **协调汇总**: 发送给 coordinator 生成最终 P6 报告 +5. **强制执行**: 所有 Critical 问题必须在 P7 前修复 +6. **输出**: 将"核心争议"部分复制到最终报告 + +**Report**: `[P6 complete] {N} issues found: {critical} critical, {high} high, {medium} medium.` +``` + +## 团队管理 + +### 查看团队状态 +```bash +cat ~/.claude/teams/counter-review-team/config.json +``` + +### 向 Agent 发送消息 +```bash +SendMessage to: claim-validator +message: 开始审查任务，输入文件在 ./review-inputs/ +``` + +### 关闭团队 +```bash +SendMessage to: "*" +message: {"type": "shutdown_request", "reason": "任务完成"} +``` + +## 注意事项 + +1. **必须发现 ≥3 个问题** - 如果 coordinator 报告 <3 个问题，需要重新审查 +2. **Critical 问题必须修复** - 才能进入 P7 +3. **保留所有审查记录** - 作为研究方法论的一部分 +4. **中文输入中文输出** - 所有 agent 支持中英文双语 diff --git a/deep-research/references/enterprise_analysis_frameworks.md b/deep-research/references/enterprise_analysis_frameworks.md new file mode 100644 index 00000000..209c03aa --- /dev/null +++ b/deep-research/references/enterprise_analysis_frameworks.md @@ -0,0 +1,135 @@ +# Enterprise Analysis Frameworks + +Apply these frameworks after completing the six-dimension data collection. Execute in order: SWOT → Competitive Barriers → Risk Matrix → Comprehensive Scoring. + +## SWOT Analysis Template + +Each SWOT entry MUST include evidence and source attribution. + +``` +| | Positive Factors | Negative Factors | +|--------------|-----------------------------------|-----------------------------------| +| **Internal** | **S (Strengths)** | **W (Weaknesses)** | +| | 1. {description} | 1. {description} | +| | • Evidence: {data/fact} | • Evidence: {data/fact} | +| | • Source: {citation} | • Source: {citation} | +| | • Impact: {assessment} | • Impact: {assessment} | +| | | | +| **External** | **O (Opportunities)** | **T (Threats)** | +| | 1. {description} | 1. {description} | +| | • Evidence: {trend/policy} | • Evidence: {pressure/risk} | +| | • Source: {citation} | • Source: {citation} | +| | • Probability: {assessment} | • Probability: {assessment} | +| | • Impact: {assessment} | • Impact: {assessment} | +``` + +**Requirements**: +- Each quadrant: 3-5 entries minimum +- Every entry must have evidence with source +- S/W must be data-backed (not opinions) +- O/T must include probability and impact estimates + +**Strategic Implications Matrix** (generate after SWOT): +- **SO Strategy** (leverage strengths to capture opportunities): 1-2 specific recommendations +- **WO Strategy** (overcome weaknesses to seize opportunities): 1-2 specific recommendations +- **ST Strategy** (use strengths to counter threats): 1-2 specific recommendations +- **WT Strategy** (mitigate weaknesses to avoid threats): 1-2 specific recommendations + +## Competitive Barrier Quantification Framework + +7 barrier dimensions with weighted scoring: + +| Dimension | Weight | Strong | Moderate | Weak | +|-----------|--------|--------|----------|------| +| **Network Effects** | 20% | 4.5 — Clear network effects (social platforms, marketplaces) | 3.0 — Exists but replaceable | 1.5 — Minimal network effects | +| **Scale Economies** | 15% | 4.0 — Unit cost drops 30%+ with scale | 2.5 — Cost drops 10-30% | 1.0 — Cost drops <10% | +| **Brand Value** | 15% | 4.0 — Category leader, high pricing power | 2.5 — Known brand, competitive | 1.0 — Commodity brand, price-sensitive | +| **Technology/Patents** | 15% | 4.0 — Core patents, hard to circumvent | 2.5 — Some patent protection | 1.0 — Peripheral patents only | +| **Switching Costs** | 15% | 4.0 — High lock-in (data, ecosystem) | 2.5 — Moderate switching friction | 1.0 — Low switching cost | +| **Regulatory Licenses** | 10% | 3.5 — Heavy regulation, hard to obtain | 2.0 — Standard regulatory requirements | 0.5 — Light regulation | +| **Data Assets** | 10% | 3.5 — Massive proprietary high-quality data | 2.0 — Some data accumulation | 0.5 — Limited or public data | + +**Scoring**: Total = Σ(dimension score × weight) + +**Rating Scale**: +| Score | Rating | Interpretation | +|-------|--------|---------------| +| ≥3.5 | A+ | Exceptional moat | +| ≥2.8 | A | Strong moat | +| ≥2.0 | B+ | Good moat | +| ≥1.5 | B | Moderate moat | +| ≥1.0 | C+ | Limited moat | +| <1.0 | C | Weak moat | + +**Output format**: Present a scorecard table with each dimension's strength rating, raw score, justification (with evidence), and the weighted total with final rating. + +## Risk Matrix Framework + +Assess 8 mandatory risk categories: + +### Risk Assessment Scales + +**Probability**: +| Level | Range | Score | +|-------|-------|-------| +| High | >70% | 0.7-1.0 | +| Medium | 30-70% | 0.3-0.7 | +| Low | <30% | 0.0-0.3 | + +**Impact**: +| Level | Description | Score | +|-------|-------------|-------| +| High | >30% revenue impact | 3 | +| Medium | 10-30% revenue impact | 2 | +| Low | <10% revenue impact | 1 | + +**Risk Level**: Risk Value = Probability Score × Impact Score +| Color | Level | Threshold | +|-------|-------|-----------| +| Red | High risk | ≥2.5 | +| Yellow | Medium risk | 1.0 – 2.5 | +| Green | Low risk | <1.0 | + +### 8 Mandatory Risk Categories + +| # | Category | Typical Triggers | +|---|----------|-----------------| +| 1 | Market risk | Industry slowdown, demand shifts | +| 2 | Competitive risk | New entrants, incumbents pivoting | +| 3 | Technology risk | Tech obsolescence, disruption | +| 4 | Regulatory risk | Policy tightening, compliance cost | +| 5 | Financial risk | Cash flow stress, debt levels | +| 6 | Operational risk | Key talent loss, supply chain | +| 7 | Talent risk | Brain drain, recruiting difficulty | +| 8 | Geopolitical risk | Trade friction, data localization | + +### Risk Table Format + +| Category | Specific Risk | Probability | Impact | Risk Value | Level | Evidence/Triggers | Current Mitigations | Recommended Actions | +|----------|--------------|-------------|--------|------------|-------|-------------------|--------------------|--------------------| + +**Requirements**: +- All 8 categories must be assessed (no skipping) +- Each risk entry must cite specific evidence or triggers +- Provide current mitigations AND recommended actions +- High risks: require immediate action plans +- Medium risks: require monitoring plans +- Low risks: require periodic review schedule + +## Comprehensive Scoring (Final Section) + +After completing SWOT, barriers, and risk matrix, generate a comprehensive scorecard: + +``` +| Dimension | Score | Weight | Weighted | Key Evidence | +|-----------|-------|--------|----------|-------------| +| Business Quality | X/10 | 25% | | | +| Competitive Position | X/10 | 20% | | | +| Financial Health | X/10 | 20% | | | +| Growth Potential | X/10 | 15% | | | +| Risk Profile | X/10 | 10% | | | +| Management Quality | X/10 | 10% | | | +| **Total** | | 100% | **X/10** | | +``` + +Every score must reference specific evidence from the six-dimension data collection. diff --git a/deep-research/references/enterprise_quality_checklist.md b/deep-research/references/enterprise_quality_checklist.md new file mode 100644 index 00000000..b09d9255 --- /dev/null +++ b/deep-research/references/enterprise_quality_checklist.md @@ -0,0 +1,160 @@ +# Enterprise Research Quality Checklist + +Three-level quality control executed at each stage transition. + +## L1: Data Collection Quality (after each dimension) + +### Per-Dimension Checks + +| Check Item | Standard | Method | Pass Condition | +|-----------|----------|--------|---------------| +| Source count | Key data points ≥2 sources | Count source annotations | ≥90% compliance | +| Source attribution | All data has source marked | Check citations in draft | ≥95% completeness | +| Cross-validation pass rate | Data deviation ≤10% | Compare multi-source data | ≥95% validation pass | +| Timeliness | Financial: ≤2 years; News: ≤6 months | Check timestamps | 100% compliance | + +**Result handling**: All pass → proceed. Partial fail → supplement sources. Critical fail → re-collect dimension. + +### Dimension-Specific Checklists + +**D1 Company Fundamentals** (target: 11/11): +- [ ] Legal entity boundaries clarified +- [ ] Founding date with month/year +- [ ] Headquarters city identified +- [ ] Founder/CEO confirmed (≥2 sources) +- [ ] Employee count with year +- [ ] Listing status (exchange, ticker) +- [ ] Latest valuation/market cap with date +- [ ] Core business one-liner +- [ ] Funding history ≥3 rounds +- [ ] ≥5 milestone events in timeline +- [ ] Ownership structure: controller identified + +**D2 Business & Products** (target: 7/7): +- [ ] ≥3 business segments identified +- [ ] Revenue share per segment +- [ ] ≥3 core products analyzed +- [ ] User metrics (DAU/MAU) with numbers +- [ ] Monetization model per product +- [ ] Revenue breakdown (segment/geography/customer) +- [ ] Growth/decline trend per segment + +**D3 Competitive Position** (target: 7/7): +- [ ] Industry clearly defined +- [ ] Market size quantified +- [ ] Company rank established +- [ ] Market share with number +- [ ] ≥3 competitors identified +- [ ] Multi-dimension comparison table complete +- [ ] ≥5 barrier dimensions assessed with scores + +**D4 Financial & Operations** (target: 9/9): +- [ ] Revenue: 3-year data +- [ ] Net income: 3-year data +- [ ] Gross margin: 3-year data +- [ ] Net margin: 3-year data +- [ ] Operating cash flow: 3-year data +- [ ] R&D expense: 3-year data +- [ ] Key financial data cross-validated (≥2 sources) +- [ ] Metric definitions consistent across years +- [ ] ≥3 efficiency metrics (ROE/ROA/etc.) + +**D5 Recent Developments** (target: 5/5): +- [ ] ≥5 recent events (within 6 months) +- [ ] Events span ≥3 event types +- [ ] Each event has impact assessment +- [ ] ≥2 strategic direction signals identified +- [ ] Most recent event within 1 month + +**D6 Internal/Proprietary** (target: 2/2): +- [ ] Internal knowledge base queried (or limitation noted) +- [ ] Internal document search executed (or limitation noted) + +## L2: Analysis Quality (after analysis frameworks applied) + +| Check Item | Standard | Method | Pass Condition | +|-----------|----------|--------|---------------| +| SWOT completeness | Each quadrant ≥3 entries | Entry count | Full coverage | +| SWOT evidence | Every entry has data backing | Check "Evidence" fields | 100% evidenced | +| Risk matrix coverage | All 8 categories assessed | Category checklist | 100% covered | +| Barrier quantification | All 7 dimensions scored | Check scorecard completeness | 100% scored | +| Conclusion support | All conclusions trace to evidence | Trace each conclusion | 100% supported | + +**Result handling**: All pass → proceed to writing. Partial fail → supplement analysis evidence. Critical fail → re-execute analysis framework. + +## L3: Document Quality (after report drafted) + +| Check Item | Standard | Method | Pass Condition | +|-----------|----------|--------|---------------| +| Structure compliance | Follows 7-chapter template | Compare against template | ≥95% compliance | +| Table format consistency | All tables uniformly formatted | Visual inspection | 100% uniform | +| Readability | Paragraphs ≤450 chars; ≥3 parallel items use lists | Paragraph length check | ≥95% compliance | +| Data annotation | All data has source + year | Citation audit | 100% complete | +| Appendix completeness | Includes source index + glossary | Content check | 100% complete | + +**Result handling**: All pass → deliver. Partial fail → format optimization. Critical fail → regenerate document. + +## Enterprise Report Structure (7 Chapters) + +``` +# {Company Name} Research Report + +> Executive Summary: {1-2 sentence core conclusion} + +--- + +## 1. Company Overview +### 1.1 Basic Information (table) +### 1.2 Development Timeline +### 1.3 Funding History (table) +### 1.4 Ownership Structure & Control +### 1.5 Core Management Team (table) + +## 2. Business & Product Structure +### 2.1 Business Landscape Overview +### 2.2 Core Product Matrix (table) +### 2.3 Revenue Structure Analysis +### 2.4 Business Development Trends + +## 3. Market & Competitive Position +### 3.1 Industry Position Analysis +### 3.2 Competitive Comparison (table) +### 3.3 Competitive Barrier Assessment (scorecard) + +## 4. Financial & Operations Analysis +### 4.1 Key Financial Metrics (3-year comparison table) +### 4.2 Operating Efficiency Assessment +### 4.3 Financial Health Summary + +## 5. Risks & Concerns +### 5.1 Risk Matrix Analysis (8-category table) +### 5.2 Key Risk Deep-Dives +### 5.3 Risk Mitigation Recommendations + +## 6. Recent Developments +### 6.1 Major Recent Events (table) +### 6.2 Strategic Signal Interpretation + +## 7. Comprehensive Assessment & Conclusion +### 7.1 SWOT Summary +### 7.2 Comprehensive Scorecard +### 7.3 Core Conclusions & Outlook + +--- + +## Appendices +### A. Data Source Index +### B. Glossary +### C. Disclaimer +``` + +## Quality Control Four Dimensions + +Apply throughout all stages: + +| Dimension | Focus | Key Checks | +|-----------|-------|------------| +| **Accuracy** | Data correctness | Source attribution, fact verification, cross-validation, error tolerance | +| **Completeness** | Information coverage | Dimension coverage, key element presence, conclusion support, risk coverage | +| **Timeliness** | Data currency | Data freshness, trend capture, signal detection, dynamic updates | +| **Consistency** | Uniform standards | Metric definitions aligned, format unified, style consistent, terminology standardized | diff --git a/deep-research/references/enterprise_research_methodology.md b/deep-research/references/enterprise_research_methodology.md new file mode 100644 index 00000000..be8680e4 --- /dev/null +++ b/deep-research/references/enterprise_research_methodology.md @@ -0,0 +1,164 @@ +# Enterprise Research Methodology + +## Six-Dimension Data Collection + +Enterprise research requires parallel collection across six dimensions. Execute all six in order, writing findings to a structured draft after each dimension. + +### Dimension 1: Company Fundamentals + +``` +Step 1.1: Confirm legal entity +├── Clarify parent/subsidiary/affiliate boundaries +├── Query: "{company} legal entity corporate structure" +├── Output: Entity scope statement +└── Verify: Map operating entities to brands + +Step 1.2: Basic information +├── Query round 1: "{company} founding date headquarters founder" +├── Query round 2: "{company} company overview profile" +├── Query round 3: "{company} CEO management team executives" +├── Source priority: Official site > Regulatory filings > Authoritative media +└── Output: Basic info table (name, founded, HQ, CEO, employees, listing status) + +Step 1.3: Funding history +├── Query: "{company} funding rounds valuation IPO" +├── Key fields: round, amount, investors, post-money valuation, date +└── Output: Funding timeline table + +Step 1.4: Ownership structure +├── Query: "{company} ownership structure beneficial owner" +├── Key fields: controller identity, economic interest %, voting rights %, control mechanisms (dual-class etc.) +└── Output: Ownership summary +``` + +### Dimension 2: Business & Products + +``` +Step 2.1: Business landscape scan +├── Query round 1: "{company} product lines business segments" +├── Query round 2: "{company} revenue breakdown by segment" +├── Query round 3: "{company} business model monetization" +├── Key fields: segment name, positioning, revenue share, YoY growth, synergies +└── Output: Business landscape table + +Step 2.2: Core product analysis +├── Query: "{company} core products DAU MAU user base" +├── Per product: positioning, target users, scale (DAU/MAU), market share, monetization, competitive advantage, trends +└── Output: Product matrix table + +Step 2.3: Revenue structure analysis +├── Source: Financial reports (deep extraction) +├── Breakdown by: segment, geography, customer type, pricing model +└── Output: Revenue structure summary +``` + +### Dimension 3: Competitive Position + +``` +Step 3.1: Industry position +├── Query: "{company} industry ranking market share" +├── Key fields: industry definition, TAM/SAM/SOM, company rank, share, concentration (CR3/CR5) +└── Output: Industry position analysis + +Step 3.2: Competitor identification & comparison +├── Query round 1: "{company} competitors" +├── Query round 2: "{company} vs {competitor A} comparison" +├── Query round 3: "{company} vs {competitor B} differences" +├── Comparison dimensions: founding, revenue, market share, core products, user scale, valuation/market cap, strengths, weaknesses +├── Minimum: ≥3 competitors identified +└── Output: Competitive comparison table + +Step 3.3: Competitive barriers assessment +├── Use quantified barrier framework (see enterprise_analysis_frameworks.md) +├── 7 dimensions: network effects, scale economies, brand, technology/patents, switching costs, regulatory licenses, data assets +└── Output: Barrier scorecard with rating +``` + +### Dimension 4: Financial & Operations + +``` +Step 4.1: Financial data collection +├── Query: "{company} financial results {year} revenue profit" +├── Core metrics (3-year minimum): revenue, revenue growth, net income, gross margin, net margin, operating cash flow, R&D expense, R&D ratio +└── Output: Financial metrics table (3+ years) + +Step 4.2: Operating efficiency analysis +├── Query: "{company} ROE ROA efficiency per-employee" +├── Efficiency metrics: ROE, ROA, revenue per employee, accounts receivable days, debt-to-equity +└── Output: Operating efficiency table + +Step 4.3: Cross-validation +├── Require ≥2 independent sources for key financial data +├── Sources: company filings (primary), regulatory filings, authoritative financial data providers +├── Deviation rules: +│ ├── ≤10%: Pass +│ ├── 10-20%: Flag with explanation +│ └── >20%: Require third-party verification +└── Output: Validation record +``` + +### Dimension 5: Recent Developments + +``` +Step 5.1: Recent news scan (past 6 months) +├── Query round 1: "{company} latest news {current year}" +├── Query round 2: "{company} strategy pivot latest developments" +├── Query round 3: "{company} executive changes leadership" +├── Query round 4: "{company} partnership acquisition latest" +├── Query round 5: "{company} product launch new release" +├── Event types: product launches, fundraising/capital, strategy shifts, executive changes, M&A/partnerships, regulatory/compliance +├── Minimum: ≥5 events identified +└── Output: Major events table + +Step 5.2: Strategic signal interpretation +├── Dimensions: expansion signals, contraction signals, transformation signals, risk signals +└── Output: Strategic signal analysis +``` + +### Dimension 6: Internal/Proprietary Sources + +``` +Step 6.1: Internal knowledge base query (if available) +├── Query 1: "our company's relationship with {target company}" +├── Query 2: "internal assessment of {target company}" +├── Query 3: "{target company} competitive analysis" +├── Query 4: "{target company} industry research" +└── Output: Internal perspective supplementary info + +Step 6.2: If no internal sources available +├── State explicitly: "No internal/proprietary sources available for this research" +├── Compensate with additional public source depth +└── Note limitation in final report +``` + +## Data Source Priority Matrix + +| Priority | Source Type | Reliability | Timeliness | Use Case | +|----------|-----------|-------------|------------|----------| +| **P0** | Official filings / annual reports | 10/10 | High | Core financial data | +| **P0** | Company website / announcements | 10/10 | High | Basic info, updates | +| **P1** | Regulatory filings | 9/10 | High | Ownership, licenses | +| **P1** | Authoritative industry reports | 9/10 | Medium | Market position, trends | +| **P2** | Mainstream financial media | 8/10 | High | News, analysis | +| **P2** | Professional research institutions | 8/10 | Medium | Deep analysis, forecasts | +| **P3** | Social media / forums | 5/10 | High | Sentiment signals only | + +**Rule**: P0 + P1 are primary sources. P2 for validation. P3 for reference only, never as sole source. + +## Cross-Validation Rules + +| Data Type | Min Sources | Max Deviation | Primary Source | Fallback Sources | +|-----------|------------|---------------|----------------|-----------------| +| Financial data | 2 | 10% | Official financial reports | Regulatory filings, analyst reports | +| Market share | 2 | 15% | Industry reports | Company disclosures, third-party analysis | +| Management info | 1 | N/A | Company official sources | Regulatory filings, reputable media | +| User metrics | 2 | 20% | Company disclosures | Third-party analytics, industry reports | + +## Search Strategy Best Practices + +1. **Multi-angle queries**: 3 different query angles per topic +2. **Time filtering**: Prioritize data within last 12 months for operational data, last 3 years for financial trends +3. **Site restriction**: Use `site:` for authoritative domains when possible +4. **Language diversity**: Query in both English and the company's primary language +5. **Exclude noise**: Use `-` to exclude irrelevant results +6. **Progressive depth**: Start broad, then narrow based on gaps identified diff --git a/deep-research/references/quality_gates.md b/deep-research/references/quality_gates.md new file mode 100644 index 00000000..b92488a6 --- /dev/null +++ b/deep-research/references/quality_gates.md @@ -0,0 +1,77 @@ +# Quality Gates V6 + +## Gate 1: Task Notes Quality (after P2) + +| Check | Standard | Lightweight | Fix | +|-------|----------|-------------|-----| +| All tasks completed | 100% | 100% | Re-dispatch failed tasks | +| Sources per task | >= 2 | >= 1 | Run additional searches | +| Findings per task | >= 3 | >= 2 | Deepen search or fetch more | +| DEEP tasks have Deep Read Notes | 100% | 100% | Fetch and read top source | +| All source URLs from actual search | 100% | 100% | Remove any invented URL | + +## Gate 2: Citation Registry (after P3) + +| Check | Standard | Lightweight | Fix | +|-------|----------|-------------|-----| +| Total approved sources | >= 12 | >= 6 | Flag thin areas for P6 | +| Unique domains | >= 5 | >= 3 | Diversify in re-search | +| Max single-source share | <= 25% | <= 30% | Find alternatives | +| Official source coverage | >= 30% for standard | >= 20% for lightweight | Add official sources | +| Source-type balance | official + academic + secondary at least 2 types | same | Fill missing type +| Dropped sources listed | All | All | Must be explicit | +| No duplicate URLs | 0 duplicates | 0 | Merge during P3 | + +## Gate 3: Draft Quality (after P5) + +| Check | Standard | Lightweight | Fix | +|-------|----------|-------------|-----| +| Every [n] in registry | 100% | 100% | Remove or fix | +| No dropped source cited | 0 violations | 0 | Remove immediately | +| Citation density | >= 1 per 200 words | >= 1 per 300 words | Add citations | +| Every section has confidence marker | 100% | 100% | Add missing | +| High-confidence claims backed by official source | 100% | 100% | Downgrade or re-source | +| Counter-claim recorded for major sections | 100% | 70% | Add opposing interpretation | +| Total word count | 3000-8000 | 2000-4000 | Adjust scope | + +## Gate 4: Notes Traceability (after P6) + +| Check | Threshold | Fix | +|-------|-----------|-----| +| Every specific claim traceable to a task note finding | 100% | 100% | Remove or mark [unverified] | +| Every statistic/number appears in some task note | 100% | 100% | Remove or verify | +| No claim contradicts a task note | 0 contradictions | 0 | Rewrite to match notes | +| Claims with recency sensitivity include source date and AS_OF | 100% | 100% | Add date metadata | +| P6 found >= 3 issues | Must | Re-examine harder if 0 found | + +## Gate 5: Verification (after P7) + +| Check | Threshold | Fix | +|-------|-----------|-----| +| Registry cross-check: all [n] valid | 100% | 100% | Remove invalid [n] | +| Spot-check: 5+ claims traced to notes | >= 4/5 pass | Fix failing claims | +| No dropped source resurrected | 0 | Remove immediately | +| Source concentration check for key claims | None > 25% | diversify | + +## Anti-Hallucination Patterns + +| Pattern | Where to detect | Fix | +|---------|----------------|-----| +| URL not from any subagent search | P7 registry check | Remove citation | +| Claim not in any task note | P6 traceability check | Remove or mark [unverified] | +| Number more precise than source | P6 ("73.2%" when note says "about 70%") | Use note's precision | +| Source authority inflated | P3 registry building | Re-score from notes | +| Source type mismatched to claim | P3 + P6 | Reclassify or replace source | +| "Studies show..." without naming study | P6 | Name specific source or remove | +| Dropped source reappears | P7 cross-check | Remove immediately | +| Subagent invented a URL | Gate 1 (lead verifies subagent notes) | Remove from notes before P3 | + +## Chinese-Specific Patterns + +| Pattern | Fix | +|---------|-----| +| Fake CNKI URL format | Remove, note gap | +| "某专家表示" without name/institution | Name or remove | +| "据统计" without data source | Add source or qualitative language | +| Fabricated institution report | Verify existence or remove | +| 旧模型信息未标注 AS_OF | 降级置信度并重搜 | diff --git a/deep-research/references/report_template_v6.md b/deep-research/references/report_template_v6.md new file mode 100644 index 00000000..ab4c3f9d --- /dev/null +++ b/deep-research/references/report_template_v6.md @@ -0,0 +1,82 @@ +# {{TITLE}} + +> 研究日期: {{DATE}} | 来源数量: {{SOURCE_COUNT}} | 字数: ~{{WORD_COUNT}} | 模式: {{MODE}} | AS_OF: {{AS_OF}} | 官方源占比: {{OFFICIAL_SHARE}} + +## 摘要 / Executive Summary + +{{200-400 words summarizing key findings, methodology, conclusions, and risks.}} + +--- + +## 目录 + +{{Auto-generate from actual section headers below.}} + +--- + +{{BODY SECTIONS — Adapt to topic type and include opposing interpretation per section.}} + +For each section: + +## N. [Topic-Specific Section Title] + +{{Section content with inline citations [1][2]. +Standard mode: 500-1000 words per section. +Lightweight mode: 300-600 words per section. + +Rules: +- 每个事实性论点都需要引用 [n] +- 数字/百分比必须有来源 +- 出现不同证据时要成对给出支持与反驳 +}} + +**置信度:** High/Medium/Low + +**依据:** {{Why this confidence level — source agreement, evidence quality, data availability}} + +**反方解释:** {{One explicit opposing interpretation with supporting citations if any, or [unverified] if insufficient.}} + +--- + +{{COUNTER-REVIEW SUMMARY}} + +- **核心争议 1:** [主张 A 与反向证据 B 对比] [n][m] +- **核心争议 2:** ... + +## 关键发现 / Key Findings + +{{3-5 findings in Standard mode, 2-3 in Lightweight. Each finding should:}} +- 具体结论 +- 对应引文 +- 信心说明 + +Example: +- **发现 1:** [Most important discovery] [3][7] +- **发现 2:** [Second most important] [1][4] + +--- + +## 局限性与未来方向 / Limitations & Future Directions + +### 本研究局限 +{{Be explicit: +- What topics/angles couldn't be covered and why +- Methodological limits (web-accessible sources, paywall, language, timing) +- Source coverage gaps and counter-claim evidence gaps +}} + +### 未来方向 +{{Concrete suggestions for follow-up research with priority and responsible evidence type.}} + +--- + +## 参考文献 / References + +[1] Author/Org. "Title". Source-Type: official/academic/secondary-industry/journalism/community/other. As Of: YYYY-MM-DD. URL. +[2] Author/Org. "Title". Source-Type: ... As Of: YYYY-MM-DD. URL. + +Rules: +- Every [n] in body MUST have matching entry here +- Every entry here MUST be cited at least once +- Source-Type and As Of fields are mandatory +- All URLs MUST come from actual search results (P2 source pool) diff --git a/deep-research/references/research_notes_format.md b/deep-research/references/research_notes_format.md new file mode 100644 index 00000000..49d11967 --- /dev/null +++ b/deep-research/references/research_notes_format.md @@ -0,0 +1,147 @@ +# Research Notes Format Specification + +The research notes are the ONLY communication channel between subagents and +the lead agent. Every fact in the final report must be traceable to a line in +these notes. No exceptions. + +## File Structure + +``` +workspace/research-notes/ + task-a.md Subagent A writes (history expert) + task-b.md Subagent B writes (transport historian) + task-c.md Subagent C writes (telecom analyst) + task-d.md Subagent D writes (comparative analyst) + registry.md Lead agent builds from task-*.md (P3) +``` + +## Per-Task Notes Format + +Each `task-{id}.md` file follows this exact structure: + +```markdown +--- +task_id: a +role: Economic Historian +status: complete +sources_found: 4 +--- + +## Sources + +[1] Before AI skeptics, Luddites raged against the machine | https://www.nationalgeographic.com/... | Source-Type: secondary-industry | As Of: 2025-08 | Authority: 8/10 +[2] Rage against the machine | https://www.cam.ac.uk/research/news/rage-against-the-machine | Source-Type: academic | As Of: 2024-04 | Authority: 8/10 +[3] Luddite | https://en.wikipedia.org/wiki/Luddite | Source-Type: community | As Of: 2026-03 | Authority: 7/10 +[4] Learning from the Luddites | https://forum.effectivealtruism.org/... | Source-Type: community | As Of: 2025-10 | Authority: 6/10 + +## Findings + +- Luddite movement began March 11, 1811 in Arnold, Nottinghamshire. [3] +- Luddites were skilled craftspeople, not anti-technology extremists. [1][2] +- In the 100M-person textile industry, Luddites never exceeded a few thousand. [2] +- Government crushed movement: 12 executed at York Assizes, Jan 1813. [3] +- Movement collapsed by 1817 under military repression. [1] +- Full textile mechanization transition took 50-90 years (1760s-1850s). [4] +- Textile workers' real wages dropped ~70% during transition. [4] +- Key lesson for AI: Luddites organized AFTER displacement began, losing leverage. [4] + +## Deep Read Notes + +### Source [1]: National Geographic — Luddites and AI +Key data: destroyed up to 10,000 pounds of frames in first year alone. + Movement spread from Nottinghamshire to Yorkshire and Lancashire in 1812. + Children made up 2/3 of workforce at Cromford factory. +Key insight: Luddites attacked the SYSTEM of exploitation, not machines per se. + They protested manufacturers circumventing standard labor practices. +Useful for: framing section on historical displacement, correcting "anti-tech" myth + +### Source [2]: Cambridge University +Key data: Luddites were "elite craftspeople" not working class broadly. + Yorkshire croppers had 7-year apprenticeships. Movement was localized, never exceeded a few thousand. +Key insight: The movement was smaller and more elite than popular history suggests. +Useful for: nuancing the scale of historical resistance + +## Gaps + +- Could not find quantitative data on how many specific jobs were lost to textile machines +- No Chinese-language academic sources on Luddite movement found +- Alternative explanation: displacement narrative may be partly confounded by wartime demand shocks +``` + +## Source Line Format + +Each source line in the `## Sources` section must contain exactly: +``` +[n] Title | URL | Source-Type: one-of{official|academic|secondary-industry|journalism|community|other} | As Of: YYYY-MM(or YYYY) | Authority: score/10 +``` + +Rules: +- [n] numbers are LOCAL to this task file (start at [1]) +- Lead agent will reassign GLOBAL [n] numbers in registry.md +- URL must be from an actual search result (subagent MUST NOT invent URLs) +- `Authority` score follows guide in quality-gates.md +- `As Of` must be provided; use `undated` if unknown +- High-confidence claims in final report must use `official` or `academic` sources + +## Findings Line Format + +Each finding must be: +- One sentence of specific, factual information +- End with source number(s) in brackets: [1] or [1][2] +- Max 10 findings per task (forces prioritization) +- No vague claims like "research shows..." — name what specifically + +Good: `Full textile mechanization transition took 50-90 years (1760s-1850s). [4]` +Bad: `The transition took a long time. [4]` +Bad: `Studies suggest that it was a lengthy process.` (no source, vague) + +## Deep Read Notes Format + +For each source that was web_fetched (full article read): +- Key data: specific, numeric evidence from article +- Key insight: the one thing this source says that others don't +- Useful for: which final section this supports + +Max 4 lines per source. This is a research notebook, not a summary. + +## Gaps Section + +List what the subagent searched for but could NOT find, and possible counter-readings. +This signals where evidence is thin and confidence should be lowered. + +## Registry Format (built by lead agent in P3) + +The `registry.md` file merges all task sources into a global registry and adds source-type / as-of fields. + +```markdown +# Citation Registry +Built from: task-a.md, task-b.md, task-c.md, task-d.md + +## Approved Sources + +[1] National Geographic — Luddites | https://www.nationalgeographic.com/... | Source-Type: secondary-industry | As Of: 2026-03 | Auth: 8 | From: task-a +[2] Cambridge — Rage against machine | https://www.cam.ac.uk/... | Source-Type: academic | As Of: 2012-04 | Auth: 8 | From: task-a +[3] OpenAI — Day Horse Lost Job | https://blogs.microsoft.com/... | Source-Type: official | As Of: 2026-01 | Auth: 8 | From: task-b +... +[N] Last source + +## Dropped + +x Quora answer | https://www.quora.com/... | Source-Type: community | As Of: 2024-10 | Auth: 3 | Reason: below threshold +x Study.com | https://study.com/... | Source-Type: secondary-industry | As Of: undated | Auth: 4 | Reason: better sources available + +## Stats + +Total evaluated: 22 +Approved: 16 +Dropped: 6 +Unique domains: 12 +Source-type: official 4 / academic 3 / secondary-industry 5 / journalism 2 / community 2 +Max single-source share: 3/16 = 19% (pass) +``` + +Rules for registry: +- [n] numbers here are FINAL — they appear unchanged in the report +- Every [n] in the report must exist in the Approved list +- Every Dropped source must NEVER appear in the report +- If two tasks found the same URL, keep it once with the higher authority score diff --git a/deep-research/references/source_accessibility_policy.md b/deep-research/references/source_accessibility_policy.md new file mode 100644 index 00000000..9da0ac58 --- /dev/null +++ b/deep-research/references/source_accessibility_policy.md @@ -0,0 +1,179 @@ +# Source Accessibility Policy + +**Version**: V6.1 +**Purpose**: Distinguish between legitimate exclusive information advantages and circular verification traps + +--- + +## The Problem + +In the "字节跳动" case study, we made a **methodology error**: + +**What happened**: +1. User asked to research **their own company**: "字节跳动某子公司" +2. We accessed user's **own Spaceship account** (their private registrar) +3. Found 25 domains **the user already owned** +4. Reported back: "The company owns these 25 domains" + +**Why this is wrong**: +- This is **circular reasoning**, not research +- User asked us to *discover* information about their company +- We instead *queried* their private data and presented it as findings +- It's like looking in someone's wallet to tell them how much money they have + +**The real question**: Can an external investigator confirm this company exists? +**Answer**: No (WHOIS privacy, no public records) + +--- + +## Core Principle: No Circular Verification + +### ❌ FORBIDDEN: Self-Verification + +When researching **the user's own assets/company/identity**: + +| Scenario | WRONG | RIGHT | +|----------|-------|-------| +| User's company | "I found in YOUR registrar that YOU own these domains" | "Public WHOIS shows privacy protection - ownership not externally verifiable" | +| User's identity | "I checked YOUR email and found YOUR address" | "Please provide address if relevant to the research" | +| User's property | "I accessed YOUR bank to see YOUR balance" | Not applicable to research | + +**Rule**: Cannot use user's private data to "discover" what user already knows about themselves. + +--- + +### ✅ ALLOWED: Exclusive Information Advantage + +When researching **third parties** (competitors, markets, investments): + +| Source Type | Example | Usage | +|-------------|---------|-------| +| **User's paid subscriptions** | Crunchbase Pro, PitchBook, Wind | ✅ Use to research competitors | +| **User's proprietary databases** | Internal CRM, industry databases | ✅ Use to research market | +| **User's private APIs** | Trading APIs, data feeds | ✅ Use for investment research | +| **User's internal documents** | Prior research, memos | ✅ Use as background for new research | + +**Rule**: User's exclusive information sources are competitive advantages - USE THEM for third-party research. + +--- + +## The Distinction + +``` +Research Target: 字节跳动某子公司 +├─ Is this the user's own company? → YES +├─ Can we use user's private data about it? → NO (circular) +└─ Must rely on: Public sources only + +Research Target: 竞争对手公司 X +├─ Is this the user's own company? → NO +├─ Can we use user's Crunchbase Pro? → YES (competitive advantage) +└─ Can use: Public + User's exclusive sources +``` + +--- + +## Corrected Methodology + +### When Researching User's Own Company + +**Approach**: External investigator perspective + +``` +User: "Research my company 字节跳动子公司" + +CORRECT RESPONSE: +1. Search public sources (WHOIS, web, news) +2. Find: Website placeholder, privacy-protected WHOIS, no news +3. Report: "From public perspective: minimal footprint, cannot verify ownership" +4. Gap: "Internal data not accessible to external investigators" + +INCORRECT RESPONSE: +1. Access user's Spaceship account +2. Find: 25 domains user already knows they own +3. Report: "The company owns 25 domains" (user already knows this!) +``` + +### When User Provides Exclusive Sources + +**Approach**: Leverage competitive advantage + +``` +User: "Research competitor X, I have Crunchbase Pro" +User: "Here's my API key: xxx" + +CORRECT RESPONSE: +1. Use provided Crunchbase Pro API +2. Find: Funding history, team info not in public sources +3. Report: "Per Crunchbase Pro [exclusive source], X raised $Y in Series Z" +4. Cite: Accessibility: exclusive (user-provided) +``` + +--- + +## Source Classification + +### public ✅ +- Available to any external researcher +- Examples: Public websites, news, SEC filings + +### exclusive-user-provided ✅ (FOR THIRD-PARTY RESEARCH) +- User's paid subscriptions, private APIs, internal databases +- **USE for**: Researching competitors, markets, investments +- **DO NOT USE for**: Verifying user's own assets/identity + +### private-user-owned ❌ (FOR SELF-RESEARCH) +- User's own accounts, emails, personal data +- **DO NOT USE**: Creates circular verification + +--- + +## Information Black Box Protocol + +When an entity (including user's own company) has no public footprint: + +1. **Document what external researcher would find**: + - WHOIS: Privacy protected + - Web search: No results + - News: No coverage + +2. **Report honestly**: + ``` + Public sources found: 0 + External visibility: None + Verdict: Cannot verify from public perspective + Note: User may have private information not available to external investigators + ``` + +3. **Do NOT**: + - Use user's private data to "fill gaps" + - Present user's private knowledge as "discovered evidence" + +--- + +## Checklist + +When starting research, determine: + +1. **Who is the research target?** + - User's own company/asset? → Public sources ONLY + - Third party? → Can use user's exclusive sources + +2. **Am I discovering or querying?** + - Discovering new info? → Research + - Querying user's own data? → Circular, not allowed + +3. **Would this finding surprise the user?** + - Yes → Legitimate research + - No (they already know) → Probably circular verification + +--- + +## Summary + +| Situation | Can Use User's Private Data? | Why? | +|-----------|------------------------------|------| +| Research user's own company | ❌ NO | Circular verification | +| Research competitor using user's Crunchbase | ✅ YES | Competitive advantage | +| Research market using user's database | ✅ YES | Exclusive information | +| "Discover" user's own domain ownership | ❌ NO | User already knows this | diff --git a/deep-research/references/subagent_prompt.md b/deep-research/references/subagent_prompt.md new file mode 100644 index 00000000..1b3b97fb --- /dev/null +++ b/deep-research/references/subagent_prompt.md @@ -0,0 +1,116 @@ +# Subagent Prompt Template + +This file defines the prompt structure sent to each research subagent. +The lead agent fills in the `{variables}` and dispatches. + +## Prompt + +``` +You are a research specialist with the role: {role}. + +## Your Task + +{objective} + +## Search Queries (start with these, adjust as needed) + +1. {query_1} +2. {query_2} +3. {query_3} (optional) + +## Instructions + +1. Run 2-4 web searches using the queries above (and variations). +2. For the best 2-3 results, use web_fetch to read the full article. +3. For each discovered source, assign: + - Source-Type: official|academic|secondary-industry|journalism|community|other + - As Of: YYYY-MM or YYYY (publication date or last verified) +4. Assess each source's authority (1-10 scale). +5. Write ALL findings to the file: {output_path} +6. Record at least one explicit counter-claim candidate in `Gaps`. +7. Use EXACTLY the format below. Do not deviate. + +## Output Format (write this to {output_path}) + +--- +task_id: {task_id} +role: {role} +status: complete +sources_found: {N} +--- + +## Sources + +[1] {Title} | {URL} | Source-Type: {Type} | As Of: {YYYY-MM-or-YYYY} | Authority: {score}/10 +[2] {Title} | {URL} | Source-Type: {Type} | As Of: {YYYY-MM-or-YYYY} | Authority: {score}/10 +... + +## Findings + +- {Specific fact, with source number}. [1] +- {Specific fact, with source number and confidence}. [2] +- {Another fact}. [1] +... (max 10 findings, each one sentence, each with source number) + +## Deep Read Notes + +### Source [1]: {Title} +Key data: {specific numbers, dates, percentages extracted from full text} +Key insight: {the one thing this source contributes that others don't} +Useful for: {which aspect of the broader research question} + +### Source [2]: {Title} +Key data: ... +Key insight: ... +Useful for: ... + +## Gaps + +- {What you searched for but could NOT find} +- {Alternative interpretation or methodological limitation} + +## END + +Do not include any content after the Gaps section. +Do not summarize your process. Write the findings file and stop. +``` + +## Depth Levels + +**DEEP** — web_fetch 2-3 full articles and write detailed Deep Read Notes. +Use for: core tasks where specific data points and expert analysis are critical. + +**SCAN** — rely mainly on search snippets, fetches at most 1 article. +Use for: supplementary tasks like source mapping. + +## Environment-Specific Dispatch + +### Claude Code +```bash +# Single task +claude -p "$(cat workspace/prompts/task-a.md)" \ + --allowedTools web_search,web_fetch,write \ + > workspace/research-notes/task-a.md + +# Parallel dispatch +for task in a b c; do + claude -p "$(cat workspace/prompts/task-${task}.md)" \ + --allowedTools web_search,web_fetch,write \ + > workspace/research-notes/task-${task}.md & +done +wait +``` + +### Cowork +Spawn subagent tasks via the subagent dispatch mechanism. + +### DeerFlow / OpenClaw +Use the `task` tool: + +```python +task( + prompt=task_a_prompt, + tools=["web_search", "web_fetch", "write_file"], + output_path="workspace/research-notes/task-a.md" +) +``` diff --git a/demos/README.md b/demos/README.md index e43db9be..aeb565b7 100644 --- a/demos/README.md +++ b/demos/README.md @@ -14,7 +14,7 @@ demos/ │ └── package-skill.tape # Package for distribution ├── github-ops/ │ └── create-pr.tape # Create pull requests -├── markdown-tools/ +├── doc-to-markdown/ │ └── convert-docs.tape # Convert documents └── generate_all_demos.sh # Generate all GIFs ``` diff --git a/markdown-tools/SKILL.md b/doc-to-markdown/SKILL.md similarity index 57% rename from markdown-tools/SKILL.md rename to doc-to-markdown/SKILL.md index 8bc71f48..0dea85ed 100644 --- a/markdown-tools/SKILL.md +++ b/doc-to-markdown/SKILL.md @@ -1,57 +1,68 @@ --- -name: markdown-tools -description: Converts documents to markdown with multi-tool orchestration for best quality. Supports Quick Mode (fast, single tool) and Heavy Mode (best quality, multi-tool merge). Use when converting PDF/DOCX/PPTX files to markdown, extracting images from documents, validating conversion quality, or needing LLM-optimized document output. +name: doc-to-markdown +description: Converts DOCX/PDF/PPTX to high-quality Markdown with automatic post-processing. Fixes pandoc grid tables, simple tables, image paths, CJK bold spacing, attribute noise, and code blocks. Benchmarked best-in-class (7.6/10) against Docling, MarkItDown, Pandoc raw, and Mammoth. Trigger on "convert document", "docx to markdown", "parse word", "doc to markdown", "解析word", "转换文档". --- -# Markdown Tools +# Doc to Markdown -Convert documents to high-quality markdown with intelligent multi-tool orchestration. +Convert documents to high-quality markdown with intelligent multi-tool orchestration and automatic DOCX post-processing. -## Dual Mode Architecture - -| Mode | Speed | Quality | Use Case | -|------|-------|---------|----------| -| **Quick** (default) | Fast | Good | Drafts, simple documents | -| **Heavy** | Slower | Best | Final documents, complex layouts | +**Architecture**: Pandoc (best-in-class extraction) + 8 post-processing fixes (our value-add). ## Quick Start -### Installation - ```bash -# Required: PDF/DOCX/PPTX support -uv tool install "markitdown[pdf]" -pip install pymupdf4llm -brew install pandoc -``` +# DOCX → Markdown (one command, zero manual fixes) +uv run --with pymupdf4llm --with markitdown scripts/convert.py document.docx -o output.md --assets-dir ./media -### Basic Conversion - -```bash -# Quick Mode (default) - fast, single best tool +# PDF → Markdown uv run --with pymupdf4llm --with markitdown scripts/convert.py document.pdf -o output.md -# Heavy Mode - multi-tool parallel execution with merge -uv run --with pymupdf4llm --with markitdown scripts/convert.py document.pdf -o output.md --heavy - -# Check available tools -uv run scripts/convert.py --list-tools +# Run tests +uv run --with pytest pytest scripts/test_convert.py -v ``` -## Tool Selection Matrix +## Dual Mode -| Format | Quick Mode Tool | Heavy Mode Tools | -|--------|----------------|------------------| +| Mode | Speed | Quality | Use Case | +|------|-------|---------|----------| +| **Quick** (default) | Fast | Good | Drafts, simple documents | +| **Heavy** | Slower | Best | Final documents, complex layouts | + +## Tool Selection + +| Format | Quick Mode | Heavy Mode | +|--------|-----------|------------| | PDF | pymupdf4llm | pymupdf4llm + markitdown | -| DOCX | pandoc | pandoc + markitdown | +| DOCX | pandoc + post-processing | pandoc + markitdown | | PPTX | markitdown | markitdown + pandoc | | XLSX | markitdown | markitdown | -### Tool Characteristics +## DOCX Post-Processing (automatic) + +When converting DOCX via pandoc, 8 cleanups are applied automatically: -- **pymupdf4llm**: LLM-optimized PDF conversion with native table detection and image extraction -- **markitdown**: Microsoft's universal converter, good for Office formats -- **pandoc**: Excellent structure preservation for DOCX/PPTX +| Problem | Fix | Test coverage | +|---------|-----|---------------| +| Grid tables (`+:---+`) | Single-column → blockquote, multi-column → pipe table | `TestPostprocessPipeline` | +| Simple tables (` ---- ----`) | Multi-column images → pipe table with captions | `TestSimpleTable` | +| Image path nesting (`media/media/`) | Flatten to `media/`, absolute → relative | `test_stats_tracking` | +| Pandoc attributes (`{width="..."}`) | Removed | `test_pandoc_attributes_removed` | +| CJK bold spacing (`**粗体**中文`) | Add space around `**` for CJK bold spans | `TestCjkBoldSpacing` (15 cases) | +| Indented dashed code blocks | → fenced ``` with language detection | `test_code_block_with_language` | +| Escaped brackets (`\[...\]`) | → `[...]` | `test_escaped_brackets_fixed` | +| Double-bracket links (`[[text]](url)`) | → `[text](url)` | `test_double_bracket_links_fixed` | + +### CJK Bold Spacing — why and how + +DOCX uses run-level styling (no spaces between bold/normal runs in CJK text). Markdown renderers need whitespace around `**` to recognize bold boundaries. + +**Rule**: if a `**content**` span contains any CJK character, ensure both sides have a space — unless already spaced or at line boundary. This handles CJK punctuation, emoji adjacency, and mixed content. + +``` +Before: 打开**飞书**，就可以 → some renderers fail to bold +After: 打开 **飞书** ，就可以 → universally renders correctly +``` ## Heavy Mode Workflow @@ -117,7 +128,7 @@ python scripts/merge_outputs.py output1.md output2.md -o merged.md --verbose ## Path Conversion (Windows/WSL) ```bash -# Windows → WSL conversion +# Windows to WSL conversion python scripts/convert_path.py "C:\Users\name\Documents\file.pdf" # Output: /mnt/c/Users/name/Documents/file.pdf ``` @@ -147,7 +158,8 @@ brew install pandoc | Script | Purpose | |--------|---------| -| `convert.py` | Main orchestrator with Quick/Heavy mode | +| `convert.py` | Main orchestrator with Quick/Heavy mode + DOCX post-processing | +| `test_convert.py` | 31 tests covering all post-processing functions | | `merge_outputs.py` | Merge multiple markdown outputs | | `validate_output.py` | Quality validation with HTML report | | `extract_pdf_images.py` | PDF image extraction with metadata | @@ -155,6 +167,7 @@ brew install pandoc ## References +- `references/benchmark-2026-03-22.md` - 5-tool benchmark (Docling/MarkItDown/Pandoc/Mammoth/ours) - `references/heavy-mode-guide.md` - Detailed Heavy Mode documentation - `references/tool-comparison.md` - Tool capabilities comparison - `references/conversion-examples.md` - Batch operation examples diff --git a/doc-to-markdown/references/benchmark-2026-03-22.md b/doc-to-markdown/references/benchmark-2026-03-22.md new file mode 100644 index 00000000..27b20690 --- /dev/null +++ b/doc-to-markdown/references/benchmark-2026-03-22.md @@ -0,0 +1,163 @@ +# DOCX→Markdown 转换方案基准测试 + +> **测试日期**：2026-03-22 +> +> **测试文件**：`助教-【腾讯云🦞】小白实践 OpenClaw 保姆级教程.docx`（19MB，77 张图片，含 grid table 布局、JSON 代码块、多列图片并排、信息框） +> +> **测试方法**：5 个方案对同一文件转换，按 5 个维度各 10 分制打分 + +--- + +## 综合评分 + +| 维度 | Docling (IBM) | MarkItDown (MS) | Pandoc | Mammoth | **doc-to-markdown（我们）** | +|------|:---:|:---:|:---:|:---:|:---:| +| 表格质量 | 5 | 3 | 5 | 1~3 | **6** | +| 图片提取 | 4 | 2 | **10** | 5 | 7 | +| 文本完整性 | 8 | 7 | **9** | 7 | **9** | +| 格式清洁度 | 5 | 5 | 5 | 3 | **7** | +| 代码块 | 2 | 1 | N/A | 1 | **9** | +| **综合** | **4.8** | **3.6** | **7.3** | **3.4~3.6** | **7.6** | + +--- + +## 各方案详细分析 + +### 1. IBM Docling（综合 4.8） + +- **版本**：docling 2.x + Granite-Docling-258M +- **架构**：AI 驱动（VLM 视觉语言模型），DocTags 中间格式 → Markdown + +**致命问题**： +- 图片引用全部是 `` 占位符（77 张图 0 张可显示），`ImageRefMode` API 对 DOCX 不可用 +- 标题层级全部丢失（0 个 `#`），所有标题退化为粗体文本 +- 代码块为零，JSON 和命令全部输出为普通段落 +- `api_key` 被错误转义为 `api\_key` + +**优点**： +- 文本内容完整，中文/emoji/链接保留良好 +- 无 grid table 或 HTML 残留 +- 表格语法正确（pipe table），但内容是占位符 + +**结论**：Docling 的优势在 PDF（AAAI 2025 论文场景），DOCX 支持远未达到生产级别。 + +### 2. Microsoft MarkItDown（综合 3.6） + +- **版本**：markitdown 0.1.5 +- **架构**：底层调用 mammoth → HTML → markdownify → Markdown + +**致命问题**： +- 77 张图片全部是截断的 base64 占位符（`data:image/png;base64,...`），默认 `keep_data_uris=False` 主动丢弃图片数据 +- 标题全部变为粗体文本（mammoth 无法识别 WPS 自定义样式） +- 代码块为零，JSON 被塞入表格单元格 +- 有序列表编号全部错误（输出为 `1. 1. 1.`） + +**优点**： +- 无 HTML 标签残留 +- 文本内容基本完整 + +**结论**：MarkItDown 的 markdownify 后处理反而引入破坏性截断。轻量场景可用，复杂 DOCX 不可靠。 + +### 3. Pandoc（综合 7.3） + +- **版本**：pandoc 3.9 +- **架构**：Haskell 原生 AST 解析，支持 60+ 格式 + +**测试了 3 种参数**： + +| 参数 | 结果 | +|------|------| +| `-t gfm` | 最差：24 个 HTML `` 嵌套，74 个 HTML `` | +| `-t markdown` | 最佳：grid table（可后处理），无 HTML | +| `-t markdown-raw_html-...` | 与 markdown 完全相同，参数无效果 | + +**问题**： +- Grid table 不可避免（原 docx 有多行单元格和嵌套表格，pipe table 无法表达） +- `{width="..." height="..."}` 68 处 +- `{.underline}` 6 处 +- 反斜杠过度转义 37 处 + +**优点**： +- 图片提取 10/10（77 张全部正确，路径结构一致） +- 文本完整性 9/10（内容、链接、emoji 全部保留） +- 最成熟稳定的底层引擎 + +**结论**：Pandoc 是最可靠的底层引擎，输出质量最高但需要后处理清洗 pandoc 私有语法。 + +### 4. Mammoth（综合 3.4~3.6） + +- **版本**：mammoth 1.11.0 +- **架构**：python-docx 解析 → HTML/Markdown（Markdown 支持已废弃） + +**测试了 2 种方式**： + +| 方式 | 综合 | +|------|------| +| 方式A：直接转 Markdown | 3.4（表格完全丢失） | +| 方式B：转 HTML → markdownify | 3.6（有表格但嵌套被压扁） | + +**致命问题**： +- 标题全部丢失（WPS `styles.xml` 中样式定义为空，mammoth 无法映射 Heading） +- 代码块为零 +- 图片全部 base64 内嵌，单文件 28MB +- 方式B 中 markdownify 丢失 14 张图片（63/77） + +**结论**：Mammoth 的 Markdown 支持已废弃，对 WPS 导出的 docx 兼容性差。不推荐。 + +### 5. doc-to-markdown / 我们的方案（综合 7.6） + +- **版本**：doc-to-markdown 1.0（基于 pandoc + 6 个后处理函数） +- **架构**：Pandoc 转换 → 自动后处理（grid table 清理、图片路径修复、属性清理、代码块修复、转义修复） + +**后处理实际效果**： + +| 后处理函数 | 修复数量 | +|-----------|---------| +| `_convert_grid_tables` | 11 处 grid table → pipe table / blockquote | +| `_clean_pandoc_attributes` | 3437 字符属性清理 | +| `_fix_code_blocks` | 22 处缩进虚线 → ``` 代码块 | +| `_fix_escaped_brackets` | 10 处 | +| `_fix_double_bracket_links` | 1 处 | +| `_fix_image_paths` | 77 张图片路径修复 | + +**已知问题（待修复）**： +- 图片路径双层嵌套 bug：`--assets-dir` 指定目录内被 pandoc 再建一层 `media/` +- 2 处 grid table 残留（文末并排图片组未完全转换） + +**优点**： +- 代码块识别 9/10（JSON 带语言标签，命令行正确包裹） +- 格式清洁度 7/10（attributes、转义、grid table 大部分清理干净） +- 文本完整性 9/10（关键内容全部保留） + +**结论**：综合最优，核心价值在 pandoc 后处理层。剩余 2 个 bug 可修。 + +--- + +## 架构决策 + +``` +最终方案：Pandoc（底层引擎）+ doc-to-markdown 后处理（增值层） + +理由： +1. Pandoc 图片提取最可靠（10/10），文本最完整（9/10） +2. Pandoc 的问题（grid table、属性、转义）全部可后处理解决 +3. Docling/MarkItDown/Mammoth 的致命问题（图片丢失、标题丢失）无法后处理修复 +4. 后处理层是我们的核心竞争力，成本低、可迭代 +``` + +--- + +## 测试文件特征 + +本次测试文件的难点在于： + +| 特征 | 说明 | 影响 | +|------|------|------| +| WPS 导出 | 非标准 Word 样式（Style ID 2/3 而非 Heading 1/2） | mammoth/markitdown/docling 标题全丢 | +| 多列图片布局 | 2x2、1x4 图片网格用表格排版 | pandoc 输出 grid table | +| 信息框/提示框 | 单列表格包裹文字 | pandoc 输出 grid table | +| 嵌套表格 | 表格内套表格 | pipe table 无法表达 | +| JSON 代码块 | 非代码块样式，用文本框/缩进表示 | 多数工具无法识别为代码 | +| 19MB 文件 | 77 张截图嵌入 | base64 方案导致 28MB 输出 | + +这些特征代表了真实世界中 WPS/飞书文档导出 docx 的典型困难，是有效的基准测试场景。 diff --git a/markdown-tools/references/conversion-examples.md b/doc-to-markdown/references/conversion-examples.md similarity index 100% rename from markdown-tools/references/conversion-examples.md rename to doc-to-markdown/references/conversion-examples.md diff --git a/markdown-tools/references/heavy-mode-guide.md b/doc-to-markdown/references/heavy-mode-guide.md similarity index 98% rename from markdown-tools/references/heavy-mode-guide.md rename to doc-to-markdown/references/heavy-mode-guide.md index 442cb6e1..3befcab7 100644 --- a/markdown-tools/references/heavy-mode-guide.md +++ b/doc-to-markdown/references/heavy-mode-guide.md @@ -1,6 +1,6 @@ # Heavy Mode Guide -Detailed documentation for markdown-tools Heavy Mode conversion. +Detailed documentation for doc-to-markdown Heavy Mode conversion. ## Overview diff --git a/markdown-tools/references/tool-comparison.md b/doc-to-markdown/references/tool-comparison.md similarity index 100% rename from markdown-tools/references/tool-comparison.md rename to doc-to-markdown/references/tool-comparison.md diff --git a/doc-to-markdown/scripts/convert.py b/doc-to-markdown/scripts/convert.py new file mode 100755 index 00000000..a3c97475 --- /dev/null +++ b/doc-to-markdown/scripts/convert.py @@ -0,0 +1,1171 @@ +#!/usr/bin/env python3 +""" +Multi-tool document to markdown converter with intelligent orchestration. + +Supports Quick Mode (fast, single tool) and Heavy Mode (best quality, multi-tool merge). +DOCX files get automatic post-processing to fix pandoc artifacts. + +Usage: + # Quick Mode (default) - fast, single best tool + uv run --with pymupdf4llm --with markitdown scripts/convert.py document.pdf -o output.md + + # Heavy Mode - multi-tool parallel execution with merge + uv run --with pymupdf4llm --with markitdown scripts/convert.py document.pdf -o output.md --heavy + + # DOCX deep mode - python-docx direct parsing (experimental) + uv run --with python-docx scripts/convert.py document.docx -o output.md --docx-deep + + # With image extraction + uv run --with pymupdf4llm scripts/convert.py document.pdf -o output.md --assets-dir ./images + +Dependencies: + - pymupdf4llm: PDF conversion (LLM-optimized) + - markitdown: PDF/DOCX/PPTX conversion + - pandoc: DOCX/PPTX conversion (system install: brew install pandoc) + - python-docx: DOCX deep parsing (optional, for --docx-deep) +""" + +import argparse +import json +import re +import subprocess +import sys +import shutil +import zipfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + + +@dataclass +class ConversionResult: + """Result from a single tool conversion.""" + markdown: str + tool: str + images: list[str] = field(default_factory=list) + success: bool = True + error: str = "" + + +# ── Post-processing stats ──────────────────────────────────────────────────── + +@dataclass +class PostProcessStats: + """Track what the DOCX post-processor fixed.""" + grid_tables_converted: int = 0 + image_paths_fixed: int = 0 + attributes_removed: int = 0 + code_blocks_fixed: int = 0 + escaped_brackets_fixed: int = 0 + double_brackets_fixed: int = 0 + + def any_fixes(self) -> bool: + return any( + getattr(self, f) > 0 + for f in self.__dataclass_fields__ + ) + + def summary(self) -> str: + parts = [] + if self.grid_tables_converted: + parts.append(f"grid tables: {self.grid_tables_converted}") + if self.image_paths_fixed: + parts.append(f"image paths: {self.image_paths_fixed}") + if self.attributes_removed: + parts.append(f"attributes: {self.attributes_removed}") + if self.code_blocks_fixed: + parts.append(f"code blocks: {self.code_blocks_fixed}") + if self.escaped_brackets_fixed: + parts.append(f"escaped brackets: {self.escaped_brackets_fixed}") + if self.double_brackets_fixed: + parts.append(f"double brackets: {self.double_brackets_fixed}") + return ", ".join(parts) if parts else "no fixes needed" + + +# ── DOCX post-processing ───────────────────────────────────────────────────── + +# Regex patterns compiled once +_RE_GRID_BORDER = re.compile(r"^\+[:=-][-:=]+(?:\+[:=-][-:=]+)*\+$") +_RE_GRID_ROW = re.compile(r"^\|(.+)\|$") +_RE_NESTED_GRID_BORDER = re.compile(r"^\|\s*\+[:=-][-:=]+\+\s*\|$") +_RE_PANDOC_ATTR = re.compile(r"\{[^}]*(?:width|height)\s*=\s*\"[^\"]*\"[^}]*\}") +_RE_PANDOC_CLASS = re.compile(r"\{\.(?:underline|mark)\}") +_RE_DOUBLE_BRACKET_LINK = re.compile(r"\[\[([^\]]+)\]\(([^)]+)\)") +_RE_DOUBLE_BRACKET_CLOSED = re.compile(r"\[\[([^\]]+)\]\]\(([^)]+)\)") +_RE_DOUBLE_BRACKET_ATTR_LINK = re.compile(r"\[\[([^\]]+)\]\{[^}]*\}\]\(([^)]+)\)") +_RE_ESCAPED_BRACKET = re.compile(r"\\(\[|])") +# Matches single-column dashed line: " ------" +# AND multi-column simple table border: " ---- -----" +_RE_DASHED_LINE = re.compile(r"^(\s{2,})-{3,}[\s-]*$") +_RE_ESCAPED_QUOTE = re.compile(r'\\"') +# CJK + fullwidth punctuation range for bold spacing checks +_RE_CJK_PUNCT = re.compile(r'[\u4e00-\u9fff\u3000-\u303f\uff01-\uffef，。、；：！？（）【】「」《》""'']') +_RE_BOLD_PAIR = re.compile(r'\*\*(.+?)\*\*') + + +def _is_grid_border(line: str) -> bool: + """Check if a line is a grid table border like +---+ or +:---+.""" + stripped = line.strip() + return bool(_RE_GRID_BORDER.match(stripped)) + + +def _is_nested_grid_border(line: str) -> bool: + """Check if a line is a nested grid border like | +---+ |.""" + stripped = line.strip() + return bool(_RE_NESTED_GRID_BORDER.match(stripped)) + + +def _count_grid_columns(border_line: str) -> int: + """Count columns in a grid table border line.""" + stripped = border_line.strip() + if not stripped.startswith("+"): + return 0 + # Count + separators minus 1 = number of columns + return stripped.count("+") - 1 + + + +# Languages recognized as code block hints in pandoc dashed-line blocks +_KNOWN_CODE_LANGS = frozenset({ + "json", "bash", "shell", "python", "javascript", "js", + "html", "css", "yaml", "xml", "sql", "plain text", + "text", "plaintext", "typescript", "ts", "go", "rust", + "java", "c", "cpp", "ruby", "php", +}) + + +def _build_pipe_table(rows: list[list[str]]) -> list[str]: + """Build a standard markdown pipe table from rows of cells.""" + if not rows: + return [] + col_count = max(len(r) for r in rows) + lines = [ + "| " + " | ".join([""] * col_count) + " |", + "| " + " | ".join(["---"] * col_count) + " |", + ] + for row in rows: + padded = row + [""] * (col_count - len(row)) + lines.append("| " + " | ".join(padded) + " |") + return lines + + +def _collect_images(directory: Path) -> list[str]: + """Collect image files from a directory (single glob pass).""" + if not directory.exists(): + return [] + image_exts = {".png", ".jpg", ".jpeg", ".gif", ".webp"} + return sorted( + str(p) for p in directory.rglob("*") + if p.suffix.lower() in image_exts + ) + + +def _convert_grid_tables(text: str, stats: PostProcessStats) -> str: + """Convert pandoc grid tables to standard markdown. + + Single-column grid tables (info boxes) -> blockquotes. + Multi-column grid tables (side-by-side images) -> split into individual elements. + Nested grid tables are flattened. + """ + lines = text.split("\n") + result = [] + i = 0 + + while i < len(lines): + line = lines[i] + + # Detect grid table start + if _is_grid_border(line): + # Collect the entire grid table + table_lines = [line] + i += 1 + while i < len(lines): + table_lines.append(lines[i]) + if _is_grid_border(lines[i]) and len(table_lines) > 1: + i += 1 + break + i += 1 + else: + # Reached end of file without closing border + # Just output as-is + result.extend(table_lines) + continue + + stats.grid_tables_converted += 1 + num_cols = _count_grid_columns(table_lines[0]) + + # Extract content lines (skip borders) + content_lines = [] + for tl in table_lines: + if _is_grid_border(tl) or _is_nested_grid_border(tl): + continue + m = _RE_GRID_ROW.match(tl.strip()) + if m: + content_lines.append(m.group(1).strip()) + else: + # Non-standard line inside grid, keep content + stripped = tl.strip() + if stripped and stripped != "|": + content_lines.append(stripped) + + if num_cols <= 1: + # Single column -> blockquote + result.append("") + for cl in content_lines: + # Strip outer pipes if present from nested grids + cleaned = cl.strip() + if cleaned.startswith("|") and cleaned.endswith("|"): + cleaned = cleaned[1:-1].strip() + # Skip nested grid borders + if _RE_GRID_BORDER.match(cleaned): + continue + if cleaned: + result.append(f"> {cleaned}") + else: + result.append(">") + result.append("") + else: + # Multi-column -> convert to standard pipe table + # Parse rows: each content_line is a row, split by | into cells + table_rows = [] + for cl in content_lines: + cells = [c.strip() for c in cl.split("|") if c.strip() and not _RE_GRID_BORDER.match(c.strip())] + if cells: + table_rows.append(cells) + + if table_rows: + result.append("") + result.extend(_build_pipe_table(table_rows)) + result.append("") + else: + result.append(line) + i += 1 + + return "\n".join(result) + + +def _fix_image_paths(text: str, assets_dir: Optional[Path], stats: PostProcessStats) -> str: + """Fix pandoc's double media path and verify images exist. + + Pandoc extracts to /media/ but references as + /media/media/. Fix the references. + Also flatten the actual directory if needed. + """ + def fix_path(m: re.Match) -> str: + alt = m.group(1) + path = m.group(2) + new_path = path + + # Fix double media/ path + if "media/media/" in path: + new_path = path.replace("media/media/", "media/") + stats.image_paths_fixed += 1 + + return f"![{alt}]({new_path})" + + text = re.sub(r"!\[([^\]]*)\]\(([^)]+)\)", fix_path, text) + + # Flatten double media/ nesting if present (pandoc artifact) + if assets_dir: + double_media = assets_dir / "media" / "media" + single_media = assets_dir / "media" + try: + for f in double_media.iterdir(): + dest = single_media / f.name + if not dest.exists(): + shutil.move(str(f), str(dest)) + double_media.rmdir() + except (FileNotFoundError, OSError): + pass + + return text + + +def _clean_pandoc_attributes(text: str, stats: PostProcessStats) -> str: + """Remove pandoc attribute annotations from markdown. + + Removes: {width="..." height="..."}, {.underline}, {.mark}, etc. + """ + count_before = len(text) + + # Remove width/height attributes on images + text = _RE_PANDOC_ATTR.sub("", text) + + # Remove class attributes like {.underline} + text = _RE_PANDOC_CLASS.sub("", text) + + if len(text) != count_before: + # Rough count of removals + stats.attributes_removed = count_before - len(text) + + return text + + +def _is_code_content(lines: list[str]) -> bool: + """Heuristic: decide if content between dashed lines is code or a note/callout. + + Code indicators: + - Has a language hint on the first line + - Contains JSON/code-like syntax ({, }, =, ;, ->, //) + - Contains URLs with protocols + - Has backslash line continuations + + Note indicators: + - Mostly CJK/prose text without code syntax + - Short single-line content + """ + text = "\n".join(lines) + stripped = text.strip() + + if not stripped: + return False + + # Code syntax indicators + code_chars = set('{}[]();=<>/\\') + code_char_count = sum(1 for c in stripped if c in code_chars) + + # If >5% of content is code syntax characters, treat as code + if len(stripped) > 0 and code_char_count / len(stripped) > 0.05: + return True + + # JSON-like structure + if stripped.startswith("{") or stripped.startswith("["): + return True + + # Command-like (starts with common command patterns) + first_line = lines[0].strip() if lines else "" + if re.match(r"^(curl|wget|npm|pip|brew|apt|docker|git|ssh|cd|ls|cat|echo|python|node|uv)\s", first_line): + return True + + return False + + +def _fix_code_blocks(text: str, stats: PostProcessStats) -> str: + """Convert pandoc's indented dashed-line blocks to fenced code blocks or blockquotes. + + Pandoc wraps both code and notes in: + ------------------------------------------------------------------ + content here + + ------------------------------------------------------------------ + + With language hint -> code block: + ```json + content here + ``` + + Without language hint + prose content -> blockquote: + > content here + + Without language hint + code-like content -> code block: + ``` + content here + ``` + """ + lines = text.split("\n") + result = [] + i = 0 + + known_langs = _KNOWN_CODE_LANGS + + while i < len(lines): + line = lines[i] + + # Detect indented dashed line (2+ leading spaces, 3+ dashes) + if _RE_DASHED_LINE.match(line): + # Check if this is a pandoc simple table (multiple dashed columns + # on the same line, or content between dashes contains images) + # Simple table pattern: " ---- ----" (multiple dash groups separated by spaces) + # Gap can be 1+ spaces (pandoc uses varying gaps) + dash_parts = [p for p in line.strip().split() if p.strip()] + is_simple_table_border = len(dash_parts) > 1 and all( + re.match(r"^-+$", p.strip()) for p in dash_parts + ) + + if is_simple_table_border: + # This is a pandoc simple table border - collect rows until + # next simple table border, convert to pipe table + table_rows = [] + j = i + 1 + while j < len(lines): + next_line = lines[j] + # Check for closing simple table border + next_parts = [p for p in next_line.strip().split() if p.strip()] + is_next_border = len(next_parts) > 1 and all( + re.match(r"^-+$", p.strip()) for p in next_parts + ) + if is_next_border: + j += 1 + break + if next_line.strip(): + # Split by 2+ spaces to get columns (pandoc uses varying gaps) + cells = [c.strip() for c in re.split(r"\s{2,}", next_line.strip()) if c.strip()] + if cells: + table_rows.append(cells) + j += 1 + + if table_rows: + stats.code_blocks_fixed += 1 + result.append("") + result.extend(_build_pipe_table(table_rows)) + result.append("") + + i = j + continue + + # Not a simple table - look for content and closing dashed line + block_content = [] + lang_hint = "" + j = i + 1 + + while j < len(lines): + next_line = lines[j] + + if _RE_DASHED_LINE.match(next_line): + # Found closing dashed line + j += 1 + break + + block_content.append(next_line) + j += 1 + else: + # No closing dashed line found - not a block, keep as-is + result.append(line) + i += 1 + continue + + # If content contains images, treat as simple table (single-column) + has_images = any("![" in cl for cl in block_content) + if has_images: + result.append("") + for cl in block_content: + cl = cl.strip() + if cl: + result.append(cl) + result.append("") + i = j + continue + + # Check if first line is a language hint (e.g., " JSON\", " Plain Text\") + has_lang_hint = False + if block_content: + first = block_content[0].strip() + first_clean = first.rstrip("\\").strip() + if first_clean.lower() in known_langs: + lang_hint = first_clean.lower() + if lang_hint in ("plain text", "text", "plaintext"): + lang_hint = "" # No language tag for plain text + has_lang_hint = True + block_content = block_content[1:] + + # Clean content: remove leading 2-space indent, fix escaped quotes + cleaned = [] + for cl in block_content: + if cl.startswith(" "): + cl = cl[2:] + cl = cl.replace('\\"', '"') + if cl.endswith("\\"): + cl = cl[:-1] + cleaned.append(cl) + + # Remove trailing/leading empty lines + while cleaned and not cleaned[-1].strip(): + cleaned.pop() + while cleaned and not cleaned[0].strip(): + cleaned.pop(0) + + if cleaned: + stats.code_blocks_fixed += 1 + + # Decide: code block vs blockquote + if has_lang_hint or _is_code_content(cleaned): + # Code block — try to pretty-print JSON + code_lines = cleaned + if lang_hint == "json": + try: + raw = "\n".join(cleaned) + parsed = json.loads(raw) + code_lines = json.dumps(parsed, indent=2, ensure_ascii=False).split("\n") + except (json.JSONDecodeError, ValueError): + pass # Keep original if not valid JSON + + result.append("") + result.append(f"```{lang_hint}") + result.extend(code_lines) + result.append("```") + result.append("") + else: + # Note/callout -> blockquote + result.append("") + for cl in cleaned: + if cl.strip(): + result.append(f"> {cl}") + else: + result.append(">") + result.append("") + + i = j + else: + result.append(line) + i += 1 + + return "\n".join(result) + + +def _fix_escaped_brackets(text: str, stats: PostProcessStats) -> str: + r"""Fix pandoc's escaped brackets: \[ -> [, \] -> ].""" + count = len(_RE_ESCAPED_BRACKET.findall(text)) + if count: + stats.escaped_brackets_fixed = count + text = _RE_ESCAPED_BRACKET.sub(r"\1", text) + return text + + +def _fix_double_bracket_links(text: str, stats: PostProcessStats) -> str: + """Fix double-bracket links: [[text]{.underline}](url) -> [text](url).""" + count = 0 + + def _replace_link(m: re.Match) -> str: + nonlocal count + count += 1 + return f"[{m.group(1)}]({m.group(2)})" + + text = _RE_DOUBLE_BRACKET_ATTR_LINK.sub(_replace_link, text) + text = _RE_DOUBLE_BRACKET_CLOSED.sub(_replace_link, text) + text = _RE_DOUBLE_BRACKET_LINK.sub(_replace_link, text) + + stats.double_brackets_fixed = count + return text + + +def _fix_cjk_bold_spacing(text: str) -> str: + """Add space around **bold** spans that contain CJK characters. + + DOCX uses run-level styling for bold — no spaces between runs in CJK text. + Markdown renderers need whitespace around ** to recognize bold boundaries. + + Rule: if a **content** span contains any CJK character, ensure both sides + have a space (unless already spaced or at line boundary). This handles: + - CJK directly touching **: 打开**飞书** → 打开 **飞书** + - Emoji touching **: **密码】**➡️ → **密码】** ➡️ + - Already spaced: 已有 **粗体** → unchanged + - English bold: English **bold** text → unchanged + """ + result = [] + last_end = 0 + + for m in _RE_BOLD_PAIR.finditer(text): + start, end = m.start(), m.end() + content = m.group(1) + + result.append(text[last_end:start]) + + # Only add spaces for bold spans containing CJK + if _RE_CJK_PUNCT.search(content): + # Space before ** if previous char is not whitespace + if start > 0 and text[start - 1] not in (' ', '\t', '\n'): + result.append(' ') + + result.append(m.group(0)) + + # Space after ** if next char is not whitespace + if end < len(text) and text[end] not in (' ', '\t', '\n'): + result.append(' ') + else: + result.append(m.group(0)) + + last_end = end + + result.append(text[last_end:]) + return ''.join(result) + + +def _cleanup_excessive_blank_lines(text: str) -> str: + """Collapse 3+ consecutive blank lines to 2.""" + return re.sub(r"\n{4,}", "\n\n\n", text) + + +def postprocess_docx_markdown( + text: str, + assets_dir: Optional[Path] = None, +) -> tuple[str, PostProcessStats]: + """Apply all DOCX-specific post-processing to pandoc markdown output. + + Returns (cleaned_text, stats). + """ + stats = PostProcessStats() + + # Order matters: grid tables first (they contain images with attributes) + text = _convert_grid_tables(text, stats) + text = _fix_image_paths(text, assets_dir, stats) + text = _clean_pandoc_attributes(text, stats) + text = _fix_code_blocks(text, stats) + text = _fix_double_bracket_links(text, stats) + text = _fix_escaped_brackets(text, stats) + text = _fix_cjk_bold_spacing(text) + text = _cleanup_excessive_blank_lines(text) + + return text, stats + + +# ── DOCX deep parsing (python-docx) ────────────────────────────────────────── + +def convert_with_docx_deep( + file_path: Path, assets_dir: Optional[Path] = None +) -> ConversionResult: + """Convert DOCX using python-docx direct parsing (experimental). + + More precise than pandoc for: + - Table structure preservation + - Comment extraction + - Image extraction with position info + """ + try: + from docx import Document + from docx.opc.constants import RELATIONSHIP_TYPE as RT + except ImportError: + return ConversionResult( + markdown="", + tool="docx-deep", + success=False, + error="python-docx not installed. Run: pip install python-docx", + ) + + try: + doc = Document(str(file_path)) + md_parts = [] + images = [] + image_counter = 0 + + # Extract images from docx zip + if assets_dir: + assets_dir.mkdir(parents=True, exist_ok=True) + media_dir = assets_dir / "media" + media_dir.mkdir(exist_ok=True) + + with zipfile.ZipFile(str(file_path), "r") as zf: + for name in zf.namelist(): + if name.startswith("word/media/"): + img_name = Path(name).name + img_dest = media_dir / img_name + with zf.open(name) as src, open(img_dest, "wb") as dst: + dst.write(src.read()) + images.append(str(img_dest)) + + # Process paragraphs + for para in doc.paragraphs: + style_name = para.style.name if para.style else "" + text = para.text.strip() + + if not text: + md_parts.append("") + continue + + # Headings + if style_name.startswith("Heading"): + try: + level = int(style_name.split()[-1]) + except (ValueError, IndexError): + level = 1 + md_parts.append(f"{'#' * level} {text}") + md_parts.append("") + continue + + # Check for bold-only paragraphs (often sub-headings in Chinese docs) + all_bold = all(run.bold for run in para.runs if run.text.strip()) + if all_bold and para.runs and len(text) < 100: + md_parts.append(f"**{text}**") + md_parts.append("") + continue + + # Regular paragraph + md_parts.append(text) + md_parts.append("") + + # Process tables + for table in doc.tables: + md_parts.append("") + rows = table.rows + if not rows: + continue + + # Header row + header_cells = [cell.text.strip() for cell in rows[0].cells] + md_parts.append("| " + " | ".join(header_cells) + " |") + md_parts.append("| " + " | ".join(["---"] * len(header_cells)) + " |") + + # Data rows + for row in rows[1:]: + cells = [cell.text.strip() for cell in row.cells] + md_parts.append("| " + " | ".join(cells) + " |") + md_parts.append("") + + markdown = "\n".join(md_parts) + + return ConversionResult( + markdown=markdown, + tool="docx-deep", + images=images, + success=True, + ) + except Exception as e: + return ConversionResult( + markdown="", tool="docx-deep", success=False, error=str(e) + ) + + +# ── Existing tool converters ───────────────────────────────────────────────── + +def check_tool_available(tool: str) -> bool: + """Check if a conversion tool is available.""" + if tool == "pymupdf4llm": + try: + import pymupdf4llm + return True + except ImportError: + return False + elif tool == "markitdown": + try: + import markitdown + return True + except ImportError: + return False + elif tool == "pandoc": + return shutil.which("pandoc") is not None + elif tool == "docx-deep": + try: + from docx import Document + return True + except ImportError: + return False + return False + + +def select_tools(file_path: Path, mode: str) -> list[str]: + """Select conversion tools based on file type and mode.""" + ext = file_path.suffix.lower() + + # Tool preferences by format + tool_map = { + ".pdf": { + "quick": ["pymupdf4llm", "markitdown"], # fallback order + "heavy": ["pymupdf4llm", "markitdown"], + }, + ".docx": { + "quick": ["pandoc", "markitdown"], + "heavy": ["pandoc", "markitdown"], + }, + ".doc": { + "quick": ["pandoc", "markitdown"], + "heavy": ["pandoc", "markitdown"], + }, + ".pptx": { + "quick": ["markitdown", "pandoc"], + "heavy": ["markitdown", "pandoc"], + }, + ".xlsx": { + "quick": ["markitdown"], + "heavy": ["markitdown"], + }, + } + + tools = tool_map.get(ext, {"quick": ["markitdown"], "heavy": ["markitdown"]}) + + if mode == "quick": + # Return first available tool + for tool in tools["quick"]: + if check_tool_available(tool): + return [tool] + return [] + else: # heavy + # Return all available tools + return [t for t in tools["heavy"] if check_tool_available(t)] + + +def convert_with_pymupdf4llm( + file_path: Path, assets_dir: Optional[Path] = None +) -> ConversionResult: + """Convert using PyMuPDF4LLM (best for PDFs).""" + try: + import pymupdf4llm + + kwargs = {} + images = [] + + if assets_dir: + assets_dir.mkdir(parents=True, exist_ok=True) + kwargs["write_images"] = True + kwargs["image_path"] = str(assets_dir) + kwargs["dpi"] = 150 + + # Use best table detection strategy + kwargs["table_strategy"] = "lines_strict" + + md_text = pymupdf4llm.to_markdown(str(file_path), **kwargs) + + if assets_dir: + images = _collect_images(assets_dir) + + return ConversionResult( + markdown=md_text, tool="pymupdf4llm", images=images, success=True + ) + except Exception as e: + return ConversionResult( + markdown="", tool="pymupdf4llm", success=False, error=str(e) + ) + + +def convert_with_markitdown( + file_path: Path, assets_dir: Optional[Path] = None +) -> ConversionResult: + """Convert using markitdown.""" + try: + # markitdown CLI approach + result = subprocess.run( + ["markitdown", str(file_path)], + capture_output=True, + text=True, + timeout=120, + ) + + if result.returncode != 0: + return ConversionResult( + markdown="", + tool="markitdown", + success=False, + error=result.stderr, + ) + + return ConversionResult( + markdown=result.stdout, tool="markitdown", success=True + ) + except FileNotFoundError: + # Try Python API + try: + from markitdown import MarkItDown + + md = MarkItDown() + result = md.convert(str(file_path)) + return ConversionResult( + markdown=result.text_content, tool="markitdown", success=True + ) + except Exception as e: + return ConversionResult( + markdown="", tool="markitdown", success=False, error=str(e) + ) + except Exception as e: + return ConversionResult( + markdown="", tool="markitdown", success=False, error=str(e) + ) + + +def convert_with_pandoc( + file_path: Path, assets_dir: Optional[Path] = None +) -> ConversionResult: + """Convert using pandoc. + + Pandoc's --extract-media=DIR creates a media/ subdirectory inside DIR. + We point --extract-media at assets_dir's parent so pandoc's media/ + subdirectory lands exactly at assets_dir (when assets_dir ends with 'media'), + or we use a temp dir and move files afterward. + """ + try: + cmd = ["pandoc", str(file_path), "-t", "markdown", "--wrap=none"] + + extract_dir = None + if assets_dir: + assets_dir.mkdir(parents=True, exist_ok=True) + # Pandoc always creates a media/ subdirectory inside --extract-media. + # Point it at the parent so media/ lands at assets_dir. + if assets_dir.name == "media": + extract_dir = assets_dir.parent + else: + extract_dir = assets_dir + cmd.extend(["--extract-media", str(extract_dir)]) + + result = subprocess.run( + cmd, capture_output=True, text=True, timeout=120 + ) + + if result.returncode != 0: + return ConversionResult( + markdown="", tool="pandoc", success=False, error=result.stderr + ) + + md = result.stdout + + # Convert absolute image paths to relative paths based on output location + if extract_dir: + abs_media = str(extract_dir / "media") + # Replace absolute paths with relative 'media/' prefix + md = md.replace(abs_media + "/", "media/") + + images = _collect_images(assets_dir) if assets_dir else [] + + return ConversionResult( + markdown=md, tool="pandoc", images=images, success=True + ) + except Exception as e: + return ConversionResult( + markdown="", tool="pandoc", success=False, error=str(e) + ) + + +def convert_single( + file_path: Path, tool: str, assets_dir: Optional[Path] = None +) -> ConversionResult: + """Run a single conversion tool.""" + converters = { + "pymupdf4llm": convert_with_pymupdf4llm, + "markitdown": convert_with_markitdown, + "pandoc": convert_with_pandoc, + "docx-deep": convert_with_docx_deep, + } + + converter = converters.get(tool) + if not converter: + return ConversionResult( + markdown="", tool=tool, success=False, error=f"Unknown tool: {tool}" + ) + + return converter(file_path, assets_dir) + + +def merge_results(results: list[ConversionResult]) -> ConversionResult: + """Merge results from multiple tools, selecting best segments.""" + if not results: + return ConversionResult(markdown="", tool="none", success=False) + + # Filter successful results + successful = [r for r in results if r.success and r.markdown.strip()] + if not successful: + # Return first error + return results[0] if results else ConversionResult( + markdown="", tool="none", success=False + ) + + if len(successful) == 1: + return successful[0] + + # Multiple successful results - merge them + # Strategy: Compare key metrics and select best + best = successful[0] + best_score = score_markdown(best.markdown) + + for result in successful[1:]: + score = score_markdown(result.markdown) + if score > best_score: + best = result + best_score = score + + # Merge images from all results + all_images = [] + seen = set() + for result in successful: + for img in result.images: + if img not in seen: + all_images.append(img) + seen.add(img) + + best.images = all_images + best.tool = f"merged({','.join(r.tool for r in successful)})" + + return best + + +def score_markdown(md: str) -> float: + """Score markdown quality for comparison.""" + score = 0.0 + + # Length (more content is generally better) + score += min(len(md) / 10000, 5.0) # Cap at 5 points + + # Tables (proper markdown tables) + table_count = md.count("|---|") + md.count("| ---") + score += min(table_count * 0.5, 3.0) + + # Images (referenced images) + image_count = md.count("![") + score += min(image_count * 0.3, 2.0) + + # Headings (proper hierarchy) + h1_count = md.count("\n# ") + h2_count = md.count("\n## ") + h3_count = md.count("\n### ") + if h1_count > 0 and h2_count >= h1_count: + score += 1.0 # Good hierarchy + + # Lists (structured content) + list_count = md.count("\n- ") + md.count("\n* ") + md.count("\n1. ") + score += min(list_count * 0.1, 2.0) + + # Penalize pandoc artifacts (grid tables, attributes) + artifact_count = md.count("+:---") + md.count("+---+") + artifact_count += md.count('{width="') + md.count("{.underline}") + score -= artifact_count * 0.5 + + return score + + +def main(): + parser = argparse.ArgumentParser( + description="Convert documents to markdown with multi-tool orchestration", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Quick mode (default) + python convert.py document.pdf -o output.md + + # Heavy mode (best quality) + python convert.py document.pdf -o output.md --heavy + + # DOCX deep mode (python-docx parsing) + python convert.py document.docx -o output.md --docx-deep + + # With custom assets directory + python convert.py document.pdf -o output.md --assets-dir ./images + """, + ) + parser.add_argument("input", type=Path, nargs="?", help="Input document path") + parser.add_argument( + "-o", "--output", type=Path, help="Output markdown file" + ) + parser.add_argument( + "--heavy", + action="store_true", + help="Enable Heavy Mode (multi-tool, best quality)", + ) + parser.add_argument( + "--docx-deep", + action="store_true", + help="Use python-docx direct parsing (experimental, DOCX only)", + ) + parser.add_argument( + "--no-postprocess", + action="store_true", + help="Disable DOCX post-processing (keep raw pandoc output)", + ) + parser.add_argument( + "--assets-dir", + type=Path, + default=None, + help="Directory for extracted images (default: _assets/)", + ) + parser.add_argument( + "--tool", + choices=["pymupdf4llm", "markitdown", "pandoc", "docx-deep"], + help="Force specific tool (overrides auto-selection)", + ) + parser.add_argument( + "--list-tools", + action="store_true", + help="List available tools and exit", + ) + + args = parser.parse_args() + + # List tools mode + if args.list_tools: + tools = ["pymupdf4llm", "markitdown", "pandoc", "docx-deep"] + print("Available conversion tools:") + for tool in tools: + status = "+" if check_tool_available(tool) else "-" + print(f" {status} {tool}") + sys.exit(0) + + # Validate input + if args.input is None: + parser.error("the following arguments are required: input") + if not args.input.exists(): + print(f"Error: Input file not found: {args.input}", file=sys.stderr) + sys.exit(1) + + # Determine output path + output_path = args.output or args.input.with_suffix(".md") + + # Determine assets directory + assets_dir = args.assets_dir + if assets_dir is None: + assets_dir = output_path.parent / f"{output_path.stem}_assets" + + is_docx = args.input.suffix.lower() in (".docx", ".doc") + + # Handle --docx-deep mode + if args.docx_deep: + if not is_docx: + print("Error: --docx-deep only works with DOCX files.", file=sys.stderr) + sys.exit(1) + tools = ["docx-deep"] + elif args.tool: + tools = [args.tool] if check_tool_available(args.tool) else [] + else: + # Select tools + mode = "heavy" if args.heavy else "quick" + tools = select_tools(args.input, mode) + + mode = "docx-deep" if args.docx_deep else ("heavy" if args.heavy else "quick") + + if not tools: + print("Error: No conversion tools available.", file=sys.stderr) + print("Install with:", file=sys.stderr) + print(" pip install pymupdf4llm", file=sys.stderr) + print(" uv tool install markitdown[pdf]", file=sys.stderr) + print(" brew install pandoc", file=sys.stderr) + sys.exit(1) + + print(f"Converting: {args.input}") + print(f"Mode: {mode.upper()}") + print(f"Tools: {', '.join(tools)}") + + # Run conversions + results = [] + for tool in tools: + print(f" Running {tool}...", end=" ", flush=True) + + # Use separate assets dirs for each tool in heavy mode + tool_assets = None + if assets_dir and mode == "heavy" and len(tools) > 1: + tool_assets = assets_dir / tool + elif assets_dir: + tool_assets = assets_dir + + result = convert_single(args.input, tool, tool_assets) + results.append(result) + + if result.success: + print(f"ok ({len(result.markdown):,} chars, {len(result.images)} images)") + else: + print(f"FAIL ({result.error[:50]}...)") + + # Merge results if heavy mode + if mode == "heavy" and len(results) > 1: + print(" Merging results...", end=" ", flush=True) + final = merge_results(results) + print(f"ok (using {final.tool})") + else: + final = merge_results(results) + + if not final.success: + print(f"Error: Conversion failed: {final.error}", file=sys.stderr) + sys.exit(1) + + # Apply DOCX post-processing + if is_docx and not args.no_postprocess and "pandoc" in final.tool: + print(" Post-processing DOCX output...", end=" ", flush=True) + final.markdown, pp_stats = postprocess_docx_markdown( + final.markdown, assets_dir + ) + print(f"ok ({pp_stats.summary()})") + + # Write output + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(final.markdown) + + print(f"\nOutput: {output_path}") + print(f" Size: {len(final.markdown):,} characters") + if final.images: + print(f" Images: {len(final.images)} extracted") + + +if __name__ == "__main__": + main() diff --git a/markdown-tools/scripts/convert_path.py b/doc-to-markdown/scripts/convert_path.py similarity index 100% rename from markdown-tools/scripts/convert_path.py rename to doc-to-markdown/scripts/convert_path.py diff --git a/markdown-tools/scripts/extract_pdf_images.py b/doc-to-markdown/scripts/extract_pdf_images.py similarity index 100% rename from markdown-tools/scripts/extract_pdf_images.py rename to doc-to-markdown/scripts/extract_pdf_images.py diff --git a/markdown-tools/scripts/merge_outputs.py b/doc-to-markdown/scripts/merge_outputs.py similarity index 100% rename from markdown-tools/scripts/merge_outputs.py rename to doc-to-markdown/scripts/merge_outputs.py diff --git a/doc-to-markdown/scripts/test_convert.py b/doc-to-markdown/scripts/test_convert.py new file mode 100644 index 00000000..d334209b --- /dev/null +++ b/doc-to-markdown/scripts/test_convert.py @@ -0,0 +1,242 @@ +"""Tests for doc-to-markdown convert.py post-processing functions. + +Run: uv run pytest scripts/test_convert.py -v +""" + +import pytest +import re +import sys +from pathlib import Path + +# Import the module under test +sys.path.insert(0, str(Path(__file__).parent)) +from convert import ( + _fix_cjk_bold_spacing, + _build_pipe_table, + _collect_images, + PostProcessStats, + postprocess_docx_markdown, +) + + +# ── CJK Bold Spacing ───────────────────────────────────────────────────────── + + +class TestCjkBoldSpacing: + """Test _fix_cjk_bold_spacing: spaces between **bold** and CJK chars.""" + + def test_bold_followed_by_cjk_punctuation(self): + """**text** directly touching CJK colon → add space after **.""" + inp = "**打开阶跃开放平台链接**：https://platform.stepfun.com/" + out = _fix_cjk_bold_spacing(inp) + assert "**打开阶跃开放平台链接** ：" in out + + def test_cjk_before_bold(self): + """CJK char directly before ** → add space before **.""" + assert _fix_cjk_bold_spacing("可用**手机号**进行") == "可用 **手机号** 进行" + + def test_bold_with_emoji_neighbor(self): + """**text** touching emoji ➡️ → still add space (CJK content rule).""" + inp = "点击**【接口密码】**➡️**【创建新的密钥**】" + out = _fix_cjk_bold_spacing(inp) + # Each CJK-containing bold span should have spaces on both sides + assert "点击 **【接口密码】** ➡️" in out + assert "➡️ **【创建新的密钥**" in out + + def test_full_emoji_line(self): + """Complete line with emoji separators between bold spans.""" + inp = "点击**【接口密码】**➡️**【创建新的密钥**】➡️**【输入密钥名称】**（输入你想取的名称），生成API Key" + out = _fix_cjk_bold_spacing(inp) + assert "点击 **【接口密码】** ➡️" in out + assert "**【输入密钥名称】** （输入" in out + + def test_bold_between_cjk(self): + """CJK **text** CJK → spaces on both sides.""" + assert _fix_cjk_bold_spacing("打开**飞书**，就可以") == "打开 **飞书** ，就可以" + + def test_bold_with_chinese_quotes(self): + """Bold containing Chinese quotes.""" + inp = '有个**"企鹅戴龙虾头套的机器人"**，开始' + out = _fix_cjk_bold_spacing(inp) + assert '**"企鹅戴龙虾头套的机器人"** ，' in out + + def test_multiple_bold_spans(self): + """Multiple bold spans in one line.""" + assert _fix_cjk_bold_spacing("这是**测试**和**验证**的效果") == "这是 **测试** 和 **验证** 的效果" + + def test_already_spaced(self): + """Already has spaces → no double spaces.""" + inp = "已有空格 **粗体** 不需要再加" + assert _fix_cjk_bold_spacing(inp) == inp + + def test_english_unchanged(self): + """English bold text should not be modified.""" + inp = "English **bold** text should not change" + assert _fix_cjk_bold_spacing(inp) == inp + + def test_line_start_bold(self): + """Bold at line start followed by CJK.""" + assert _fix_cjk_bold_spacing("**重要**内容") == "**重要** 内容" + + def test_line_start_bold_standalone(self): + """Bold at line start with no CJK neighbor → no change.""" + assert _fix_cjk_bold_spacing("**这是纯粗体不需要改**") == "**这是纯粗体不需要改**" + + def test_no_bold(self): + """Text without bold markers → unchanged.""" + inp = "这是普通文本，没有粗体" + assert _fix_cjk_bold_spacing(inp) == inp + + def test_empty_string(self): + assert _fix_cjk_bold_spacing("") == "" + + def test_bold_at_line_end(self): + """Bold at line end → no trailing space needed.""" + assert _fix_cjk_bold_spacing("内容是**粗体**") == "内容是 **粗体**" + + def test_mixed_cjk_and_english_bold(self): + """English bold between CJK → no change (no CJK in content).""" + inp = "请使用 **API Key** 进行认证" + assert _fix_cjk_bold_spacing(inp) == inp + + +# ── Pipe Table Builder ──────────────────────────────────────────────────────── + + +class TestBuildPipeTable: + """Test _build_pipe_table: rows → markdown pipe table.""" + + def test_basic_table(self): + rows = [["a", "b"], ["c", "d"]] + result = _build_pipe_table(rows) + assert result == [ + "| | |", + "| --- | --- |", + "| a | b |", + "| c | d |", + ] + + def test_uneven_rows(self): + """Rows with different column counts → padded.""" + rows = [["a", "b", "c"], ["d"]] + result = _build_pipe_table(rows) + assert "| d | | |" in result + + def test_single_cell(self): + rows = [["only"]] + result = _build_pipe_table(rows) + assert len(result) == 3 # header + sep + 1 row + + def test_empty_rows(self): + assert _build_pipe_table([]) == [] + + def test_image_with_caption(self): + """Images and captions should pair correctly in table.""" + rows = [ + ["![](img1.png)", "![](img2.png)"], + ["Step 1", "Step 2"], + ] + result = _build_pipe_table(rows) + assert "| ![](img1.png) | ![](img2.png) |" in result + assert "| Step 1 | Step 2 |" in result + + +# ── Full Post-Processing Pipeline ───────────────────────────────────────────── + + +class TestPostprocessPipeline: + """Integration tests for the full postprocess_docx_markdown pipeline.""" + + def test_grid_table_single_column_to_blockquote(self): + """Single-column grid table → blockquote.""" + inp = """+:---+ +| 注意事项 | ++----+""" + out, stats = postprocess_docx_markdown(inp) + assert "> 注意事项" in out + assert "+:---+" not in out + + def test_pandoc_attributes_removed(self): + """Pandoc {width=...} and {.underline} removed.""" + inp = '![](img.png){width="5in" height="3in"} and [text]{.underline}' + out, stats = postprocess_docx_markdown(inp) + assert "{width=" not in out + assert "{.underline}" not in out + assert "![](img.png)" in out + + def test_escaped_brackets_fixed(self): + r"""Pandoc \[ and \] → [ and ].""" + inp = r"你 \[在飞书里\] 发消息" + out, stats = postprocess_docx_markdown(inp) + assert "你 [在飞书里] 发消息" in out + + def test_double_bracket_links_fixed(self): + """[[text]](url) → [text](url).""" + inp = "[[点击跳转]](https://example.com)" + out, stats = postprocess_docx_markdown(inp) + assert "[点击跳转](https://example.com)" in out + + def test_code_block_with_language(self): + """Indented dashed block with JSON language hint → ```json.""" + inp = """ ------------------------------------------------------------------ + JSON\\ + {\\ + "provider": "stepfun"\\ + } + ------------------------------------------------------------------""" + out, stats = postprocess_docx_markdown(inp) + assert "```json" in out + assert '"provider": "stepfun"' in out + assert "---" not in out + + def test_code_block_plain_text_to_blockquote(self): + """Indented dashed block with plain text → blockquote.""" + inp = """ -------------------------- + 注意：这是一条重要提示 + --------------------------""" + out, stats = postprocess_docx_markdown(inp) + assert "> 注意：这是一条重要提示" in out + + def test_cjk_bold_spacing_in_pipeline(self): + """CJK bold spacing is applied in the full pipeline.""" + inp = "打开**飞书**，就可以看到" + out, stats = postprocess_docx_markdown(inp) + assert "打开 **飞书** ，就可以看到" in out + + def test_excessive_blank_lines_collapsed(self): + """4+ blank lines → 2 blank lines.""" + inp = "line1\n\n\n\n\nline2" + out, stats = postprocess_docx_markdown(inp) + assert out.count("\n") < 5 + + def test_stats_tracking(self): + """Stats object correctly tracks fix counts.""" + inp = '![](media/media/img.png){width="5in"}' + out, stats = postprocess_docx_markdown(inp) + assert stats.attributes_removed > 0 + + +# ── Simple Table (pandoc) ───────────────────────────────────────────────────── + + +class TestSimpleTable: + """Test pandoc simple table (indented dashes with spaces) → pipe table.""" + + def test_two_column_image_table(self): + """Two images side by side in simple table → pipe table.""" + inp = """ ---- ---- + ![](img1.png) ![](img2.png) + + ---- ----""" + out, stats = postprocess_docx_markdown(inp) + assert "| ![](img1.png) | ![](img2.png) |" in out + assert "----" not in out + + def test_four_column_image_table(self): + """Four images in simple table → 4-column pipe table.""" + inp = """ ---------- ---------- ---------- ---------- + ![](a.png) ![](b.png) ![](c.png) ![](d.png) + + ---------- ---------- ---------- ----------""" + out, stats = postprocess_docx_markdown(inp) + assert "| ![](a.png) | ![](b.png) | ![](c.png) | ![](d.png) |" in out diff --git a/markdown-tools/scripts/validate_output.py b/doc-to-markdown/scripts/validate_output.py similarity index 100% rename from markdown-tools/scripts/validate_output.py rename to doc-to-markdown/scripts/validate_output.py diff --git a/douban-skill/SKILL.md b/douban-skill/SKILL.md new file mode 100644 index 00000000..a1ce9782 --- /dev/null +++ b/douban-skill/SKILL.md @@ -0,0 +1,131 @@ +--- +name: douban-skill +description: > + Export and sync Douban (豆瓣) book/movie/music/game collections to local CSV files via Frodo API. + Supports full export (all history) and RSS incremental sync (recent items). + Use when the user wants to export Douban reading/watching/listening/gaming history, + back up their Douban data, set up incremental sync, or mentions 豆瓣/douban collections. + Triggers on: 豆瓣, douban, 读书记录, 观影记录, 书影音, 导出豆瓣, export, backup, sync, collection. +--- + +# Douban Collection Export + +Export Douban user collections (books, movies, music, games) to CSV files. +Douban has no official data export; the official API shut down in 2018. + +## What This Skill Can Do + +- Full export of all book/movie/music/game collections via Frodo API +- RSS incremental sync for daily updates (last ~10 items) +- CSV output with UTF-8 BOM (Excel-compatible), cross-platform (macOS/Windows/Linux) +- No login, no cookies, no browser required +- Pre-flight user ID validation (fail fast on wrong ID) + +## What This Skill Cannot Do + +- Cannot export reviews (长评), notes (读书笔记), or broadcasts (广播) +- Cannot filter by single category in one run (exports all 4 types together) +- Cannot access private profiles (returns 0 items silently) + +## Why Frodo API (Do NOT Use Web Scraping) + +Douban uses PoW (Proof of Work) challenges on web pages, blocking all HTTP scraping. +We tested 7 approaches — only the Frodo API works. **Do NOT attempt** web scraping, +`browser_cookie3`+`requests`, `curl` with cookies, or Jina Reader. + +See [references/troubleshooting.md](references/troubleshooting.md) for the complete +failure log of all 7 tested approaches and why each failed. + +## Security & Privacy + +The API key and HMAC secret in the script are Douban's **public mobile app credentials**, +extracted from the APK. They are shared by all Douban app users and do not identify you. +No personal credentials are used or stored. Data is fetched only from `frodo.douban.com`. + +## Full Export (Primary Method) + +```bash +DOUBAN_USER= python3 scripts/douban-frodo-export.py +``` + +**Finding the user ID:** Profile URL `douban.com/people//` — the ID is after `/people/`. +If the user provides a full URL, the script auto-extracts the ID. + +**Environment variables:** +- `DOUBAN_USER` (required): Douban user ID (alphanumeric or numeric, or full profile URL) +- `DOUBAN_OUTPUT_DIR` (optional): Override output directory + +**Default output** (auto-detected per platform): +- macOS: `~/Downloads/douban-sync//` +- Windows: `%USERPROFILE%\Downloads\douban-sync\\` +- Linux: `~/Downloads/douban-sync//` + +**Dependencies:** Python 3.6+ standard library only (works with `python3` or `uv run`). + +**Example console output:** +``` +Douban Export for user: your_douban_id +Output directory: /Users/you/Downloads/douban-sync/your_douban_id + +=== 读过 (book) === + Total: 639 + Fetched 0-50 (50/639) + Fetched 50-100 (100/639) + ... + Fetched 597-639 (639/639) + Collected: 639 + +=== 在读 (book) === + Total: 75 + ... + +--- Writing CSV files --- + 书.csv: 996 rows + 影视.csv: 238 rows + 音乐.csv: 0 rows + 游戏.csv: 0 rows + +Done! 1234 total items exported to /Users/you/Downloads/douban-sync/your_douban_id +``` + +## RSS Incremental Sync (Complementary) + +```bash +DOUBAN_USER= node scripts/douban-rss-sync.mjs +``` + +RSS returns only the latest ~10 items (no pagination). Use Full Export first, then RSS for daily updates. + +## Output Format + +Four CSV files per user: + +``` +Downloads/douban-sync// +├── 书.csv (读过 + 在读 + 想读) +├── 影视.csv (看过 + 在看 + 想看) +├── 音乐.csv (听过 + 在听 + 想听) +└── 游戏.csv (玩过 + 在玩 + 想玩) +``` + +Columns: `title, url, date, rating, status, comment` +- `rating`: ★ to ★★★★★ (empty if unrated) +- `date`: YYYY-MM-DD (when the user marked it) +- Safe to run multiple times (overwrites with fresh data) +- Row counts may be slightly below Douban's displayed count due to delisted items + +## Workflow + +1. Ask for Douban user ID (from profile URL, or accept full URL) +2. Run: `DOUBAN_USER= python3 scripts/douban-frodo-export.py` +3. Verify: row counts in console output should match, check with `wc -l /*.csv` +4. (Optional) Set up RSS sync for daily incremental updates + +## Troubleshooting + +See [references/troubleshooting.md](references/troubleshooting.md) for: +- Frodo API auth details (HMAC-SHA1 signature computation) +- Common errors (code 996 signature error, rate limits, pagination quirks) +- Complete failure log of all 7 tested approaches with root causes +- Alternative approaches (豆伴 extension, Tampermonkey script, browser console) +- API endpoint reference with response format diff --git a/douban-skill/references/troubleshooting.md b/douban-skill/references/troubleshooting.md new file mode 100644 index 00000000..501b115a --- /dev/null +++ b/douban-skill/references/troubleshooting.md @@ -0,0 +1,265 @@ +# Troubleshooting & Technical Reference + +## How Frodo API Auth Works + +The Frodo API is Douban's mobile app backend at `frodo.douban.com`. It uses HMAC-SHA1 signature +authentication instead of the PoW challenges used on web pages. + +**Signature computation:** +1. Build raw string: `GET` + `&` + URL-encoded(**path only**) + `&` + timestamp +2. HMAC-SHA1 with secret key `bf7dddc7c9cfe6f7` +3. Base64-encode the result → this is `_sig` + +**Critical:** Sign only the URL **path** (e.g., `/api/v2/user/xxx/interests`), never the +full URL with query parameters. This was our first signature error — code 996. + +**Required query parameters:** +- `apiKey`: `0dad551ec0f84ed02907ff5c42e8ec70` (Douban mobile app's public API key) +- `_ts`: Unix timestamp in **seconds** (string) +- `_sig`: The computed HMAC-SHA1 signature +- `os_rom`: `android` + +**Required headers:** +- `User-Agent`: Must look like a Douban Android app client string + +**Python implementation:** +```python +import hmac, hashlib, base64, urllib.parse + +def compute_signature(url_path, timestamp): + raw = '&'.join(['GET', urllib.parse.quote(url_path, safe=''), timestamp]) + sig = hmac.new(b'bf7dddc7c9cfe6f7', raw.encode(), hashlib.sha1) + return base64.b64encode(sig.digest()).decode() +``` + +## Common Errors + +### Signature Error (code 996) + +```json +{"msg": "invalid_request_996", "code": 996} +``` + +**Cause:** The `_sig` parameter doesn't match the expected value. + +**Debug checklist:** +1. Are you signing only the **path**, not the full URL with query params? +2. Does `_ts` in the signature match `_ts` in the query params exactly? +3. Is `_ts` a string of Unix seconds (not milliseconds)? +4. Are you using `urllib.parse.quote(path, safe='')` (encoding `/` as `%2F`)? + +### Pagination Returns Fewer Items Than Expected + +Some pages return fewer than the requested `count` (e.g., 48 instead of 50). This happens +when items have been delisted from Douban's catalog but still count toward the total. + +**This was our biggest silent bug.** The first version of the export script used +`len(page_items) < count_per_page` as the stop condition. Result: only 499 out of 639 +books were exported, with no error message. The fix: + +```python +# WRONG: stops early when a page has fewer items due to delisted content +if len(interests) < count_per_page: + break + +# CORRECT: check against the total count reported by the API +if len(all_items) >= total: + break +start += len(interests) # advance by actual count, not page_size +``` + +### Rating Scale Confusion + +The Frodo API returns **two different ratings** per item: + +| Field | Scale | Meaning | +|-------|-------|---------| +| `interest.rating` | `{value: 1-5, max: 5}` | **User's personal rating** | +| `subject.rating` | `{value: 0-10, max: 10}` | Douban community average | + +Our first version divided all values by 2, which halved the user's rating (2 stars → 1 star). +The fix: check `max` field to determine scale. + +```python +# Correct conversion +if max_val <= 5: + stars = int(val) # value is already 1-5 +else: + stars = int(val / 2) # value is 2-10, convert to 1-5 +``` + +### HTTP 403 / Rate Limiting + +The Frodo API is generally tolerant, but excessive requests may trigger rate limiting. + +**Tested intervals:** +- 1.5s between pages + 2s between categories: 1234 items exported without issues +- 0s (no delay): Not tested, not recommended + +If you hit 403, increase delays to 3s/5s and retry after a few minutes. + +## Detailed Failure Log: All 7 Tested Approaches + +### Approach 1: `requests` + `browser_cookie3` (Python) + +**What we tried:** Extract Chrome cookies via `browser_cookie3`, use `requests` with those cookies. + +**What happened:** +1. First request succeeded — we saw "639 books" in the page title +2. Subsequent requests returned "禁止访问" (Forbidden) page +3. The HTML contained no items despite HTTP 200 status + +**Root cause:** Douban's PoW challenge. The first request sometimes passes (cached/grace period), +but subsequent requests trigger the PoW redirect to `sec.douban.com`. Python `requests` cannot +execute the SHA-512 proof-of-work JavaScript. + +### Approach 2: `curl` with browser cookies + +**What we tried:** Export cookies from Chrome, use `curl` with full browser headers (User-Agent, +Accept, Referer, Accept-Language). + +**What happened:** HTTP 302 redirect to `https://www.douban.com/misc/sorry?original-url=...` + +**Root cause:** Same PoW issue. Even with `NO_PROXY` set to bypass local proxy, the IP was +already rate-limited from approach 1's requests. + +### Approach 3: Jina Reader (`r.jina.ai`) + +**What we tried:** `curl -s "https://r.jina.ai/https://book.douban.com/people//collect"` + +**What happened:** HTTP 200 but content was "403 Forbidden" — Jina's server got blocked. + +**Root cause:** Jina's scraping infrastructure also cannot solve Douban's PoW challenges. + +### Approach 4: Chrome DevTools MCP (Playwright browser) + +**What we tried:** Navigate to Douban pages in the Playwright browser via Chrome DevTools MCP. +Injected cookies via `document.cookie` in evaluate_script. + +**What happened:** +1. `mcp__chrome-devtools__navigate_page` → page title was "403 Forbidden" +2. After cookie injection → still redirected to `/misc/sorry` + +**Root cause:** The Chrome DevTools MCP connects to a Playwright browser instance, not the +user's actual Chrome. Even after injecting cookies, the IP was already banned from earlier +requests. Also, HttpOnly cookies (like `dbcl2`) can't be set via `document.cookie`. + +### Approach 5: `opencli douban marks` + +**What we tried:** `opencli douban marks --uid --status all --limit 0 -f csv` + +**What happened:** **Partial success** — exported 24 movie records successfully. + +**Limitation:** `opencli douban` only implements `marks` (movies). No book/music/game support. +The `opencli generate` and `opencli cascade` commands failed to discover APIs for +`book.douban.com` because Douban books use server-rendered HTML with no discoverable API. + +### Approach 6: Agent Reach + +**What we tried:** Installed `agent-reach` (17-platform CLI tool). Checked for Douban support. + +**What happened:** Agent Reach has no Douban channel. Its web reader (Jina) also gets 403. + +### Approach 7: Node.js HTTP scraper (from douban-sync-skill) + +**What we tried:** The `douban-scraper.mjs` from the cosformula/douban-sync-skill. + +**Status:** User rejected the command before it ran — based on prior failures, it would hit +the same PoW blocking. The script uses `fetch()` with a fake User-Agent, which is exactly +what approaches 1-3 proved doesn't work. + +## Alternative Approaches (Not Blocked) + +These approaches work but have different tradeoffs compared to the Frodo API: + +### 豆伴 (Tofu) Chrome Extension (605 stars) + +- GitHub: `doufen-org/tofu` +- Uses Douban's **Rexxar API** (`m.douban.com/rexxar/api/v2/user/{uid}/interests`) +- Most comprehensive: backs up books, movies, music, games, reviews, notes, photos, etc. +- **Current status (April 2026):** Mainline v0.12.x is broken due to MV3 migration + anti-scraping. + PR #121 (v0.13.0) fixes both issues but is not yet merged. +- **Risk:** Makes many API calls as logged-in user → may trigger account lockout + +### Tampermonkey Userscript (bambooom/douban-backup, 162 stars) + +- Greasemonkey/Tampermonkey: `https://greasyfork.org/en/scripts/420999` +- Runs inside real browser → inherits PoW-solved session +- Adds "export" button on collection pages → auto-paginates → downloads CSV +- Suitable for one-time manual export + +### Browser Console Script (built into old skill) + +- Paste `fetch()`-based extraction script into browser DevTools console +- Zero blocking risk (same-origin request from authenticated session) +- Most manual approach — user must paste script and copy clipboard + +## API Endpoint Reference + +### User Interests (Collections) + +``` +GET https://frodo.douban.com/api/v2/user/{user_id}/interests + ?type={book|movie|music|game} + &status={done|doing|mark} + &start={offset} + &count={page_size, max 50} + &apiKey=0dad551ec0f84ed02907ff5c42e8ec70 + &_ts={unix_timestamp_seconds} + &_sig={hmac_sha1_signature} + &os_rom=android +``` + +**Response:** +```json +{ + "count": 50, + "start": 0, + "total": 639, + "interests": [ + { + "comment": "短评文本", + "rating": {"value": 4, "max": 5, "star_count": 4.0}, + "create_time": "2026-03-21 18:23:10", + "status": "done", + "id": 4799352304, + "subject": { + "id": "36116375", + "title": "书名", + "url": "https://book.douban.com/subject/36116375/", + "rating": {"value": 7.8, "max": 10, "count": 14} + } + } + ] +} +``` + +**Important distinctions:** +- `interest.rating` = user's personal rating (max 5) +- `subject.rating` = Douban community average (max 10) +- `interest.create_time` = when the user marked it (not the item's publish date) +- `status`: `done` = 读过/看过/听过/玩过, `doing` = 在读/在看/在听/在玩, `mark` = 想读/想看/想听/想玩 + +### Other Known Frodo Endpoints (Not Used by This Skill) + +| Endpoint | Returns | +|----------|---------| +| `/api/v2/book/{id}` | Book detail | +| `/api/v2/movie/{id}` | Movie detail | +| `/api/v2/group/{id}/topics` | Group discussion topics | +| `/api/v2/group/topic/{id}` | Single topic with comments | +| `/api/v2/subject_collection/{type}/items` | Douban curated lists | + +### Mouban Proxy Service (Third-Party) + +`mouban.mythsman.com` is a Go service that pre-crawls Douban data. If a user has been indexed, +it returns data instantly without hitting Douban directly. Endpoints: + +| Endpoint | Returns | +|----------|---------| +| `GET /guest/check_user?id={douban_id}` | User profile + counts | +| `GET /guest/user_book?id={id}&action={wish\|do\|collect}` | Book entries | +| `GET /guest/user_movie?id={id}&action=...` | Movie entries | + +**Caveat:** Data freshness depends on when the service last crawled the user. First request +for a new user triggers a background crawl (takes minutes to hours). Third-party dependency. diff --git a/douban-skill/scripts/douban-frodo-export.py b/douban-skill/scripts/douban-frodo-export.py new file mode 100644 index 00000000..2c6a0038 --- /dev/null +++ b/douban-skill/scripts/douban-frodo-export.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +""" +Douban Collection Full Export via Frodo API (Mobile App Backend) + +Exports all book/movie/music/game collections to CSV files. +No login or cookies required — uses HMAC-SHA1 signature auth. + +The API key and HMAC secret are Douban's mobile app credentials, extracted from +the public APK. They are the same for all users and do not identify you. No +personal credentials are used or stored. Data is fetched only from frodo.douban.com. + +Usage: + DOUBAN_USER= python3 douban-frodo-export.py + DOUBAN_USER= DOUBAN_OUTPUT_DIR=/custom/path python3 douban-frodo-export.py + +Environment: + DOUBAN_USER (required): Douban user ID from profile URL + DOUBAN_OUTPUT_DIR (optional): Override output directory +""" + +import hmac +import hashlib +import base64 +import csv +import json +import os +import platform +import re +import socket +import sys +import time +import urllib.parse +import urllib.request +import urllib.error + +# --- Frodo API Auth --- +# Public credentials from the Douban Android APK, shared by all app users. +API_KEY = '0dad551ec0f84ed02907ff5c42e8ec70' +HMAC_SECRET = b'bf7dddc7c9cfe6f7' +BASE_URL = 'https://frodo.douban.com' +USER_AGENT = ( + 'api-client/1 com.douban.frodo/7.22.0.beta9(231) Android/23 ' + 'product/Mate40 vendor/HUAWEI model/Mate40 brand/HUAWEI ' + 'rom/android network/wifi platform/AndroidPad' +) + +# --- Rate Limiting --- +# 1.5s between pages, 2s between categories. Tested with 1200+ items. +PAGE_DELAY = 1.5 +CATEGORY_DELAY = 2.0 +ITEMS_PER_PAGE = 50 +MAX_PAGES_SAFETY = 500 # Guard against infinite pagination loops + +# --- Category Definitions --- +CATEGORIES = [ + ('book', 'done', '读过', '书.csv'), + ('book', 'doing', '在读', '书.csv'), + ('book', 'mark', '想读', '书.csv'), + ('movie', 'done', '看过', '影视.csv'), + ('movie', 'doing', '在看', '影视.csv'), + ('movie', 'mark', '想看', '影视.csv'), + ('music', 'done', '听过', '音乐.csv'), + ('music', 'doing', '在听', '音乐.csv'), + ('music', 'mark', '想听', '音乐.csv'), + ('game', 'done', '玩过', '游戏.csv'), + ('game', 'doing', '在玩', '游戏.csv'), + ('game', 'mark', '想玩', '游戏.csv'), +] + +URL_PREFIX = { + 'book': 'https://book.douban.com/subject/', + 'movie': 'https://movie.douban.com/subject/', + 'music': 'https://music.douban.com/subject/', + 'game': 'https://www.douban.com/game/', +} + +CSV_FIELDS = ['title', 'url', 'date', 'rating', 'status', 'comment'] + + +def get_download_dir(): + """Get the platform-appropriate Downloads directory.""" + system = platform.system() + if system == 'Darwin': + return os.path.expanduser('~/Downloads') + elif system == 'Windows': + return os.path.join(os.environ.get('USERPROFILE', os.path.expanduser('~')), 'Downloads') + else: + return os.path.expanduser('~/Downloads') + + +def get_output_dir(user_id): + """Determine output directory from env or platform default.""" + base = os.environ.get('DOUBAN_OUTPUT_DIR') + if not base: + base = os.path.join(get_download_dir(), 'douban-sync') + return os.path.join(base, user_id) + + +def compute_signature(url_path, timestamp): + """Compute Frodo API HMAC-SHA1 signature. + + Signs: METHOD & url_encoded_path & timestamp (path only, no query params). + """ + raw = '&'.join(['GET', urllib.parse.quote(url_path, safe=''), timestamp]) + sig = hmac.new(HMAC_SECRET, raw.encode(), hashlib.sha1) + return base64.b64encode(sig.digest()).decode() + + +def fetch_json(url, params): + """Make an authenticated GET request to the Frodo API. + + Returns (data_dict, status_code). Catches HTTP errors, network errors, + and timeouts — all return a synthetic error dict so the caller can retry. + """ + query = urllib.parse.urlencode(params) + full_url = f'{url}?{query}' + req = urllib.request.Request(full_url, headers={'User-Agent': USER_AGENT}) + try: + with urllib.request.urlopen(req, timeout=15) as resp: + return json.loads(resp.read().decode('utf-8')), resp.status + except urllib.error.HTTPError as e: + body = e.read().decode('utf-8', errors='replace')[:200] + return {'error': body, 'code': e.code}, e.code + except urllib.error.URLError as e: + return {'error': f'Network error: {e.reason}'}, 0 + except socket.timeout: + return {'error': 'Request timed out'}, 0 + except json.JSONDecodeError as e: + return {'error': f'Invalid JSON response: {e}'}, 0 + + +def preflight_check(user_id): + """Verify user exists by fetching one page of book interests. + + Returns True if the user has any data, False if the user ID appears invalid. + Prints a warning and continues if the check itself fails (network issue). + """ + api_path = f'/api/v2/user/{user_id}/interests' + ts = str(int(time.time())) + sig = compute_signature(api_path, ts) + params = { + 'type': 'book', 'status': 'done', 'start': 0, 'count': 1, + 'apiKey': API_KEY, '_ts': ts, '_sig': sig, 'os_rom': 'android', + } + data, code = fetch_json(f'{BASE_URL}{api_path}', params) + if code == 0: + print(f'Warning: Could not verify user ID (network issue). Proceeding anyway.') + return True + if code != 200: + print(f'Error: API returned HTTP {code} for user "{user_id}".') + print(f' Check that the user ID is correct (from douban.com/people//).') + return False + total = data.get('total', -1) + if total == -1: + print(f'Warning: Unexpected API response. Proceeding anyway.') + return True + return True + + +def fetch_all_interests(user_id, type_name, status): + """Fetch all items for a given type+status combination. + + Paginates through the API, checking against the reported total + (not page size) to handle pages with fewer items due to delisted content. + """ + api_path = f'/api/v2/user/{user_id}/interests' + all_items = [] + start = 0 + total = None + retries = 0 + max_retries = 3 + page_count = 0 + + while page_count < MAX_PAGES_SAFETY: + page_count += 1 + ts = str(int(time.time())) + sig = compute_signature(api_path, ts) + params = { + 'type': type_name, 'status': status, + 'start': start, 'count': ITEMS_PER_PAGE, + 'apiKey': API_KEY, '_ts': ts, '_sig': sig, 'os_rom': 'android', + } + + data, status_code = fetch_json(f'{BASE_URL}{api_path}', params) + + if status_code != 200: + retries += 1 + if retries > max_retries: + print(f' Error: HTTP {status_code} after {max_retries} retries, stopping.') + print(f' See references/troubleshooting.md for common errors.') + break + delay = 5 * (2 ** (retries - 1)) + print(f' HTTP {status_code}, retry {retries}/{max_retries}, waiting {delay}s...') + time.sleep(delay) + continue + + retries = 0 + + if total is None: + total = data.get('total', 0) + if total == 0: + return [] + print(f' Total: {total}') + + interests = data.get('interests', []) + if not interests: + break + + all_items.extend(interests) + print(f' Fetched {start}-{start + len(interests)} ({len(all_items)}/{total})') + + if len(all_items) >= total: + break + start += len(interests) + time.sleep(PAGE_DELAY) + + return all_items + + +def extract_rating(interest): + """Convert Frodo API rating to star string. + + Frodo returns {value: N, max: 5} where N is 1-5. + Some older entries may use max=10 scale (value 2-10). + API values are typically integers; round() handles any edge cases. + """ + r = interest.get('rating') + if not r or not isinstance(r, dict): + return '' + val = r.get('value', 0) + max_val = r.get('max', 5) + if not val: + return '' + stars = round(val) if max_val <= 5 else round(val / 2) + return '★' * max(0, min(5, stars)) + + +def interest_to_row(interest, type_name, status_cn): + """Convert a single Frodo API interest object to a CSV row dict.""" + subject = interest.get('subject', {}) + sid = subject.get('id', '') + prefix = URL_PREFIX.get(type_name, 'https://www.douban.com/subject/') + url = f'{prefix}{sid}/' if sid else subject.get('url', '') + + date_raw = interest.get('create_time', '') or '' + date = date_raw[:10] if re.match(r'\d{4}-\d{2}-\d{2}', date_raw) else '' + + return { + 'title': subject.get('title', ''), + 'url': url, + 'date': date, + 'rating': extract_rating(interest), + 'status': status_cn, + 'comment': interest.get('comment', ''), + } + + +def write_csv(filepath, rows): + """Write rows to a CSV file with UTF-8 BOM for Excel compatibility.""" + with open(filepath, 'w', newline='', encoding='utf-8-sig') as f: + writer = csv.DictWriter(f, fieldnames=CSV_FIELDS) + writer.writeheader() + writer.writerows(rows) + + +def main(): + user_id = os.environ.get('DOUBAN_USER', '').strip() + if not user_id: + print('Error: DOUBAN_USER environment variable is required') + print('Usage: DOUBAN_USER= python3 douban-frodo-export.py') + sys.exit(1) + + # Extract ID from URL if user pasted a full profile URL + url_match = re.search(r'douban\.com/people/([A-Za-z0-9._-]+)', user_id) + if url_match: + user_id = url_match.group(1) + + if not re.match(r'^[A-Za-z0-9._-]+$', user_id): + print(f'Error: DOUBAN_USER contains invalid characters: {user_id}') + sys.exit(1) + + output_dir = get_output_dir(user_id) + os.makedirs(output_dir, exist_ok=True) + print(f'Douban Export for user: {user_id}') + print(f'Output directory: {output_dir}\n') + + # Pre-flight: verify user ID is valid before spending time on full export + if not preflight_check(user_id): + sys.exit(1) + + # Collect data grouped by output file, then write all at the end. + file_data = {} + grand_total = 0 + + for type_name, status, status_cn, outfile in CATEGORIES: + print(f'=== {status_cn} ({type_name}) ===') + items = fetch_all_interests(user_id, type_name, status) + + if outfile not in file_data: + file_data[outfile] = [] + + for item in items: + file_data[outfile].append(interest_to_row(item, type_name, status_cn)) + + count = len(items) + grand_total += count + if count > 0: + print(f' Collected: {count}\n') + else: + print(f' (empty)\n') + + time.sleep(CATEGORY_DELAY) + + # Write CSV files + print('--- Writing CSV files ---') + for filename, rows in file_data.items(): + filepath = os.path.join(output_dir, filename) + write_csv(filepath, rows) + print(f' {filename}: {len(rows)} rows') + + print(f'\nDone! {grand_total} total items exported to {output_dir}') + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print('\n\nExport interrupted by user.') + sys.exit(130) diff --git a/douban-skill/scripts/douban-rss-sync.mjs b/douban-skill/scripts/douban-rss-sync.mjs new file mode 100644 index 00000000..c90b85c4 --- /dev/null +++ b/douban-skill/scripts/douban-rss-sync.mjs @@ -0,0 +1,190 @@ +#!/usr/bin/env node +/** + * Douban RSS → CSV incremental sync + * + * Pulls the public RSS feed, parses new entries, appends to CSV files. + * No login required. Returns only the ~10 most recent items. + * Best used for daily sync after a full Frodo API export. + * + * Usage: + * DOUBAN_USER= node douban-rss-sync.mjs + * + * Environment: + * DOUBAN_USER (required): Douban user ID + * DOUBAN_OUTPUT_DIR (optional): Override output directory + */ + +import https from 'node:https'; +import http from 'node:http'; +import fs from 'node:fs'; +import path from 'node:path'; +import os from 'node:os'; + +let DOUBAN_USER = process.env.DOUBAN_USER; +if (!DOUBAN_USER) { console.error('Error: DOUBAN_USER env var is required'); process.exit(1); } +// Extract ID from full URL if provided (e.g., https://www.douban.com/people/foo/) +const urlMatch = DOUBAN_USER.match(/douban\.com\/people\/([A-Za-z0-9._-]+)/); +if (urlMatch) DOUBAN_USER = urlMatch[1]; +if (!/^[A-Za-z0-9._-]+$/.test(DOUBAN_USER)) { console.error('Error: DOUBAN_USER contains invalid characters'); process.exit(1); } + +function getDownloadDir() { + if (process.platform === 'win32') { + return path.join(process.env.USERPROFILE || os.homedir(), 'Downloads'); + } + return path.join(os.homedir(), 'Downloads'); +} + +const BASE_DIR = process.env.DOUBAN_OUTPUT_DIR || path.join(getDownloadDir(), 'douban-sync'); +const DOUBAN_OUTPUT_DIR = path.join(BASE_DIR, DOUBAN_USER); +const STATE_FILE = path.join(DOUBAN_OUTPUT_DIR, '.douban-rss-state.json'); +const RSS_URL = `https://www.douban.com/feed/people/${DOUBAN_USER}/interests`; + +const CATEGORY_MAP = [ + { pattern: /^读过/, file: '书.csv', status: '读过' }, + { pattern: /^(?:在读|最近在读)/, file: '书.csv', status: '在读' }, + { pattern: /^想读/, file: '书.csv', status: '想读' }, + { pattern: /^看过/, file: '影视.csv', status: '看过' }, + { pattern: /^(?:在看|最近在看)/, file: '影视.csv', status: '在看' }, + { pattern: /^想看/, file: '影视.csv', status: '想看' }, + { pattern: /^听过/, file: '音乐.csv', status: '听过' }, + { pattern: /^(?:在听|最近在听)/, file: '音乐.csv', status: '在听' }, + { pattern: /^想听/, file: '音乐.csv', status: '想听' }, + { pattern: /^玩过/, file: '游戏.csv', status: '玩过' }, + { pattern: /^(?:在玩|最近在玩)/, file: '游戏.csv', status: '在玩' }, + { pattern: /^想玩/, file: '游戏.csv', status: '想玩' }, +]; + +const CSV_HEADER = '\ufefftitle,url,date,rating,status,comment\n'; +const RATING_MAP = { '力荐': '★★★★★', '推荐': '★★★★', '还行': '★★★', '较差': '★★', '很差': '★' }; + +function httpGet(url, redirects = 0) { + if (redirects > 5) return Promise.reject(new Error('Too many redirects')); + return new Promise((resolve, reject) => { + const mod = url.startsWith('https') ? https : http; + const req = mod.get(url, { headers: { 'User-Agent': 'Mozilla/5.0' }, timeout: 15000 }, res => { + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + return httpGet(new URL(res.headers.location, url).href, redirects + 1).then(resolve, reject); + } + if (res.statusCode >= 400) return reject(new Error(`HTTP ${res.statusCode} for ${url}`)); + let data = ''; + res.on('data', c => data += c); + res.on('end', () => resolve(data)); + }); + req.on('error', reject); + req.on('timeout', () => { req.destroy(); reject(new Error('Request timeout')); }); + }); +} + +function csvEscape(str) { + if (!str) return ''; + if (str.includes(',') || str.includes('"') || str.includes('\n') || str.includes('\r')) { + return '"' + str.replace(/"/g, '""') + '"'; + } + return str; +} + +function parseItems(xml) { + const items = []; + const itemRegex = /([\s\S]*?)<\/item>/g; + let match; + while ((match = itemRegex.exec(xml)) !== null) { + const block = match[1]; + const get = tag => { + const m = block.match(new RegExp(`<${tag}[^>]*>(?:)?<\\/${tag}>`)); + return m ? m[1].trim() : ''; + }; + const title = get('title'); + const link = get('link'); + const guid = get('guid'); + const pubDate = get('pubDate'); + const desc = get('description'); + const ratingMatch = desc.match(/推荐:\s*(力荐|推荐|还行|较差|很差)/); + const rating = ratingMatch ? RATING_MAP[ratingMatch[1]] || '' : ''; + const commentMatch = desc.match(/短评:\s*([^<]+)/); + const comment = commentMatch ? commentMatch[1].trim() : ''; + items.push({ title, link, guid, pubDate, rating, comment }); + } + return items; +} + +function loadState() { + try { return JSON.parse(fs.readFileSync(STATE_FILE, 'utf8')); } + catch { return { lastSyncGuids: [] }; } +} + +function saveState(state) { fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 2)); } + +function extractName(title) { + for (const { pattern } of CATEGORY_MAP) { + if (pattern.test(title)) return title.replace(pattern, ''); + } + return title; +} + +function isAlreadyInFile(filePath, link) { + try { + const content = fs.readFileSync(filePath, 'utf8'); + // Exact URL match as CSV field — avoid false positives from substring matches + // (e.g., /subject/1234/ matching /subject/12345/) + return content.includes(',' + link + ',') || + content.includes(',' + link + '\n') || + content.includes(',' + link + '\r'); + } catch { return false; } +} + +function formatDate(pubDateStr) { + try { + const direct = pubDateStr.match(/(\d{4}-\d{2}-\d{2})/); + if (direct) return direct[1]; + const d = new Date(pubDateStr); + const cst = new Date(d.getTime() + 8 * 3600000); + return cst.toISOString().split('T')[0]; + } catch { return ''; } +} + +function ensureCsvFile(filePath) { + if (!fs.existsSync(filePath)) { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, CSV_HEADER); + } +} + +function appendToCsv(filePath, entry, status) { + ensureCsvFile(filePath); + const name = extractName(entry.title); + const date = formatDate(entry.pubDate); + const line = [csvEscape(name), csvEscape(entry.link), csvEscape(date), + csvEscape(entry.rating), csvEscape(status), csvEscape(entry.comment)].join(',') + '\n'; + fs.appendFileSync(filePath, line); +} + +async function main() { + console.log(`Douban RSS Sync for user: ${DOUBAN_USER}`); + console.log(`Output: ${DOUBAN_OUTPUT_DIR}\n`); + console.log('Fetching RSS feed...'); + const xml = await httpGet(RSS_URL); + const items = parseItems(xml); + console.log(`Found ${items.length} items in feed`); + + const state = loadState(); + const knownGuids = new Set(state.lastSyncGuids || []); + let newCount = 0; + + for (const item of items) { + if (knownGuids.has(item.guid)) continue; + const cat = CATEGORY_MAP.find(c => c.pattern.test(item.title)); + if (!cat) { console.log(` Skip (unknown category): ${item.title}`); continue; } + const filePath = path.join(DOUBAN_OUTPUT_DIR, cat.file); + if (isAlreadyInFile(filePath, item.link)) { console.log(` Skip (exists): ${item.title}`); continue; } + console.log(` + ${item.title} → ${cat.file}`); + appendToCsv(filePath, item, cat.status); + newCount++; + } + + state.lastSyncGuids = items.map(i => i.guid); + state.lastSync = new Date().toISOString(); + saveState(state); + console.log(`\nDone. ${newCount} new entries added.`); +} + +main().catch(err => { console.error('Error:', err.message); process.exit(1); }); diff --git a/github-contributor/SKILL.md b/github-contributor/SKILL.md index 103c7732..70b7666a 100644 --- a/github-contributor/SKILL.md +++ b/github-contributor/SKILL.md @@ -118,6 +118,23 @@ gh search repos "topic:cli" --sort=stars --limit=20 ## PR Excellence +### The High-Quality PR Formula + +Based on real-world successful contributions to major open-source projects: + +``` +1. Deep investigation (post to issue, not PR) +2. Minimal, surgical fix (only change what's necessary) +3. Regression test (prevent future breakage) +4. CHANGELOG entry (if project uses it) +5. End-to-end validation (prove bug exists, prove fix works) +6. Clear PR structure (~50 lines, focused) +7. Professional communication +8. Separate concerns (detailed analysis in issue, fix summary in PR) +9. No internal/irrelevant details +10. Responsive to feedback +``` + ### Before Writing Code ``` @@ -127,6 +144,38 @@ Pre-PR Checklist: - [ ] Comment on issue to claim it - [ ] Understand project conventions - [ ] Set up development environment +- [ ] Trace through git history for context +- [ ] Identify root cause with evidence +``` + +### Investigation Phase (Post to Issue) + +**Do this BEFORE coding**: + +1. **Reproduce the bug** with exact commands and output +2. **Trace git history** to understand context + ```bash + git log --all --grep="keyword" --oneline + git blame file.ts | grep "relevant_line" + ``` +3. **Link related issues/PRs** that provide context +4. **Post detailed analysis to issue** (not PR) + - Timeline of related changes + - Root cause explanation + - Why previous approaches didn't work + +**Example structure**: +```markdown +## Investigation + +I traced this through the codebase history: + +1. [Date]: #[PR] introduced [feature] +2. [Date]: #[PR] added [workaround] because [reason] +3. [Date]: #[PR] changed [parameter] +4. Now: Safe to [fix] because [explanation] + +[Detailed evidence with code references] ``` ### Writing the PR @@ -140,76 +189,151 @@ docs(readme): update installation instructions for Windows refactor(validation): extract validation logic into separate module ``` -**Evidence loop**: Prove the change with a reproducible fail -> fix -> pass loop. +**Keep PR description focused** (~50 lines): +- Summary (1-2 sentences) +- Root cause (technical, with code refs) +- Changes (bullet list) +- Why it's safe +- Testing approach +- Related issues + +**Move detailed investigation to issue comments**, not PR. + +### Evidence Loop + +**Critical**: Prove the change with a reproducible fail → fix → pass loop. + +1. **Reproduce failure** with original version + ```bash + # Test with original version + npm install -g package@original-version + [command that triggers bug] + # Capture: error messages, exit codes, timestamps + ``` -- Reproduce the failure with one command. -- Apply the fix. -- Rerun the exact same command and capture the passing output. -- Compare baseline vs fixed vs reference behavior. -- Redact local paths, secrets, tokens, and internal hostnames before posting logs or screenshots. +2. **Apply fix** and test with patched version + ```bash + # Test with fixed version + npm install -g package@fixed-version + [same command] + # Capture: success output, normal exit codes + ``` -**Description**: Structured and thorough +3. **Document both** with timestamps, PIDs, exit codes, logs + +4. **Redact sensitive info**: + - Local absolute paths (`/Users/...`, `/home/...`) + - Secrets/tokens/API keys + - Internal URLs/hostnames + - Recheck every pasted block before submitting + +**Description**: Focused and reviewable (~50 lines) ````markdown ## Summary -[What this PR does in 1-2 sentences] +[1-2 sentences: what this fixes and why] -## Motivation -[Why this change is needed] +## Root Cause +[Technical explanation with code references] ## Changes -- [Change 1] -- [Change 2] +- [Actual code changes] +- [Tests added] +- [Docs updated] -## Evidence Loop -Command: -```bash -# Baseline (before fix) -[command] +## Why This Is Safe +[Explain why it won't break anything] -# Fixed (after fix, same command) -[command] -``` +## Testing -Raw output: +### Test 1: Reproduce Bug (Original Version) +Command: `[command]` +Result: ```text -[paste baseline output] +[failure output with timestamps, exit codes] ``` +### Test 2: Validate Fix (Patched Version) +Command: `[same command]` +Result: ```text -[paste fixed output] +[success output with timestamps, exit codes] ``` -## Comparison -| Case | Command / Scenario | Result | Evidence | -|------|--------------------|--------|----------| -| Baseline | `[same command]` | Fail | [raw output block] | -| Fixed | `[same command]` | Pass | [raw output block] | -| Reference | [spec, issue, or main branch behavior] | Expected | [link or note] | +## Related +- Fixes #[issue] +- Related: #[other issues/PRs] +```` -## Sources/Attribution -- [Issue, docs, or code references] +**What NOT to include in PR**: +- ❌ Detailed timeline analysis (put in issue) +- ❌ Historical context (put in issue) +- ❌ Internal tooling mentions +- ❌ Speculation or uncertainty +- ❌ Walls of text (>100 lines) -## Risks -- [Potential downside and impact] +### Code Changes Best Practices -## Rollback Plan -- Revert commit(s): [hash] -- Restore previous behavior with: [command] +**Minimal, surgical fixes**: +- ✅ Only change what's necessary to fix the bug +- ✅ Add regression test to prevent future breakage +- ✅ Update CHANGELOG if project uses it +- ❌ Don't refactor surrounding code +- ❌ Don't add "improvements" beyond the fix +- ❌ Don't change unrelated files -## Testing -[How you tested this] +**Example** (OpenClaw PR #39763): +``` +Files changed: 2 +- src/infra/process-respawn.ts (3 lines removed, 1 added) +- src/infra/process-respawn.test.ts (regression test added) -## Screenshots (if UI) -[Before/After images] -```` +Result: 278K star project, clean approval +``` + +### Separation of Concerns + +**Issue comments**: Detailed investigation +- Timeline analysis +- Historical context +- Related PRs/issues +- Root cause deep dive + +**PR description**: Focused on the fix +- Summary (1-2 sentences) +- Root cause (technical) +- Changes (bullet list) +- Testing validation +- ~50 lines total + +**Separate test comment**: End-to-end validation +- Test with original version (prove bug) +- Test with fixed version (prove fix) +- Full logs with timestamps ### After Submitting -- Respond to feedback promptly +- Monitor CI results +- Respond to feedback promptly (within 24 hours) - Make requested changes quickly - Be grateful for reviews -- Don't argue, discuss +- Don't argue, discuss professionally +- If you need to update PR: + - Add new commits (don't force push during review) + - Explain what changed in comment + - Re-request review when ready + +**Professional responses**: +``` +✅ "Good point! I've updated the implementation to..." +✅ "Thanks for catching that. Fixed in commit abc123." +✅ "I see what you mean. I chose this approach because... + Would you prefer if I changed it to...?" + +❌ "That's just your opinion." +❌ "It works on my machine." +❌ "This is how I always do it." +``` ## Building Reputation @@ -243,35 +367,70 @@ Level 4: Maintainer status ### Don't -- Submit drive-by PRs without context +- Submit drive-by PRs without investigation +- Include detailed timeline in PR (put in issue) +- Mention internal tooling or infrastructure - Argue with maintainers - Ignore code style guidelines - Make massive changes without discussion - Ghost after submitting +- Refactor code unrelated to the fix +- Add "improvements" beyond what was requested +- Force push during review (unless asked) ### Do +- Investigate thoroughly BEFORE coding +- Post detailed analysis to issue, not PR +- Keep PR focused and minimal (~50 lines) - Start with small, focused PRs - Follow project conventions exactly +- Add regression tests +- Update CHANGELOG if project uses it - Communicate proactively - Accept feedback gracefully - Build relationships over time +- Test with both original and fixed versions +- Redact sensitive info from logs ## Workflow Template ``` -Contribution Workflow: +High-Quality Contribution Workflow: + +Investigation Phase: - [ ] Find project with "good first issue" - [ ] Read contribution guidelines - [ ] Comment on issue to claim +- [ ] Reproduce bug with original version +- [ ] Trace git history for context +- [ ] Identify root cause with evidence +- [ ] Post detailed analysis to issue + +Implementation Phase: - [ ] Fork and set up locally -- [ ] Make focused changes -- [ ] Run evidence loop (reproduce fail -> apply fix -> rerun same command pass) -- [ ] Add baseline vs fixed vs reference comparison -- [ ] Redact paths/secrets/internal hosts in logs and screenshots -- [ ] Test thoroughly -- [ ] Write clear PR description -- [ ] Respond to review feedback +- [ ] Make minimal, focused changes +- [ ] Add regression test +- [ ] Update CHANGELOG (if applicable) +- [ ] Follow project conventions exactly + +Validation Phase: +- [ ] Test with original version (prove bug exists) +- [ ] Test with fixed version (prove fix works) +- [ ] Document both with timestamps/logs +- [ ] Redact paths/secrets/internal hosts + +Submission Phase: +- [ ] Write focused PR description (~50 lines) +- [ ] Link to detailed issue analysis +- [ ] Post end-to-end test results +- [ ] Ensure CI passes + +Review Phase: +- [ ] Respond to feedback within 24 hours +- [ ] Make requested changes quickly +- [ ] Don't force push during review +- [ ] Thank reviewers - [ ] Celebrate when merged! 🎉 ``` @@ -310,3 +469,28 @@ Types: `feat`, `fix`, `docs`, `style`, `refactor`, `test`, `chore` - `references/pr_checklist.md` - Complete PR quality checklist - `references/project_evaluation.md` - How to evaluate projects - `references/communication_templates.md` - Issue/PR templates +- `references/high_quality_pr_case_study.md` - Real-world successful PR walkthrough (OpenClaw #39763) + +## Success Indicators + +You know you have a high-quality PR when: + +- ✅ Maintainers understand the problem immediately +- ✅ Reviewers can verify the fix easily +- ✅ CI passes on first try +- ✅ No "can you explain..." questions +- ✅ Minimal back-and-forth +- ✅ Quick approval + +## Key Metrics for Quality PRs + +Based on successful contributions to major projects: + +- **Files changed**: 1-3 (focused scope) +- **Lines changed**: 10-50 (minimal fix) +- **PR description**: ~50 lines (concise) +- **Issue investigation**: 100-300 lines (thorough) +- **Time to first draft**: 2-3 days (proper investigation) +- **Time to ready**: 3-5 days (including validation) +- **Response time**: <24 hours (professional) + diff --git a/github-contributor/references/high_quality_pr_case_study.md b/github-contributor/references/high_quality_pr_case_study.md new file mode 100644 index 00000000..8942ab43 --- /dev/null +++ b/github-contributor/references/high_quality_pr_case_study.md @@ -0,0 +1,328 @@ +# High-Quality PR: Real-World Case Study + +Based on OpenClaw PR #39763 - A successful bug fix contribution to a 278K star TypeScript project. + +## What Made This PR High-Quality + +### 1. Complete Evidence Chain + +**Issue → Root Cause → Fix → Validation** + +``` +✅ Original bug report with symptoms +✅ Deep investigation with timeline analysis +✅ Root cause identified in source code +✅ Minimal, surgical fix +✅ End-to-end testing with before/after comparison +✅ Regression test added +``` + +### 2. Thorough Investigation Before Coding + +**Timeline Analysis** (Posted to issue, not PR): +- Traced bug through 3 years of related changes +- Identified when workaround was added (#29078) +- Explained why removing workaround is now safe +- Linked to all relevant historical PRs and issues + +**Key insight**: Detailed investigation goes in the issue, not the PR. Keep PR focused on the fix. + +### 3. Minimal, Focused Changes + +**Files changed**: 2 +- `src/infra/process-respawn.ts` - 3 lines removed, 1 line added +- `src/infra/process-respawn.test.ts` - Updated tests + regression test + +**What we didn't do**: +- ❌ Refactor surrounding code +- ❌ Add "improvements" beyond the fix +- ❌ Change unrelated files +- ❌ Add extensive comments + +### 4. Regression Test Added + +```typescript +test("launchd path never returns failed status", () => { + const result = detectSupervisor("launchd"); + expect(result.mode).not.toBe("failed"); + expect(result.mode).toBe("supervised"); +}); +``` + +**Why this matters**: Prevents the bug from being reintroduced. + +### 5. CHANGELOG Entry + +Following project conventions: +```markdown +## [Unreleased] + +### Fixed +- **darwin/launchd**: Remove `kickstart -k` self-restart to prevent race condition with launchd bootout (#39763) +``` + +**Key**: Check if project maintains CHANGELOG and follow their format exactly. + +### 6. Clear PR Structure + +**Title**: `fix(darwin): remove launchd kickstart race condition` +- Conventional commit format +- Scope indicates platform +- Clear what was fixed + +**Body** (~50 lines, trimmed from original 136): +```markdown +## Summary +[2 sentences: what + why] + +## Root Cause +[Technical explanation with code references] + +## Changes +- [Bullet list of actual changes] + +## Why This Is Safe +[Explain why the fix won't break anything] + +## Testing +[How it was validated] + +## Related +- Fixes #39760 +- Related: #27650, #29078 +``` + +### 7. Separation of Concerns + +**Issue comment**: Detailed timeline, investigation, evidence +**PR description**: Focused on the fix, testing, safety +**Separate test comment**: End-to-end validation results + +**Why**: Keeps PR reviewable. Detailed context available but not blocking review. + +### 8. End-to-End Testing + +**Test 1**: Reproduced bug with original version +``` +Result: Bootstrap failed: 5, SIGKILL, exit code -9 ✅ +``` + +**Test 2**: Validated fix with patched version +``` +Result: Clean restart, no errors, normal exit code ✅ +``` + +**Evidence**: Posted full logs with timestamps, PIDs, exit codes. + +### 9. What We Avoided + +**❌ Don't mention internal tooling**: +- We had a custom monitor script that auto-remediated the bug +- We initially mentioned it in PR comments +- **Removed it** because it's not part of OpenClaw - would confuse maintainers + +**❌ Don't over-explain in PR**: +- Moved detailed timeline analysis to issue +- Kept PR focused on fix validation + +**❌ Don't add noise**: +- No "I think this might work" comments +- No "please review" pings +- No unnecessary updates + +### 10. Professional Communication + +**In issue**: +```markdown +## Timeline Analysis + +I traced this through the codebase history: + +1. 2023-05: #27650 set ThrottleInterval to 60s +2. 2023-08: #29078 added kickstart workaround +3. Later: ThrottleInterval reduced to 1s +4. Now: Safe to remove kickstart + +[Detailed evidence with links] +``` + +**In PR**: +```markdown +## Testing Complete ✅ + +End-to-end testing confirms: +1. Bug reproduced with 2026.3.7 +2. Fix validated with PR branch +3. Ready for review + +Full logs: [link to issue comment] +``` + +## The High-Quality PR Formula + +``` +1. Deep investigation (post to issue) +2. Minimal, surgical fix +3. Regression test +4. CHANGELOG entry (if project uses it) +5. End-to-end validation +6. Clear PR structure +7. Professional communication +8. Separate concerns (issue vs PR) +9. No internal/irrelevant details +10. Responsive to feedback +``` + +## PR Lifecycle + +``` +Day 1: Investigation + ├─ Reproduce bug locally + ├─ Trace through codebase history + ├─ Identify root cause + └─ Post detailed analysis to issue + +Day 2: Implementation + ├─ Create minimal fix + ├─ Add regression test + ├─ Update CHANGELOG + └─ Test locally + +Day 3: Validation + ├─ Test with original version (reproduce bug) + ├─ Test with fixed version (validate fix) + ├─ Document test results + └─ Submit PR + +Day 4: Refinement + ├─ Trim PR description (move details to issue) + ├─ Add context about historical changes + ├─ Post end-to-end test results + └─ Mark ready for review + +Day 5+: Review cycle + ├─ Respond to feedback promptly + ├─ Make requested changes + └─ Wait for CI and approval +``` + +## Key Metrics + +**OpenClaw PR #39763**: +- Files changed: 2 +- Lines added: ~20 (including tests) +- Lines removed: 3 +- PR description: ~50 lines +- Issue investigation: ~200 lines +- Time to first draft: 3 days +- Time to ready: 4 days + +## What Maintainers Look For + +Based on this experience: + +1. **Does it fix the problem?** ✅ Bug reproduced and fixed +2. **Is it minimal?** ✅ Only changed what's necessary +3. **Will it break anything?** ✅ Explained why it's safe +4. **Can it regress?** ✅ Added regression test +5. **Is it documented?** ✅ CHANGELOG entry +6. **Is it tested?** ✅ End-to-end validation +7. **Is it reviewable?** ✅ Clear structure, focused scope + +## Anti-Patterns We Avoided + +1. ❌ **Drive-by PR**: "Here's a fix, hope it works" + - ✅ We did: Deep investigation, thorough testing + +2. ❌ **Kitchen sink PR**: "Fixed bug + refactored + added features" + - ✅ We did: Minimal, focused fix only + +3. ❌ **No evidence PR**: "Trust me, it works" + - ✅ We did: Reproduced bug, validated fix, posted logs + +4. ❌ **Wall of text PR**: 500-line description + - ✅ We did: Trimmed to ~50 lines, moved details to issue + +5. ❌ **Ghost PR**: Submit and disappear + - ✅ We did: Responsive, iterative refinement + +## Lessons Learned + +### Investigation Phase +- Trace through git history to understand context +- Link to all related issues and PRs +- Post detailed analysis to issue, not PR + +### Implementation Phase +- Make the smallest possible change +- Add regression test +- Follow project conventions exactly + +### Validation Phase +- Test with original version (prove bug exists) +- Test with fixed version (prove fix works) +- Document both with timestamps and logs + +### Communication Phase +- Keep PR focused and reviewable +- Move detailed context to issue +- Remove internal/irrelevant details +- Be professional and responsive + +## Template for Future PRs + +```markdown +## Summary +[1-2 sentences: what this fixes and why] + +## Root Cause +[Technical explanation with code references] + +## Changes +- [Actual code changes] +- [Tests added] +- [Docs updated] + +## Why This Is Safe +[Explain why it won't break anything] + +## Testing +[How you validated the fix] + +### Test 1: Reproduce Bug +Command: `[command]` +Result: [failure output] + +### Test 2: Validate Fix +Command: `[same command]` +Result: [success output] + +## Related +- Fixes #[issue] +- Related: #[other issues] +``` + +## Success Indicators + +You know you have a high-quality PR when: + +- ✅ Maintainers understand the problem immediately +- ✅ Reviewers can verify the fix easily +- ✅ CI passes on first try +- ✅ No "can you explain..." questions +- ✅ Minimal back-and-forth +- ✅ Quick approval + +## Final Checklist + +Before submitting: +- [ ] Bug reproduced with original version +- [ ] Fix validated with patched version +- [ ] Regression test added +- [ ] CHANGELOG updated (if applicable) +- [ ] PR description is focused (~50 lines) +- [ ] Detailed investigation in issue, not PR +- [ ] No internal tooling mentioned +- [ ] No local paths/secrets in logs +- [ ] All tests pass +- [ ] Follows project conventions diff --git a/github-contributor/references/pr_checklist.md b/github-contributor/references/pr_checklist.md index 319a1b5d..0e07aba7 100644 --- a/github-contributor/references/pr_checklist.md +++ b/github-contributor/references/pr_checklist.md @@ -2,13 +2,37 @@ Complete checklist for creating high-quality pull requests. -## Before Starting +# PR Quality Checklist + +Complete checklist for creating high-quality pull requests based on successful contributions to major open-source projects. + +## Investigation Phase (Before Coding) - [ ] Read CONTRIBUTING.md thoroughly - [ ] Check for existing PRs addressing same issue - [ ] Comment on issue to express interest -- [ ] Wait for maintainer acknowledgment (if required) -- [ ] Understand project's code style +- [ ] Reproduce bug with original version +- [ ] Trace git history for context (`git log --all --grep="keyword"`) +- [ ] Identify root cause with code references +- [ ] Post detailed investigation to issue (not PR) +- [ ] Link all related issues/PRs + +### Investigation Template (Post to Issue) + +```markdown +## Investigation + +I traced this through the codebase history: + +1. [Date]: #[PR] introduced [feature] +2. [Date]: #[PR] added [workaround] because [reason] +3. [Date]: #[PR] changed [parameter] +4. Now: Safe to [fix] because [explanation] + +[Detailed evidence with code references] +``` + +## Before Starting ## Environment Setup @@ -28,11 +52,27 @@ refactor/extract-validation-logic ## During Development -- [ ] Make small, focused commits +- [ ] Make minimal, focused changes (only what's necessary) +- [ ] Add regression test to prevent future breakage +- [ ] Update CHANGELOG if project uses it - [ ] Follow project's commit message format -- [ ] Add tests for new functionality -- [ ] Update documentation if needed - [ ] Run linter/formatter before committing +- [ ] Don't refactor unrelated code +- [ ] Don't add "improvements" beyond the fix + +### What to Change + +✅ **Do change**: +- Code directly related to the fix +- Tests for the fix +- CHANGELOG entry +- Relevant documentation + +❌ **Don't change**: +- Surrounding code (refactoring) +- Unrelated files +- Code style of existing code +- Add features beyond the fix ## Before Submitting @@ -46,10 +86,25 @@ refactor/extract-validation-logic ### Evidence Loop -- [ ] Reproduce the failure with one command and capture baseline output +- [ ] Test with original version and capture failure output - [ ] Apply the fix -- [ ] Rerun the exact same command and capture passing output -- [ ] Add baseline vs fixed vs reference comparison +- [ ] Test with fixed version and capture success output +- [ ] Document both tests with timestamps, exit codes, PIDs +- [ ] Compare baseline vs fixed behavior + +### Testing Commands + +```bash +# Test 1: Reproduce bug with original version +npm install -g package@original-version +[command that triggers bug] +# Capture: error messages, exit codes, timestamps + +# Test 2: Validate fix with patched version +npm install -g package@fixed-version +[same command] +# Capture: success output, normal exit codes +``` ### Redaction Gate @@ -67,15 +122,32 @@ refactor/extract-validation-logic ### PR Description -- [ ] Clear, descriptive title -- [ ] Summary of changes -- [ ] Motivation/context -- [ ] Testing approach -- [ ] Screenshots (for UI changes) +- [ ] Clear, descriptive title (conventional commit format) +- [ ] Focused description (~50 lines, not >100) +- [ ] Summary (1-2 sentences) +- [ ] Root cause (technical, with code refs) +- [ ] Changes (bullet list) +- [ ] Why it's safe +- [ ] Testing validation - [ ] Related issues linked -- [ ] Sources/Attribution section added -- [ ] Risks section added -- [ ] Rollback Plan section added +- [ ] No detailed timeline (move to issue) +- [ ] No internal tooling mentions +- [ ] No speculation or uncertainty + +### PR Description Length Guide + +✅ **Good**: ~50 lines +- Summary: 2 lines +- Root cause: 5 lines +- Changes: 5 lines +- Why safe: 5 lines +- Testing: 20 lines +- Related: 3 lines + +❌ **Too long**: >100 lines +- Move detailed investigation to issue +- Move timeline analysis to issue +- Keep PR focused on the fix ## PR Title Format @@ -96,82 +168,51 @@ chore(deps): update lodash to 4.17.21 ````markdown ## Summary -Brief description of what this PR does. - -## Motivation +[1-2 sentences: what this fixes and why] -Why is this change needed? Link to issue if applicable. +## Root Cause -Closes #123 +[Technical explanation with code references] ## Changes -- Change 1 -- Change 2 -- Change 3 +- [Actual code changes] +- [Tests added] +- [Docs updated] -## Evidence Loop +## Why This Is Safe -Command: -```bash -# Baseline (before fix) -[command] +[Explain why it won't break anything] -# Fixed (after fix, same command) -[command] -``` +## Testing -Raw output: +### Test 1: Reproduce Bug (Original Version) +Command: `[command]` +Result: ```text -[baseline output] +[failure output with timestamps, exit codes] ``` +### Test 2: Validate Fix (Patched Version) +Command: `[same command]` +Result: ```text -[fixed output] +[success output with timestamps, exit codes] ``` -## Comparison (Baseline vs Fixed vs Reference) - -| Case | Command / Scenario | Result | Evidence | -|------|--------------------|--------|----------| -| Baseline | `[same command]` | Fail | [raw output block] | -| Fixed | `[same command]` | Pass | [raw output block] | -| Reference | [spec, issue, or main behavior] | Expected | [link or note] | - -## Sources/Attribution - -- [Issue, docs, benchmark source, or code reference] - -## Risks +## Related -- [Risk and impact] - -## Rollback Plan - -- Revert commit(s): [hash] -- Restore previous behavior with: [command] - -## Testing - -How did you test this change? - -- [ ] Unit tests added/updated -- [ ] Manual testing performed -- [ ] Integration tests pass - -## Screenshots (if applicable) - -| Before | After | -|--------|-------| -| image | image | - -## Checklist - -- [ ] Tests pass -- [ ] Documentation updated -- [ ] Code follows project style +- Fixes #[issue] +- Related: #[other issues/PRs] ```` +**What NOT to include**: +- ❌ Detailed timeline analysis (put in issue) +- ❌ Historical context (put in issue) +- ❌ Internal tooling mentions +- ❌ Speculation or uncertainty +- ❌ Walls of text (>100 lines) + ## Comparison Table Template ```markdown @@ -185,10 +226,34 @@ How did you test this change? ## After Submitting - [ ] Monitor for CI results -- [ ] Respond to review comments promptly +- [ ] Respond to review comments within 24 hours - [ ] Make requested changes quickly - [ ] Thank reviewers for their time - [ ] Don't force push after review starts (unless asked) +- [ ] Add new commits during review (don't amend) +- [ ] Explain what changed in follow-up comments +- [ ] Re-request review when ready + +## Separation of Concerns + +### Issue Comments (Detailed Investigation) +- Timeline analysis +- Historical context +- Related PRs/issues +- Root cause deep dive +- 100-300 lines OK + +### PR Description (Focused on Fix) +- Summary (1-2 sentences) +- Root cause (technical) +- Changes (bullet list) +- Testing validation +- ~50 lines total + +### Separate Test Comment (End-to-End Validation) +- Test with original version +- Test with fixed version +- Full logs with timestamps ## Review Response Etiquette diff --git a/github-contributor/references/project_evaluation.md b/github-contributor/references/project_evaluation.md index 8916de75..94d9d5ff 100644 --- a/github-contributor/references/project_evaluation.md +++ b/github-contributor/references/project_evaluation.md @@ -2,17 +2,24 @@ How to evaluate open-source projects before contributing. +## Prerequisites + +- Install GitHub CLI and verify availability: `gh --version` +- Authenticate before running commands: `gh auth status || gh auth login` + ## Quick Health Check ```bash # Check recent activity -gh repo view owner/repo --json updatedAt,stargazersCount,openIssues +gh repo view owner/repo \ + --json updatedAt,stargazerCount,issues \ + --jq '{updatedAt, stargazers: .stargazerCount, openIssues: .issues.totalCount}' # Check PR response time gh pr list --repo owner/repo --state merged --limit 10 # Check issue activity -gh issue list --repo owner/repo --limit 20 +gh issue list --repo owner/repo --state=open --limit 20 ``` ## Evaluation Criteria @@ -131,7 +138,7 @@ gh search repos "topic:cli" --sort=stars gh search repos "language:python" --sort=stars # Find with good first issues -gh search issues "good first issue" --language=rust +gh search issues "good first issue" --language=rust --state=open ``` ### By Need diff --git a/markdown-tools/scripts/convert.py b/markdown-tools/scripts/convert.py deleted file mode 100755 index 9ac6f36c..00000000 --- a/markdown-tools/scripts/convert.py +++ /dev/null @@ -1,434 +0,0 @@ -#!/usr/bin/env python3 -""" -Multi-tool document to markdown converter with intelligent orchestration. - -Supports Quick Mode (fast, single tool) and Heavy Mode (best quality, multi-tool merge). - -Usage: - # Quick Mode (default) - fast, single best tool - uv run --with pymupdf4llm --with markitdown scripts/convert.py document.pdf -o output.md - - # Heavy Mode - multi-tool parallel execution with merge - uv run --with pymupdf4llm --with markitdown scripts/convert.py document.pdf -o output.md --heavy - - # With image extraction - uv run --with pymupdf4llm scripts/convert.py document.pdf -o output.md --assets-dir ./images - -Dependencies: - - pymupdf4llm: PDF conversion (LLM-optimized) - - markitdown: PDF/DOCX/PPTX conversion - - pandoc: DOCX/PPTX conversion (system install: brew install pandoc) -""" - -import argparse -import subprocess -import sys -import tempfile -import shutil -from dataclasses import dataclass, field -from pathlib import Path -from typing import Optional - - -@dataclass -class ConversionResult: - """Result from a single tool conversion.""" - markdown: str - tool: str - images: list[str] = field(default_factory=list) - success: bool = True - error: str = "" - - -def check_tool_available(tool: str) -> bool: - """Check if a conversion tool is available.""" - if tool == "pymupdf4llm": - try: - import pymupdf4llm - return True - except ImportError: - return False - elif tool == "markitdown": - try: - import markitdown - return True - except ImportError: - return False - elif tool == "pandoc": - return shutil.which("pandoc") is not None - return False - - -def select_tools(file_path: Path, mode: str) -> list[str]: - """Select conversion tools based on file type and mode.""" - ext = file_path.suffix.lower() - - # Tool preferences by format - tool_map = { - ".pdf": { - "quick": ["pymupdf4llm", "markitdown"], # fallback order - "heavy": ["pymupdf4llm", "markitdown"], - }, - ".docx": { - "quick": ["pandoc", "markitdown"], - "heavy": ["pandoc", "markitdown"], - }, - ".doc": { - "quick": ["pandoc", "markitdown"], - "heavy": ["pandoc", "markitdown"], - }, - ".pptx": { - "quick": ["markitdown", "pandoc"], - "heavy": ["markitdown", "pandoc"], - }, - ".xlsx": { - "quick": ["markitdown"], - "heavy": ["markitdown"], - }, - } - - tools = tool_map.get(ext, {"quick": ["markitdown"], "heavy": ["markitdown"]}) - - if mode == "quick": - # Return first available tool - for tool in tools["quick"]: - if check_tool_available(tool): - return [tool] - return [] - else: # heavy - # Return all available tools - return [t for t in tools["heavy"] if check_tool_available(t)] - - -def convert_with_pymupdf4llm( - file_path: Path, assets_dir: Optional[Path] = None -) -> ConversionResult: - """Convert using PyMuPDF4LLM (best for PDFs).""" - try: - import pymupdf4llm - - kwargs = {} - images = [] - - if assets_dir: - assets_dir.mkdir(parents=True, exist_ok=True) - kwargs["write_images"] = True - kwargs["image_path"] = str(assets_dir) - kwargs["dpi"] = 150 - - # Use best table detection strategy - kwargs["table_strategy"] = "lines_strict" - - md_text = pymupdf4llm.to_markdown(str(file_path), **kwargs) - - # Collect extracted images - if assets_dir and assets_dir.exists(): - images = [str(p) for p in assets_dir.glob("*.png")] - images.extend([str(p) for p in assets_dir.glob("*.jpg")]) - - return ConversionResult( - markdown=md_text, tool="pymupdf4llm", images=images, success=True - ) - except Exception as e: - return ConversionResult( - markdown="", tool="pymupdf4llm", success=False, error=str(e) - ) - - -def convert_with_markitdown( - file_path: Path, assets_dir: Optional[Path] = None -) -> ConversionResult: - """Convert using markitdown.""" - try: - # markitdown CLI approach - result = subprocess.run( - ["markitdown", str(file_path)], - capture_output=True, - text=True, - timeout=120, - ) - - if result.returncode != 0: - return ConversionResult( - markdown="", - tool="markitdown", - success=False, - error=result.stderr, - ) - - return ConversionResult( - markdown=result.stdout, tool="markitdown", success=True - ) - except FileNotFoundError: - # Try Python API - try: - from markitdown import MarkItDown - - md = MarkItDown() - result = md.convert(str(file_path)) - return ConversionResult( - markdown=result.text_content, tool="markitdown", success=True - ) - except Exception as e: - return ConversionResult( - markdown="", tool="markitdown", success=False, error=str(e) - ) - except Exception as e: - return ConversionResult( - markdown="", tool="markitdown", success=False, error=str(e) - ) - - -def convert_with_pandoc( - file_path: Path, assets_dir: Optional[Path] = None -) -> ConversionResult: - """Convert using pandoc.""" - try: - cmd = ["pandoc", str(file_path), "-t", "markdown", "--wrap=none"] - - if assets_dir: - assets_dir.mkdir(parents=True, exist_ok=True) - cmd.extend(["--extract-media", str(assets_dir)]) - - result = subprocess.run( - cmd, capture_output=True, text=True, timeout=120 - ) - - if result.returncode != 0: - return ConversionResult( - markdown="", tool="pandoc", success=False, error=result.stderr - ) - - images = [] - if assets_dir and assets_dir.exists(): - images = [str(p) for p in assets_dir.rglob("*.png")] - images.extend([str(p) for p in assets_dir.rglob("*.jpg")]) - - return ConversionResult( - markdown=result.stdout, tool="pandoc", images=images, success=True - ) - except Exception as e: - return ConversionResult( - markdown="", tool="pandoc", success=False, error=str(e) - ) - - -def convert_single( - file_path: Path, tool: str, assets_dir: Optional[Path] = None -) -> ConversionResult: - """Run a single conversion tool.""" - converters = { - "pymupdf4llm": convert_with_pymupdf4llm, - "markitdown": convert_with_markitdown, - "pandoc": convert_with_pandoc, - } - - converter = converters.get(tool) - if not converter: - return ConversionResult( - markdown="", tool=tool, success=False, error=f"Unknown tool: {tool}" - ) - - return converter(file_path, assets_dir) - - -def merge_results(results: list[ConversionResult]) -> ConversionResult: - """Merge results from multiple tools, selecting best segments.""" - if not results: - return ConversionResult(markdown="", tool="none", success=False) - - # Filter successful results - successful = [r for r in results if r.success and r.markdown.strip()] - if not successful: - # Return first error - return results[0] if results else ConversionResult( - markdown="", tool="none", success=False - ) - - if len(successful) == 1: - return successful[0] - - # Multiple successful results - merge them - # Strategy: Compare key metrics and select best - best = successful[0] - best_score = score_markdown(best.markdown) - - for result in successful[1:]: - score = score_markdown(result.markdown) - if score > best_score: - best = result - best_score = score - - # Merge images from all results - all_images = [] - seen = set() - for result in successful: - for img in result.images: - if img not in seen: - all_images.append(img) - seen.add(img) - - best.images = all_images - best.tool = f"merged({','.join(r.tool for r in successful)})" - - return best - - -def score_markdown(md: str) -> float: - """Score markdown quality for comparison.""" - score = 0.0 - - # Length (more content is generally better) - score += min(len(md) / 10000, 5.0) # Cap at 5 points - - # Tables (proper markdown tables) - table_count = md.count("|---|") + md.count("| ---") - score += min(table_count * 0.5, 3.0) - - # Images (referenced images) - image_count = md.count("![") - score += min(image_count * 0.3, 2.0) - - # Headings (proper hierarchy) - h1_count = md.count("\n# ") - h2_count = md.count("\n## ") - h3_count = md.count("\n### ") - if h1_count > 0 and h2_count >= h1_count: - score += 1.0 # Good hierarchy - - # Lists (structured content) - list_count = md.count("\n- ") + md.count("\n* ") + md.count("\n1. ") - score += min(list_count * 0.1, 2.0) - - return score - - -def main(): - parser = argparse.ArgumentParser( - description="Convert documents to markdown with multi-tool orchestration", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -Examples: - # Quick mode (default) - python convert.py document.pdf -o output.md - - # Heavy mode (best quality) - python convert.py document.pdf -o output.md --heavy - - # With custom assets directory - python convert.py document.pdf -o output.md --assets-dir ./images - """, - ) - parser.add_argument("input", type=Path, help="Input document path") - parser.add_argument( - "-o", "--output", type=Path, help="Output markdown file" - ) - parser.add_argument( - "--heavy", - action="store_true", - help="Enable Heavy Mode (multi-tool, best quality)", - ) - parser.add_argument( - "--assets-dir", - type=Path, - default=None, - help="Directory for extracted images (default: _assets/)", - ) - parser.add_argument( - "--tool", - choices=["pymupdf4llm", "markitdown", "pandoc"], - help="Force specific tool (overrides auto-selection)", - ) - parser.add_argument( - "--list-tools", - action="store_true", - help="List available tools and exit", - ) - - args = parser.parse_args() - - # List tools mode - if args.list_tools: - tools = ["pymupdf4llm", "markitdown", "pandoc"] - print("Available conversion tools:") - for tool in tools: - status = "✓" if check_tool_available(tool) else "✗" - print(f" {status} {tool}") - sys.exit(0) - - # Validate input - if not args.input.exists(): - print(f"Error: Input file not found: {args.input}", file=sys.stderr) - sys.exit(1) - - # Determine output path - output_path = args.output or args.input.with_suffix(".md") - - # Determine assets directory - assets_dir = args.assets_dir - if assets_dir is None and args.heavy: - assets_dir = output_path.parent / f"{output_path.stem}_assets" - - # Select tools - mode = "heavy" if args.heavy else "quick" - if args.tool: - tools = [args.tool] if check_tool_available(args.tool) else [] - else: - tools = select_tools(args.input, mode) - - if not tools: - print("Error: No conversion tools available.", file=sys.stderr) - print("Install with:", file=sys.stderr) - print(" pip install pymupdf4llm", file=sys.stderr) - print(" uv tool install markitdown[pdf]", file=sys.stderr) - print(" brew install pandoc", file=sys.stderr) - sys.exit(1) - - print(f"Converting: {args.input}") - print(f"Mode: {mode.upper()}") - print(f"Tools: {', '.join(tools)}") - - # Run conversions - results = [] - for tool in tools: - print(f" Running {tool}...", end=" ", flush=True) - - # Use separate assets dirs for each tool in heavy mode - tool_assets = None - if assets_dir and mode == "heavy" and len(tools) > 1: - tool_assets = assets_dir / tool - elif assets_dir: - tool_assets = assets_dir - - result = convert_single(args.input, tool, tool_assets) - results.append(result) - - if result.success: - print(f"✓ ({len(result.markdown):,} chars, {len(result.images)} images)") - else: - print(f"✗ ({result.error[:50]}...)") - - # Merge results if heavy mode - if mode == "heavy" and len(results) > 1: - print(" Merging results...", end=" ", flush=True) - final = merge_results(results) - print(f"✓ (using {final.tool})") - else: - final = merge_results(results) - - if not final.success: - print(f"Error: Conversion failed: {final.error}", file=sys.stderr) - sys.exit(1) - - # Write output - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(final.markdown) - - print(f"\nOutput: {output_path}") - print(f" Size: {len(final.markdown):,} characters") - if final.images: - print(f" Images: {len(final.images)} extracted") - - -if __name__ == "__main__": - main() diff --git a/meeting-minutes-taker/SKILL.md b/meeting-minutes-taker/SKILL.md index 6971e650..5ae91359 100644 --- a/meeting-minutes-taker/SKILL.md +++ b/meeting-minutes-taker/SKILL.md @@ -11,7 +11,7 @@ Transform raw meeting transcripts into comprehensive, evidence-based meeting min ## Quick Start **Pre-processing (Optional but Recommended):** -- **Document conversion**: Use `markdown-tools` skill to convert .docx/.pdf to Markdown first (preserves tables/images) +- **Document conversion**: Use `doc-to-markdown` skill to convert .docx/.pdf to Markdown first (preserves tables/images) - **Transcript cleanup**: Use `transcript-fixer` skill to fix ASR/STT errors if transcript quality is poor - **Context file**: Prepare `context.md` with team directory for accurate speaker identification @@ -457,7 +457,7 @@ If v3 has a flowchart for "Status Query Mechanism" but v1/v2 don't have it, that **Full pipeline for .docx transcripts:** ``` -Step 0: markdown-tools # Convert .docx → Markdown (preserves tables/images) +Step 0: doc-to-markdown # Convert .docx → Markdown (preserves tables/images) ↓ Step 0.5: transcript-fixer # Fix ASR errors (optional, if quality is poor) ↓ diff --git a/pdf-creator/SKILL.md b/pdf-creator/SKILL.md index ff942554..ac3c809b 100644 --- a/pdf-creator/SKILL.md +++ b/pdf-creator/SKILL.md @@ -1,55 +1,60 @@ --- name: pdf-creator -description: Create PDF documents from markdown with proper Chinese font support using weasyprint. This skill should be used when converting markdown to PDF, generating formal documents (legal, trademark filings, reports), or when Chinese typography is required. Triggers include "convert to PDF", "generate PDF", "markdown to PDF", or any request for creating printable documents. +description: Create PDF documents from markdown with proper Chinese font support. Supports theme system (default for formal docs, warm-terra for training materials) and dual backend (weasyprint or Chrome). Triggers include "convert to PDF", "generate PDF", "markdown to PDF", or any request for creating printable documents. --- # PDF Creator -Create professional PDF documents from markdown with proper Chinese font support. +Create professional PDF documents from markdown with Chinese font support and theme system. ## Quick Start -Convert a single markdown file: - ```bash -cd ~/workspace/claude-code-skills/pdf-creator -uv run --with weasyprint --with markdown scripts/md_to_pdf.py input.md output.pdf -``` +# Default theme (formal: Songti SC + black/grey) +uv run --with weasyprint scripts/md_to_pdf.py input.md output.pdf -Batch convert multiple files: +# Warm theme (training: PingFang SC + terra cotta) +uv run --with weasyprint scripts/md_to_pdf.py input.md --theme warm-terra -```bash -uv run --with weasyprint --with markdown scripts/batch_convert.py *.md --output-dir ./pdfs +# No weasyprint? Use Chrome backend (auto-detected if weasyprint unavailable) +python scripts/md_to_pdf.py input.md --theme warm-terra --backend chrome + +# List available themes +python scripts/md_to_pdf.py --list-themes dummy.md ``` -macOS ARM (Homebrew) 的 `DYLD_LIBRARY_PATH` 会自动检测配置，无需手动设置。 +## Themes + +Stored in `themes/*.css`. Each theme is a standalone CSS file. -## Font Configuration +| Theme | Font | Color | Best for | +|-------|------|-------|----------| +| `default` | Songti SC + Heiti SC | Black/grey | Legal docs, contracts, formal reports | +| `warm-terra` | PingFang SC | Terra cotta (#d97756) + warm neutrals | Course outlines, training materials, workshops | -The scripts use these Chinese fonts (with fallbacks): +To create a new theme: copy `themes/default.css`, modify, save as `themes/your-theme.css`. -| Font Type | Primary | Fallbacks | -|-----------|---------|-----------| -| Body text | Songti SC | SimSun, STSong, Noto Serif CJK SC | -| Headings | Heiti SC | SimHei, STHeiti, Noto Sans CJK SC | +## Backends -## Output Specifications +The script auto-detects the best available backend: -- **Page size**: A4 -- **Margins**: 2.5cm top/bottom, 2cm left/right -- **Body font**: 12pt, 1.8 line height -- **Max file size**: Designed to stay under 2MB for form submissions +| Backend | Install | Pros | Cons | +|---------|---------|------|------| +| `weasyprint` | `pip install weasyprint` | Precise CSS rendering, no browser needed | Requires system libs (cairo, pango) | +| `chrome` | Google Chrome installed | Zero Python deps, great CJK support | Larger binary, slightly less CSS control | -## Common Use Cases +Override with `--backend chrome` or `--backend weasyprint`. -1. **Legal documents**: Trademark filings, contracts, evidence lists -2. **Reports**: Business reports, technical documentation -3. **Formal letters**: Official correspondence requiring print format +## Batch Convert + +```bash +uv run --with weasyprint scripts/batch_convert.py *.md --output-dir ./pdfs +``` ## Troubleshooting -**Problem**: Chinese characters display as boxes -**Solution**: Ensure Songti SC or other Chinese fonts are installed on the system +**Chinese characters display as boxes**: Ensure Chinese fonts are installed (Songti SC, PingFang SC, etc.) + +**weasyprint import error**: Run with `uv run --with weasyprint` or use `--backend chrome` instead. -**Problem**: `weasyprint` import error -**Solution**: Run with `uv run --with weasyprint --with markdown` to ensure dependencies +**Chrome header/footer appearing**: The script passes `--no-pdf-header-footer`. If it still appears, your Chrome version may not support this flag — update Chrome. diff --git a/pdf-creator/scripts/md_to_pdf.py b/pdf-creator/scripts/md_to_pdf.py index cf79ea4e..4fd9ed6a 100644 --- a/pdf-creator/scripts/md_to_pdf.py +++ b/pdf-creator/scripts/md_to_pdf.py @@ -1,226 +1,301 @@ #!/usr/bin/env python3 """ -Markdown to PDF converter with Chinese font support. +Markdown to PDF converter with Chinese font support and theme system. -Converts markdown files to PDF using weasyprint, with proper Chinese typography. -Designed for formal documents (trademark filings, legal documents, reports). +Converts markdown files to PDF using: + - pandoc (markdown → HTML) + - weasyprint or headless Chrome (HTML → PDF), auto-detected Usage: python md_to_pdf.py input.md output.pdf - python md_to_pdf.py input.md # outputs input.pdf + python md_to_pdf.py input.md --theme warm-terra + python md_to_pdf.py input.md --theme default --backend chrome + python md_to_pdf.py input.md # outputs input.pdf, default theme, auto backend -Requirements: - pip install weasyprint markdown +Themes: + Stored in ../themes/*.css. Built-in themes: + - default: Songti SC + black/grey, formal documents + - warm-terra: PingFang SC + terra cotta, training/workshop materials - macOS environment setup (if needed): - export DYLD_LIBRARY_PATH="/opt/homebrew/lib:$DYLD_LIBRARY_PATH" +Requirements: + pandoc (system install, e.g. brew install pandoc) + weasyprint (pip install weasyprint) OR Google Chrome (for --backend chrome) """ +from __future__ import annotations + +import argparse import os import platform import re +import shutil +import subprocess import sys +import tempfile from pathlib import Path -# Auto-configure library path on macOS ARM (Homebrew) — must be before weasyprint import -if platform.system() == 'Darwin': - _homebrew_lib = '/opt/homebrew/lib' +SCRIPT_DIR = Path(__file__).resolve().parent +THEMES_DIR = SCRIPT_DIR.parent / "themes" + +# macOS ARM: auto-configure library path for weasyprint +if platform.system() == "Darwin": + _homebrew_lib = "/opt/homebrew/lib" if Path(_homebrew_lib).is_dir(): - _cur = os.environ.get('DYLD_LIBRARY_PATH', '') + _cur = os.environ.get("DYLD_LIBRARY_PATH", "") if _homebrew_lib not in _cur: - os.environ['DYLD_LIBRARY_PATH'] = f"{_homebrew_lib}:{_cur}" if _cur else _homebrew_lib - -import markdown -from weasyprint import CSS, HTML - - -# CSS with Chinese font support -CSS_STYLES = """ -@page { - size: A4; - margin: 2.5cm 2cm; -} - -body { - font-family: 'Songti SC', 'SimSun', 'STSong', 'Noto Serif CJK SC', serif; - font-size: 12pt; - line-height: 1.8; - color: #000; - width: 100%; -} - -h1 { - font-family: 'Heiti SC', 'SimHei', 'STHeiti', 'Noto Sans CJK SC', sans-serif; - font-size: 18pt; - font-weight: bold; - text-align: center; - margin-top: 0; - margin-bottom: 1.5em; -} - -h2 { - font-family: 'Heiti SC', 'SimHei', 'STHeiti', 'Noto Sans CJK SC', sans-serif; - font-size: 14pt; - font-weight: bold; - margin-top: 1.5em; - margin-bottom: 0.8em; -} - -h3 { - font-family: 'Heiti SC', 'SimHei', 'STHeiti', 'Noto Sans CJK SC', sans-serif; - font-size: 12pt; - font-weight: bold; - margin-top: 1em; - margin-bottom: 0.5em; -} - -p { - margin: 0.8em 0; - text-align: justify; -} - -ul, ol { - margin: 0.8em 0; - padding-left: 2em; -} - -li { - margin: 0.4em 0; -} - -table { - border-collapse: collapse; - width: 100%; - margin: 1em 0; - font-size: 10pt; - table-layout: fixed; -} - -th, td { - border: 1px solid #666; - padding: 8px 6px; - text-align: left; - overflow-wrap: break-word; - word-break: normal; -} - -th { - background-color: #f0f0f0; - font-weight: bold; -} - -hr { - border: none; - border-top: 1px solid #ccc; - margin: 1.5em 0; -} - -strong { - font-weight: bold; -} - -code { - font-family: 'SF Mono', 'Monaco', 'Menlo', monospace; - font-size: 10pt; - background-color: #f5f5f5; - padding: 0.2em 0.4em; - border-radius: 3px; -} - -pre { - background-color: #f5f5f5; - padding: 1em; - overflow-x: auto; - font-size: 10pt; - line-height: 1.4; - border-radius: 4px; -} - -blockquote { - border-left: 3px solid #ccc; - margin: 1em 0; - padding-left: 1em; - color: #555; -} -""" + os.environ["DYLD_LIBRARY_PATH"] = ( + f"{_homebrew_lib}:{_cur}" if _cur else _homebrew_lib + ) + + +def _find_chrome() -> str | None: + """Find Chrome/Chromium binary path.""" + candidates = [ + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Chromium.app/Contents/MacOS/Chromium", + shutil.which("google-chrome"), + shutil.which("chromium"), + shutil.which("chrome"), + ] + for c in candidates: + if c and Path(c).exists(): + return str(c) + return None + + +def _has_weasyprint() -> bool: + """Check if weasyprint is importable.""" + try: + import weasyprint # noqa: F401 + + return True + except ImportError: + return False + + +def _detect_backend() -> str: + """Auto-detect best available backend: weasyprint > chrome.""" + if _has_weasyprint(): + return "weasyprint" + if _find_chrome(): + return "chrome" + print( + "Error: No PDF backend found. Install weasyprint (pip install weasyprint) " + "or Google Chrome.", + file=sys.stderr, + ) + sys.exit(1) + + +def _load_theme(theme_name: str) -> str: + """Load CSS from themes directory.""" + theme_file = THEMES_DIR / f"{theme_name}.css" + if not theme_file.exists(): + available = [f.stem for f in THEMES_DIR.glob("*.css")] + print( + f"Error: Theme '{theme_name}' not found. Available: {available}", + file=sys.stderr, + ) + sys.exit(1) + return theme_file.read_text(encoding="utf-8") + + +def _list_themes() -> list[str]: + """List available theme names.""" + if not THEMES_DIR.exists(): + return [] + return sorted(f.stem for f in THEMES_DIR.glob("*.css")) def _ensure_list_spacing(text: str) -> str: """Ensure blank lines before list items for proper markdown parsing. - The Python markdown library requires a blank line before a list when it - follows a paragraph. Without it, list items render as plain text. + Both Python markdown library and pandoc require a blank line before a list + when it follows a paragraph. Without it, list items render as plain text. """ - lines = text.split('\n') + lines = text.split("\n") result = [] - list_re = re.compile(r'^(\s*)([-*+]|\d+\.)\s') + list_re = re.compile(r"^(\s*)([-*+]|\d+\.)\s") for i, line in enumerate(lines): if i > 0 and list_re.match(line): prev = lines[i - 1] if prev.strip() and not list_re.match(prev): - result.append('') + result.append("") result.append(line) - return '\n'.join(result) + return "\n".join(result) -def markdown_to_pdf(md_file: str, pdf_file: str | None = None) -> str: - """ - Convert markdown file to PDF with Chinese font support. - - Args: - md_file: Path to input markdown file - pdf_file: Path to output PDF file (optional, defaults to same name as input) +def _md_to_html(md_file: str) -> str: + """Convert markdown to HTML using pandoc with list spacing preprocessing.""" + if not shutil.which("pandoc"): + print( + "Error: pandoc not found. Install with: brew install pandoc", + file=sys.stderr, + ) + sys.exit(1) - Returns: - Path to generated PDF file - """ - md_path = Path(md_file) + md_content = Path(md_file).read_text(encoding="utf-8") + md_content = _ensure_list_spacing(md_content) - if pdf_file is None: - pdf_file = str(md_path.with_suffix('.pdf')) + result = subprocess.run( + ["pandoc", "-f", "markdown", "-t", "html"], + input=md_content, + capture_output=True, + text=True, + ) + if result.returncode != 0: + print(f"Error: pandoc failed: {result.stderr}", file=sys.stderr) + sys.exit(1) - # Read and preprocess markdown content - md_content = _ensure_list_spacing(md_path.read_text(encoding='utf-8')) + return result.stdout - # Convert to HTML - html_content = markdown.markdown( - md_content, - extensions=['tables', 'fenced_code', 'codehilite', 'toc'] - ) - # Create full HTML document - full_html = f""" +def _build_full_html(html_content: str, css: str, title: str) -> str: + """Wrap HTML content in a full document with CSS.""" + return f""" - {md_path.stem} + {title} + {html_content} """ - # Generate PDF - HTML(string=full_html).write_pdf(pdf_file, stylesheets=[CSS(string=CSS_STYLES)]) +def _render_weasyprint(full_html: str, pdf_file: str, css: str) -> None: + """Render PDF using weasyprint.""" + from weasyprint import CSS, HTML + + HTML(string=full_html).write_pdf(pdf_file, stylesheets=[CSS(string=css)]) + + +def _render_chrome(full_html: str, pdf_file: str) -> None: + """Render PDF using headless Chrome.""" + chrome = _find_chrome() + if not chrome: + print("Error: Chrome not found.", file=sys.stderr) + sys.exit(1) + + with tempfile.NamedTemporaryFile( + suffix=".html", mode="w", encoding="utf-8", delete=False + ) as f: + f.write(full_html) + html_path = f.name + + try: + result = subprocess.run( + [ + chrome, + "--headless", + "--disable-gpu", + "--no-pdf-header-footer", + f"--print-to-pdf={pdf_file}", + html_path, + ], + capture_output=True, + text=True, + ) + if not Path(pdf_file).exists(): + print( + f"Error: Chrome failed to generate PDF. stderr: {result.stderr}", + file=sys.stderr, + ) + sys.exit(1) + finally: + Path(html_path).unlink(missing_ok=True) + + +def markdown_to_pdf( + md_file: str, + pdf_file: str | None = None, + theme: str = "default", + backend: str | None = None, +) -> str: + """ + Convert markdown file to PDF. + + Args: + md_file: Path to input markdown file + pdf_file: Path to output PDF (optional, defaults to same name as input) + theme: Theme name (from themes/ directory) + backend: 'weasyprint', 'chrome', or None (auto-detect) + + Returns: + Path to generated PDF file + """ + md_path = Path(md_file) + if pdf_file is None: + pdf_file = str(md_path.with_suffix(".pdf")) + + if backend is None: + backend = _detect_backend() + + css = _load_theme(theme) + html_content = _md_to_html(md_file) + full_html = _build_full_html(html_content, css, md_path.stem) + + if backend == "weasyprint": + _render_weasyprint(full_html, pdf_file, css) + elif backend == "chrome": + _render_chrome(full_html, pdf_file) + else: + print(f"Error: Unknown backend '{backend}'", file=sys.stderr) + sys.exit(1) + + size_kb = Path(pdf_file).stat().st_size / 1024 + print(f"Generated: {pdf_file} ({size_kb:.0f}KB, theme={theme}, backend={backend})") return pdf_file def main(): - if len(sys.argv) < 2: - print("Usage: python md_to_pdf.py [output.pdf]") - print("\nConverts markdown to PDF with Chinese font support.") - sys.exit(1) + available_themes = _list_themes() - md_file = sys.argv[1] - pdf_file = sys.argv[2] if len(sys.argv) > 2 else None + parser = argparse.ArgumentParser( + description="Markdown to PDF with Chinese font support and themes." + ) + parser.add_argument("input", help="Input markdown file") + parser.add_argument("output", nargs="?", help="Output PDF file (optional)") + parser.add_argument( + "--theme", + default="default", + choices=available_themes or ["default"], + help=f"CSS theme (available: {', '.join(available_themes) or 'default'})", + ) + parser.add_argument( + "--backend", + choices=["weasyprint", "chrome"], + default=None, + help="PDF rendering backend (default: auto-detect)", + ) + parser.add_argument( + "--list-themes", + action="store_true", + help="List available themes and exit", + ) - if not Path(md_file).exists(): - print(f"Error: File not found: {md_file}") + args = parser.parse_args() + + if args.list_themes: + for t in available_themes: + marker = " (default)" if t == "default" else "" + css_file = THEMES_DIR / f"{t}.css" + first_line = "" + for line in css_file.read_text().splitlines(): + line = line.strip() + if line.startswith("*") and "—" in line: + first_line = line.lstrip("* ").strip() + break + print(f" {t}{marker}: {first_line}") + sys.exit(0) + + if not Path(args.input).exists(): + print(f"Error: File not found: {args.input}", file=sys.stderr) sys.exit(1) - output = markdown_to_pdf(md_file, pdf_file) - print(f"Generated: {output}") + markdown_to_pdf(args.input, args.output, args.theme, args.backend) if __name__ == "__main__": diff --git a/pdf-creator/scripts/tests/test_list_rendering.py b/pdf-creator/scripts/tests/test_list_rendering.py new file mode 100755 index 00000000..a7944c97 --- /dev/null +++ b/pdf-creator/scripts/tests/test_list_rendering.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +""" +Test list rendering in PDF generation. + +Verifies that markdown lists are correctly rendered in PDFs, +even when they don't have blank lines before them. + +The original markdown files are NOT modified - preprocessing +happens in memory during conversion. +""" + +import subprocess +import sys +import tempfile +from pathlib import Path + +# Test markdown content with various list scenarios +TEST_MARKDOWN = """# 测试列表解析 + +## 场景1：列表前有空行（正常） + +这是一段文字。 + +- 列表项 1 +- 列表项 2 +- 列表项 3 + +## 场景2：列表前没有空行（关键测试） + +这是一段文字。 +- 列表项 1 +- 列表项 2 +- 列表项 3 + +## 场景3：有序列表前没有空行 + +这是一段文字。 +1. 第一项 +2. 第二项 +3. 第三项 + +## 场景4：有序列表前有空行（正常） + +这是一段文字。 + +1. 第一项 +2. 第二项 +3. 第三项 +""" + + +def run_test(): + """Run the list rendering test.""" + # Create temporary files + with tempfile.NamedTemporaryFile(mode='w', suffix='.md', delete=False, encoding='utf-8') as md_file: + md_file.write(TEST_MARKDOWN) + md_path = md_file.name + + pdf_path = md_path.replace('.md', '.pdf') + txt_path = md_path.replace('.md', '.txt') + + try: + # Generate PDF + script_dir = Path(__file__).parent.parent + md_to_pdf = script_dir / 'md_to_pdf.py' + + print(f"生成 PDF: {md_path} -> {pdf_path}") + result = subprocess.run( + ['uv', 'run', '--with', 'weasyprint', str(md_to_pdf), md_path, pdf_path], + capture_output=True, text=True, cwd=script_dir.parent + ) + + if result.returncode != 0: + print(f"❌ PDF 生成失败: {result.stderr}") + return False + + print(f"✅ PDF 已生成") + + # Extract text from PDF + result = subprocess.run( + ['pdftotext', pdf_path, txt_path], + capture_output=True, text=True + ) + + if result.returncode != 0: + print(f"❌ 文本提取失败: {result.stderr}") + return False + + # Read extracted text + with open(txt_path, 'r', encoding='utf-8') as f: + pdf_text = f.read() + + # Verify original file was not modified + with open(md_path, 'r', encoding='utf-8') as f: + original_content = f.read() + + if original_content != TEST_MARKDOWN: + print("❌ 原始文件被修改了！") + return False + + print("✅ 原始文件未被修改") + + # Verify list rendering + print("\n=== 列表渲染验证 ===") + + tests_passed = 0 + tests_total = 4 + + # Test 1: List with blank line before it + if '• 列表项 1' in pdf_text: + print("✅ 场景1: 列表前有空行 - 正确渲染") + tests_passed += 1 + else: + print("❌ 场景1: 列表前有空行 - 渲染失败") + + # Test 2: Critical test - list without blank line before it + scene2_start = pdf_text.find('场景2') + scene2_section = pdf_text[scene2_start:scene2_start+200] if scene2_start != -1 else "" + + if '• 列表项 1' in scene2_section and '- 列表项 1' not in scene2_section: + print("✅ 场景2: 列表前没有空行 - 正确渲染（关键测试）") + tests_passed += 1 + else: + print("❌ 场景2: 列表前没有空行 - 渲染失败") + print(f" 实际内容: {scene2_section}") + + # Test 3: Ordered list without blank line + scene3_start = pdf_text.find('场景3') + scene3_section = pdf_text[scene3_start:scene3_start+200] if scene3_start != -1 else "" + + if '1. 第一项' in scene3_section and '2. 第二项' in scene3_section: + print("✅ 场景3: 有序列表前没有空行 - 正确渲染") + tests_passed += 1 + else: + print("❌ 场景3: 有序列表前没有空行 - 渲染失败") + + # Test 4: Ordered list with blank line + if '1. 第一项' in pdf_text and '2. 第二项' in pdf_text: + print("✅ 场景4: 有序列表前有空行 - 正确渲染") + tests_passed += 1 + else: + print("❌ 场景4: 有序列表前有空行 - 渲染失败") + + print(f"\n=== 测试结果: {tests_passed}/{tests_total} 通过 ===") + + if tests_passed == tests_total: + print("\n✅ 所有测试通过！") + print(f"\n生成的文件:") + print(f" Markdown: {md_path}") + print(f" PDF: {pdf_path}") + print(f" Text: {txt_path}") + return True + else: + print(f"\n❌ {tests_total - tests_passed} 个测试失败") + return False + + except Exception as e: + print(f"❌ 测试失败: {e}") + import traceback + traceback.print_exc() + return False + + +if __name__ == '__main__': + success = run_test() + sys.exit(0 if success else 1) diff --git a/pdf-creator/themes/default.css b/pdf-creator/themes/default.css new file mode 100644 index 00000000..255cd1a2 --- /dev/null +++ b/pdf-creator/themes/default.css @@ -0,0 +1,88 @@ +/* + * Default — PDF theme for formal documents + * + * Color palette: black/grey, no accent color + * Font: Songti SC (body) + Heiti SC (headings) + * Best for: legal documents, trademark filings, contracts, formal reports + * + * This is the original built-in theme from md_to_pdf.py, extracted for reference. + */ + +@page { + size: A4; + margin: 2.5cm 2cm; +} + +body { + font-family: 'Songti SC', 'SimSun', 'STSong', 'Noto Serif CJK SC', serif; + font-size: 12pt; + line-height: 1.8; + color: #000; + width: 100%; +} + +h1 { + font-family: 'Heiti SC', 'SimHei', 'STHeiti', 'Noto Sans CJK SC', sans-serif; + font-size: 18pt; + font-weight: bold; + text-align: center; + margin-top: 0; + margin-bottom: 1.5em; +} + +h2 { + font-family: 'Heiti SC', 'SimHei', 'STHeiti', 'Noto Sans CJK SC', sans-serif; + font-size: 14pt; + font-weight: bold; + margin-top: 1.5em; + margin-bottom: 0.8em; +} + +h3 { + font-family: 'Heiti SC', 'SimHei', 'STHeiti', 'Noto Sans CJK SC', sans-serif; + font-size: 12pt; + font-weight: bold; + margin-top: 1em; + margin-bottom: 0.5em; +} + +p { + margin: 0.8em 0; + text-align: justify; +} + +ul, ol { + margin: 0.8em 0; + padding-left: 2em; +} + +li { + margin: 0.4em 0; +} + +table { + border-collapse: collapse; + width: 100%; + margin: 1em 0; + font-size: 10pt; + table-layout: fixed; +} + +th, td { + border: 1px solid #666; + padding: 8px 6px; + text-align: left; + overflow-wrap: break-word; + word-break: normal; +} + +th { + background-color: #f0f0f0; + font-weight: bold; +} + +hr { + border: none; + border-top: 1px solid #ccc; + margin: 1.5em 0; +} diff --git a/pdf-creator/themes/warm-terra.css b/pdf-creator/themes/warm-terra.css new file mode 100644 index 00000000..3af0f800 --- /dev/null +++ b/pdf-creator/themes/warm-terra.css @@ -0,0 +1,121 @@ +/* + * Warm Terra — PDF theme for workshop/training documents + * + * Color palette: terra cotta (#d97756) + warm neutrals + * Font: PingFang SC (macOS) / Microsoft YaHei (Windows) + * Best for: course outlines, training materials, workshop agendas + * + * Usage with md_to_pdf.py: + * python md_to_pdf.py input.md output.pdf --theme warm-terra + * + * Usage with pandoc + Chrome (fallback): + * pandoc input.md -o /tmp/out.html --standalone -H <(cat this-file.css wrapped in + + +

Eval Set Review: __SKILL_NAME_PLACEHOLDER__

+

Current description: __SKILL_DESCRIPTION_PLACEHOLDER__

+ +
+ + +
+ +
+ + + + + + + + +
QueryShould TriggerActions
+ +

+ + + + diff --git a/skill-creator/eval-viewer/generate_review.py b/skill-creator/eval-viewer/generate_review.py new file mode 100644 index 00000000..7fa59786 --- /dev/null +++ b/skill-creator/eval-viewer/generate_review.py @@ -0,0 +1,471 @@ +#!/usr/bin/env python3 +"""Generate and serve a review page for eval results. + +Reads the workspace directory, discovers runs (directories with outputs/), +embeds all output data into a self-contained HTML page, and serves it via +a tiny HTTP server. Feedback auto-saves to feedback.json in the workspace. + +Usage: + python generate_review.py [--port PORT] [--skill-name NAME] + python generate_review.py --previous-feedback /path/to/old/feedback.json + +No dependencies beyond the Python stdlib are required. +""" + +import argparse +import base64 +import json +import mimetypes +import os +import re +import signal +import subprocess +import sys +import time +import webbrowser +from functools import partial +from http.server import HTTPServer, BaseHTTPRequestHandler +from pathlib import Path + +# Files to exclude from output listings +METADATA_FILES = {"transcript.md", "user_notes.md", "metrics.json"} + +# Extensions we render as inline text +TEXT_EXTENSIONS = { + ".txt", ".md", ".json", ".csv", ".py", ".js", ".ts", ".tsx", ".jsx", + ".yaml", ".yml", ".xml", ".html", ".css", ".sh", ".rb", ".go", ".rs", + ".java", ".c", ".cpp", ".h", ".hpp", ".sql", ".r", ".toml", +} + +# Extensions we render as inline images +IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp"} + +# MIME type overrides for common types +MIME_OVERRIDES = { + ".svg": "image/svg+xml", + ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", +} + + +def get_mime_type(path: Path) -> str: + ext = path.suffix.lower() + if ext in MIME_OVERRIDES: + return MIME_OVERRIDES[ext] + mime, _ = mimetypes.guess_type(str(path)) + return mime or "application/octet-stream" + + +def find_runs(workspace: Path) -> list[dict]: + """Recursively find directories that contain an outputs/ subdirectory.""" + runs: list[dict] = [] + _find_runs_recursive(workspace, workspace, runs) + runs.sort(key=lambda r: (r.get("eval_id", float("inf")), r["id"])) + return runs + + +def _find_runs_recursive(root: Path, current: Path, runs: list[dict]) -> None: + if not current.is_dir(): + return + + outputs_dir = current / "outputs" + if outputs_dir.is_dir(): + run = build_run(root, current) + if run: + runs.append(run) + return + + skip = {"node_modules", ".git", "__pycache__", "skill", "inputs"} + for child in sorted(current.iterdir()): + if child.is_dir() and child.name not in skip: + _find_runs_recursive(root, child, runs) + + +def build_run(root: Path, run_dir: Path) -> dict | None: + """Build a run dict with prompt, outputs, and grading data.""" + prompt = "" + eval_id = None + + # Try eval_metadata.json + for candidate in [run_dir / "eval_metadata.json", run_dir.parent / "eval_metadata.json"]: + if candidate.exists(): + try: + metadata = json.loads(candidate.read_text()) + prompt = metadata.get("prompt", "") + eval_id = metadata.get("eval_id") + except (json.JSONDecodeError, OSError): + pass + if prompt: + break + + # Fall back to transcript.md + if not prompt: + for candidate in [run_dir / "transcript.md", run_dir / "outputs" / "transcript.md"]: + if candidate.exists(): + try: + text = candidate.read_text() + match = re.search(r"## Eval Prompt\n\n([\s\S]*?)(?=\n##|$)", text) + if match: + prompt = match.group(1).strip() + except OSError: + pass + if prompt: + break + + if not prompt: + prompt = "(No prompt found)" + + run_id = str(run_dir.relative_to(root)).replace("/", "-").replace("\\", "-") + + # Collect output files + outputs_dir = run_dir / "outputs" + output_files: list[dict] = [] + if outputs_dir.is_dir(): + for f in sorted(outputs_dir.iterdir()): + if f.is_file() and f.name not in METADATA_FILES: + output_files.append(embed_file(f)) + + # Load grading if present + grading = None + for candidate in [run_dir / "grading.json", run_dir.parent / "grading.json"]: + if candidate.exists(): + try: + grading = json.loads(candidate.read_text()) + except (json.JSONDecodeError, OSError): + pass + if grading: + break + + return { + "id": run_id, + "prompt": prompt, + "eval_id": eval_id, + "outputs": output_files, + "grading": grading, + } + + +def embed_file(path: Path) -> dict: + """Read a file and return an embedded representation.""" + ext = path.suffix.lower() + mime = get_mime_type(path) + + if ext in TEXT_EXTENSIONS: + try: + content = path.read_text(errors="replace") + except OSError: + content = "(Error reading file)" + return { + "name": path.name, + "type": "text", + "content": content, + } + elif ext in IMAGE_EXTENSIONS: + try: + raw = path.read_bytes() + b64 = base64.b64encode(raw).decode("ascii") + except OSError: + return {"name": path.name, "type": "error", "content": "(Error reading file)"} + return { + "name": path.name, + "type": "image", + "mime": mime, + "data_uri": f"data:{mime};base64,{b64}", + } + elif ext == ".pdf": + try: + raw = path.read_bytes() + b64 = base64.b64encode(raw).decode("ascii") + except OSError: + return {"name": path.name, "type": "error", "content": "(Error reading file)"} + return { + "name": path.name, + "type": "pdf", + "data_uri": f"data:{mime};base64,{b64}", + } + elif ext == ".xlsx": + try: + raw = path.read_bytes() + b64 = base64.b64encode(raw).decode("ascii") + except OSError: + return {"name": path.name, "type": "error", "content": "(Error reading file)"} + return { + "name": path.name, + "type": "xlsx", + "data_b64": b64, + } + else: + # Binary / unknown — base64 download link + try: + raw = path.read_bytes() + b64 = base64.b64encode(raw).decode("ascii") + except OSError: + return {"name": path.name, "type": "error", "content": "(Error reading file)"} + return { + "name": path.name, + "type": "binary", + "mime": mime, + "data_uri": f"data:{mime};base64,{b64}", + } + + +def load_previous_iteration(workspace: Path) -> dict[str, dict]: + """Load previous iteration's feedback and outputs. + + Returns a map of run_id -> {"feedback": str, "outputs": list[dict]}. + """ + result: dict[str, dict] = {} + + # Load feedback + feedback_map: dict[str, str] = {} + feedback_path = workspace / "feedback.json" + if feedback_path.exists(): + try: + data = json.loads(feedback_path.read_text()) + feedback_map = { + r["run_id"]: r["feedback"] + for r in data.get("reviews", []) + if r.get("feedback", "").strip() + } + except (json.JSONDecodeError, OSError, KeyError): + pass + + # Load runs (to get outputs) + prev_runs = find_runs(workspace) + for run in prev_runs: + result[run["id"]] = { + "feedback": feedback_map.get(run["id"], ""), + "outputs": run.get("outputs", []), + } + + # Also add feedback for run_ids that had feedback but no matching run + for run_id, fb in feedback_map.items(): + if run_id not in result: + result[run_id] = {"feedback": fb, "outputs": []} + + return result + + +def generate_html( + runs: list[dict], + skill_name: str, + previous: dict[str, dict] | None = None, + benchmark: dict | None = None, +) -> str: + """Generate the complete standalone HTML page with embedded data.""" + template_path = Path(__file__).parent / "viewer.html" + template = template_path.read_text() + + # Build previous_feedback and previous_outputs maps for the template + previous_feedback: dict[str, str] = {} + previous_outputs: dict[str, list[dict]] = {} + if previous: + for run_id, data in previous.items(): + if data.get("feedback"): + previous_feedback[run_id] = data["feedback"] + if data.get("outputs"): + previous_outputs[run_id] = data["outputs"] + + embedded = { + "skill_name": skill_name, + "runs": runs, + "previous_feedback": previous_feedback, + "previous_outputs": previous_outputs, + } + if benchmark: + embedded["benchmark"] = benchmark + + data_json = json.dumps(embedded) + + return template.replace("/*__EMBEDDED_DATA__*/", f"const EMBEDDED_DATA = {data_json};") + + +# --------------------------------------------------------------------------- +# HTTP server (stdlib only, zero dependencies) +# --------------------------------------------------------------------------- + +def _kill_port(port: int) -> None: + """Kill any process listening on the given port.""" + try: + result = subprocess.run( + ["lsof", "-ti", f":{port}"], + capture_output=True, text=True, timeout=5, + ) + for pid_str in result.stdout.strip().split("\n"): + if pid_str.strip(): + try: + os.kill(int(pid_str.strip()), signal.SIGTERM) + except (ProcessLookupError, ValueError): + pass + if result.stdout.strip(): + time.sleep(0.5) + except subprocess.TimeoutExpired: + pass + except FileNotFoundError: + print("Note: lsof not found, cannot check if port is in use", file=sys.stderr) + +class ReviewHandler(BaseHTTPRequestHandler): + """Serves the review HTML and handles feedback saves. + + Regenerates the HTML on each page load so that refreshing the browser + picks up new eval outputs without restarting the server. + """ + + def __init__( + self, + workspace: Path, + skill_name: str, + feedback_path: Path, + previous: dict[str, dict], + benchmark_path: Path | None, + *args, + **kwargs, + ): + self.workspace = workspace + self.skill_name = skill_name + self.feedback_path = feedback_path + self.previous = previous + self.benchmark_path = benchmark_path + super().__init__(*args, **kwargs) + + def do_GET(self) -> None: + if self.path == "/" or self.path == "/index.html": + # Regenerate HTML on each request (re-scans workspace for new outputs) + runs = find_runs(self.workspace) + benchmark = None + if self.benchmark_path and self.benchmark_path.exists(): + try: + benchmark = json.loads(self.benchmark_path.read_text()) + except (json.JSONDecodeError, OSError): + pass + html = generate_html(runs, self.skill_name, self.previous, benchmark) + content = html.encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) + elif self.path == "/api/feedback": + data = b"{}" + if self.feedback_path.exists(): + data = self.feedback_path.read_bytes() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + else: + self.send_error(404) + + def do_POST(self) -> None: + if self.path == "/api/feedback": + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) + try: + data = json.loads(body) + if not isinstance(data, dict) or "reviews" not in data: + raise ValueError("Expected JSON object with 'reviews' key") + self.feedback_path.write_text(json.dumps(data, indent=2) + "\n") + resp = b'{"ok":true}' + self.send_response(200) + except (json.JSONDecodeError, OSError, ValueError) as e: + resp = json.dumps({"error": str(e)}).encode() + self.send_response(500) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(resp))) + self.end_headers() + self.wfile.write(resp) + else: + self.send_error(404) + + def log_message(self, format: str, *args: object) -> None: + # Suppress request logging to keep terminal clean + pass + + +def main() -> None: + parser = argparse.ArgumentParser(description="Generate and serve eval review") + parser.add_argument("workspace", type=Path, help="Path to workspace directory") + parser.add_argument("--port", "-p", type=int, default=3117, help="Server port (default: 3117)") + parser.add_argument("--skill-name", "-n", type=str, default=None, help="Skill name for header") + parser.add_argument( + "--previous-workspace", type=Path, default=None, + help="Path to previous iteration's workspace (shows old outputs and feedback as context)", + ) + parser.add_argument( + "--benchmark", type=Path, default=None, + help="Path to benchmark.json to show in the Benchmark tab", + ) + parser.add_argument( + "--static", "-s", type=Path, default=None, + help="Write standalone HTML to this path instead of starting a server", + ) + args = parser.parse_args() + + workspace = args.workspace.resolve() + if not workspace.is_dir(): + print(f"Error: {workspace} is not a directory", file=sys.stderr) + sys.exit(1) + + runs = find_runs(workspace) + if not runs: + print(f"No runs found in {workspace}", file=sys.stderr) + sys.exit(1) + + skill_name = args.skill_name or workspace.name.replace("-workspace", "") + feedback_path = workspace / "feedback.json" + + previous: dict[str, dict] = {} + if args.previous_workspace: + previous = load_previous_iteration(args.previous_workspace.resolve()) + + benchmark_path = args.benchmark.resolve() if args.benchmark else None + benchmark = None + if benchmark_path and benchmark_path.exists(): + try: + benchmark = json.loads(benchmark_path.read_text()) + except (json.JSONDecodeError, OSError): + pass + + if args.static: + html = generate_html(runs, skill_name, previous, benchmark) + args.static.parent.mkdir(parents=True, exist_ok=True) + args.static.write_text(html) + print(f"\n Static viewer written to: {args.static}\n") + sys.exit(0) + + # Kill any existing process on the target port + port = args.port + _kill_port(port) + handler = partial(ReviewHandler, workspace, skill_name, feedback_path, previous, benchmark_path) + try: + server = HTTPServer(("127.0.0.1", port), handler) + except OSError: + # Port still in use after kill attempt — find a free one + server = HTTPServer(("127.0.0.1", 0), handler) + port = server.server_address[1] + + url = f"http://localhost:{port}" + print(f"\n Eval Viewer") + print(f" ─────────────────────────────────") + print(f" URL: {url}") + print(f" Workspace: {workspace}") + print(f" Feedback: {feedback_path}") + if previous: + print(f" Previous: {args.previous_workspace} ({len(previous)} runs)") + if benchmark_path: + print(f" Benchmark: {benchmark_path}") + print(f"\n Press Ctrl+C to stop.\n") + + webbrowser.open(url) + + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nStopped.") + server.server_close() + + +if __name__ == "__main__": + main() diff --git a/skill-creator/eval-viewer/viewer.html b/skill-creator/eval-viewer/viewer.html new file mode 100644 index 00000000..6d8e9634 --- /dev/null +++ b/skill-creator/eval-viewer/viewer.html @@ -0,0 +1,1325 @@ + + + + + + Eval Review + + + + + + + +
+
+
+

Eval Review:

+
Review each output and leave feedback below. Navigate with arrow keys or buttons. When done, copy feedback and paste into Claude Code.
+
+
+
+ + + + + +
+
+ +
+
Prompt
+
+
+
+
+ + +
+
Output
+
+
No output files found
+
+
+ + + + + + + + +
+
Your Feedback
+
+ + + +
+
+
+ + +
+ + +
+
+
No benchmark data available. Run a benchmark to see quantitative results here.
+
+
+
+ + +
+
+

Review Complete

+

Your feedback has been saved. Go back to your Claude Code session and tell Claude you're done reviewing.

+
+ +
+
+
+ + +
+ + + + diff --git a/skill-creator/references/prerequisites.md b/skill-creator/references/prerequisites.md new file mode 100644 index 00000000..0873263a --- /dev/null +++ b/skill-creator/references/prerequisites.md @@ -0,0 +1,117 @@ +# Skill Creator Prerequisites + +Auto-detect and install all dependencies before starting skill creation. This prevents failures mid-workflow (e.g., discovering gitleaks is missing only at the packaging step). + +## Quick Check Script + +Run all checks in one go: + +```bash +echo "=== Skill Creator Prerequisites ===" +echo -n "Python 3: "; python3 --version 2>/dev/null || echo "MISSING" +echo -n "PyYAML: "; python3 -c "import yaml; print('OK')" 2>/dev/null || echo "MISSING" +echo -n "gitleaks: "; gitleaks version 2>/dev/null || echo "MISSING" +echo -n "claude CLI: "; which claude 2>/dev/null || echo "MISSING" +echo -n "anthropic SDK: "; python3 -c "import anthropic; print('OK')" 2>/dev/null || echo "MISSING (optional)" +echo -n "uv: "; uv --version 2>/dev/null || echo "MISSING (optional)" +``` + +## Dependencies by Phase + +| Dependency | Required For | Phase | Severity | +|-----------|-------------|-------|----------| +| Python 3.7+ | All scripts | All | **Blocking** | +| PyYAML | `quick_validate.py`, `package_skill.py` | Validation, Packaging | **Blocking** | +| gitleaks | `security_scan.py` | Security Review (Step 6) | **Blocking for packaging** | +| claude CLI | `run_eval.py`, `run_loop.py` | Testing, Description Optimization | **Blocking for evals** | +| anthropic SDK | `improve_description.py`, `run_loop.py` | Description Optimization | Optional (only for desc optimization) | +| uv | Skills that bundle Python scripts | Export/Runtime | Optional (skill-specific) | +| webbrowser | `generate_review.py` (viewer) | Eval Review | Optional (can use `--static` fallback) | + +## Auto-Installation + +### PyYAML (required) + +```bash +# Preferred: via uv +uv pip install --system pyyaml + +# Alternative: via pip +pip3 install pyyaml + +# Verify +python3 -c "import yaml; print(yaml.__version__)" +``` + +### gitleaks (required for packaging) + +```bash +# macOS +brew install gitleaks + +# Linux +wget https://github.com/gitleaks/gitleaks/releases/download/v8.21.2/gitleaks_8.21.2_linux_x64.tar.gz +tar -xzf gitleaks_8.21.2_linux_x64.tar.gz && sudo mv gitleaks /usr/local/bin/ + +# Verify +gitleaks version +``` + +### anthropic SDK (optional, for description optimization) + +```bash +# Preferred: via uv +uv pip install --system anthropic + +# Alternative: via pip +pip3 install anthropic + +# Verify +python3 -c "import anthropic; print('OK')" +``` + +Also requires `ANTHROPIC_API_KEY` environment variable to be set. + +### claude CLI (required for evals) + +The `claude` CLI (Claude Code) must be installed and available in PATH. If the user is already running this skill inside Claude Code, this is already satisfied. + +```bash +# Verify +which claude && claude --version +``` + +If missing, the user needs to install Claude Code from https://claude.ai/claude-code. + +## Script Invocation + +All scripts must be run from the skill-creator root directory using module syntax: + +```bash +# CORRECT — run from skill-creator directory +cd +python3 -m scripts.package_skill +python3 -m scripts.security_scan +python3 -m scripts.aggregate_benchmark --skill-name + +# WRONG — direct invocation fails with ModuleNotFoundError +python3 scripts/package_skill.py # ImportError: No module named 'scripts' +``` + +This is because the scripts use relative imports (`from scripts.quick_validate import ...`). + +## Presenting Results to User + +After running all checks, present a summary table: + +``` +Skill Creator Prerequisites: + [x] Python 3.12.0 + [x] PyYAML 6.0.1 + [x] gitleaks 8.21.2 + [x] claude CLI (running inside Claude Code) + [ ] anthropic SDK — not installed (only needed for description optimization) + [x] uv 0.6.x +``` + +If any **blocking** dependency is missing and auto-install fails, clearly explain what the user needs to do and stop before proceeding to skill creation. diff --git a/skill-creator/references/schemas.md b/skill-creator/references/schemas.md new file mode 100644 index 00000000..b6eeaa2d --- /dev/null +++ b/skill-creator/references/schemas.md @@ -0,0 +1,430 @@ +# JSON Schemas + +This document defines the JSON schemas used by skill-creator. + +--- + +## evals.json + +Defines the evals for a skill. Located at `evals/evals.json` within the skill directory. + +```json +{ + "skill_name": "example-skill", + "evals": [ + { + "id": 1, + "prompt": "User's example prompt", + "expected_output": "Description of expected result", + "files": ["evals/files/sample1.pdf"], + "expectations": [ + "The output includes X", + "The skill used script Y" + ] + } + ] +} +``` + +**Fields:** +- `skill_name`: Name matching the skill's frontmatter +- `evals[].id`: Unique integer identifier +- `evals[].prompt`: The task to execute +- `evals[].expected_output`: Human-readable description of success +- `evals[].files`: Optional list of input file paths (relative to skill root) +- `evals[].expectations`: List of verifiable statements + +--- + +## history.json + +Tracks version progression in Improve mode. Located at workspace root. + +```json +{ + "started_at": "2026-01-15T10:30:00Z", + "skill_name": "pdf", + "current_best": "v2", + "iterations": [ + { + "version": "v0", + "parent": null, + "expectation_pass_rate": 0.65, + "grading_result": "baseline", + "is_current_best": false + }, + { + "version": "v1", + "parent": "v0", + "expectation_pass_rate": 0.75, + "grading_result": "won", + "is_current_best": false + }, + { + "version": "v2", + "parent": "v1", + "expectation_pass_rate": 0.85, + "grading_result": "won", + "is_current_best": true + } + ] +} +``` + +**Fields:** +- `started_at`: ISO timestamp of when improvement started +- `skill_name`: Name of the skill being improved +- `current_best`: Version identifier of the best performer +- `iterations[].version`: Version identifier (v0, v1, ...) +- `iterations[].parent`: Parent version this was derived from +- `iterations[].expectation_pass_rate`: Pass rate from grading +- `iterations[].grading_result`: "baseline", "won", "lost", or "tie" +- `iterations[].is_current_best`: Whether this is the current best version + +--- + +## grading.json + +Output from the grader agent. Located at `/grading.json`. + +```json +{ + "expectations": [ + { + "text": "The output includes the name 'John Smith'", + "passed": true, + "evidence": "Found in transcript Step 3: 'Extracted names: John Smith, Sarah Johnson'" + }, + { + "text": "The spreadsheet has a SUM formula in cell B10", + "passed": false, + "evidence": "No spreadsheet was created. The output was a text file." + } + ], + "summary": { + "passed": 2, + "failed": 1, + "total": 3, + "pass_rate": 0.67 + }, + "execution_metrics": { + "tool_calls": { + "Read": 5, + "Write": 2, + "Bash": 8 + }, + "total_tool_calls": 15, + "total_steps": 6, + "errors_encountered": 0, + "output_chars": 12450, + "transcript_chars": 3200 + }, + "timing": { + "executor_duration_seconds": 165.0, + "grader_duration_seconds": 26.0, + "total_duration_seconds": 191.0 + }, + "claims": [ + { + "claim": "The form has 12 fillable fields", + "type": "factual", + "verified": true, + "evidence": "Counted 12 fields in field_info.json" + } + ], + "user_notes_summary": { + "uncertainties": ["Used 2023 data, may be stale"], + "needs_review": [], + "workarounds": ["Fell back to text overlay for non-fillable fields"] + }, + "eval_feedback": { + "suggestions": [ + { + "assertion": "The output includes the name 'John Smith'", + "reason": "A hallucinated document that mentions the name would also pass" + } + ], + "overall": "Assertions check presence but not correctness." + } +} +``` + +**Fields:** +- `expectations[]`: Graded expectations with evidence +- `summary`: Aggregate pass/fail counts +- `execution_metrics`: Tool usage and output size (from executor's metrics.json) +- `timing`: Wall clock timing (from timing.json) +- `claims`: Extracted and verified claims from the output +- `user_notes_summary`: Issues flagged by the executor +- `eval_feedback`: (optional) Improvement suggestions for the evals, only present when the grader identifies issues worth raising + +--- + +## metrics.json + +Output from the executor agent. Located at `/outputs/metrics.json`. + +```json +{ + "tool_calls": { + "Read": 5, + "Write": 2, + "Bash": 8, + "Edit": 1, + "Glob": 2, + "Grep": 0 + }, + "total_tool_calls": 18, + "total_steps": 6, + "files_created": ["filled_form.pdf", "field_values.json"], + "errors_encountered": 0, + "output_chars": 12450, + "transcript_chars": 3200 +} +``` + +**Fields:** +- `tool_calls`: Count per tool type +- `total_tool_calls`: Sum of all tool calls +- `total_steps`: Number of major execution steps +- `files_created`: List of output files created +- `errors_encountered`: Number of errors during execution +- `output_chars`: Total character count of output files +- `transcript_chars`: Character count of transcript + +--- + +## timing.json + +Wall clock timing for a run. Located at `/timing.json`. + +**How to capture:** When a subagent task completes, the task notification includes `total_tokens` and `duration_ms`. Save these immediately — they are not persisted anywhere else and cannot be recovered after the fact. + +```json +{ + "total_tokens": 84852, + "duration_ms": 23332, + "total_duration_seconds": 23.3, + "executor_start": "2026-01-15T10:30:00Z", + "executor_end": "2026-01-15T10:32:45Z", + "executor_duration_seconds": 165.0, + "grader_start": "2026-01-15T10:32:46Z", + "grader_end": "2026-01-15T10:33:12Z", + "grader_duration_seconds": 26.0 +} +``` + +--- + +## benchmark.json + +Output from Benchmark mode. Located at `benchmarks//benchmark.json`. + +```json +{ + "metadata": { + "skill_name": "pdf", + "skill_path": "/path/to/pdf", + "executor_model": "claude-sonnet-4-20250514", + "analyzer_model": "most-capable-model", + "timestamp": "2026-01-15T10:30:00Z", + "evals_run": [1, 2, 3], + "runs_per_configuration": 3 + }, + + "runs": [ + { + "eval_id": 1, + "eval_name": "Ocean", + "configuration": "with_skill", + "run_number": 1, + "result": { + "pass_rate": 0.85, + "passed": 6, + "failed": 1, + "total": 7, + "time_seconds": 42.5, + "tokens": 3800, + "tool_calls": 18, + "errors": 0 + }, + "expectations": [ + {"text": "...", "passed": true, "evidence": "..."} + ], + "notes": [ + "Used 2023 data, may be stale", + "Fell back to text overlay for non-fillable fields" + ] + } + ], + + "run_summary": { + "with_skill": { + "pass_rate": {"mean": 0.85, "stddev": 0.05, "min": 0.80, "max": 0.90}, + "time_seconds": {"mean": 45.0, "stddev": 12.0, "min": 32.0, "max": 58.0}, + "tokens": {"mean": 3800, "stddev": 400, "min": 3200, "max": 4100} + }, + "without_skill": { + "pass_rate": {"mean": 0.35, "stddev": 0.08, "min": 0.28, "max": 0.45}, + "time_seconds": {"mean": 32.0, "stddev": 8.0, "min": 24.0, "max": 42.0}, + "tokens": {"mean": 2100, "stddev": 300, "min": 1800, "max": 2500} + }, + "delta": { + "pass_rate": "+0.50", + "time_seconds": "+13.0", + "tokens": "+1700" + } + }, + + "notes": [ + "Assertion 'Output is a PDF file' passes 100% in both configurations - may not differentiate skill value", + "Eval 3 shows high variance (50% ± 40%) - may be flaky or model-dependent", + "Without-skill runs consistently fail on table extraction expectations", + "Skill adds 13s average execution time but improves pass rate by 50%" + ] +} +``` + +**Fields:** +- `metadata`: Information about the benchmark run + - `skill_name`: Name of the skill + - `timestamp`: When the benchmark was run + - `evals_run`: List of eval names or IDs + - `runs_per_configuration`: Number of runs per config (e.g. 3) +- `runs[]`: Individual run results + - `eval_id`: Numeric eval identifier + - `eval_name`: Human-readable eval name (used as section header in the viewer) + - `configuration`: Must be `"with_skill"` or `"without_skill"` (the viewer uses this exact string for grouping and color coding) + - `run_number`: Integer run number (1, 2, 3...) + - `result`: Nested object with `pass_rate`, `passed`, `total`, `time_seconds`, `tokens`, `errors` +- `run_summary`: Statistical aggregates per configuration + - `with_skill` / `without_skill`: Each contains `pass_rate`, `time_seconds`, `tokens` objects with `mean` and `stddev` fields + - `delta`: Difference strings like `"+0.50"`, `"+13.0"`, `"+1700"` +- `notes`: Freeform observations from the analyzer + +**Important:** The viewer reads these field names exactly. Using `config` instead of `configuration`, or putting `pass_rate` at the top level of a run instead of nested under `result`, will cause the viewer to show empty/zero values. Always reference this schema when generating benchmark.json manually. + +--- + +## comparison.json + +Output from blind comparator. Located at `/comparison-N.json`. + +```json +{ + "winner": "A", + "reasoning": "Output A provides a complete solution with proper formatting and all required fields. Output B is missing the date field and has formatting inconsistencies.", + "rubric": { + "A": { + "content": { + "correctness": 5, + "completeness": 5, + "accuracy": 4 + }, + "structure": { + "organization": 4, + "formatting": 5, + "usability": 4 + }, + "content_score": 4.7, + "structure_score": 4.3, + "overall_score": 9.0 + }, + "B": { + "content": { + "correctness": 3, + "completeness": 2, + "accuracy": 3 + }, + "structure": { + "organization": 3, + "formatting": 2, + "usability": 3 + }, + "content_score": 2.7, + "structure_score": 2.7, + "overall_score": 5.4 + } + }, + "output_quality": { + "A": { + "score": 9, + "strengths": ["Complete solution", "Well-formatted", "All fields present"], + "weaknesses": ["Minor style inconsistency in header"] + }, + "B": { + "score": 5, + "strengths": ["Readable output", "Correct basic structure"], + "weaknesses": ["Missing date field", "Formatting inconsistencies", "Partial data extraction"] + } + }, + "expectation_results": { + "A": { + "passed": 4, + "total": 5, + "pass_rate": 0.80, + "details": [ + {"text": "Output includes name", "passed": true} + ] + }, + "B": { + "passed": 3, + "total": 5, + "pass_rate": 0.60, + "details": [ + {"text": "Output includes name", "passed": true} + ] + } + } +} +``` + +--- + +## analysis.json + +Output from post-hoc analyzer. Located at `/analysis.json`. + +```json +{ + "comparison_summary": { + "winner": "A", + "winner_skill": "path/to/winner/skill", + "loser_skill": "path/to/loser/skill", + "comparator_reasoning": "Brief summary of why comparator chose winner" + }, + "winner_strengths": [ + "Clear step-by-step instructions for handling multi-page documents", + "Included validation script that caught formatting errors" + ], + "loser_weaknesses": [ + "Vague instruction 'process the document appropriately' led to inconsistent behavior", + "No script for validation, agent had to improvise" + ], + "instruction_following": { + "winner": { + "score": 9, + "issues": ["Minor: skipped optional logging step"] + }, + "loser": { + "score": 6, + "issues": [ + "Did not use the skill's formatting template", + "Invented own approach instead of following step 3" + ] + } + }, + "improvement_suggestions": [ + { + "priority": "high", + "category": "instructions", + "suggestion": "Replace 'process the document appropriately' with explicit steps", + "expected_impact": "Would eliminate ambiguity that caused inconsistent behavior" + } + ], + "transcript_insights": { + "winner_execution_pattern": "Read skill -> Followed 5-step process -> Used validation script", + "loser_execution_pattern": "Read skill -> Unclear on approach -> Tried 3 different methods" + } +} +``` diff --git a/skill-creator/references/skill-development-methodology.md b/skill-creator/references/skill-development-methodology.md new file mode 100644 index 00000000..0d93b075 --- /dev/null +++ b/skill-creator/references/skill-development-methodology.md @@ -0,0 +1,149 @@ +# Skill Development Methodology + +综合 Anthropic 官方最佳实践、skill-creator 工作流、社区经验和实战教训的完整方法论。 + +本文档只包含 SKILL.md 中**没有覆盖**的内容。SKILL.md 已经详细描述的流程（Prior Art 8 渠道表、决策矩阵、Inline vs Fork、测试用例格式、描述优化循环等）不在此重复——请直接参考 SKILL.md 对应章节。 + +## Phase 1: 先手动解决问题，不要上来就建 skill + +SKILL.md 的 "Capture Intent" 章节覆盖了意图收集的 4 个问题和 skill 类型分类。本节补充一个被忽略的前置步骤： + +**不要一开始就写 skill。** 先用 Claude Code 正常解决用户的问题，在过程中积累经验——哪些方案有效、哪些失败、最终的 working solution 是什么。如果你没有亲自失败过，你写不出能防止别人失败的 skill。 + +很多 skill 都是从"把我们刚做的变成一个 skill"中诞生的。先从对话历史中提取已验证的模式（SKILL.md "Capture Intent" 第三段已提及），然后才开始规划 skill 结构。 + +## Phase 2: 用 Agent Team 做并行调研 + +SKILL.md 的 "Prior Art Research" 章节覆盖了 8 个搜索渠道、clone-and-verify 检查清单、和 Adopt/Extend/Build 决策矩阵。本节补充 SKILL.md 未提及的**并行调研模式**： + +遇到不确定的技术方案时，不要串行尝试（太慢），也不要凭经验猜（太危险）。同时启动 3+ 个研究 agent，每个负责一个调研方向： + +| Agent | 职责 | 搜索范围 | +|-------|------|---------| +| 工具调研 | 找已有成熟工具 | GitHub stars、npm/PyPI、社区 skill 注册表 | +| API 调研 | 找可用 API 端点 | 官方文档、逆向工程、移动端 API | +| 约束调研 | 理解技术限制 | 反爬机制、认证要求、平台限制 | + +每个 agent 必须独立验证（读源码、确认 API 可达、检查最近提交日期），不能只看 README。 + +**案例**：开发一个数据导出 skill 时，3 个 agent 并行跑了 5-20 分钟，分别发现：一个关键工具当前版本 broken（605 stars 但 PR 待合并）、一个未公开的移动端 API（唯一可行方案）、目标平台升级了 PoW 反爬（所有 HTTP 抓取失效）。没有并行研究，这些信息需要串行试错 3+ 小时才能获得。 + +## Phase 3: 用真实数据验证原型 + +SKILL.md 的 Evaluation-Driven Development 流程覆盖了"先跑 baseline → 建 eval → 迭代"的过程。本节补充两个 SKILL.md 未强调的验证原则： + +### 3.1 数据完整性验证 + +"it runs without errors" ≠ "it exported all items correctly"。必须： +- 对比 API 报告的 total 和实际导出行数 +- 检查字段格式（评分、日期、编码是否符合预期） +- 用不同规模的数据测试（0 条、100 条、1000+ 条） + +**常见静默 bug**： +- 分页逻辑：某些页面返回的数据量少于请求值（如请求 50 条返回 48 条），被误判为最后一页导致提前终止。修复：检查 `total` 而非 `page_size` +- 数据转换：API 返回 `{value: 2, max: 5}` 表示 2/5 星，但代码按 `max: 10` 处理后变成 1 星。修复：检查 `max` 字段确定 scale + +### 3.2 记录失败 + +详细记录每个失败方案的方法、失败模式、根因。这些将成为 skill 中 "Do NOT attempt" 部分的内容——这是 skill 最独特的价值，防止未来的 agent 重走弯路。 + +失败记录的结构： + +| 方案 | 结果 | 根因 | +|------|------|------| +| 方案名称 | 具体失败表现（HTTP 状态码、错误信息） | 架构层面的原因分析 | + +## Phase 4: Skill 写作补充原则 + +SKILL.md 的 "Skill Writing Guide" 已覆盖 frontmatter、progressive disclosure、bundled resources、命名规范等。本节补充 SKILL.md 未提及的内容层面原则： + +### 4.1 写清楚 skill 不能做什么 + +防止 agent 尝试不可能的操作。例如： +- "Cannot export reviews (长评) — different API endpoint, not implemented" +- "Cannot filter by single category — exports all 4 types together" + +### 4.2 写清楚失败过什么 + +在 SKILL.md 或 references 中保留失败方案的摘要（详见 Phase 3.2），加上明确的"Do NOT attempt"警告。这比正面指令更有效——agent 看到 7 种方案的失败记录后，不会尝试第 8 种类似方案。 + +### 4.3 安全说明 + +如果脚本包含 API key、HMAC 密钥或其他凭据，必须解释来源和安全性。例如："These are the app's public credentials extracted from the APK, shared by all users. No personal credentials are used." + +### 4.4 Console output 示例 + +展示一次成功运行的完整控制台输出。让 agent 知道"正确运行"长什么样，方便验证（SKILL.md Phase 5 的 self-verification）。 + +### 4.5 脚本健壮性 + +SKILL.md 的 "Solve, don't punt" 覆盖了基本错误处理。补充实战中发现的常见遗漏： +- 只捕获 HTTPError，遗漏 URLError / socket.timeout / JSONDecodeError +- 无限分页循环（API 异常时）——需要 max-page 安全阀 +- CSV 中的换行符/回车符——`csvEscape` 必须处理 `\r` +- 用户输入是完整 URL 而非 ID——脚本应自动提取 + +## Phase 5: 测试迭代补充 + +SKILL.md 的测试流程非常详细（A/B 测试、断言、评分、viewer）。本节补充两个 SKILL.md 未覆盖的实操教训： + +### 5.1 删除竞争的旧 skill + +如果系统中存在旧版 skill（关键词冲突），eval agent 会被旧 skill 截胡，导致测试结果完全无效。必须在测试前删除旧 skill。 + +**信号**：eval agent 使用了不同于预期的脚本或方法 → 检查是否有同名/同领域的旧 skill 被加载。 + +### 5.2 量化迭代对比 + +SKILL.md 提到 timing.json 和 benchmark，但未给出具体应跟踪哪些指标。推荐： + +| 指标 | 为什么重要 | +|------|-----------| +| 数据完整性（实际/预期） | 核心正确性 | +| 执行时间 | 用户体验 | +| Token 消耗 | 成本 | +| 工具调用次数 | skill 引导效率——次数越少说明 skill 的指令越清晰 | +| 错误数 | 必须为 0 | + +**案例对比**：某 skill 迭代后，工具调用从 31 次降到 8 次（74% 减少）、Token 从 72K 降到 41K（43% 减少），说明 skill 的指令让 agent 不再需要自己摸索。 + +## Phase 6: Counter Review — 用 Agent Team 做对抗性审查 + +这是 SKILL.md 未覆盖的独立环节。SKILL.md 的 "Improving the skill" 章节关注用户反馈驱动的迭代，但没有系统化的多视角审查流程。 + +### 6.1 第一轮：3 个视角并行 + +用 Task 工具同时启动 3 个 review agent： + +| Reviewer | 视角 | 关注点 | +|----------|------|--------| +| Skill 质量 | 对标 Anthropic 最佳实践 | 描述质量、简洁性、progressive disclosure、可操作性、错误预防、示例、术语一致性 | +| 代码健壮性 | 高级工程师找 bug | 错误处理、安全性、跨平台、边界情况、依赖、幂等性 | +| 用户视角 | 首次使用者体验 | 首次成功率、输入容错、输出预期、隐私顾虑、失败恢复 | + +### 6.2 修复后 Final Gate + +修复所有 Critical 和 HIGH 问题后，再启动 final gate reviewers 验证修复正确性。评分 >= 8 才放行。 + +### 6.3 常见发现模式 + +根据实战经验，reviewer 经常发现的问题类型： +- **SKILL.md 和 references 内容重复**（每次都会犯，包括本文档自己） +- **异常类型遗漏**（只捕获 HTTPError，漏掉 URLError/socket.timeout） +- **substring 误匹配**（`content.includes(url)` 导致 `/1234/` 匹配 `/12345/`） +- **docstring 与实际行为不一致**（写了 "4.5 → 5" 但实际行为是 "4.5 → 4"） +- **误导性注释**（注释说"每个分类写入后立即保存"但代码在最后才写入） +- **时间敏感数据**（特定日期的测试结果、版本号——下周就过时了） + +## Phase 7 & 8: Description Optimization + Packaging + +SKILL.md 已完整覆盖描述优化循环（20 个 eval query、60/40 train/test split、5 轮迭代）和打包流程（prerequisites、security scan、marketplace.json）。无补充。 + +## 来源 + +| 来源 | 本文档引用的独有贡献 | +|------|-------------------| +| Anthropic Official | Evaluation-driven development、conciseness imperative（已由 SKILL.md 覆盖，本文不重复） | +| skill-creator SKILL.md | 完整工作流和工具链（本文引用但不复制，请直接参考 SKILL.md） | +| 社区经验 | 激活率数据（20%→90%）、Encoded Preference > Capability Uplift | +| 实战教训 | 并行研究 agent、失败记录的价值、竞争 skill 删除、量化迭代对比、Counter Review 流程 | diff --git a/skill-creator/scripts/__init__.py b/skill-creator/scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/skill-creator/scripts/aggregate_benchmark.py b/skill-creator/scripts/aggregate_benchmark.py new file mode 100755 index 00000000..3e66e8c1 --- /dev/null +++ b/skill-creator/scripts/aggregate_benchmark.py @@ -0,0 +1,401 @@ +#!/usr/bin/env python3 +""" +Aggregate individual run results into benchmark summary statistics. + +Reads grading.json files from run directories and produces: +- run_summary with mean, stddev, min, max for each metric +- delta between with_skill and without_skill configurations + +Usage: + python aggregate_benchmark.py + +Example: + python aggregate_benchmark.py benchmarks/2026-01-15T10-30-00/ + +The script supports two directory layouts: + + Workspace layout (from skill-creator iterations): + / + └── eval-N/ + ├── with_skill/ + │ ├── run-1/grading.json + │ └── run-2/grading.json + └── without_skill/ + ├── run-1/grading.json + └── run-2/grading.json + + Legacy layout (with runs/ subdirectory): + / + └── runs/ + └── eval-N/ + ├── with_skill/ + │ └── run-1/grading.json + └── without_skill/ + └── run-1/grading.json +""" + +import argparse +import json +import math +import sys +from datetime import datetime, timezone +from pathlib import Path + + +def calculate_stats(values: list[float]) -> dict: + """Calculate mean, stddev, min, max for a list of values.""" + if not values: + return {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0} + + n = len(values) + mean = sum(values) / n + + if n > 1: + variance = sum((x - mean) ** 2 for x in values) / (n - 1) + stddev = math.sqrt(variance) + else: + stddev = 0.0 + + return { + "mean": round(mean, 4), + "stddev": round(stddev, 4), + "min": round(min(values), 4), + "max": round(max(values), 4) + } + + +def load_run_results(benchmark_dir: Path) -> dict: + """ + Load all run results from a benchmark directory. + + Returns dict keyed by config name (e.g. "with_skill"/"without_skill", + or "new_skill"/"old_skill"), each containing a list of run results. + """ + # Support both layouts: eval dirs directly under benchmark_dir, or under runs/ + runs_dir = benchmark_dir / "runs" + if runs_dir.exists(): + search_dir = runs_dir + elif list(benchmark_dir.glob("eval-*")): + search_dir = benchmark_dir + else: + print(f"No eval directories found in {benchmark_dir} or {benchmark_dir / 'runs'}") + return {} + + results: dict[str, list] = {} + + for eval_idx, eval_dir in enumerate(sorted(search_dir.glob("eval-*"))): + metadata_path = eval_dir / "eval_metadata.json" + if metadata_path.exists(): + try: + with open(metadata_path) as mf: + eval_id = json.load(mf).get("eval_id", eval_idx) + except (json.JSONDecodeError, OSError): + eval_id = eval_idx + else: + try: + eval_id = int(eval_dir.name.split("-")[1]) + except ValueError: + eval_id = eval_idx + + # Discover config directories dynamically rather than hardcoding names + for config_dir in sorted(eval_dir.iterdir()): + if not config_dir.is_dir(): + continue + # Skip non-config directories (inputs, outputs, etc.) + if not list(config_dir.glob("run-*")): + continue + config = config_dir.name + if config not in results: + results[config] = [] + + for run_dir in sorted(config_dir.glob("run-*")): + run_number = int(run_dir.name.split("-")[1]) + grading_file = run_dir / "grading.json" + + if not grading_file.exists(): + print(f"Warning: grading.json not found in {run_dir}") + continue + + try: + with open(grading_file) as f: + grading = json.load(f) + except json.JSONDecodeError as e: + print(f"Warning: Invalid JSON in {grading_file}: {e}") + continue + + # Extract metrics + result = { + "eval_id": eval_id, + "run_number": run_number, + "pass_rate": grading.get("summary", {}).get("pass_rate", 0.0), + "passed": grading.get("summary", {}).get("passed", 0), + "failed": grading.get("summary", {}).get("failed", 0), + "total": grading.get("summary", {}).get("total", 0), + } + + # Extract timing — check grading.json first, then sibling timing.json + timing = grading.get("timing", {}) + result["time_seconds"] = timing.get("total_duration_seconds", 0.0) + timing_file = run_dir / "timing.json" + if result["time_seconds"] == 0.0 and timing_file.exists(): + try: + with open(timing_file) as tf: + timing_data = json.load(tf) + result["time_seconds"] = timing_data.get("total_duration_seconds", 0.0) + result["tokens"] = timing_data.get("total_tokens", 0) + except json.JSONDecodeError: + pass + + # Extract metrics if available + metrics = grading.get("execution_metrics", {}) + result["tool_calls"] = metrics.get("total_tool_calls", 0) + if not result.get("tokens"): + result["tokens"] = metrics.get("output_chars", 0) + result["errors"] = metrics.get("errors_encountered", 0) + + # Extract expectations — viewer requires fields: text, passed, evidence + raw_expectations = grading.get("expectations", []) + for exp in raw_expectations: + if "text" not in exp or "passed" not in exp: + print(f"Warning: expectation in {grading_file} missing required fields (text, passed, evidence): {exp}") + result["expectations"] = raw_expectations + + # Extract notes from user_notes_summary + notes_summary = grading.get("user_notes_summary", {}) + notes = [] + notes.extend(notes_summary.get("uncertainties", [])) + notes.extend(notes_summary.get("needs_review", [])) + notes.extend(notes_summary.get("workarounds", [])) + result["notes"] = notes + + results[config].append(result) + + return results + + +def aggregate_results(results: dict) -> dict: + """ + Aggregate run results into summary statistics. + + Returns run_summary with stats for each configuration and delta. + """ + run_summary = {} + configs = list(results.keys()) + + for config in configs: + runs = results.get(config, []) + + if not runs: + run_summary[config] = { + "pass_rate": {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0}, + "time_seconds": {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0}, + "tokens": {"mean": 0, "stddev": 0, "min": 0, "max": 0} + } + continue + + pass_rates = [r["pass_rate"] for r in runs] + times = [r["time_seconds"] for r in runs] + tokens = [r.get("tokens", 0) for r in runs] + + run_summary[config] = { + "pass_rate": calculate_stats(pass_rates), + "time_seconds": calculate_stats(times), + "tokens": calculate_stats(tokens) + } + + # Calculate delta between the first two configs (if two exist) + if len(configs) >= 2: + primary = run_summary.get(configs[0], {}) + baseline = run_summary.get(configs[1], {}) + else: + primary = run_summary.get(configs[0], {}) if configs else {} + baseline = {} + + delta_pass_rate = primary.get("pass_rate", {}).get("mean", 0) - baseline.get("pass_rate", {}).get("mean", 0) + delta_time = primary.get("time_seconds", {}).get("mean", 0) - baseline.get("time_seconds", {}).get("mean", 0) + delta_tokens = primary.get("tokens", {}).get("mean", 0) - baseline.get("tokens", {}).get("mean", 0) + + run_summary["delta"] = { + "pass_rate": f"{delta_pass_rate:+.2f}", + "time_seconds": f"{delta_time:+.1f}", + "tokens": f"{delta_tokens:+.0f}" + } + + return run_summary + + +def generate_benchmark(benchmark_dir: Path, skill_name: str = "", skill_path: str = "") -> dict: + """ + Generate complete benchmark.json from run results. + """ + results = load_run_results(benchmark_dir) + run_summary = aggregate_results(results) + + # Build runs array for benchmark.json + runs = [] + for config in results: + for result in results[config]: + runs.append({ + "eval_id": result["eval_id"], + "configuration": config, + "run_number": result["run_number"], + "result": { + "pass_rate": result["pass_rate"], + "passed": result["passed"], + "failed": result["failed"], + "total": result["total"], + "time_seconds": result["time_seconds"], + "tokens": result.get("tokens", 0), + "tool_calls": result.get("tool_calls", 0), + "errors": result.get("errors", 0) + }, + "expectations": result["expectations"], + "notes": result["notes"] + }) + + # Determine eval IDs from results + eval_ids = sorted(set( + r["eval_id"] + for config in results.values() + for r in config + )) + + benchmark = { + "metadata": { + "skill_name": skill_name or "", + "skill_path": skill_path or "", + "executor_model": "", + "analyzer_model": "", + "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "evals_run": eval_ids, + "runs_per_configuration": 3 + }, + "runs": runs, + "run_summary": run_summary, + "notes": [] # To be filled by analyzer + } + + return benchmark + + +def generate_markdown(benchmark: dict) -> str: + """Generate human-readable benchmark.md from benchmark data.""" + metadata = benchmark["metadata"] + run_summary = benchmark["run_summary"] + + # Determine config names (excluding "delta") + configs = [k for k in run_summary if k != "delta"] + config_a = configs[0] if len(configs) >= 1 else "config_a" + config_b = configs[1] if len(configs) >= 2 else "config_b" + label_a = config_a.replace("_", " ").title() + label_b = config_b.replace("_", " ").title() + + lines = [ + f"# Skill Benchmark: {metadata['skill_name']}", + "", + f"**Model**: {metadata['executor_model']}", + f"**Date**: {metadata['timestamp']}", + f"**Evals**: {', '.join(map(str, metadata['evals_run']))} ({metadata['runs_per_configuration']} runs each per configuration)", + "", + "## Summary", + "", + f"| Metric | {label_a} | {label_b} | Delta |", + "|--------|------------|---------------|-------|", + ] + + a_summary = run_summary.get(config_a, {}) + b_summary = run_summary.get(config_b, {}) + delta = run_summary.get("delta", {}) + + # Format pass rate + a_pr = a_summary.get("pass_rate", {}) + b_pr = b_summary.get("pass_rate", {}) + lines.append(f"| Pass Rate | {a_pr.get('mean', 0)*100:.0f}% ± {a_pr.get('stddev', 0)*100:.0f}% | {b_pr.get('mean', 0)*100:.0f}% ± {b_pr.get('stddev', 0)*100:.0f}% | {delta.get('pass_rate', '—')} |") + + # Format time + a_time = a_summary.get("time_seconds", {}) + b_time = b_summary.get("time_seconds", {}) + lines.append(f"| Time | {a_time.get('mean', 0):.1f}s ± {a_time.get('stddev', 0):.1f}s | {b_time.get('mean', 0):.1f}s ± {b_time.get('stddev', 0):.1f}s | {delta.get('time_seconds', '—')}s |") + + # Format tokens + a_tokens = a_summary.get("tokens", {}) + b_tokens = b_summary.get("tokens", {}) + lines.append(f"| Tokens | {a_tokens.get('mean', 0):.0f} ± {a_tokens.get('stddev', 0):.0f} | {b_tokens.get('mean', 0):.0f} ± {b_tokens.get('stddev', 0):.0f} | {delta.get('tokens', '—')} |") + + # Notes section + if benchmark.get("notes"): + lines.extend([ + "", + "## Notes", + "" + ]) + for note in benchmark["notes"]: + lines.append(f"- {note}") + + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser( + description="Aggregate benchmark run results into summary statistics" + ) + parser.add_argument( + "benchmark_dir", + type=Path, + help="Path to the benchmark directory" + ) + parser.add_argument( + "--skill-name", + default="", + help="Name of the skill being benchmarked" + ) + parser.add_argument( + "--skill-path", + default="", + help="Path to the skill being benchmarked" + ) + parser.add_argument( + "--output", "-o", + type=Path, + help="Output path for benchmark.json (default: /benchmark.json)" + ) + + args = parser.parse_args() + + if not args.benchmark_dir.exists(): + print(f"Directory not found: {args.benchmark_dir}") + sys.exit(1) + + # Generate benchmark + benchmark = generate_benchmark(args.benchmark_dir, args.skill_name, args.skill_path) + + # Determine output paths + output_json = args.output or (args.benchmark_dir / "benchmark.json") + output_md = output_json.with_suffix(".md") + + # Write benchmark.json + with open(output_json, "w") as f: + json.dump(benchmark, f, indent=2) + print(f"Generated: {output_json}") + + # Write benchmark.md + markdown = generate_markdown(benchmark) + with open(output_md, "w") as f: + f.write(markdown) + print(f"Generated: {output_md}") + + # Print summary + run_summary = benchmark["run_summary"] + configs = [k for k in run_summary if k != "delta"] + delta = run_summary.get("delta", {}) + + print(f"\nSummary:") + for config in configs: + pr = run_summary[config]["pass_rate"]["mean"] + label = config.replace("_", " ").title() + print(f" {label}: {pr*100:.1f}% pass rate") + print(f" Delta: {delta.get('pass_rate', '—')}") + + +if __name__ == "__main__": + main() diff --git a/skill-creator/scripts/generate_report.py b/skill-creator/scripts/generate_report.py new file mode 100755 index 00000000..959e30a0 --- /dev/null +++ b/skill-creator/scripts/generate_report.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +"""Generate an HTML report from run_loop.py output. + +Takes the JSON output from run_loop.py and generates a visual HTML report +showing each description attempt with check/x for each test case. +Distinguishes between train and test queries. +""" + +import argparse +import html +import json +import sys +from pathlib import Path + + +def generate_html(data: dict, auto_refresh: bool = False, skill_name: str = "") -> str: + """Generate HTML report from loop output data. If auto_refresh is True, adds a meta refresh tag.""" + history = data.get("history", []) + holdout = data.get("holdout", 0) + title_prefix = html.escape(skill_name + " \u2014 ") if skill_name else "" + + # Get all unique queries from train and test sets, with should_trigger info + train_queries: list[dict] = [] + test_queries: list[dict] = [] + if history: + for r in history[0].get("train_results", history[0].get("results", [])): + train_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)}) + if history[0].get("test_results"): + for r in history[0].get("test_results", []): + test_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)}) + + refresh_tag = ' \n' if auto_refresh else "" + + html_parts = [""" + + + +""" + refresh_tag + """ """ + title_prefix + """Skill Description Optimization + + + + + + +

""" + title_prefix + """Skill Description Optimization

+
+ Optimizing your skill's description. This page updates automatically as Claude tests different versions of your skill's description. Each row is an iteration — a new description attempt. The columns show test queries: green checkmarks mean the skill triggered correctly (or correctly didn't trigger), red crosses mean it got it wrong. The "Train" score shows performance on queries used to improve the description; the "Test" score shows performance on held-out queries the optimizer hasn't seen. When it's done, Claude will apply the best-performing description to your skill. +
+"""] + + # Summary section + best_test_score = data.get('best_test_score') + best_train_score = data.get('best_train_score') + html_parts.append(f""" +
+

Original: {html.escape(data.get('original_description', 'N/A'))}

+

Best: {html.escape(data.get('best_description', 'N/A'))}

+

Best Score: {data.get('best_score', 'N/A')} {'(test)' if best_test_score else '(train)'}

+

Iterations: {data.get('iterations_run', 0)} | Train: {data.get('train_size', '?')} | Test: {data.get('test_size', '?')}

+
+""") + + # Legend + html_parts.append(""" +
+ Query columns: + Should trigger + Should NOT trigger + Train + Test +
+""") + + # Table header + html_parts.append(""" +
+ + + + + + + +""") + + # Add column headers for train queries + for qinfo in train_queries: + polarity = "positive-col" if qinfo["should_trigger"] else "negative-col" + html_parts.append(f' \n') + + # Add column headers for test queries (different color) + for qinfo in test_queries: + polarity = "positive-col" if qinfo["should_trigger"] else "negative-col" + html_parts.append(f' \n') + + html_parts.append(""" + + +""") + + # Find best iteration for highlighting + if test_queries: + best_iter = max(history, key=lambda h: h.get("test_passed") or 0).get("iteration") + else: + best_iter = max(history, key=lambda h: h.get("train_passed", h.get("passed", 0))).get("iteration") + + # Add rows for each iteration + for h in history: + iteration = h.get("iteration", "?") + train_passed = h.get("train_passed", h.get("passed", 0)) + train_total = h.get("train_total", h.get("total", 0)) + test_passed = h.get("test_passed") + test_total = h.get("test_total") + description = h.get("description", "") + train_results = h.get("train_results", h.get("results", [])) + test_results = h.get("test_results", []) + + # Create lookups for results by query + train_by_query = {r["query"]: r for r in train_results} + test_by_query = {r["query"]: r for r in test_results} if test_results else {} + + # Compute aggregate correct/total runs across all retries + def aggregate_runs(results: list[dict]) -> tuple[int, int]: + correct = 0 + total = 0 + for r in results: + runs = r.get("runs", 0) + triggers = r.get("triggers", 0) + total += runs + if r.get("should_trigger", True): + correct += triggers + else: + correct += runs - triggers + return correct, total + + train_correct, train_runs = aggregate_runs(train_results) + test_correct, test_runs = aggregate_runs(test_results) + + # Determine score classes + def score_class(correct: int, total: int) -> str: + if total > 0: + ratio = correct / total + if ratio >= 0.8: + return "score-good" + elif ratio >= 0.5: + return "score-ok" + return "score-bad" + + train_class = score_class(train_correct, train_runs) + test_class = score_class(test_correct, test_runs) + + row_class = "best-row" if iteration == best_iter else "" + + html_parts.append(f""" + + + + +""") + + # Add result for each train query + for qinfo in train_queries: + r = train_by_query.get(qinfo["query"], {}) + did_pass = r.get("pass", False) + triggers = r.get("triggers", 0) + runs = r.get("runs", 0) + + icon = "✓" if did_pass else "✗" + css_class = "pass" if did_pass else "fail" + + html_parts.append(f' \n') + + # Add result for each test query (with different background) + for qinfo in test_queries: + r = test_by_query.get(qinfo["query"], {}) + did_pass = r.get("pass", False) + triggers = r.get("triggers", 0) + runs = r.get("runs", 0) + + icon = "✓" if did_pass else "✗" + css_class = "pass" if did_pass else "fail" + + html_parts.append(f' \n') + + html_parts.append(" \n") + + html_parts.append(""" +
IterTrainTestDescription{html.escape(qinfo["query"])}{html.escape(qinfo["query"])}
{iteration}{train_correct}/{train_runs}{test_correct}/{test_runs}{html.escape(description)}{icon}{triggers}/{runs}{icon}{triggers}/{runs}
+
+""") + + html_parts.append(""" + + +""") + + return "".join(html_parts) + + +def main(): + parser = argparse.ArgumentParser(description="Generate HTML report from run_loop output") + parser.add_argument("input", help="Path to JSON output from run_loop.py (or - for stdin)") + parser.add_argument("-o", "--output", default=None, help="Output HTML file (default: stdout)") + parser.add_argument("--skill-name", default="", help="Skill name to include in the report title") + args = parser.parse_args() + + if args.input == "-": + data = json.load(sys.stdin) + else: + data = json.loads(Path(args.input).read_text()) + + html_output = generate_html(data, skill_name=args.skill_name) + + if args.output: + Path(args.output).write_text(html_output) + print(f"Report written to {args.output}", file=sys.stderr) + else: + print(html_output) + + +if __name__ == "__main__": + main() diff --git a/skill-creator/scripts/improve_description.py b/skill-creator/scripts/improve_description.py new file mode 100755 index 00000000..06bcec76 --- /dev/null +++ b/skill-creator/scripts/improve_description.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +"""Improve a skill description based on eval results. + +Takes eval results (from run_eval.py) and generates an improved description +by calling `claude -p` as a subprocess (same auth pattern as run_eval.py — +uses the session's Claude Code auth, no separate ANTHROPIC_API_KEY needed). +""" + +import argparse +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +from scripts.utils import parse_skill_md + + +def _call_claude(prompt: str, model: str | None, timeout: int = 300) -> str: + """Run `claude -p` with the prompt on stdin and return the text response. + + Prompt goes over stdin (not argv) because it embeds the full SKILL.md + body and can easily exceed comfortable argv length. + """ + cmd = ["claude", "-p", "--output-format", "text"] + if model: + cmd.extend(["--model", model]) + + # Remove CLAUDECODE env var to allow nesting claude -p inside a + # Claude Code session. The guard is for interactive terminal conflicts; + # programmatic subprocess usage is safe. Same pattern as run_eval.py. + env = {k: v for k, v in os.environ.items() if k != "CLAUDECODE"} + + result = subprocess.run( + cmd, + input=prompt, + capture_output=True, + text=True, + env=env, + timeout=timeout, + ) + if result.returncode != 0: + raise RuntimeError( + f"claude -p exited {result.returncode}\nstderr: {result.stderr}" + ) + return result.stdout + + +def improve_description( + skill_name: str, + skill_content: str, + current_description: str, + eval_results: dict, + history: list[dict], + model: str, + test_results: dict | None = None, + log_dir: Path | None = None, + iteration: int | None = None, +) -> str: + """Call Claude to improve the description based on eval results.""" + failed_triggers = [ + r for r in eval_results["results"] + if r["should_trigger"] and not r["pass"] + ] + false_triggers = [ + r for r in eval_results["results"] + if not r["should_trigger"] and not r["pass"] + ] + + # Build scores summary + train_score = f"{eval_results['summary']['passed']}/{eval_results['summary']['total']}" + if test_results: + test_score = f"{test_results['summary']['passed']}/{test_results['summary']['total']}" + scores_summary = f"Train: {train_score}, Test: {test_score}" + else: + scores_summary = f"Train: {train_score}" + + prompt = f"""You are optimizing a skill description for a Claude Code skill called "{skill_name}". A "skill" is sort of like a prompt, but with progressive disclosure -- there's a title and description that Claude sees when deciding whether to use the skill, and then if it does use the skill, it reads the .md file which has lots more details and potentially links to other resources in the skill folder like helper files and scripts and additional documentation or examples. + +The description appears in Claude's "available_skills" list. When a user sends a query, Claude decides whether to invoke the skill based solely on the title and on this description. Your goal is to write a description that triggers for relevant queries, and doesn't trigger for irrelevant ones. + +Here's the current description: + +"{current_description}" + + +Current scores ({scores_summary}): + +""" + if failed_triggers: + prompt += "FAILED TO TRIGGER (should have triggered but didn't):\n" + for r in failed_triggers: + prompt += f' - "{r["query"]}" (triggered {r["triggers"]}/{r["runs"]} times)\n' + prompt += "\n" + + if false_triggers: + prompt += "FALSE TRIGGERS (triggered but shouldn't have):\n" + for r in false_triggers: + prompt += f' - "{r["query"]}" (triggered {r["triggers"]}/{r["runs"]} times)\n' + prompt += "\n" + + if history: + prompt += "PREVIOUS ATTEMPTS (do NOT repeat these — try something structurally different):\n\n" + for h in history: + train_s = f"{h.get('train_passed', h.get('passed', 0))}/{h.get('train_total', h.get('total', 0))}" + test_s = f"{h.get('test_passed', '?')}/{h.get('test_total', '?')}" if h.get('test_passed') is not None else None + score_str = f"train={train_s}" + (f", test={test_s}" if test_s else "") + prompt += f'\n' + prompt += f'Description: "{h["description"]}"\n' + if "results" in h: + prompt += "Train results:\n" + for r in h["results"]: + status = "PASS" if r["pass"] else "FAIL" + prompt += f' [{status}] "{r["query"][:80]}" (triggered {r["triggers"]}/{r["runs"]})\n' + if h.get("note"): + prompt += f'Note: {h["note"]}\n' + prompt += "\n\n" + + prompt += f""" + +Skill content (for context on what the skill does): + +{skill_content} + + +Based on the failures, write a new and improved description that is more likely to trigger correctly. When I say "based on the failures", it's a bit of a tricky line to walk because we don't want to overfit to the specific cases you're seeing. So what I DON'T want you to do is produce an ever-expanding list of specific queries that this skill should or shouldn't trigger for. Instead, try to generalize from the failures to broader categories of user intent and situations where this skill would be useful or not useful. The reason for this is twofold: + +1. Avoid overfitting +2. The list might get loooong and it's injected into ALL queries and there might be a lot of skills, so we don't want to blow too much space on any given description. + +Concretely, your description should not be more than about 100-200 words, even if that comes at the cost of accuracy. There is a hard limit of 1024 characters — descriptions over that will be truncated, so stay comfortably under it. + +Here are some tips that we've found to work well in writing these descriptions: +- The skill should be phrased in the imperative -- "Use this skill for" rather than "this skill does" +- The skill description should focus on the user's intent, what they are trying to achieve, vs. the implementation details of how the skill works. +- The description competes with other skills for Claude's attention — make it distinctive and immediately recognizable. +- If you're getting lots of failures after repeated attempts, change things up. Try different sentence structures or wordings. + +I'd encourage you to be creative and mix up the style in different iterations since you'll have multiple opportunities to try different approaches and we'll just grab the highest-scoring one at the end. + +Please respond with only the new description text in tags, nothing else.""" + + text = _call_claude(prompt, model) + + match = re.search(r"(.*?)", text, re.DOTALL) + description = match.group(1).strip().strip('"') if match else text.strip().strip('"') + + transcript: dict = { + "iteration": iteration, + "prompt": prompt, + "response": text, + "parsed_description": description, + "char_count": len(description), + "over_limit": len(description) > 1024, + } + + # Safety net: the prompt already states the 1024-char hard limit, but if + # the model blew past it anyway, make one fresh single-turn call that + # quotes the too-long version and asks for a shorter rewrite. (The old + # SDK path did this as a true multi-turn; `claude -p` is one-shot, so we + # inline the prior output into the new prompt instead.) + if len(description) > 1024: + shorten_prompt = ( + f"{prompt}\n\n" + f"---\n\n" + f"A previous attempt produced this description, which at " + f"{len(description)} characters is over the 1024-character hard limit:\n\n" + f'"{description}"\n\n' + f"Rewrite it to be under 1024 characters while keeping the most " + f"important trigger words and intent coverage. Respond with only " + f"the new description in tags." + ) + shorten_text = _call_claude(shorten_prompt, model) + match = re.search(r"(.*?)", shorten_text, re.DOTALL) + shortened = match.group(1).strip().strip('"') if match else shorten_text.strip().strip('"') + + transcript["rewrite_prompt"] = shorten_prompt + transcript["rewrite_response"] = shorten_text + transcript["rewrite_description"] = shortened + transcript["rewrite_char_count"] = len(shortened) + description = shortened + + transcript["final_description"] = description + + if log_dir: + log_dir.mkdir(parents=True, exist_ok=True) + log_file = log_dir / f"improve_iter_{iteration or 'unknown'}.json" + log_file.write_text(json.dumps(transcript, indent=2)) + + return description + + +def main(): + parser = argparse.ArgumentParser(description="Improve a skill description based on eval results") + parser.add_argument("--eval-results", required=True, help="Path to eval results JSON (from run_eval.py)") + parser.add_argument("--skill-path", required=True, help="Path to skill directory") + parser.add_argument("--history", default=None, help="Path to history JSON (previous attempts)") + parser.add_argument("--model", required=True, help="Model for improvement") + parser.add_argument("--verbose", action="store_true", help="Print thinking to stderr") + args = parser.parse_args() + + skill_path = Path(args.skill_path) + if not (skill_path / "SKILL.md").exists(): + print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr) + sys.exit(1) + + eval_results = json.loads(Path(args.eval_results).read_text()) + history = [] + if args.history: + history = json.loads(Path(args.history).read_text()) + + name, _, content = parse_skill_md(skill_path) + current_description = eval_results["description"] + + if args.verbose: + print(f"Current: {current_description}", file=sys.stderr) + print(f"Score: {eval_results['summary']['passed']}/{eval_results['summary']['total']}", file=sys.stderr) + + new_description = improve_description( + skill_name=name, + skill_content=content, + current_description=current_description, + eval_results=eval_results, + history=history, + model=args.model, + ) + + if args.verbose: + print(f"Improved: {new_description}", file=sys.stderr) + + # Output as JSON with both the new description and updated history + output = { + "description": new_description, + "history": history + [{ + "description": current_description, + "passed": eval_results["summary"]["passed"], + "failed": eval_results["summary"]["failed"], + "total": eval_results["summary"]["total"], + "results": eval_results["results"], + }], + } + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/skill-creator/scripts/package_skill.py b/skill-creator/scripts/package_skill.py index eb665363..bd10fe28 100755 --- a/skill-creator/scripts/package_skill.py +++ b/skill-creator/scripts/package_skill.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ -Skill Packager - Creates a distributable zip file of a skill folder +Skill Packager - Creates a distributable .skill file of a skill folder Usage: python utils/package_skill.py [output-directory] @@ -10,15 +10,45 @@ python utils/package_skill.py skills/public/my-skill ./dist """ +import fnmatch +import re import sys import zipfile -import re from pathlib import Path -from quick_validate import validate_skill -from security_scan import calculate_skill_hash - - -def validate_security_marker(skill_path: Path) -> tuple[bool, str]: +from typing import Optional, Tuple + +SCRIPT_DIR = Path(__file__).resolve().parent +PACKAGE_ROOT = SCRIPT_DIR.parent +if str(PACKAGE_ROOT) not in sys.path: + sys.path.insert(0, str(PACKAGE_ROOT)) + +from scripts.quick_validate import validate_skill +from scripts.security_scan import calculate_skill_hash + +# Patterns to exclude when packaging skills. +EXCLUDE_DIRS = {"__pycache__", "node_modules"} +EXCLUDE_GLOBS = {"*.pyc"} +EXCLUDE_FILES = {".DS_Store"} +# Directories excluded only at the skill root (not when nested deeper). +ROOT_EXCLUDE_DIRS = {"evals"} + + +def should_exclude(rel_path: Path) -> bool: + """Check if a path should be excluded from packaging.""" + parts = rel_path.parts + if any(part in EXCLUDE_DIRS for part in parts): + return True + # rel_path is relative to skill_path.parent, so parts[0] is the skill + # folder name and parts[1] (if present) is the first subdir. + if len(parts) > 1 and parts[1] in ROOT_EXCLUDE_DIRS: + return True + name = rel_path.name + if name in EXCLUDE_FILES: + return True + return any(fnmatch.fnmatch(name, pat) for pat in EXCLUDE_GLOBS) + + +def validate_security_marker(skill_path: Path) -> Tuple[bool, str]: """ Validate security marker file exists and hash matches current content @@ -58,54 +88,54 @@ def validate_security_marker(skill_path: Path) -> tuple[bool, str]: def package_skill(skill_path, output_dir=None): """ - Package a skill folder into a zip file. + Package a skill folder into a .skill file. Args: skill_path: Path to the skill folder - output_dir: Optional output directory for the zip file (defaults to current directory) + output_dir: Optional output directory for the .skill file (defaults to current directory) Returns: - Path to the created zip file, or None if error + Path to the created .skill file, or None if error """ skill_path = Path(skill_path).resolve() # Validate skill folder exists if not skill_path.exists(): - print(f"❌ Error: Skill folder not found: {skill_path}") + print(f"Error: Skill folder not found: {skill_path}") return None if not skill_path.is_dir(): - print(f"❌ Error: Path is not a directory: {skill_path}") + print(f"Error: Path is not a directory: {skill_path}") return None # Validate SKILL.md exists skill_md = skill_path / "SKILL.md" if not skill_md.exists(): - print(f"❌ Error: SKILL.md not found in {skill_path}") + print(f"Error: SKILL.md not found in {skill_path}") return None # Step 1: Validate skill structure and metadata - print("🔍 Step 1: Validating skill structure...") + print("Step 1: Validating skill structure...") valid, message = validate_skill(skill_path) if not valid: - print(f"❌ FAILED: {message}") + print(f"FAILED: {message}") print(" Fix validation errors before packaging.") return None - print(f"✅ PASSED: {message}\n") + print(f"PASSED: {message}\n") # Step 2: Validate security scan (HARD REQUIREMENT) - print("🔍 Step 2: Validating security scan...") + print("Step 2: Validating security scan...") is_valid, message = validate_security_marker(skill_path) if not is_valid: - print(f"❌ BLOCKED: {message}") + print(f"BLOCKED: {message}") print(f" You MUST run: python scripts/security_scan.py {skill_path.name}") print(" Security review is MANDATORY before packaging.") return None - print(f"✅ PASSED: {message}\n") + print(f"PASSED: {message}\n") # Step 3: Package the skill - print("📦 Step 3: Creating package...") + print("Step 3: Creating package...") # Determine output location skill_name = skill_path.name @@ -115,24 +145,27 @@ def package_skill(skill_path, output_dir=None): else: output_path = Path.cwd() - zip_filename = output_path / f"{skill_name}.zip" + skill_filename = output_path / f"{skill_name}.skill" - # Create the zip file + # Create the .skill file (zip format) try: - with zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) as zipf: - # Walk through the skill directory + with zipfile.ZipFile(skill_filename, 'w', zipfile.ZIP_DEFLATED) as zipf: + # Walk through the skill directory, excluding build artifacts for file_path in skill_path.rglob('*'): - if file_path.is_file(): - # Calculate the relative path within the zip - arcname = file_path.relative_to(skill_path.parent) - zipf.write(file_path, arcname) - print(f" Added: {arcname}") - - print(f"\n✅ Successfully packaged skill to: {zip_filename}") - return zip_filename + if not file_path.is_file(): + continue + arcname = file_path.relative_to(skill_path.parent) + if should_exclude(arcname): + print(f" Skipped: {arcname}") + continue + zipf.write(file_path, arcname) + print(f" Added: {arcname}") + + print(f"\nSuccessfully packaged skill to: {skill_filename}") + return skill_filename except Exception as e: - print(f"❌ Error creating zip file: {e}") + print(f"Error creating .skill file: {e}") return None @@ -147,7 +180,7 @@ def main(): skill_path = sys.argv[1] output_dir = sys.argv[2] if len(sys.argv) > 2 else None - print(f"📦 Packaging skill: {skill_path}") + print(f"Packaging skill: {skill_path}") if output_dir: print(f" Output directory: {output_dir}") print() diff --git a/skill-creator/scripts/quick_validate.py b/skill-creator/scripts/quick_validate.py index 2d022000..78eac04b 100755 --- a/skill-creator/scripts/quick_validate.py +++ b/skill-creator/scripts/quick_validate.py @@ -6,6 +6,7 @@ import sys import os import re +import yaml from pathlib import Path @@ -57,7 +58,7 @@ def find_path_references(content: str) -> list[str]: if any(x in line_lower for x in [ 'example:', 'examples:', 'e.g.', 'for example', '- **example', '- example:', 'such as', - 'pattern:', 'usage:', '❌', '✅', + 'pattern:', 'usage:', '\u274c', '\u2705', '- **allowed', '- **best practice', 'would be helpful', 'like `scripts/', 'like `references/', 'like `assets/', ]): @@ -92,6 +93,14 @@ def validate_path_references(skill_path: Path, content: str) -> tuple[bool, list return len(missing) == 0, missing +# Define allowed properties (union of official and our extensions) +ALLOWED_PROPERTIES = { + 'name', 'description', 'license', 'allowed-tools', 'metadata', + 'compatibility', 'context', 'agent', 'disable-model-invocation', + 'user-invocable', 'model', 'argument-hint', 'hooks', +} + + def validate_skill(skill_path): """Basic validation of a skill""" skill_path = Path(skill_path) @@ -111,10 +120,10 @@ def validate_skill(skill_path): if not match: return False, "Invalid frontmatter format" - frontmatter = match.group(1) + frontmatter_text = match.group(1) # Check for invalid indentation characters in frontmatter - invalid_indent = find_invalid_frontmatter_indentation(frontmatter) + invalid_indent = find_invalid_frontmatter_indentation(frontmatter_text) if invalid_indent: samples = ", ".join( f"line {line_no} ({describe_whitespace(ch)})" @@ -126,29 +135,62 @@ def validate_skill(skill_path): f"Found: {samples}{more}" ) + # Parse YAML frontmatter + try: + frontmatter = yaml.safe_load(frontmatter_text) + if not isinstance(frontmatter, dict): + return False, "Frontmatter must be a YAML dictionary" + except yaml.YAMLError as e: + return False, f"Invalid YAML in frontmatter: {e}" + + # Check for unexpected properties + unexpected_keys = set(frontmatter.keys()) - ALLOWED_PROPERTIES + if unexpected_keys: + return False, ( + f"Unexpected key(s) in SKILL.md frontmatter: {', '.join(sorted(unexpected_keys))}. " + f"Allowed properties are: {', '.join(sorted(ALLOWED_PROPERTIES))}" + ) + # Check required fields - if 'name:' not in frontmatter: - return False, "Missing 'name' in frontmatter" - if 'description:' not in frontmatter: + if 'description' not in frontmatter: return False, "Missing 'description' in frontmatter" - # Extract name for validation - name_match = re.search(r'name:\s*(.+)', frontmatter) - if name_match: - name = name_match.group(1).strip() - # Check naming convention (hyphen-case: lowercase with hyphens) - if not re.match(r'^[a-z0-9-]+$', name): - return False, f"Name '{name}' should be hyphen-case (lowercase letters, digits, and hyphens only)" - if name.startswith('-') or name.endswith('-') or '--' in name: - return False, f"Name '{name}' cannot start/end with hyphen or contain consecutive hyphens" + # Extract name for validation (optional per official spec, but validate if present) + name = frontmatter.get('name', '') + if isinstance(name, str): + name = name.strip() + if name: + # Check naming convention (kebab-case: lowercase with hyphens) + if not re.match(r'^[a-z0-9-]+$', name): + return False, f"Name '{name}' should be kebab-case (lowercase letters, digits, and hyphens only)" + if name.startswith('-') or name.endswith('-') or '--' in name: + return False, f"Name '{name}' cannot start/end with hyphen or contain consecutive hyphens" + # Check name length (max 64 characters per spec) + if len(name) > 64: + return False, f"Name is too long ({len(name)} characters). Maximum is 64 characters." + elif name is not None: + return False, f"Name must be a string, got {type(name).__name__}" # Extract and validate description - desc_match = re.search(r'description:\s*(.+)', frontmatter) - if desc_match: - description = desc_match.group(1).strip() + description = frontmatter.get('description', '') + if not isinstance(description, str): + return False, f"Description must be a string, got {type(description).__name__}" + description = description.strip() + if description: # Check for angle brackets if '<' in description or '>' in description: return False, "Description cannot contain angle brackets (< or >)" + # Check description length (max 1024 characters per spec) + if len(description) > 1024: + return False, f"Description is too long ({len(description)} characters). Maximum is 1024 characters." + + # Validate compatibility field if present (optional) + compatibility = frontmatter.get('compatibility', '') + if compatibility: + if not isinstance(compatibility, str): + return False, f"Compatibility must be a string, got {type(compatibility).__name__}" + if len(compatibility) > 500: + return False, f"Compatibility is too long ({len(compatibility)} characters). Maximum is 500 characters." # Validate path references exist paths_valid, missing_paths = validate_path_references(skill_path, content) @@ -161,7 +203,7 @@ def validate_skill(skill_path): if len(sys.argv) != 2: print("Usage: python quick_validate.py ") sys.exit(1) - + valid, message = validate_skill(sys.argv[1]) print(message) sys.exit(0 if valid else 1) diff --git a/skill-creator/scripts/run_eval.py b/skill-creator/scripts/run_eval.py new file mode 100755 index 00000000..e58c70be --- /dev/null +++ b/skill-creator/scripts/run_eval.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +"""Run trigger evaluation for a skill description. + +Tests whether a skill's description causes Claude to trigger (read the skill) +for a set of queries. Outputs results as JSON. +""" + +import argparse +import json +import os +import select +import subprocess +import sys +import time +import uuid +from concurrent.futures import ProcessPoolExecutor, as_completed +from pathlib import Path + +from scripts.utils import parse_skill_md + + +def find_project_root() -> Path: + """Find the project root by walking up from cwd looking for .claude/. + + Mimics how Claude Code discovers its project root, so the command file + we create ends up where claude -p will look for it. + """ + current = Path.cwd() + for parent in [current, *current.parents]: + if (parent / ".claude").is_dir(): + return parent + return current + + +def run_single_query( + query: str, + skill_name: str, + skill_description: str, + timeout: int, + project_root: str, + model: str | None = None, +) -> bool: + """Run a single query and return whether the skill was triggered. + + Creates a command file in .claude/commands/ so it appears in Claude's + available_skills list, then runs `claude -p` with the raw query. + Uses --include-partial-messages to detect triggering early from + stream events (content_block_start) rather than waiting for the + full assistant message, which only arrives after tool execution. + """ + unique_id = uuid.uuid4().hex[:8] + clean_name = f"{skill_name}-skill-{unique_id}" + project_commands_dir = Path(project_root) / ".claude" / "commands" + command_file = project_commands_dir / f"{clean_name}.md" + + try: + project_commands_dir.mkdir(parents=True, exist_ok=True) + # Use YAML block scalar to avoid breaking on quotes in description + indented_desc = "\n ".join(skill_description.split("\n")) + command_content = ( + f"---\n" + f"description: |\n" + f" {indented_desc}\n" + f"---\n\n" + f"# {skill_name}\n\n" + f"This skill handles: {skill_description}\n" + ) + command_file.write_text(command_content) + + cmd = [ + "claude", + "-p", query, + "--output-format", "stream-json", + "--verbose", + "--include-partial-messages", + ] + if model: + cmd.extend(["--model", model]) + + # Remove CLAUDECODE env var to allow nesting claude -p inside a + # Claude Code session. The guard is for interactive terminal conflicts; + # programmatic subprocess usage is safe. + env = {k: v for k, v in os.environ.items() if k != "CLAUDECODE"} + + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + cwd=project_root, + env=env, + ) + + triggered = False + start_time = time.time() + buffer = "" + # Track state for stream event detection + pending_tool_name = None + accumulated_json = "" + + try: + while time.time() - start_time < timeout: + if process.poll() is not None: + remaining = process.stdout.read() + if remaining: + buffer += remaining.decode("utf-8", errors="replace") + break + + ready, _, _ = select.select([process.stdout], [], [], 1.0) + if not ready: + continue + + chunk = os.read(process.stdout.fileno(), 8192) + if not chunk: + break + buffer += chunk.decode("utf-8", errors="replace") + + while "\n" in buffer: + line, buffer = buffer.split("\n", 1) + line = line.strip() + if not line: + continue + + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + + # Early detection via stream events + if event.get("type") == "stream_event": + se = event.get("event", {}) + se_type = se.get("type", "") + + if se_type == "content_block_start": + cb = se.get("content_block", {}) + if cb.get("type") == "tool_use": + tool_name = cb.get("name", "") + if tool_name in ("Skill", "Read"): + pending_tool_name = tool_name + accumulated_json = "" + else: + return False + + elif se_type == "content_block_delta" and pending_tool_name: + delta = se.get("delta", {}) + if delta.get("type") == "input_json_delta": + accumulated_json += delta.get("partial_json", "") + if clean_name in accumulated_json: + return True + + elif se_type in ("content_block_stop", "message_stop"): + if pending_tool_name: + return clean_name in accumulated_json + if se_type == "message_stop": + return False + + # Fallback: full assistant message + elif event.get("type") == "assistant": + message = event.get("message", {}) + for content_item in message.get("content", []): + if content_item.get("type") != "tool_use": + continue + tool_name = content_item.get("name", "") + tool_input = content_item.get("input", {}) + if tool_name == "Skill" and clean_name in tool_input.get("skill", ""): + triggered = True + elif tool_name == "Read" and clean_name in tool_input.get("file_path", ""): + triggered = True + return triggered + + elif event.get("type") == "result": + return triggered + finally: + # Clean up process on any exit path (return, exception, timeout) + if process.poll() is None: + process.kill() + process.wait() + + return triggered + finally: + if command_file.exists(): + command_file.unlink() + + +def run_eval( + eval_set: list[dict], + skill_name: str, + description: str, + num_workers: int, + timeout: int, + project_root: Path, + runs_per_query: int = 1, + trigger_threshold: float = 0.5, + model: str | None = None, +) -> dict: + """Run the full eval set and return results.""" + results = [] + + with ProcessPoolExecutor(max_workers=num_workers) as executor: + future_to_info = {} + for item in eval_set: + for run_idx in range(runs_per_query): + future = executor.submit( + run_single_query, + item["query"], + skill_name, + description, + timeout, + str(project_root), + model, + ) + future_to_info[future] = (item, run_idx) + + query_triggers: dict[str, list[bool]] = {} + query_items: dict[str, dict] = {} + for future in as_completed(future_to_info): + item, _ = future_to_info[future] + query = item["query"] + query_items[query] = item + if query not in query_triggers: + query_triggers[query] = [] + try: + query_triggers[query].append(future.result()) + except Exception as e: + print(f"Warning: query failed: {e}", file=sys.stderr) + query_triggers[query].append(False) + + for query, triggers in query_triggers.items(): + item = query_items[query] + trigger_rate = sum(triggers) / len(triggers) + should_trigger = item["should_trigger"] + if should_trigger: + did_pass = trigger_rate >= trigger_threshold + else: + did_pass = trigger_rate < trigger_threshold + results.append({ + "query": query, + "should_trigger": should_trigger, + "trigger_rate": trigger_rate, + "triggers": sum(triggers), + "runs": len(triggers), + "pass": did_pass, + }) + + passed = sum(1 for r in results if r["pass"]) + total = len(results) + + return { + "skill_name": skill_name, + "description": description, + "results": results, + "summary": { + "total": total, + "passed": passed, + "failed": total - passed, + }, + } + + +def main(): + parser = argparse.ArgumentParser(description="Run trigger evaluation for a skill description") + parser.add_argument("--eval-set", required=True, help="Path to eval set JSON file") + parser.add_argument("--skill-path", required=True, help="Path to skill directory") + parser.add_argument("--description", default=None, help="Override description to test") + parser.add_argument("--num-workers", type=int, default=10, help="Number of parallel workers") + parser.add_argument("--timeout", type=int, default=30, help="Timeout per query in seconds") + parser.add_argument("--runs-per-query", type=int, default=3, help="Number of runs per query") + parser.add_argument("--trigger-threshold", type=float, default=0.5, help="Trigger rate threshold") + parser.add_argument("--model", default=None, help="Model to use for claude -p (default: user's configured model)") + parser.add_argument("--verbose", action="store_true", help="Print progress to stderr") + args = parser.parse_args() + + eval_set = json.loads(Path(args.eval_set).read_text()) + skill_path = Path(args.skill_path) + + if not (skill_path / "SKILL.md").exists(): + print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr) + sys.exit(1) + + name, original_description, content = parse_skill_md(skill_path) + description = args.description or original_description + project_root = find_project_root() + + if args.verbose: + print(f"Evaluating: {description}", file=sys.stderr) + + output = run_eval( + eval_set=eval_set, + skill_name=name, + description=description, + num_workers=args.num_workers, + timeout=args.timeout, + project_root=project_root, + runs_per_query=args.runs_per_query, + trigger_threshold=args.trigger_threshold, + model=args.model, + ) + + if args.verbose: + summary = output["summary"] + print(f"Results: {summary['passed']}/{summary['total']} passed", file=sys.stderr) + for r in output["results"]: + status = "PASS" if r["pass"] else "FAIL" + rate_str = f"{r['triggers']}/{r['runs']}" + print(f" [{status}] rate={rate_str} expected={r['should_trigger']}: {r['query'][:70]}", file=sys.stderr) + + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/skill-creator/scripts/run_loop.py b/skill-creator/scripts/run_loop.py new file mode 100755 index 00000000..30a263d6 --- /dev/null +++ b/skill-creator/scripts/run_loop.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +"""Run the eval + improve loop until all pass or max iterations reached. + +Combines run_eval.py and improve_description.py in a loop, tracking history +and returning the best description found. Supports train/test split to prevent +overfitting. +""" + +import argparse +import json +import random +import sys +import tempfile +import time +import webbrowser +from pathlib import Path + +from scripts.generate_report import generate_html +from scripts.improve_description import improve_description +from scripts.run_eval import find_project_root, run_eval +from scripts.utils import parse_skill_md + + +def split_eval_set(eval_set: list[dict], holdout: float, seed: int = 42) -> tuple[list[dict], list[dict]]: + """Split eval set into train and test sets, stratified by should_trigger.""" + random.seed(seed) + + # Separate by should_trigger + trigger = [e for e in eval_set if e["should_trigger"]] + no_trigger = [e for e in eval_set if not e["should_trigger"]] + + # Shuffle each group + random.shuffle(trigger) + random.shuffle(no_trigger) + + # Calculate split points + n_trigger_test = max(1, int(len(trigger) * holdout)) + n_no_trigger_test = max(1, int(len(no_trigger) * holdout)) + + # Split + test_set = trigger[:n_trigger_test] + no_trigger[:n_no_trigger_test] + train_set = trigger[n_trigger_test:] + no_trigger[n_no_trigger_test:] + + return train_set, test_set + + +def run_loop( + eval_set: list[dict], + skill_path: Path, + description_override: str | None, + num_workers: int, + timeout: int, + max_iterations: int, + runs_per_query: int, + trigger_threshold: float, + holdout: float, + model: str, + verbose: bool, + live_report_path: Path | None = None, + log_dir: Path | None = None, +) -> dict: + """Run the eval + improvement loop.""" + project_root = find_project_root() + name, original_description, content = parse_skill_md(skill_path) + current_description = description_override or original_description + + # Split into train/test if holdout > 0 + if holdout > 0: + train_set, test_set = split_eval_set(eval_set, holdout) + if verbose: + print(f"Split: {len(train_set)} train, {len(test_set)} test (holdout={holdout})", file=sys.stderr) + else: + train_set = eval_set + test_set = [] + + history = [] + exit_reason = "unknown" + + for iteration in range(1, max_iterations + 1): + if verbose: + print(f"\n{'='*60}", file=sys.stderr) + print(f"Iteration {iteration}/{max_iterations}", file=sys.stderr) + print(f"Description: {current_description}", file=sys.stderr) + print(f"{'='*60}", file=sys.stderr) + + # Evaluate train + test together in one batch for parallelism + all_queries = train_set + test_set + t0 = time.time() + all_results = run_eval( + eval_set=all_queries, + skill_name=name, + description=current_description, + num_workers=num_workers, + timeout=timeout, + project_root=project_root, + runs_per_query=runs_per_query, + trigger_threshold=trigger_threshold, + model=model, + ) + eval_elapsed = time.time() - t0 + + # Split results back into train/test by matching queries + train_queries_set = {q["query"] for q in train_set} + train_result_list = [r for r in all_results["results"] if r["query"] in train_queries_set] + test_result_list = [r for r in all_results["results"] if r["query"] not in train_queries_set] + + train_passed = sum(1 for r in train_result_list if r["pass"]) + train_total = len(train_result_list) + train_summary = {"passed": train_passed, "failed": train_total - train_passed, "total": train_total} + train_results = {"results": train_result_list, "summary": train_summary} + + if test_set: + test_passed = sum(1 for r in test_result_list if r["pass"]) + test_total = len(test_result_list) + test_summary = {"passed": test_passed, "failed": test_total - test_passed, "total": test_total} + test_results = {"results": test_result_list, "summary": test_summary} + else: + test_results = None + test_summary = None + + history.append({ + "iteration": iteration, + "description": current_description, + "train_passed": train_summary["passed"], + "train_failed": train_summary["failed"], + "train_total": train_summary["total"], + "train_results": train_results["results"], + "test_passed": test_summary["passed"] if test_summary else None, + "test_failed": test_summary["failed"] if test_summary else None, + "test_total": test_summary["total"] if test_summary else None, + "test_results": test_results["results"] if test_results else None, + # For backward compat with report generator + "passed": train_summary["passed"], + "failed": train_summary["failed"], + "total": train_summary["total"], + "results": train_results["results"], + }) + + # Write live report if path provided + if live_report_path: + partial_output = { + "original_description": original_description, + "best_description": current_description, + "best_score": "in progress", + "iterations_run": len(history), + "holdout": holdout, + "train_size": len(train_set), + "test_size": len(test_set), + "history": history, + } + live_report_path.write_text(generate_html(partial_output, auto_refresh=True, skill_name=name)) + + if verbose: + def print_eval_stats(label, results, elapsed): + pos = [r for r in results if r["should_trigger"]] + neg = [r for r in results if not r["should_trigger"]] + tp = sum(r["triggers"] for r in pos) + pos_runs = sum(r["runs"] for r in pos) + fn = pos_runs - tp + fp = sum(r["triggers"] for r in neg) + neg_runs = sum(r["runs"] for r in neg) + tn = neg_runs - fp + total = tp + tn + fp + fn + precision = tp / (tp + fp) if (tp + fp) > 0 else 1.0 + recall = tp / (tp + fn) if (tp + fn) > 0 else 1.0 + accuracy = (tp + tn) / total if total > 0 else 0.0 + print(f"{label}: {tp+tn}/{total} correct, precision={precision:.0%} recall={recall:.0%} accuracy={accuracy:.0%} ({elapsed:.1f}s)", file=sys.stderr) + for r in results: + status = "PASS" if r["pass"] else "FAIL" + rate_str = f"{r['triggers']}/{r['runs']}" + print(f" [{status}] rate={rate_str} expected={r['should_trigger']}: {r['query'][:60]}", file=sys.stderr) + + print_eval_stats("Train", train_results["results"], eval_elapsed) + if test_summary: + print_eval_stats("Test ", test_results["results"], 0) + + if train_summary["failed"] == 0: + exit_reason = f"all_passed (iteration {iteration})" + if verbose: + print(f"\nAll train queries passed on iteration {iteration}!", file=sys.stderr) + break + + if iteration == max_iterations: + exit_reason = f"max_iterations ({max_iterations})" + if verbose: + print(f"\nMax iterations reached ({max_iterations}).", file=sys.stderr) + break + + # Improve the description based on train results + if verbose: + print(f"\nImproving description...", file=sys.stderr) + + t0 = time.time() + # Strip test scores from history so improvement model can't see them + blinded_history = [ + {k: v for k, v in h.items() if not k.startswith("test_")} + for h in history + ] + new_description = improve_description( + skill_name=name, + skill_content=content, + current_description=current_description, + eval_results=train_results, + history=blinded_history, + model=model, + log_dir=log_dir, + iteration=iteration, + ) + improve_elapsed = time.time() - t0 + + if verbose: + print(f"Proposed ({improve_elapsed:.1f}s): {new_description}", file=sys.stderr) + + current_description = new_description + + # Find the best iteration by TEST score (or train if no test set) + if test_set: + best = max(history, key=lambda h: h["test_passed"] or 0) + best_score = f"{best['test_passed']}/{best['test_total']}" + else: + best = max(history, key=lambda h: h["train_passed"]) + best_score = f"{best['train_passed']}/{best['train_total']}" + + if verbose: + print(f"\nExit reason: {exit_reason}", file=sys.stderr) + print(f"Best score: {best_score} (iteration {best['iteration']})", file=sys.stderr) + + return { + "exit_reason": exit_reason, + "original_description": original_description, + "best_description": best["description"], + "best_score": best_score, + "best_train_score": f"{best['train_passed']}/{best['train_total']}", + "best_test_score": f"{best['test_passed']}/{best['test_total']}" if test_set else None, + "final_description": current_description, + "iterations_run": len(history), + "holdout": holdout, + "train_size": len(train_set), + "test_size": len(test_set), + "history": history, + } + + +def main(): + parser = argparse.ArgumentParser(description="Run eval + improve loop") + parser.add_argument("--eval-set", required=True, help="Path to eval set JSON file") + parser.add_argument("--skill-path", required=True, help="Path to skill directory") + parser.add_argument("--description", default=None, help="Override starting description") + parser.add_argument("--num-workers", type=int, default=10, help="Number of parallel workers") + parser.add_argument("--timeout", type=int, default=30, help="Timeout per query in seconds") + parser.add_argument("--max-iterations", type=int, default=5, help="Max improvement iterations") + parser.add_argument("--runs-per-query", type=int, default=3, help="Number of runs per query") + parser.add_argument("--trigger-threshold", type=float, default=0.5, help="Trigger rate threshold") + parser.add_argument("--holdout", type=float, default=0.4, help="Fraction of eval set to hold out for testing (0 to disable)") + parser.add_argument("--model", required=True, help="Model for improvement") + parser.add_argument("--verbose", action="store_true", help="Print progress to stderr") + parser.add_argument("--report", default="auto", help="Generate HTML report at this path (default: 'auto' for temp file, 'none' to disable)") + parser.add_argument("--results-dir", default=None, help="Save all outputs (results.json, report.html, log.txt) to a timestamped subdirectory here") + args = parser.parse_args() + + eval_set = json.loads(Path(args.eval_set).read_text()) + skill_path = Path(args.skill_path) + + if not (skill_path / "SKILL.md").exists(): + print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr) + sys.exit(1) + + name, _, _ = parse_skill_md(skill_path) + + # Set up live report path + if args.report != "none": + if args.report == "auto": + timestamp = time.strftime("%Y%m%d_%H%M%S") + live_report_path = Path(tempfile.gettempdir()) / f"skill_description_report_{skill_path.name}_{timestamp}.html" + else: + live_report_path = Path(args.report) + # Open the report immediately so the user can watch + live_report_path.write_text("

Starting optimization loop...

") + webbrowser.open(str(live_report_path)) + else: + live_report_path = None + + # Determine output directory (create before run_loop so logs can be written) + if args.results_dir: + timestamp = time.strftime("%Y-%m-%d_%H%M%S") + results_dir = Path(args.results_dir) / timestamp + results_dir.mkdir(parents=True, exist_ok=True) + else: + results_dir = None + + log_dir = results_dir / "logs" if results_dir else None + + output = run_loop( + eval_set=eval_set, + skill_path=skill_path, + description_override=args.description, + num_workers=args.num_workers, + timeout=args.timeout, + max_iterations=args.max_iterations, + runs_per_query=args.runs_per_query, + trigger_threshold=args.trigger_threshold, + holdout=args.holdout, + model=args.model, + verbose=args.verbose, + live_report_path=live_report_path, + log_dir=log_dir, + ) + + # Save JSON output + json_output = json.dumps(output, indent=2) + print(json_output) + if results_dir: + (results_dir / "results.json").write_text(json_output) + + # Write final HTML report (without auto-refresh) + if live_report_path: + live_report_path.write_text(generate_html(output, auto_refresh=False, skill_name=name)) + print(f"\nReport: {live_report_path}", file=sys.stderr) + + if results_dir and live_report_path: + (results_dir / "report.html").write_text(generate_html(output, auto_refresh=False, skill_name=name)) + + if results_dir: + print(f"Results saved to: {results_dir}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/skill-creator/scripts/utils.py b/skill-creator/scripts/utils.py new file mode 100644 index 00000000..51b6a07d --- /dev/null +++ b/skill-creator/scripts/utils.py @@ -0,0 +1,47 @@ +"""Shared utilities for skill-creator scripts.""" + +from pathlib import Path + + + +def parse_skill_md(skill_path: Path) -> tuple[str, str, str]: + """Parse a SKILL.md file, returning (name, description, full_content).""" + content = (skill_path / "SKILL.md").read_text() + lines = content.split("\n") + + if lines[0].strip() != "---": + raise ValueError("SKILL.md missing frontmatter (no opening ---)") + + end_idx = None + for i, line in enumerate(lines[1:], start=1): + if line.strip() == "---": + end_idx = i + break + + if end_idx is None: + raise ValueError("SKILL.md missing frontmatter (no closing ---)") + + name = "" + description = "" + frontmatter_lines = lines[1:end_idx] + i = 0 + while i < len(frontmatter_lines): + line = frontmatter_lines[i] + if line.startswith("name:"): + name = line[len("name:"):].strip().strip('"').strip("'") + elif line.startswith("description:"): + value = line[len("description:"):].strip() + # Handle YAML multiline indicators (>, |, >-, |-) + if value in (">", "|", ">-", "|-"): + continuation_lines: list[str] = [] + i += 1 + while i < len(frontmatter_lines) and (frontmatter_lines[i].startswith(" ") or frontmatter_lines[i].startswith("\t")): + continuation_lines.append(frontmatter_lines[i].strip()) + i += 1 + description = " ".join(continuation_lines) + continue + else: + description = value.strip('"').strip("'") + i += 1 + + return name, description, content diff --git a/technical-change-tracker/SKILL.md b/technical-change-tracker/SKILL.md new file mode 100644 index 00000000..ba6c9743 --- /dev/null +++ b/technical-change-tracker/SKILL.md @@ -0,0 +1,49 @@ +--- +name: technical-change-tracker +description: | + Track code changes with structured JSON records and accessible HTML output for AI session continuity. Use when user says /tc, /tc init, /tc create, /tc update, /tc status, /tc resume, /tc close, /tc export, /tc dashboard, or /tc retro. +user-invocable: true +tools: Read, Write, Edit, Glob, Grep, Bash +--- + +# Technical Change (TC) Tracker + +Track every code change with structured JSON records and accessible HTML output. +Enables seamless AI bot session handoff across projects. + +## Install + +```bash +git clone https://github.com/Elkidogz/technical-change-skill.git +``` + +## Commands + +| Command | Description | +|---------|-------------| +| `/tc init` | Initialize TC tracking in current project | +| `/tc create ` | Create a new TC record | +| `/tc update ` | Update a TC (status, files, tests) | +| `/tc status [tc-id]` | View TC status | +| `/tc resume ` | Resume from previous session handoff | +| `/tc close ` | Deploy and close a TC | +| `/tc export` | Regenerate all HTML from JSON | +| `/tc dashboard` | Regenerate dashboard | +| `/tc retro ` | Batch-create TCs from project history | + +## Features + +- **JSON records** with append-only revision history and field-level change tracking +- **State machine**: Planned > In Progress > Blocked > Implemented > Tested > Deployed +- **Test cases** with log snippet evidence and manual approval +- **AI session handoff**: progress, next steps, blockers, key context, decisions +- **Non-blocking**: TC bookkeeping runs as background subagent, never interrupts coding +- **Retroactive**: `/tc retro` batch-creates TCs from existing project history +- **WCAG AA+ HTML**: dark theme, rem-based fonts, CSS-only dashboard filters +- **Zero dependencies**: Python stdlib only + +## Repository + +https://github.com/Elkidogz/technical-change-skill + +MIT License. diff --git a/terraform-skill/SKILL.md b/terraform-skill/SKILL.md new file mode 100644 index 00000000..7356c8f6 --- /dev/null +++ b/terraform-skill/SKILL.md @@ -0,0 +1,233 @@ +--- +name: terraform-skill +description: Operational traps for Terraform provisioners, multi-environment isolation, and zero-to-deployment reliability. Covers provisioner timing races, SSH connection conflicts, DNS record duplication, volume permissions, database bootstrap gaps, snapshot cross-contamination, Cloudflare credential format errors, hardcoded domains in Caddyfiles/compose, and init-data-only-on-first-boot pitfalls. Activate when writing null_resource provisioners, creating multi-environment Terraform setups, debugging containers that are Restarting/unhealthy after terraform apply, setting up fresh instances with cloud-init, or any IaC code that SSHs into remote hosts. Also activate when the user mentions terraform plan/apply errors, provisioner failures, infrastructure drift, TLS certificate errors, or Caddy/gateway configuration. +--- + +# Terraform Operational Traps + +Failure patterns from real deployments. Every item caused an incident. Organized as: **exact error → root cause → copy-paste fix**. + +## Provisioner traps (symptom → fix) + +### `docker: not found` in remote-exec + +cloud-init still installing Docker when provisioner SSHs in. + +```hcl +provisioner "remote-exec" { + inline = [ + "cloud-init status --wait || true", + "which docker || { echo 'FATAL: Docker not ready'; exit 1; }", + ] +} +``` + +### `rsync: connection unexpectedly closed` in local-exec + +Terraform holds its SSH connection open; local-exec rsync opens a second one that gets rejected. Never use local-exec for file transfer to remote. Use tarball + file provisioner: + +```hcl +provisioner "local-exec" { + command = "tar czf /tmp/src.tar.gz --exclude=node_modules --exclude=.git -C ${path.module}/../../.. myproject" +} +provisioner "file" { + source = "/tmp/src.tar.gz" + destination = "/tmp/src.tar.gz" +} +provisioner "remote-exec" { + inline = ["tar xzf /tmp/src.tar.gz -C /data/ && rm -f /tmp/src.tar.gz"] +} +``` + +macOS BSD tar: `--exclude` must come BEFORE the source argument. + +### `cloud-init status` shows "running" forever + +`apt-get -y` does not suppress debconf dialogs. Packages like `iptables-persistent` block on TTY prompts. + +```yaml +- | + echo iptables-persistent iptables-persistent/autosave_v4 boolean true | debconf-set-selections + echo iptables-persistent iptables-persistent/autosave_v6 boolean true | debconf-set-selections + DEBIAN_FRONTEND=noninteractive apt-get install -y iptables-persistent +``` + +Known offenders: `iptables-persistent`, `postfix`, `mysql-server`, `wireshark-common`. + +### `EACCES: permission denied` in container logs, container Restarting + +Host volume dirs are root-owned; container runs as non-root (uid 1001). Fix before `docker compose up`: + +```bash +mkdir -p /data/myapp/data /data/myapp/logs +chown -R 1001:1001 /data/myapp/data /data/myapp/logs +``` + +Find UID: grep `adduser.*-u` or `USER` in Dockerfile. + +### Provisioner fails but no diagnostic output + +`set -e` exits on first error, hiding subsequent `docker logs` output. Use `set -u` without `-e`, put one verification gate at the end: + +```hcl +provisioner "remote-exec" { + inline = [ + "set -u", + "docker compose up -d", + "sleep 15", + "docker logs myapp --tail 20 2>&1 || true", + "docker ps --format 'table {{.Names}}\\t{{.Status}}' || true", + "docker ps --filter name=myapp --format '{{.Status}}' | grep -q healthy || exit 1", + ] +} +``` + +### Container `Restarting` — database tables missing + +DB migrations not in provisioner. PostgreSQL `docker-entrypoint-initdb.d` only runs on empty data dir. Explicitly create DB + run migrations: + +```bash +# After postgres healthy: +docker exec pg psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname='mydb'" | grep -q 1 \ + || docker exec pg psql -U postgres -c "CREATE DATABASE mydb;" + +# Idempotent migrations: +for f in migrations/*.sql; do + VER=$(basename $f) + APPLIED=$($PSQL -tAc "SELECT 1 FROM schema_migrations WHERE version='$VER'" | tr -d ' ') + [ "$APPLIED" = "1" ] && continue + { echo 'BEGIN;'; cat $f; echo 'COMMIT;'; } | $PSQL + $PSQL -tAc "INSERT INTO schema_migrations(version) VALUES ('$VER') ON CONFLICT DO NOTHING" +done +``` + +### `docker compose build` ignores env var override + +Compose reads build args from `.env` file, not shell env. `VAR=x docker compose build` does NOT work. + +```bash +# WRONG +DOCKER_WITH_PROXY_MODE=disabled docker compose build + +# RIGHT +grep -q DOCKER_WITH_PROXY_MODE .env || echo 'DOCKER_WITH_PROXY_MODE=disabled' >> .env +docker compose build +``` + +### TLS handshake fails: `Invalid format for Authorization header` + +Caddy DNS-01 ACME needs a Cloudflare **API Token** (`cfut_` prefix, 40+ chars, Bearer auth). A **Global API Key** (37 hex chars, X-Auth-Key auth) causes `HTTP 400 Code:6003`. Production may appear to work because it has cached certificates; fresh environments fail on first cert request. + +```bash +# Verify token format before deploy: +TOKEN=$(grep CLOUDFLARE_API_TOKEN .env | cut -d= -f2) +echo "$TOKEN" | grep -q "^cfut_" || echo "FATAL: needs API Token, not Global Key" +``` + +Create scoped token via API: +```bash +curl -s "https://api.cloudflare.com/client/v4/user/tokens" -X POST \ + -H "X-Auth-Email: $CF_EMAIL" -H "X-Auth-Key: $CF_GLOBAL_KEY" \ + -d '{"name":"caddy-dns-acme","policies":[{"effect":"allow", + "resources":{"com.cloudflare.api.account.zone.":"*"}, + "permission_groups":[ + {"id":"4755a26eedb94da69e1066d98aa820be","name":"DNS Write"}, + {"id":"c8fed203ed3043cba015a93ad1616f1f","name":"Zone Read"}]}]}' +``` + +### TLS fails on staging but works on production — hardcoded domains + +Caddyfile or compose has literal domain names. Staging Caddy loads production config, tries to get certs for domains it doesn't own → ACME fails. + +**Caddyfile**: Use `{$VAR}` — Caddy evaluates env vars at startup. +```caddy +# WRONG +gpt-6.pro { tls { dns cloudflare {env.CLOUDFLARE_API_TOKEN} } } + +# RIGHT +{$LOBEHUB_DOMAIN} { tls { dns cloudflare {env.CLOUDFLARE_API_TOKEN} } } +``` + +**Compose**: Use `${VAR:?required}` — fail-fast if unset. +```yaml +# WRONG +- APP_URL=https://gpt-6.pro + +# RIGHT +- APP_URL=${APP_URL:?APP_URL is required} +``` + +Pass the env var to the gateway container so Caddy can read it: +```yaml +environment: + - LOBEHUB_DOMAIN=${LOBEHUB_DOMAIN:?LOBEHUB_DOMAIN is required} + - CLOUDFLARE_API_TOKEN=${CLOUDFLARE_API_TOKEN:?required for DNS-01 TLS} +``` + +### OAuth login fails: `Social sign in failed` + +Casdoor `init_data.json` contains hardcoded redirect URIs. `--createDatabase=true` only applies init_data on first-ever DB creation — not on restarts. Fix via SQL in provisioner: + +```bash +# Replace production domain with staging in existing Casdoor DB +$PSQL -c "UPDATE application SET redirect_uris = REPLACE(redirect_uris, + 'gpt-6.pro', 'staging.gpt-6.pro') + WHERE name='lobechat' + AND redirect_uris LIKE '%gpt-6.pro%' + AND redirect_uris NOT LIKE '%staging.gpt-6.pro%';" +``` + +Also check `AUTH_CASDOOR_ISSUER` — it must match the Casdoor subdomain (`auth.staging.example.com`), not the app root domain. + +## Multi-environment isolation + +Before creating a second environment, grep `.tf` files for hardcoded names. See [references/multi-env-isolation.md](references/multi-env-isolation.md) for the complete matrix. + +**Will fail on apply** (globally unique): + +| Resource | Scope | Fix | +|---|---|---| +| SSH key pair | Region | `"${env}-deploy"` | +| SLS log project | Account | `"${env}-logs"` | +| CloudMonitor contact | Account | `"${env}-ops"` | + +**DNS duplication trap**: Two environments creating A records for the same name in the same Cloudflare zone → two independent record IDs → DNS round-robin → ~50% traffic to wrong instance. Fix: use subdomain isolation (`staging.example.com`) or separate zones. Remember to create DNS records for ALL subdomains Caddy serves (e.g., `auth.staging`, `minio.staging`). + +**Snapshot cross-contamination**: Unfiltered `data "alicloud_ecs_snapshots"` returns ALL account snapshots. New env inherits old 100GB snapshot, fails creating 40GB disk. Gate with variable: + +```hcl +locals { + latest_snapshot_id = var.enable_snapshot_recovery && length(local.available_snapshots) > 0 + ? local.available_snapshots[0].snapshot_id : null +} +``` + +Do NOT add `count` to the data source — changes its state address, causes drift. + +## Pre-deploy validation + +Run a validation script **before** `terraform apply` to catch configuration errors locally. This eliminates the deploy→discover→fix→redeploy cycle. + +Key checks (see [references/pre-deploy-validation.md](references/pre-deploy-validation.md)): +1. `terraform validate` — syntax +2. No hardcoded domains in Caddyfiles or compose files +3. Required env vars present (`LOBEHUB_DOMAIN`, `CLAUDE4DEV_DOMAIN`, `CLOUDFLARE_API_TOKEN`, `APP_URL`, etc.) +4. Cloudflare API Token format (not Global API Key) +5. DNS records exist for all Caddy-served domains +6. Casdoor issuer URL matches `auth.*` subdomain +7. SSH private key exists + +Integrate into Makefile: `make pre-deploy ENV=staging` before `make apply`. + +## Zero-to-deployment + +Fresh disks expose every implicit dependency. See [references/zero-to-deploy-checklist.md](references/zero-to-deploy-checklist.md). + +Key items that break provisioners on fresh instances: +1. **Directories**: `mkdir -p /data/{svc1,svc2}` in cloud-init — `file` provisioner fails if target dir missing +2. **Databases**: Explicit `CREATE DATABASE` — PG init scripts only run on empty data dir +3. **Migrations**: Tracked in `schema_migrations` table, applied idempotently +4. **Provisioner ordering**: `depends_on` between resources sharing Docker networks +5. **Memory**: Stop non-critical containers during Docker build on small instances (≤8GB) +6. **Domain parameterization**: Every domain in Caddyfile/compose must be `{$VAR}` / `${VAR:?required}` +7. **Credential format**: Caddy needs API Token (`cfut_`), not Global API Key diff --git a/terraform-skill/references/multi-env-isolation.md b/terraform-skill/references/multi-env-isolation.md new file mode 100644 index 00000000..bf4fd2e1 --- /dev/null +++ b/terraform-skill/references/multi-env-isolation.md @@ -0,0 +1,130 @@ +# Multi-Environment Isolation Checklist + +When creating a second Terraform environment (`staging`, `lab`, etc.) in the same cloud account alongside production, every item below must be verified. Skip one and you get silent name collisions or cross-contamination. + +## Terraform state isolation + +Two environments MUST use different state paths. Same OSS/S3 bucket is fine — different prefix isolates completely: + +```hcl +# production +backend "oss" { + bucket = "myproject-terraform-state" + prefix = "environments/production" +} + +# staging +backend "oss" { + bucket = "myproject-terraform-state" # same bucket OK + prefix = "environments/staging" # different prefix = isolated state +} +``` + +**Verification**: `terraform state list` in one environment must show ZERO resources from the other. + +## Resource naming collision matrix + +Grep every `.tf` file for hardcoded names. Every globally-unique resource will collide. + +### Must rename (apply will fail) + +| Resource | Uniqueness scope | Fix pattern | +|---|---|---| +| SSH key pair (`key_pair_name`) | Region | `"${env}-deploy"` | +| SLS log project (`project_name`) | Account | `"${env}-logs"` | +| CloudMonitor contact (`alarm_contact_name`) | Account | `"${env}-ops"` | +| CloudMonitor contact group | Account | `"${env}-ops"` | + +### Should rename (won't fail but causes confusion) + +| Resource | Issue if same name | +|---|---| +| Security group name | Two SGs with same name in same VPC, can't tell apart in console | +| ECS instance name/hostname | Two instances named `myapp-spot` in console | +| Data disk name | Same in disk list | +| Auto snapshot policy name | Same in policy list | +| SLS machine group name | Logs from both instances land in same group | + +### Pattern: Use a module name variable + +```hcl +# production main.tf +module "app" { + source = "../../modules/spot-with-data-disk" + name = "production-spot" # flows to instance_name, disk_name, snapshot_policy_name +} + +# staging main.tf +module "app" { + source = "../../modules/spot-with-data-disk" + name = "staging-spot" # all child resource names auto-isolated +} +``` + +## DNS record isolation + +### The duplication trap + +Two Terraform environments creating A records for `@` (root) in the same Cloudflare zone: +- Each gets its own Cloudflare record ID (independent) +- Cloudflare now has TWO A records for the same domain +- DNS round-robins between the two IPs +- ~50% of traffic goes to the wrong instance + +### Correct patterns + +**Pattern A: Subdomain isolation** (recommended for staging/lab): +```hcl +# Production: root domain records +resource "cloudflare_dns_record" "prod" { + name = "@" # gpt-6.pro +} + +# Staging: subdomain records only +resource "cloudflare_dns_record" "staging" { + name = "staging" # staging.gpt-6.pro +} +``` + +**Pattern B: Separate zones** (for fully independent deployments): +Each environment gets its own domain/zone. No shared Cloudflare zone IDs. + +**Pattern C: One environment owns DNS** (production): +Only production has DNS resources. Other environments access via IP only. + +### Destroy safety + +When one environment is destroyed: +- Its DNS records are deleted (by their specific Cloudflare record IDs) +- Other environments' DNS records are NOT affected +- **Verify before destroy**: Compare DNS record IDs between environments: + ```bash + terraform state show 'cloudflare_dns_record.app["root"]' | grep "^id" + ``` + IDs must be different. + +## Shared resources (safe to share) + +These are referenced but NOT managed by the second environment: + +| Resource | Why safe | +|---|---| +| VPC / VSwitch | Referenced by ID, not created | +| Cloudflare zone ID | Referenced, records are independent | +| OSS state bucket | Different prefix = different state | +| SSH public key content | Same key, different key pair resource | +| Cloud provider credentials | Same account, different resources | + +## Makefile pattern for multi-environment + +```makefile +ENV ?= production +ENV_DIR := environments/$(ENV) + +init: ; cd $(ENV_DIR) && terraform init +plan: ; cd $(ENV_DIR) && terraform plan -out=tfplan +apply: ; cd $(ENV_DIR) && terraform apply tfplan +drift: ; cd $(ENV_DIR) && terraform plan -detailed-exitcode +``` + +Usage: `make plan ENV=staging` diff --git a/terraform-skill/references/pre-deploy-validation.md b/terraform-skill/references/pre-deploy-validation.md new file mode 100644 index 00000000..6e1c485e --- /dev/null +++ b/terraform-skill/references/pre-deploy-validation.md @@ -0,0 +1,82 @@ +# Pre-Deploy Validation Pattern + +Run before `terraform apply` to catch configuration errors locally. Eliminates the deploy→discover→fix→redeploy cycle that wastes hours. + +## Why this matters + +Every hardcoded value becomes a bug when creating a second environment. Production accumulates implicit state over time (cached TLS certs, manually created databases, hand-edited configs). Fresh instances expose all of these as failures. A pre-deploy script catches them before they reach the remote. + +## Validation categories + +### 1. Terraform syntax +```bash +terraform validate +``` + +### 2. Hardcoded domains +```bash +# Caddyfiles: should use {$VAR} not literal domains +grep -v "^#" gateway/conf.d/*.caddy | grep -c "example\.com" # should be 0 + +# Compose: should use ${VAR:?required} not literal domains +grep -v "^#" docker-compose.production.yml | grep -c "example\.com" # should be 0 +``` + +### 3. Required env vars +Check that every `${VAR:?required}` in compose has a matching entry in `.env`: +```bash +for VAR in LOBEHUB_DOMAIN CLAUDE4DEV_DOMAIN CLOUDFLARE_API_TOKEN APP_URL AUTH_URL; do + grep -q "^$VAR=" .env || echo "FAIL: $VAR missing" +done +``` + +### 4. Cloudflare credential format +Caddy's Cloudflare plugin uses Bearer auth. Global API Keys (37 hex chars) fail with `Invalid format for Authorization header`. +```bash +TOKEN=$(grep CLOUDFLARE_API_TOKEN .env | cut -d= -f2) +echo "$TOKEN" | grep -qE "^cfut_|^[A-Za-z0-9_-]{40,}$" || echo "FAIL: looks like Global API Key, not API Token" +``` + +### 5. DNS ↔ Caddy consistency +Every domain Caddy serves needs a DNS record. Check live resolution: +```bash +for DOMAIN in staging.example.com auth.staging.example.com; do + curl -sf "https://dns.google/resolve?name=$DOMAIN&type=A" | python3 -c \ + "import sys,json; d=json.load(sys.stdin); exit(0 if d.get('Answer') else 1)" \ + || echo "FAIL: $DOMAIN not resolving" +done +``` + +### 6. Casdoor issuer consistency +`AUTH_CASDOOR_ISSUER` must point to `auth.`, not the app's root domain: +```bash +ISSUER=$(grep AUTH_CASDOOR_ISSUER .env | cut -d= -f2) +DOMAIN=$(grep LOBEHUB_DOMAIN .env | cut -d= -f2) +[ "$ISSUER" = "https://auth.$DOMAIN" ] || echo "FAIL: issuer should be https://auth.$DOMAIN" +``` + +### 7. SSH key exists +```bash +[ -f ~/.ssh/id_ed25519 ] || echo "FAIL: SSH key not found" +``` + +## Makefile integration + +```makefile +pre-deploy: + @./scripts/validate-env.sh $(ENV) + +# Enforce: plan requires pre-deploy to pass +plan: pre-deploy + cd $(ENV_DIR) && terraform plan -out=tfplan +``` + +## Anti-pattern: deploy-and-pray + +The opposite of pre-deploy validation is the "deploy and see what breaks" cycle: +1. `terraform apply` → fails +2. SSH in to debug → discover error +3. Fix locally → commit → re-apply → fails differently +4. Repeat 5-10 times + +Each cycle takes 3-5 minutes (plan + apply + provisioner). Pre-deploy catches 80% of issues in <5 seconds locally. diff --git a/terraform-skill/references/zero-to-deploy-checklist.md b/terraform-skill/references/zero-to-deploy-checklist.md new file mode 100644 index 00000000..a975c268 --- /dev/null +++ b/terraform-skill/references/zero-to-deploy-checklist.md @@ -0,0 +1,168 @@ +# Zero-to-Deployment Checklist + +A fresh instance with an empty data disk exposes every implicit dependency that production silently relies on. This checklist covers everything that must be explicitly created before services will start. + +## Pre-flight: cloud-init must handle + +These run at OS boot, before Terraform provisioners: + +- [ ] **Mount data disk**: Format if new (`blkid` check), mount to `/data`, add to fstab +- [ ] **Create service directories**: `mkdir -p /data/{service1,service2,...}` — file provisioners fail if target dir doesn't exist +- [ ] **Install Docker + Compose**: Curl installer, enable systemd service +- [ ] **Configure swap**: `fallocate` on data disk (NOT system disk) +- [ ] **SSH hardening**: key-only auth, no password root login +- [ ] **Firewall**: UFW + DOCKER-USER iptables chain +- [ ] **Debconf preseed**: For any package with interactive prompts (iptables-persistent, etc.) +- [ ] **Signal readiness**: Write timestamp to `/data/cloud-init.log` + +## Provisioner ordering + +Terraform provisioners execute in declaration order within a resource, but resources execute in parallel unless `depends_on` is set. + +``` +lobehub_deploy ──────────────────→ channel_sync (depends_on lobehub) + → casdoor_sync (depends_on lobehub) + → minio_sync (depends_on lobehub) + +claude4dev_deploy (depends_on lobehub_deploy) + ├─ wait for cloud-init + ├─ upload source (tarball via file provisioner) + ├─ upload .env (staging variant) + ├─ start stateful (postgres, redis) --no-recreate + ├─ run DB migrations + ├─ build stateless images + ├─ fix volume permissions + ├─ start stateless (relay, api, frontend, gateway) + └─ verify health +``` + +## Database bootstrap + +### PostgreSQL databases + +PostgreSQL `docker-entrypoint-initdb.d` scripts only run when the data directory is empty (first-ever start). On subsequent starts — even if a database doesn't exist — init scripts are skipped. + +**Fix**: Explicitly create databases in provisioner: +```bash +# Wait for postgres healthy +sleep 10 +# Create database if missing (idempotent) +docker exec my-postgres psql -U postgres -tc \ + "SELECT 1 FROM pg_database WHERE datname='mydb'" | grep -q 1 \ + || docker exec my-postgres psql -U postgres -c "CREATE DATABASE mydb;" +``` + +### Schema migrations + +Migrations must be idempotent. Track applied versions: +```bash +PSQL='docker compose exec -T postgres psql -v ON_ERROR_STOP=1 -U myuser -d mydb' + +# Create tracking table +$PSQL -tAc "CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ DEFAULT now() +)" + +# Apply each migration file in order +for f in migrations/*.sql; do + VER=$(basename $f) + APPLIED=$($PSQL -tAc "SELECT 1 FROM schema_migrations WHERE version='$VER'" | tr -d ' ') + if [ "$APPLIED" = "1" ]; then + echo "Skip: $VER" + else + echo "Apply: $VER" + { echo 'BEGIN;'; cat $f; echo 'COMMIT;'; } | $PSQL + $PSQL -tAc "INSERT INTO schema_migrations(version) VALUES ('$VER') ON CONFLICT DO NOTHING" + fi +done +``` + +## Docker build on remote + +### Proxy mode + +Docker Compose reads build args from `.env` via `${VAR:-default}`. Command-line env vars do NOT override `.env` values for compose interpolation. + +```bash +# WRONG: compose still reads DOCKER_WITH_PROXY_MODE from .env +DOCKER_WITH_PROXY_MODE=disabled docker compose build myapp + +# RIGHT: modify .env so compose reads the correct value +grep -q DOCKER_WITH_PROXY_MODE .env || echo 'DOCKER_WITH_PROXY_MODE=disabled' >> .env +docker compose build myapp +``` + +### Memory management + +Building Docker images while 10+ containers run can OOM on small instances (8GB). Strategy: + +```bash +# Stop non-critical containers to free RAM +cd /data/other-project && docker compose stop search-engine analytics-db || true + +# Build (memory-intensive) +cd /data/myproject && docker compose build myapp + +# Restart stopped containers +cd /data/other-project && docker compose up -d search-engine analytics-db || true +``` + +## Volume permissions + +Containers running as non-root need writable volume directories: + +```bash +# Before docker compose up: +mkdir -p data-dir logs-dir +chown -R 1001:1001 data-dir logs-dir # match container UID +``` + +Find the UID from the Dockerfile: +```dockerfile +RUN adduser -S myuser -u 1001 -G mygroup +USER myuser # runs as uid 1001 +``` + +## Environment-specific .env files + +Production `.env` contains production URLs. Staging needs its own `.env` with: + +| Variable | Production | Staging | +|---|---|---| +| `FRONTEND_URL` | `https://myapp.com` | `https://staging.myapp.com` | +| `CORS_ORIGIN` | `https://myapp.com` | `https://staging.myapp.com` | +| `NEW_API_URL` | `http://api-container:3000` | Same (internal Docker network) | +| `DOCKER_WITH_PROXY_MODE` | `required` (if behind proxy) | `disabled` (direct internet) | + +**Pattern**: Create `.env.staging` alongside `.env`. In Terraform: +```hcl +locals { + env_src = "${local.repo}/.env.staging" # staging-specific +} + +provisioner "file" { + source = local.env_src + destination = "${local.deploy_dir}/.env" +} +``` + +Rsync must exclude `.env` files (otherwise production .env overwrites staging .env): +``` +--exclude=.env --exclude='.env.*' +``` + +## Verification template + +After all services start, verify in the provisioner (not ad-hoc SSH): + +```bash +sleep 20 +echo '=== Service logs ===' +docker logs my-critical-service --tail 20 2>&1 || true +echo '=== All containers ===' +docker ps --format 'table {{.Names}}\t{{.Status}}' 2>&1 || true +# Final gate (only line that can fail) +docker ps --filter name=my-critical-service --format '{{.Status}}' | grep -q healthy \ + || { echo 'FATAL: service unhealthy'; exit 1; } +``` diff --git a/transcript-fixer/SKILL.md b/transcript-fixer/SKILL.md index 4830456c..11060cb5 100644 --- a/transcript-fixer/SKILL.md +++ b/transcript-fixer/SKILL.md @@ -29,40 +29,54 @@ powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | ie ## Quick Start -**Recommended: Use Enhanced Wrapper** (auto-detects API key, opens HTML diff): +**Default: Native AI Correction (no API key needed)** + +When invoked from Claude Code, the skill uses a two-phase approach: +1. **Dictionary phase** (script): Apply 700+ learned correction rules instantly +2. **AI phase** (Claude native): Claude reads the text directly and fixes ASR errors, adds paragraph breaks, removes filler words ```bash # First time: Initialize database uv run scripts/fix_transcription.py --init -# Process transcript with enhanced UX -uv run scripts/fix_transcript_enhanced.py input.md --output ./corrected +# Phase 1: Dictionary corrections (instant, free) +uv run scripts/fix_transcription.py --input meeting.md --stage 1 ``` -The enhanced wrapper automatically: -- Detects GLM API key from shell configs (checks lines near `ANTHROPIC_BASE_URL`) -- Moves output files to specified directory -- Opens HTML visual diff in browser for immediate feedback +After Stage 1, Claude should: +1. Read the Stage 1 output in ~3000-char chunks +2. Identify ASR errors (homophones, technical terms, broken sentences) +3. Present corrections in a table for user review (high/medium confidence) +4. Apply confirmed corrections and save stable patterns to dictionary +5. Optionally: add paragraph breaks and remove excessive filler words -**Alternative: Use Core Script Directly**: +**Alternative: API-Based Batch Processing** (for automation or large volumes): ```bash -# 1. Set API key (if not auto-detected) +# Set API key for automated AI corrections export GLM_API_KEY="" # From https://open.bigmodel.cn/ -# 2. Add common corrections (5-10 terms) -uv run scripts/fix_transcription.py --add "错误词" "正确词" --domain general +# Run full pipeline (dict + API AI + diff report) +uv run scripts/fix_transcript_enhanced.py input.md --output ./corrected +``` -# 3. Run full correction pipeline -uv run scripts/fix_transcription.py --input meeting.md --stage 3 +**Timestamp repair**: +```bash +uv run scripts/fix_transcript_timestamps.py meeting.txt --in-place +``` -# 4. Review learned patterns after 3-5 runs -uv run scripts/fix_transcription.py --review-learned +**Split transcript into sections and rebase each section to `00:00:00`**: +```bash +uv run scripts/split_transcript_sections.py meeting.txt \ + --first-section-name "课前聊天" \ + --section "正式上课::好，无缝切换嘛。对。那个曹总连上了吗？那个网页。" \ + --section "课后复盘::我们复盘一下。" \ + --rebase-to-zero ``` **Output files**: - `*_stage1.md` - Dictionary corrections applied -- `*_stage2.md` - AI corrections applied (final version) +- `*_corrected.txt` - Final version (native mode) or `*_stage2.md` (API mode) - `*_对比.html` - Visual diff (open in browser for best experience) **Generate word-level diff** (recommended for reviewing corrections): @@ -102,14 +116,15 @@ This creates an HTML file showing word-by-word differences with clear highlighti ## Core Workflow -Three-stage pipeline stores corrections in `~/.transcript-fixer/corrections.db`: +Two-phase pipeline stores corrections in `~/.transcript-fixer/corrections.db`: 1. **Initialize** (first time): `uv run scripts/fix_transcription.py --init` 2. **Add domain corrections**: `--add "错误词" "正确词" --domain ` -3. **Process transcript**: `--input file.md --stage 3` -4. **Review learned patterns**: `--review-learned` and `--approve` high-confidence suggestions +3. **Phase 1 — Dictionary**: `--input file.md --stage 1` (instant, free) +4. **Phase 2 — AI Correction**: Claude reads output and fixes ASR errors natively (default), or use `--stage 3` with `GLM_API_KEY` for API mode +5. **Save stable patterns**: `--add "错误词" "正确词"` after each fix session +6. **Review learned patterns**: `--review-learned` and `--approve` high-confidence suggestions -**Stages**: Dictionary (instant, free) → AI via GLM API (parallel) → Full pipeline **Domains**: `general`, `embodied_ai`, `finance`, `medical`, or custom names including Chinese (e.g., `火星加速器`, `具身智能`) **Learning**: Patterns appearing ≥3 times at ≥80% confidence move from AI to dictionary @@ -117,23 +132,96 @@ See `references/workflow_guide.md` for detailed workflows, `references/script_pa ## Critical Workflow: Dictionary Iteration -**MUST save corrections after each fix.** This is the skill's core value. +**Save stable, reusable ASR patterns after each fix.** This is the skill's core value. -After fixing errors manually, immediately save to dictionary: +After fixing errors manually, immediately save stable corrections to dictionary: ```bash uv run scripts/fix_transcription.py --add "错误词" "正确词" --domain general ``` +Do **not** save one-off deletions, ambiguous context-only rewrites, or section-specific cleanup to the dictionary. + See `references/iteration_workflow.md` for complete iteration guide with checklist. -## AI Fallback Strategy +## FALSE POSITIVE RISKS -- READ BEFORE ADDING CORRECTIONS + +Dictionary-based corrections are powerful but dangerous. Adding the wrong rule silently corrupts every future transcript. The `--add` command runs safety checks automatically, but you must understand the risks. + +### What is safe to add + +- **ASR-specific gibberish**: "巨升智能" -> "具身智能" (no real word sounds like "巨升智能") +- **Long compound errors**: "语音是别" -> "语音识别" (4+ chars, unlikely to collide) +- **English transliteration errors**: "japanese 3 pro" -> "Gemini 3 Pro" + +### What is NEVER safe to add + +- **Common Chinese words**: "仿佛", "正面", "犹豫", "传说", "增加", "教育" -- these appear correctly in normal text. Replacing them corrupts transcripts from better ASR models. +- **Words <=2 characters**: Almost any 2-char Chinese string is a valid word or part of one. "线数" inside "产线数据" becomes "产线束据". +- **Both sides are real words**: "仿佛->反复", "犹豫->抑郁" -- both forms are valid Chinese. The "error" is only an error for one specific ASR model. + +### When in doubt, use a context rule instead + +Context rules use regex patterns that match only in specific surroundings, avoiding false positives: +```bash +# Instead of: --add "线数" "线束" +# Use a context rule in the database: +sqlite3 ~/.transcript-fixer/corrections.db "INSERT INTO context_rules (pattern, replacement, description, priority) VALUES ('(?线束 (not inside 产线数据)', 10);" +``` + +### Auditing the dictionary + +Run `--audit` periodically to scan all rules for false positive risks: +```bash +uv run scripts/fix_transcription.py --audit +uv run scripts/fix_transcription.py --audit --domain manufacturing +``` + +### Forcing a risky addition + +If you understand the risks and still want to add a flagged rule: +```bash +uv run scripts/fix_transcription.py --add "仿佛" "反复" --domain general --force +``` + +## Native AI Correction (Default Mode) + +**Claude IS the AI.** When running inside Claude Code, use Claude's own language understanding for Stage 2 corrections instead of calling an external API. This is the default behavior — no API key needed. + +### Workflow + +1. **Run Stage 1** (dictionary): `uv run scripts/fix_transcription.py --input file.md --stage 1` +2. **Read the text** in ~3000-character chunks (use `cut -c-` for single-line files) +3. **Identify ASR errors** — look for: + - Homophone errors (同音字): "上海文" → "上下文", "扩种" → "扩充" + - Broken sentence boundaries: "很大程。路上" → "很大程度上" + - Technical terms: "Web coding" → "Vibe Coding" + - Missing/extra characters: "沉沉默" → "沉默" +4. **Present corrections** in a table with confidence levels before applying: + - High confidence: clear ASR errors with unambiguous corrections + - Medium confidence: context-dependent, need user confirmation +5. **Apply corrections** to a copy of the file (never modify the original) +6. **Save stable patterns** to dictionary: `--add "错误词" "正确词" --domain general` +7. **Generate word diff**: `uv run scripts/generate_word_diff.py original.md corrected.md diff.html` + +### Enhanced AI Capabilities (Native Mode Only) + +Native mode can do things the API mode cannot: + +- **Intelligent paragraph breaks**: Add `\n\n` at logical topic transitions in continuous text +- **Filler word reduction**: Remove excessive repetition (这个这个这个 → 这个, 都都都都 → 都) +- **Interactive review**: Present corrections for user confirmation before applying +- **Context-aware judgment**: Use full document context to resolve ambiguous errors + +### When to Use API Mode Instead + +Use `GLM_API_KEY` + Stage 3 for: +- Batch processing multiple files in automation +- When Claude Code is not available (standalone script usage) +- Consistent reproducible processing without interactive review -When GLM API is unavailable (503, network issues), the script outputs `[CLAUDE_FALLBACK]` marker. +### Legacy Fallback Marker -Claude Code should then: -1. Analyze the text directly for ASR errors -2. Fix using Edit tool -3. **MUST save corrections to dictionary** with `--add` +When the script outputs `[CLAUDE_FALLBACK]` (GLM API error), switch to native mode automatically. ## Database Operations @@ -153,8 +241,8 @@ sqlite3 ~/.transcript-fixer/corrections.db "SELECT value FROM system_config WHER | Stage | Description | Speed | Cost | |-------|-------------|-------|------| | 1 | Dictionary only | Instant | Free | -| 2 | AI only | ~10s | API calls | -| 3 | Full pipeline | ~10s | API calls | +| 1 + Native | Dictionary + Claude AI (default) | ~1min | Free | +| 3 | Dictionary + API AI + diff report | ~10s | API calls | ## Bundled Resources @@ -162,7 +250,9 @@ sqlite3 ~/.transcript-fixer/corrections.db "SELECT value FROM system_config WHER - `ensure_deps.py` - Initialize shared virtual environment (run once, optional) - `fix_transcript_enhanced.py` - Enhanced wrapper (recommended for interactive use) - `fix_transcription.py` - Core CLI (for automation) +- `fix_transcript_timestamps.py` - Normalize/repair speaker timestamps and optionally rebase to zero - `generate_word_diff.py` - Generate word-level diff HTML for reviewing corrections +- `split_transcript_sections.py` - Split a transcript by marker phrases and optionally rebase each section - `examples/bulk_import.py` - Bulk import example **References** (load as needed): diff --git a/transcript-fixer/references/iteration_workflow.md b/transcript-fixer/references/iteration_workflow.md index ba94a462..17383118 100644 --- a/transcript-fixer/references/iteration_workflow.md +++ b/transcript-fixer/references/iteration_workflow.md @@ -16,7 +16,7 @@ The core value of transcript-fixer is building a personalized correction diction └─────────────────────────────────────────────────┘ ``` -**Key principle**: Every correction you make should be saved to the dictionary. This transforms one-time work into permanent value. +**Key principle**: Every stable, reusable ASR correction you make should be saved to the dictionary. This transforms one-time work into permanent value without polluting the database. ## Workflow Checklist @@ -34,7 +34,7 @@ Correction Progress: ## Save Corrections Immediately -After fixing any transcript, save corrections: +After fixing any transcript, save stable corrections: ```bash # Single correction @@ -122,3 +122,12 @@ Patterns appearing ≥3 times at ≥80% confidence are suggested for review. 3. **Use domains**: Organize corrections by topic for better precision 4. **Verify**: Always run --list to confirm saves 5. **Review suggestions**: Periodically check --review-learned for auto-detected patterns + +## What NOT to Save to Dictionary + +Do **not** save these as reusable dictionary entries: + +- Full-sentence deletions +- One-off section headers or meeting-specific boilerplate +- Context-only disambiguations such as `cloud -> Claude` when `cloud` can also be legitimate +- File-local cleanup after section splitting or timestamp rebasing diff --git a/transcript-fixer/references/script_parameters.md b/transcript-fixer/references/script_parameters.md index a5537abc..6cbeab3d 100644 --- a/transcript-fixer/references/script_parameters.md +++ b/transcript-fixer/references/script_parameters.md @@ -9,6 +9,8 @@ Detailed command-line parameters and usage examples for transcript-fixer Python - [Correction Management](#correction-management) - [Correction Workflow](#correction-workflow) - [Learning Commands](#learning-commands) +- [fix_transcript_timestamps.py](#fix_transcript_timestampspy) - Normalize/repair speaker timestamps +- [split_transcript_sections.py](#split_transcript_sectionspy) - Split transcript into named sections - [diff_generator.py](#diffgeneratorpy) - Generate comparison reports - [Common Workflows](#common-workflows) - [Exit Codes](#exit-codes) @@ -74,6 +76,59 @@ python scripts/fix_transcription.py --input meeting.md --stage 3 --output ./corr - `2` - GLM_API_KEY environment variable not set (Stage 2 or 3 only) - `3` - API request failed +## fix_transcript_timestamps.py + +Normalize speaker timestamp lines such as `天生 00:21` or `Speaker 7 01:31:10`. + +### Syntax + +```bash +python scripts/fix_transcript_timestamps.py [--output FILE | --in-place | --check] +``` + +### Key Parameters + +- `--format {hhmmss,preserve}`: output timestamp style +- `--rebase-to-zero`: reset the first detected speaker timestamp to `00:00:00` +- `--rollover-backjump-seconds`: threshold for treating `59:58 -> 00:05` as a new hour +- `--jitter-seconds`: tolerated small backward jitter before flagging anomaly + +### Usage Examples + +```bash +# Normalize mixed MM:SS / HH:MM:SS +python scripts/fix_transcript_timestamps.py meeting.txt --in-place + +# Rebase a split transcript so it starts at 00:00:00 +python scripts/fix_transcript_timestamps.py workshop-class.txt --in-place --rebase-to-zero + +# Only inspect anomalies, do not write +python scripts/fix_transcript_timestamps.py meeting.txt --check +``` + +## split_transcript_sections.py + +Split a transcript into named sections using marker phrases. Useful for workshop transcripts that include setup chat, class, and debrief in one file. + +### Syntax + +```bash +python scripts/split_transcript_sections.py \ + --first-section-name \ + --section "Name::Marker" \ + --section "Name::Marker" +``` + +### Usage Example + +```bash +python scripts/split_transcript_sections.py workshop.txt \ + --first-section-name "课前聊天" \ + --section "正式上课::好，无缝切换嘛。对。那个曹总连上了吗？那个网页。" \ + --section "课后复盘::我们复盘一下。" \ + --rebase-to-zero +``` + ## generate_diff_report.py Multi-format diff report generator for comparing correction stages. diff --git a/transcript-fixer/references/workflow_guide.md b/transcript-fixer/references/workflow_guide.md index b6132b6d..5dbc7dad 100644 --- a/transcript-fixer/references/workflow_guide.md +++ b/transcript-fixer/references/workflow_guide.md @@ -17,6 +17,7 @@ Detailed step-by-step workflows for transcript correction and management. - [5. Stage-by-Stage Execution](#5-stage-by-stage-execution) - [6. Context-Aware Rules](#6-context-aware-rules) - [7. Diff Report Generation](#7-diff-report-generation) + - [8. Workshop Transcript Split + Timestamp Rebase](#8-workshop-transcript-split--timestamp-rebase) - [Batch Processing](#batch-processing) - [Process Multiple Files](#process-multiple-files) - [Parallel Processing](#parallel-processing) @@ -400,6 +401,30 @@ See `file_formats.md` for context_rules schema. See `script_parameters.md` for advanced diff options. +### 8. Workshop Transcript Split + Timestamp Rebase + +**Goal**: Split a long workshop transcript into sections such as setup chat, class, and debrief, then make each section start from `00:00:00`. + +**Steps**: + +1. **Correct transcript text first** (dictionary + AI/manual review) +2. **Pick marker phrases** for each section boundary +3. **Split and rebase**: + +```bash +uv run scripts/split_transcript_sections.py workshop.txt \ + --first-section-name "课前聊天" \ + --section "正式上课::好，无缝切换嘛。对。那个曹总连上了吗？那个网页。" \ + --section "课后复盘::我们复盘一下。" \ + --rebase-to-zero +``` + +4. **If you already split the files**, rebase a single file directly: + +```bash +uv run scripts/fix_transcript_timestamps.py class.txt --in-place --rebase-to-zero +``` + ## Batch Processing ### Process Multiple Files diff --git a/transcript-fixer/scripts/cli/__init__.py b/transcript-fixer/scripts/cli/__init__.py index 6e6e5b0c..ca8d6207 100644 --- a/transcript-fixer/scripts/cli/__init__.py +++ b/transcript-fixer/scripts/cli/__init__.py @@ -9,6 +9,7 @@ from .commands import ( cmd_init, cmd_add_correction, + cmd_audit, cmd_list_corrections, cmd_run_correction, cmd_review_learned, @@ -25,6 +26,7 @@ __all__ = [ 'cmd_init', 'cmd_add_correction', + 'cmd_audit', 'cmd_list_corrections', 'cmd_run_correction', 'cmd_review_learned', diff --git a/transcript-fixer/scripts/cli/argument_parser.py b/transcript-fixer/scripts/cli/argument_parser.py index 086d67f9..a418fd08 100644 --- a/transcript-fixer/scripts/cli/argument_parser.py +++ b/transcript-fixer/scripts/cli/argument_parser.py @@ -37,12 +37,24 @@ def create_argument_parser() -> argparse.ArgumentParser: dest="add_correction", help="Add correction" ) + parser.add_argument( + "--force", + action="store_true", + default=False, + help="Force --add even when safety checks detect risks (common word, substring collision)" + ) parser.add_argument( "--list", action="store_true", dest="list_corrections", help="List all corrections" ) + parser.add_argument( + "--audit", + action="store_true", + dest="audit_dictionary", + help="Audit all active corrections for false positive risks (common words, short text, substring collisions)" + ) # Correction workflow parser.add_argument( @@ -62,8 +74,8 @@ def create_argument_parser() -> argparse.ArgumentParser: ) parser.add_argument( "--domain", "-d", - default="general", - help="Correction domain" + default=None, + help="Correction domain (default: all domains)" ) # Learning commands diff --git a/transcript-fixer/scripts/cli/commands.py b/transcript-fixer/scripts/cli/commands.py index 4caa832c..dbac6553 100644 --- a/transcript-fixer/scripts/cli/commands.py +++ b/transcript-fixer/scripts/cli/commands.py @@ -43,26 +43,106 @@ def cmd_init(args: argparse.Namespace) -> None: def cmd_add_correction(args: argparse.Namespace) -> None: - """Add a single correction""" + """Add a single correction with safety checks""" service = _get_service() + force = getattr(args, 'force', False) try: - service.add_correction(args.from_text, args.to_text, args.domain) - print(f"✅ Added: '{args.from_text}' → '{args.to_text}' (domain: {args.domain})") + service.add_correction( + args.from_text, args.to_text, args.domain, force=force, + ) + print(f"Added: '{args.from_text}' -> '{args.to_text}' (domain: {args.domain})") except Exception as e: - print(f"❌ Error: {e}") + print(f"Error: {e}", file=sys.stderr) sys.exit(1) +def cmd_audit(args: argparse.Namespace) -> None: + """Audit all active corrections for false positive risks""" + service = _get_service() + domain = getattr(args, 'domain', None) + + print(f"\nAuditing corrections" + (f" (domain: {domain})" if domain else " (all domains)") + "...") + print("=" * 70) + + issues = service.audit_dictionary(domain) + + if not issues: + corrections = service.get_corrections(domain) + print(f"\nAll {len(corrections)} corrections passed safety checks.") + return + + # Categorize + error_count = 0 + warning_count = 0 + for from_text, warnings in issues.items(): + for w in warnings: + if w.level == "error": + error_count += 1 + else: + warning_count += 1 + + corrections = service.get_corrections(domain) + print(f"\nScanned {len(corrections)} corrections. " + f"Found issues in {len(issues)} rules:") + print(f" Errors: {error_count} (should be removed or converted to context rules)") + print(f" Warnings: {warning_count} (review recommended)") + print() + + # Print details grouped by severity + for severity in ["error", "warning"]: + label = "ERRORS" if severity == "error" else "WARNINGS" + relevant = { + ft: [w for w in ws if w.level == severity] + for ft, ws in issues.items() + } + relevant = {ft: ws for ft, ws in relevant.items() if ws} + + if not relevant: + continue + + print(f"--- {label} ({len(relevant)} rules) ---") + for from_text, warnings in sorted(relevant.items()): + to_text = corrections.get(from_text, "?") + print(f"\n '{from_text}' -> '{to_text}'") + for w in warnings: + print(f" [{w.category}] {w.message}") + print(f" Suggestion: {w.suggestion}") + print() + + if error_count > 0: + print( + f"ACTION REQUIRED: {error_count} error(s) found. These rules are " + f"actively causing false positives and should be removed or " + f"converted to context rules." + ) + print( + f"To remove a rule: " + f"sqlite3 ~/.transcript-fixer/corrections.db " + f"\"UPDATE corrections SET is_active=0 WHERE from_text='...';\"" + ) + + def cmd_list_corrections(args: argparse.Namespace) -> None: """List all corrections""" service = _get_service() corrections = service.get_corrections(args.domain) - print(f"\n📋 Corrections (domain: {args.domain})") + if args.domain: + header = f"domain: {args.domain}, {len(corrections)} total" + else: + header = f"all domains, {len(corrections)} total" + + print(f"\n📋 Corrections ({header})") print("=" * 60) - for wrong, correct in sorted(corrections.items()): - print(f" '{wrong}' → '{correct}'") - print(f"\nTotal: {len(corrections)} corrections\n") + + if args.domain: + for wrong, correct in sorted(corrections.items()): + print(f" '{wrong}' → '{correct}'") + else: + all_corrections = service.repository.get_all_corrections(active_only=True) + for c in all_corrections: + print(f" [{c.domain}] '{c.from_text}' → '{c.to_text}'") + print() def cmd_run_correction(args: argparse.Namespace) -> None: @@ -83,12 +163,23 @@ def cmd_run_correction(args: argparse.Namespace) -> None: # Load corrections and rules corrections = service.get_corrections(args.domain) context_rules = service.load_context_rules() + domain_stats = service.get_domain_stats() # Read input file print(f"📖 Reading: {input_path.name}") with open(input_path, 'r', encoding='utf-8') as f: original_text = f.read() - print(f" File size: {len(original_text):,} characters\n") + print(f" File size: {len(original_text):,} characters") + + # Show domain loading info + if args.domain: + print(f"📚 Loaded {len(corrections)} corrections (domain: {args.domain})") + elif domain_stats: + parts = ", ".join(f"{d}: {n}" for d, n in sorted(domain_stats.items())) + print(f"📚 Loaded {len(corrections)} corrections ({parts})") + else: + print(f"📚 No corrections in database") + print() # Stage 1: Dictionary corrections stage1_changes = [] @@ -109,7 +200,17 @@ def cmd_run_correction(args: argparse.Namespace) -> None: stage1_file = output_dir / f"{input_path.stem}_stage1.md" with open(stage1_file, 'w', encoding='utf-8') as f: f.write(stage1_text) - print(f"💾 Saved: {stage1_file.name}\n") + print(f"💾 Saved: {stage1_file.name}") + + # Hint when 0 corrections and other domains have rules + if summary['total_changes'] == 0 and args.domain and domain_stats: + other = {d: n for d, n in domain_stats.items() if d != args.domain} + if other: + parts = ", ".join(f"{d} ({n})" for d, n in sorted(other.items())) + total = sum(other.values()) + print(f"hint: no rules in domain '{args.domain}'. Available: {parts}") + print(f"hint: run without --domain to use all {total} rules") + print() # Stage 2: AI corrections stage2_changes = [] diff --git a/transcript-fixer/scripts/core/correction_service.py b/transcript-fixer/scripts/core/correction_service.py index 4d53c50c..daf70cff 100644 --- a/transcript-fixer/scripts/core/correction_service.py +++ b/transcript-fixer/scripts/core/correction_service.py @@ -12,6 +12,7 @@ import re import os +import sys import logging from pathlib import Path from typing import Dict, List, Optional, Tuple @@ -23,6 +24,10 @@ DatabaseError ) +# Import safety check for common words +sys.path.insert(0, str(Path(__file__).parent.parent)) +from utils.common_words import check_correction_safety, audit_corrections, SafetyWarning + logger = logging.getLogger(__name__) @@ -178,10 +183,15 @@ def add_correction( domain: str = "general", source: str = "manual", confidence: float = 1.0, - notes: Optional[str] = None + notes: Optional[str] = None, + force: bool = False, ) -> int: """ - Add a correction with full validation. + Add a correction with full validation and safety checks. + + Safety checks detect common Chinese words and substring collision + risks that would cause false positives. Pass force=True to bypass + (errors become warnings printed to stderr). Args: from_text: Original (incorrect) text @@ -190,12 +200,13 @@ def add_correction( source: Origin of correction confidence: Confidence score notes: Optional notes + force: If True, downgrade safety errors to warnings Returns: ID of inserted correction Raises: - ValidationError: If validation fails + ValidationError: If validation or safety check fails """ # Comprehensive validation self.validate_correction_text(from_text, "from_text") @@ -210,6 +221,34 @@ def add_correction( f"from_text and to_text are identical: '{from_text}'" ) + # Safety check: detect common words and substring collisions + safety_warnings = check_correction_safety(from_text, to_text, strict=True) + + if safety_warnings: + errors = [w for w in safety_warnings if w.level == "error"] + warns = [w for w in safety_warnings if w.level == "warning"] + + if errors and not force: + # Block the addition + msg_parts = [] + for w in errors: + msg_parts.append(f"[{w.category}] {w.message}") + msg_parts.append(f" Suggestion: {w.suggestion}") + raise ValidationError( + f"Safety check BLOCKED adding '{from_text}' -> '{to_text}':\n" + + "\n".join(msg_parts) + + "\n\nUse --force to override (at your own risk)." + ) + + # Print warnings (errors downgraded by --force, or genuine warnings) + all_to_print = errors + warns if force else warns + if all_to_print: + for w in all_to_print: + prefix = "FORCED" if w.level == "error" else "WARNING" + logger.warning( + f"[{prefix}] [{w.category}] {w.message} | {w.suggestion}" + ) + # Get current user added_by = os.getenv("USER") or os.getenv("USERNAME") or "unknown" @@ -382,6 +421,14 @@ def export_corrections(self, domain: str = "general") -> Dict[str, str]: # ==================== Statistics and Reporting ==================== + def get_domain_stats(self) -> Dict[str, int]: + """Get count of active corrections per domain.""" + all_corrections = self.repository.get_all_corrections(active_only=True) + stats: Dict[str, int] = {} + for c in all_corrections: + stats[c.domain] = stats.get(c.domain, 0) + 1 + return stats + def get_statistics(self, domain: Optional[str] = None) -> Dict[str, any]: """ Get correction statistics. @@ -423,6 +470,31 @@ def get_statistics(self, domain: Optional[str] = None) -> Dict[str, any]: return stats + # ==================== Audit Operations ==================== + + def audit_dictionary( + self, + domain: Optional[str] = None, + ) -> Dict[str, List[SafetyWarning]]: + """ + Audit all active corrections for safety issues. + + Scans every rule and flags: + - from_text that is a common Chinese word (false positive risk) + - from_text that is <= 2 characters (high collision risk) + - from_text that appears as substring in common words (collateral damage) + - Both from_text and to_text being common words (bidirectional risk) + + Args: + domain: Optional domain filter (None = all domains) + + Returns: + Dict mapping from_text to list of SafetyWarnings. + Only entries with issues are included. + """ + corrections = self.get_corrections(domain) + return audit_corrections(corrections) + # ==================== Helper Methods ==================== def _detect_conflicts( diff --git a/transcript-fixer/scripts/core/dictionary_processor.py b/transcript-fixer/scripts/core/dictionary_processor.py index 15a95863..6a1907e7 100644 --- a/transcript-fixer/scripts/core/dictionary_processor.py +++ b/transcript-fixer/scripts/core/dictionary_processor.py @@ -14,9 +14,17 @@ from __future__ import annotations import re +import sys +import logging +from pathlib import Path from typing import Dict, List, Tuple from dataclasses import dataclass +sys.path.insert(0, str(Path(__file__).parent.parent)) +from utils.common_words import ALL_COMMON_WORDS + +logger = logging.getLogger(__name__) + @dataclass class Change: @@ -96,7 +104,16 @@ def _apply_context_rules(self, text: str) -> Tuple[str, List[Change]]: return corrected, changes def _apply_dictionary(self, text: str) -> Tuple[str, List[Change]]: - """Apply simple dictionary replacements""" + """ + Apply dictionary replacements with substring safety checks. + + Safety layers (applied in order at each match site): + 1. Superset check: if to_text already exists at the match position, + skip to prevent duplication (e.g., "金流"→"现金流" inside "现金流"). + This applies to ALL rules regardless of length. + 2. Boundary check (short rules only, <=3 chars): if the match is inside + a longer common word, skip to prevent collateral damage. + """ changes = [] corrected = text @@ -104,32 +121,167 @@ def _apply_dictionary(self, text: str) -> Tuple[str, List[Change]]: if wrong not in corrected: continue - # Find all occurrences - occurrences = [] - start = 0 - while True: - pos = corrected.find(wrong, start) - if pos == -1: - break - line_num = corrected[:pos].count('\n') + 1 - occurrences.append(line_num) - start = pos + len(wrong) - - # Track changes - for line_num in occurrences: - changes.append(Change( - line_number=line_num, - from_text=wrong, - to_text=correct, - rule_type="dictionary", - rule_name="corrections_dict" - )) - - # Apply replacement - corrected = corrected.replace(wrong, correct) + # All rules go through position-aware replacement to get + # the superset check. Short rules additionally get the + # boundary check against common words. + needs_boundary_check = len(wrong) <= 3 + corrected, new_changes = self._apply_with_safety_checks( + corrected, wrong, correct, needs_boundary_check, + ) + changes.extend(new_changes) return corrected, changes + def _find_occurrences(self, text: str, target: str) -> List[int]: + """Find all line numbers where target appears in text.""" + occurrences = [] + start = 0 + while True: + pos = text.find(target, start) + if pos == -1: + break + line_num = text[:pos].count('\n') + 1 + occurrences.append(line_num) + start = pos + len(target) + return occurrences + + def _apply_with_safety_checks( + self, + text: str, + wrong: str, + correct: str, + check_boundaries: bool, + ) -> Tuple[str, List[Change]]: + """ + Apply replacement at each match position with two safety layers: + + 1. Superset check (all rules): When to_text contains from_text + (e.g., "金流"→"现金流"), check if the surrounding text already + forms to_text. If so, skip — the text is already correct. + + 2. Boundary check (short rules only): Check if the match is inside + a longer common word (e.g., "天差" inside "天差地别"). + """ + changes = [] + result_parts = [] + search_start = 0 + + while search_start < len(text): + pos = text.find(wrong, search_start) + if pos == -1: + result_parts.append(text[search_start:]) + break + + # Safety layer 1: superset check. + # If to_text contains from_text, the replacement could create + # duplication. Check if to_text already exists at this position. + if self._already_corrected(text, pos, wrong, correct): + result_parts.append(text[search_start:pos + len(wrong)]) + search_start = pos + len(wrong) + logger.debug( + f"Skipped '{wrong}' at pos {pos}: " + f"already corrected ('{correct}' present)" + ) + continue + + # Safety layer 2: boundary check (short rules only). + if check_boundaries and self._is_inside_longer_word( + text, pos, wrong + ): + result_parts.append(text[search_start:pos + len(wrong)]) + search_start = pos + len(wrong) + logger.debug( + f"Skipped '{wrong}' at pos {pos}: part of longer word" + ) + continue + + # Safe to replace + line_num = text[:pos].count('\n') + 1 + changes.append(Change( + line_number=line_num, + from_text=wrong, + to_text=correct, + rule_type="dictionary", + rule_name="corrections_dict" + )) + + result_parts.append(text[search_start:pos]) + result_parts.append(correct) + search_start = pos + len(wrong) + + return "".join(result_parts), changes + + @staticmethod + def _already_corrected( + text: str, pos: int, from_text: str, to_text: str + ) -> bool: + """ + Check if to_text already exists at the match position, meaning + the text is already in the corrected form. + + This catches the case where from_text is a substring of to_text + (e.g., "金流" is inside "现金流"). If the surrounding text already + forms "现金流", replacing "金流" would produce "现现金流". + + Returns True if the replacement should be skipped. + """ + if from_text not in to_text: + # to_text doesn't contain from_text, so no superset risk. + return False + + to_len = len(to_text) + from_len = len(from_text) + + # Find all positions where from_text appears inside to_text. + # For each, check if the surrounding text matches to_text. + offset = 0 + while True: + idx = to_text.find(from_text, offset) + if idx == -1: + break + + # If to_text were at text position (pos - idx), from_text at pos + # would be the substring starting at idx within to_text. + candidate_start = pos - idx + candidate_end = candidate_start + to_len + + if (candidate_start >= 0 + and candidate_end <= len(text) + and text[candidate_start:candidate_end] == to_text): + return True + + offset = idx + 1 + + return False + + @staticmethod + def _is_inside_longer_word(text: str, pos: int, match: str) -> bool: + """ + Check if the match at `pos` is embedded inside a longer common word. + + Looks at a window around the match and checks all possible substrings + of that window against the common words set. + """ + match_len = len(match) + # Check windows of 2 to 5 characters that overlap with the match + max_word_len = 5 + window_start = max(0, pos - (max_word_len - 1)) + window_end = min(len(text), pos + match_len + (max_word_len - 1)) + window = text[window_start:window_end] + + # Position of the match within the window + match_offset = pos - window_start + + # Check all substrings that contain the match position + for length in range(match_len + 1, min(max_word_len + 1, len(window) + 1)): + for start in range(max(0, match_offset + match_len - length), + min(match_offset + 1, len(window) - length + 1)): + substr = window[start:start + length] + if substr != match and substr in ALL_COMMON_WORDS: + return True + + return False + def get_summary(self, changes: List[Change]) -> Dict[str, int]: """Generate summary statistics""" summary = { diff --git a/transcript-fixer/scripts/fix_transcript_timestamps.py b/transcript-fixer/scripts/fix_transcript_timestamps.py new file mode 100644 index 00000000..17754d29 --- /dev/null +++ b/transcript-fixer/scripts/fix_transcript_timestamps.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +"""Normalize and repair speaker timestamp lines in ASR transcripts. + +This script targets transcript lines shaped like: + + 天生 00:21 + Speaker 11 01:31:10 + +Common fixes: +- Normalize mixed `MM:SS` / `HH:MM:SS` into one format +- Repair hour rollovers for `MM:SS` timestamps after long recordings +- Optionally rebase a split transcript so it starts at 00:00:00 +- Report suspicious backward jumps instead of silently rewriting them +""" + +from __future__ import annotations + +import argparse +import re +import sys +from dataclasses import dataclass +from pathlib import Path + + +SPEAKER_TS_RE = re.compile(r"^(?P.+?)\s+(?P\d{1,2}:\d{2}(?::\d{2})?)\s*$") + + +@dataclass +class TimestampEvent: + line_no: int + original: str + fixed: str + reason: str + + +@dataclass +class RepairResult: + changed_lines: list[TimestampEvent] + anomalies: list[TimestampEvent] + repaired_text: str + matched_count: int + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Repair speaker timestamp lines in transcript files." + ) + parser.add_argument("input", help="Input transcript file") + parser.add_argument( + "--output", + help="Output path. Defaults to _timestamps_fixed when not using --check.", + ) + parser.add_argument( + "--in-place", + action="store_true", + help="Overwrite the input file", + ) + parser.add_argument( + "--check", + action="store_true", + help="Only analyze and report issues without writing a file", + ) + parser.add_argument( + "--format", + choices=("hhmmss", "preserve"), + default="hhmmss", + help="Timestamp output format. Default: hhmmss", + ) + parser.add_argument( + "--rollover-backjump-seconds", + type=int, + default=15 * 60, + help=( + "Backward jump threshold for treating MM:SS as a new hour rollover. " + "Default: 900" + ), + ) + parser.add_argument( + "--jitter-seconds", + type=int, + default=5, + help="Allowed small backward jitter before flagging an anomaly. Default: 5", + ) + parser.add_argument( + "--rebase-to-zero", + action="store_true", + help="Rebase the first detected speaker timestamp in the file to 00:00:00.", + ) + return parser.parse_args() + + +def to_seconds(parts: list[int]) -> int: + if len(parts) == 2: + minutes, seconds = parts + return minutes * 60 + seconds + hours, minutes, seconds = parts + return hours * 3600 + minutes * 60 + seconds + + +def format_timestamp(total_seconds: int, output_format: str) -> str: + hours = total_seconds // 3600 + minutes = (total_seconds % 3600) // 60 + seconds = total_seconds % 60 + + if output_format == "preserve" and hours == 0: + return f"{minutes:02d}:{seconds:02d}" + return f"{hours:02d}:{minutes:02d}:{seconds:02d}" + + +def repair_timestamps( + text: str, + *, + output_format: str, + rollover_backjump_seconds: int, + jitter_seconds: int, + rebase_to_zero: bool, +) -> RepairResult: + lines = text.splitlines(keepends=True) + repaired_lines: list[str] = [] + changed_lines: list[TimestampEvent] = [] + anomalies: list[TimestampEvent] = [] + + current_hour = 0 + last_abs_seconds: int | None = None + first_abs_seconds: int | None = None + matched_count = 0 + + for line_no, line in enumerate(lines, start=1): + stripped = line.rstrip("\r\n") + newline = line[len(stripped) :] + match = SPEAKER_TS_RE.match(stripped) + + if not match: + repaired_lines.append(line) + continue + + matched_count += 1 + prefix = match.group("prefix") + raw_ts = match.group("ts") + parts = [int(part) for part in raw_ts.split(":")] + + if len(parts) == 3: + abs_seconds = to_seconds(parts) + current_hour = parts[0] + + if ( + last_abs_seconds is not None + and abs_seconds + jitter_seconds < last_abs_seconds + ): + anomalies.append( + TimestampEvent( + line_no=line_no, + original=raw_ts, + fixed=format_timestamp(abs_seconds, output_format), + reason="explicit_hhmmss_backwards", + ) + ) + else: + minutes, seconds = parts + candidate = current_hour * 3600 + minutes * 60 + seconds + + if last_abs_seconds is not None: + while candidate + rollover_backjump_seconds < last_abs_seconds: + current_hour += 1 + candidate = current_hour * 3600 + minutes * 60 + seconds + + if candidate + jitter_seconds < last_abs_seconds: + anomalies.append( + TimestampEvent( + line_no=line_no, + original=raw_ts, + fixed=format_timestamp(candidate, output_format), + reason="small_backjump_after_rollover_check", + ) + ) + + abs_seconds = candidate + + if first_abs_seconds is None: + first_abs_seconds = abs_seconds + + display_seconds = ( + abs_seconds - first_abs_seconds + if rebase_to_zero and first_abs_seconds is not None + else abs_seconds + ) + + fixed_ts = format_timestamp(display_seconds, output_format) + repaired_lines.append(f"{prefix} {fixed_ts}{newline}") + + if fixed_ts != raw_ts: + changed_lines.append( + TimestampEvent( + line_no=line_no, + original=raw_ts, + fixed=fixed_ts, + reason="normalized_or_rollover_fixed", + ) + ) + + last_abs_seconds = abs_seconds + + return RepairResult( + changed_lines=changed_lines, + anomalies=anomalies, + repaired_text="".join(repaired_lines), + matched_count=matched_count, + ) + + +def default_output_path(input_path: Path) -> Path: + return input_path.with_name( + f"{input_path.stem}_timestamps_fixed{input_path.suffix}" + ) + + +def print_summary(result: RepairResult) -> None: + print(f"Matched speaker timestamp lines: {result.matched_count}") + print(f"Rewritten timestamp lines: {len(result.changed_lines)}") + print(f"Anomalies flagged: {len(result.anomalies)}") + + if result.changed_lines: + print("\nChanged lines:") + for event in result.changed_lines[:20]: + print( + f" L{event.line_no}: {event.original} -> {event.fixed} " + f"({event.reason})" + ) + if len(result.changed_lines) > 20: + print(f" ... {len(result.changed_lines) - 20} more") + + if result.anomalies: + print("\nAnomalies:") + for event in result.anomalies[:20]: + print( + f" L{event.line_no}: {event.original} -> {event.fixed} " + f"({event.reason})" + ) + if len(result.anomalies) > 20: + print(f" ... {len(result.anomalies) - 20} more") + + +def main() -> int: + args = parse_args() + input_path = Path(args.input).expanduser().resolve() + + if not input_path.exists(): + print(f"Input file not found: {input_path}", file=sys.stderr) + return 1 + + if args.check and args.in_place: + print("--check and --in-place cannot be used together", file=sys.stderr) + return 1 + + text = input_path.read_text(encoding="utf-8") + result = repair_timestamps( + text, + output_format=args.format, + rollover_backjump_seconds=args.rollover_backjump_seconds, + jitter_seconds=args.jitter_seconds, + rebase_to_zero=args.rebase_to_zero, + ) + + print_summary(result) + + if args.check: + return 0 if not result.anomalies else 2 + + if args.in_place: + output_path = input_path + elif args.output: + output_path = Path(args.output).expanduser().resolve() + else: + output_path = default_output_path(input_path) + + output_path.write_text(result.repaired_text, encoding="utf-8") + print(f"\nWrote repaired transcript: {output_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/transcript-fixer/scripts/fix_transcription.py b/transcript-fixer/scripts/fix_transcription.py index 6e63379b..323ba17d 100755 --- a/transcript-fixer/scripts/fix_transcription.py +++ b/transcript-fixer/scripts/fix_transcription.py @@ -31,6 +31,7 @@ from cli import ( cmd_init, cmd_add_correction, + cmd_audit, cmd_list_corrections, cmd_run_correction, cmd_review_learned, @@ -89,6 +90,8 @@ def main() -> None: elif args.add_correction: args.from_text, args.to_text = args.add_correction cmd_add_correction(args) + elif getattr(args, 'audit_dictionary', False): + cmd_audit(args) elif args.list_corrections: cmd_list_corrections(args) elif args.review_learned: diff --git a/transcript-fixer/scripts/split_transcript_sections.py b/transcript-fixer/scripts/split_transcript_sections.py new file mode 100644 index 00000000..89733ba7 --- /dev/null +++ b/transcript-fixer/scripts/split_transcript_sections.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Split a transcript into named sections and optionally rebase timestamps. + +Example: + uv run scripts/split_transcript_sections.py meeting.txt \ + --first-section-name "课前聊天" \ + --section "正式上课::好，无缝切换嘛。对。那个曹总连上了吗？那个网页。" \ + --section "课后复盘::我们复盘一下。" \ + --rebase-to-zero +""" + +from __future__ import annotations + +import argparse +import re +import sys +from dataclasses import dataclass +from pathlib import Path + +from fix_transcript_timestamps import repair_timestamps + + +INVALID_FILENAME_CHARS = re.compile(r'[\\/:*?"<>|]+') + + +@dataclass +class SectionSpec: + name: str + marker: str + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Split a transcript into named sections using marker phrases." + ) + parser.add_argument("input", help="Input transcript file") + parser.add_argument( + "--first-section-name", + default="part-1", + help="Name for the section before the first marker. Default: part-1", + ) + parser.add_argument( + "--section", + action="append", + required=True, + metavar="NAME::MARKER", + help=( + "Section boundary definition. Split starts at the first occurrence of MARKER " + "and names that segment NAME." + ), + ) + parser.add_argument( + "--output-dir", + help="Directory for output files. Defaults to the input file directory.", + ) + parser.add_argument( + "--rebase-to-zero", + action="store_true", + help="Rebase each output section so its first speaker timestamp becomes 00:00:00.", + ) + parser.add_argument( + "--format", + choices=("hhmmss", "preserve"), + default="hhmmss", + help="Timestamp output format when rebasing. Default: hhmmss", + ) + parser.add_argument( + "--rollover-backjump-seconds", + type=int, + default=15 * 60, + help="Backward jump threshold for MM:SS hour rollover repair. Default: 900", + ) + parser.add_argument( + "--jitter-seconds", + type=int, + default=5, + help="Allowed small backward jitter before flagging an anomaly. Default: 5", + ) + return parser.parse_args() + + +def parse_section_arg(value: str) -> SectionSpec: + if "::" not in value: + raise ValueError(f"Invalid --section value: {value!r}") + name, marker = value.split("::", 1) + name = name.strip() + marker = marker.strip() + if not name or not marker: + raise ValueError(f"Invalid --section value: {value!r}") + return SectionSpec(name=name, marker=marker) + + +def sanitize_filename_component(name: str) -> str: + return INVALID_FILENAME_CHARS.sub("-", name).strip().replace(" ", "-") + + +def split_text_by_markers( + text: str, + *, + first_section_name: str, + sections: list[SectionSpec], +) -> list[tuple[str, str]]: + boundaries: list[tuple[int, SectionSpec]] = [] + search_from = 0 + + for spec in sections: + idx = text.find(spec.marker, search_from) + if idx == -1: + raise ValueError(f"Marker not found for section {spec.name!r}: {spec.marker!r}") + boundaries.append((idx, spec)) + search_from = idx + 1 + + result: list[tuple[str, str]] = [] + prev_idx = 0 + prev_name = first_section_name + + for idx, spec in boundaries: + result.append((prev_name, text[prev_idx:idx].rstrip() + "\n")) + prev_idx = idx + prev_name = spec.name + + result.append((prev_name, text[prev_idx:].rstrip() + "\n")) + return result + + +def maybe_rebase( + text: str, + *, + rebase_to_zero: bool, + output_format: str, + rollover_backjump_seconds: int, + jitter_seconds: int, +) -> str: + if not rebase_to_zero: + return text + result = repair_timestamps( + text, + output_format=output_format, + rollover_backjump_seconds=rollover_backjump_seconds, + jitter_seconds=jitter_seconds, + rebase_to_zero=True, + ) + return result.repaired_text + + +def main() -> int: + args = parse_args() + input_path = Path(args.input).expanduser().resolve() + + if not input_path.exists(): + print(f"Input file not found: {input_path}", file=sys.stderr) + return 1 + + try: + sections = [parse_section_arg(value) for value in args.section] + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 1 + + text = input_path.read_text(encoding="utf-8") + + try: + parts = split_text_by_markers( + text, + first_section_name=args.first_section_name, + sections=sections, + ) + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 1 + + output_dir = ( + Path(args.output_dir).expanduser().resolve() + if args.output_dir + else input_path.parent + ) + output_dir.mkdir(parents=True, exist_ok=True) + + for idx, (name, content) in enumerate(parts, start=1): + content = maybe_rebase( + content, + rebase_to_zero=args.rebase_to_zero, + output_format=args.format, + rollover_backjump_seconds=args.rollover_backjump_seconds, + jitter_seconds=args.jitter_seconds, + ) + safe_name = sanitize_filename_component(name) + output_path = output_dir / f"{input_path.stem}-{idx:02d}-{safe_name}{input_path.suffix}" + output_path.write_text(content, encoding="utf-8") + print(f"{output_path}\t{len(content.splitlines())} lines") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/transcript-fixer/scripts/tests/test_common_words_safety.py b/transcript-fixer/scripts/tests/test_common_words_safety.py new file mode 100644 index 00000000..3d4e205c --- /dev/null +++ b/transcript-fixer/scripts/tests/test_common_words_safety.py @@ -0,0 +1,675 @@ +#!/usr/bin/env python3 +""" +Tests for common word safety checks and boundary-aware replacement. + +Covers the three classes of production bugs: +1. Common words added as corrections cause false positives +2. Substring matching causes collateral damage +3. Short common words should never be dictionary entries +""" + +import unittest +import tempfile +import shutil +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from utils.common_words import ( + check_correction_safety, + audit_corrections, + SafetyWarning, + ALL_COMMON_WORDS, + COMMON_WORDS_2CHAR, + SUBSTRING_COLLISION_MAP, +) +from core.dictionary_processor import DictionaryProcessor +from core.correction_repository import CorrectionRepository +from core.correction_service import CorrectionService, ValidationError + + +class TestSafetyChecks(unittest.TestCase): + """Test the check_correction_safety function.""" + + def test_common_word_blocked_strict(self): + """Adding a common word like '仿佛' should produce an error in strict mode.""" + warnings = check_correction_safety("仿佛", "反复", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "Expected at least one error for '仿佛'") + self.assertTrue( + any(w.category == "common_word" for w in errors), + "Expected 'common_word' category", + ) + + def test_common_word_warning_nonstrict(self): + """In non-strict mode, common words produce warnings, not errors.""" + warnings = check_correction_safety("仿佛", "反复", strict=False) + errors = [w for w in warnings if w.level == "error"] + self.assertEqual(len(errors), 0, "Non-strict mode should have no errors") + warns = [w for w in warnings if w.level == "warning"] + self.assertTrue(len(warns) > 0, "Expected at least one warning") + + def test_both_common_words_flagged(self): + """When both from_text and to_text are common words, flag with 'both_common'.""" + warnings = check_correction_safety("正面", "正念", strict=True) + both = [w for w in warnings if w.category == "both_common"] + # "正面" is common, "正念" may or may not be -- but at least common_word should fire + common = [w for w in warnings if w.category == "common_word"] + self.assertTrue(len(common) > 0) + + def test_short_text_warning(self): + """2-char text not in common words list still gets a short_text warning.""" + # Use something unlikely to be in the common words list + warnings = check_correction_safety("zz", "xx", strict=True) + short_warns = [w for w in warnings if w.category == "short_text"] + self.assertTrue(len(short_warns) > 0, "Expected short_text warning for 2-char text") + + def test_known_substring_collision(self): + """'线数' is in SUBSTRING_COLLISION_MAP and should trigger collision warning.""" + warnings = check_correction_safety("线数", "线束", strict=True) + collisions = [w for w in warnings if w.category == "substring_collision"] + self.assertTrue(len(collisions) > 0, "Expected substring_collision for '线数'") + + def test_safe_correction_no_warnings(self): + """A safe, domain-specific correction should produce no warnings.""" + # "巨升智能" -> "具身智能" is a genuine ASR error, not a common word + warnings = check_correction_safety("巨升智能", "具身智能", strict=True) + self.assertEqual(len(warnings), 0, f"Expected no warnings, got: {warnings}") + + def test_long_from_text_safe(self): + """Long from_text (>4 chars) should not trigger short text or collision warnings.""" + warnings = check_correction_safety("语音识别错误", "语音识别模型", strict=True) + short_warns = [w for w in warnings if w.category == "short_text"] + self.assertEqual(len(short_warns), 0) + + # --- Production false positives from the bug report --- + + def test_production_false_positive_fangfu(self): + """'仿佛→反复' was a real production false positive.""" + warnings = check_correction_safety("仿佛", "反复", strict=True) + self.assertTrue(len(warnings) > 0) + + def test_production_false_positive_zhengmian(self): + """'正面→正念' was a real production false positive.""" + warnings = check_correction_safety("正面", "正念", strict=True) + self.assertTrue(len(warnings) > 0) + + def test_production_false_positive_youyu(self): + """'犹豫→抑郁' was a real production false positive.""" + warnings = check_correction_safety("犹豫", "抑郁", strict=True) + self.assertTrue(len(warnings) > 0) + + def test_production_false_positive_chuanshuo(self): + """'传说→穿梭' was a real production false positive.""" + warnings = check_correction_safety("传说", "穿梭", strict=True) + self.assertTrue(len(warnings) > 0) + + def test_production_false_positive_yanji(self): + """'演技→眼界' was a real production false positive.""" + warnings = check_correction_safety("演技", "眼界", strict=True) + self.assertTrue(len(warnings) > 0) + + def test_production_false_positive_zengjia(self): + """'增加→工站/环节' was a real production false positive.""" + warnings = check_correction_safety("增加", "工站", strict=True) + self.assertTrue(len(warnings) > 0) + + +class TestAuditCorrections(unittest.TestCase): + """Test the audit_corrections function.""" + + def test_audit_finds_known_bad_rules(self): + """Audit should flag the production false positives.""" + corrections = { + "仿佛": "反复", + "正面": "正念", + "线数": "线束", + "巨升智能": "具身智能", # This one is fine + } + issues = audit_corrections(corrections) + + self.assertIn("仿佛", issues) + self.assertIn("正面", issues) + self.assertIn("线数", issues) + self.assertNotIn("巨升智能", issues) + + def test_audit_empty_dict(self): + """Audit of empty dict returns empty.""" + issues = audit_corrections({}) + self.assertEqual(len(issues), 0) + + +class TestBoundaryAwareReplacement(unittest.TestCase): + """Test DictionaryProcessor's boundary-aware replacement logic.""" + + def test_substring_collision_prevented(self): + """'线数→线束' should NOT match inside '产线数据'.""" + processor = DictionaryProcessor({"线数": "线束"}, []) + text = "这条产线数据很重要" + result, changes = processor.process(text) + self.assertEqual(result, "这条产线数据很重要", + "Should NOT replace '线数' inside '产线数据'") + self.assertEqual(len(changes), 0) + + def test_standalone_match_replaced(self): + """'线数→线束' SHOULD match when it's standalone (not inside a longer word).""" + processor = DictionaryProcessor({"线数": "线束"}, []) + text = "检查线数是否正确" + result, changes = processor.process(text) + # "线数" here is standalone (not inside a common word), + # so it should be replaced + self.assertEqual(result, "检查线束是否正确") + self.assertEqual(len(changes), 1) + + def test_long_correction_not_affected(self): + """Corrections longer than 3 chars use standard replacement.""" + processor = DictionaryProcessor({"巨升智能": "具身智能"}, []) + text = "今天讨论巨升智能的进展" + result, changes = processor.process(text) + self.assertEqual(result, "今天讨论具身智能的进展") + self.assertEqual(len(changes), 1) + + def test_multiple_replacements_mixed(self): + """Mix of safe and unsafe positions should be handled correctly.""" + processor = DictionaryProcessor({"数据": "数据集"}, []) + text = "大数据分析和数据清洗" + result, changes = processor.process(text) + # "数据" inside "大数据" should be protected + # "数据" standalone should be replaced + # Both are common words, so boundary check applies + # The exact behavior depends on what's in ALL_COMMON_WORDS + # At minimum, the processor should not crash + self.assertIsInstance(result, str) + + def test_no_corrections_no_changes(self): + """Empty corrections dict produces no changes.""" + processor = DictionaryProcessor({}, []) + text = "原始文本" + result, changes = processor.process(text) + self.assertEqual(result, "原始文本") + self.assertEqual(len(changes), 0) + + def test_context_rules_still_work(self): + """Context rules (regex) are unaffected by boundary checks.""" + context_rules = [{ + "pattern": r"股价系统", + "replacement": "框架系统", + "description": "ASR error fix" + }] + processor = DictionaryProcessor({}, context_rules) + text = "股价系统需要优化" + result, changes = processor.process(text) + self.assertEqual(result, "框架系统需要优化") + self.assertEqual(len(changes), 1) + + +class TestSupersetReplacementBug(unittest.TestCase): + """ + Bug 1: When to_text contains from_text as a substring, and the + surrounding text already forms to_text, the replacement must be skipped. + + Production example: rule "金流"→"现金流", input "现金流断了" + Without fix: "现现金流断了" (WRONG -- duplicated prefix) + With fix: "现金流断了" (correct -- already in target form) + + This check must work for ALL rule lengths, not just short rules. + """ + + def test_suffix_superset_skip(self): + """from_text is a suffix of to_text: 金流→现金流 inside 现金流.""" + processor = DictionaryProcessor({"金流": "现金流"}, []) + result, changes = processor.process("现金流断了") + self.assertEqual(result, "现金流断了") + self.assertEqual(len(changes), 0) + + def test_suffix_superset_standalone_replaced(self): + """Standalone from_text should still be replaced.""" + processor = DictionaryProcessor({"金流": "现金流"}, []) + result, changes = processor.process("金流断了") + self.assertEqual(result, "现金流断了") + self.assertEqual(len(changes), 1) + + def test_prefix_superset_skip(self): + """from_text is a prefix of to_text: 现金→现金流 inside 现金流.""" + processor = DictionaryProcessor({"现金": "现金流"}, []) + result, changes = processor.process("现金流断了") + self.assertEqual(result, "现金流断了") + self.assertEqual(len(changes), 0) + + def test_middle_superset_skip(self): + """from_text is in the middle of to_text.""" + processor = DictionaryProcessor({"金流": "现金流通"}, []) + result, changes = processor.process("现金流通畅") + self.assertEqual(result, "现金流通畅") + self.assertEqual(len(changes), 0) + + def test_long_rule_superset_skip(self): + """Superset check must also work for long rules (>3 chars).""" + processor = DictionaryProcessor({"金流断裂": "现金流断裂"}, []) + result, changes = processor.process("现金流断裂了") + self.assertEqual(result, "现金流断裂了") + self.assertEqual(len(changes), 0) + + def test_long_rule_superset_standalone_replaced(self): + """Long rule standalone should still be replaced.""" + processor = DictionaryProcessor({"金流断裂": "现金流断裂"}, []) + result, changes = processor.process("金流断裂了") + self.assertEqual(result, "现金流断裂了") + self.assertEqual(len(changes), 1) + + def test_superset_with_unknown_words(self): + """Superset check works regardless of common_words membership.""" + # Use words NOT in ALL_COMMON_WORDS + processor = DictionaryProcessor({"资流": "投资流"}, []) + result, changes = processor.process("投资流断了") + self.assertEqual(result, "投资流断了") + self.assertEqual(len(changes), 0) + + def test_superset_mixed_positions(self): + """One occurrence is already correct, another is standalone.""" + processor = DictionaryProcessor({"金流": "现金流"}, []) + result, changes = processor.process("现金流好，金流差") + self.assertEqual(result, "现金流好，现金流差") + self.assertEqual(len(changes), 1) + + def test_no_superset_normal_replacement(self): + """When to_text does NOT contain from_text, normal replacement.""" + processor = DictionaryProcessor({"金流": "资金链"}, []) + result, changes = processor.process("金流断了") + self.assertEqual(result, "资金链断了") + self.assertEqual(len(changes), 1) + + +class TestIdiomCompoundProtection(unittest.TestCase): + """ + Bug 2: Short rules must not corrupt idioms and compound words. + + Production examples: + - "天差"→"偏差" inside "天差地别" => "偏差地别" (broken idiom) + - "亮亮"→"亮哥" inside "漂漂亮亮" => "漂漂亮哥" (broken phrase) + + Defense: _is_inside_longer_word checks common_words set. + """ + + def test_tiancha_inside_idiom(self): + """天差→偏差 must not break 天差地别.""" + processor = DictionaryProcessor({"天差": "偏差"}, []) + result, changes = processor.process("天差地别") + self.assertEqual(result, "天差地别") + self.assertEqual(len(changes), 0) + + def test_liangliang_inside_compound(self): + """亮亮→亮哥 must not break 漂漂亮亮.""" + processor = DictionaryProcessor({"亮亮": "亮哥"}, []) + result, changes = processor.process("漂漂亮亮") + self.assertEqual(result, "漂漂亮亮") + self.assertEqual(len(changes), 0) + + def test_tiancha_standalone_replaced(self): + """Standalone 天差 (not inside idiom) should be replaced.""" + processor = DictionaryProcessor({"天差": "偏差"}, []) + # 天差 alone, not followed by 地别 or other idiom continuation + result, changes = processor.process("误差天差太大了") + # Whether this gets replaced depends on common_words; at minimum + # it should not crash. If 天差 is in common words, it stays. + self.assertIsInstance(result, str) + + +class TestValidPhraseProtection(unittest.TestCase): + """ + Bug 3: Short rules must not corrupt valid phrases where from_text + is a legitimate substring. + + Production example: + - "被看"→"被砍" inside "被看见" => "被砍见" + + Defense: _is_inside_longer_word checks common_words set. + """ + + def test_beikan_inside_beikanjian(self): + """被看→被砍 must not break 被看见.""" + processor = DictionaryProcessor({"被看": "被砍"}, []) + result, changes = processor.process("被看见") + self.assertEqual(result, "被看见") + self.assertEqual(len(changes), 0) + + def test_beikan_in_sentence(self): + """被看→被砍 must not break 被看见 in a full sentence.""" + processor = DictionaryProcessor({"被看": "被砍"}, []) + result, changes = processor.process("他被看见了") + self.assertEqual(result, "他被看见了") + self.assertEqual(len(changes), 0) + + +class TestServiceSafetyIntegration(unittest.TestCase): + """Integration tests: CorrectionService rejects unsafe corrections.""" + + def setUp(self): + self.test_dir = Path(tempfile.mkdtemp()) + self.db_path = self.test_dir / "test.db" + self.repository = CorrectionRepository(self.db_path) + self.service = CorrectionService(self.repository) + + def tearDown(self): + self.service.close() + shutil.rmtree(self.test_dir) + + def test_common_word_rejected(self): + """Adding a common word correction is blocked by default.""" + with self.assertRaises(ValidationError) as ctx: + self.service.add_correction("仿佛", "反复", "general") + self.assertIn("Safety check BLOCKED", str(ctx.exception)) + + def test_common_word_forced(self): + """Adding a common word with force=True succeeds.""" + correction_id = self.service.add_correction( + "仿佛", "反复", "general", force=True, + ) + self.assertIsInstance(correction_id, int) + self.assertGreater(correction_id, 0) + + def test_safe_correction_accepted(self): + """A genuine ASR correction is accepted without force.""" + correction_id = self.service.add_correction( + "巨升智能", "具身智能", "general", + ) + self.assertIsInstance(correction_id, int) + + def test_audit_on_service(self): + """audit_dictionary method returns issues for unsafe rules.""" + # Force-add some unsafe rules + self.service.add_correction("仿佛", "反复", "general", force=True) + self.service.add_correction("巨升智能", "具身智能", "general") + + issues = self.service.audit_dictionary("general") + self.assertIn("仿佛", issues) + self.assertNotIn("巨升智能", issues) + + +class TestProductionFalsePositivesCoverage(unittest.TestCase): + """ + Verify ALL production false positives from the 2026-03 manual review + are present in the safety system and correctly caught. + + Each test corresponds to a specific word that caused real damage in production. + If any of these tests fail, it means the safety net has a gap. + """ + + # --- Category 1: Lifestyle domain --- + + def test_baojian_blocked(self): + """'保健' (lifestyle/beauty) must be caught.""" + self.assertIn("保健", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("保健", "宝剑", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'保健' must produce an error") + + def test_neihan_blocked(self): + """'内涵' (lifestyle/beauty) must be caught.""" + self.assertIn("内涵", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("内涵", "内含", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'内涵' must produce an error") + + def test_zhengjing_blocked(self): + """'正经' (lifestyle) must be caught.""" + self.assertIn("正经", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("正经", "正劲", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'正经' must produce an error") + + # --- Category 1: Manufacturing domain --- + + def test_jingong_blocked(self): + """'仅供' (manufacturing) must be caught.""" + self.assertIn("仅供", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("仅供", "紧供", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'仅供' must produce an error") + + def test_gongqi_blocked(self): + """'供气' (manufacturing) must be caught.""" + self.assertIn("供气", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("供气", "工器", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'供气' must produce an error") + + def test_chutou_blocked(self): + """'出头' (manufacturing) must be caught.""" + self.assertIn("出头", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("出头", "初投", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'出头' must produce an error") + + def test_jikou_blocked(self): + """'几口' (manufacturing) must be caught.""" + self.assertIn("几口", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("几口", "集口", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'几口' must produce an error") + + # --- Category 1: Various domains --- + + def test_liangben_blocked(self): + """'两本' must be caught.""" + self.assertIn("两本", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("两本", "量本", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'两本' must produce an error") + + def test_chuwu_blocked(self): + """'初五' must be caught.""" + self.assertIn("初五", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("初五", "出误", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'初五' must produce an error") + + def test_lijie_blocked(self): + """'力竭' must be caught.""" + self.assertIn("力竭", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("力竭", "立杰", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'力竭' must produce an error") + + def test_chongyu_blocked(self): + """'充于' must be caught.""" + self.assertIn("充于", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("充于", "冲余", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'充于' must produce an error") + + def test_shuju_blocked(self): + """'数据' must be caught.""" + self.assertIn("数据", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("数据", "束据", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'数据' must produce an error") + + # --- Category 1: Substring collision sources --- + + def test_beikan_blocked(self): + """'被看' (general) must be caught.""" + self.assertIn("被看", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("被看", "被砍", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'被看' must produce an error") + + def test_tiancha_blocked(self): + """'天差' (education) must be caught.""" + self.assertIn("天差", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("天差", "偏差", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'天差' must produce an error") + + def test_liangliang_blocked(self): + """'亮亮' (manufacturing) must be caught.""" + self.assertIn("亮亮", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("亮亮", "亮哥", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'亮亮' must produce an error") + + def test_jinliu_blocked(self): + """'金流' (manufacturing) must be caught.""" + self.assertIn("金流", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("金流", "现金流", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'金流' must produce an error") + + # --- Category 1: Substring issue sources --- + + def test_kanjian_blocked(self): + """'看见' must be caught (caused substring issues).""" + self.assertIn("看见", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("看见", "砍件", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'看见' must produce an error") + + def test_fenzhong_blocked(self): + """'分钟' must be caught (caused substring issues).""" + self.assertIn("分钟", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("分钟", "份种", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'分钟' must produce an error") + + +class TestSubstringCollisionMapCoverage(unittest.TestCase): + """ + Verify all production substring collision patterns are in the map. + + Each test reproduces a real corruption pattern from production: + a short word matched inside a longer valid phrase and corrupted it. + """ + + def test_xianshu_collision_exists(self): + """'线数' inside '产线数据' -> corrupts to '产线束据'.""" + self.assertIn("线数", SUBSTRING_COLLISION_MAP) + self.assertIn("产线数据", SUBSTRING_COLLISION_MAP["线数"]) + + def test_jinliu_collision_exists(self): + """'金流' inside '现金流' -> corrupts to '现现金流'.""" + self.assertIn("金流", SUBSTRING_COLLISION_MAP) + self.assertIn("现金流", SUBSTRING_COLLISION_MAP["金流"]) + + def test_beikan_collision_exists(self): + """'被看' inside '被看见' -> corrupts to '被砍见'.""" + self.assertIn("被看", SUBSTRING_COLLISION_MAP) + self.assertIn("被看见", SUBSTRING_COLLISION_MAP["被看"]) + + def test_liangliang_collision_exists(self): + """'亮亮' inside '漂漂亮亮' -> corrupts to '漂漂亮哥'.""" + self.assertIn("亮亮", SUBSTRING_COLLISION_MAP) + self.assertIn("漂漂亮亮", SUBSTRING_COLLISION_MAP["亮亮"]) + + def test_tiancha_collision_exists(self): + """'天差' inside '天差地别' -> corrupts idiom to '偏差地别'.""" + self.assertIn("天差", SUBSTRING_COLLISION_MAP) + self.assertIn("天差地别", SUBSTRING_COLLISION_MAP["天差"]) + + def test_collision_safety_check_fires(self): + """check_correction_safety must flag entries in SUBSTRING_COLLISION_MAP.""" + for short_word in ["金流", "被看", "亮亮", "天差"]: + warnings = check_correction_safety(short_word, "dummy", strict=True) + collision_warnings = [ + w for w in warnings if w.category == "substring_collision" + ] + self.assertTrue( + len(collision_warnings) > 0, + f"'{short_word}' must trigger substring_collision warning", + ) + + +class TestBoundaryAwareProductionCollisions(unittest.TestCase): + """ + End-to-end tests: verify DictionaryProcessor does NOT corrupt + longer valid phrases when a short correction matches inside them. + + Each test reproduces an exact production corruption scenario. + """ + + def test_jinliu_inside_xianjinliu(self): + """'金流→现金流' must NOT corrupt '现金流' to '现现金流'.""" + processor = DictionaryProcessor({"金流": "现金流"}, []) + text = "公司的现金流很健康" + result, changes = processor.process(text) + self.assertEqual(result, "公司的现金流很健康", + "Must NOT replace '金流' inside '现金流'") + self.assertEqual(len(changes), 0) + + def test_beikan_inside_beikanjian(self): + """'被看→被砍' must NOT corrupt '被看见' to '被砍见'.""" + processor = DictionaryProcessor({"被看": "被砍"}, []) + text = "他被看见了" + result, changes = processor.process(text) + self.assertEqual(result, "他被看见了", + "Must NOT replace '被看' inside '被看见'") + self.assertEqual(len(changes), 0) + + def test_liangliang_inside_piaopiaoliangliag(self): + """'亮亮→亮哥' must NOT corrupt '漂漂亮亮' to '漂漂亮哥'.""" + processor = DictionaryProcessor({"亮亮": "亮哥"}, []) + text = "打扮得漂漂亮亮的" + result, changes = processor.process(text) + self.assertEqual(result, "打扮得漂漂亮亮的", + "Must NOT replace '亮亮' inside '漂漂亮亮'") + self.assertEqual(len(changes), 0) + + def test_tiancha_inside_tianchadiebie(self): + """'天差→偏差' must NOT corrupt '天差地别' to '偏差地别'.""" + processor = DictionaryProcessor({"天差": "偏差"}, []) + text = "两者天差地别" + result, changes = processor.process(text) + self.assertEqual(result, "两者天差地别", + "Must NOT replace '天差' inside '天差地别'") + self.assertEqual(len(changes), 0) + + def test_kanjian_not_corrupted_by_beikan(self): + """'被看→被砍' must NOT corrupt '看见' if '被看见' is in text.""" + processor = DictionaryProcessor({"被看": "被砍"}, []) + text = "我被看见了，别人也看见了" + result, changes = processor.process(text) + # '被看见' contains '被看' -- boundary check must protect it + self.assertNotIn("被砍", result, + "Must NOT corrupt any instance of '被看' inside '被看见'") + + +class TestAuditCatchesAllProductionFalsePositives(unittest.TestCase): + """ + Verify audit_corrections flags every single production false positive + when they appear in a corrections dictionary. + """ + + def test_audit_catches_all_category1_words(self): + """Every Category 1 word must be flagged by audit_corrections.""" + all_false_positives = { + # lifestyle + "仿佛": "反复", "正面": "正念", "犹豫": "抑郁", + "传说": "穿梭", "演技": "眼界", "无果": "无过", + "旗号": "期号", "应急": "应集", "正经": "正劲", + # lifestyle/beauty + "保健": "宝剑", "内涵": "内含", + # manufacturing + "仅供": "紧供", "供气": "工器", "出头": "初投", "几口": "集口", + # lifestyle previously disabled + "增加": "工站", "教育": "叫于", "大一": "答疑", + "曲线": "去先", "分母": "份母", + # various domains + "两本": "量本", "初五": "出误", "数据": "束据", + "力竭": "立杰", "充于": "冲余", + # substring collision sources + "被看": "被砍", "天差": "偏差", "亮亮": "亮哥", "金流": "现金流", + # substring issue words + "看见": "砍件", "分钟": "份种", + } + + issues = audit_corrections(all_false_positives) + + for word in all_false_positives: + self.assertIn( + word, issues, + f"audit_corrections MUST flag '{word}' but did not" + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/transcript-fixer/scripts/tests/test_correction_service.py b/transcript-fixer/scripts/tests/test_correction_service.py index d81963db..a5521e54 100644 --- a/transcript-fixer/scripts/tests/test_correction_service.py +++ b/transcript-fixer/scripts/tests/test_correction_service.py @@ -160,6 +160,31 @@ def test_remove_nonexistent_correction(self): success = self.service.remove_correction("nonexistent", "general") self.assertFalse(success) + # ==================== Domain Stats Tests ==================== + + def test_get_corrections_all_domains(self): + """domain=None loads all domains.""" + self.service.add_correction("a", "b", "general") + self.service.add_correction("c", "d", "finance") + all_corr = self.service.get_corrections(None) + self.assertEqual(len(all_corr), 2) + self.assertIn("a", all_corr) + self.assertIn("c", all_corr) + + def test_get_domain_stats(self): + """get_domain_stats returns per-domain counts.""" + self.service.add_correction("a", "b", "general") + self.service.add_correction("c", "d", "finance") + self.service.add_correction("e", "f", "finance") + stats = self.service.get_domain_stats() + self.assertEqual(stats["general"], 1) + self.assertEqual(stats["finance"], 2) + + def test_get_domain_stats_empty(self): + """get_domain_stats returns empty dict when no corrections.""" + stats = self.service.get_domain_stats() + self.assertEqual(stats, {}) + # ==================== Import/Export Tests ==================== def test_import_corrections(self): diff --git a/transcript-fixer/scripts/tests/test_fix_transcript_timestamps.py b/transcript-fixer/scripts/tests/test_fix_transcript_timestamps.py new file mode 100644 index 00000000..b7fd4579 --- /dev/null +++ b/transcript-fixer/scripts/tests/test_fix_transcript_timestamps.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Tests for transcript timestamp normalization and rebasing.""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from fix_transcript_timestamps import repair_timestamps + + +class TestFixTranscriptTimestamps(unittest.TestCase): + def test_rollover_fix(self): + text = ( + "甲 58:50\n" + "内容 A\n" + "乙 59:58\n" + "内容 B\n" + "丙 00:05\n" + "内容 C\n" + ) + result = repair_timestamps( + text, + output_format="hhmmss", + rollover_backjump_seconds=15 * 60, + jitter_seconds=5, + rebase_to_zero=False, + ) + self.assertIn("甲 00:58:50", result.repaired_text) + self.assertIn("乙 00:59:58", result.repaired_text) + self.assertIn("丙 01:00:05", result.repaired_text) + self.assertEqual(len(result.anomalies), 0) + + def test_rebase_to_zero(self): + text = ( + "甲 01:31:10\n" + "内容 A\n" + "乙 01:31:12\n" + "内容 B\n" + ) + result = repair_timestamps( + text, + output_format="hhmmss", + rollover_backjump_seconds=15 * 60, + jitter_seconds=5, + rebase_to_zero=True, + ) + self.assertIn("甲 00:00:00", result.repaired_text) + self.assertIn("乙 00:00:02", result.repaired_text) + + +if __name__ == "__main__": + unittest.main() diff --git a/transcript-fixer/scripts/tests/test_split_transcript_sections.py b/transcript-fixer/scripts/tests/test_split_transcript_sections.py new file mode 100644 index 00000000..d896e2b6 --- /dev/null +++ b/transcript-fixer/scripts/tests/test_split_transcript_sections.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Tests for transcript section splitting.""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from split_transcript_sections import ( + SectionSpec, + sanitize_filename_component, + split_text_by_markers, +) + + +class TestSplitTranscriptSections(unittest.TestCase): + def test_split_text_by_markers(self): + text = ( + "预热内容\n" + "开始安装\n" + "装环境内容\n" + "我们复盘一下。\n" + "复盘内容\n" + ) + parts = split_text_by_markers( + text, + first_section_name="课前聊天", + sections=[ + SectionSpec(name="正式上课", marker="开始安装"), + SectionSpec(name="课后复盘", marker="我们复盘一下。"), + ], + ) + self.assertEqual(parts[0][0], "课前聊天") + self.assertEqual(parts[1][0], "正式上课") + self.assertEqual(parts[2][0], "课后复盘") + self.assertTrue(parts[1][1].startswith("开始安装")) + self.assertTrue(parts[2][1].startswith("我们复盘一下。")) + + def test_sanitize_filename_component(self): + self.assertEqual(sanitize_filename_component("课后/复盘"), "课后-复盘") + + +if __name__ == "__main__": + unittest.main() diff --git a/transcript-fixer/scripts/utils/common_words.py b/transcript-fixer/scripts/utils/common_words.py new file mode 100644 index 00000000..16b498c1 --- /dev/null +++ b/transcript-fixer/scripts/utils/common_words.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +""" +Common Chinese Words Safety Check + +Detects when a correction's from_text is a common Chinese word, +which would cause false positive replacements across transcripts. + +This is the core defense against the "仿佛→反复" class of bugs: +valid corrections for one ASR model that corrupt correct text from better models. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import List, Set + + +# High-frequency Chinese words that should NEVER be dictionary correction sources. +# These are words that appear correctly in normal Chinese text and replacing them +# would cause widespread collateral damage. +# +# Organized by category for maintainability. Not exhaustive -- the heuristic +# checks below catch additional cases. +# +# IMPORTANT: This list was curated from actual production false positives found +# in a 187-video transcription run (2026-03). Each entry caused real damage. + +COMMON_WORDS_2CHAR: Set[str] = { + # --- Production false positives (confirmed damage, 2026-03 run) --- + # lifestyle domain + "仿佛", "正面", "犹豫", "传说", "演技", "无果", "旗号", "应急", "正经", + # lifestyle/beauty domain + "保健", "内涵", + # manufacturing domain + "仅供", "供气", "出头", "几口", + # lifestyle - previously disabled then re-confirmed + "增加", "教育", "大一", "曲线", "分母", + # various domains - discovered in manual review + "两本", "初五", "数据", "力竭", "充于", + # general/manufacturing/education - substring collision sources + "被看", "天差", "亮亮", "金流", + # caused substring issues in production + "看见", "分钟", + # --- High-frequency general vocabulary --- + "我们", "他们", "你们", "这个", "那个", "什么", "怎么", "为什么", + "可以", "因为", "所以", "但是", "虽然", "如果", "已经", "正在", + "需要", "应该", "可能", "一定", "非常", "比较", "特别", "一般", + "开始", "结束", "继续", "发展", "问题", "方法", "工作", "时间", + "学习", "研究", "分析", "讨论", "了解", "知道", "觉得", "认为", + "希望", "表示", "提出", "建议", "要求", "计划", "设计", "管理", + "技术", "系统", "数据", "网络", "平台", "产品", "服务", "市场", + "企业", "公司", "团队", "项目", "客户", "用户", "资源", "成本", + "效果", "质量", "安全", "标准", "流程", "模式", "策略", "方案", + "结构", "功能", "接口", "模块", "组件", "测试", "部署", "运维", + "目标", "任务", "进度", "优化", "调整", "更新", "升级", "维护", + "配置", "参数", "设置", "选项", "状态", "信息", "内容", "格式", + "教育", "培训", "实践", "经验", "能力", "水平", "素质", "思维", + "创新", "合作", "沟通", "交流", "反馈", "评估", "考核", "激励", + # --- Common verbs and adjectives --- + "实现", "完成", "处理", "解决", "执行", "操作", "运行", "启动", + "关闭", "打开", "保存", "删除", "修改", "添加", "移除", "查看", + "搜索", "过滤", "排序", "导入", "导出", "上传", "下载", "同步", + "重要", "关键", "核心", "基本", "主要", "次要", "简单", "复杂", + "明确", "清晰", "具体", "详细", "准确", "完整", "稳定", "灵活", + # --- Domain terms that look like ASR errors but are valid --- + "线数", "曲线", "分母", "正面", "旗号", "无果", "演技", +} + +# Common 3+ character words that should also be protected. +# These serve dual purpose: +# 1. Never used as correction sources (same as 2-char words) +# 2. Used by DictionaryProcessor._is_inside_longer_word() to detect +# when a short correction target is embedded inside a valid longer word +COMMON_WORDS_3PLUS: Set[str] = { + "自动化", "智能化", "数字化", "信息化", "标准化", "规范化", + "产线数", "服务器", "数据库", "操作系统", "人工智能", "机器学习", + "深度学习", "自然语言", "计算机视觉", "强化学习", + "区块链", "云计算", "大数据", "物联网", "互联网", + # --- Production collision targets (longer words containing short false positives) --- + # These must be here so _is_inside_longer_word() can detect them + "产线数据", "现金流", "资金流", "现金流量", "资金流向", + "被看见", "被看到", "被看作", "被看成", "被看好", + "漂漂亮亮", "亮亮堂堂", "明明亮亮", + "天差地别", "天差地远", + "被看见", "没看见", + "出头露面", "出头之日", + "正月初五", "大年初五", + "保健品", "保健操", "医疗保健", + "文化内涵", + "无果而终", + # --- Common Chinese idioms/phrases containing short words --- + # These are needed to prevent idiom corruption + "正面临", "正面对", + "应急响应", "应急预案", "应急处理", + "仅供参考", "仅供参阅", +} + +# Words that commonly contain other words as substrings. +# Key: the short word, Value: common words containing it. +# Used to warn about substring collision risk. +SUBSTRING_COLLISION_MAP: dict[str, list[str]] = { + "线数": ["产线数据", "曲线数", "线数量"], + "增加": ["新增加", "增加值"], + "数据": ["大数据", "数据库", "数据集", "元数据"], + "服务": ["服务器", "服务端", "微服务", "云服务"], + "测试": ["单元测试", "集成测试", "压力测试", "测试用例"], + "模型": ["大模型", "模型训练", "预训练模型"], + "学习": ["学习率", "深度学习", "机器学习", "强化学习"], + "正面": ["正面临", "正面对"], + "应急": ["应急响应", "应急预案", "应急处理"], + "无果": ["无果而终", "毫无果断"], + # --- Production substring collision patterns (2026-03 manual review) --- + # "线数" inside "产线数据" → corrupts to "产线束据" + # (already covered above) + # "金流" inside "现金流" → corrupts to "现现金流" (replacement contains match) + "金流": ["现金流", "资金流", "资金流向", "现金流量"], + # "被看" inside "被看见" → corrupts to "被砍见" + "被看": ["被看见", "被看到", "被看作", "被看成", "被看好"], + # "亮亮" inside "漂漂亮亮" → corrupts to "漂漂亮哥" + "亮亮": ["漂漂亮亮", "亮亮堂堂", "明明亮亮"], + # "天差" inside "天差地别" → corrupts idiom to "偏差地别" + "天差": ["天差地别", "天差地远"], + # "看见" inside longer phrases → substring collision risk + "看见": ["被看见", "看见过", "没看见"], + # "分钟" inside longer phrases → substring collision risk + "分钟": ["几分钟", "十分钟", "三十分钟", "一分钟"], + # "出头" common in phrases + "出头": ["出头露面", "出头之日", "冒出头"], + # "初五" common in date phrases + "初五": ["正月初五", "大年初五"], + # "保健" common in compound words + "保健": ["保健品", "保健操", "医疗保健"], + # "内涵" common in compound words + "内涵": ["内涵段子", "文化内涵"], +} + +ALL_COMMON_WORDS: Set[str] = COMMON_WORDS_2CHAR | COMMON_WORDS_3PLUS + + +@dataclass +class SafetyWarning: + """A warning about a potentially dangerous correction rule.""" + level: str # "error" (should block) or "warning" (should confirm) + category: str # "common_word", "short_text", "substring_collision" + message: str + suggestion: str # What to do instead + + +def check_correction_safety( + from_text: str, + to_text: str, + strict: bool = True, +) -> List[SafetyWarning]: + """ + Check if a correction rule is safe to add to the dictionary. + + This is the main entry point. Returns a list of warnings/errors. + Empty list = safe to add. + + Args: + from_text: The text to be replaced (the "wrong" text) + to_text: The replacement text (the "correct" text) + strict: If True, common word matches are errors; if False, warnings + + Returns: + List of SafetyWarning objects (empty = safe) + """ + warnings: List[SafetyWarning] = [] + + # Check 1: Is from_text a known common word? + if from_text in ALL_COMMON_WORDS: + level = "error" if strict else "warning" + warnings.append(SafetyWarning( + level=level, + category="common_word", + message=( + f"'{from_text}' is a common Chinese word that appears correctly " + f"in normal text. Replacing it with '{to_text}' will cause " + f"false positives across all transcripts." + ), + suggestion=( + f"Use a context rule instead: add a regex pattern that matches " + f"'{from_text}' only in the specific context where it's an ASR error. " + f"Example: match '{from_text}' only when preceded/followed by specific characters." + ), + )) + + # Check 2: Is from_text very short (<=2 chars)? + if len(from_text) <= 2: + # Even if not in our common words list, 2-char Chinese words are risky + if from_text not in ALL_COMMON_WORDS: + # Not already flagged above -- add a length warning + warnings.append(SafetyWarning( + level="warning", + category="short_text", + message=( + f"'{from_text}' is only {len(from_text)} character(s). " + f"Short corrections have high false positive risk in Chinese " + f"because they match as substrings inside longer words." + ), + suggestion=( + f"Verify '{from_text}' is never a valid word in any context. " + f"If unsure, use a context rule with surrounding text patterns instead." + ), + )) + + # Check 3: Could from_text match as a substring inside common words? + # This catches the "线数" matching inside "产线数据" bug. + if from_text in SUBSTRING_COLLISION_MAP: + collisions = SUBSTRING_COLLISION_MAP[from_text] + warnings.append(SafetyWarning( + level="error" if strict else "warning", + category="substring_collision", + message=( + f"'{from_text}' is a substring of common words: " + f"{', '.join(collisions)}. " + f"Replacing '{from_text}' with '{to_text}' will corrupt these words." + ), + suggestion=( + f"Use a context rule with negative lookahead/lookbehind to exclude " + f"matches inside these common words. Example regex: " + f"'(? None: + """ + Check if from_text appears as a substring in any common word, + where the common word is NOT the from_text itself. + """ + if len(from_text) > 4: + # Long enough that substring collisions are unlikely to be problematic + return + + collisions: List[str] = [] + for word in ALL_COMMON_WORDS: + if word == from_text: + continue + if from_text in word: + collisions.append(word) + + if collisions: + # Only show first 5 to avoid spam + shown = collisions[:5] + more = f" (and {len(collisions) - 5} more)" if len(collisions) > 5 else "" + warnings.append(SafetyWarning( + level="warning", + category="substring_collision", + message=( + f"'{from_text}' appears inside {len(collisions)} common word(s): " + f"{', '.join(shown)}{more}. " + f"This replacement may cause collateral damage." + ), + suggestion=( + f"Review whether '{from_text}→{to_text}' could corrupt any of " + f"these words. Consider using a context rule instead." + ), + )) + + +def audit_corrections( + corrections: dict[str, str], +) -> dict[str, List[SafetyWarning]]: + """ + Audit all corrections in a dictionary for safety issues. + + Used by the --audit command. + + Args: + corrections: Dict of {from_text: to_text} + + Returns: + Dict of {from_text: [warnings]} for entries with issues. + Entries with no issues are not included. + """ + results: dict[str, List[SafetyWarning]] = {} + + for from_text, to_text in corrections.items(): + warnings = check_correction_safety(from_text, to_text, strict=False) + if warnings: + results[from_text] = warnings + + return results diff --git a/transcript-fixer/scripts/utils/security.py b/transcript-fixer/scripts/utils/security.py index 69e6185c..b98cfb9b 100644 --- a/transcript-fixer/scripts/utils/security.py +++ b/transcript-fixer/scripts/utils/security.py @@ -43,7 +43,7 @@ def mask_secret(secret: str, visible_chars: int = 4) -> str: Masked string like "7fb3...DPRR" Examples: - >>> mask_secret("7fb3ab7b186242288fe93a27227b7149.bJCOEAsUfejvWDPR") + >>> mask_secret("example-fake-api-key-1234567890abcdef.test") '7fb3...DPRR' >>> mask_secret("short") @@ -248,7 +248,7 @@ class SecretStr: Wrapper for secrets that prevents accidental logging. Usage: - api_key = SecretStr("7fb3ab7b186242288fe93a27227b7149.bJCOEAsUfejvWDPR") + api_key = SecretStr("example-fake-api-key-1234567890abcdef.test") print(api_key) # Prints: SecretStr(7fb3...DPRR) print(api_key.get()) # Get actual value when needed diff --git a/tunnel-doctor/SKILL.md b/tunnel-doctor/SKILL.md index 865fbd17..84e9a581 100644 --- a/tunnel-doctor/SKILL.md +++ b/tunnel-doctor/SKILL.md @@ -27,15 +27,21 @@ Proxy/VPN tools on macOS create conflicts at five independent layers. Layers 1-3 Determine which scenario applies: - **Browser returns HTTP 503, but `curl` and SSH both work** → System proxy bypass conflict (Step 2C) +- **`local.` fails in browser/default `curl`, but direct/no-proxy request works** → Local vanity domain proxy interception (Step 2C-1) - **Tailscale ping works, SSH works, but curl/HTTP times out** → HTTP proxy env var conflict (Step 2A) - **Tailscale ping works, SSH/TCP times out** → Route conflict (Step 2B) - **Remote dev server auth redirects to `localhost` → browser can't follow** → SSH tunnel needed (Step 2D) - **`make status` / scripts curl to localhost fail with proxy** → localhost proxy interception (Step 2E) - **`git push/pull` fails with `FATAL: failed to begin relaying via HTTP`** → SSH double tunnel (Step 2F) -- **`docker pull` fails with `TLS handshake timeout` or `docker build` can't fetch base images** → VM/container proxy propagation (Step 2G) +- **`docker build` `RUN apk/apt` fails with `Connection refused` instantly** → OrbStack transparent proxy + TUN conflict (Step 2G-1, fix: `--network host`) +- **`docker pull` fails with `TLS handshake timeout`** → VM proxy misconfiguration (Step 2G-2, fix: `docker.json` with `host.internal`) +- **Container healthcheck `(unhealthy)` but app runs fine** → Lowercase proxy env var leak (Step 2G-4, fix: clear `http_proxy`+`HTTP_PROXY`) +- **`docker build` can't fetch base images** → VM/container proxy propagation (Step 2G) - **`git clone` fails with `Connection closed by 198.18.x.x`** → TUN DNS hijack for SSH (Step 2H) - **SSH connects but `operation not permitted`** → Tailscale SSH config issue (Step 4) - **SSH connects but `be-child ssh` exits code 1** → WSL snap sandbox issue (Step 5) +- **TCP port 22 reachable (`nc -z` succeeds) but SSH fails with `kex_exchange_identification: Connection closed`** → Tailscale SSH proxy intercept on WSL (Step 5A) +- **`tailscale ssh` returns "not available on App Store builds"** → Wrong Tailscale distribution on macOS (Step 5B) **Key distinctions**: - SSH does NOT use `http_proxy`/`NO_PROXY` env vars. If SSH works but HTTP doesn't → Layer 2. @@ -43,7 +49,30 @@ Determine which scenario applies: - If `tailscale ping` works but regular `ping` doesn't → Layer 1 (route table corrupted). - If `ssh -T git@github.com` works but `git push` fails intermittently → Layer 4 (double tunnel). - If host `curl https://...` works but `docker pull` times out → Layer 5 (VM proxy propagation). +- If `docker pull` works but `docker build` `RUN apk add` fails instantly with `Connection refused` → OrbStack transparent proxy broken by TUN (Step 2G-1). +- If container healthcheck shows `(unhealthy)` but app works → lowercase `http_proxy` leaked into container (Step 2G-4). - If DNS resolves to `198.18.x.x` virtual IPs → TUN DNS hijack (Step 2H). +- If `nc -z` succeeds on port 22 but SSH gets no banner (`kex_exchange_identification`) → Tailscale SSH proxy intercept (Step 5A). Confirm with `tcpdump -i any port 22` on the remote — 0 packets means Tailscale intercepts above the kernel. +- If `tailscale ssh` fails with "not available on App Store builds" → install Standalone Tailscale (Step 5B). + +### Fast Path: Run Automated Checks + +For common macOS conflicts (env proxy, system proxy exceptions, direct/proxy path split, local TLS trust), run: + +```bash +python3 scripts/quick_diagnose.py --host local.claude4.dev --url https://local.claude4.dev/health +``` + +Optional route ownership check for a Tailscale destination: + +```bash +python3 scripts/quick_diagnose.py --host --url http://:/health --tailscale-ip <100.x.x.x> +``` + +Interpretation: +- `direct=PASS` + `forced_proxy=FAIL` = host must bypass proxy (`skip-proxy` + `NO_PROXY`). +- `strict_tls=FAIL` + `direct=PASS` = path is reachable; trust issue only (install/trust local CA). +- `host in scutil exceptions: no` = browser/system clients still likely proxied. ### Step 2A: Fix HTTP Proxy Environment Variables @@ -76,6 +105,18 @@ export NO_PROXY=localhost,127.0.0.1,.ts.net,100.64.0.0/10,192.168.*,10.*,172.16. **NO_PROXY syntax pitfalls** — see [references/proxy_conflict_reference.md](references/proxy_conflict_reference.md) for the compatibility matrix. +**Go `net/http` CIDR caveat**: Go's standard `net/http` does NOT support CIDR notation in `NO_PROXY`. Setting `NO_PROXY=100.64.0.0/10` works for curl and Python, but Go programs (including Tailscale-adjacent tooling) will still send traffic through the proxy. The fix is to use MagicDNS hostnames (e.g., `workstation-4090-wsl`) instead of raw IPs, or add explicit hostnames to `NO_PROXY`: + +```bash +# WRONG for Go programs — CIDR is silently ignored +NO_PROXY=100.64.0.0/10 go-program http://100.101.102.103:8002/health # → goes through proxy + +# CORRECT — use hostname (matched as suffix) or explicit IP +export NO_PROXY=localhost,127.0.0.1,.ts.net,workstation-4090-wsl,100.101.102.103,192.168.*,10.*,172.16.* +``` + +This is especially relevant when accessing Tailscale services from Go-based tools (e.g., custom CLIs, Go test suites hitting remote APIs). + Verify the fix: ```bash @@ -107,6 +148,19 @@ gateway: 192.168.x.1 # Default gateway interface: en0 # Physical interface, NOT Tailscale ``` +**Important**: Not all `utun` interfaces are Tailscale's. Verify which utun belongs to Tailscale before concluding the route is correct: + +```bash +# Find Tailscale's utun interface (has a 100.x.x.x IP) +ifconfig | grep -A2 'inet 100\.' +``` + +Quick indicators by MTU: +- **MTU 1280** → typically Tailscale +- **MTU 4064** → typically Shadowrocket TUN + +If `route -n get` shows traffic going to a utun with MTU 4064, it is hitting Shadowrocket's TUN, not Tailscale — this is still a route conflict even though the interface name starts with `utun`. + Confirm with full route table: ```bash @@ -150,6 +204,41 @@ skip-proxy = 192.168.0.0/16, 10.0.0.0/8, 172.16.0.0/12, 100.64.0.0/10, localhost - `skip-proxy`: Bypasses the HTTP proxy layer only. Traffic still flows through the TUN interface and Tailscale utun handles it. Safe. - `tun-excluded-routes`: Removes the CIDR from the TUN routing entirely. This creates a competing `en0` route that overrides Tailscale. Breaks everything. +#### Step 2C-1: Fix Local Vanity Domain Interception (`local.`) + +**Symptom**: `https://local.` fails in browser or default `curl`, but succeeds with direct/no-proxy command: + +```bash +env -u http_proxy -u https_proxy curl -k -I https://local./health +# -> 200 +curl -I https://local./health +# -> proxy CONNECT then TLS reset/failure +``` + +**Root cause**: The domain is routed through system/shell proxy instead of local direct path. + +**Fix**: +1. Add domain to proxy app bypass list (`skip-proxy` for Shadowrocket). +2. Add domain to shell bypass list (`NO_PROXY`/`no_proxy`). +3. If local TLS uses internal CA, trust the local root certificate. + +```bash +# ~/.zshrc +export NO_PROXY=localhost,127.0.0.1,.ts.net,100.64.0.0/10,192.168.*,10.*,172.16.*,local.,www.local. +export no_proxy="$NO_PROXY" +``` + +**Verification**: + +```bash +python3 scripts/quick_diagnose.py --host local. --url https://local./health +``` + +Expected: +- `host in NO_PROXY: yes` +- `host in scutil exceptions: yes` +- `ambient=PASS` and `direct=PASS` + ### Step 2D: Fix Auth Redirect for Remote Dev (SSH Tunnel) **Symptom**: Dev server runs on a remote machine (e.g., Mac Mini via Tailscale). You access `http://:3010` in the browser. Login/signup works, but after auth, the app redirects to `http://localhost:3010/` which fails — `localhost` on your machine isn't running the dev server. @@ -234,7 +323,7 @@ GIT_SSH_COMMAND="ssh -o ProxyCommand=none" git push origin main ### Step 2G: Fix VM/Container Runtime Proxy Propagation (Docker pull/build failures) -**Symptom**: `docker pull` or `docker build` fails with `net/http: TLS handshake timeout` or `Internal Server Error` from `auth.docker.io`, while host `curl` to the same URLs works fine. +**Symptom**: `docker pull` or `docker build` fails with `net/http: TLS handshake timeout`, `Connection refused` from Alpine/Debian repos, or `Internal Server Error` from `auth.docker.io`, while host `curl` to the same URLs works fine. **Applies to**: OrbStack, Docker Desktop, or any VM-based Docker runtime on macOS with Shadowrocket/Clash TUN active. @@ -247,66 +336,160 @@ VM process (Docker): Docker daemon → VM bridge → host network → TUN → The TUN handles host-originated traffic correctly but may drop or delay VM-bridged traffic (different TCP stack, MTU, keepalive behavior). -**Three sub-problems and their fixes**: +**Critical distinction: `docker pull` vs `docker build` use different proxy paths**: + +| Operation | Proxy source | What controls it | +|-----------|-------------|------------------| +| `docker pull` | Docker daemon config | `~/.orbstack/config/docker.json` or `docker info` | +| `docker build` (`RUN apt/apk`) | Build container env | `--build-arg http_proxy=...` or `--network host` | +| `docker run` | Container env | `-e http_proxy=...` or inherited from daemon | + +Fixing `docker.json` alone will NOT fix `docker build` — the `RUN` commands inside the build container don't inherit daemon proxy settings. + +**Diagnosis** — identify which sub-problem: + +```bash +# 1. Can the Docker daemon pull images? +docker pull --quiet alpine:latest 2>&1 + +# 2. Can a RUN command inside a build reach the internet? +docker build --no-cache - <<'EOF' 2>&1 +FROM alpine:latest +RUN apk update && echo "APK OK" +EOF + +# 3. Can a running container reach the internet? +docker run --rm alpine:latest sh -c "apk update 2>&1 | head -3" +``` + +**Four sub-problems and their fixes**: + +#### 2G-1: `docker build` fails but host works (most common with OrbStack + Shadowrocket) + +**Symptom**: `RUN apk add` or `RUN apt-get install` inside `docker build` fails with `Connection refused` instantly (< 0.2s), even though host `curl` to the same URL works. + +**Root cause**: OrbStack's `network_proxy: auto` creates a transparent proxy inside the VM that intercepts all HTTPS traffic. When Shadowrocket TUN is also active, the transparent proxy's upstream connection breaks — it redirects HTTPS to `127.0.0.1` inside the VM, which has nothing listening. + +**Diagnosis**: + +```bash +# Verify: inside the container, HTTPS goes to 127.0.0.1 (broken transparent proxy) +docker run --rm alpine:latest sh -c "wget -q --timeout=5 -O /dev/null https://dl-cdn.alpinelinux.org/ 2>&1" +# → "wget: can't connect to remote host (127.0.0.1): Connection refused" +# ^^^^^^^^^^^^ This is the smoking gun + +# Verify: --network host bypasses the VM bridge and works +docker run --rm --network host alpine:latest sh -c "apk update 2>&1 | head -3" +# → "v3.23.x ... OK: 27431 distinct packages available" ← Works! +``` + +**Fix** — use `--network host` for docker build: + +```bash +docker build --network host -f Dockerfile -t myimage . +``` + +This bypasses OrbStack's VM network bridge entirely. The build container uses the host's network stack directly, where Shadowrocket TUN correctly handles traffic. + +**Trade-off**: `--network host` disables build-time network isolation. For CI/CD, prefer fixing the proxy config (2G-2). For local development, `--network host` is the pragmatic fix. + +**Permanent fix** — if all your builds need this, add to `~/.docker/daemon.json` or use a shell alias: + +```bash +# Shell alias (add to ~/.zshrc) +alias docker-build='docker build --network host' +``` + +#### 2G-2: OrbStack auto-detects and caches proxy config -#### 2G-1: OrbStack auto-detects and caches proxy (most common) +OrbStack's `network_proxy: auto` reads `http_proxy` from the shell environment and configures the Docker daemon. The config is stored in `~/.orbstack/config/docker.json`. -OrbStack's `network_proxy: auto` reads `http_proxy` from the shell environment and writes it to `~/.orbstack/config/docker.json`. **Crucially**, `orbctl config set network_proxy none` does NOT clean up `docker.json` — the cached proxy persists. +**Key behaviors**: +- `network_proxy: auto` — OrbStack reads host env, creates transparent proxy in VM +- `network_proxy: none` — Disables transparent proxy, but VM bridge traffic still routes through TUN (may timeout) +- `docker.json` — Controls `docker pull` proxy, NOT `docker build` RUN commands **Diagnosis**: ```bash -# OrbStack config says "none" but Docker still shows proxy -orbctl config get network_proxy # → "none" -docker info | grep -i proxy # → HTTP Proxy: http://127.0.0.1:1082 ← stale! +# Check all three layers +echo "=== OrbStack config ===" +orbctl config get network_proxy -# The real source of truth: +echo "=== docker.json (daemon proxy) ===" cat ~/.orbstack/config/docker.json -# → {"proxies": {"http-proxy": "http://127.0.0.1:1082", ...}} ← cached! + +echo "=== Docker info (effective proxy) ===" +docker info | grep -iE "proxy|No Proxy" ``` -**Fix** — DON'T remove the proxy. Instead, add precise `no-proxy` to prevent localhost interception while keeping the proxy as the VM's outbound channel: +**Fix** — configure `docker.json` with `host.internal` (OrbStack resolves this to the host IP): ```bash -# Write corrected config (keeps proxy, adds no-proxy for local traffic) python3 -c " -import json +import json, os config = { 'proxies': { - 'http-proxy': 'http://127.0.0.1:1082', - 'https-proxy': 'http://127.0.0.1:1082', + 'http-proxy': 'http://host.internal:1082', + 'https-proxy': 'http://host.internal:1082', 'no-proxy': 'localhost,127.0.0.1,::1,192.168.128.0/24,100.64.0.0/10,host.internal,*.local' } } -json.dump(config, open('$HOME/.orbstack/config/docker.json', 'w'), indent=2) +path = os.path.expanduser('~/.orbstack/config/docker.json') +json.dump(config, open(path, 'w'), indent=2) +print('Written:', path) " -# Full restart (not just docker engine) +# Full restart required orbctl stop && sleep 3 && orbctl start ``` -**Why NOT remove the proxy**: When TUN is active, removing the Docker proxy means VM traffic goes directly through the bridge → TUN path, which causes TLS handshake timeouts. The proxy provides a working outbound channel because OrbStack maps host `127.0.0.1` into the VM. +**Important**: Use `host.internal` (OrbStack-specific), NOT `127.0.0.1` (points to VM loopback) and NOT `host.docker.internal` (may not resolve in all contexts). -#### 2G-2: Removing proxy makes Docker worse (counter-intuitive) +**Why NOT remove the proxy**: When TUN is active, removing the Docker proxy means VM traffic goes directly through the bridge → TUN path, which causes TLS handshake timeouts. The proxy provides a working outbound channel. + +#### 2G-3: Removing proxy makes Docker worse (counter-intuitive) | Docker config | Traffic path | Result | |---------------|-------------|--------| -| Proxy ON, no `no-proxy` | Docker → proxy → TUN → internet | Docker Hub ✅, localhost probes ❌ | -| Proxy OFF | Docker → VM bridge → host → TUN → internet | TLS timeout ❌ | -| **Proxy ON + `no-proxy`** | **External: Docker → proxy → internet ✅; Local: Docker → direct ✅** | **Both work ✅** | +| Proxy ON (`127.0.0.1`), no `no-proxy` | Docker → VM proxy → ??? | `docker pull` may work, localhost probes ❌ | +| Proxy ON (`host.internal`), + `no-proxy` | External: Docker → host proxy → internet; Local: direct | **Both work ✅** | +| Proxy OFF (`network_proxy: none`) | Docker → VM bridge → host → TUN → internet | TLS timeout ❌ | +| **`--network host` (build only)** | **Build container → host network → TUN → internet** | **Build works ✅** | + +**Decision tree**: +- `docker pull` broken → Fix `docker.json` with `host.internal` proxy (2G-2) +- `docker build` broken → Use `--network host` (2G-1) OR pass `--build-arg http_proxy=http://host.internal:1082` +- Both broken → Fix both: `docker.json` + `--network host` + +#### 2G-4: Deploy scripts and container healthchecks probe localhost through proxy + +Deploy scripts that `curl localhost` inside containers or Docker healthchecks that use `wget http://localhost` will route through the proxy if env vars leak into the container. + +**Common symptoms**: +- Container healthcheck shows `(unhealthy)` but the app inside is running fine +- `wget: can't connect to remote host (127.0.0.1): Connection refused` in healthcheck logs (proxy port, not app port) -#### 2G-3: Deploy scripts probe localhost through proxy +**Root cause**: Docker inherits uppercase AND lowercase proxy env vars from the host. Many tools only clear uppercase (`HTTP_PROXY=`) but forget lowercase (`http_proxy=http://127.0.0.1:1082`). The healthcheck `wget` uses lowercase. -Deploy scripts that `curl localhost` inside the Docker environment will route through the proxy. Fix by adding `NO_PROXY` at the script level: +**Fix in docker-compose.yml** — clear BOTH cases: + +```yaml +environment: + # Must clear both uppercase and lowercase — wget/curl check different vars + - HTTP_PROXY= + - HTTPS_PROXY= + - http_proxy= + - https_proxy= + - NO_PROXY=* + - no_proxy=* +``` + +**Fix in deploy scripts**: ```bash -# In deploy.sh or similar scripts: _local_bypass="localhost,127.0.0.1,::1" -if [[ -n "${NO_PROXY:-}" ]]; then - export NO_PROXY="${_local_bypass},${NO_PROXY}" -else - export NO_PROXY="${_local_bypass}" -fi +export NO_PROXY="${_local_bypass}${NO_PROXY:+,${NO_PROXY}}" export no_proxy="$NO_PROXY" # Use 127.0.0.1 instead of localhost in probe URLs (some proxy implementations @@ -324,8 +507,15 @@ docker info | grep -iE "proxy|No Proxy" # Pull test docker pull --quiet hello-world -# Local probe test -curl -s http://127.0.0.1:3001/health +# Build test (the real verification) +docker build --network host --no-cache - <<'EOF' +FROM alpine:latest +RUN apk update && echo "BUILD OK" +EOF + +# Container env check (no proxy leak) +docker exec env | grep -i proxy +# Expected: all empty or not set ``` ### Step 2H: Fix TUN DNS Hijack for SSH/Git (198.18.x.x virtual IPs) @@ -453,13 +643,89 @@ sudo tailscale up --ssh **Important**: The new installation may assign a different Tailscale IP. Check with `tailscale status --self`. +### Step 5A: Fix Tailscale SSH Proxy Silent Failure on WSL + +**Symptom**: TCP port 22 is reachable (`nc -z -w 5 22` succeeds), but SSH fails immediately with: + +``` +kex_exchange_identification: Connection closed by remote host +``` + +No SSH banner is ever received. This happens even with apt-installed Tailscale (not snap). + +**Root cause**: When `tailscale up --ssh` is enabled on WSL, Tailscale intercepts port 22 connections at the application layer (above the kernel network stack). If Tailscale's built-in SSH proxy malfunctions, it accepts the TCP connection but immediately closes it before sending the SSH banner. + +**Key diagnostic** — on the WSL instance: + +```bash +# This will show 0 packets even during active SSH attempts +sudo tcpdump -i any port 22 -c 5 -w /dev/null 2>&1 +``` + +Zero packets means Tailscale is intercepting connections before they reach the kernel network stack. The kernel's `sshd` never sees the connection. + +**Distinction from Step 5**: Step 5 covers snap sandbox issues where `be-child ssh` fails. This is a different problem — Tailscale's SSH proxy itself silently fails, regardless of installation method. + +**Fix** — disable Tailscale's SSH proxy and use regular sshd: + +```bash +# On the WSL instance: +sudo tailscale up --ssh=false + +# Verify sshd is running +sudo service ssh status +# If not running: +sudo service ssh start + +# Verify from the client machine: +ssh -o ConnectTimeout=10 @ 'echo SSH_OK' +``` + +After disabling Tailscale SSH, connections go through the kernel network stack to `sshd` as normal. The Tailscale ACL `"action": "accept"` in Step 4 is no longer relevant — authentication is handled by `sshd` using SSH keys or passwords. + +**When to keep `--ssh` enabled**: Only if you specifically need Tailscale's SSH features (ACL-based access control, no SSH key management). If standard sshd works, prefer `--ssh=false` for reliability. + +### Step 5B: Fix App Store Tailscale on macOS (Missing `tailscale ssh`) + +**Symptom**: Running `tailscale ssh` returns: + +``` +The 'tailscale ssh' subcommand is not available on macOS builds +distributed through the App Store or TestFlight. +``` + +**Root cause**: The App Store version of Tailscale for macOS is sandboxed and does not include the `tailscale ssh` subcommand. + +**Fix** — install the Standalone version: + +1. Uninstall the App Store version (delete from /Applications) +2. Download the Standalone build from https://pkgs.tailscale.com/stable/#macos +3. Install to /Applications + +**Post-install CLI setup**: The standalone `tailscale` CLI binary is embedded inside the app bundle. Add an alias to your shell config: + +```bash +# ~/.zshrc +alias tailscale="/Applications/Tailscale.app/Contents/MacOS/Tailscale" +``` + +Verify: + +```bash +source ~/.zshrc +tailscale version +tailscale ssh @ # Should work now +``` + ### Step 6: Verify End-to-End Run a complete connectivity test: ```bash -# 1. Check route is correct +# 1. Check route is correct (must show Tailscale's utun, not en0 or Shadowrocket's utun) route -n get +# Also confirm which utun is Tailscale's: +ifconfig | grep -A2 'inet 100\.' # 2. Test TCP connectivity nc -z -w 5 22 @@ -468,7 +734,7 @@ nc -z -w 5 22 ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no @ 'echo SSH_OK && hostname && whoami' ``` -All three must pass. If step 1 fails, revisit Step 3. If step 2 fails, check WSL sshd or firewall. If step 3 fails, revisit Steps 4-5. +All three must pass. If step 1 fails, revisit Step 3. If step 1 shows wrong utun (e.g., Shadowrocket's utun with MTU 4064 instead of Tailscale's with MTU 1280), that is also a route conflict. If step 2 passes but step 3 fails with `kex_exchange_identification`, revisit Step 5A (Tailscale SSH proxy intercept). If step 2 fails, check WSL sshd or firewall. If step 3 fails with other errors, revisit Steps 4-5. ## SOP: Remote Development via Tailscale @@ -557,7 +823,13 @@ Each `-L` flag is independent. If one port is already bound locally, `ExitOnForw ### 4. SSH Non-Login Shell Setup -SSH non-login shells don't load `~/.zshrc`, so nvm/Homebrew tools and proxy env vars are unavailable. Prefix all remote commands with `source ~/.zshrc 2>/dev/null;`. See [references/proxy_conflict_reference.md § SSH Non-Login Shell Pitfall](references/proxy_conflict_reference.md) for details and examples. +**This is a frequent source of "it works interactively but fails in scripts" bugs.** SSH non-login shells don't load `~/.zshrc` (or `~/.bashrc` on Linux), so tools installed via nvm, Homebrew, uv, cargo, or any shell-level manager won't be in `$PATH`. Proxy env vars set in `~/.zshrc` also won't be loaded. + +This affects **all** remote commands run via `ssh user@host "command"`, including CI/CD pipelines, cron-triggered SSH, and Makefile remote targets. Prefix all remote commands with `source ~/.zshrc 2>/dev/null;` (macOS) or `source ~/.bashrc 2>/dev/null;` (Linux/WSL). + +**Common failure**: `ssh user@host "uv run ..."` or `ssh user@host "node ..."` returns `command not found` even though the command works in an interactive SSH session. + +See [references/proxy_conflict_reference.md § SSH Non-Login Shell Pitfall](references/proxy_conflict_reference.md) for details and examples. For Makefile targets that run remote commands: diff --git a/tunnel-doctor/references/proxy_conflict_reference.md b/tunnel-doctor/references/proxy_conflict_reference.md index 03f35e3b..34fed4cf 100644 --- a/tunnel-doctor/references/proxy_conflict_reference.md +++ b/tunnel-doctor/references/proxy_conflict_reference.md @@ -68,6 +68,14 @@ NO_PROXY="" curl -s -X POST "http://:8080/api/ curl --noproxy '*' -s --connect-timeout 2 "http://192.168.31.110:8080/api/read" | head -1 ``` +**Port conflict warning**: Shadowrocket's config API listens on port 8080 by default, which may conflict with other services (e.g., whisper.cpp server, development proxies). If the API returns unexpected content (HTML, JSON from another service), verify what is actually listening on the port: + +```bash +lsof -nP -iTCP:8080 | head -5 +``` + +If another service owns port 8080, you need to either stop that service or access the Shadowrocket API from a different device on the same network. + **Critical**: Use `--data-binary`, NOT `-d`. The `-d` flag URL-encodes the content, corrupting `#`, `=`, `&` and other characters in the config. This **destroys the entire configuration** — all rules, settings, and proxy groups are lost. The user must restore from backup. ```bash @@ -88,6 +96,31 @@ tun-excluded-routes = 10.0.0.0/8, 127.0.0.0/8, 169.254.0.0/16, 172.16.0.0/12, 19 Note: `100.64.0.0/10` is intentionally absent. +### Complete Working Reference Config for Tailscale Compatibility + +This is a validated reference showing the correct relationship between `skip-proxy`, `tun-excluded-routes`, and `[Rule]` for Tailscale coexistence: + +``` +[General] +# skip-proxy: bypass the HTTP proxy for these destinations (fixes browser 503) +# 100.64.0.0/10 MUST be here for browser access to Tailscale IPs +skip-proxy = 192.168.0.0/16, 10.0.0.0/8, 172.16.0.0/12, 100.64.0.0/10, localhost, *.local, captive.apple.com + +# tun-excluded-routes: CIDRs excluded from TUN routing (sent directly via physical interface) +# 100.64.0.0/10 must NOT be here — including it creates an en0 route that overrides Tailscale +tun-excluded-routes = 10.0.0.0/8, 127.0.0.0/8, 169.254.0.0/16, 172.16.0.0/12, 192.0.0.0/24, 192.0.2.0/24, 192.88.99.0/24, 192.168.0.0/16, 198.51.100.0/24, 203.0.113.0/24, 224.0.0.0/4, 255.255.255.255/32 + +[Rule] +# Tailscale traffic enters TUN but is passed through without proxying +IP-CIDR,100.64.0.0/10,DIRECT +# ... other rules ... +``` + +**Key points**: +- `skip-proxy` — YES, include `100.64.0.0/10` (browser bypass) +- `tun-excluded-routes` — NO, never include `100.64.0.0/10` (would hijack routing) +- `[Rule]` — YES, include `IP-CIDR,100.64.0.0/10,DIRECT` (TUN passthrough) + ## Clash / ClashX Pro ### The Fix @@ -151,12 +184,15 @@ export NO_PROXY=localhost,127.0.0.1,.ts.net,100.64.0.0/10,192.168.*,10.*,172.16. ### NO_PROXY Syntax Pitfalls -| Syntax | curl | Python requests | Node.js | Meaning | -|--------|------|-----------------|---------|---------| -| `.ts.net` | ✅ | ✅ | ✅ | Domain suffix match (correct) | -| `*.ts.net` | ❌ | ✅ | varies | Glob — curl does NOT support this | -| `100.64.0.0/10` | ✅ 7.86+ | ✅ 2.25+ | ❌ native | CIDR notation | -| `100.*` | ✅ | ✅ | ✅ | Too broad — covers public IPs `100.0-63.*` and `100.128-255.*` | +| Syntax | curl | Python requests | Go `net/http` | Node.js | Meaning | +|--------|------|-----------------|---------------|---------|---------| +| `.ts.net` | ✅ | ✅ | ✅ | ✅ | Domain suffix match (correct) | +| `*.ts.net` | ❌ | ✅ | ❌ | varies | Glob — curl and Go do NOT support this | +| `100.64.0.0/10` | ✅ 7.86+ | ✅ 2.25+ | ❌ | ❌ native | CIDR notation — Go silently ignores it | +| `100.*` | ✅ | ✅ | ❌ | ✅ | Too broad — covers public IPs `100.0-63.*` and `100.128-255.*` | +| `workstation-name` | ✅ | ✅ | ✅ | ✅ | Exact hostname match (safest for Go) | + +**Go `net/http` warning**: Go's proxy bypass logic (`httpproxy.Config.ProxyFunc`) does not implement CIDR matching. `NO_PROXY=100.64.0.0/10` is silently ignored — Go programs will still route traffic through the proxy. Use MagicDNS hostnames (e.g., `workstation-4090-wsl`) or explicit IPs (e.g., `100.101.102.103`) instead of CIDR ranges when Go programs need to bypass the proxy. **Key rule**: Always use `.ts.net` (leading dot, no asterisk) for domain suffix matching. This is the most portable syntax across all HTTP clients. @@ -282,9 +318,9 @@ Connection setup is slightly slower (~6s vs ~2s) because TUN routing has more ne ## General Principles -### Four Conflict Layers +### Five Conflict Layers -Proxy tools create conflicts at four independent layers on macOS. Layers 1-3 affect Tailscale connectivity; Layer 4 affects SSH git operations through the same proxy infrastructure: +Proxy tools create conflicts at five independent layers on macOS. Layers 1-3 affect Tailscale connectivity; Layer 4 affects SSH git operations; Layer 5 affects VM/container runtimes: | Layer | Setting | What it controls | Symptom when wrong | |-------|---------|------------------|--------------------| @@ -292,6 +328,7 @@ Proxy tools create conflicts at four independent layers on macOS. Layers 1-3 aff | 2. HTTP env vars | `http_proxy` / `NO_PROXY` | CLI tools (curl, wget, Python, Node.js) | `curl` times out, SSH works, browser works | | 3. System proxy | `skip-proxy` | Browser and system HTTP clients | Browser 503, `curl` works (both with/without proxy), SSH works | | 4. SSH ProxyCommand | `ProxyCommand connect -H` | SSH git operations (push/pull/clone) | `ssh -T` works, `git push` fails intermittently with `failed to begin relaying via HTTP` | +| 5. VM/Container proxy | Docker/OrbStack proxy config | `docker pull`, `docker build` | Host `curl` works, `docker pull` times out (TLS handshake timeout) | **Each layer is independent.** A fix at one layer doesn't help the others. You may need fixes at multiple layers simultaneously. @@ -398,4 +435,6 @@ If a proxy config change breaks Tailscale connectivity: sudo route delete -net 100.64.0.0/10 ``` +**Important**: Manually deleting a bad `en0` route with `sudo route delete` is only a temporary fix. Shadowrocket will re-add the route when the VPN connection is next reconnected or toggled. The only permanent fix is modifying the Shadowrocket configuration to remove `100.64.0.0/10` from `tun-excluded-routes` (it should never be there). + If `tun-excluded-routes` was modified, reverting it and restarting Shadowrocket will restore Tailscale's routing immediately. diff --git a/tunnel-doctor/scripts/quick_diagnose.py b/tunnel-doctor/scripts/quick_diagnose.py new file mode 100755 index 00000000..a53b833b --- /dev/null +++ b/tunnel-doctor/scripts/quick_diagnose.py @@ -0,0 +1,522 @@ +#!/usr/bin/env python3 +""" +Quick tunnel/proxy conflict diagnostics for macOS. + +This script detects the most common local and Tailscale networking conflicts: +1) Shell proxy env + NO_PROXY mismatch +2) System proxy exceptions mismatch +3) Proxy path failure vs direct path success +4) Local TLS trust issues +5) Route ownership conflicts for a Tailscale IP (optional) +""" + +import argparse +import json +import os +import re +import shlex +import socket +import subprocess +import sys +from typing import Dict, List, Optional, Tuple + + +def run(cmd: List[str], env: Optional[Dict[str, str]] = None) -> Tuple[int, str, str]: + proc = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + return proc.returncode, proc.stdout.strip(), proc.stderr.strip() + + +def split_csv(value: str) -> List[str]: + if not value: + return [] + return [item.strip() for item in value.split(",") if item.strip()] + + +def match_proxy_pattern(host: str, pattern: str) -> bool: + p = pattern.strip().lower() + h = host.strip().lower() + if not p or not h: + return False + + # CIDR / wildcard IP patterns are not domain checks. + if "/" in p and not p.startswith("*."): + return False + if re.match(r"^\d+\.\d+\.\d+\.\*$", p): + return False + + if p == h: + return True + if p.startswith("*."): + suffix = p[1:] # keep leading dot + return h.endswith(suffix) + if p.startswith("."): + return h.endswith(p) + return False + + +def has_host_bypass(host: str, patterns: List[str]) -> bool: + return any(match_proxy_pattern(host, item) for item in patterns) + + +def parse_scutil_proxy() -> Dict[str, object]: + code, stdout, _stderr = run(["scutil", "--proxy"]) + if code != 0: + return {"raw": "", "exceptions": [], "http_enabled": False, "https_enabled": False} + + raw = stdout + exceptions: List[str] = [] + for line in raw.splitlines(): + m = re.search(r"\d+\s*:\s*(.+)$", line) + if m and "ExceptionsList" not in line: + exceptions.append(m.group(1).strip()) + + http_enabled = bool(re.search(r"HTTPEnable\s*:\s*1", raw)) + https_enabled = bool(re.search(r"HTTPSEnable\s*:\s*1", raw)) + + return { + "raw": raw, + "exceptions": exceptions, + "http_enabled": http_enabled, + "https_enabled": https_enabled, + } + + +def resolve_host(host: str) -> List[str]: + ips = [] + try: + infos = socket.getaddrinfo(host, None) + for item in infos: + ip = item[4][0] + if ip not in ips: + ips.append(ip) + except socket.gaierror: + pass + return ips + + +def curl_status(url: str, timeout: int, mode: str, proxy_url: Optional[str] = None) -> Dict[str, object]: + cmd = [ + "curl", + "-k", + "-sS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + str(timeout), + url, + ] + + env = os.environ.copy() + if mode == "direct": + for key in ( + "http_proxy", + "https_proxy", + "HTTP_PROXY", + "HTTPS_PROXY", + "all_proxy", + "ALL_PROXY", + ): + env.pop(key, None) + elif mode == "forced_proxy" and proxy_url: + cmd.extend(["--proxy", proxy_url]) + + code, stdout, stderr = run(cmd, env=env) + http_code = stdout if stdout else "000" + ok = code == 0 and http_code.isdigit() and http_code != "000" + + return { + "ok": ok, + "http_code": http_code, + "exit_code": code, + "stderr": stderr, + "command": " ".join(shlex.quote(x) for x in cmd), + } + + +def strict_tls_check(url: str, timeout: int) -> Dict[str, object]: + cmd = [ + "curl", + "-sS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + str(timeout), + url, + ] + env = os.environ.copy() + for key in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"): + env.pop(key, None) + code, stdout, stderr = run(cmd, env=env) + cert_issue = "certificate" in stderr.lower() or "ssl" in stderr.lower() + return { + "ok": code == 0 and stdout != "000", + "http_code": stdout if stdout else "000", + "exit_code": code, + "stderr": stderr, + "cert_issue": cert_issue, + } + + +def find_tailscale_utun() -> Optional[str]: + """Find which utun interface belongs to Tailscale (has a 100.x.x.x IP).""" + code, stdout, _ = run(["ifconfig"]) + if code != 0: + return None + current_iface = "" + for line in stdout.splitlines(): + # Interface header line (e.g., "utun7: flags=...") + m = re.match(r"^(\w+):", line) + if m: + current_iface = m.group(1) + # Look for Tailscale CGNAT IP on a utun interface + if current_iface.startswith("utun") and "inet 100." in line: + return current_iface + return None + + +def get_iface_mtu(iface: str) -> Optional[int]: + """Get MTU of a network interface.""" + code, stdout, _ = run(["ifconfig", iface]) + if code != 0: + return None + m = re.search(r"mtu\s+(\d+)", stdout) + return int(m.group(1)) if m else None + + +def route_check(tailscale_ip: str) -> Dict[str, object]: + code, stdout, stderr = run(["route", "-n", "get", tailscale_ip]) + if code != 0: + return {"ok": False, "interface": "", "gateway": "", "raw": stderr or stdout} + + interface = "" + gateway = "" + for line in stdout.splitlines(): + line = line.strip() + if line.startswith("interface:"): + interface = line.split(":", 1)[1].strip() + if line.startswith("gateway:"): + gateway = line.split(":", 1)[1].strip() + + # Identify which utun is Tailscale's and whether the route points to it + tailscale_utun = find_tailscale_utun() + route_mtu = get_iface_mtu(interface) if interface else None + is_tailscale_iface = (interface == tailscale_utun) if tailscale_utun else None + wrong_utun = ( + interface.startswith("utun") + and tailscale_utun is not None + and interface != tailscale_utun + ) + + return { + "ok": True, + "interface": interface, + "gateway": gateway, + "tailscale_utun": tailscale_utun or "", + "route_iface_mtu": route_mtu, + "is_tailscale_iface": is_tailscale_iface, + "wrong_utun": wrong_utun, + "raw": stdout, + } + + +def pick_proxy_url() -> Optional[str]: + for key in ("http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY"): + value = os.environ.get(key) + if value: + return value + return None + + +def build_report( + host: str, + url: str, + timeout: int, + tailscale_ip: Optional[str], +) -> Dict[str, object]: + no_proxy = os.environ.get("NO_PROXY", "") + no_proxy_lc = os.environ.get("no_proxy", "") + no_proxy_entries = split_csv(no_proxy if no_proxy else no_proxy_lc) + + scutil_info = parse_scutil_proxy() + scutil_exceptions = scutil_info["exceptions"] + + proxy_url = pick_proxy_url() + direct = curl_status(url, timeout, mode="direct") + ambient = curl_status(url, timeout, mode="ambient") + forced_proxy = curl_status(url, timeout, mode="forced_proxy", proxy_url=proxy_url) if proxy_url else None + strict_tls = strict_tls_check(url, timeout) if url.startswith("https://") else None + + host_ips = resolve_host(host) + host_in_no_proxy = has_host_bypass(host, no_proxy_entries) + host_in_scutil_exceptions = has_host_bypass(host, scutil_exceptions) + + findings: List[Dict[str, str]] = [] + + if not host_ips: + findings.append( + { + "level": "error", + "title": "Host resolution failed", + "detail": f"{host} could not be resolved. Check DNS/hosts first.", + "fix": f"Add a hosts entry if this is local: 127.0.0.1 {host}", + } + ) + + if proxy_url and not host_in_no_proxy: + findings.append( + { + "level": "warn", + "title": "NO_PROXY missing target host", + "detail": f"Proxy is enabled ({proxy_url}) but NO_PROXY does not match {host}.", + "fix": ( + "Add host to NO_PROXY/no_proxy, e.g. " + f"NO_PROXY=...,{host}" + ), + } + ) + + if (scutil_info["http_enabled"] or scutil_info["https_enabled"]) and not host_in_scutil_exceptions: + findings.append( + { + "level": "warn", + "title": "System proxy exception missing target host", + "detail": f"scutil active exceptions do not include {host}.", + "fix": ( + "Add host to proxy app skip/bypass list (Shadowrocket/Clash/Surge), " + "then reload profile." + ), + } + ) + + if direct["ok"] and forced_proxy and not forced_proxy["ok"]: + findings.append( + { + "level": "error", + "title": "Proxy path is broken for target host", + "detail": ( + "Direct access works, but forced proxy tunnel fails. " + "Traffic must bypass proxy for this host." + ), + "fix": ( + f"Add {host} to both NO_PROXY and proxy app skip-proxy/DIRECT rules." + ), + } + ) + + if not ambient["ok"] and direct["ok"]: + findings.append( + { + "level": "error", + "title": "Ambient shell path fails while direct path works", + "detail": ( + "Current shell env/proxy settings break default access." + ), + "fix": ( + "Use NO_PROXY for this host, or temporarily unset proxy env for local verification." + ), + } + ) + + if strict_tls and not strict_tls["ok"] and direct["ok"] and strict_tls["cert_issue"]: + findings.append( + { + "level": "warn", + "title": "TLS trust issue detected", + "detail": "Network path is reachable, but strict TLS validation failed.", + "fix": "Trust local CA certificate (for local/internal TLS) or use a valid public cert.", + } + ) + + route_info = route_check(tailscale_ip) if tailscale_ip else None + if route_info and route_info["ok"]: + iface = str(route_info["interface"]) + ts_utun = str(route_info.get("tailscale_utun", "")) + route_mtu = route_info.get("route_iface_mtu") + wrong_utun = route_info.get("wrong_utun", False) + + if iface.startswith("en"): + findings.append( + { + "level": "error", + "title": "Possible route hijack for Tailscale destination", + "detail": f"route -n get {tailscale_ip} resolved to {iface}, not utun*.", + "fix": ( + "Check proxy TUN excluded-routes. Do not exclude 100.64.0.0/10 from TUN route table." + ), + } + ) + elif wrong_utun: + mtu_hint = f" (MTU {route_mtu})" if route_mtu else "" + findings.append( + { + "level": "error", + "title": "Route points to wrong utun interface", + "detail": ( + f"route -n get {tailscale_ip} resolved to {iface}{mtu_hint}, " + f"but Tailscale is on {ts_utun}. " + f"Likely hitting Shadowrocket/VPN TUN (MTU 4064) instead of Tailscale (MTU 1280)." + ), + "fix": ( + "Check proxy TUN excluded-routes and rule ordering. " + "Ensure IP-CIDR,100.64.0.0/10,DIRECT is in proxy rules." + ), + } + ) + + summary = { + "host": host, + "url": url, + "host_ips": host_ips, + "proxy_url": proxy_url or "", + "env_no_proxy": no_proxy if no_proxy else no_proxy_lc, + "host_in_no_proxy": host_in_no_proxy, + "scutil_http_enabled": scutil_info["http_enabled"], + "scutil_https_enabled": scutil_info["https_enabled"], + "host_in_scutil_exceptions": host_in_scutil_exceptions, + "connectivity": { + "ambient": ambient, + "direct": direct, + "forced_proxy": forced_proxy, + "strict_tls": strict_tls, + }, + "tailscale_route": route_info, + "findings": findings, + } + return summary + + +def print_human(report: Dict[str, object]) -> int: + print("=== Tunnel Doctor Quick Diagnose ===") + print(f"Host: {report['host']}") + print(f"URL: {report['url']}") + ips = report["host_ips"] + print(f"Resolved IPs: {', '.join(ips) if ips else 'N/A'}") + print("") + + print("Proxy Context") + print(f"- proxy env: {report['proxy_url'] or '(not set)'}") + print(f"- host in NO_PROXY: {'yes' if report['host_in_no_proxy'] else 'no'}") + print( + "- system proxy enabled: " + f"HTTP={'yes' if report['scutil_http_enabled'] else 'no'} " + f"HTTPS={'yes' if report['scutil_https_enabled'] else 'no'}" + ) + print(f"- host in scutil exceptions: {'yes' if report['host_in_scutil_exceptions'] else 'no'}") + print("") + + conn = report["connectivity"] + print("Connectivity Checks") + for key in ("ambient", "direct", "forced_proxy", "strict_tls"): + value = conn.get(key) + if not value: + continue + ok = "PASS" if value.get("ok") else "FAIL" + print( + f"- {key:12s}: {ok} " + f"(http={value.get('http_code', '000')}, exit={value.get('exit_code', 'n/a')})" + ) + stderr = value.get("stderr") + if stderr: + print(f" stderr: {stderr}") + print("") + + route = report.get("tailscale_route") + if route: + if route.get("ok"): + print("Tailscale Route Check") + print(f"- route interface: {route.get('interface') or 'N/A'}") + route_mtu = route.get("route_iface_mtu") + if route_mtu: + print(f" route iface MTU: {route_mtu}") + print(f"- gateway: {route.get('gateway') or 'N/A'}") + ts_utun = route.get("tailscale_utun") + if ts_utun: + print(f"- tailscale utun: {ts_utun}") + is_ts = route.get("is_tailscale_iface") + if is_ts is True: + print(" route → Tailscale utun: YES (correct)") + elif is_ts is False: + print(" route → Tailscale utun: NO (MISMATCH — see findings)") + else: + print("- tailscale utun: (not detected — is Tailscale running?)") + print("") + else: + print("Tailscale Route Check") + print(f"- failed: {route.get('raw', '')}") + print("") + + findings = report["findings"] + if not findings: + print("Result: no high-confidence conflict found.") + print("If browser still fails, verify proxy app profile mode/rule order and reload profile.") + return 0 + + print("Findings") + severity_order = {"error": 0, "warn": 1, "info": 2} + findings_sorted = sorted(findings, key=lambda x: severity_order.get(str(x.get("level")), 99)) + for idx, item in enumerate(findings_sorted, start=1): + print(f"{idx}. [{item['level'].upper()}] {item['title']}") + print(f" Detail: {item['detail']}") + print(f" Fix: {item['fix']}") + + return 1 if any(x["level"] == "error" for x in findings) else 0 + + +def main() -> int: + if sys.platform != "darwin": + print("This script is designed for macOS only.", file=sys.stderr) + return 2 + + parser = argparse.ArgumentParser( + description="Quick diagnostics for Tailscale + proxy conflicts on macOS." + ) + parser.add_argument("--host", default="local.claude4.dev", help="Target host to diagnose.") + parser.add_argument( + "--url", + default="", + help="Full URL to test. Default: https:///health", + ) + parser.add_argument( + "--timeout", + type=int, + default=8, + help="curl timeout seconds (default: 8).", + ) + parser.add_argument( + "--tailscale-ip", + default="", + help="Optional Tailscale IP for route ownership check (e.g. 100.101.102.103).", + ) + parser.add_argument( + "--json", + action="store_true", + help="Output JSON report.", + ) + args = parser.parse_args() + + url = args.url.strip() or f"https://{args.host}/health" + report = build_report( + host=args.host.strip(), + url=url, + timeout=max(args.timeout, 1), + tailscale_ip=args.tailscale_ip.strip() or None, + ) + + if args.json: + print(json.dumps(report, indent=2, ensure_ascii=False)) + return 0 + return print_human(report) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/youtube-downloader/references/internal-sop.md b/youtube-downloader/references/internal-sop.md new file mode 100644 index 00000000..5fdd1be3 --- /dev/null +++ b/youtube-downloader/references/internal-sop.md @@ -0,0 +1,12 @@ +# YouTube Downloader Internal SOP + +Use this SOP to avoid common yt-dlp failures and confusion: + +1. Quote YouTube URLs in shell commands (zsh treats `?` as glob). Example: `'https://www.youtube.com/watch?v=VIDEO_ID'`. +2. Ensure proxy is active for both yt-dlp and PO Token providers (HTTP_PROXY/HTTPS_PROXY/ALL_PROXY). +3. If you see "Sign in to confirm you're not a bot", request cookie permission and use browser cookies. +4. Start the PO Token provider before downloading. Prefer Docker bgutil; fall back to browser-based WPC when Docker is unavailable or fails. +5. Use `web_safari` client when cookies are present; otherwise use `mweb` for PO tokens. +6. Keep the browser window open while WPC is minting tokens and make sure it can reach YouTube through the same proxy. +7. If you see "Only images are available" or "Requested format is not available", treat it as PO token failure and retry after fixing provider/browser state. +8. If you see SSL EOF or fragment errors, treat it as proxy instability. Retry with progressive formats or switch to a more stable proxy.